feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <[email protected]>
This commit is contained in:
DenozordecandCursor committed
1 parent
924f111e48
commit
f8e0520817
197 files changed
+18655
-1416
No files matched your search
@@ -4,6 +4,10 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -78,39 +82,110 @@ func (p *Postgres) GetASNPrefixCache(asn int64) (*store.ASNPrefixCacheEntry, boo
|
||||
|
||||
func (p *Postgres) SetASNPrefixCache(asn int64, holder string, prefixes []string) error {
|
||||
ctx := context.Background()
|
||||
normalized, err := canonicalASNPrefixes(prefixes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := p.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO asn_prefix_cache (asn, holder, fetched_at)
|
||||
VALUES ($1, $2, now())
|
||||
ON CONFLICT (asn) DO UPDATE SET
|
||||
holder = EXCLUDED.holder,
|
||||
fetched_at = EXCLUDED.fetched_at`, asn, holder)
|
||||
INSERT INTO asn_prefix_cache (asn, holder, fetched_at)
|
||||
VALUES ($1, $2, now())
|
||||
ON CONFLICT (asn) DO UPDATE SET
|
||||
holder = EXCLUDED.holder,
|
||||
fetched_at = EXCLUDED.fetched_at`, asn, holder)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if asnCacheRowTableExists(ctx, tx) {
|
||||
existing, err := listASNCachePrefixes(ctx, tx, asn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if prefixSetEqual(existing, normalized) {
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM asn_prefix_cache_row WHERE asn = $1`, asn); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, pfx := range prefixes {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO asn_prefix_cache_row (asn, prefix) VALUES ($1, $2::cidr)`, asn, pfx); err != nil {
|
||||
return err
|
||||
if len(normalized) > 0 {
|
||||
if _, err := tx.CopyFrom(ctx, pgx.Identifier{"asn_prefix_cache_row"}, []string{"asn", "prefix"}, pgx.CopyFromSlice(len(normalized), func(i int) ([]any, error) {
|
||||
pfx, err := netip.ParsePrefix(normalized[i])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("repository: asn cache prefix %q: %w", normalized[i], err)
|
||||
}
|
||||
return []any{asn, pfx.Masked()}, nil
|
||||
})); err != nil {
|
||||
return fmt.Errorf("repository: copy asn cache rows: %w", err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
raw, err := json.Marshal(prefixes)
|
||||
raw, err := json.Marshal(normalized)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE asn_prefix_cache SET prefixes_json = $2::jsonb WHERE asn = $1`, asn, string(raw)); err != nil {
|
||||
UPDATE asn_prefix_cache SET prefixes_json = $2::jsonb WHERE asn = $1`, asn, string(raw)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func listASNCachePrefixes(ctx context.Context, tx pgx.Tx, asn int64) ([]string, error) {
|
||||
rows, err := tx.Query(ctx, `SELECT prefix::text FROM asn_prefix_cache_row WHERE asn = $1`, asn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var s string
|
||||
if err := rows.Scan(&s); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return canonicalASNPrefixes(out)
|
||||
}
|
||||
|
||||
func canonicalASNPrefixes(in []string) ([]string, error) {
|
||||
seen := make(map[string]struct{}, len(in))
|
||||
out := make([]string, 0, len(in))
|
||||
for _, raw := range in {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
continue
|
||||
}
|
||||
pfx, err := netip.ParsePrefix(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("repository: asn cache prefix %q: %w", raw, err)
|
||||
}
|
||||
s := pfx.Masked().String()
|
||||
if _, ok := seen[s]; ok {
|
||||
continue
|
||||
}
|
||||
seen[s] = struct{}{}
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func prefixSetEqual(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
|
||||
"evobgp/internal/db"
|
||||
"evobgp/internal/store"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestPostgresImplementsBackend(t *testing.T) {
|
||||
@@ -48,7 +50,11 @@ func TestPostgresModuleCRUDIntegration(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tenant := "01TESTTENANT00000000000001"
|
||||
tenant := uuid.NewString()
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO tenant (id, name, slug) VALUES ($1, $2, $3)`, tenant, "integration", "integration-"+tenant); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _, _ = pool.Exec(ctx, `DELETE FROM tenant WHERE id = $1`, tenant) }()
|
||||
mod, err := pg.CreateModule(tenant, &store.Module{
|
||||
Name: "audit-test",
|
||||
Type: "IP_RANGES",
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"evobgp/internal/store"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
const deploymentColumns = `job_id::text,tenant_id::text,speaker_id::text,revision_id::text,generation,request_sequence,state,confirmation,error_message,dispatch_attempts,retry_round,health_received_at,health_revision_id,health_source_at,health_error,health_observation_token`
|
||||
|
||||
func scanDeployment(row pgx.Row) (store.DeploymentSpeaker, error) {
|
||||
var d store.DeploymentSpeaker
|
||||
err := row.Scan(&d.JobID, &d.TenantID, &d.SpeakerID, &d.RevisionID, &d.Generation, &d.RequestSequence, &d.State, &d.Confirmation, &d.Error, &d.DispatchAttempts, &d.RetryRound, &d.HealthReceivedAt, &d.HealthRevisionID, &d.HealthSourceAt, &d.HealthError, &d.ObservationToken)
|
||||
return d, err
|
||||
}
|
||||
|
||||
func deploymentFence(ctx context.Context, tx pgx.Tx, f store.JobFence) (int64, error) {
|
||||
var seq int64
|
||||
err := tx.QueryRow(ctx, `SELECT request_sequence FROM job_audit WHERE id=$1::uuid AND tenant_id=$2::uuid AND owner_id=$3 AND fencing_token=$4 AND status='running' AND NOT cancel_requested AND lease_until>clock_timestamp() FOR UPDATE`, f.JobID, f.TenantID, f.OwnerID, f.Token).Scan(&seq)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return 0, store.ErrJobLeaseLost
|
||||
}
|
||||
return seq, err
|
||||
}
|
||||
|
||||
func (p *Postgres) BeginDeploymentSpeaker(ctx context.Context, f store.JobFence, speaker, revision string) (store.DeploymentSpeaker, error) {
|
||||
tx, err := p.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return store.DeploymentSpeaker{}, err
|
||||
}
|
||||
defer rollbackWithTimeout(tx)
|
||||
seq, err := deploymentFence(ctx, tx, f)
|
||||
if err != nil {
|
||||
return store.DeploymentSpeaker{}, err
|
||||
}
|
||||
var generation, currentSeq int64
|
||||
var currentJob, currentRevision string
|
||||
err = tx.QueryRow(ctx, `SELECT deployment_generation,deployment_request_sequence,COALESCE(deployment_job_id::text,''),COALESCE(published_revision_id::text,'') FROM bgp_speaker WHERE id=$1::uuid AND tenant_id=$2::uuid FOR UPDATE`, speaker, f.TenantID).Scan(&generation, ¤tSeq, ¤tJob, ¤tRevision)
|
||||
if err != nil {
|
||||
return store.DeploymentSpeaker{}, err
|
||||
}
|
||||
if f.ExpectedGeneration != 0 && currentJob != f.JobID && (generation != f.ExpectedGeneration || currentRevision != revision) {
|
||||
return store.DeploymentSpeaker{}, store.ErrJobLeaseLost
|
||||
}
|
||||
reserve := func() error {
|
||||
tag, e := tx.Exec(ctx, `UPDATE bgp_speaker SET delivery_owner=$2,delivery_token=$3,delivery_until=clock_timestamp()+($4::bigint*interval '1 millisecond') WHERE id=$1::uuid AND (delivery_until IS NULL OR delivery_until<=clock_timestamp())`, speaker, f.OwnerID, f.Token, store.DeploymentDeliveryWindow.Milliseconds())
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return store.ErrDeploymentBusy
|
||||
}
|
||||
return nil
|
||||
}
|
||||
existing, err := scanDeployment(tx.QueryRow(ctx, `SELECT `+deploymentColumns+` FROM deployment_speaker WHERE job_id=$1::uuid AND speaker_id=$2::uuid`, f.JobID, speaker))
|
||||
if err == nil {
|
||||
if currentJob != f.JobID || generation != existing.Generation || existing.RevisionID != revision {
|
||||
return store.DeploymentSpeaker{}, store.ErrJobLeaseLost
|
||||
}
|
||||
if existing.State != "applied" {
|
||||
if err = reserve(); err != nil {
|
||||
return store.DeploymentSpeaker{}, err
|
||||
}
|
||||
}
|
||||
if _, err = deploymentFence(ctx, tx, f); err != nil {
|
||||
return store.DeploymentSpeaker{}, err
|
||||
}
|
||||
return existing, tx.Commit(ctx)
|
||||
}
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return store.DeploymentSpeaker{}, err
|
||||
}
|
||||
if f.ExpectedGeneration != 0 {
|
||||
seq = currentSeq
|
||||
}
|
||||
if currentSeq >= seq && f.ExpectedGeneration == 0 {
|
||||
return store.DeploymentSpeaker{}, store.ErrJobLeaseLost
|
||||
}
|
||||
var valid bool
|
||||
err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM config_revision WHERE id=$1::uuid AND tenant_id=$2::uuid)`, revision, f.TenantID).Scan(&valid)
|
||||
if err != nil {
|
||||
return store.DeploymentSpeaker{}, err
|
||||
}
|
||||
if !valid {
|
||||
return store.DeploymentSpeaker{}, store.ErrNotFound
|
||||
}
|
||||
if err = reserve(); err != nil {
|
||||
return store.DeploymentSpeaker{}, err
|
||||
}
|
||||
d, err := scanDeployment(tx.QueryRow(ctx, `INSERT INTO deployment_speaker(job_id,tenant_id,speaker_id,revision_id,generation,request_sequence,state,retry_round) VALUES($1::uuid,$2::uuid,$3::uuid,$4::uuid,$5,$6,'published',$7) RETURNING `+deploymentColumns, f.JobID, f.TenantID, speaker, revision, generation+1, seq, f.RetryRound))
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE bgp_speaker SET deployment_generation=$2,deployment_request_sequence=$3,deployment_job_id=$4::uuid,published_revision_id=$5::uuid,published_at=now(),updated_at=now() WHERE id=$1::uuid`, speaker, d.Generation, seq, f.JobID, revision)
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
if _, err = deploymentFence(ctx, tx, f); err != nil {
|
||||
return store.DeploymentSpeaker{}, err
|
||||
}
|
||||
return d, tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (p *Postgres) RecordDeploymentSpeaker(ctx context.Context, f store.JobFence, d store.DeploymentSpeaker, state, confirmation, message string) error {
|
||||
if d.JobID != f.JobID || d.TenantID != f.TenantID {
|
||||
return store.ErrJobLeaseLost
|
||||
}
|
||||
if state == "applied" && confirmation != "agent_reported" && confirmation != "local_applied" {
|
||||
return store.ErrInvalidInput
|
||||
}
|
||||
tx, err := p.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rollbackWithTimeout(tx)
|
||||
if _, err = deploymentFence(ctx, tx, f); err != nil {
|
||||
return err
|
||||
}
|
||||
var speaker string
|
||||
err = tx.QueryRow(ctx, `SELECT id::text FROM bgp_speaker WHERE id=$1::uuid AND tenant_id=$2::uuid AND deployment_job_id=$3::uuid AND deployment_generation=$4 AND published_revision_id=$5::uuid AND delivery_owner=$6 AND delivery_token=$7 AND delivery_until>clock_timestamp() FOR UPDATE`, d.SpeakerID, f.TenantID, f.JobID, d.Generation, d.RevisionID, f.OwnerID, f.Token).Scan(&speaker)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return store.ErrJobLeaseLost
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE deployment_speaker SET state=$3,confirmation=$4,error_message=$5,dispatch_attempts=dispatch_attempts+CASE WHEN $3='dispatched' THEN 1 ELSE 0 END,updated_at=now() WHERE job_id=$1::uuid AND speaker_id=$2::uuid AND generation=$6`, f.JobID, speaker, state, confirmation, message, d.Generation)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if state != "dispatched" && state != "unknown" {
|
||||
if _, err = tx.Exec(ctx, `UPDATE bgp_speaker SET delivery_until=NULL,delivery_owner=NULL WHERE id=$1::uuid`, speaker); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if state == "applied" {
|
||||
_, err = tx.Exec(ctx, `UPDATE bgp_speaker SET last_applied_revision_id=$2::uuid,updated_at=now() WHERE id=$1::uuid`, speaker, d.RevisionID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err = deploymentFence(ctx, tx, f); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (p *Postgres) ListDeploymentSpeakers(ctx context.Context, tenant, job string) ([]store.DeploymentSpeaker, error) {
|
||||
rows, err := p.pool.Query(ctx, `SELECT `+deploymentColumns+` FROM deployment_speaker WHERE tenant_id=$1::uuid AND job_id=$2::uuid ORDER BY speaker_id`, tenant, job)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []store.DeploymentSpeaker
|
||||
for rows.Next() {
|
||||
d, e := scanDeployment(rows)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (p *Postgres) ListPendingDeployments(ctx context.Context, limit int) ([]store.DeploymentSpeaker, error) {
|
||||
if limit <= 0 || limit > 100 {
|
||||
limit = 100
|
||||
}
|
||||
rows, err := p.pool.Query(ctx, `SELECT `+deploymentColumns+` FROM deployment_speaker WHERE (job_id,speaker_id) IN (SELECT d.job_id,d.speaker_id FROM deployment_speaker d JOIN bgp_speaker s ON s.id=d.speaker_id AND s.deployment_job_id=d.job_id AND s.deployment_generation=d.generation JOIN job_audit j ON j.id=d.job_id WHERE d.state<>'applied' AND d.retry_round<3 AND j.status IN ('succeeded','failed') AND NOT j.cancel_requested AND (s.delivery_until IS NULL OR s.delivery_until<=clock_timestamp()) AND d.updated_at < clock_timestamp()-((10*(d.retry_round+1))::bigint*interval '1 second')) ORDER BY updated_at LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []store.DeploymentSpeaker
|
||||
for rows.Next() {
|
||||
d, e := scanDeployment(rows)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ListDeploymentObservations rotates through current generations by oldest CP
|
||||
// receipt time; the bounded batch prevents a large fleet monopolizing a worker.
|
||||
func (p *Postgres) ListDeploymentObservations(ctx context.Context, limit int) ([]store.DeploymentSpeaker, error) {
|
||||
if limit <= 0 || limit > 100 {
|
||||
limit = 10
|
||||
}
|
||||
rows, err := p.pool.Query(ctx, `WITH picked AS (SELECT d.job_id AS picked_job,d.speaker_id AS picked_speaker FROM deployment_speaker d JOIN bgp_speaker s ON s.id=d.speaker_id AND s.deployment_job_id=d.job_id AND s.deployment_generation=d.generation JOIN job_audit j ON j.id=d.job_id WHERE j.status IN ('succeeded','failed') AND NOT j.cancel_requested AND (s.delivery_until IS NULL OR s.delivery_until<=clock_timestamp()) ORDER BY d.health_received_at NULLS FIRST,d.speaker_id FOR UPDATE OF d SKIP LOCKED LIMIT $1) UPDATE deployment_speaker d SET health_observation_token=d.health_observation_token+1 FROM picked WHERE d.job_id=picked.picked_job AND d.speaker_id=picked.picked_speaker RETURNING `+deploymentColumns, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []store.DeploymentSpeaker
|
||||
for rows.Next() {
|
||||
d, e := scanDeployment(rows)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ObserveDeployment accepts only a health report for the still-current generation
|
||||
// outside any delivery reservation. It never treats a legacy report as verified.
|
||||
func (p *Postgres) ObserveDeployment(ctx context.Context, d store.DeploymentSpeaker, o store.DeploymentObservation) error {
|
||||
tx, err := p.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rollbackWithTimeout(tx)
|
||||
var speaker string
|
||||
err = tx.QueryRow(ctx, `SELECT id::text FROM bgp_speaker WHERE id=$1::uuid AND tenant_id=$2::uuid AND deployment_job_id=$3::uuid AND deployment_generation=$4 AND published_revision_id=$5::uuid AND (delivery_until IS NULL OR delivery_until<=clock_timestamp()) FOR UPDATE`, d.SpeakerID, d.TenantID, d.JobID, d.Generation, d.RevisionID).Scan(&speaker)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return store.ErrJobLeaseLost
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `UPDATE deployment_speaker SET health_observation_token=health_observation_token+1,health_received_at=clock_timestamp(),health_revision_id=$3,health_source_at=$4,health_error=$5,state=CASE WHEN $5='' AND $3<>'' THEN CASE WHEN $3=revision_id::text THEN 'applied' ELSE 'unknown' END ELSE state END,confirmation=CASE WHEN $5='' AND $3<>'' THEN CASE WHEN $3=revision_id::text THEN 'agent_reported' ELSE 'unconfirmed' END ELSE confirmation END WHERE job_id=$1::uuid AND speaker_id=$2::uuid AND health_observation_token=$6`, d.JobID, speaker, o.RevisionID, o.SourceAt, o.Error, d.ObservationToken)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return store.ErrJobLeaseLost
|
||||
}
|
||||
if o.Error == "" && o.RevisionID == d.RevisionID {
|
||||
if _, err = tx.Exec(ctx, `UPDATE bgp_speaker SET last_applied_revision_id=$2::uuid,updated_at=now() WHERE id=$1::uuid`, speaker, d.RevisionID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestDeploymentLeaseGenerationAndRestartIntegration(t *testing.T) {
|
||||
p, tenant := queueIntegration(t)
|
||||
ctx := context.Background()
|
||||
speaker, revision, newer := uuid.NewString(), uuid.NewString(), uuid.NewString()
|
||||
for _, id := range []string{revision, newer} {
|
||||
if _, err := p.pool.Exec(ctx, `INSERT INTO config_revision(id,tenant_id,content_hash) VALUES($1,$2,'deployment-test')`, id, tenant); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := p.pool.Exec(ctx, `INSERT INTO bgp_speaker(id,tenant_id,role) VALUES($1,$2,'replica')`, speaker, tenant); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
enqueue := func() store.JobRecord {
|
||||
t.Helper()
|
||||
j, _, err := p.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: "deploy_apply"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return j
|
||||
}
|
||||
claim := func(owner string) store.JobRecord {
|
||||
t.Helper()
|
||||
rows, err := p.ClaimJobs(ctx, owner, 1, time.Minute)
|
||||
if err != nil || len(rows) != 1 {
|
||||
t.Fatalf("claim: %+v %v", rows, err)
|
||||
}
|
||||
return rows[0]
|
||||
}
|
||||
fence := func(j store.JobRecord) store.JobFence {
|
||||
return store.JobFence{JobID: j.ID, TenantID: j.TenantID, OwnerID: j.OwnerID, Token: j.FencingToken}
|
||||
}
|
||||
first := enqueue()
|
||||
first = claim("deployment-one")
|
||||
d, err := p.BeginDeploymentSpeaker(ctx, fence(first), speaker, revision)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second := enqueue()
|
||||
second = claim("deployment-two")
|
||||
if _, err = p.BeginDeploymentSpeaker(ctx, fence(second), speaker, newer); !errors.Is(err, store.ErrDeploymentBusy) {
|
||||
t.Fatalf("overlapping network ownership: %v", err)
|
||||
}
|
||||
if err = p.RecordDeploymentSpeaker(ctx, fence(first), d, "unknown", "unconfirmed", "response lost"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = p.pool.Exec(ctx, `UPDATE job_audit SET lease_until=clock_timestamp()-interval '1 second' WHERE id=$1`, first.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = p.RecordDeploymentSpeaker(ctx, fence(first), d, "applied", "agent_reported", ""); !errors.Is(err, store.ErrJobLeaseLost) {
|
||||
t.Fatalf("expired ACK accepted: %v", err)
|
||||
}
|
||||
reclaimed := claim("deployment-reclaimed")
|
||||
if reclaimed.ID != first.ID || reclaimed.FencingToken <= first.FencingToken {
|
||||
t.Fatal("job not reclaimed with fresh token")
|
||||
}
|
||||
if _, err = p.BeginDeploymentSpeaker(ctx, fence(reclaimed), speaker, revision); !errors.Is(err, store.ErrDeploymentBusy) {
|
||||
t.Fatalf("reclaim overlaps uncertain request: %v", err)
|
||||
}
|
||||
if _, err = p.pool.Exec(ctx, `UPDATE bgp_speaker SET delivery_until=clock_timestamp()-interval '1 second' WHERE id=$1`, speaker); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A fresh repository object sees the committed partial progress after restart.
|
||||
restarted := &Postgres{pool: p.pool}
|
||||
rows, err := restarted.ListDeploymentSpeakers(ctx, tenant, first.ID)
|
||||
if err != nil || len(rows) != 1 || rows[0].State != "unknown" {
|
||||
t.Fatalf("restart progress: %+v %v", rows, err)
|
||||
}
|
||||
d2, err := restarted.BeginDeploymentSpeaker(ctx, fence(second), speaker, newer)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d2.Generation <= d.Generation {
|
||||
t.Fatal("speaker generation did not advance")
|
||||
}
|
||||
if err = p.RecordDeploymentSpeaker(ctx, fence(reclaimed), d, "applied", "agent_reported", ""); !errors.Is(err, store.ErrJobLeaseLost) {
|
||||
t.Fatalf("late old-generation ACK accepted: %v", err)
|
||||
}
|
||||
if err = p.RecordDeploymentSpeaker(ctx, fence(second), d2, "applied", "agent_reported", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var published, applied string
|
||||
if err = p.pool.QueryRow(ctx, `SELECT published_revision_id::text,last_applied_revision_id::text FROM bgp_speaker WHERE id=$1`, speaker).Scan(&published, &applied); err != nil || published != newer || applied != newer {
|
||||
t.Fatalf("fenced pointers: %s %s %v", published, applied, err)
|
||||
}
|
||||
d2Replay, err := p.BeginDeploymentSpeaker(ctx, fence(second), speaker, newer)
|
||||
if err != nil || d2Replay.State != "applied" || d2Replay.Generation != d2.Generation {
|
||||
t.Fatalf("applied replay: %+v %v", d2Replay, err)
|
||||
}
|
||||
second.Status = "succeeded"
|
||||
if err = p.FinishJob(ctx, second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldPoll, err := p.ListDeploymentObservations(ctx, 10)
|
||||
if err != nil || len(oldPoll) != 1 {
|
||||
t.Fatalf("old poll %+v %v", oldPoll, err)
|
||||
}
|
||||
newPoll, err := p.ListDeploymentObservations(ctx, 10)
|
||||
if err != nil || len(newPoll) != 1 {
|
||||
t.Fatalf("new poll %+v %v", newPoll, err)
|
||||
}
|
||||
// A stale snapshot must not downgrade the current acknowledged state when
|
||||
// the latest poll times out or the old agent omits revision fields.
|
||||
newPoll[0].State = "unknown"
|
||||
newPoll[0].Confirmation = "unconfirmed"
|
||||
if err = p.ObserveDeployment(ctx, newPoll[0], store.DeploymentObservation{Error: "timeout"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = p.ObserveDeployment(ctx, oldPoll[0], store.DeploymentObservation{RevisionID: revision}); !errors.Is(err, store.ErrJobLeaseLost) {
|
||||
t.Fatalf("late health accepted: %v", err)
|
||||
}
|
||||
progress, err := p.ListDeploymentSpeakers(ctx, tenant, second.ID)
|
||||
if err != nil || len(progress) != 1 || progress[0].State != "applied" || progress[0].Confirmation != "agent_reported" || progress[0].HealthError != "timeout" {
|
||||
t.Fatalf("health lost ACK %+v %v", progress, err)
|
||||
}
|
||||
// Historical deployment records must survive ordinary revision retention.
|
||||
if _, err = p.pool.Exec(ctx, `DELETE FROM config_revision WHERE id=$1::uuid`, revision); err != nil {
|
||||
t.Fatalf("history blocks revision pruning: %v", err)
|
||||
}
|
||||
history, err := p.ListDeploymentSpeakers(ctx, tenant, first.ID)
|
||||
if err != nil || len(history) != 1 || history[0].RevisionID != revision {
|
||||
t.Fatalf("pruning removed history %+v %v", history, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"evobgp/internal/db"
|
||||
"evobgp/internal/store"
|
||||
"evobgp/migrations"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestJobQueueFreshSeedIntegration(t *testing.T) {
|
||||
dsn := os.Getenv("EVOBGP_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("EVOBGP_TEST_DATABASE_URL not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
admin, err := pgxpool.New(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer admin.Close()
|
||||
schema := "queue_seed_" + uuid.NewString()
|
||||
quoted := `"` + schema + `"`
|
||||
if _, err = admin.Exec(ctx, `CREATE SCHEMA `+quoted); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if _, e := admin.Exec(ctx, `DROP SCHEMA `+quoted+` CASCADE`); e != nil {
|
||||
t.Error(e)
|
||||
}
|
||||
}()
|
||||
cfg, err := pgxpool.ParseConfig(dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.ConnConfig.RuntimeParams["search_path"] = schema
|
||||
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err = db.ApplyPostgresMigrations(ctx, pool, migrations.Postgres, "postgres"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, err := NewPostgres(ctx, pool, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tenant, cdn, ip, revision, speaker := p.DemoIDs()
|
||||
if tenant == "" || tenant == store.SystemJobTenantID || cdn == "" || ip == "" || revision == "" || speaker == "" {
|
||||
t.Fatalf("system tenant blocked demo seed: %q %q %q %q %q", tenant, cdn, ip, revision, speaker)
|
||||
}
|
||||
}
|
||||
|
||||
func queueIntegration(t *testing.T) (*Postgres, string) {
|
||||
t.Helper()
|
||||
dsn := os.Getenv("EVOBGP_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("EVOBGP_TEST_DATABASE_URL not set")
|
||||
}
|
||||
pool, e := db.OpenPostgresPool(context.Background(), dsn)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
tenant := uuid.NewString()
|
||||
if _, e = pool.Exec(context.Background(), `INSERT INTO tenant(id,name,slug) VALUES($1,'queue integration',$2)`, tenant, "queue-"+tenant); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = pool.Exec(context.Background(), `DELETE FROM tenant WHERE id=$1`, tenant) })
|
||||
return &Postgres{pool: pool}, tenant
|
||||
}
|
||||
|
||||
func TestJobAuditClaimAndRestartIntegration(t *testing.T) {
|
||||
p, tenant := queueIntegration(t)
|
||||
ctx := context.Background()
|
||||
j, _, e := p.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: "test"})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
results := make(chan []store.JobRecord, 2)
|
||||
errs := make(chan error, 2)
|
||||
for _, owner := range []string{"replica-a", "replica-b"} {
|
||||
wg.Add(1)
|
||||
go func(owner string) {
|
||||
defer wg.Done()
|
||||
items, e := p.ClaimJobs(ctx, owner, 1, time.Minute)
|
||||
results <- items
|
||||
errs <- e
|
||||
}(owner)
|
||||
}
|
||||
wg.Wait()
|
||||
close(results)
|
||||
close(errs)
|
||||
for e := range errs {
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
var first store.JobRecord
|
||||
count := 0
|
||||
for items := range results {
|
||||
for _, item := range items {
|
||||
if item.ID != j.ID {
|
||||
t.Fatalf("unexpected job %+v", item)
|
||||
}
|
||||
first = item
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("claims=%d", count)
|
||||
}
|
||||
if _, e = p.HeartbeatJob(ctx, j.ID, first.OwnerID, first.FencingToken, time.Hour); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if items, e := p.ClaimJobs(ctx, "other", 1, time.Minute); e != nil || len(items) != 0 {
|
||||
t.Fatalf("stole live lease: %+v %v", items, e)
|
||||
}
|
||||
if _, e = p.pool.Exec(ctx, `UPDATE job_audit SET lease_until=now()-interval '1 second' WHERE id=$1`, j.ID); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
pool, e := db.OpenPostgresPool(ctx, os.Getenv("EVOBGP_TEST_DATABASE_URL"))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer pool.Close()
|
||||
restarted := &Postgres{pool: pool}
|
||||
items, e := restarted.ClaimJobs(ctx, "restarted", 1, time.Minute)
|
||||
if e != nil || len(items) != 1 {
|
||||
t.Fatalf("restart claim: %+v %v", items, e)
|
||||
}
|
||||
if items[0].FencingToken <= first.FencingToken {
|
||||
t.Fatal("token did not advance")
|
||||
}
|
||||
first.Status = "succeeded"
|
||||
if e = p.FinishJob(ctx, first); !errors.Is(e, store.ErrJobLeaseLost) {
|
||||
t.Fatalf("stale finish: %v", e)
|
||||
}
|
||||
if _, e = p.CancelJob(ctx, tenant, j.ID); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
items[0].Status = "succeeded"
|
||||
if e = restarted.FinishJob(ctx, items[0]); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
got, e := p.GetJob(ctx, tenant, j.ID)
|
||||
if e != nil || got.Status != "cancelled" {
|
||||
t.Fatalf("cancel/finish race: %+v %v", got, e)
|
||||
}
|
||||
listed, _, _, e := restarted.ListJobs(ctx, tenant, "cancelled", "", "", 10)
|
||||
if e != nil || len(listed) != 1 {
|
||||
t.Fatalf("shared terminal list %+v %v", listed, e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobAuditIdempotencyRowIntegration(t *testing.T) {
|
||||
p, tenant := queueIntegration(t)
|
||||
ctx := context.Background()
|
||||
key := uuid.NewString()
|
||||
in := store.JobEnqueue{TenantID: tenant, Kind: "test", IdempotencyKey: &key, Meta: map[string]any{"input": "same"}}
|
||||
var wg sync.WaitGroup
|
||||
ids := make(chan string, 8)
|
||||
errs := make(chan error, 8)
|
||||
for i := 0; i < 8; i++ {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); j, _, e := p.EnqueueJob(ctx, in); ids <- j.ID; errs <- e }()
|
||||
}
|
||||
wg.Wait()
|
||||
close(ids)
|
||||
close(errs)
|
||||
for e := range errs {
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
id := ""
|
||||
for got := range ids {
|
||||
if id == "" {
|
||||
id = got
|
||||
}
|
||||
if got != id {
|
||||
t.Fatalf("different IDs %s / %s", id, got)
|
||||
}
|
||||
}
|
||||
items, e := p.ClaimJobs(ctx, "replica", 1, time.Minute)
|
||||
if e != nil || len(items) != 1 {
|
||||
t.Fatalf("claim %+v %v", items, e)
|
||||
}
|
||||
items[0].Status = "succeeded"
|
||||
items[0].Meta = map[string]any{"result": "done"}
|
||||
if e = p.FinishJob(ctx, items[0]); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
got, created, e := p.EnqueueJob(ctx, in)
|
||||
if e != nil || created || got.ID != id || got.Status != "succeeded" {
|
||||
t.Fatalf("terminal replay %+v %v %v", got, created, e)
|
||||
}
|
||||
in.Meta = map[string]any{"input": "changed"}
|
||||
if _, _, e = p.EnqueueJob(ctx, in); !errors.Is(e, store.ErrJobConflict) {
|
||||
t.Fatalf("changed request accepted: %v", e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobQueueRetryBoundsAndFailureIntegration(t *testing.T) {
|
||||
p, tenant := queueIntegration(t)
|
||||
ctx := context.Background()
|
||||
t.Setenv("EVOBGP_JOB_MAX_TENANT_PENDING", "1")
|
||||
j, _, e := p.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: "test"})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, _, e = p.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: "test"}); !errors.Is(e, store.ErrJobQueueFull) {
|
||||
t.Fatalf("unbounded queue: %v", e)
|
||||
}
|
||||
if _, e = p.pool.Exec(ctx, `UPDATE job_audit SET attempt=3 WHERE id=$1`, j.ID); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = p.ReapExpiredJobs(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
items, e := p.ClaimJobs(ctx, "worker", 1, time.Minute)
|
||||
if e != nil || len(items) != 0 {
|
||||
t.Fatalf("exhausted claim %+v %v", items, e)
|
||||
}
|
||||
got, e := p.GetJob(ctx, tenant, j.ID)
|
||||
if e != nil || got.Status != "failed" {
|
||||
t.Fatalf("exhausted status %+v %v", got, e)
|
||||
}
|
||||
if _, e = p.RetryJob(ctx, tenant, j.ID); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
items, e = p.ClaimJobs(ctx, "worker", 1, time.Minute)
|
||||
if e != nil || len(items) != 1 || items[0].Attempt != 1 {
|
||||
t.Fatalf("operator retry %+v %v", items, e)
|
||||
}
|
||||
if e = p.ReleaseJob(ctx, j.ID, items[0].OwnerID, items[0].FencingToken); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
items, e = p.ClaimJobs(ctx, "worker-2", 1, time.Minute)
|
||||
if e != nil || len(items) != 1 || items[0].Attempt != 1 {
|
||||
t.Fatalf("shutdown consumed retry %+v %v", items, e)
|
||||
}
|
||||
failedPool, e := db.OpenPostgresPool(ctx, os.Getenv("EVOBGP_TEST_DATABASE_URL"))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
failedPool.Close()
|
||||
broken := &Postgres{pool: failedPool}
|
||||
if accepted, created, e := broken.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: "test"}); e == nil || created || accepted.ID != "" {
|
||||
t.Fatalf("accepted on failed DB: %+v %v %v", accepted, created, e)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestJobEnqueueTenantLocksDiffer(t *testing.T) {
|
||||
const tenantA = "11111111-1111-1111-1111-111111111111"
|
||||
const tenantB = "22222222-2222-2222-2222-222222222222"
|
||||
a := jobEnqueueTenantLock(tenantA)
|
||||
b := jobEnqueueTenantLock(tenantB)
|
||||
if a == b || a == jobEnqueueGlobalLock || b == jobEnqueueGlobalLock {
|
||||
t.Fatalf("locks a=%d b=%d global=%d", a, b, jobEnqueueGlobalLock)
|
||||
}
|
||||
if jobEnqueueTenantLock(tenantA) != a {
|
||||
t.Fatal("tenant lock is not stable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnqueueJobTenantLocksDoNotOverlapIntegration(t *testing.T) {
|
||||
p, tenantA := queueIntegration(t)
|
||||
ctx := context.Background()
|
||||
tenantB := uuid.NewString()
|
||||
if _, err := p.pool.Exec(ctx, `INSERT INTO tenant(id,name,slug) VALUES($1,'queue b',$2)`, tenantB, "queue-b-"+tenantB); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = p.pool.Exec(context.Background(), `DELETE FROM tenant WHERE id=$1`, tenantB)
|
||||
})
|
||||
|
||||
conn, err := p.pool.Acquire(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Release()
|
||||
tx, err := conn.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.Background()) }()
|
||||
if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1::bigint)`, jobEnqueueTenantLock(tenantA)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
otherDone := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, err := p.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenantB, Kind: "test"})
|
||||
otherDone <- err
|
||||
}()
|
||||
select {
|
||||
case err := <-otherDone:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("tenant B enqueue shared tenant A lock")
|
||||
}
|
||||
|
||||
short, cancel := context.WithTimeout(ctx, 400*time.Millisecond)
|
||||
defer cancel()
|
||||
if _, _, err = p.EnqueueJob(short, store.JobEnqueue{TenantID: tenantA, Kind: "test"}); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("same tenant did not wait: %v", err)
|
||||
}
|
||||
|
||||
key := "idem-key"
|
||||
in := store.JobEnqueue{TenantID: tenantB, Kind: "test", IdempotencyKey: &key, Meta: map[string]any{"n": 1}}
|
||||
first, created, err := p.EnqueueJob(ctx, in)
|
||||
if err != nil || !created || first.ID == "" {
|
||||
t.Fatalf("first=%+v created=%v err=%v", first, created, err)
|
||||
}
|
||||
in.Meta = map[string]any{"n": 2}
|
||||
if _, _, err = p.EnqueueJob(ctx, in); !errors.Is(err, store.ErrJobConflict) {
|
||||
t.Fatalf("idempotency conflict: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,371 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// jobEnqueueGlobalLock serializes the global pending-job count. It is not held
|
||||
// while idempotency is resolved; that uses jobEnqueueTenantLock.
|
||||
const jobEnqueueGlobalLock int64 = 38042341
|
||||
|
||||
const jobColumns = `id::text, tenant_id::text, kind, status, idempotency_key, module_id::text, meta_json, created_at, started_at, finished_at, error_message, COALESCE(owner_id,''), fencing_token, attempt, cancel_requested, lease_until, request_sequence, COALESCE(request_hash,'')`
|
||||
|
||||
func scanJob(row pgx.Row) (store.JobRecord, error) {
|
||||
var j store.JobRecord
|
||||
var meta []byte
|
||||
err := row.Scan(&j.ID, &j.TenantID, &j.Kind, &j.Status, &j.IdempotencyKey, &j.ModuleID, &meta, &j.CreatedAt, &j.StartedAt, &j.FinishedAt, &j.Error, &j.OwnerID, &j.FencingToken, &j.Attempt, &j.CancelRequested, &j.LeaseUntil, &j.RequestSequence, &j.RequestHash)
|
||||
if err != nil {
|
||||
return j, err
|
||||
}
|
||||
if err = json.Unmarshal(meta, &j.Meta); err != nil {
|
||||
return j, fmt.Errorf("repository: decode job metadata: %w", err)
|
||||
}
|
||||
return j, nil
|
||||
}
|
||||
|
||||
func jobLimit(env string, fallback int) int {
|
||||
n, e := strconv.Atoi(os.Getenv(env))
|
||||
if e == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func rollbackWithTimeout(tx pgx.Tx) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = tx.Rollback(ctx)
|
||||
}
|
||||
|
||||
func (p *Postgres) CountJobsByStatus(ctx context.Context) (map[string]int64, error) {
|
||||
rows, err := p.pool.Query(ctx, `SELECT status,count(*) FROM job_audit GROUP BY status`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]int64{}
|
||||
for rows.Next() {
|
||||
var status string
|
||||
var n int64
|
||||
if err := rows.Scan(&status, &n); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[status] = n
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (p *Postgres) FindJobByIdempotencyKey(ctx context.Context, tenant, key string) (store.JobRecord, error) {
|
||||
j, err := scanJob(p.pool.QueryRow(ctx, `SELECT `+jobColumns+` FROM job_audit WHERE tenant_id=$1::uuid AND idempotency_key=$2`, tenant, key))
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return j, store.ErrJobNotFound
|
||||
}
|
||||
return j, err
|
||||
}
|
||||
|
||||
// EnqueueJob inserts exactly one immutable request and returns the committed row.
|
||||
func (p *Postgres) EnqueueJob(ctx context.Context, in store.JobEnqueue) (store.JobRecord, bool, error) {
|
||||
h, err := store.JobRequestHash(in)
|
||||
if err != nil {
|
||||
return store.JobRecord{}, false, fmt.Errorf("repository: hash job: %w", err)
|
||||
}
|
||||
meta, err := json.Marshal(in.Meta)
|
||||
if err != nil {
|
||||
return store.JobRecord{}, false, fmt.Errorf("repository: encode job: %w", err)
|
||||
}
|
||||
tx, err := p.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return store.JobRecord{}, false, err
|
||||
}
|
||||
defer rollbackWithTimeout(tx)
|
||||
if err = lockJobEnqueue(ctx, tx, jobEnqueueTenantLock(in.TenantID)); err != nil {
|
||||
return store.JobRecord{}, false, err
|
||||
}
|
||||
if in.IdempotencyKey != nil && *in.IdempotencyKey != "" {
|
||||
var existingHash string
|
||||
j, e := scanJob(tx.QueryRow(ctx, `SELECT `+jobColumns+` FROM job_audit WHERE tenant_id=$1::uuid AND idempotency_key=$2 FOR UPDATE`, in.TenantID, *in.IdempotencyKey))
|
||||
if e == nil {
|
||||
if err = tx.QueryRow(ctx, `SELECT COALESCE(request_hash,'') FROM job_audit WHERE id=$1::uuid`, j.ID).Scan(&existingHash); err != nil {
|
||||
return store.JobRecord{}, false, err
|
||||
}
|
||||
if existingHash != h {
|
||||
return store.JobRecord{}, false, store.ErrJobConflict
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return store.JobRecord{}, false, err
|
||||
}
|
||||
return j, false, nil
|
||||
}
|
||||
if !errors.Is(e, pgx.ErrNoRows) {
|
||||
return store.JobRecord{}, false, e
|
||||
}
|
||||
}
|
||||
var tenantN int
|
||||
if err = tx.QueryRow(ctx, `SELECT count(*)::int FROM job_audit WHERE status IN ('queued','running') AND tenant_id=$1::uuid`, in.TenantID).Scan(&tenantN); err != nil {
|
||||
return store.JobRecord{}, false, err
|
||||
}
|
||||
if tenantN >= jobLimit("EVOBGP_JOB_MAX_TENANT_PENDING", 1000) {
|
||||
return store.JobRecord{}, false, store.ErrJobQueueFull
|
||||
}
|
||||
if err = lockJobEnqueue(ctx, tx, jobEnqueueGlobalLock); err != nil {
|
||||
return store.JobRecord{}, false, err
|
||||
}
|
||||
var global int
|
||||
if err = tx.QueryRow(ctx, `SELECT count(*)::int FROM job_audit WHERE status IN ('queued','running')`).Scan(&global); err != nil {
|
||||
return store.JobRecord{}, false, err
|
||||
}
|
||||
if global >= jobLimit("EVOBGP_JOB_MAX_PENDING", 10000) {
|
||||
return store.JobRecord{}, false, store.ErrJobQueueFull
|
||||
}
|
||||
id := uuid.NewString()
|
||||
var idem, mod any
|
||||
if in.IdempotencyKey != nil && *in.IdempotencyKey != "" {
|
||||
idem = *in.IdempotencyKey
|
||||
}
|
||||
if in.ModuleID != nil && *in.ModuleID != "" {
|
||||
mod = *in.ModuleID
|
||||
}
|
||||
j, err := scanJob(tx.QueryRow(ctx, `INSERT INTO job_audit(id,tenant_id,kind,status,idempotency_key,module_id,meta_json,request_hash,request_json) VALUES($1::uuid,$2::uuid,$3,'queued',$4,$5::uuid,$6::jsonb,$7,$6::jsonb) RETURNING `+jobColumns, id, in.TenantID, in.Kind, idem, mod, meta, h))
|
||||
if err != nil {
|
||||
return store.JobRecord{}, false, err
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return store.JobRecord{}, false, err
|
||||
}
|
||||
return j, true, nil
|
||||
}
|
||||
|
||||
// ClaimJobs atomically acquires available rows with a fresh fencing token.
|
||||
func (p *Postgres) ClaimJobs(ctx context.Context, owner string, limit int, lease time.Duration) ([]store.JobRecord, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if limit > 64 {
|
||||
limit = 64
|
||||
}
|
||||
if lease <= 0 {
|
||||
lease = time.Minute
|
||||
}
|
||||
rows, err := p.pool.Query(ctx, `WITH picked AS (
|
||||
SELECT id AS picked_id FROM job_audit WHERE ((status='queued' AND available_at<=now()) OR (status='running' AND lease_until<clock_timestamp() AND NOT cancel_requested)) AND attempt<3
|
||||
ORDER BY available_at,created_at,request_sequence,id FOR UPDATE SKIP LOCKED LIMIT $1
|
||||
) UPDATE job_audit j SET status='running',owner_id=$2,lease_until=clock_timestamp()+$3::bigint * interval '1 millisecond',attempt=j.attempt+1,fencing_token=j.fencing_token+1,started_at=COALESCE(j.started_at,now()) FROM picked WHERE j.id=picked.picked_id RETURNING `+jobColumns, limit, owner, lease.Milliseconds())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]store.JobRecord, 0, limit)
|
||||
for rows.Next() {
|
||||
j, e := scanJob(rows)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
out = append(out, j)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ReapExpiredJobs marks orphaned cancellations and exhausted retries terminal.
|
||||
// It is separate from ClaimJobs so a once-per-second claim does not rewrite job_audit.
|
||||
func (p *Postgres) ReapExpiredJobs(ctx context.Context) error {
|
||||
if _, err := p.pool.Exec(ctx, `UPDATE job_audit SET status='cancelled',finished_at=now(),owner_id=NULL,lease_until=NULL WHERE status='running' AND lease_until<clock_timestamp() AND cancel_requested`); err != nil {
|
||||
return fmt.Errorf("repository: reap cancelled jobs: %w", err)
|
||||
}
|
||||
if _, err := p.pool.Exec(ctx, `UPDATE job_audit SET status='failed',finished_at=now(),error_message='retry limit reached',owner_id=NULL,lease_until=NULL WHERE ((status='running' AND lease_until<clock_timestamp()) OR status='queued') AND attempt>=3`); err != nil {
|
||||
return fmt.Errorf("repository: reap exhausted jobs: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HeartbeatJob extends a live lease and reports a cross-replica cancel request.
|
||||
func (p *Postgres) HeartbeatJob(ctx context.Context, id, owner string, token int64, lease time.Duration) (bool, error) {
|
||||
if lease <= 0 {
|
||||
lease = time.Minute
|
||||
}
|
||||
var cancelled bool
|
||||
err := p.pool.QueryRow(ctx, `UPDATE job_audit SET lease_until=clock_timestamp()+$4::bigint * interval '1 millisecond' WHERE id=$1::uuid AND owner_id=$2 AND fencing_token=$3 AND status='running' AND lease_until>clock_timestamp() RETURNING cancel_requested`, id, owner, token, lease.Milliseconds()).Scan(&cancelled)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, store.ErrJobLeaseLost
|
||||
}
|
||||
return cancelled, err
|
||||
}
|
||||
|
||||
// CheckpointJob persists mutable progress without altering the immutable request.
|
||||
func (p *Postgres) CheckpointJob(ctx context.Context, j store.JobRecord) error {
|
||||
meta, err := json.Marshal(j.Meta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tag, err := p.pool.Exec(ctx, `UPDATE job_audit SET meta_json=$4::jsonb WHERE id=$1::uuid AND owner_id=$2 AND fencing_token=$3 AND status='running' AND lease_until>clock_timestamp()`, j.ID, j.OwnerID, j.FencingToken, meta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return store.ErrJobLeaseLost
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FinishJob commits terminal status only for the current live lease owner.
|
||||
func (p *Postgres) FinishJob(ctx context.Context, j store.JobRecord) error {
|
||||
meta, err := json.Marshal(j.Meta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tag, err := p.pool.Exec(ctx, `UPDATE job_audit SET status=CASE WHEN cancel_requested THEN 'cancelled' ELSE $4 END,error_message=CASE WHEN cancel_requested THEN NULL ELSE $5 END,finished_at=now(),meta_json=$6::jsonb,owner_id=NULL,lease_until=NULL WHERE id=$1::uuid AND owner_id=$2 AND fencing_token=$3 AND status='running' AND lease_until>clock_timestamp()`, j.ID, j.OwnerID, j.FencingToken, j.Status, j.Error, meta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return store.ErrJobLeaseLost
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReleaseJob returns an owned running job to the queue during graceful shutdown.
|
||||
func (p *Postgres) ReleaseJob(ctx context.Context, id, owner string, token int64) error {
|
||||
tag, err := p.pool.Exec(ctx, `UPDATE job_audit SET status=CASE WHEN cancel_requested THEN 'cancelled' ELSE 'queued' END,finished_at=CASE WHEN cancel_requested THEN now() ELSE NULL END,owner_id=NULL,lease_until=NULL,available_at=now(),attempt=GREATEST(attempt-1,0) WHERE id=$1::uuid AND owner_id=$2 AND fencing_token=$3 AND status='running'`, id, owner, token)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return store.ErrJobLeaseLost
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetJob returns a tenant-scoped job from shared storage.
|
||||
func (p *Postgres) GetJob(ctx context.Context, tenant, id string) (store.JobRecord, error) {
|
||||
if _, err := uuid.Parse(id); err != nil {
|
||||
return store.JobRecord{}, store.ErrJobNotFound
|
||||
}
|
||||
j, err := scanJob(p.pool.QueryRow(ctx, `SELECT `+jobColumns+` FROM job_audit WHERE id=$1::uuid AND tenant_id=$2::uuid`, id, tenant))
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return j, store.ErrJobNotFound
|
||||
}
|
||||
return j, err
|
||||
}
|
||||
|
||||
// ListJobs returns a bounded shared page in creation order.
|
||||
func (p *Postgres) ListJobs(ctx context.Context, tenant, status, kind, cursor string, limit int) ([]store.JobRecord, string, bool, error) {
|
||||
if limit <= 0 || limit > 100 {
|
||||
limit = 50
|
||||
}
|
||||
off, _ := strconv.Atoi(cursor)
|
||||
if off < 0 {
|
||||
off = 0
|
||||
}
|
||||
rows, err := p.pool.Query(ctx, `SELECT `+jobColumns+` FROM job_audit WHERE tenant_id=$1::uuid AND ($2='' OR status=$2) AND ($3='' OR kind=$3) ORDER BY created_at DESC,request_sequence DESC,id DESC OFFSET $4 LIMIT $5`, tenant, status, kind, off, limit+1)
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]store.JobRecord, 0, limit+1)
|
||||
for rows.Next() {
|
||||
j, e := scanJob(rows)
|
||||
if e != nil {
|
||||
return nil, "", false, e
|
||||
}
|
||||
out = append(out, j)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
more := len(out) > limit
|
||||
if more {
|
||||
out = out[:limit]
|
||||
}
|
||||
next := ""
|
||||
if more {
|
||||
next = strconv.Itoa(off + limit)
|
||||
}
|
||||
return out, next, more, nil
|
||||
}
|
||||
|
||||
// CancelJob atomically cancels queued work or signals its active owner.
|
||||
func (p *Postgres) CancelJob(ctx context.Context, tenant, id string) (store.JobRecord, error) {
|
||||
j, err := scanJob(p.pool.QueryRow(ctx, `UPDATE job_audit SET status=CASE WHEN status='queued' THEN 'cancelled' ELSE status END,finished_at=CASE WHEN status='queued' THEN now() ELSE finished_at END,cancel_requested=CASE WHEN status='running' THEN true ELSE cancel_requested END WHERE id=$1::uuid AND tenant_id=$2::uuid RETURNING `+jobColumns, id, tenant))
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return j, store.ErrJobNotFound
|
||||
}
|
||||
return j, err
|
||||
}
|
||||
|
||||
// RetryJob explicitly schedules a failed operation under its original ID.
|
||||
func (p *Postgres) RetryJob(ctx context.Context, tenant, id string) (store.JobRecord, error) {
|
||||
tx, err := p.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return store.JobRecord{}, err
|
||||
}
|
||||
defer rollbackWithTimeout(tx)
|
||||
if err = lockJobEnqueue(ctx, tx, jobEnqueueTenantLock(tenant)); err != nil {
|
||||
return store.JobRecord{}, err
|
||||
}
|
||||
old, err := scanJob(tx.QueryRow(ctx, `SELECT `+jobColumns+` FROM job_audit WHERE id=$1::uuid AND tenant_id=$2::uuid FOR UPDATE`, id, tenant))
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return store.JobRecord{}, store.ErrJobNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return store.JobRecord{}, err
|
||||
}
|
||||
if old.Status != "failed" {
|
||||
return store.JobRecord{}, store.ErrJobConflict
|
||||
}
|
||||
var pending int
|
||||
if err = tx.QueryRow(ctx, `SELECT count(*)::int FROM job_audit WHERE status IN ('queued','running') AND tenant_id=$1::uuid`, tenant).Scan(&pending); err != nil {
|
||||
return store.JobRecord{}, err
|
||||
}
|
||||
if pending >= jobLimit("EVOBGP_JOB_MAX_TENANT_PENDING", 1000) {
|
||||
return store.JobRecord{}, store.ErrJobQueueFull
|
||||
}
|
||||
if err = lockJobEnqueue(ctx, tx, jobEnqueueGlobalLock); err != nil {
|
||||
return store.JobRecord{}, err
|
||||
}
|
||||
var global int
|
||||
if err = tx.QueryRow(ctx, `SELECT count(*)::int FROM job_audit WHERE status IN ('queued','running')`).Scan(&global); err != nil {
|
||||
return store.JobRecord{}, err
|
||||
}
|
||||
if global >= jobLimit("EVOBGP_JOB_MAX_PENDING", 10000) {
|
||||
return store.JobRecord{}, store.ErrJobQueueFull
|
||||
}
|
||||
j, err := scanJob(tx.QueryRow(ctx, `UPDATE job_audit SET status='queued',available_at=now(),started_at=NULL,finished_at=NULL,error_message=NULL,cancel_requested=false,attempt=0,owner_id=NULL,lease_until=NULL,meta_json=COALESCE(request_json,meta_json) WHERE id=$1::uuid AND tenant_id=$2::uuid AND status='failed' RETURNING `+jobColumns, id, tenant))
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return j, store.ErrJobConflict
|
||||
}
|
||||
if err != nil {
|
||||
return j, err
|
||||
}
|
||||
return j, tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func lockJobEnqueue(ctx context.Context, tx pgx.Tx, key int64) error {
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1::bigint)`, key); err != nil {
|
||||
return fmt.Errorf("repository: lock job enqueue: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// jobEnqueueTenantLock is the per-tenant advisory key for idempotency and the tenant pending limit.
|
||||
func jobEnqueueTenantLock(tenantID string) int64 {
|
||||
h := fnv.New64a()
|
||||
_, _ = h.Write([]byte("evobgp-job-enqueue-v1\x00"))
|
||||
_, _ = h.Write([]byte(strings.TrimSpace(tenantID)))
|
||||
key := int64(h.Sum64())
|
||||
if key == jobEnqueueGlobalLock {
|
||||
key ^= 0x5a17
|
||||
}
|
||||
return key
|
||||
}
|
||||
@@ -2,10 +2,11 @@ package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
@@ -13,155 +14,240 @@ import (
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// moduleSnapshotRowExistsOnce caches the schema check for the process lifetime.
|
||||
// The table is created by migrations and never disappears at runtime; DB errors are
|
||||
// not cached so a transient outage falls back to the JSON path only once.
|
||||
var moduleSnapshotRowExistsOnce sync.Once
|
||||
|
||||
var moduleSnapshotRowExistsCached bool
|
||||
|
||||
func moduleSnapshotRowTableExists(ctx context.Context, q queryRower) bool {
|
||||
moduleSnapshotRowExistsOnce.Do(func() {
|
||||
var n int
|
||||
err := q.QueryRow(ctx, `
|
||||
SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema = 'public' AND table_name = 'module_prefix_snapshot_row'
|
||||
LIMIT 1`).Scan(&n)
|
||||
// A query error means the backend is unreachable or information_schema is hidden;
|
||||
// treat as "absent" so callers fall back to prefixes_json, but retry next call.
|
||||
if err != nil {
|
||||
moduleSnapshotRowExistsOnce = sync.Once{}
|
||||
return
|
||||
}
|
||||
moduleSnapshotRowExistsCached = true
|
||||
})
|
||||
return moduleSnapshotRowExistsCached
|
||||
// ListModuleSnapshotSourceRows returns persisted rows for one source key.
|
||||
func (p *Postgres) ListModuleSnapshotSourceRows(ctx context.Context, tenantID, moduleID, source string) ([]store.PrefixRow, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := p.pool.Query(ctx, `
|
||||
SELECT prefix, community_id::text, source
|
||||
FROM module_prefix_snapshot_row
|
||||
WHERE tenant_id = $1::uuid AND module_id = $2::uuid AND source = $3
|
||||
ORDER BY ord`, tenantID, moduleID, source)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("repository: read snapshot source rows: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out, err := readModuleSnapshotRows(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) GetModulePrefixSnapshot(tenantID, moduleID string) (*store.ModulePrefixSnapshot, bool, error) {
|
||||
ctx := context.Background()
|
||||
func (p *Postgres) GetModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string) (*store.ModulePrefixSnapshot, bool, error) {
|
||||
tx, err := p.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly})
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("repository: begin snapshot read: %w", err)
|
||||
}
|
||||
defer rollbackSnapshotTx(tx)
|
||||
snap, ok, err := readModuleSnapshotTx(ctx, tx, tenantID, moduleID)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, false, fmt.Errorf("repository: commit snapshot read: %w", err)
|
||||
}
|
||||
return snap, ok, nil
|
||||
}
|
||||
|
||||
func readModuleSnapshotTx(ctx context.Context, tx pgx.Tx, tenantID, moduleID string) (*store.ModulePrefixSnapshot, bool, error) {
|
||||
var inputHash string
|
||||
var collectedAt time.Time
|
||||
err := p.pool.QueryRow(ctx, `
|
||||
SELECT input_hash, collected_at
|
||||
var version int64
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT input_hash, collected_at, snapshot_version
|
||||
FROM module_prefix_snapshot
|
||||
WHERE tenant_id = $1 AND module_id = $2`,
|
||||
tenantID, moduleID).Scan(&inputHash, &collectedAt)
|
||||
tenantID, moduleID).Scan(&inputHash, &collectedAt, &version)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, false, nil
|
||||
}
|
||||
return nil, false, err
|
||||
}
|
||||
var prefixes []store.PrefixRow
|
||||
if moduleSnapshotRowTableExists(ctx, p.pool) {
|
||||
rows, qerr := p.pool.Query(ctx, `
|
||||
rows, qerr := tx.Query(ctx, `
|
||||
SELECT prefix, community_id::text, source
|
||||
FROM module_prefix_snapshot_row
|
||||
WHERE tenant_id = $1::uuid AND module_id = $2::uuid
|
||||
ORDER BY ord`, tenantID, moduleID)
|
||||
if qerr != nil {
|
||||
return nil, false, qerr
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var pr store.PrefixRow
|
||||
var comm *string
|
||||
if err := rows.Scan(&pr.Prefix, &comm, &pr.Source); err != nil {
|
||||
continue
|
||||
}
|
||||
pr.CommunityID = comm
|
||||
prefixes = append(prefixes, pr)
|
||||
}
|
||||
} else {
|
||||
var raw []byte
|
||||
if err := p.pool.QueryRow(ctx, `
|
||||
SELECT prefixes_json FROM module_prefix_snapshot
|
||||
WHERE tenant_id = $1 AND module_id = $2`, tenantID, moduleID).Scan(&raw); err == nil && len(raw) > 0 {
|
||||
_ = json.Unmarshal(raw, &prefixes)
|
||||
}
|
||||
if qerr != nil {
|
||||
return nil, false, fmt.Errorf("repository: read snapshot rows: %w", qerr)
|
||||
}
|
||||
defer rows.Close()
|
||||
prefixes, err := readModuleSnapshotRows(rows)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return &store.ModulePrefixSnapshot{
|
||||
InputHash: inputHash,
|
||||
CollectedAt: collectedAt.UTC(),
|
||||
Version: version,
|
||||
Prefixes: prefixes,
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) SetModulePrefixSnapshot(tenantID, moduleID, inputHash string, prefixes []store.PrefixRow) error {
|
||||
type moduleSnapshotRows interface {
|
||||
Next() bool
|
||||
Scan(...any) error
|
||||
Err() error
|
||||
}
|
||||
|
||||
func readModuleSnapshotRows(rows moduleSnapshotRows) ([]store.PrefixRow, error) {
|
||||
var prefixes []store.PrefixRow
|
||||
for rows.Next() {
|
||||
var pr store.PrefixRow
|
||||
var comm *string
|
||||
if err := rows.Scan(&pr.Prefix, &comm, &pr.Source); err != nil {
|
||||
return nil, fmt.Errorf("repository: scan snapshot row: %w", err)
|
||||
}
|
||||
pr.CommunityID = comm
|
||||
prefixes = append(prefixes, pr)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("repository: iterate snapshot rows: %w", err)
|
||||
}
|
||||
return prefixes, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) SetModulePrefixSnapshot(ctx context.Context, tenantID, moduleID, inputHash string, prefixes []store.PrefixRow) error {
|
||||
if strings.TrimSpace(tenantID) == "" || strings.TrimSpace(moduleID) == "" || strings.TrimSpace(inputHash) == "" {
|
||||
return store.ErrInvalidInput
|
||||
}
|
||||
ctx := context.Background()
|
||||
return p.UpdateModulePrefixSnapshot(ctx, tenantID, moduleID, func(*store.ModulePrefixSnapshot) (*store.ModulePrefixSnapshot, error) {
|
||||
return &store.ModulePrefixSnapshot{InputHash: inputHash, Prefixes: prefixes}, nil
|
||||
})
|
||||
}
|
||||
|
||||
func rollbackSnapshotTx(tx pgx.Tx) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = tx.Rollback(ctx)
|
||||
}
|
||||
|
||||
// UpdateModulePrefixSnapshot serializes read, mutation and write on the same transaction connection.
|
||||
func (p *Postgres) UpdateModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string, update store.ModuleSnapshotUpdate) error {
|
||||
if update == nil || strings.TrimSpace(tenantID) == "" || strings.TrimSpace(moduleID) == "" {
|
||||
return store.ErrInvalidInput
|
||||
}
|
||||
ctx, cancel, timeoutInstalled := store.SnapshotWriteBudget(ctx)
|
||||
defer cancel()
|
||||
tx, err := p.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("repository: begin snapshot update: %w", err)
|
||||
}
|
||||
defer rollbackSnapshotTx(tx)
|
||||
if err := setSnapshotStatementTimeout(ctx, tx, timeoutInstalled); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1::bigint)`, moduleSnapshotAdvisoryKey(tenantID, moduleID)); err != nil {
|
||||
return fmt.Errorf("repository: lock snapshot transaction: %w", err)
|
||||
}
|
||||
current, _, err := readModuleSnapshotTx(ctx, tx, tenantID, moduleID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
_, err = tx.Exec(ctx, `
|
||||
next, err := update(current)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if next != nil && strings.TrimSpace(next.InputHash) == "" {
|
||||
return store.ErrInvalidInput
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if next == nil {
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM module_prefix_snapshot WHERE tenant_id=$1::uuid AND module_id=$2::uuid`, tenantID, moduleID); err != nil {
|
||||
return fmt.Errorf("repository: delete module snapshot: %w", err)
|
||||
}
|
||||
} else if !store.SameSnapshotContent(current, next) {
|
||||
if err := writeModuleSnapshotTx(ctx, tx, tenantID, moduleID, next); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
hash := ""
|
||||
if next != nil {
|
||||
hash = next.InputHash
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `UPDATE module SET input_hash=$3 WHERE tenant_id=$1::uuid AND id=$2::uuid AND deleted_at IS NULL`, tenantID, moduleID, hash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("repository: update module input hash: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return store.ErrNotFound
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("repository: commit module snapshot: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeModuleSnapshotTx(ctx context.Context, tx pgx.Tx, tenantID, moduleID string, next *store.ModulePrefixSnapshot) error {
|
||||
_, err := tx.Exec(ctx, `
|
||||
INSERT INTO module_prefix_snapshot (tenant_id, module_id, input_hash, collected_at)
|
||||
VALUES ($1::uuid, $2::uuid, $3, now())
|
||||
ON CONFLICT (tenant_id, module_id) DO UPDATE SET
|
||||
input_hash = EXCLUDED.input_hash,
|
||||
collected_at = EXCLUDED.collected_at`,
|
||||
tenantID, moduleID, inputHash)
|
||||
tenantID, moduleID, next.InputHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if moduleSnapshotRowTableExists(ctx, tx) {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
if _, err := tx.Exec(ctx, `
|
||||
DELETE FROM module_prefix_snapshot_row
|
||||
WHERE tenant_id = $1::uuid AND module_id = $2::uuid`, tenantID, moduleID); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(prefixes) > 0 {
|
||||
_, err = tx.CopyFrom(
|
||||
ctx,
|
||||
pgx.Identifier{"module_prefix_snapshot_row"},
|
||||
[]string{"tenant_id", "module_id", "ord", "prefix", "community_id", "source"},
|
||||
pgx.CopyFromSlice(len(prefixes), func(i int) ([]any, error) {
|
||||
pr := prefixes[i]
|
||||
var comm any
|
||||
if pr.CommunityID != nil && strings.TrimSpace(*pr.CommunityID) != "" {
|
||||
comm = strings.TrimSpace(*pr.CommunityID)
|
||||
}
|
||||
src := pr.Source
|
||||
if strings.TrimSpace(src) == "" {
|
||||
src = "render"
|
||||
}
|
||||
return []any{tenantID, moduleID, i, strings.TrimSpace(pr.Prefix), comm, src}, nil
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
raw, err := json.Marshal(prefixes)
|
||||
return err
|
||||
}
|
||||
if len(next.Prefixes) > 0 {
|
||||
_, err = tx.CopyFrom(
|
||||
ctx,
|
||||
pgx.Identifier{"module_prefix_snapshot_row"},
|
||||
[]string{"tenant_id", "module_id", "ord", "prefix", "community_id", "source"},
|
||||
pgx.CopyFromSlice(len(next.Prefixes), func(i int) ([]any, error) {
|
||||
pr := next.Prefixes[i]
|
||||
var comm any
|
||||
if pr.CommunityID != nil && strings.TrimSpace(*pr.CommunityID) != "" {
|
||||
comm = strings.TrimSpace(*pr.CommunityID)
|
||||
}
|
||||
src := pr.Source
|
||||
if strings.TrimSpace(src) == "" {
|
||||
src = "render"
|
||||
}
|
||||
return []any{tenantID, moduleID, i, strings.TrimSpace(pr.Prefix), comm, src}, nil
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE module_prefix_snapshot SET prefixes_json = $3::jsonb
|
||||
WHERE tenant_id = $1::uuid AND module_id = $2::uuid`,
|
||||
tenantID, moduleID, string(raw)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Postgres) DeleteModulePrefixSnapshot(tenantID, moduleID string) error {
|
||||
ctx := context.Background()
|
||||
_, err := p.pool.Exec(ctx, `
|
||||
DELETE FROM module_prefix_snapshot WHERE tenant_id = $1 AND module_id = $2`,
|
||||
tenantID, moduleID)
|
||||
return err
|
||||
func (p *Postgres) DeleteModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string) error {
|
||||
return p.UpdateModulePrefixSnapshot(ctx, tenantID, moduleID, func(*store.ModulePrefixSnapshot) (*store.ModulePrefixSnapshot, error) {
|
||||
return nil, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (p *Postgres) SetModuleInputHash(tenantID, moduleID, hash string) error {
|
||||
ctx := context.Background()
|
||||
tag, err := p.pool.Exec(ctx, `
|
||||
func (p *Postgres) SetModuleInputHash(ctx context.Context, tenantID, moduleID, hash string) error {
|
||||
ctx, cancel, timeoutInstalled := store.SnapshotWriteBudget(ctx)
|
||||
defer cancel()
|
||||
tx, err := p.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("repository: begin module hash update: %w", err)
|
||||
}
|
||||
defer rollbackSnapshotTx(tx)
|
||||
if err := setSnapshotStatementTimeout(ctx, tx, timeoutInstalled); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1::bigint)`, moduleSnapshotAdvisoryKey(tenantID, moduleID)); err != nil {
|
||||
return fmt.Errorf("repository: lock module hash update: %w", err)
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE module SET input_hash = $3
|
||||
WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL`,
|
||||
moduleID, tenantID, hash)
|
||||
@@ -171,22 +257,29 @@ func (p *Postgres) SetModuleInputHash(tenantID, moduleID, hash string) error {
|
||||
if tag.RowsAffected() == 0 {
|
||||
return store.ErrNotFound
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// setSnapshotStatementTimeout bounds a lock when the caller did not set a deadline.
|
||||
func setSnapshotStatementTimeout(ctx context.Context, tx pgx.Tx, installed bool) error {
|
||||
if !installed {
|
||||
return nil
|
||||
}
|
||||
ms := store.SnapshotStatementTimeout(ctx).Milliseconds()
|
||||
if ms < 1 {
|
||||
ms = 1
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `SELECT set_config('statement_timeout', $1, true)`, strconv.FormatInt(ms, 10)); err != nil {
|
||||
return fmt.Errorf("repository: set snapshot statement timeout: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Postgres) LockModuleSnapshot(tenantID, moduleID string) func() {
|
||||
ctx := context.Background()
|
||||
conn, err := p.pool.Acquire(ctx)
|
||||
if err != nil {
|
||||
return func() {}
|
||||
}
|
||||
key := strings.TrimSpace(tenantID) + "\x00" + strings.TrimSpace(moduleID)
|
||||
if _, err := conn.Exec(ctx, `SELECT pg_advisory_lock(hashtext($1))`, key); err != nil {
|
||||
conn.Release()
|
||||
return func() {}
|
||||
}
|
||||
return func() {
|
||||
_, _ = conn.Exec(ctx, `SELECT pg_advisory_unlock(hashtext($1))`, key)
|
||||
conn.Release()
|
||||
}
|
||||
func moduleSnapshotAdvisoryKey(tenantID, moduleID string) int64 {
|
||||
h := fnv.New64a()
|
||||
_, _ = h.Write([]byte("evobgp-module-snapshot-v1\x00"))
|
||||
_, _ = h.Write([]byte(strings.TrimSpace(tenantID)))
|
||||
_, _ = h.Write([]byte{0})
|
||||
_, _ = h.Write([]byte(strings.TrimSpace(moduleID)))
|
||||
return int64(h.Sum64())
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func snapshotIntegrationModule(t *testing.T, p *Postgres, tenant string) string {
|
||||
t.Helper()
|
||||
id := uuid.NewString()
|
||||
if _, err := p.pool.Exec(context.Background(), `INSERT INTO module(id,tenant_id,type,name) VALUES($1,$2,'CDN_CIDRS',$3)`, id, tenant, id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func TestPostgresModuleSnapshotAtomicRMWMaxPoolOneIntegration(t *testing.T) {
|
||||
p, tenant := queueIntegration(t)
|
||||
module := snapshotIntegrationModule(t, p, tenant)
|
||||
cfg := p.pool.Config()
|
||||
cfg.MaxConns = 1
|
||||
smallPool, err := pgxpool.NewWithConfig(context.Background(), cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer smallPool.Close()
|
||||
small := &Postgres{pool: smallPool}
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 2)
|
||||
for i := 0; i < 2; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
errs <- small.UpdateModulePrefixSnapshot(context.Background(), tenant, module, func(cur *store.ModulePrefixSnapshot) (*store.ModulePrefixSnapshot, error) {
|
||||
var rows []store.PrefixRow
|
||||
if cur != nil {
|
||||
rows = append(rows, cur.Prefixes...)
|
||||
}
|
||||
rows = append(rows, store.PrefixRow{Prefix: fmt.Sprintf("192.0.%d.0/24", i), Source: fmt.Sprintf("cdn:%d", i)})
|
||||
return &store.ModulePrefixSnapshot{InputHash: "atomic", Prefixes: rows}, nil
|
||||
})
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
snap, ok, err := small.GetModulePrefixSnapshot(context.Background(), tenant, module)
|
||||
if err != nil || !ok || len(snap.Prefixes) != 2 {
|
||||
t.Fatalf("lost concurrent update: %+v %v %v", snap, ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresModuleSnapshotAdvisoryWaitCancellationIntegration(t *testing.T) {
|
||||
p, tenant := queueIntegration(t)
|
||||
module := snapshotIntegrationModule(t, p, tenant)
|
||||
other := snapshotIntegrationModule(t, p, tenant)
|
||||
cfg := p.pool.Config()
|
||||
cfg.MaxConns = 3
|
||||
testPool, err := pgxpool.NewWithConfig(context.Background(), cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer testPool.Close()
|
||||
locked := &Postgres{pool: testPool}
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
firstDone := make(chan error, 1)
|
||||
go func() {
|
||||
firstDone <- locked.UpdateModulePrefixSnapshot(context.Background(), tenant, module, func(*store.ModulePrefixSnapshot) (*store.ModulePrefixSnapshot, error) {
|
||||
close(started)
|
||||
<-release
|
||||
return &store.ModulePrefixSnapshot{InputHash: "first"}, nil
|
||||
})
|
||||
}()
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(5 * time.Second):
|
||||
close(release)
|
||||
t.Fatal("first transaction did not acquire lock")
|
||||
}
|
||||
defer func() {
|
||||
select {
|
||||
case <-release:
|
||||
default:
|
||||
close(release)
|
||||
}
|
||||
}()
|
||||
waitCtx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
err = locked.UpdateModulePrefixSnapshot(waitCtx, tenant, module, func(*store.ModulePrefixSnapshot) (*store.ModulePrefixSnapshot, error) {
|
||||
return &store.ModulePrefixSnapshot{InputHash: "second"}, nil
|
||||
})
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("same-key wait did not cancel: %v", err)
|
||||
}
|
||||
otherCtx, otherCancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer otherCancel()
|
||||
if err := locked.SetModulePrefixSnapshot(otherCtx, tenant, other, "other", nil); err != nil {
|
||||
t.Fatalf("unrelated module blocked: %v", err)
|
||||
}
|
||||
close(release)
|
||||
if err := <-firstDone; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresModuleSnapshotPartialWriteRollbackIntegration(t *testing.T) {
|
||||
p, tenant := queueIntegration(t)
|
||||
module := snapshotIntegrationModule(t, p, tenant)
|
||||
ctx := context.Background()
|
||||
if err := p.SetModulePrefixSnapshot(ctx, tenant, module, "before", []store.PrefixRow{{Prefix: "192.0.2.0/24", Source: "cdn:before"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := p.UpdateModulePrefixSnapshot(ctx, tenant, module, func(cur *store.ModulePrefixSnapshot) (*store.ModulePrefixSnapshot, error) {
|
||||
if cur == nil || len(cur.Prefixes) != 1 {
|
||||
return nil, errors.New("missing prior snapshot")
|
||||
}
|
||||
invalidCommunity := "not-a-uuid"
|
||||
return &store.ModulePrefixSnapshot{InputHash: "after", Prefixes: []store.PrefixRow{{Prefix: "198.51.100.0/24", CommunityID: &invalidCommunity, Source: "cdn:after"}}}, nil
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("invalid copy unexpectedly committed")
|
||||
}
|
||||
snap, ok, err := p.GetModulePrefixSnapshot(ctx, tenant, module)
|
||||
if err != nil || !ok || snap.InputHash != "before" || len(snap.Prefixes) != 1 || snap.Prefixes[0].Prefix != "192.0.2.0/24" {
|
||||
t.Fatalf("partial write visible: %+v %v %v", snap, ok, err)
|
||||
}
|
||||
var hash string
|
||||
if err := p.pool.QueryRow(ctx, `SELECT input_hash FROM module WHERE id=$1`, module).Scan(&hash); err != nil || hash != "before" {
|
||||
t.Fatalf("module input hash changed after rollback: %q %v", hash, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresModuleSnapshotKilledBackendRollsBackIntegration(t *testing.T) {
|
||||
p, tenant := queueIntegration(t)
|
||||
module := snapshotIntegrationModule(t, p, tenant)
|
||||
ctx := context.Background()
|
||||
if err := p.SetModulePrefixSnapshot(ctx, tenant, module, "before", []store.PrefixRow{{Prefix: "192.0.2.0/24"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := p.pool.Config()
|
||||
cfg.MaxConns = 1
|
||||
cfg.ConnConfig.RuntimeParams["application_name"] = "snapshot-kill-" + uuid.NewString()
|
||||
smallPool, err := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer smallPool.Close()
|
||||
small := &Postgres{pool: smallPool}
|
||||
killedBackend := false
|
||||
err = small.UpdateModulePrefixSnapshot(ctx, tenant, module, func(*store.ModulePrefixSnapshot) (*store.ModulePrefixSnapshot, error) {
|
||||
var pid int
|
||||
if err := p.pool.QueryRow(ctx, `SELECT pid FROM pg_stat_activity WHERE application_name=$1 AND state='idle in transaction' ORDER BY pid LIMIT 1`, cfg.ConnConfig.RuntimeParams["application_name"]).Scan(&pid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var killed bool
|
||||
if err := p.pool.QueryRow(ctx, `SELECT pg_terminate_backend($1)`, pid).Scan(&killed); err != nil || !killed {
|
||||
return nil, fmt.Errorf("terminate snapshot backend: killed=%v err=%w", killed, err)
|
||||
}
|
||||
killedBackend = true
|
||||
return &store.ModulePrefixSnapshot{InputHash: "after", Prefixes: []store.PrefixRow{{Prefix: "198.51.100.0/24"}}}, nil
|
||||
})
|
||||
if !killedBackend {
|
||||
t.Fatalf("backend kill was not exercised: %v", err)
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("killed backend committed snapshot")
|
||||
}
|
||||
verifyCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
snap, ok, err := p.GetModulePrefixSnapshot(verifyCtx, tenant, module)
|
||||
if err != nil || !ok || snap.InputHash != "before" || snap.Prefixes[0].Prefix != "192.0.2.0/24" {
|
||||
t.Fatalf("snapshot changed after backend loss: %+v %v %v", snap, ok, err)
|
||||
}
|
||||
var hash string
|
||||
if err := p.pool.QueryRow(ctx, `SELECT input_hash FROM module WHERE id=$1`, module).Scan(&hash); err != nil || hash != "before" {
|
||||
t.Fatalf("module input hash changed after backend loss: %q %v", hash, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"evobgp/internal/store"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestPostgresModuleSnapshotMethodsRespectCancellationIntegration(t *testing.T) {
|
||||
p, tenant := queueIntegration(t)
|
||||
module := uuid.NewString()
|
||||
if _, err := p.pool.Exec(context.Background(), `INSERT INTO module(id,tenant_id,type,name) VALUES($1,$2,'CDN_CIDRS','cancel-snapshot')`, module, tenant); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, _, err := p.GetModulePrefixSnapshot(ctx, tenant, module); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if err := p.SetModulePrefixSnapshot(ctx, tenant, module, "hash", []store.PrefixRow{{Prefix: "192.0.2.0/24"}}); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("set: %v", err)
|
||||
}
|
||||
if err := p.DeleteModulePrefixSnapshot(ctx, tenant, module); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("delete: %v", err)
|
||||
}
|
||||
if err := p.SetModuleInputHash(ctx, tenant, module, "hash"); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("input hash: %v", err)
|
||||
}
|
||||
rev := uuid.NewString()
|
||||
err := p.CreateRenderRevision(ctx, rev, tenant, module, nil, "cancel-hash", map[string]string{"bird.conf": "router id 192.0.2.1;\n"}, []store.PrefixRow{{Prefix: "203.0.113.0/24", Source: "test"}})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("revision: %v", err)
|
||||
}
|
||||
var n int
|
||||
if err := p.pool.QueryRow(context.Background(), `SELECT count(*)::int FROM config_revision WHERE id=$1::uuid`, rev).Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 0 {
|
||||
t.Fatal("cancelled revision write committed")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type partialSnapshotRows struct {
|
||||
returned bool
|
||||
scanErr error
|
||||
rowsErr error
|
||||
}
|
||||
|
||||
func (r *partialSnapshotRows) Next() bool {
|
||||
if r.returned {
|
||||
return false
|
||||
}
|
||||
r.returned = true
|
||||
return true
|
||||
}
|
||||
|
||||
func (r *partialSnapshotRows) Scan(dest ...any) error {
|
||||
if r.scanErr != nil {
|
||||
return r.scanErr
|
||||
}
|
||||
*dest[0].(*string) = "192.0.2.0/24"
|
||||
*dest[1].(**string) = nil
|
||||
*dest[2].(*string) = "cdn:test"
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *partialSnapshotRows) Err() error { return r.rowsErr }
|
||||
|
||||
func TestReadModuleSnapshotRowsRejectsPartialResults(t *testing.T) {
|
||||
sentinel := errors.New("simulated read failure")
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
rows *partialSnapshotRows
|
||||
}{
|
||||
{"scan", &partialSnapshotRows{scanErr: sentinel}},
|
||||
{"iteration", &partialSnapshotRows{rowsErr: sentinel}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
prefixes, err := readModuleSnapshotRows(tc.rows)
|
||||
if prefixes != nil || !errors.Is(err, sentinel) {
|
||||
t.Fatalf("partial snapshot returned: %+v %v", prefixes, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -115,6 +115,41 @@ func (p *Postgres) PeerSessionCountsByState() map[string]int {
|
||||
return out
|
||||
}
|
||||
|
||||
// ObserveStoreMetrics reads persisted revision and peer inventory aggregates for a Prometheus scrape.
|
||||
// An error invalidates the entire sample so a database outage cannot appear as an empty store.
|
||||
func (p *Postgres) ObserveStoreMetrics(ctx context.Context) (max, sum, peers int, states map[string]int, err error) {
|
||||
err = p.pool.QueryRow(ctx, `
|
||||
SELECT
|
||||
COALESCE(MAX((meta_json->>'materialized_prefix_count')::int), 0),
|
||||
COALESCE(SUM((meta_json->>'materialized_prefix_count')::int), 0)
|
||||
FROM config_revision`).Scan(&max, &sum)
|
||||
if err != nil {
|
||||
return 0, 0, 0, nil, fmt.Errorf("repository: collect stored revision metrics: %w", err)
|
||||
}
|
||||
err = p.pool.QueryRow(ctx, `SELECT COUNT(*) FROM bgp_peer`).Scan(&peers)
|
||||
if err != nil {
|
||||
return 0, 0, 0, nil, fmt.Errorf("repository: collect peer inventory metrics: %w", err)
|
||||
}
|
||||
rows, err := p.pool.Query(ctx, `SELECT COALESCE(meta_json->>'session_state','unknown'), COUNT(*) FROM bgp_peer GROUP BY 1`)
|
||||
if err != nil {
|
||||
return 0, 0, 0, nil, fmt.Errorf("repository: collect stored peer states: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
states = make(map[string]int)
|
||||
for rows.Next() {
|
||||
var state string
|
||||
var count int
|
||||
if err = rows.Scan(&state, &count); err != nil {
|
||||
return 0, 0, 0, nil, fmt.Errorf("repository: scan stored peer states: %w", err)
|
||||
}
|
||||
states[state] = count
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return 0, 0, 0, nil, fmt.Errorf("repository: read stored peer states: %w", err)
|
||||
}
|
||||
return max, sum, peers, states, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) ListModules(tenantID string) []*store.Module {
|
||||
ctx := context.Background()
|
||||
rows, err := p.pool.Query(ctx, `
|
||||
@@ -333,7 +368,9 @@ func (p *Postgres) CreateModule(tenantID string, in *store.Module) (*store.Modul
|
||||
}
|
||||
|
||||
func (p *Postgres) invalidateModuleInputHash(tenantID, moduleID string) {
|
||||
_ = p.SetModuleInputHash(tenantID, moduleID, "")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = p.SetModuleInputHash(ctx, tenantID, moduleID, "")
|
||||
}
|
||||
|
||||
func (p *Postgres) invalidateTenantModuleHashes(tenantID string) {
|
||||
@@ -1059,7 +1096,7 @@ func (p *Postgres) LatestPublishedRevision(speakerID string) (string, time.Time,
|
||||
|
||||
func (p *Postgres) ListTenantIDs() ([]string, error) {
|
||||
ctx := context.Background()
|
||||
rows, err := p.pool.Query(ctx, `SELECT id::text FROM tenant ORDER BY id`)
|
||||
rows, err := p.pool.Query(ctx, `SELECT id::text FROM tenant WHERE id <> $1::uuid ORDER BY id`, store.SystemJobTenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1075,14 +1112,24 @@ func (p *Postgres) ListTenantIDs() ([]string, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) CreateRenderRevision(revisionID, tenantID, moduleID string, parentRevisionID *string, contentHash string, previewFragments map[string]string, prefixes []store.PrefixRow) error {
|
||||
func (p *Postgres) CreateRenderRevision(ctx context.Context, revisionID, tenantID, moduleID string, parentRevisionID *string, contentHash string, previewFragments map[string]string, prefixes []store.PrefixRow) error {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
ctx, cancel, _ := store.SnapshotWriteBudget(ctx)
|
||||
defer cancel()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(revisionID) == "" {
|
||||
return store.ErrInvalidInput
|
||||
}
|
||||
if _, err := p.GetModule(tenantID, moduleID); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx := context.Background()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if previewFragments == nil {
|
||||
previewFragments = map[string]string{}
|
||||
}
|
||||
@@ -1092,9 +1139,12 @@ func (p *Postgres) CreateRenderRevision(revisionID, tenantID, moduleID string, p
|
||||
}
|
||||
tx, err := p.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("repository: begin render revision: %w", err)
|
||||
}
|
||||
defer rollbackWithTimeout(tx)
|
||||
if err := setRevisionStatementTimeout(ctx, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
|
||||
var parent any
|
||||
if parentRevisionID != nil && strings.TrimSpace(*parentRevisionID) != "" {
|
||||
@@ -1102,8 +1152,8 @@ func (p *Postgres) CreateRenderRevision(revisionID, tenantID, moduleID string, p
|
||||
}
|
||||
revID := strings.TrimSpace(revisionID)
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO config_revision (id, tenant_id, module_id, content_hash, parent_revision_id, meta_json)
|
||||
VALUES ($1::uuid, $2::uuid, $3::uuid, $4, $5::uuid, $6::jsonb)`,
|
||||
INSERT INTO config_revision (id, tenant_id, module_id, content_hash, parent_revision_id, meta_json)
|
||||
VALUES ($1::uuid, $2::uuid, $3::uuid, $4, $5::uuid, $6::jsonb)`,
|
||||
revID, tenantID, moduleID, strings.TrimSpace(contentHash), parent, meta)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1111,16 +1161,32 @@ func (p *Postgres) CreateRenderRevision(revisionID, tenantID, moduleID string, p
|
||||
if err := insertRevisionPreview(ctx, tx, revID, previewFragments); err != nil {
|
||||
return err
|
||||
}
|
||||
snapID, err := p.ensurePrefixSnapshot(ctx, tx, contentHash, prefixes)
|
||||
snapID, err := p.ensurePrefixSnapshot(ctx, tx, prefixes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := p.linkRevisionPrefixSnapshot(ctx, tx, revID, snapID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
if err := insertRenderManifestTx(ctx, tx, revID, tenantID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("repository: commit render revision: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// setRevisionStatementTimeout bounds the revision transaction the same way snapshot writes do:
|
||||
// the caller deadline wins, otherwise EVOBGP_SNAPSHOT_TIMEOUT (default 2 minutes).
|
||||
func setRevisionStatementTimeout(ctx context.Context, tx pgx.Tx) error {
|
||||
ms := store.SnapshotStatementTimeout(ctx).Milliseconds()
|
||||
if ms < 1 {
|
||||
ms = 1
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `SELECT set_config('statement_timeout', $1, true)`, strconv.FormatInt(ms, 10)); err != nil {
|
||||
return fmt.Errorf("repository: set revision statement timeout: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func TestObserveStoreMetricsReturnsDatabaseError(t *testing.T) {
|
||||
pool, err := pgxpool.New(context.Background(), "postgres://localhost/evobgp")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pool.Close()
|
||||
pg := &Postgres{pool: pool}
|
||||
_, _, _, _, err = pg.ObserveStoreMetrics(context.Background())
|
||||
if err == nil {
|
||||
t.Fatal("expected closed pool error")
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package repository
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"evobgp/internal/store"
|
||||
@@ -48,35 +49,39 @@ func (p *Postgres) revisionPrefixSnapshotID(ctx context.Context, revisionID stri
|
||||
return *snap, true
|
||||
}
|
||||
|
||||
func (p *Postgres) ensurePrefixSnapshot(ctx context.Context, db execQuerier, contentHash string, prefixes []store.PrefixRow) (string, error) {
|
||||
func (p *Postgres) ensurePrefixSnapshot(ctx context.Context, db execQuerier, prefixes []store.PrefixRow) (string, error) {
|
||||
if !prefixSnapshotTableExists(ctx, db) {
|
||||
return "", nil
|
||||
}
|
||||
hash := normalizeSnapshotHash(contentHash)
|
||||
hash := store.PrefixSnapshotHash(prefixes)
|
||||
snapID, err := p.resolvePrefixSnapshotID(ctx, db, hash)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var rowCount int
|
||||
_ = db.QueryRow(ctx, `SELECT COUNT(*)::int FROM prefix_snapshot_row WHERE snapshot_id = $1::uuid`, snapID).Scan(&rowCount)
|
||||
if rowCount > 0 {
|
||||
if rowCount > 0 || len(prefixes) == 0 {
|
||||
return snapID, nil
|
||||
}
|
||||
for i, pr := range prefixes {
|
||||
var comm any
|
||||
if pr.CommunityID != nil && strings.TrimSpace(*pr.CommunityID) != "" {
|
||||
comm = strings.TrimSpace(*pr.CommunityID)
|
||||
}
|
||||
src := pr.Source
|
||||
if strings.TrimSpace(src) == "" {
|
||||
src = "render"
|
||||
}
|
||||
if _, err := db.Exec(ctx, `
|
||||
INSERT INTO prefix_snapshot_row (snapshot_id, ord, prefix, community_id, source)
|
||||
VALUES ($1::uuid, $2, $3, $4::uuid, $5)`,
|
||||
snapID, i, strings.TrimSpace(pr.Prefix), comm, src); err != nil {
|
||||
return "", err
|
||||
}
|
||||
_, err = db.CopyFrom(
|
||||
ctx,
|
||||
pgx.Identifier{"prefix_snapshot_row"},
|
||||
[]string{"snapshot_id", "ord", "prefix", "community_id", "source"},
|
||||
pgx.CopyFromSlice(len(prefixes), func(i int) ([]any, error) {
|
||||
pr := prefixes[i]
|
||||
var comm any
|
||||
if pr.CommunityID != nil && strings.TrimSpace(*pr.CommunityID) != "" {
|
||||
comm = strings.TrimSpace(*pr.CommunityID)
|
||||
}
|
||||
src := pr.Source
|
||||
if strings.TrimSpace(src) == "" {
|
||||
src = "render"
|
||||
}
|
||||
return []any{snapID, i, strings.TrimSpace(pr.Prefix), comm, src}, nil
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("repository: copy prefix snapshot rows: %w", err)
|
||||
}
|
||||
return snapID, nil
|
||||
}
|
||||
|
||||
@@ -27,7 +27,8 @@ func TestEnsurePrefixSnapshotFillsEmptyExistingIntegration(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
hash := normalizeSnapshotHash("sha256:empty-fill-" + uuid.NewString())
|
||||
prefixes := []store.PrefixRow{{Prefix: "203.0.113.1/32", Source: "test"}}
|
||||
hash := store.PrefixSnapshotHash(prefixes)
|
||||
snapID := uuid.NewString()
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO prefix_snapshot (id, content_hash) VALUES ($1::uuid, $2)`, snapID, hash); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -42,9 +43,7 @@ func TestEnsurePrefixSnapshotFillsEmptyExistingIntegration(t *testing.T) {
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
|
||||
got, err := pg.ensurePrefixSnapshot(ctx, tx, "sha256:"+hash, []store.PrefixRow{
|
||||
{Prefix: "203.0.113.1/32", Source: "test"},
|
||||
})
|
||||
got, err := pg.ensurePrefixSnapshot(ctx, tx, prefixes)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -79,14 +78,13 @@ func TestEnsurePrefixSnapshotIdempotentIntegration(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
contentHash := "sha256:idempotent-" + uuid.NewString()
|
||||
prefixes := []store.PrefixRow{{Prefix: "198.51.100.0/24", Source: "test"}}
|
||||
|
||||
tx1, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id1, err := pg.ensurePrefixSnapshot(ctx, tx1, contentHash, prefixes)
|
||||
id1, err := pg.ensurePrefixSnapshot(ctx, tx1, prefixes)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -102,7 +100,7 @@ func TestEnsurePrefixSnapshotIdempotentIntegration(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = tx2.Rollback(ctx) }()
|
||||
id2, err := pg.ensurePrefixSnapshot(ctx, tx2, contentHash, prefixes)
|
||||
id2, err := pg.ensurePrefixSnapshot(ctx, tx2, prefixes)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -49,6 +49,7 @@ func loadRevisionPreview(ctx context.Context, q queryRower, revisionID string, m
|
||||
type execQuerier interface {
|
||||
queryRower
|
||||
Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error)
|
||||
CopyFrom(ctx context.Context, tableName pgx.Identifier, columnNames []string, rowSrc pgx.CopyFromSource) (int64, error)
|
||||
}
|
||||
|
||||
func insertRevisionPreview(ctx context.Context, db execQuerier, revisionID string, preview map[string]string) error {
|
||||
|
||||
@@ -27,7 +27,13 @@ func sqlPrunableRevisionsWhere(tenantParam, cutoffParam string) string {
|
||||
}
|
||||
|
||||
func (p *Postgres) EstimateRevisionPrune(tenantID string, cutoff time.Time, retentionMinutes int) (store.RevisionPruneEstimate, error) {
|
||||
ctx := context.Background()
|
||||
return p.estimateRevisionPrune(context.Background(), tenantID, cutoff, retentionMinutes)
|
||||
}
|
||||
|
||||
func (p *Postgres) estimateRevisionPrune(ctx context.Context, tenantID string, cutoff time.Time, retentionMinutes int) (store.RevisionPruneEstimate, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
out := store.RevisionPruneEstimate{
|
||||
RetentionMinutes: retentionMinutes,
|
||||
CutoffAt: cutoff.UTC(),
|
||||
@@ -88,8 +94,14 @@ func (p *Postgres) EstimateRevisionPrune(tenantID string, cutoff time.Time, rete
|
||||
}
|
||||
|
||||
func (p *Postgres) PruneRevisionsWithStats(tenantID string, cutoff time.Time) (store.RevisionPruneResult, error) {
|
||||
ctx := context.Background()
|
||||
est, err := p.EstimateRevisionPrune(tenantID, cutoff, 0)
|
||||
return p.pruneRevisionsWithStats(context.Background(), tenantID, cutoff)
|
||||
}
|
||||
|
||||
func (p *Postgres) pruneRevisionsWithStats(ctx context.Context, tenantID string, cutoff time.Time) (store.RevisionPruneResult, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
est, err := p.estimateRevisionPrune(ctx, tenantID, cutoff, 0)
|
||||
if err != nil {
|
||||
return store.RevisionPruneResult{}, err
|
||||
}
|
||||
@@ -97,6 +109,9 @@ func (p *Postgres) PruneRevisionsWithStats(tenantID string, cutoff time.Time) (s
|
||||
|
||||
where := sqlPrunableRevisionsWhere("$1", "$2")
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return out, err
|
||||
}
|
||||
cmd, err := p.pool.Exec(ctx, `
|
||||
DELETE FROM config_revision AS cr
|
||||
WHERE cr.id IN (
|
||||
@@ -116,6 +131,9 @@ func (p *Postgres) PruneRevisionsWithStats(tenantID string, cutoff time.Time) (s
|
||||
}
|
||||
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return out, err
|
||||
}
|
||||
snaps, rows, err := p.pruneUnreferencedPrefixSnapshots(ctx, revisionPruneBatchSize)
|
||||
if err != nil {
|
||||
return out, err
|
||||
@@ -129,8 +147,8 @@ func (p *Postgres) PruneRevisionsWithStats(tenantID string, cutoff time.Time) (s
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) PruneRevisionsBefore(tenantID string, cutoff time.Time) (int, error) {
|
||||
res, err := p.PruneRevisionsWithStats(tenantID, cutoff)
|
||||
func (p *Postgres) PruneRevisionsBefore(ctx context.Context, tenantID string, cutoff time.Time) (int, error) {
|
||||
res, err := p.pruneRevisionsWithStats(ctx, tenantID, cutoff)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
@@ -3,13 +3,14 @@ package repository
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"evobgp/internal/store"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func (p *Postgres) seedDemo(ctx context.Context) error {
|
||||
var n int
|
||||
if err := p.pool.QueryRow(ctx, `SELECT COUNT(*)::int FROM tenant`).Scan(&n); err != nil {
|
||||
if err := p.pool.QueryRow(ctx, `SELECT COUNT(*)::int FROM tenant WHERE id<>$1::uuid`, store.SystemJobTenantID).Scan(&n); err != nil {
|
||||
return err
|
||||
}
|
||||
if n > 0 {
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"evobgp/internal/store"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
func (p *Postgres) GetRenderGeneration(ctx context.Context, tenantID string) (int64, error) {
|
||||
var generation int64
|
||||
err := p.pool.QueryRow(ctx, `SELECT generation FROM tenant_render_generation WHERE tenant_id=$1::uuid`, tenantID).Scan(&generation)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return 0, store.ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("repository: render generation: %w", err)
|
||||
}
|
||||
return generation, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) ListModuleSnapshotHeaders(ctx context.Context, tenantID string) ([]store.ModuleSnapshotUse, error) {
|
||||
rows, err := p.pool.Query(ctx, `
|
||||
SELECT module_id::text, snapshot_version, COALESCE(input_hash, '')
|
||||
FROM module_prefix_snapshot
|
||||
WHERE tenant_id = $1::uuid
|
||||
ORDER BY module_id`, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("repository: list snapshot headers: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []store.ModuleSnapshotUse
|
||||
for rows.Next() {
|
||||
var item store.ModuleSnapshotUse
|
||||
if err := rows.Scan(&item.ModuleID, &item.SnapshotVersion, &item.InputHash); err != nil {
|
||||
return nil, fmt.Errorf("repository: scan snapshot header: %w", err)
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("repository: iterate snapshot headers: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) LatestRenderManifest(ctx context.Context, tenantID string) (*store.RenderManifestRecord, bool, error) {
|
||||
var revID string
|
||||
var gen *int64
|
||||
var raw []byte
|
||||
err := p.pool.QueryRow(ctx, `
|
||||
SELECT cr.id::text, m.config_generation, m.manifest_json
|
||||
FROM config_revision cr
|
||||
LEFT JOIN render_revision_manifest m ON m.revision_id = cr.id
|
||||
WHERE cr.tenant_id = $1::uuid
|
||||
ORDER BY cr.created_at DESC, cr.id DESC
|
||||
LIMIT 1`, tenantID).Scan(&revID, &gen, &raw)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("repository: latest render manifest: %w", err)
|
||||
}
|
||||
if gen == nil || len(raw) == 0 {
|
||||
return nil, false, nil
|
||||
}
|
||||
var doc struct {
|
||||
Snapshots []store.ModuleSnapshotUse `json:"snapshots"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return nil, false, fmt.Errorf("repository: decode render manifest: %w", err)
|
||||
}
|
||||
return &store.RenderManifestRecord{
|
||||
RevisionID: revID,
|
||||
Generation: *gen,
|
||||
Snapshots: doc.Snapshots,
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
func insertRenderManifestTx(ctx context.Context, tx pgx.Tx, revisionID, tenantID string) error {
|
||||
var gen int64
|
||||
err := tx.QueryRow(ctx, `
|
||||
SELECT generation FROM tenant_render_generation
|
||||
WHERE tenant_id = $1::uuid
|
||||
FOR UPDATE`, tenantID).Scan(&gen)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
gen = 1
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("repository: lock render generation: %w", err)
|
||||
}
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT module_id::text, snapshot_version, COALESCE(input_hash, '')
|
||||
FROM module_prefix_snapshot
|
||||
WHERE tenant_id = $1::uuid
|
||||
ORDER BY module_id`, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("repository: read snapshot versions: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var snaps []store.ModuleSnapshotUse
|
||||
for rows.Next() {
|
||||
var item store.ModuleSnapshotUse
|
||||
if err := rows.Scan(&item.ModuleID, &item.SnapshotVersion, &item.InputHash); err != nil {
|
||||
return fmt.Errorf("repository: scan snapshot version: %w", err)
|
||||
}
|
||||
snaps = append(snaps, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return fmt.Errorf("repository: iterate snapshot versions: %w", err)
|
||||
}
|
||||
body, err := json.Marshal(struct {
|
||||
Snapshots []store.ModuleSnapshotUse `json:"snapshots"`
|
||||
}{Snapshots: snaps})
|
||||
if err != nil {
|
||||
return fmt.Errorf("repository: encode render manifest: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO render_revision_manifest (revision_id, tenant_id, config_generation, manifest_json)
|
||||
VALUES ($1::uuid, $2::uuid, $3, $4::jsonb)`,
|
||||
revisionID, tenantID, gen, body); err != nil {
|
||||
return fmt.Errorf("repository: insert render manifest: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
const telemetryColumns = `tenant_id::text,target_id,COALESCE(speaker_id::text,''),kind,local_authority,config_fingerprint,bgp_json,health_json,next_poll_at,failure_count,discovery_cursor`
|
||||
const telemetryClaimColumns = `t.tenant_id::text,t.target_id,COALESCE(t.speaker_id::text,''),t.kind,t.local_authority,t.config_fingerprint,t.bgp_json,t.health_json,t.next_poll_at,t.failure_count,t.discovery_cursor,t.fencing_token`
|
||||
|
||||
func scanTelemetry(row pgx.Row) (store.TelemetrySnapshot, error) {
|
||||
var s store.TelemetrySnapshot
|
||||
var bgp, health []byte
|
||||
err := row.Scan(&s.TenantID, &s.TargetID, &s.SpeakerID, &s.Kind, &s.LocalAuthority, &s.ConfigFingerprint, &bgp, &health, &s.NextPollAt, &s.FailureCount, &s.DiscoveryCursor)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
if err := json.Unmarshal(bgp, &s.BGP); err != nil {
|
||||
return s, fmt.Errorf("repository: decode telemetry BGP: %w", err)
|
||||
}
|
||||
if err := json.Unmarshal(health, &s.Health); err != nil {
|
||||
return s, fmt.Errorf("repository: decode telemetry health: %w", err)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// ClaimTelemetry inserts at most one bounded batch of new targets and leases due rows.
|
||||
func (p *Postgres) ClaimTelemetry(ctx context.Context, owner string, limit int, lease time.Duration, localAuthority string) ([]store.TelemetryClaim, error) {
|
||||
if owner == "" || limit <= 0 {
|
||||
return nil, store.ErrInvalidInput
|
||||
}
|
||||
if limit > 64 {
|
||||
limit = 64
|
||||
}
|
||||
if lease <= 0 {
|
||||
lease = 15 * time.Second
|
||||
}
|
||||
_, err := p.pool.Exec(ctx, `INSERT INTO telemetry_snapshot(tenant_id,target_id,speaker_id,kind)
|
||||
SELECT s.tenant_id,s.id::text,s.id,'remote_agent' FROM bgp_speaker s
|
||||
WHERE s.role<>'master' AND COALESCE(s.meta_json->>'agent_domain','')<>'' AND COALESCE(s.meta_json->>'agent_secret','')<>''
|
||||
AND NOT EXISTS (SELECT 1 FROM telemetry_snapshot t WHERE t.tenant_id=s.tenant_id AND t.target_id=s.id::text)
|
||||
ORDER BY s.id LIMIT $1 ON CONFLICT DO NOTHING`, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("repository: materialize telemetry targets: %w", err)
|
||||
}
|
||||
if localAuthority != "" {
|
||||
_, err = p.pool.Exec(ctx, `INSERT INTO telemetry_snapshot(tenant_id,target_id,kind,local_authority)
|
||||
SELECT t.id,'cp-local/'||t.id::text||'/'||$3,'local_bird',$3 FROM tenant t
|
||||
WHERE t.id<>$2::uuid AND NOT EXISTS (SELECT 1 FROM telemetry_snapshot x WHERE x.tenant_id=t.id AND x.target_id='cp-local/'||t.id::text||'/'||$3)
|
||||
ORDER BY t.id LIMIT $1 ON CONFLICT DO NOTHING`, limit, store.SystemJobTenantID, localAuthority)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("repository: materialize local telemetry targets: %w", err)
|
||||
}
|
||||
}
|
||||
rows, err := p.pool.Query(ctx, `WITH picked AS (
|
||||
SELECT tenant_id,target_id FROM telemetry_snapshot WHERE next_poll_at<=clock_timestamp()
|
||||
AND (lease_until IS NULL OR lease_until<=clock_timestamp()) AND (kind<>'local_bird' OR local_authority=$4)
|
||||
AND (kind='local_bird' OR EXISTS (SELECT 1 FROM bgp_speaker s WHERE s.id=telemetry_snapshot.speaker_id AND s.role<>'master'
|
||||
AND COALESCE(s.meta_json->>'agent_domain','')<>'' AND COALESCE(s.meta_json->>'agent_secret','')<>''))
|
||||
ORDER BY next_poll_at,tenant_id,target_id FOR UPDATE SKIP LOCKED LIMIT $1
|
||||
) UPDATE telemetry_snapshot t SET owner_id=$2,lease_until=clock_timestamp()+$3::bigint*interval '1 millisecond',fencing_token=t.fencing_token+1
|
||||
FROM picked WHERE t.tenant_id=picked.tenant_id AND t.target_id=picked.target_id
|
||||
RETURNING `+telemetryClaimColumns, limit, owner, lease.Milliseconds(), localAuthority)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("repository: claim telemetry: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
claims := make([]store.TelemetryClaim, 0, limit)
|
||||
for rows.Next() {
|
||||
var c store.TelemetryClaim
|
||||
var bgp, health []byte
|
||||
if err = rows.Scan(&c.TenantID, &c.TargetID, &c.SpeakerID, &c.Kind, &c.LocalAuthority, &c.ConfigFingerprint, &bgp, &health, &c.NextPollAt, &c.FailureCount, &c.DiscoveryCursor, &c.FencingToken); err != nil {
|
||||
return nil, fmt.Errorf("repository: scan telemetry claim: %w", err)
|
||||
}
|
||||
if err = json.Unmarshal(bgp, &c.BGP); err != nil {
|
||||
return nil, fmt.Errorf("repository: decode telemetry BGP: %w", err)
|
||||
}
|
||||
if err = json.Unmarshal(health, &c.Health); err != nil {
|
||||
return nil, fmt.Errorf("repository: decode telemetry health: %w", err)
|
||||
}
|
||||
c.OwnerID = owner
|
||||
claims = append(claims, c)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("repository: iterate telemetry claims: %w", err)
|
||||
}
|
||||
rows.Close()
|
||||
for i := range claims {
|
||||
c := &claims[i]
|
||||
if c.Kind == "remote_agent" {
|
||||
c.Speaker, err = p.getTelemetrySpeaker(ctx, c.TenantID, c.SpeakerID)
|
||||
} else {
|
||||
var spID string
|
||||
err = p.pool.QueryRow(ctx, `SELECT id::text FROM bgp_speaker WHERE tenant_id=$1::uuid AND role='master' ORDER BY id LIMIT 1`, c.TenantID).Scan(&spID)
|
||||
if err == pgx.ErrNoRows {
|
||||
err = nil
|
||||
} else if err == nil {
|
||||
c.SpeakerID = spID
|
||||
c.Speaker, err = p.getTelemetrySpeaker(ctx, c.TenantID, spID)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("repository: resolve telemetry speaker: %w", err)
|
||||
}
|
||||
c.ConfigFingerprint = store.TelemetrySpeakerFingerprint(c.Speaker)
|
||||
}
|
||||
return claims, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) getTelemetrySpeaker(ctx context.Context, tenantID, speakerID string) (*store.Speaker, error) {
|
||||
var sp store.Speaker
|
||||
err := p.pool.QueryRow(ctx, `SELECT id::text,tenant_id::text,role,COALESCE(endpoint,''),COALESCE(meta_json::text,'{}') FROM bgp_speaker WHERE tenant_id=$1::uuid AND id=$2::uuid`, tenantID, speakerID).Scan(&sp.ID, &sp.TenantID, &sp.Role, &sp.Endpoint, &sp.MetaJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &sp, nil
|
||||
}
|
||||
|
||||
// CompleteTelemetry records an attempt only while the claim still owns its lease.
|
||||
func (p *Postgres) CompleteTelemetry(ctx context.Context, claim store.TelemetryClaim, result store.TelemetryResult, nextPoll time.Time) error {
|
||||
tx, err := p.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("repository: begin telemetry completion: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
var oldBGP, oldHealth []byte
|
||||
var token int64
|
||||
var owner string
|
||||
var leaseUntil time.Time
|
||||
var pending bool
|
||||
var failures int
|
||||
var oldFingerprint string
|
||||
var discoveryCursor int
|
||||
err = tx.QueryRow(ctx, `SELECT bgp_json,health_json,fencing_token,COALESCE(owner_id,''),COALESCE(lease_until,'epoch'::timestamptz),refresh_pending,failure_count,config_fingerprint,discovery_cursor
|
||||
FROM telemetry_snapshot WHERE tenant_id=$1::uuid AND target_id=$2 FOR UPDATE`, claim.TenantID, claim.TargetID).Scan(&oldBGP, &oldHealth, &token, &owner, &leaseUntil, &pending, &failures, &oldFingerprint, &discoveryCursor)
|
||||
if err == pgx.ErrNoRows || err == nil && (token != claim.FencingToken || owner != claim.OwnerID) {
|
||||
return store.ErrTelemetryLeaseLost
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("repository: read telemetry lease: %w", err)
|
||||
}
|
||||
var dbNow time.Time
|
||||
if err = tx.QueryRow(ctx, `SELECT clock_timestamp()`).Scan(&dbNow); err != nil {
|
||||
return fmt.Errorf("repository: telemetry clock: %w", err)
|
||||
}
|
||||
if !leaseUntil.After(dbNow) {
|
||||
return store.ErrTelemetryLeaseLost
|
||||
}
|
||||
if claim.Speaker != nil {
|
||||
var current store.Speaker
|
||||
err = tx.QueryRow(ctx, `SELECT id::text,role,COALESCE(endpoint,''),COALESCE(meta_json::text,'{}') FROM bgp_speaker WHERE id=$1::uuid AND tenant_id=$2::uuid FOR SHARE`, claim.Speaker.ID, claim.TenantID).Scan(¤t.ID, ¤t.Role, ¤t.Endpoint, ¤t.MetaJSON)
|
||||
if err == pgx.ErrNoRows || err == nil && store.TelemetrySpeakerFingerprint(¤t) != claim.ConfigFingerprint {
|
||||
return store.ErrTelemetryLeaseLost
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("repository: verify telemetry config: %w", err)
|
||||
}
|
||||
}
|
||||
if claim.Kind == "local_bird" {
|
||||
var masterID string
|
||||
err = tx.QueryRow(ctx, `SELECT id::text FROM bgp_speaker WHERE tenant_id=$1::uuid AND role='master' ORDER BY id LIMIT 1`, claim.TenantID).Scan(&masterID)
|
||||
if err == pgx.ErrNoRows {
|
||||
masterID = ""
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("repository: verify local telemetry master: %w", err)
|
||||
}
|
||||
if masterID != claim.SpeakerID {
|
||||
return store.ErrTelemetryLeaseLost
|
||||
}
|
||||
}
|
||||
var bgp store.TelemetryBGP
|
||||
var health store.TelemetryHealth
|
||||
if err = json.Unmarshal(oldBGP, &bgp); err != nil {
|
||||
return fmt.Errorf("repository: decode previous telemetry BGP: %w", err)
|
||||
}
|
||||
if err = json.Unmarshal(oldHealth, &health); err != nil {
|
||||
return fmt.Errorf("repository: decode previous telemetry health: %w", err)
|
||||
}
|
||||
if oldFingerprint != claim.ConfigFingerprint {
|
||||
bgp = store.TelemetryBGP{}
|
||||
health = store.TelemetryHealth{}
|
||||
discoveryCursor = 0
|
||||
}
|
||||
if result.BGP.Source != "" {
|
||||
if result.BGP.Error != "" {
|
||||
result.BGP.Sessions = nil
|
||||
result.BGP.LastSuccessAt = bgp.LastSuccessAt
|
||||
} else {
|
||||
result.BGP.LastSuccessAt = result.BGP.ReceivedAt
|
||||
}
|
||||
bgp = result.BGP
|
||||
}
|
||||
if result.Health.Source != "" {
|
||||
if result.Health.OK {
|
||||
result.Health.LastSuccessAt = result.Health.ReceivedAt
|
||||
} else {
|
||||
result.Health.LastSuccessAt = health.LastSuccessAt
|
||||
}
|
||||
health = result.Health
|
||||
}
|
||||
if bgp.Error != "" || health.Source != "" && !health.OK {
|
||||
failures++
|
||||
} else {
|
||||
failures = 0
|
||||
}
|
||||
if pending {
|
||||
nextPoll = dbNow
|
||||
}
|
||||
bgpJSON, err := json.Marshal(bgp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("repository: encode telemetry BGP: %w", err)
|
||||
}
|
||||
healthJSON, err := json.Marshal(health)
|
||||
if err != nil {
|
||||
return fmt.Errorf("repository: encode telemetry health: %w", err)
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE telemetry_snapshot SET bgp_json=$5::jsonb,health_json=$6::jsonb,next_poll_at=$7,failure_count=$8,
|
||||
config_fingerprint=$9,speaker_id=$10::uuid
|
||||
WHERE tenant_id=$1::uuid AND target_id=$2 AND owner_id=$3 AND fencing_token=$4`, claim.TenantID, claim.TargetID, claim.OwnerID, claim.FencingToken, bgpJSON, healthJSON, nextPoll, failures, claim.ConfigFingerprint, nullTelemetryUUID(claim.SpeakerID))
|
||||
if err != nil {
|
||||
return fmt.Errorf("repository: write telemetry: %w", err)
|
||||
}
|
||||
if result.BGP.Source != "" && result.BGP.Error == "" {
|
||||
if discoveryCursor, err = writeTelemetryDiscoveries(ctx, tx, claim, result.BGP, discoveryCursor); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `UPDATE telemetry_snapshot SET owner_id=NULL,lease_until=NULL,refresh_pending=false,discovery_cursor=$5 WHERE tenant_id=$1::uuid AND target_id=$2 AND owner_id=$3 AND fencing_token=$4 AND lease_until>clock_timestamp()`, claim.TenantID, claim.TargetID, claim.OwnerID, claim.FencingToken, discoveryCursor)
|
||||
if err != nil {
|
||||
return fmt.Errorf("repository: release telemetry lease: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() != 1 {
|
||||
return store.ErrTelemetryLeaseLost
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("repository: commit telemetry: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeTelemetryDiscoveries(ctx context.Context, tx pgx.Tx, claim store.TelemetryClaim, bgp store.TelemetryBGP, cursor int) (int, error) {
|
||||
sessions, next := store.TelemetryDiscoveryBatch(bgp.Sessions, cursor, 256)
|
||||
for _, sess := range sessions {
|
||||
neighbor, ok := store.NormalizePeerNeighborString(sess.Neighbor)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
var known bool
|
||||
if err := tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM bgp_peer WHERE tenant_id=$1::uuid AND neighbor=$2::inet)`, claim.TenantID, neighbor).Scan(&known); err != nil {
|
||||
return 0, fmt.Errorf("repository: check discovered neighbor: %w", err)
|
||||
}
|
||||
if known {
|
||||
continue
|
||||
}
|
||||
neighborID := strings.TrimSpace(sess.NeighborID)
|
||||
spID := nullTelemetryUUID(claim.SpeakerID)
|
||||
seenAt := bgp.ReceivedAt
|
||||
_, err := tx.Exec(ctx, `INSERT INTO bgp_peer_discovery(id,tenant_id,speaker_id,neighbor_id,neighbor,remote_asn,protocol_name,session_state,first_seen_at,last_seen_at)
|
||||
VALUES($1,$2::uuid,$3::uuid,$4,$5::inet,$6,$7,$8,$9,$9) ON CONFLICT DO NOTHING`, uuid.NewString(), claim.TenantID, spID, neighborID, neighbor, sess.NeighborAS, sess.Name, sess.State, seenAt)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("repository: insert discovered neighbor: %w", err)
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE bgp_peer_discovery SET protocol_name=$5,session_state=$6,last_seen_at=$7,
|
||||
speaker_id=COALESCE($8::uuid,speaker_id),updated_at=clock_timestamp()
|
||||
WHERE tenant_id=$1::uuid AND status<>'rejected' AND last_seen_at<=$7
|
||||
AND ((neighbor_id<>'' AND neighbor_id=$2) OR (neighbor_id='' AND neighbor=$3::inet AND remote_asn=$4))`, claim.TenantID, neighborID, neighbor, sess.NeighborAS, sess.Name, sess.State, seenAt, spID)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("repository: update discovered neighbor: %w", err)
|
||||
}
|
||||
}
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func nullTelemetryUUID(id string) any {
|
||||
if strings.TrimSpace(id) == "" {
|
||||
return nil
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// ListTelemetry returns committed snapshots for one tenant without touching the agents.
|
||||
func (p *Postgres) ListTelemetry(ctx context.Context, tenantID string) ([]store.TelemetrySnapshot, error) {
|
||||
rows, err := p.pool.Query(ctx, `SELECT `+telemetryColumns+` FROM telemetry_snapshot t WHERE tenant_id=$1::uuid AND (kind='local_bird' OR EXISTS
|
||||
(SELECT 1 FROM bgp_speaker s WHERE s.id=t.speaker_id AND s.role<>'master' AND COALESCE(s.meta_json->>'agent_domain','')<>'' AND COALESCE(s.meta_json->>'agent_secret','')<>'')) ORDER BY target_id`, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("repository: list telemetry: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]store.TelemetrySnapshot, 0)
|
||||
for rows.Next() {
|
||||
s, err := scanTelemetry(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// RequestTelemetryRefresh advances due times once per cooldown window per target.
|
||||
func (p *Postgres) RequestTelemetryRefresh(ctx context.Context, tenantID string, cooldown time.Duration) (bool, error) {
|
||||
if cooldown <= 0 {
|
||||
cooldown = 10 * time.Second
|
||||
}
|
||||
tag, err := p.pool.Exec(ctx, `UPDATE telemetry_snapshot SET last_refresh_requested_at=clock_timestamp(),
|
||||
refresh_pending=CASE WHEN lease_until>clock_timestamp() THEN true ELSE refresh_pending END,
|
||||
next_poll_at=CASE WHEN lease_until>clock_timestamp() THEN next_poll_at ELSE clock_timestamp() END
|
||||
WHERE tenant_id=$1::uuid AND (kind='local_bird' OR speaker_id IS NOT NULL)
|
||||
AND (last_refresh_requested_at IS NULL OR last_refresh_requested_at<=clock_timestamp()-$2::bigint*interval '1 millisecond')`, tenantID, cooldown.Milliseconds())
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("repository: request telemetry refresh: %w", err)
|
||||
}
|
||||
return tag.RowsAffected() > 0, nil
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
"evobgp/internal/store"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
func TestTelemetryLeaseSnapshotAndDiscoveryIntegration(t *testing.T) {
|
||||
p, tenant := queueIntegration(t)
|
||||
ctx := context.Background()
|
||||
speaker := uuid.NewString()
|
||||
_, err := p.pool.Exec(ctx, `INSERT INTO bgp_speaker(id,tenant_id,role,meta_json) VALUES($1,$2,'replica',$3::jsonb)`, speaker, tenant, `{"agent_domain":"node.example","agent_secret":"secret"}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, err := p.ClaimTelemetry(ctx, "owner-a", 1, time.Minute, "")
|
||||
if err != nil || len(first) != 1 || first[0].SpeakerID != speaker {
|
||||
t.Fatalf("first claim=%+v %v", first, err)
|
||||
}
|
||||
if second, err := p.ClaimTelemetry(ctx, "owner-b", 1, time.Minute, ""); err != nil || len(second) != 0 {
|
||||
t.Fatalf("overlapping claim=%+v %v", second, err)
|
||||
}
|
||||
at := time.Now().UTC()
|
||||
result := store.TelemetryResult{BGP: store.TelemetryBGP{TelemetryObservation: store.TelemetryObservation{Source: "agent_bird_protocols", ObservedAt: at, ReceivedAt: at}, Sessions: []birdfmt.BGPSession{{Name: birdfmt.DynamicPeerProtocolPrefix + "test", Neighbor: "192.0.2.8", NeighborAS: 64512, State: "Established"}}}, Health: store.TelemetryHealth{TelemetryObservation: store.TelemetryObservation{Source: "agent_health", ObservedAt: at, ReceivedAt: at}, OK: true}}
|
||||
if err := p.CompleteTelemetry(ctx, first[0], result, at.Add(time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other := &Postgres{pool: p.pool}
|
||||
snaps, err := other.ListTelemetry(ctx, tenant)
|
||||
if err != nil || len(snaps) != 1 || len(snaps[0].BGP.Sessions) != 1 || snaps[0].ConfigFingerprint == "" {
|
||||
t.Fatalf("restarted repository snapshot=%+v %v", snaps, err)
|
||||
}
|
||||
discoveries, err := other.ListPeerDiscoveries(tenant, store.PeerDiscoveryPending)
|
||||
if err != nil || len(discoveries) != 1 {
|
||||
t.Fatalf("fenced discovery=%+v %v", discoveries, err)
|
||||
}
|
||||
if requested, err := other.RequestTelemetryRefresh(ctx, tenant, time.Minute); err != nil || !requested {
|
||||
t.Fatalf("refresh=%v %v", requested, err)
|
||||
}
|
||||
if requested, err := other.RequestTelemetryRefresh(ctx, tenant, time.Minute); err != nil || requested {
|
||||
t.Fatalf("refresh not coalesced=%v %v", requested, err)
|
||||
}
|
||||
newClaim, err := other.ClaimTelemetry(ctx, "owner-b", 1, time.Minute, "")
|
||||
if err != nil || len(newClaim) != 1 || newClaim[0].FencingToken <= first[0].FencingToken {
|
||||
t.Fatalf("new lease=%+v %v", newClaim, err)
|
||||
}
|
||||
if err := p.CompleteTelemetry(ctx, first[0], result, at.Add(time.Hour)); !errors.Is(err, store.ErrTelemetryLeaseLost) {
|
||||
t.Fatalf("late result accepted: %v", err)
|
||||
}
|
||||
|
||||
// Expire the lease inside the discovery write, after the initial fence check.
|
||||
// The final fence must roll back BOTH snapshot and discovery atomically.
|
||||
function := pgx.Identifier{"test_telemetry_expiry_" + uuid.NewString()}.Sanitize()
|
||||
trigger := pgx.Identifier{"test_telemetry_expiry_" + uuid.NewString()}.Sanitize()
|
||||
_, err = p.pool.Exec(ctx, `CREATE FUNCTION `+function+`() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NEW.tenant_id::text = TG_ARGV[0] THEN
|
||||
UPDATE telemetry_snapshot SET lease_until='epoch'::timestamptz
|
||||
WHERE tenant_id=NEW.tenant_id AND target_id=TG_ARGV[1];
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END $$`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = p.pool.Exec(context.Background(), `DROP FUNCTION IF EXISTS `+function+`() CASCADE`) })
|
||||
// Both arguments are generated UUIDs, not untrusted SQL input.
|
||||
_, err = p.pool.Exec(ctx, `CREATE TRIGGER `+trigger+` BEFORE INSERT ON bgp_peer_discovery FOR EACH ROW EXECUTE FUNCTION `+function+`('`+tenant+`','`+speaker+`')`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result.BGP.Sessions = []birdfmt.BGPSession{{Name: birdfmt.DynamicPeerProtocolPrefix + "rollback", Neighbor: "192.0.2.9", NeighborAS: 64512, State: "Established"}}
|
||||
if err = other.CompleteTelemetry(ctx, newClaim[0], result, at.Add(time.Hour)); !errors.Is(err, store.ErrTelemetryLeaseLost) {
|
||||
t.Fatalf("completion after lease expired during discovery: %v", err)
|
||||
}
|
||||
snaps, err = p.ListTelemetry(ctx, tenant)
|
||||
if err != nil || len(snaps) != 1 || len(snaps[0].BGP.Sessions) != 1 || snaps[0].BGP.Sessions[0].Neighbor != "192.0.2.8" {
|
||||
t.Fatalf("expired completion changed snapshot: %+v %v", snaps, err)
|
||||
}
|
||||
discoveries, err = p.ListPeerDiscoveries(tenant, store.PeerDiscoveryPending)
|
||||
if err != nil || len(discoveries) != 1 {
|
||||
t.Fatalf("expired completion changed discoveries: %+v %v", discoveries, err)
|
||||
}
|
||||
neighbor, ok := store.NormalizePeerNeighborString(discoveries[0].Neighbor)
|
||||
if !ok || neighbor != "192.0.2.8" {
|
||||
t.Fatalf("expired completion inserted discovery: %+v", discoveries)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user