perf: incremental render scale, SQL pagination and observability

- BIRD filter chunking (500 CIDR); bounded job worker pool - ListModulesPage SQL push-down; revision diff limit; batch revision prune - DB pool tuning; Prometheus pipeline/job metrics - Coalesce module_refresh via idempotency; GET /jobs/{id}/report - JobAuditWriter foundation for job_audit persistence Co-authored-by: Cursor <[email protected]>
2026-05-21 10:45:46 +07:00
parent a8c5e9701f
commit dd7d43c2c2
8 changed files with 336 additions and 32 deletions
@@ -0,0 +1,50 @@
+package repository
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+// JobAuditWriter persists async job lifecycle rows to job_audit (optional cross-process queue foundation).
+type JobAuditWriter struct {
+	pool *pgxpool.Pool
+}
+
+func NewJobAuditWriter(pool *pgxpool.Pool) *JobAuditWriter {
+	if pool == nil {
+		return nil
+	}
+	return &JobAuditWriter{pool: pool}
+}
+
+// UpsertRunning inserts or updates a running job row (best-effort).
+func (w *JobAuditWriter) UpsertRunning(ctx context.Context, tenantID, jobID, kind string, idempotencyKey *string, meta map[string]any) {
+	if w == nil || w.pool == nil {
+		return
+	}
+	metaJSON, _ := json.Marshal(meta)
+	var idem any
+	if idempotencyKey != nil && *idempotencyKey != "" {
+		idem = *idempotencyKey
+	}
+	_, _ = w.pool.Exec(ctx, `
+		INSERT INTO job_audit (id, tenant_id, kind, status, idempotency_key, meta_json, created_at, started_at)
+		VALUES ($1::uuid, $2::uuid, $3, 'running', $4, $5::jsonb, now(), now())
+		ON CONFLICT (tenant_id, idempotency_key) WHERE idempotency_key IS NOT NULL
+		DO UPDATE SET status='running', started_at=now(), meta_json=EXCLUDED.meta_json`,
+		jobID, tenantID, kind, idem, metaJSON)
+}
+
+// MarkTerminal updates job_audit terminal state (best-effort).
+func (w *JobAuditWriter) MarkTerminal(ctx context.Context, tenantID, jobID, status string, errMsg *string, finishedAt time.Time) {
+	if w == nil || w.pool == nil {
+		return
+	}
+	_, _ = w.pool.Exec(ctx, `
+		UPDATE job_audit SET status=$3, error_message=$4, finished_at=$5
+		WHERE id=$1::uuid AND tenant_id=$2::uuid`,
+		jobID, tenantID, status, errMsg, finishedAt.UTC())
+}
@@ -162,6 +162,75 @@ func (p *Postgres) ListModules(tenantID string) []*store.Module {
 	return out
 }

+func (p *Postgres) ListModulesPage(tenantID, cursor string, limit int) ([]*store.Module, string, bool) {
+	if limit <= 0 {
+		limit = 50
+	}
+	off := 0
+	if cursor != "" {
+		if n, err := strconv.Atoi(cursor); err == nil && n >= 0 {
+			off = n
+		}
+	}
+	ctx := context.Background()
+	rows, err := p.pool.Query(ctx, `
+		SELECT id, type, name, enabled, priority, doh_profile_id::text, doh_resolver_policy,
+			refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at
+		FROM module WHERE tenant_id = $1 AND deleted_at IS NULL
+		ORDER BY priority, name
+		LIMIT $2 OFFSET $3`, tenantID, limit+1, off)
+	if err != nil {
+		return nil, "", false
+	}
+	defer rows.Close()
+	var out []*store.Module
+	moduleByID := make(map[string]*store.Module)
+	for rows.Next() {
+		var m store.Module
+		m.TenantID = tenantID
+		var doh, dc, cron *string
+		var refresh *int32
+		var last *time.Time
+		if err := rows.Scan(&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last); err != nil {
+			continue
+		}
+		m.DohResolverPolicy = store.NormalizeDohResolverPolicy(m.DohResolverPolicy)
+		if refresh != nil {
+			m.RefreshIntervalSec = int(*refresh)
+		}
+		if cron != nil {
+			m.CronExpr = *cron
+		}
+		if doh != nil && *doh != "" {
+			m.DohProfileID = doh
+		}
+		if dc != nil && *dc != "" {
+			m.DefaultCommunityID = dc
+		}
+		if last != nil {
+			t := last.UTC()
+			m.LastRefreshedAt = &t
+		}
+		out = append(out, &m)
+		moduleByID[m.ID] = &m
+	}
+	if err := p.batchFillModuleDohFields(ctx, moduleByID); err != nil {
+		return nil, "", false
+	}
+	more := len(out) > limit
+	if more {
+		out = out[:limit]
+	}
+	next := ""
+	if more {
+		next = fmt.Sprintf("%d", off+limit)
+	}
+	if len(out) == 0 {
+		return nil, "", false
+	}
+	return out, next, more
+}
+
 func (p *Postgres) GetModule(tenantID, moduleID string) (*store.Module, error) {
 	ctx := context.Background()
 	var m store.Module
@@ -762,6 +831,8 @@ func (p *Postgres) CreateRollbackRevision(tenantID, sourceRevisionID string) (st
 	return newID, nil
 }

+const maxRevisionDiffRows = 5000
+
 func (p *Postgres) RevisionDiff(tenantID, aID, bID string) (map[string]any, error) {
 	if _, err := p.GetRevision(tenantID, aID); err != nil {
 		return nil, err
@@ -786,7 +857,7 @@ func (p *Postgres) RevisionDiff(tenantID, aID, bID string) (map[string]any, erro
 			SELECT prefix FROM revision_materialized_prefix WHERE revision_id=$1::uuid
 			EXCEPT
 			SELECT prefix FROM revision_materialized_prefix WHERE revision_id=$2::uuid
-		) s ORDER BY 1`, bID, aID)
+		) s ORDER BY 1 LIMIT $3`, bID, aID, maxRevisionDiffRows+1)
 	if err != nil {
 		return nil, err
 	}
@@ -798,13 +869,18 @@ func (p *Postgres) RevisionDiff(tenantID, aID, bID string) (map[string]any, erro
 			continue
 		}
 		added = append(added, s)
+		if len(added) > maxRevisionDiffRows {
+			added = added[:maxRevisionDiffRows]
+			break
+		}
 	}
+	addedTruncated := len(added) >= maxRevisionDiffRows
 	rowsRem, err := p.pool.Query(ctx, `
 		SELECT prefix::text FROM (
 			SELECT prefix FROM revision_materialized_prefix WHERE revision_id=$1::uuid
 			EXCEPT
 			SELECT prefix FROM revision_materialized_prefix WHERE revision_id=$2::uuid
-		) s ORDER BY 1`, aID, bID)
+		) s ORDER BY 1 LIMIT $3`, aID, bID, maxRevisionDiffRows+1)
 	if err != nil {
 		return nil, err
 	}
@@ -816,40 +892,56 @@ func (p *Postgres) RevisionDiff(tenantID, aID, bID string) (map[string]any, erro
 			continue
 		}
 		removed = append(removed, s)
+		if len(removed) > maxRevisionDiffRows {
+			removed = removed[:maxRevisionDiffRows]
+			break
+		}
 	}
 	return map[string]any{
 		"revision_a": aID,
 		"revision_b": bID,
 		"prefixes": map[string]any{
 			"added": added, "removed": removed, "unchanged_count": unchanged,
+			"truncated": addedTruncated || len(removed) >= maxRevisionDiffRows,
 		},
 	}, nil
 }

 func (p *Postgres) PruneRevisionsBefore(tenantID string, cutoff time.Time) (int, error) {
 	ctx := context.Background()
-	cmd, err := p.pool.Exec(ctx, `
-		DELETE FROM config_revision AS cr
-		WHERE cr.tenant_id = $1
-		  AND cr.created_at < $2
-		  AND cr.id <> (
-			SELECT id
-			FROM config_revision
-			WHERE tenant_id = $1
-			ORDER BY created_at DESC
-			LIMIT 1
-		  )
-		  AND NOT EXISTS (
-			SELECT 1
-			FROM bgp_speaker AS sp
-			WHERE sp.tenant_id = $1
-			  AND (sp.last_applied_revision_id = cr.id OR sp.published_revision_id = cr.id)
-		  )`,
-		tenantID, cutoff.UTC())
-	if err != nil {
-		return 0, err
+	total := 0
+	const batchSize = 50
+	for {
+		cmd, err := p.pool.Exec(ctx, `
+			DELETE FROM config_revision AS cr
+			WHERE cr.id IN (
+				SELECT id FROM config_revision
+				WHERE tenant_id = $1
+				  AND created_at < $2
+				  AND id <> (
+					SELECT id FROM config_revision
+					WHERE tenant_id = $1
+					ORDER BY created_at DESC
+					LIMIT 1
+				  )
+				  AND NOT EXISTS (
+					SELECT 1 FROM bgp_speaker AS sp
+					WHERE sp.tenant_id = $1
+					  AND (sp.last_applied_revision_id = config_revision.id OR sp.published_revision_id = config_revision.id)
+				  )
+				ORDER BY created_at ASC
+				LIMIT $3
+			)`, tenantID, cutoff.UTC(), batchSize)
+		if err != nil {
+			return total, err
+		}
+		n := int(cmd.RowsAffected())
+		total += n
+		if n < batchSize {
+			break
+		}
 	}
-	return int(cmd.RowsAffected()), nil
+	return total, nil
 }

 func (p *Postgres) SetLastAppliedRevision(tenantID, speakerID, revisionID string) error {