perf: incremental render scale, SQL pagination and observability

- BIRD filter chunking (500 CIDR); bounded job worker pool
- ListModulesPage SQL push-down; revision diff limit; batch revision prune
- DB pool tuning; Prometheus pipeline/job metrics
- Coalesce module_refresh via idempotency; GET /jobs/{id}/report
- JobAuditWriter foundation for job_audit persistence

Co-authored-by: Cursor <[email protected]>

internal/observability/metrics.go

@@ -80,6 +80,32 @@ var (
 		Help:      "Prefix row count after CIDR aggregation on tenant render.",
 		Buckets:   prometheus.ExponentialBuckets(1, 2, 16),
 	})
+
+	pipelineRefreshDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: namespace,
+		Name:      "pipeline_refresh_duration_seconds",
+		Help:      "Module refresh ingest duration by module type.",
+		Buckets:   prometheus.ExponentialBuckets(0.05, 2, 14),
+	}, []string{"module_type"})
+
+	renderPrefixCount = promauto.NewHistogram(prometheus.HistogramOpts{
+		Namespace: namespace,
+		Name:      "render_prefix_count",
+		Help:      "Materialized prefix count per tenant render.",
+		Buckets:   prometheus.ExponentialBuckets(10, 2, 16),
+	})
+
+	jobQueueActive = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Name:      "job_queue_active",
+		Help:      "Currently running in-process async jobs.",
+	})
+
+	jobQueueCapacity = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Name:      "job_queue_capacity",
+		Help:      "Maximum concurrent in-process async jobs.",
+	})
 )
 
 var (
@@ -99,6 +125,27 @@ func RecordPrefixAggregation(rawCount, aggregatedCount int, duration time.Durati
 	prefixAggregationDuration.Observe(duration.Seconds())
 	prefixAggregationRawCount.Observe(float64(rawCount))
 	prefixAggregationAggregatedCount.Observe(float64(aggregatedCount))
+	renderPrefixCount.Observe(float64(aggregatedCount))
+}
+
+// RecordPipelineRefresh records module ingest duration.
+func RecordPipelineRefresh(moduleType string, duration time.Duration) {
+	if moduleType == "" {
+		moduleType = "unknown"
+	}
+	pipelineRefreshDuration.WithLabelValues(moduleType).Observe(duration.Seconds())
+}
+
+// RecordJobQueueDepth updates in-process job worker utilization gauges.
+func RecordJobQueueDepth(active, capacity int) {
+	if active < 0 {
+		active = 0
+	}
+	if capacity < 0 {
+		capacity = 0
+	}
+	jobQueueActive.Set(float64(active))
+	jobQueueCapacity.Set(float64(capacity))
 }
 
 // RecordJobTerminal increments jobs_finished_total for terminal statuses.