perf: incremental render scale, SQL pagination and observability

- BIRD filter chunking (500 CIDR); bounded job worker pool
- ListModulesPage SQL push-down; revision diff limit; batch revision prune
- DB pool tuning; Prometheus pipeline/job metrics
- Coalesce module_refresh via idempotency; GET /jobs/{id}/report
- JobAuditWriter foundation for job_audit persistence

Co-authored-by: Cursor <[email protected]>

internal/httpapi/routes.go

@@ -69,6 +69,7 @@ func (s *Server) registerV1(m *http.ServeMux) {
 	m.HandleFunc("GET /bird/status", s.handleBirdStatus)
 	m.HandleFunc("GET /jobs", s.handleListJobs)
 	m.HandleFunc("GET /jobs/{job_id}", s.handleGetJob)
+	m.HandleFunc("GET /jobs/{job_id}/report", s.handleGetJobReport)
 	m.HandleFunc("POST /jobs/{job_id}/cancel", s.handleCancelJob)
 	m.HandleFunc("GET /speakers/{speaker_id}/revisions/latest", s.handleNodeLatestRevision)
 	m.HandleFunc("GET /speakers/{speaker_id}/bundle/{revision_id}", s.handleNodeBundle)
@@ -201,6 +202,22 @@ func (s *Server) handleListModules(w http.ResponseWriter, r *http.Request) {
 	}
 
 	filtered := make([]*store.Module, 0)
+	limit := parseListLimit(r)
+	cursor := r.URL.Query().Get("cursor")
+	if typeFilter == "" && enabledFilter == nil {
+		page, next, more := s.store.ListModulesPage(a.TenantID, cursor, limit)
+		for _, mod := range page {
+			filtered = append(filtered, mod)
+		}
+		items := make([]map[string]any, 0, len(filtered))
+		for _, mod := range filtered {
+			items = append(items, moduleJSON(mod))
+		}
+		writeJSON(w, http.StatusOK, map[string]any{
+			"items": items, "next_cursor": strPtrOrNull(next), "has_more": more,
+		})
+		return
+	}
 	for _, mod := range s.store.ListModules(a.TenantID) {
 		if typeFilter != "" && mod.Type != typeFilter {
 			continue
@@ -534,7 +551,8 @@ func (s *Server) enqueueModuleRefreshIfEnabled(tenantID, moduleID, trigger strin
 		return
 	}
 	mid := moduleID
-	_, _, _ = s.jobs.Enqueue(tenantID, jobs.KindModuleRefresh, nil, &mid, map[string]any{
+	key := "module_refresh:" + moduleID
+	_, _, _ = s.jobs.Enqueue(tenantID, jobs.KindModuleRefresh, &key, &mid, map[string]any{
 		"module_id": moduleID,
 		"trigger":   trigger,
 	})
@@ -601,6 +619,9 @@ func (s *Server) handleRevisionPreview(w http.ResponseWriter, r *http.Request) {
 	for k, v := range rev.PreviewFragments {
 		obj[k] = v
 	}
+	if expanded := pipeline.BuildExpandedBirdPreview(rev.PreviewFragments); expanded != "" {
+		obj[pipeline.AuxBirdFullExpandedKey()] = expanded
+	}
 	writeJSON(w, http.StatusOK, obj)
 }
 
@@ -855,6 +876,44 @@ func (s *Server) handleGetJob(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, j.Snapshot())
 }
 
+func (s *Server) handleGetJobReport(w http.ResponseWriter, r *http.Request) {
+	a, ok := authFromContext(r.Context())
+	if !ok {
+		writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth")
+		return
+	}
+	if !s.requireAtLeast(w, a, "viewer") {
+		return
+	}
+	j, err := s.jobs.Get(a.TenantID, r.PathValue("job_id"))
+	if err != nil {
+		writeProblem(w, http.StatusNotFound, "Not Found", "job not found")
+		return
+	}
+	snap := j.Snapshot()
+	meta, _ := snap["meta"].(map[string]any)
+	out := map[string]any{
+		"job_id":    snap["job_id"],
+		"kind":      snap["kind"],
+		"status":    snap["status"],
+		"meta":      meta,
+		"error":     snap["error"],
+		"created_at": snap["created_at"],
+	}
+	if meta != nil {
+		if v, ok := meta["log_entries"]; ok {
+			out["log_entries"] = v
+		}
+		if v, ok := meta["log_total"]; ok {
+			out["log_total"] = v
+		}
+		if v, ok := meta["revision_id"]; ok {
+			out["revision_id"] = v
+		}
+	}
+	writeJSON(w, http.StatusOK, out)
+}
+
 func (s *Server) handleCancelJob(w http.ResponseWriter, r *http.Request) {
 	a, ok := authFromContext(r.Context())
 	if !ok {