feat(jobs): add durable PG queue reclaim, slog, and richer metrics

JSON slog в ключевых пакетах; Prometheus path_group, job_audit_depth, upstream breaker; job_audit ClaimQueued/ReclaimStaleRunning + Adopt loop для HA после рестарта.

Co-authored-by: Cursor <[email protected]>

internal/observability/metrics.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"net/http"
 	"strconv"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -30,7 +31,7 @@ var (
 			Name:      "http_requests_total",
 			Help:      "HTTP requests handled by the API mux (excludes /metrics).",
 		},
-		[]string{"method", "code"},
+		[]string{"method", "code", "path_group"},
 	)
 
 	jobsFinished = promauto.NewCounterVec(
@@ -106,6 +107,18 @@ var (
 		Name:      "job_queue_capacity",
 		Help:      "Maximum concurrent in-process async jobs.",
 	})
+
+	jobAuditDepth = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Name:      "job_audit_depth",
+		Help:      "Rows in job_audit by status (durable queue).",
+	}, []string{"status"})
+
+	upstreamBreakerOpen = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Name:      "upstream_breaker_open",
+		Help:      "1 if per-host HTTP circuit breaker is open.",
+	}, []string{"host"})
 )
 
 var (
@@ -158,6 +171,28 @@ func RecordJobTerminal(kind, status string) {
 	}
 }
 
+// RecordJobAuditDepth updates durable job_audit depth gauges.
+func RecordJobAuditDepth(counts map[string]int64) {
+	for status, n := range counts {
+		if status == "" {
+			status = "unknown"
+		}
+		jobAuditDepth.WithLabelValues(status).Set(float64(n))
+	}
+}
+
+// SetUpstreamBreakerOpen records circuit breaker state for a host (1=open, 0=closed).
+func SetUpstreamBreakerOpen(host string, open bool) {
+	if host == "" {
+		host = "_"
+	}
+	v := 0.0
+	if open {
+		v = 1
+	}
+	upstreamBreakerOpen.WithLabelValues(host).Set(v)
+}
+
 // SetBuildInfo sets evobgp_build_info gauge (idempotent labels).
 func SetBuildInfo(version, gitSHA string) {
 	if version == "" {
@@ -292,15 +327,32 @@ func MetricsHandler() http.Handler {
 	return promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{})
 }
 
-// HTTPMiddleware records method and status code for all wrapped requests.
+// HTTPMiddleware records method, status code, and path group for all wrapped requests.
 func HTTPMiddleware(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		sw := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
 		next.ServeHTTP(sw, r)
-		httpRequests.WithLabelValues(r.Method, strconv.Itoa(sw.status)).Inc()
+		httpRequests.WithLabelValues(r.Method, strconv.Itoa(sw.status), pathGroup(r.URL.Path)).Inc()
 	})
 }
 
+func pathGroup(path string) string {
+	if path == "" {
+		return "/"
+	}
+	parts := strings.Split(strings.Trim(path, "/"), "/")
+	if len(parts) == 0 || parts[0] == "" {
+		return "/"
+	}
+	if parts[0] == "v1" && len(parts) >= 2 {
+		return "/v1/" + parts[1]
+	}
+	if parts[0] == "metrics" || parts[0] == "version" {
+		return "/" + parts[0]
+	}
+	return "/" + parts[0]
+}
+
 type statusRecorder struct {
 	http.ResponseWriter
 	status int