feat(jobs): add durable PG queue reclaim, slog, and richer metrics

JSON slog в ключевых пакетах; Prometheus path_group, job_audit_depth, upstream breaker; job_audit ClaimQueued/ReclaimStaleRunning + Adopt loop для HA после рестарта.

Co-authored-by: Cursor <[email protected]>
This commit is contained in:
Denozordec
2026-07-31 12:26:18 +07:00
co-authored by Cursor
parent 13e3d21ce2
commit 8fe74c1d3b
18 changed files with 448 additions and 51 deletions
+9 -2
View File
@@ -7,6 +7,8 @@ import (
"io"
"net/http"
"time"
"evobgp/internal/observability"
)
const DefaultTimeout = 45 * time.Second
@@ -69,19 +71,24 @@ func DoWithBreaker(ctx context.Context, hc *http.Client, req *http.Request, maxA
if req == nil || req.URL == nil {
return nil, fmt.Errorf("httpclient: nil request")
}
br := breakerForHost(req.URL.Hostname())
host := req.URL.Hostname()
br := breakerForHost(host)
if !br.allow() {
return nil, fmt.Errorf("httpclient: circuit open for %s", req.URL.Hostname())
observability.SetUpstreamBreakerOpen(host, true)
return nil, fmt.Errorf("httpclient: circuit open for %s", host)
}
resp, err := DoWithRetry(ctx, hc, req, maxAttempts)
if err != nil {
br.recordFailure()
observability.SetUpstreamBreakerOpen(host, !br.allow())
return nil, err
}
if resp.StatusCode >= 500 {
br.recordFailure()
observability.SetUpstreamBreakerOpen(host, !br.allow())
return resp, nil
}
br.recordSuccess()
observability.SetUpstreamBreakerOpen(host, false)
return resp, nil
}