feat(jobs): add durable PG queue reclaim, slog, and richer metrics
JSON slog в ключевых пакетах; Prometheus path_group, job_audit_depth, upstream breaker; job_audit ClaimQueued/ReclaimStaleRunning + Adopt loop для HA после рестарта. Co-authored-by: Cursor <[email protected]>
This commit is contained in:
@@ -7,6 +7,8 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/observability"
|
||||
)
|
||||
|
||||
const DefaultTimeout = 45 * time.Second
|
||||
@@ -69,19 +71,24 @@ func DoWithBreaker(ctx context.Context, hc *http.Client, req *http.Request, maxA
|
||||
if req == nil || req.URL == nil {
|
||||
return nil, fmt.Errorf("httpclient: nil request")
|
||||
}
|
||||
br := breakerForHost(req.URL.Hostname())
|
||||
host := req.URL.Hostname()
|
||||
br := breakerForHost(host)
|
||||
if !br.allow() {
|
||||
return nil, fmt.Errorf("httpclient: circuit open for %s", req.URL.Hostname())
|
||||
observability.SetUpstreamBreakerOpen(host, true)
|
||||
return nil, fmt.Errorf("httpclient: circuit open for %s", host)
|
||||
}
|
||||
resp, err := DoWithRetry(ctx, hc, req, maxAttempts)
|
||||
if err != nil {
|
||||
br.recordFailure()
|
||||
observability.SetUpstreamBreakerOpen(host, !br.allow())
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode >= 500 {
|
||||
br.recordFailure()
|
||||
observability.SetUpstreamBreakerOpen(host, !br.allow())
|
||||
return resp, nil
|
||||
}
|
||||
br.recordSuccess()
|
||||
observability.SetUpstreamBreakerOpen(host, false)
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user