feat(jobs): add durable PG queue reclaim, slog, and richer metrics

JSON slog в ключевых пакетах; Prometheus path_group, job_audit_depth, upstream breaker; job_audit ClaimQueued/ReclaimStaleRunning + Adopt loop для HA после рестарта.

Co-authored-by: Cursor <[email protected]>
This commit is contained in:
Denozordec
2026-07-31 12:26:18 +07:00
co-authored by Cursor
parent 13e3d21ce2
commit 8fe74c1d3b
18 changed files with 448 additions and 51 deletions
+8 -7
View File
@@ -5,6 +5,7 @@ import (
"bytes"
"context"
"encoding/json"
"evobgp/internal/logging"
"fmt"
"io"
"log"
@@ -43,14 +44,14 @@ func Run(ctx context.Context, deps *Deps) {
t := time.NewTicker(30 * time.Second)
defer t.Stop()
if deps.Jobs != nil {
log.Printf("evobgp-scheduler: active (in-process enqueue tenant_refresh)")
logging.Default().Info(fmt.Sprintf("evobgp-scheduler: active (in-process enqueue tenant_refresh)"))
} else {
log.Printf("evobgp-scheduler: active (HTTP POST .../tenant/refresh → %s)", strings.TrimSpace(deps.APIBase))
logging.Default().Info(fmt.Sprintf("evobgp-scheduler: active (HTTP POST .../tenant/refresh → %s)", strings.TrimSpace(deps.APIBase)))
}
for {
select {
case <-ctx.Done():
log.Printf("evobgp-scheduler: stopped")
logging.Default().Info(fmt.Sprintf("evobgp-scheduler: stopped"))
return
case <-t.C:
tick(context.Background(), deps)
@@ -61,7 +62,7 @@ func Run(ctx context.Context, deps *Deps) {
func tick(ctx context.Context, deps *Deps) {
tenants, err := deps.Store.ListTenantIDs()
if err != nil {
log.Printf("evobgp-scheduler: list tenants: %v", err)
logging.Default().Info(fmt.Sprintf("evobgp-scheduler: list tenants: %v", err))
return
}
now := time.Now().UTC()
@@ -84,16 +85,16 @@ func tick(ctx context.Context, deps *Deps) {
"trigger": "scheduler",
})
if err != nil {
log.Printf("evobgp-scheduler: enqueue tenant %s: %v", tid, err)
logging.Default().Info(fmt.Sprintf("evobgp-scheduler: enqueue tenant %s: %v", tid, err))
continue
}
if created {
log.Printf("evobgp-scheduler: queued tenant refresh for %d module(s) in tenant %s", len(due), tid)
logging.Default().Info(fmt.Sprintf("evobgp-scheduler: queued tenant refresh for %d module(s) in tenant %s", len(due), tid))
}
continue
}
if err := postTenantRefresh(ctx, deps, due, key); err != nil {
log.Printf("evobgp-scheduler: http tenant refresh %s: %v", tid, err)
logging.Default().Info(fmt.Sprintf("evobgp-scheduler: http tenant refresh %s: %v", tid, err))
}
}
}