feat(jobs): add durable PG queue reclaim, slog, and richer metrics

JSON slog в ключевых пакетах; Prometheus path_group, job_audit_depth, upstream breaker; job_audit ClaimQueued/ReclaimStaleRunning + Adopt loop для HA после рестарта.

Co-authored-by: Cursor <[email protected]>
This commit is contained in:
Denozordec
2026-07-31 12:26:18 +07:00
co-authored by Cursor
parent 13e3d21ce2
commit 8fe74c1d3b
18 changed files with 448 additions and 51 deletions
+7 -5
View File
@@ -2,6 +2,8 @@ package deploy
import (
"context"
"evobgp/internal/logging"
"fmt"
"log"
"os"
"strings"
@@ -22,18 +24,18 @@ func Run(ctx context.Context, deps *Deps) {
cfg := config.Load()
broker.LogConnect(ctx, cfg.BrokerURL)
if d := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_ACTIVE_DIR")); d != "" {
log.Printf("evobgp-deploy: EVOBGP_BIRD_ACTIVE_DIR=%q (apply via API jobs when API has same env)", d)
logging.Default().Info(fmt.Sprintf("evobgp-deploy: EVOBGP_BIRD_ACTIVE_DIR=%q (apply via API jobs when API has same env)", d))
}
if deps == nil || deps.Store == nil {
log.Fatalf("evobgp-deploy: missing store (pass deploy.Deps from BootstrapWorkers or evobgp-all)")
}
t := time.NewTicker(90 * time.Second)
defer t.Stop()
log.Printf("evobgp-deploy: active (speaker published vs applied drift log)")
logging.Default().Info(fmt.Sprintf("evobgp-deploy: active (speaker published vs applied drift log)"))
for {
select {
case <-ctx.Done():
log.Printf("evobgp-deploy: stopped")
logging.Default().Info(fmt.Sprintf("evobgp-deploy: stopped"))
return
case <-t.C:
logDrift(context.Background(), deps.Store)
@@ -45,7 +47,7 @@ func logDrift(ctx context.Context, st store.Backend) {
_ = ctx
tenants, err := st.ListTenantIDs()
if err != nil {
log.Printf("evobgp-deploy: list tenants: %v", err)
logging.Default().Info(fmt.Sprintf("evobgp-deploy: list tenants: %v", err))
return
}
for _, tid := range tenants {
@@ -59,7 +61,7 @@ func logDrift(ctx context.Context, st store.Backend) {
applied = *sp.LastAppliedRevisionID
}
if applied != "" && applied != pub {
log.Printf("evobgp-deploy: drift speaker=%s applied=%s published=%s", sp.ID, applied, pub)
logging.Default().Info(fmt.Sprintf("evobgp-deploy: drift speaker=%s applied=%s published=%s", sp.ID, applied, pub))
}
}
}