feat(observability): add maintenance policy metrics

Prometheus: runs, duration, rows_deleted, config_changes; инкремент при CRUD и Execute.
Co-authored-by: Cursor <[email protected]>
This commit is contained in:
Denozordec
2026-06-12 13:28:20 +07:00
co-authored by Cursor
parent cbf345b25f
commit aaef47c7a7
3 changed files with 125 additions and 4 deletions
+4
View File
@@ -6,6 +6,7 @@ import (
"strings"
"evobgp/internal/jobs"
"evobgp/internal/observability"
"evobgp/internal/store"
)
@@ -107,6 +108,7 @@ func (s *Server) handleCreateMaintenancePolicy(w http.ResponseWriter, r *http.Re
return
}
_ = s.store.AppendMaintenancePolicyConfigAudit(actorPrefix(a), p.ID, "create", nil, maintenancePolicyJSON(p))
observability.IncMaintenanceConfigChange("create")
s.reloadMaintenanceConfig(r)
writeJSON(w, http.StatusCreated, maintenancePolicyJSON(p))
}
@@ -133,6 +135,7 @@ func (s *Server) handlePatchMaintenancePolicy(w http.ResponseWriter, r *http.Req
return
}
_ = s.store.AppendMaintenancePolicyConfigAudit(actorPrefix(a), id, "update", maintenancePolicyJSON(before), maintenancePolicyJSON(updated))
observability.IncMaintenanceConfigChange("update")
s.reloadMaintenanceConfig(r)
writeJSON(w, http.StatusOK, maintenancePolicyJSON(updated))
}
@@ -153,6 +156,7 @@ func (s *Server) handleDeleteMaintenancePolicy(w http.ResponseWriter, r *http.Re
return
}
_ = s.store.AppendMaintenancePolicyConfigAudit(actorPrefix(a), id, "delete", maintenancePolicyJSON(before), nil)
observability.IncMaintenanceConfigChange("delete")
s.reloadMaintenanceConfig(r)
w.WriteHeader(http.StatusNoContent)
}
+49 -4
View File
@@ -7,6 +7,7 @@ import (
"strings"
"time"
"evobgp/internal/observability"
"evobgp/internal/pgmonitor"
"evobgp/internal/store"
@@ -22,19 +23,29 @@ type PolicyExecutor struct {
// Execute runs cleanup and/or vacuum steps for a policy.
func (e *PolicyExecutor) Execute(ctx context.Context, policy *store.MaintenancePolicy, dryRun bool) (map[string]any, error) {
if e == nil || e.Pool == nil {
return nil, fmt.Errorf("maintenance: postgres not configured")
}
start := time.Now()
if policy == nil {
return nil, store.ErrInvalidInput
}
action := policyAction(policy)
record := func(status string, detail map[string]any) {
observability.RecordMaintenancePolicyRun(policy.ID, action, status, dryRun, time.Since(start), rowsDeletedFromDetail(detail))
}
if e == nil || e.Pool == nil {
record("failed", nil)
return nil, fmt.Errorf("maintenance: postgres not configured")
}
if err := ValidateTableName(policy.TableName); err != nil {
record("failed", nil)
return nil, err
}
if err := ValidateCondition(policy.Condition); err != nil {
record("failed", nil)
return nil, err
}
if !store.ValidVacuumStrategy(policy.VacuumStrategy) {
record("failed", nil)
return nil, store.ErrInvalidInput
}
@@ -50,6 +61,7 @@ func (e *PolicyExecutor) Execute(ctx context.Context, policy *store.MaintenanceP
detail[k] = v
}
if err != nil {
record("failed", detail)
return detail, err
}
}
@@ -61,6 +73,7 @@ func (e *PolicyExecutor) Execute(ctx context.Context, policy *store.MaintenanceP
detail["vacuum"] = vacDetail
}
if err != nil {
record("failed", detail)
return detail, err
}
}
@@ -68,10 +81,42 @@ func (e *PolicyExecutor) Execute(ctx context.Context, policy *store.MaintenanceP
if !dryRun && e.Store != nil {
_ = e.Store.TouchMaintenancePolicyRun(policy.ID, "succeeded", "")
}
record("succeeded", detail)
return detail, nil
}
func policyAction(p *store.MaintenancePolicy) string {
if p == nil {
return "run"
}
if p.RetentionPeriodSec != nil || p.MaxRows != nil {
if p.VacuumStrategy != store.VacuumStrategyNone {
return "cleanup_vacuum"
}
return "cleanup"
}
if p.VacuumStrategy != store.VacuumStrategyNone {
return p.VacuumStrategy
}
return "run"
}
func rowsDeletedFromDetail(detail map[string]any) int64 {
if detail == nil {
return 0
}
switch v := detail["deleted"].(type) {
case int64:
return v
case int:
return int64(v)
case float64:
return int64(v)
default:
return 0
}
}
func (e *PolicyExecutor) runCleanup(ctx context.Context, policy *store.MaintenancePolicy, dryRun bool) (map[string]any, error) {
detail := map[string]any{"cleanup": true}
limit := NormalizeBatchLimit(policy.MaxRows)
@@ -0,0 +1,72 @@
package observability
import (
"strconv"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
var (
maintenancePolicyRuns = promauto.NewCounterVec(
prometheus.CounterOpts{
Namespace: namespace,
Name: "maintenance_policy_runs_total",
Help: "Maintenance policy executions by outcome.",
},
[]string{"policy_id", "action", "status", "dry_run"},
)
maintenancePolicyDuration = promauto.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: namespace,
Name: "maintenance_policy_duration_seconds",
Help: "Maintenance policy execution duration.",
Buckets: prometheus.ExponentialBuckets(0.05, 2, 12),
},
[]string{"policy_id", "action"},
)
maintenanceRowsDeleted = promauto.NewCounterVec(
prometheus.CounterOpts{
Namespace: namespace,
Name: "maintenance_policy_rows_deleted_total",
Help: "Rows deleted by maintenance cleanup policies.",
},
[]string{"policy_id"},
)
maintenanceConfigChanges = promauto.NewCounterVec(
prometheus.CounterOpts{
Namespace: namespace,
Name: "maintenance_config_changes_total",
Help: "Maintenance policy configuration changes from UI/API.",
},
[]string{"action"},
)
)
// RecordMaintenancePolicyRun updates run counters and histograms.
func RecordMaintenancePolicyRun(policyID, action, status string, dryRun bool, duration time.Duration, rowsDeleted int64) {
if policyID == "" {
policyID = "unknown"
}
if action == "" {
action = "run"
}
dry := strconv.FormatBool(dryRun)
maintenancePolicyRuns.WithLabelValues(policyID, action, status, dry).Inc()
maintenancePolicyDuration.WithLabelValues(policyID, action).Observe(duration.Seconds())
if rowsDeleted > 0 && !dryRun {
maintenanceRowsDeleted.WithLabelValues(policyID).Add(float64(rowsDeleted))
}
}
// IncMaintenanceConfigChange increments config audit metric.
func IncMaintenanceConfigChange(action string) {
if action == "" {
action = "unknown"
}
maintenanceConfigChanges.WithLabelValues(action).Inc()
}