feat(observability): add maintenance policy metrics
Prometheus: runs, duration, rows_deleted, config_changes; инкремент при CRUD и Execute. Co-authored-by: Cursor <[email protected]>
This commit is contained in:
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"evobgp/internal/jobs"
|
||||
"evobgp/internal/observability"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
@@ -107,6 +108,7 @@ func (s *Server) handleCreateMaintenancePolicy(w http.ResponseWriter, r *http.Re
|
||||
return
|
||||
}
|
||||
_ = s.store.AppendMaintenancePolicyConfigAudit(actorPrefix(a), p.ID, "create", nil, maintenancePolicyJSON(p))
|
||||
observability.IncMaintenanceConfigChange("create")
|
||||
s.reloadMaintenanceConfig(r)
|
||||
writeJSON(w, http.StatusCreated, maintenancePolicyJSON(p))
|
||||
}
|
||||
@@ -133,6 +135,7 @@ func (s *Server) handlePatchMaintenancePolicy(w http.ResponseWriter, r *http.Req
|
||||
return
|
||||
}
|
||||
_ = s.store.AppendMaintenancePolicyConfigAudit(actorPrefix(a), id, "update", maintenancePolicyJSON(before), maintenancePolicyJSON(updated))
|
||||
observability.IncMaintenanceConfigChange("update")
|
||||
s.reloadMaintenanceConfig(r)
|
||||
writeJSON(w, http.StatusOK, maintenancePolicyJSON(updated))
|
||||
}
|
||||
@@ -153,6 +156,7 @@ func (s *Server) handleDeleteMaintenancePolicy(w http.ResponseWriter, r *http.Re
|
||||
return
|
||||
}
|
||||
_ = s.store.AppendMaintenancePolicyConfigAudit(actorPrefix(a), id, "delete", maintenancePolicyJSON(before), nil)
|
||||
observability.IncMaintenanceConfigChange("delete")
|
||||
s.reloadMaintenanceConfig(r)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/observability"
|
||||
"evobgp/internal/pgmonitor"
|
||||
"evobgp/internal/store"
|
||||
|
||||
@@ -22,19 +23,29 @@ type PolicyExecutor struct {
|
||||
|
||||
// Execute runs cleanup and/or vacuum steps for a policy.
|
||||
func (e *PolicyExecutor) Execute(ctx context.Context, policy *store.MaintenancePolicy, dryRun bool) (map[string]any, error) {
|
||||
if e == nil || e.Pool == nil {
|
||||
return nil, fmt.Errorf("maintenance: postgres not configured")
|
||||
}
|
||||
start := time.Now()
|
||||
if policy == nil {
|
||||
return nil, store.ErrInvalidInput
|
||||
}
|
||||
action := policyAction(policy)
|
||||
record := func(status string, detail map[string]any) {
|
||||
observability.RecordMaintenancePolicyRun(policy.ID, action, status, dryRun, time.Since(start), rowsDeletedFromDetail(detail))
|
||||
}
|
||||
|
||||
if e == nil || e.Pool == nil {
|
||||
record("failed", nil)
|
||||
return nil, fmt.Errorf("maintenance: postgres not configured")
|
||||
}
|
||||
if err := ValidateTableName(policy.TableName); err != nil {
|
||||
record("failed", nil)
|
||||
return nil, err
|
||||
}
|
||||
if err := ValidateCondition(policy.Condition); err != nil {
|
||||
record("failed", nil)
|
||||
return nil, err
|
||||
}
|
||||
if !store.ValidVacuumStrategy(policy.VacuumStrategy) {
|
||||
record("failed", nil)
|
||||
return nil, store.ErrInvalidInput
|
||||
}
|
||||
|
||||
@@ -50,6 +61,7 @@ func (e *PolicyExecutor) Execute(ctx context.Context, policy *store.MaintenanceP
|
||||
detail[k] = v
|
||||
}
|
||||
if err != nil {
|
||||
record("failed", detail)
|
||||
return detail, err
|
||||
}
|
||||
}
|
||||
@@ -61,6 +73,7 @@ func (e *PolicyExecutor) Execute(ctx context.Context, policy *store.MaintenanceP
|
||||
detail["vacuum"] = vacDetail
|
||||
}
|
||||
if err != nil {
|
||||
record("failed", detail)
|
||||
return detail, err
|
||||
}
|
||||
}
|
||||
@@ -68,10 +81,42 @@ func (e *PolicyExecutor) Execute(ctx context.Context, policy *store.MaintenanceP
|
||||
if !dryRun && e.Store != nil {
|
||||
_ = e.Store.TouchMaintenancePolicyRun(policy.ID, "succeeded", "")
|
||||
}
|
||||
|
||||
record("succeeded", detail)
|
||||
return detail, nil
|
||||
}
|
||||
|
||||
func policyAction(p *store.MaintenancePolicy) string {
|
||||
if p == nil {
|
||||
return "run"
|
||||
}
|
||||
if p.RetentionPeriodSec != nil || p.MaxRows != nil {
|
||||
if p.VacuumStrategy != store.VacuumStrategyNone {
|
||||
return "cleanup_vacuum"
|
||||
}
|
||||
return "cleanup"
|
||||
}
|
||||
if p.VacuumStrategy != store.VacuumStrategyNone {
|
||||
return p.VacuumStrategy
|
||||
}
|
||||
return "run"
|
||||
}
|
||||
|
||||
func rowsDeletedFromDetail(detail map[string]any) int64 {
|
||||
if detail == nil {
|
||||
return 0
|
||||
}
|
||||
switch v := detail["deleted"].(type) {
|
||||
case int64:
|
||||
return v
|
||||
case int:
|
||||
return int64(v)
|
||||
case float64:
|
||||
return int64(v)
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func (e *PolicyExecutor) runCleanup(ctx context.Context, policy *store.MaintenancePolicy, dryRun bool) (map[string]any, error) {
|
||||
detail := map[string]any{"cleanup": true}
|
||||
limit := NormalizeBatchLimit(policy.MaxRows)
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package observability
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
var (
|
||||
maintenancePolicyRuns = promauto.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "maintenance_policy_runs_total",
|
||||
Help: "Maintenance policy executions by outcome.",
|
||||
},
|
||||
[]string{"policy_id", "action", "status", "dry_run"},
|
||||
)
|
||||
|
||||
maintenancePolicyDuration = promauto.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Namespace: namespace,
|
||||
Name: "maintenance_policy_duration_seconds",
|
||||
Help: "Maintenance policy execution duration.",
|
||||
Buckets: prometheus.ExponentialBuckets(0.05, 2, 12),
|
||||
},
|
||||
[]string{"policy_id", "action"},
|
||||
)
|
||||
|
||||
maintenanceRowsDeleted = promauto.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "maintenance_policy_rows_deleted_total",
|
||||
Help: "Rows deleted by maintenance cleanup policies.",
|
||||
},
|
||||
[]string{"policy_id"},
|
||||
)
|
||||
|
||||
maintenanceConfigChanges = promauto.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "maintenance_config_changes_total",
|
||||
Help: "Maintenance policy configuration changes from UI/API.",
|
||||
},
|
||||
[]string{"action"},
|
||||
)
|
||||
)
|
||||
|
||||
// RecordMaintenancePolicyRun updates run counters and histograms.
|
||||
func RecordMaintenancePolicyRun(policyID, action, status string, dryRun bool, duration time.Duration, rowsDeleted int64) {
|
||||
if policyID == "" {
|
||||
policyID = "unknown"
|
||||
}
|
||||
if action == "" {
|
||||
action = "run"
|
||||
}
|
||||
dry := strconv.FormatBool(dryRun)
|
||||
maintenancePolicyRuns.WithLabelValues(policyID, action, status, dry).Inc()
|
||||
maintenancePolicyDuration.WithLabelValues(policyID, action).Observe(duration.Seconds())
|
||||
if rowsDeleted > 0 && !dryRun {
|
||||
maintenanceRowsDeleted.WithLabelValues(policyID).Add(float64(rowsDeleted))
|
||||
}
|
||||
}
|
||||
|
||||
// IncMaintenanceConfigChange increments config audit metric.
|
||||
func IncMaintenanceConfigChange(action string) {
|
||||
if action == "" {
|
||||
action = "unknown"
|
||||
}
|
||||
maintenanceConfigChanges.WithLabelValues(action).Inc()
|
||||
}
|
||||
Reference in New Issue
Block a user