feat(observability): add maintenance policy metrics
Prometheus: runs, duration, rows_deleted, config_changes; инкремент при CRUD и Execute. Co-authored-by: Cursor <[email protected]>
This commit is contained in:
@@ -6,6 +6,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"evobgp/internal/jobs"
|
"evobgp/internal/jobs"
|
||||||
|
"evobgp/internal/observability"
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -107,6 +108,7 @@ func (s *Server) handleCreateMaintenancePolicy(w http.ResponseWriter, r *http.Re
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
_ = s.store.AppendMaintenancePolicyConfigAudit(actorPrefix(a), p.ID, "create", nil, maintenancePolicyJSON(p))
|
_ = s.store.AppendMaintenancePolicyConfigAudit(actorPrefix(a), p.ID, "create", nil, maintenancePolicyJSON(p))
|
||||||
|
observability.IncMaintenanceConfigChange("create")
|
||||||
s.reloadMaintenanceConfig(r)
|
s.reloadMaintenanceConfig(r)
|
||||||
writeJSON(w, http.StatusCreated, maintenancePolicyJSON(p))
|
writeJSON(w, http.StatusCreated, maintenancePolicyJSON(p))
|
||||||
}
|
}
|
||||||
@@ -133,6 +135,7 @@ func (s *Server) handlePatchMaintenancePolicy(w http.ResponseWriter, r *http.Req
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
_ = s.store.AppendMaintenancePolicyConfigAudit(actorPrefix(a), id, "update", maintenancePolicyJSON(before), maintenancePolicyJSON(updated))
|
_ = s.store.AppendMaintenancePolicyConfigAudit(actorPrefix(a), id, "update", maintenancePolicyJSON(before), maintenancePolicyJSON(updated))
|
||||||
|
observability.IncMaintenanceConfigChange("update")
|
||||||
s.reloadMaintenanceConfig(r)
|
s.reloadMaintenanceConfig(r)
|
||||||
writeJSON(w, http.StatusOK, maintenancePolicyJSON(updated))
|
writeJSON(w, http.StatusOK, maintenancePolicyJSON(updated))
|
||||||
}
|
}
|
||||||
@@ -153,6 +156,7 @@ func (s *Server) handleDeleteMaintenancePolicy(w http.ResponseWriter, r *http.Re
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
_ = s.store.AppendMaintenancePolicyConfigAudit(actorPrefix(a), id, "delete", maintenancePolicyJSON(before), nil)
|
_ = s.store.AppendMaintenancePolicyConfigAudit(actorPrefix(a), id, "delete", maintenancePolicyJSON(before), nil)
|
||||||
|
observability.IncMaintenanceConfigChange("delete")
|
||||||
s.reloadMaintenanceConfig(r)
|
s.reloadMaintenanceConfig(r)
|
||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/observability"
|
||||||
"evobgp/internal/pgmonitor"
|
"evobgp/internal/pgmonitor"
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
|
|
||||||
@@ -22,19 +23,29 @@ type PolicyExecutor struct {
|
|||||||
|
|
||||||
// Execute runs cleanup and/or vacuum steps for a policy.
|
// Execute runs cleanup and/or vacuum steps for a policy.
|
||||||
func (e *PolicyExecutor) Execute(ctx context.Context, policy *store.MaintenancePolicy, dryRun bool) (map[string]any, error) {
|
func (e *PolicyExecutor) Execute(ctx context.Context, policy *store.MaintenancePolicy, dryRun bool) (map[string]any, error) {
|
||||||
if e == nil || e.Pool == nil {
|
start := time.Now()
|
||||||
return nil, fmt.Errorf("maintenance: postgres not configured")
|
|
||||||
}
|
|
||||||
if policy == nil {
|
if policy == nil {
|
||||||
return nil, store.ErrInvalidInput
|
return nil, store.ErrInvalidInput
|
||||||
}
|
}
|
||||||
|
action := policyAction(policy)
|
||||||
|
record := func(status string, detail map[string]any) {
|
||||||
|
observability.RecordMaintenancePolicyRun(policy.ID, action, status, dryRun, time.Since(start), rowsDeletedFromDetail(detail))
|
||||||
|
}
|
||||||
|
|
||||||
|
if e == nil || e.Pool == nil {
|
||||||
|
record("failed", nil)
|
||||||
|
return nil, fmt.Errorf("maintenance: postgres not configured")
|
||||||
|
}
|
||||||
if err := ValidateTableName(policy.TableName); err != nil {
|
if err := ValidateTableName(policy.TableName); err != nil {
|
||||||
|
record("failed", nil)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := ValidateCondition(policy.Condition); err != nil {
|
if err := ValidateCondition(policy.Condition); err != nil {
|
||||||
|
record("failed", nil)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if !store.ValidVacuumStrategy(policy.VacuumStrategy) {
|
if !store.ValidVacuumStrategy(policy.VacuumStrategy) {
|
||||||
|
record("failed", nil)
|
||||||
return nil, store.ErrInvalidInput
|
return nil, store.ErrInvalidInput
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -50,6 +61,7 @@ func (e *PolicyExecutor) Execute(ctx context.Context, policy *store.MaintenanceP
|
|||||||
detail[k] = v
|
detail[k] = v
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
record("failed", detail)
|
||||||
return detail, err
|
return detail, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -61,6 +73,7 @@ func (e *PolicyExecutor) Execute(ctx context.Context, policy *store.MaintenanceP
|
|||||||
detail["vacuum"] = vacDetail
|
detail["vacuum"] = vacDetail
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
record("failed", detail)
|
||||||
return detail, err
|
return detail, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -68,10 +81,42 @@ func (e *PolicyExecutor) Execute(ctx context.Context, policy *store.MaintenanceP
|
|||||||
if !dryRun && e.Store != nil {
|
if !dryRun && e.Store != nil {
|
||||||
_ = e.Store.TouchMaintenancePolicyRun(policy.ID, "succeeded", "")
|
_ = e.Store.TouchMaintenancePolicyRun(policy.ID, "succeeded", "")
|
||||||
}
|
}
|
||||||
|
record("succeeded", detail)
|
||||||
return detail, nil
|
return detail, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func policyAction(p *store.MaintenancePolicy) string {
|
||||||
|
if p == nil {
|
||||||
|
return "run"
|
||||||
|
}
|
||||||
|
if p.RetentionPeriodSec != nil || p.MaxRows != nil {
|
||||||
|
if p.VacuumStrategy != store.VacuumStrategyNone {
|
||||||
|
return "cleanup_vacuum"
|
||||||
|
}
|
||||||
|
return "cleanup"
|
||||||
|
}
|
||||||
|
if p.VacuumStrategy != store.VacuumStrategyNone {
|
||||||
|
return p.VacuumStrategy
|
||||||
|
}
|
||||||
|
return "run"
|
||||||
|
}
|
||||||
|
|
||||||
|
func rowsDeletedFromDetail(detail map[string]any) int64 {
|
||||||
|
if detail == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
switch v := detail["deleted"].(type) {
|
||||||
|
case int64:
|
||||||
|
return v
|
||||||
|
case int:
|
||||||
|
return int64(v)
|
||||||
|
case float64:
|
||||||
|
return int64(v)
|
||||||
|
default:
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (e *PolicyExecutor) runCleanup(ctx context.Context, policy *store.MaintenancePolicy, dryRun bool) (map[string]any, error) {
|
func (e *PolicyExecutor) runCleanup(ctx context.Context, policy *store.MaintenancePolicy, dryRun bool) (map[string]any, error) {
|
||||||
detail := map[string]any{"cleanup": true}
|
detail := map[string]any{"cleanup": true}
|
||||||
limit := NormalizeBatchLimit(policy.MaxRows)
|
limit := NormalizeBatchLimit(policy.MaxRows)
|
||||||
|
|||||||
@@ -0,0 +1,72 @@
|
|||||||
|
package observability
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
maintenancePolicyRuns = promauto.NewCounterVec(
|
||||||
|
prometheus.CounterOpts{
|
||||||
|
Namespace: namespace,
|
||||||
|
Name: "maintenance_policy_runs_total",
|
||||||
|
Help: "Maintenance policy executions by outcome.",
|
||||||
|
},
|
||||||
|
[]string{"policy_id", "action", "status", "dry_run"},
|
||||||
|
)
|
||||||
|
|
||||||
|
maintenancePolicyDuration = promauto.NewHistogramVec(
|
||||||
|
prometheus.HistogramOpts{
|
||||||
|
Namespace: namespace,
|
||||||
|
Name: "maintenance_policy_duration_seconds",
|
||||||
|
Help: "Maintenance policy execution duration.",
|
||||||
|
Buckets: prometheus.ExponentialBuckets(0.05, 2, 12),
|
||||||
|
},
|
||||||
|
[]string{"policy_id", "action"},
|
||||||
|
)
|
||||||
|
|
||||||
|
maintenanceRowsDeleted = promauto.NewCounterVec(
|
||||||
|
prometheus.CounterOpts{
|
||||||
|
Namespace: namespace,
|
||||||
|
Name: "maintenance_policy_rows_deleted_total",
|
||||||
|
Help: "Rows deleted by maintenance cleanup policies.",
|
||||||
|
},
|
||||||
|
[]string{"policy_id"},
|
||||||
|
)
|
||||||
|
|
||||||
|
maintenanceConfigChanges = promauto.NewCounterVec(
|
||||||
|
prometheus.CounterOpts{
|
||||||
|
Namespace: namespace,
|
||||||
|
Name: "maintenance_config_changes_total",
|
||||||
|
Help: "Maintenance policy configuration changes from UI/API.",
|
||||||
|
},
|
||||||
|
[]string{"action"},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
// RecordMaintenancePolicyRun updates run counters and histograms.
|
||||||
|
func RecordMaintenancePolicyRun(policyID, action, status string, dryRun bool, duration time.Duration, rowsDeleted int64) {
|
||||||
|
if policyID == "" {
|
||||||
|
policyID = "unknown"
|
||||||
|
}
|
||||||
|
if action == "" {
|
||||||
|
action = "run"
|
||||||
|
}
|
||||||
|
dry := strconv.FormatBool(dryRun)
|
||||||
|
maintenancePolicyRuns.WithLabelValues(policyID, action, status, dry).Inc()
|
||||||
|
maintenancePolicyDuration.WithLabelValues(policyID, action).Observe(duration.Seconds())
|
||||||
|
if rowsDeleted > 0 && !dryRun {
|
||||||
|
maintenanceRowsDeleted.WithLabelValues(policyID).Add(float64(rowsDeleted))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// IncMaintenanceConfigChange increments config audit metric.
|
||||||
|
func IncMaintenanceConfigChange(action string) {
|
||||||
|
if action == "" {
|
||||||
|
action = "unknown"
|
||||||
|
}
|
||||||
|
maintenanceConfigChanges.WithLabelValues(action).Inc()
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user