diff --git a/internal/httpapi/routes_maintenance.go b/internal/httpapi/routes_maintenance.go index 8620075..2796ac9 100644 --- a/internal/httpapi/routes_maintenance.go +++ b/internal/httpapi/routes_maintenance.go @@ -6,6 +6,7 @@ import ( "strings" "evobgp/internal/jobs" + "evobgp/internal/observability" "evobgp/internal/store" ) @@ -107,6 +108,7 @@ func (s *Server) handleCreateMaintenancePolicy(w http.ResponseWriter, r *http.Re return } _ = s.store.AppendMaintenancePolicyConfigAudit(actorPrefix(a), p.ID, "create", nil, maintenancePolicyJSON(p)) + observability.IncMaintenanceConfigChange("create") s.reloadMaintenanceConfig(r) writeJSON(w, http.StatusCreated, maintenancePolicyJSON(p)) } @@ -133,6 +135,7 @@ func (s *Server) handlePatchMaintenancePolicy(w http.ResponseWriter, r *http.Req return } _ = s.store.AppendMaintenancePolicyConfigAudit(actorPrefix(a), id, "update", maintenancePolicyJSON(before), maintenancePolicyJSON(updated)) + observability.IncMaintenanceConfigChange("update") s.reloadMaintenanceConfig(r) writeJSON(w, http.StatusOK, maintenancePolicyJSON(updated)) } @@ -153,6 +156,7 @@ func (s *Server) handleDeleteMaintenancePolicy(w http.ResponseWriter, r *http.Re return } _ = s.store.AppendMaintenancePolicyConfigAudit(actorPrefix(a), id, "delete", maintenancePolicyJSON(before), nil) + observability.IncMaintenanceConfigChange("delete") s.reloadMaintenanceConfig(r) w.WriteHeader(http.StatusNoContent) } diff --git a/internal/maintenance/policy_executor.go b/internal/maintenance/policy_executor.go index 63338c2..bc42594 100644 --- a/internal/maintenance/policy_executor.go +++ b/internal/maintenance/policy_executor.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "evobgp/internal/observability" "evobgp/internal/pgmonitor" "evobgp/internal/store" @@ -22,19 +23,29 @@ type PolicyExecutor struct { // Execute runs cleanup and/or vacuum steps for a policy. func (e *PolicyExecutor) Execute(ctx context.Context, policy *store.MaintenancePolicy, dryRun bool) (map[string]any, error) { - if e == nil || e.Pool == nil { - return nil, fmt.Errorf("maintenance: postgres not configured") - } + start := time.Now() if policy == nil { return nil, store.ErrInvalidInput } + action := policyAction(policy) + record := func(status string, detail map[string]any) { + observability.RecordMaintenancePolicyRun(policy.ID, action, status, dryRun, time.Since(start), rowsDeletedFromDetail(detail)) + } + + if e == nil || e.Pool == nil { + record("failed", nil) + return nil, fmt.Errorf("maintenance: postgres not configured") + } if err := ValidateTableName(policy.TableName); err != nil { + record("failed", nil) return nil, err } if err := ValidateCondition(policy.Condition); err != nil { + record("failed", nil) return nil, err } if !store.ValidVacuumStrategy(policy.VacuumStrategy) { + record("failed", nil) return nil, store.ErrInvalidInput } @@ -50,6 +61,7 @@ func (e *PolicyExecutor) Execute(ctx context.Context, policy *store.MaintenanceP detail[k] = v } if err != nil { + record("failed", detail) return detail, err } } @@ -61,6 +73,7 @@ func (e *PolicyExecutor) Execute(ctx context.Context, policy *store.MaintenanceP detail["vacuum"] = vacDetail } if err != nil { + record("failed", detail) return detail, err } } @@ -68,10 +81,42 @@ func (e *PolicyExecutor) Execute(ctx context.Context, policy *store.MaintenanceP if !dryRun && e.Store != nil { _ = e.Store.TouchMaintenancePolicyRun(policy.ID, "succeeded", "") } - + record("succeeded", detail) return detail, nil } +func policyAction(p *store.MaintenancePolicy) string { + if p == nil { + return "run" + } + if p.RetentionPeriodSec != nil || p.MaxRows != nil { + if p.VacuumStrategy != store.VacuumStrategyNone { + return "cleanup_vacuum" + } + return "cleanup" + } + if p.VacuumStrategy != store.VacuumStrategyNone { + return p.VacuumStrategy + } + return "run" +} + +func rowsDeletedFromDetail(detail map[string]any) int64 { + if detail == nil { + return 0 + } + switch v := detail["deleted"].(type) { + case int64: + return v + case int: + return int64(v) + case float64: + return int64(v) + default: + return 0 + } +} + func (e *PolicyExecutor) runCleanup(ctx context.Context, policy *store.MaintenancePolicy, dryRun bool) (map[string]any, error) { detail := map[string]any{"cleanup": true} limit := NormalizeBatchLimit(policy.MaxRows) diff --git a/internal/observability/maintenance_metrics.go b/internal/observability/maintenance_metrics.go new file mode 100644 index 0000000..9d7d0c9 --- /dev/null +++ b/internal/observability/maintenance_metrics.go @@ -0,0 +1,72 @@ +package observability + +import ( + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + maintenancePolicyRuns = promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "maintenance_policy_runs_total", + Help: "Maintenance policy executions by outcome.", + }, + []string{"policy_id", "action", "status", "dry_run"}, + ) + + maintenancePolicyDuration = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespace, + Name: "maintenance_policy_duration_seconds", + Help: "Maintenance policy execution duration.", + Buckets: prometheus.ExponentialBuckets(0.05, 2, 12), + }, + []string{"policy_id", "action"}, + ) + + maintenanceRowsDeleted = promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "maintenance_policy_rows_deleted_total", + Help: "Rows deleted by maintenance cleanup policies.", + }, + []string{"policy_id"}, + ) + + maintenanceConfigChanges = promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "maintenance_config_changes_total", + Help: "Maintenance policy configuration changes from UI/API.", + }, + []string{"action"}, + ) +) + +// RecordMaintenancePolicyRun updates run counters and histograms. +func RecordMaintenancePolicyRun(policyID, action, status string, dryRun bool, duration time.Duration, rowsDeleted int64) { + if policyID == "" { + policyID = "unknown" + } + if action == "" { + action = "run" + } + dry := strconv.FormatBool(dryRun) + maintenancePolicyRuns.WithLabelValues(policyID, action, status, dry).Inc() + maintenancePolicyDuration.WithLabelValues(policyID, action).Observe(duration.Seconds()) + if rowsDeleted > 0 && !dryRun { + maintenanceRowsDeleted.WithLabelValues(policyID).Add(float64(rowsDeleted)) + } +} + +// IncMaintenanceConfigChange increments config audit metric. +func IncMaintenanceConfigChange(action string) { + if action == "" { + action = "unknown" + } + maintenanceConfigChanges.WithLabelValues(action).Inc() +}