feat(db): implement PostgreSQL monitoring and maintenance features
CI / changes (push) Successful in 9s
CI / commitlint (push) Has been skipped
CI / openapi (push) Successful in 26s
CI / web (push) Successful in 33s
CI / go (push) Successful in 2m11s
CI / bird2 (push) Successful in 16s
CI / release (push) Successful in 3m27s
CI / changes (push) Successful in 9s
CI / commitlint (push) Has been skipped
CI / openapi (push) Successful in 26s
CI / web (push) Successful in 33s
CI / go (push) Successful in 2m11s
CI / bird2 (push) Successful in 16s
CI / release (push) Successful in 3m27s
Added PostgreSQL monitoring and maintenance capabilities to the API, including new endpoints for instance-level metrics, maintenance operations, and job scheduling. Updated the HTTP API to support PostgreSQL monitoring routes and integrated a background scheduler for metrics collection. Enhanced the CLI with database commands for maintenance tasks. Updated documentation to reflect these changes.
This commit is contained in:
@@ -0,0 +1,209 @@
|
||||
// Package dbcli implements control-plane PostgreSQL maintenance CLI (HTTP or local DSN).
|
||||
package dbcli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/db"
|
||||
"evobgp/internal/httpclient"
|
||||
"evobgp/internal/pgmonitor"
|
||||
)
|
||||
|
||||
// Run executes db subcommands; args exclude program name and "db".
|
||||
func Run(args []string) int {
|
||||
if len(args) == 0 {
|
||||
printUsage()
|
||||
return 2
|
||||
}
|
||||
switch args[0] {
|
||||
case "report":
|
||||
return cmdReport(args[1:])
|
||||
case "vacuum":
|
||||
return cmdMaint(args[1:], "vacuum", "/v1/postgres/vacuum")
|
||||
case "analyze":
|
||||
return cmdMaint(args[1:], "analyze", "/v1/postgres/analyze")
|
||||
case "cleanup":
|
||||
return cmdCleanup(args[1:])
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "dbcli: unknown command %q\n", args[0])
|
||||
printUsage()
|
||||
return 2
|
||||
}
|
||||
}
|
||||
|
||||
func printUsage() {
|
||||
fmt.Fprintln(os.Stderr, `usage:
|
||||
evobgp-api db report [--api-url URL] [--token TOKEN] [--format json]
|
||||
evobgp-api db vacuum [--table NAME] [--dry-run] [--api-url URL] [--token TOKEN]
|
||||
evobgp-api db analyze [--table NAME] [--dry-run] [--api-url URL] [--token TOKEN]
|
||||
evobgp-api db cleanup --policy NAME [--dry-run] [--limit N] [--api-url URL] [--token TOKEN]
|
||||
Local break-glass: set EVOBGP_DATABASE_URL (report only uses direct SQL).`)
|
||||
}
|
||||
|
||||
func cmdReport(args []string) int {
|
||||
fs := flag.NewFlagSet("report", flag.ExitOnError)
|
||||
apiURL := fs.String("api-url", "", "control plane base URL")
|
||||
token := fs.String("token", "", "Bearer token (operator)")
|
||||
format := fs.String("format", "json", "output format (json)")
|
||||
_ = fs.Parse(args)
|
||||
|
||||
if dsn := strings.TrimSpace(os.Getenv("EVOBGP_DATABASE_URL")); dsn != "" && *apiURL == "" {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
pool, err := db.OpenPostgresPool(ctx, dsn)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 1
|
||||
}
|
||||
defer pool.Close()
|
||||
svc := pgmonitor.NewService(pool)
|
||||
ov, err := svc.Overview(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 1
|
||||
}
|
||||
return writeJSONStdout(ov, *format)
|
||||
}
|
||||
if *apiURL == "" || *token == "" {
|
||||
fmt.Fprintln(os.Stderr, "report: --api-url and --token required without EVOBGP_DATABASE_URL")
|
||||
return 2
|
||||
}
|
||||
body, err := apiGET(*apiURL, *token, "/v1/monitoring/postgres/overview")
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 1
|
||||
}
|
||||
var pretty any
|
||||
if err := json.Unmarshal(body, &pretty); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 1
|
||||
}
|
||||
return writeJSONStdout(pretty, *format)
|
||||
}
|
||||
|
||||
func cmdMaint(args []string, _ string, path string) int {
|
||||
fs := flag.NewFlagSet("maint", flag.ExitOnError)
|
||||
table := fs.String("table", "", "table name")
|
||||
dryRun := fs.Bool("dry-run", false, "dry run only")
|
||||
apiURL := fs.String("api-url", "", "control plane base URL")
|
||||
token := fs.String("token", "", "Bearer token (operator)")
|
||||
_ = fs.Parse(args)
|
||||
if *apiURL == "" || *token == "" {
|
||||
fmt.Fprintln(os.Stderr, "maintenance: --api-url and --token are required")
|
||||
return 2
|
||||
}
|
||||
payload := map[string]any{"dry_run": *dryRun}
|
||||
if *table != "" {
|
||||
payload["table"] = *table
|
||||
}
|
||||
body, err := apiPOST(*apiURL, *token, path, payload)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 1
|
||||
}
|
||||
return writeRawJSON(body)
|
||||
}
|
||||
|
||||
func cmdCleanup(args []string) int {
|
||||
fs := flag.NewFlagSet("cleanup", flag.ExitOnError)
|
||||
policy := fs.String("policy", "", "cleanup policy name")
|
||||
dryRun := fs.Bool("dry-run", true, "dry run")
|
||||
limit := fs.Int("limit", 10000, "max rows")
|
||||
apiURL := fs.String("api-url", "", "control plane base URL")
|
||||
token := fs.String("token", "", "Bearer token (operator)")
|
||||
_ = fs.Parse(args)
|
||||
if *policy == "" {
|
||||
fmt.Fprintln(os.Stderr, "cleanup: --policy is required")
|
||||
return 2
|
||||
}
|
||||
if *apiURL == "" || *token == "" {
|
||||
fmt.Fprintln(os.Stderr, "cleanup: --api-url and --token are required")
|
||||
return 2
|
||||
}
|
||||
payload := map[string]any{"policy": *policy, "dry_run": *dryRun, "limit": *limit}
|
||||
body, err := apiPOST(*apiURL, *token, "/v1/postgres/cleanup", payload)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 1
|
||||
}
|
||||
return writeRawJSON(body)
|
||||
}
|
||||
|
||||
func apiGET(base, token, path string) ([]byte, error) {
|
||||
u := strings.TrimRight(base, "/") + path
|
||||
req, err := http.NewRequest(http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
resp, err := httpclient.DoWithRetry(ctx, httpclient.New(60*time.Second), req, 3)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("dbcli: GET %s: %s: %s", path, resp.Status, strings.TrimSpace(string(b)))
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func apiPOST(base, token, path string, payload map[string]any) ([]byte, error) {
|
||||
b, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
u := strings.TrimRight(base, "/") + path
|
||||
req, err := http.NewRequest(http.MethodPost, u, bytes.NewReader(b))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
resp, err := httpclient.DoWithRetry(ctx, httpclient.New(60*time.Second), req, 3)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
out, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("dbcli: POST %s: %s: %s", path, resp.Status, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func writeJSONStdout(v any, format string) int {
|
||||
if format != "json" {
|
||||
fmt.Fprintln(os.Stderr, "only json format supported")
|
||||
return 2
|
||||
}
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
if err := enc.Encode(v); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func writeRawJSON(b []byte) int {
|
||||
var v any
|
||||
if err := json.Unmarshal(b, &v); err != nil {
|
||||
_, _ = os.Stdout.Write(b)
|
||||
return 0
|
||||
}
|
||||
return writeJSONStdout(v, "json")
|
||||
}
|
||||
@@ -51,7 +51,7 @@ func BootstrapWorkers(ctx context.Context, opts Options) (store.Backend, *jobs.R
|
||||
}
|
||||
|
||||
cdnHTTP := NewCDNHTTPClient()
|
||||
wk := &jobs.Worker{Store: backend, HTTPClient: cdnHTTP}
|
||||
wk := &jobs.Worker{Store: backend, PgPool: pool, HTTPClient: cdnHTTP}
|
||||
reg := jobs.NewRegistry(wk.Process)
|
||||
wk.Registry = reg
|
||||
if pool != nil {
|
||||
|
||||
@@ -76,6 +76,8 @@ func (s *Server) registerV1(m *http.ServeMux) {
|
||||
m.HandleFunc("GET /speakers/{speaker_id}/bundle/{revision_id}", s.handleNodeBundle)
|
||||
m.HandleFunc("POST /nodes/enroll", s.handleNodeEnroll)
|
||||
s.registerCRUDRoutes(m)
|
||||
s.registerPostgresMonitoringRoutes(m)
|
||||
s.registerPostgresMaintenanceRoutes(m)
|
||||
}
|
||||
|
||||
func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/jobs"
|
||||
"evobgp/internal/pgmonitor"
|
||||
)
|
||||
|
||||
var (
|
||||
pgMaintRateMu sync.Mutex
|
||||
pgMaintLastByTK = map[string]time.Time{}
|
||||
)
|
||||
|
||||
func (s *Server) registerPostgresMaintenanceRoutes(m *http.ServeMux) {
|
||||
m.HandleFunc("POST /postgres/vacuum", s.handlePostgresVacuum)
|
||||
m.HandleFunc("POST /postgres/vacuum-analyze", s.handlePostgresVacuumAnalyze)
|
||||
m.HandleFunc("POST /postgres/analyze", s.handlePostgresAnalyze)
|
||||
m.HandleFunc("POST /postgres/reindex", s.handlePostgresReindex)
|
||||
m.HandleFunc("POST /postgres/cleanup", s.handlePostgresCleanup)
|
||||
m.HandleFunc("GET /postgres/maintenance/logs", s.handlePostgresMaintenanceLogs)
|
||||
}
|
||||
|
||||
func (s *Server) requireOperatorStrict(w http.ResponseWriter, a Auth) bool {
|
||||
if strings.ToLower(a.Role) != "operator" {
|
||||
writeProblem(w, http.StatusForbidden, "Forbidden", "operator role required")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Server) checkPgMaintRateLimit(tenantID, kind string) bool {
|
||||
key := tenantID + ":" + kind
|
||||
pgMaintRateMu.Lock()
|
||||
defer pgMaintRateMu.Unlock()
|
||||
if t, ok := pgMaintLastByTK[key]; ok && time.Since(t) < 60*time.Second {
|
||||
return false
|
||||
}
|
||||
pgMaintLastByTK[key] = time.Now().UTC()
|
||||
return true
|
||||
}
|
||||
|
||||
type pgMaintBody struct {
|
||||
Table string `json:"table"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
Index string `json:"index"`
|
||||
Policy string `json:"policy"`
|
||||
Limit int `json:"limit"`
|
||||
}
|
||||
|
||||
func (s *Server) decodePgMaintBody(r *http.Request) (pgMaintBody, bool) {
|
||||
var body pgMaintBody
|
||||
if r.Body == nil || r.ContentLength == 0 {
|
||||
return body, true
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&body); err != nil && err != io.EOF {
|
||||
return body, false
|
||||
}
|
||||
return body, true
|
||||
}
|
||||
|
||||
func (s *Server) enqueuePostgresMaint(w http.ResponseWriter, r *http.Request, a Auth, kind string, meta map[string]any) {
|
||||
if !s.requirePostgres(w) || !s.requireOperatorStrict(w, a) {
|
||||
return
|
||||
}
|
||||
if !s.checkPgMaintRateLimit(a.TenantID, kind) {
|
||||
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
|
||||
return
|
||||
}
|
||||
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
|
||||
var idemPtr *string
|
||||
if idem != "" {
|
||||
idemPtr = &idem
|
||||
}
|
||||
meta["actor_prefix"] = actorPrefix(a)
|
||||
j, _, err := s.jobs.Enqueue(a.TenantID, kind, idemPtr, nil, meta)
|
||||
if err != nil {
|
||||
writeInternalError(w, "postgres_maint_enqueue", err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
snap := j.Snapshot()
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"job_id": snap["job_id"], "status": snap["status"]})
|
||||
}
|
||||
|
||||
func (s *Server) handlePostgresVacuum(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok {
|
||||
writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth")
|
||||
return
|
||||
}
|
||||
body, ok2 := s.decodePgMaintBody(r)
|
||||
if !ok2 {
|
||||
writeProblem(w, http.StatusBadRequest, "Bad Request", invalidInputDetail)
|
||||
return
|
||||
}
|
||||
s.enqueuePostgresMaint(w, r, a, jobs.KindPostgresVacuum, map[string]any{
|
||||
"table": body.Table, "dry_run": body.DryRun, "job_title": "PostgreSQL VACUUM",
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) handlePostgresVacuumAnalyze(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok {
|
||||
writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth")
|
||||
return
|
||||
}
|
||||
body, ok2 := s.decodePgMaintBody(r)
|
||||
if !ok2 {
|
||||
writeProblem(w, http.StatusBadRequest, "Bad Request", invalidInputDetail)
|
||||
return
|
||||
}
|
||||
s.enqueuePostgresMaint(w, r, a, jobs.KindPostgresVacuumAnalyze, map[string]any{
|
||||
"table": body.Table, "dry_run": body.DryRun, "job_title": "PostgreSQL VACUUM ANALYZE",
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) handlePostgresAnalyze(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok {
|
||||
writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth")
|
||||
return
|
||||
}
|
||||
body, ok2 := s.decodePgMaintBody(r)
|
||||
if !ok2 {
|
||||
writeProblem(w, http.StatusBadRequest, "Bad Request", invalidInputDetail)
|
||||
return
|
||||
}
|
||||
s.enqueuePostgresMaint(w, r, a, jobs.KindPostgresAnalyze, map[string]any{
|
||||
"table": body.Table, "dry_run": body.DryRun, "job_title": "PostgreSQL ANALYZE",
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) handlePostgresReindex(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok {
|
||||
writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth")
|
||||
return
|
||||
}
|
||||
body, ok2 := s.decodePgMaintBody(r)
|
||||
if !ok2 {
|
||||
writeProblem(w, http.StatusBadRequest, "Bad Request", invalidInputDetail)
|
||||
return
|
||||
}
|
||||
table := body.Table
|
||||
if table == "" {
|
||||
table = body.Index
|
||||
}
|
||||
s.enqueuePostgresMaint(w, r, a, jobs.KindPostgresReindex, map[string]any{
|
||||
"table": table, "dry_run": body.DryRun, "job_title": "PostgreSQL REINDEX",
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) handlePostgresCleanup(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok {
|
||||
writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth")
|
||||
return
|
||||
}
|
||||
body, ok2 := s.decodePgMaintBody(r)
|
||||
if !ok2 {
|
||||
writeProblem(w, http.StatusBadRequest, "Bad Request", invalidInputDetail)
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(body.Policy) == "" {
|
||||
writeProblem(w, http.StatusBadRequest, "Bad Request", "policy is required")
|
||||
return
|
||||
}
|
||||
s.enqueuePostgresMaint(w, r, a, jobs.KindPostgresCleanup, map[string]any{
|
||||
"policy": body.Policy, "dry_run": body.DryRun, "limit": body.Limit,
|
||||
"job_title": "PostgreSQL cleanup",
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) handlePostgresMaintenanceLogs(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok || !s.requireAtLeast(w, a, "viewer") || !s.requirePostgres(w) {
|
||||
return
|
||||
}
|
||||
cursor := r.URL.Query().Get("cursor")
|
||||
limit := parseLimitQuery(r, 20, 100)
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
items, next, hasMore, err := pgmonitor.ListMaintenanceLogs(ctx, s.pgMonitor.Pool(), cursor, limit)
|
||||
if err != nil {
|
||||
writeInternalError(w, "postgres_maint_logs", err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"items": items, "next_cursor": next, "has_more": hasMore})
|
||||
}
|
||||
|
||||
func actorPrefix(a Auth) string {
|
||||
if len(a.Token) >= 8 {
|
||||
return a.Token[:8]
|
||||
}
|
||||
return a.Role
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (s *Server) registerPostgresMonitoringRoutes(m *http.ServeMux) {
|
||||
m.HandleFunc("GET /monitoring/postgres/overview", s.handlePostgresOverview)
|
||||
m.HandleFunc("GET /monitoring/postgres/queries", s.handlePostgresQueries)
|
||||
m.HandleFunc("GET /monitoring/postgres/locks", s.handlePostgresLocks)
|
||||
m.HandleFunc("GET /monitoring/postgres/tables", s.handlePostgresTables)
|
||||
m.HandleFunc("GET /monitoring/postgres/recommendations", s.handlePostgresRecommendations)
|
||||
m.HandleFunc("GET /monitoring/correlation", s.handleMonitoringCorrelation)
|
||||
}
|
||||
|
||||
func (s *Server) requirePostgres(w http.ResponseWriter) bool {
|
||||
if s.pgMonitor == nil {
|
||||
writeProblem(w, http.StatusServiceUnavailable, "Unavailable", "postgresql backend required")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func parseLimitQuery(r *http.Request, def, max int) int {
|
||||
if v := r.URL.Query().Get("limit"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func (s *Server) handlePostgresOverview(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok || !s.requireAtLeast(w, a, "viewer") || !s.requirePostgres(w) {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second)
|
||||
defer cancel()
|
||||
out, err := s.pgMonitor.Overview(ctx)
|
||||
if err != nil {
|
||||
writeInternalError(w, "postgres_overview", err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
func (s *Server) handlePostgresQueries(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok || !s.requireAtLeast(w, a, "viewer") || !s.requirePostgres(w) {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second)
|
||||
defer cancel()
|
||||
out, err := s.pgMonitor.TopQueries(ctx, parseLimitQuery(r, 20, 100))
|
||||
if err != nil {
|
||||
writeInternalError(w, "postgres_queries", err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
func (s *Server) handlePostgresLocks(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok || !s.requireAtLeast(w, a, "viewer") || !s.requirePostgres(w) {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second)
|
||||
defer cancel()
|
||||
out, err := s.pgMonitor.Locks(ctx)
|
||||
if err != nil {
|
||||
writeInternalError(w, "postgres_locks", err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"items": out})
|
||||
}
|
||||
|
||||
func (s *Server) handlePostgresTables(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok || !s.requireAtLeast(w, a, "viewer") || !s.requirePostgres(w) {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second)
|
||||
defer cancel()
|
||||
out, err := s.pgMonitor.Tables(ctx, parseLimitQuery(r, 20, 100))
|
||||
if err != nil {
|
||||
writeInternalError(w, "postgres_tables", err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"items": out})
|
||||
}
|
||||
|
||||
func (s *Server) handlePostgresRecommendations(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok || !s.requireAtLeast(w, a, "viewer") || !s.requirePostgres(w) {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
out, err := s.pgMonitor.Recommendations(ctx)
|
||||
if err != nil {
|
||||
writeInternalError(w, "postgres_recommendations", err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
func (s *Server) handleMonitoringCorrelation(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok || !s.requireAtLeast(w, a, "viewer") || !s.requirePostgres(w) {
|
||||
return
|
||||
}
|
||||
window := 60
|
||||
if v := r.URL.Query().Get("window"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil {
|
||||
window = n
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
out, err := s.pgMonitor.Correlation(ctx, window)
|
||||
if err != nil {
|
||||
writeInternalError(w, "monitoring_correlation", err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPostgresOverviewMemoryBackend503(t *testing.T) {
|
||||
srv, err := New(Options{SeedDemo: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1/monitoring/postgres/overview", nil)
|
||||
req.Header.Set("Authorization", "Bearer dev")
|
||||
rec := httptest.NewRecorder()
|
||||
srv.Handler().ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"evobgp/internal/jobs"
|
||||
"evobgp/internal/pgmonitor"
|
||||
"evobgp/internal/store"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
@@ -19,6 +20,7 @@ import (
|
||||
type Server struct {
|
||||
store store.Backend
|
||||
pgPool *pgxpool.Pool
|
||||
pgMonitor *pgmonitor.Service
|
||||
jobs *jobs.Registry
|
||||
bundlePriv ed25519.PrivateKey
|
||||
keyResolver *apiKeyResolver
|
||||
@@ -63,9 +65,14 @@ func New(opts Options) (*Server, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var pgMon *pgmonitor.Service
|
||||
if pool != nil {
|
||||
pgMon = pgmonitor.NewService(pool)
|
||||
}
|
||||
s := &Server{
|
||||
store: backend,
|
||||
pgPool: pool,
|
||||
pgMonitor: pgMon,
|
||||
jobs: reg,
|
||||
bundlePriv: priv,
|
||||
keyResolver: resolver,
|
||||
@@ -89,3 +96,10 @@ func (s *Server) Store() store.Backend { return s.store }
|
||||
|
||||
// Jobs exposes the in-process async job registry (for scheduler / evobgp-all).
|
||||
func (s *Server) Jobs() *jobs.Registry { return s.jobs }
|
||||
|
||||
// StartBackground starts PostgreSQL monitoring scheduler until ctx is cancelled.
|
||||
func (s *Server) StartBackground(ctx context.Context) {
|
||||
if s != nil && s.pgPool != nil {
|
||||
pgmonitor.StartScheduler(ctx, s.pgPool)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"evobgp/internal/pgmonitor"
|
||||
)
|
||||
|
||||
func (w *Worker) pgService() *pgmonitor.Service {
|
||||
if w == nil || w.PgPool == nil {
|
||||
return nil
|
||||
}
|
||||
return pgmonitor.NewService(w.PgPool)
|
||||
}
|
||||
|
||||
func (w *Worker) runPostgresMetricsRefresh(j *Job) {
|
||||
s := w.pgService()
|
||||
if s == nil {
|
||||
j.Fail("postgresql not configured")
|
||||
return
|
||||
}
|
||||
ctx, cancel := j.workContext()
|
||||
defer cancel()
|
||||
if err := s.RefreshMetricsSnapshot(ctx); err != nil {
|
||||
j.Fail(err.Error())
|
||||
return
|
||||
}
|
||||
j.Succeed()
|
||||
}
|
||||
|
||||
func (w *Worker) runPostgresSlowQueryAgg(j *Job) {
|
||||
s := w.pgService()
|
||||
if s == nil {
|
||||
j.Fail("postgresql not configured")
|
||||
return
|
||||
}
|
||||
ctx, cancel := j.workContext()
|
||||
defer cancel()
|
||||
if err := s.AggregateSlowQueries(ctx, 30); err != nil {
|
||||
j.Fail(err.Error())
|
||||
return
|
||||
}
|
||||
j.Succeed()
|
||||
}
|
||||
|
||||
func (w *Worker) runPostgresTableBloat(j *Job) {
|
||||
s := w.pgService()
|
||||
if s == nil {
|
||||
j.Fail("postgresql not configured")
|
||||
return
|
||||
}
|
||||
ctx, cancel := j.workContext()
|
||||
defer cancel()
|
||||
if err := s.EstimateTableBloat(ctx); err != nil {
|
||||
j.Fail(err.Error())
|
||||
return
|
||||
}
|
||||
j.Succeed()
|
||||
}
|
||||
|
||||
func (w *Worker) runPostgresIndexUsage(j *Job) {
|
||||
s := w.pgService()
|
||||
if s == nil {
|
||||
j.Fail("postgresql not configured")
|
||||
return
|
||||
}
|
||||
ctx, cancel := j.workContext()
|
||||
defer cancel()
|
||||
if err := s.AnalyzeIndexUsage(ctx); err != nil {
|
||||
j.Fail(err.Error())
|
||||
return
|
||||
}
|
||||
j.Succeed()
|
||||
}
|
||||
|
||||
func (w *Worker) runPostgresAutovacuumLag(j *Job) {
|
||||
s := w.pgService()
|
||||
if s == nil {
|
||||
j.Fail("postgresql not configured")
|
||||
return
|
||||
}
|
||||
ctx, cancel := j.workContext()
|
||||
defer cancel()
|
||||
if err := s.DetectAutovacuumLag(ctx); err != nil {
|
||||
j.Fail(err.Error())
|
||||
return
|
||||
}
|
||||
j.Succeed()
|
||||
}
|
||||
|
||||
func (w *Worker) runPostgresMaint(j *Job, kind string) {
|
||||
if w == nil || w.PgPool == nil {
|
||||
j.Fail("postgresql not configured")
|
||||
return
|
||||
}
|
||||
table, _ := j.Meta["table"].(string)
|
||||
dryRun, _ := j.Meta["dry_run"].(bool)
|
||||
actor, _ := j.Meta["actor_prefix"].(string)
|
||||
ctx, cancel := j.workContext()
|
||||
defer cancel()
|
||||
auditID, _ := pgmonitor.InsertMaintenanceAudit(ctx, w.PgPool, j.TenantID, actor, kind, table, dryRun)
|
||||
detail, err := pgmonitor.ExecMaintenance(ctx, w.PgPool, kind, table, dryRun)
|
||||
var errMsg *string
|
||||
status := StatusSucceeded
|
||||
if err != nil {
|
||||
s := err.Error()
|
||||
errMsg = &s
|
||||
status = StatusFailed
|
||||
j.Fail(s)
|
||||
} else {
|
||||
j.mergeMeta(map[string]any{"maintenance": detail, "audit_id": auditID})
|
||||
j.Succeed()
|
||||
}
|
||||
if auditID != "" {
|
||||
_ = pgmonitor.FinishMaintenanceAudit(ctx, w.PgPool, auditID, status, detail, errMsg)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) runPostgresCleanup(j *Job) {
|
||||
if w == nil || w.PgPool == nil {
|
||||
j.Fail("postgresql not configured")
|
||||
return
|
||||
}
|
||||
policy, _ := j.Meta["policy"].(string)
|
||||
dryRun, _ := j.Meta["dry_run"].(bool)
|
||||
limit := 0
|
||||
if v, ok := j.Meta["limit"].(float64); ok {
|
||||
limit = int(v)
|
||||
}
|
||||
actor, _ := j.Meta["actor_prefix"].(string)
|
||||
ctx, cancel := j.workContext()
|
||||
defer cancel()
|
||||
auditID, _ := pgmonitor.InsertMaintenanceAudit(ctx, w.PgPool, j.TenantID, actor, "cleanup", policy, dryRun)
|
||||
detail, err := pgmonitor.RunCleanup(ctx, w.PgPool, strings.TrimSpace(policy), dryRun, limit)
|
||||
var errMsg *string
|
||||
status := StatusSucceeded
|
||||
if err != nil {
|
||||
s := err.Error()
|
||||
errMsg = &s
|
||||
status = StatusFailed
|
||||
j.Fail(s)
|
||||
} else {
|
||||
j.mergeMeta(map[string]any{"cleanup": detail, "audit_id": auditID})
|
||||
j.Succeed()
|
||||
}
|
||||
if auditID != "" {
|
||||
_ = pgmonitor.FinishMaintenanceAudit(ctx, w.PgPool, auditID, status, detail, errMsg)
|
||||
}
|
||||
}
|
||||
|
||||
// EnqueuePostgresAnalyzerJobs enqueues periodic analyzer jobs (global tenant id).
|
||||
func EnqueuePostgresAnalyzerJobs(reg *Registry, tenantID string) {
|
||||
if reg == nil || tenantID == "" {
|
||||
return
|
||||
}
|
||||
kinds := []string{
|
||||
KindPostgresMetricsRefresh,
|
||||
KindPostgresSlowQueryAgg,
|
||||
KindPostgresTableBloat,
|
||||
KindPostgresIndexUsage,
|
||||
KindPostgresAutovacuumLag,
|
||||
}
|
||||
for _, k := range kinds {
|
||||
key := fmt.Sprintf("pgmon-%s-%s", k, tenantID)
|
||||
idem := key
|
||||
_, _, _ = reg.Enqueue(tenantID, k, &idem, nil, map[string]any{"trigger": "scheduler"})
|
||||
}
|
||||
}
|
||||
+39
-6
@@ -18,6 +18,8 @@ import (
|
||||
"evobgp/internal/observability"
|
||||
"evobgp/internal/pipeline"
|
||||
"evobgp/internal/store"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// mergeBirdPostApplyMeta attaches a birdc snapshot after deploy/reload (best-effort).
|
||||
@@ -43,17 +45,28 @@ func mergeBirdPostApplyMeta(j *Job) {
|
||||
}
|
||||
|
||||
const (
|
||||
KindModuleRefresh = "module_refresh"
|
||||
KindTenantRefresh = "tenant_refresh"
|
||||
KindPeerReconcile = "peer_reconcile"
|
||||
KindDeployApply = "deploy_apply"
|
||||
KindRevisionRollback = "revision_rollback"
|
||||
KindBirdReload = "bird_reload"
|
||||
KindModuleRefresh = "module_refresh"
|
||||
KindTenantRefresh = "tenant_refresh"
|
||||
KindPeerReconcile = "peer_reconcile"
|
||||
KindDeployApply = "deploy_apply"
|
||||
KindRevisionRollback = "revision_rollback"
|
||||
KindBirdReload = "bird_reload"
|
||||
KindPostgresMetricsRefresh = "postgres_metrics_refresh"
|
||||
KindPostgresSlowQueryAgg = "postgres_slow_query_aggregate"
|
||||
KindPostgresTableBloat = "postgres_table_bloat_estimate"
|
||||
KindPostgresIndexUsage = "postgres_index_usage_analyze"
|
||||
KindPostgresAutovacuumLag = "postgres_autovacuum_lag_detect"
|
||||
KindPostgresVacuum = "postgres_vacuum"
|
||||
KindPostgresVacuumAnalyze = "postgres_vacuum_analyze"
|
||||
KindPostgresAnalyze = "postgres_analyze"
|
||||
KindPostgresReindex = "postgres_reindex"
|
||||
KindPostgresCleanup = "postgres_cleanup"
|
||||
)
|
||||
|
||||
// Worker executes queued jobs against store.Backend (memory or SQL).
|
||||
type Worker struct {
|
||||
Store store.Backend
|
||||
PgPool *pgxpool.Pool
|
||||
HTTPClient *http.Client // optional; CDN refresh uses this (default 45s timeout).
|
||||
// Registry is set after BootstrapWorkers creates the job queue; used to chain deploy_apply after refresh/rollback.
|
||||
Registry *Registry
|
||||
@@ -155,6 +168,26 @@ func (w *Worker) Process(j *Job) {
|
||||
}
|
||||
mergeBirdPostApplyMeta(j)
|
||||
j.Succeed()
|
||||
case KindPostgresMetricsRefresh:
|
||||
w.runPostgresMetricsRefresh(j)
|
||||
case KindPostgresSlowQueryAgg:
|
||||
w.runPostgresSlowQueryAgg(j)
|
||||
case KindPostgresTableBloat:
|
||||
w.runPostgresTableBloat(j)
|
||||
case KindPostgresIndexUsage:
|
||||
w.runPostgresIndexUsage(j)
|
||||
case KindPostgresAutovacuumLag:
|
||||
w.runPostgresAutovacuumLag(j)
|
||||
case KindPostgresVacuum:
|
||||
w.runPostgresMaint(j, "vacuum")
|
||||
case KindPostgresVacuumAnalyze:
|
||||
w.runPostgresMaint(j, "vacuum_analyze")
|
||||
case KindPostgresAnalyze:
|
||||
w.runPostgresMaint(j, "analyze")
|
||||
case KindPostgresReindex:
|
||||
w.runPostgresMaint(j, "reindex")
|
||||
case KindPostgresCleanup:
|
||||
w.runPostgresCleanup(j)
|
||||
default:
|
||||
j.Fail("unknown job kind")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package pgmonitor
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type cacheEntry struct {
|
||||
at time.Time
|
||||
data any
|
||||
}
|
||||
|
||||
type ttlCache struct {
|
||||
mu sync.RWMutex
|
||||
ttl time.Duration
|
||||
items map[string]cacheEntry
|
||||
}
|
||||
|
||||
func newTTLCache(ttl time.Duration) *ttlCache {
|
||||
return &ttlCache{ttl: ttl, items: make(map[string]cacheEntry)}
|
||||
}
|
||||
|
||||
func (c *ttlCache) get(key string) (any, bool) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
e, ok := c.items[key]
|
||||
if !ok || time.Since(e.at) > c.ttl {
|
||||
return nil, false
|
||||
}
|
||||
return e.data, true
|
||||
}
|
||||
|
||||
func (c *ttlCache) set(key string, data any) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.items[key] = cacheEntry{at: time.Now().UTC(), data: data}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package pgmonitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// Correlation builds aligned timeline points from job_audit and overview cache.
|
||||
func (s *Service) Correlation(ctx context.Context, windowMinutes int) (CorrelationResponse, error) {
|
||||
if s == nil || s.pool == nil {
|
||||
return CorrelationResponse{}, fmt.Errorf("pgmonitor: postgres not configured")
|
||||
}
|
||||
if windowMinutes <= 0 {
|
||||
windowMinutes = 60
|
||||
}
|
||||
if windowMinutes > 1440 {
|
||||
windowMinutes = 1440
|
||||
}
|
||||
since := time.Now().UTC().Add(-time.Duration(windowMinutes) * time.Minute)
|
||||
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT date_trunc('minute', finished_at) AS bucket,
|
||||
percentile_cont(0.99) WITHIN GROUP (ORDER BY
|
||||
EXTRACT(EPOCH FROM (finished_at - started_at)) * 1000)
|
||||
FROM job_audit
|
||||
WHERE finished_at >= $1 AND kind IN ('module_refresh', 'tenant_refresh')
|
||||
AND status = 'succeeded' AND started_at IS NOT NULL
|
||||
GROUP BY 1
|
||||
ORDER BY 1`, since)
|
||||
if err != nil {
|
||||
return CorrelationResponse{}, fmt.Errorf("pgmonitor: correlation jobs: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
points := make(map[time.Time]*CorrelationPoint)
|
||||
for rows.Next() {
|
||||
var bucket time.Time
|
||||
var p99 *float64
|
||||
if err := rows.Scan(&bucket, &p99); err != nil {
|
||||
return CorrelationResponse{}, err
|
||||
}
|
||||
bucket = bucket.UTC()
|
||||
pt := points[bucket]
|
||||
if pt == nil {
|
||||
pt = &CorrelationPoint{Timestamp: bucket}
|
||||
points[bucket] = pt
|
||||
}
|
||||
if p99 != nil {
|
||||
pt.PipelineRefreshP99Ms = *p99
|
||||
}
|
||||
}
|
||||
|
||||
ov, err := s.Overview(ctx)
|
||||
if err == nil && ov.Database.CacheHitPct > 0 {
|
||||
now := time.Now().UTC().Truncate(time.Minute)
|
||||
pt := points[now]
|
||||
if pt == nil {
|
||||
pt = &CorrelationPoint{Timestamp: now}
|
||||
points[now] = pt
|
||||
}
|
||||
pt.CacheHitPct = ov.Database.CacheHitPct
|
||||
}
|
||||
|
||||
out := make([]CorrelationPoint, 0, len(points))
|
||||
for _, p := range points {
|
||||
out = append(out, *p)
|
||||
}
|
||||
// simple sort by time
|
||||
for i := 0; i < len(out); i++ {
|
||||
for j := i + 1; j < len(out); j++ {
|
||||
if out[j].Timestamp.Before(out[i].Timestamp) {
|
||||
out[i], out[j] = out[j], out[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
return CorrelationResponse{WindowMinutes: windowMinutes, Points: out}, nil
|
||||
}
|
||||
|
||||
// RecordCorrelationSnapshot is a hook for future Prometheus samples (no-op placeholder).
|
||||
func RecordCorrelationSnapshot(_ *pgxpool.Pool) {}
|
||||
@@ -0,0 +1,154 @@
|
||||
package pgmonitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// CleanupPolicy names safe retention policies.
|
||||
type CleanupPolicy string
|
||||
|
||||
const (
|
||||
PolicyJobAuditRetention CleanupPolicy = "job_audit_retention"
|
||||
PolicyASNCacheRetention CleanupPolicy = "asn_cache_retention"
|
||||
)
|
||||
|
||||
// CleanupRequest for POST /postgres/cleanup.
|
||||
type CleanupRequest struct {
|
||||
Policy string `json:"policy"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
Limit int `json:"limit"`
|
||||
}
|
||||
|
||||
// RunCleanup executes a named retention policy.
|
||||
func RunCleanup(ctx context.Context, pool *pgxpool.Pool, policy string, dryRun bool, limit int) (map[string]any, error) {
|
||||
if pool == nil {
|
||||
return nil, fmt.Errorf("pgmonitor: postgres not configured")
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 10000
|
||||
}
|
||||
if limit > 100000 {
|
||||
limit = 100000
|
||||
}
|
||||
detail := map[string]any{"policy": policy, "dry_run": dryRun, "limit": limit}
|
||||
switch CleanupPolicy(policy) {
|
||||
case PolicyJobAuditRetention:
|
||||
cutoff := time.Now().UTC().Add(-90 * 24 * time.Hour)
|
||||
if dryRun {
|
||||
var n int64
|
||||
err := pool.QueryRow(ctx, `
|
||||
SELECT count(*) FROM job_audit
|
||||
WHERE created_at < $1 AND status IN ('succeeded', 'failed', 'cancelled')`, cutoff).Scan(&n)
|
||||
detail["would_delete"] = n
|
||||
return detail, err
|
||||
}
|
||||
tag, err := pool.Exec(ctx, `
|
||||
DELETE FROM job_audit
|
||||
WHERE id IN (
|
||||
SELECT id FROM job_audit
|
||||
WHERE created_at < $1 AND status IN ('succeeded', 'failed', 'cancelled')
|
||||
LIMIT $2
|
||||
)`, cutoff, limit)
|
||||
if err != nil {
|
||||
return detail, err
|
||||
}
|
||||
detail["deleted"] = tag.RowsAffected()
|
||||
return detail, nil
|
||||
case PolicyASNCacheRetention:
|
||||
cutoff := time.Now().UTC().Add(-7 * 24 * time.Hour)
|
||||
if dryRun {
|
||||
var n int64
|
||||
err := pool.QueryRow(ctx, `SELECT count(*) FROM asn_prefix_cache WHERE fetched_at < $1`, cutoff).Scan(&n)
|
||||
detail["would_delete"] = n
|
||||
return detail, err
|
||||
}
|
||||
tag, err := pool.Exec(ctx, `
|
||||
DELETE FROM asn_prefix_cache WHERE fetched_at < $1`, cutoff)
|
||||
if err != nil {
|
||||
return detail, err
|
||||
}
|
||||
detail["deleted"] = tag.RowsAffected()
|
||||
return detail, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("pgmonitor: unknown cleanup policy %q", policy)
|
||||
}
|
||||
}
|
||||
|
||||
// InsertMaintenanceAudit records an audit row at job start.
|
||||
func InsertMaintenanceAudit(ctx context.Context, pool *pgxpool.Pool, tenantID, actorPrefix, kind, table string, dryRun bool) (string, error) {
|
||||
id := uuid.New().String()
|
||||
_, err := pool.Exec(ctx, `
|
||||
INSERT INTO postgres_maintenance_audit
|
||||
(id, tenant_id, actor_prefix, kind, target_table, dry_run, status, created_at)
|
||||
VALUES ($1, NULLIF($2,''), NULLIF($3,''), $4, NULLIF($5,''), $6, 'running', now())`,
|
||||
id, tenantID, actorPrefix, kind, table, dryRun)
|
||||
return id, err
|
||||
}
|
||||
|
||||
// FinishMaintenanceAudit updates terminal state.
|
||||
func FinishMaintenanceAudit(ctx context.Context, pool *pgxpool.Pool, id, status string, detail map[string]any, errMsg *string) error {
|
||||
var detailJSON []byte
|
||||
if detail != nil {
|
||||
detailJSON, _ = json.Marshal(detail)
|
||||
}
|
||||
_, err := pool.Exec(ctx, `
|
||||
UPDATE postgres_maintenance_audit
|
||||
SET status = $2, detail_json = $3::jsonb, error_message = $4,
|
||||
finished_at = now(), started_at = COALESCE(started_at, now())
|
||||
WHERE id = $1`,
|
||||
id, status, string(detailJSON), errMsg)
|
||||
return err
|
||||
}
|
||||
|
||||
// ListMaintenanceLogs returns paginated audit rows.
|
||||
func ListMaintenanceLogs(ctx context.Context, pool *pgxpool.Pool, cursor string, limit int) ([]MaintenanceLogRow, string, bool, error) {
|
||||
limit = clampLimit(limit, 20, 100)
|
||||
args := []any{limit + 1}
|
||||
q := `
|
||||
SELECT id, COALESCE(tenant_id,''), COALESCE(actor_prefix,''), kind,
|
||||
COALESCE(target_table,''), dry_run, status,
|
||||
detail_json, COALESCE(error_message,''), created_at, started_at, finished_at
|
||||
FROM postgres_maintenance_audit`
|
||||
if cursor != "" {
|
||||
q += ` WHERE created_at < (SELECT created_at FROM postgres_maintenance_audit WHERE id = $2)`
|
||||
args = append(args, cursor)
|
||||
}
|
||||
q += ` ORDER BY created_at DESC LIMIT $1`
|
||||
|
||||
rows, err := pool.Query(ctx, q, args...)
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []MaintenanceLogRow
|
||||
for rows.Next() {
|
||||
var r MaintenanceLogRow
|
||||
var detailRaw []byte
|
||||
var started, finished *time.Time
|
||||
if err := rows.Scan(&r.ID, &r.TenantID, &r.ActorPrefix, &r.Kind, &r.TargetTable,
|
||||
&r.DryRun, &r.Status, &detailRaw, &r.Error, &r.CreatedAt, &started, &finished); err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
r.StartedAt = started
|
||||
r.FinishedAt = finished
|
||||
if len(detailRaw) > 0 {
|
||||
_ = json.Unmarshal(detailRaw, &r.Detail)
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
hasMore := len(out) > limit
|
||||
if hasMore {
|
||||
out = out[:limit]
|
||||
}
|
||||
next := ""
|
||||
if hasMore && len(out) > 0 {
|
||||
next = out[len(out)-1].ID
|
||||
}
|
||||
return out, next, hasMore, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
package pgmonitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func clampLimit(limit, def, max int) int {
|
||||
if limit <= 0 {
|
||||
return def
|
||||
}
|
||||
if limit > max {
|
||||
return max
|
||||
}
|
||||
return limit
|
||||
}
|
||||
|
||||
func (s *Service) fetchOverview(ctx context.Context) (Overview, error) {
|
||||
now := time.Now().UTC()
|
||||
out := Overview{CollectedAt: now}
|
||||
|
||||
var active, idle, total, maxConn int
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT
|
||||
count(*) FILTER (WHERE state = 'active'),
|
||||
count(*) FILTER (WHERE state = 'idle'),
|
||||
count(*),
|
||||
(SELECT setting::int FROM pg_settings WHERE name = 'max_connections')
|
||||
FROM pg_stat_activity
|
||||
WHERE datname = current_database()`).Scan(&active, &idle, &total, &maxConn)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("pgmonitor: connections: %w", err)
|
||||
}
|
||||
out.Connections = Connections{Active: active, Idle: idle, Total: total, MaxConnections: maxConn}
|
||||
|
||||
var cachePct *float64
|
||||
err = s.pool.QueryRow(ctx, `
|
||||
SELECT numbackends, xact_commit, xact_rollback, deadlocks, blks_hit, blks_read,
|
||||
CASE WHEN blks_hit + blks_read > 0
|
||||
THEN round(100.0 * blks_hit::numeric / (blks_hit + blks_read), 2) END
|
||||
FROM pg_stat_database WHERE datname = current_database()`).Scan(
|
||||
&out.Database.Backends,
|
||||
&out.Database.XactCommit,
|
||||
&out.Database.XactRollback,
|
||||
&out.Database.Deadlocks,
|
||||
&out.Database.BlksHit,
|
||||
&out.Database.BlksRead,
|
||||
&cachePct,
|
||||
)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("pgmonitor: database stats: %w", err)
|
||||
}
|
||||
if cachePct != nil {
|
||||
out.Database.CacheHitPct = *cachePct
|
||||
}
|
||||
|
||||
_ = s.pool.QueryRow(ctx, `
|
||||
SELECT checkpoints_timed, checkpoints_req, buffers_checkpoint, buffers_clean,
|
||||
maxwritten_clean, buffers_backend, buffers_alloc
|
||||
FROM pg_stat_bgwriter`).Scan(
|
||||
&out.Bgwriter.CheckpointsTimed,
|
||||
&out.Bgwriter.CheckpointsReq,
|
||||
&out.Bgwriter.BuffersCheckpoint,
|
||||
&out.Bgwriter.BuffersClean,
|
||||
&out.Bgwriter.MaxWrittenClean,
|
||||
&out.Bgwriter.BuffersBackend,
|
||||
&out.Bgwriter.BuffersAlloc,
|
||||
)
|
||||
|
||||
_ = s.pool.QueryRow(ctx, `SELECT pg_database_size(current_database())`).Scan(&out.SizeBytes)
|
||||
|
||||
_ = s.pool.QueryRow(ctx, `
|
||||
SELECT
|
||||
(SELECT setting FROM pg_settings WHERE name = 'shared_buffers'),
|
||||
(SELECT setting FROM pg_settings WHERE name = 'work_mem'),
|
||||
(SELECT setting FROM pg_settings WHERE name = 'effective_cache_size')`).Scan(
|
||||
&out.MemorySettings.SharedBuffers,
|
||||
&out.MemorySettings.WorkMem,
|
||||
&out.MemorySettings.EffectiveCacheSize,
|
||||
)
|
||||
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT client_addr::text, state, sync_state,
|
||||
EXTRACT(EPOCH FROM COALESCE(write_lag, flush_lag, replay_lag)) * 1000
|
||||
FROM pg_stat_replication`)
|
||||
if err == nil {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var peer ReplicationPeer
|
||||
var lagMs *float64
|
||||
if err := rows.Scan(&peer.ClientAddr, &peer.State, &peer.SyncState, &lagMs); err != nil {
|
||||
continue
|
||||
}
|
||||
if lagMs != nil {
|
||||
v := int64(*lagMs)
|
||||
peer.LagMs = &v
|
||||
}
|
||||
out.Replication = append(out.Replication, peer)
|
||||
}
|
||||
}
|
||||
|
||||
var ext bool
|
||||
_ = s.pool.QueryRow(ctx, `
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements')`).Scan(&ext)
|
||||
out.StatementsEnabled = ext
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func queryLocks(ctx context.Context, pool *pgxpool.Pool) ([]LockRow, error) {
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT l.locktype, l.mode, l.granted, a.pid, COALESCE(a.usename, ''),
|
||||
COALESCE(a.state, ''), COALESCE(left(a.query, 300), ''),
|
||||
NOT l.granted AS blocked
|
||||
FROM pg_locks l
|
||||
JOIN pg_stat_activity a ON a.pid = l.pid
|
||||
WHERE a.datname = current_database()
|
||||
AND (NOT l.granted OR l.mode LIKE '%Exclusive%')
|
||||
ORDER BY l.granted ASC, a.query_start NULLS LAST
|
||||
LIMIT 200`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pgmonitor: locks: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []LockRow
|
||||
for rows.Next() {
|
||||
var r LockRow
|
||||
if err := rows.Scan(&r.Locktype, &r.Mode, &r.Granted, &r.PID, &r.User, &r.State, &r.Query, &r.Blocked); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func queryTables(ctx context.Context, pool *pgxpool.Pool, limit int) ([]TableStat, error) {
|
||||
limit = clampLimit(limit, 20, 100)
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT t.relname,
|
||||
pg_total_relation_size(t.relid),
|
||||
s.heap_blks_read, s.heap_blks_hit,
|
||||
t.idx_scan, t.seq_scan, t.n_dead_tup, t.last_autovacuum,
|
||||
CASE WHEN t.n_live_tup + t.n_dead_tup > 0
|
||||
THEN round(t.n_dead_tup::numeric / (t.n_live_tup + t.n_dead_tup), 4)
|
||||
ELSE 0 END
|
||||
FROM pg_statio_user_tables s
|
||||
JOIN pg_stat_user_tables t ON t.relid = s.relid
|
||||
WHERE t.schemaname = 'public'
|
||||
ORDER BY pg_total_relation_size(t.relid) DESC
|
||||
LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pgmonitor: tables: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []TableStat
|
||||
for rows.Next() {
|
||||
var r TableStat
|
||||
var last *time.Time
|
||||
if err := rows.Scan(&r.Relname, &r.TotalBytes, &r.HeapBlksRead, &r.HeapBlksHit,
|
||||
&r.IdxScan, &r.SeqScan, &r.DeadTuples, &last, &r.BloatRatio); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.LastAutovacuum = last
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// TopQueries loads from pg_stat_statements when available.
|
||||
func (s *Service) TopQueries(ctx context.Context, limit int) (QueriesResponse, error) {
|
||||
if s == nil || s.pool == nil {
|
||||
return QueriesResponse{}, errors.New("pgmonitor: postgres not configured")
|
||||
}
|
||||
limit = clampLimit(limit, 20, 100)
|
||||
now := time.Now().UTC()
|
||||
|
||||
if snap, ok, err := s.loadSnapshot(ctx, "slow_queries", 15*time.Minute); err == nil && ok {
|
||||
var items []QueryStat
|
||||
if err := decodePayload(snap.Payload, &items); err == nil {
|
||||
return QueriesResponse{CollectedAt: snap.CollectedAt, Source: "snapshot", Items: items}, nil
|
||||
}
|
||||
}
|
||||
|
||||
if !s.statementsEnabled(ctx) {
|
||||
return QueriesResponse{CollectedAt: now, Source: "live", Items: nil}, nil
|
||||
}
|
||||
items, err := queryTopStatements(ctx, s.pool, limit)
|
||||
if err != nil {
|
||||
return QueriesResponse{}, err
|
||||
}
|
||||
return QueriesResponse{CollectedAt: now, Source: "live", Items: items}, nil
|
||||
}
|
||||
|
||||
func (s *Service) statementsEnabled(ctx context.Context) bool {
|
||||
var ok bool
|
||||
_ = s.pool.QueryRow(ctx, `
|
||||
SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements')`).Scan(&ok)
|
||||
return ok
|
||||
}
|
||||
|
||||
func queryTopStatements(ctx context.Context, pool *pgxpool.Pool, limit int) ([]QueryStat, error) {
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT queryid, left(query, 500), calls, total_exec_time, mean_exec_time, rows
|
||||
FROM pg_stat_statements
|
||||
WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database())
|
||||
ORDER BY mean_exec_time DESC
|
||||
LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
if isUndefinedTable(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("pgmonitor: pg_stat_statements: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []QueryStat
|
||||
for rows.Next() {
|
||||
var r QueryStat
|
||||
if err := rows.Scan(&r.QueryID, &r.Query, &r.Calls, &r.TotalExecMs, &r.MeanExecMs, &r.Rows); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func isUndefinedTable(err error) bool {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) {
|
||||
return pgErr.Code == "42P01" || pgErr.Code == "42704"
|
||||
}
|
||||
return strings.Contains(err.Error(), "pg_stat_statements")
|
||||
}
|
||||
|
||||
func isSafeIdent(name string) bool {
|
||||
if name == "" {
|
||||
return true
|
||||
}
|
||||
for _, r := range name {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ExecMaintenance runs VACUUM/ANALYZE/REINDEX with optional dry-run (returns SQL executed or planned).
|
||||
func ExecMaintenance(ctx context.Context, pool *pgxpool.Pool, kind, table string, dryRun bool) (detail map[string]any, err error) {
|
||||
if pool == nil {
|
||||
return nil, errors.New("pgmonitor: postgres not configured")
|
||||
}
|
||||
table = strings.TrimSpace(table)
|
||||
if table != "" && !isSafeIdent(table) {
|
||||
return nil, errors.New("pgmonitor: invalid table name")
|
||||
}
|
||||
qual := ""
|
||||
if table != "" {
|
||||
qual = " " + pgx.Identifier{table}.Sanitize()
|
||||
}
|
||||
var sql string
|
||||
switch kind {
|
||||
case "vacuum":
|
||||
sql = "VACUUM" + qual
|
||||
case "vacuum_analyze":
|
||||
sql = "VACUUM ANALYZE" + qual
|
||||
case "analyze":
|
||||
sql = "ANALYZE" + qual
|
||||
case "reindex":
|
||||
if table == "" {
|
||||
return nil, errors.New("pgmonitor: reindex requires table")
|
||||
}
|
||||
sql = "REINDEX TABLE" + qual
|
||||
default:
|
||||
return nil, fmt.Errorf("pgmonitor: unknown maintenance kind %q", kind)
|
||||
}
|
||||
detail = map[string]any{"sql": sql, "dry_run": dryRun}
|
||||
if dryRun {
|
||||
return detail, nil
|
||||
}
|
||||
_, err = pool.Exec(ctx, sql)
|
||||
if err != nil {
|
||||
return detail, fmt.Errorf("pgmonitor: %s: %w", kind, err)
|
||||
}
|
||||
detail["executed"] = true
|
||||
return detail, nil
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package pgmonitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Recommendations builds heuristic items from live stats and snapshots.
|
||||
func (s *Service) Recommendations(ctx context.Context) (RecommendationsResponse, error) {
|
||||
now := time.Now().UTC()
|
||||
var items []RecommendationItem
|
||||
|
||||
ov, err := s.Overview(ctx)
|
||||
if err == nil {
|
||||
if ov.Database.CacheHitPct > 0 && ov.Database.CacheHitPct < 90 {
|
||||
items = append(items, RecommendationItem{
|
||||
Severity: "warn",
|
||||
Code: "low_cache_hit",
|
||||
Title: "Низкий cache hit ratio",
|
||||
Detail: "Buffer cache hit ниже 90%; проверьте shared_buffers и горячие seq scan.",
|
||||
})
|
||||
}
|
||||
if ov.Database.Deadlocks > 0 {
|
||||
items = append(items, RecommendationItem{
|
||||
Severity: "warn",
|
||||
Code: "deadlocks",
|
||||
Title: "Зафиксированы deadlocks",
|
||||
Detail: "Проверьте конкурирующие транзакции и порядок блокировок.",
|
||||
})
|
||||
}
|
||||
if ov.Connections.MaxConnections > 0 &&
|
||||
float64(ov.Connections.Total)/float64(ov.Connections.MaxConnections) > 0.8 {
|
||||
items = append(items, RecommendationItem{
|
||||
Severity: "critical",
|
||||
Code: "connections_high",
|
||||
Title: "Много подключений к PostgreSQL",
|
||||
Detail: "Использование max_connections выше 80%; увеличьте pool tuning или лимит.",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
tables, err := s.Tables(ctx, 30)
|
||||
if err == nil {
|
||||
for _, t := range tables {
|
||||
if t.SeqScan > 1000 && t.IdxScan < t.SeqScan/10 {
|
||||
items = append(items, RecommendationItem{
|
||||
Severity: "warn",
|
||||
Code: "missing_index",
|
||||
Title: "Высокий seq_scan",
|
||||
Detail: "Таблица часто сканируется последовательно; рассмотрите индекс.",
|
||||
Refs: []string{t.Relname},
|
||||
})
|
||||
}
|
||||
if t.BloatRatio > 0.2 && t.DeadTuples > 5000 {
|
||||
items = append(items, RecommendationItem{
|
||||
Severity: "info",
|
||||
Code: "autovacuum_lag",
|
||||
Title: "Возможный bloat / мёртвые строки",
|
||||
Detail: "Высокая доля n_dead_tup; запланируйте VACUUM.",
|
||||
Refs: []string{t.Relname},
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if snap, ok, _ := s.loadSnapshot(ctx, "unused_indexes", 30*time.Minute); ok {
|
||||
type unused struct {
|
||||
Index string `json:"index"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
}
|
||||
var list []unused
|
||||
if decodePayload(snap.Payload, &list) == nil {
|
||||
for _, u := range list {
|
||||
if u.SizeBytes < 1024*1024 {
|
||||
continue
|
||||
}
|
||||
items = append(items, RecommendationItem{
|
||||
Severity: "info",
|
||||
Code: "unused_index",
|
||||
Title: "Неиспользуемый индекс",
|
||||
Detail: "idx_scan=0; проверьте перед удалением.",
|
||||
Refs: []string{u.Index},
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
q, err := s.TopQueries(ctx, 5)
|
||||
if err == nil {
|
||||
for _, qs := range q.Items {
|
||||
if qs.MeanExecMs > 500 {
|
||||
items = append(items, RecommendationItem{
|
||||
Severity: "warn",
|
||||
Code: "slow_query",
|
||||
Title: "Медленный запрос",
|
||||
Detail: "Среднее время выполнения выше 500ms.",
|
||||
Refs: []string{qs.Query},
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return RecommendationsResponse{CollectedAt: now, Items: items}, nil
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package pgmonitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// StartScheduler runs periodic PostgreSQL analyzer snapshots until ctx is cancelled.
|
||||
func StartScheduler(ctx context.Context, pool *pgxpool.Pool) {
|
||||
if pool == nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
t5 := time.NewTicker(5 * time.Minute)
|
||||
t15 := time.NewTicker(15 * time.Minute)
|
||||
defer t5.Stop()
|
||||
defer t15.Stop()
|
||||
s := NewService(pool)
|
||||
runLight := func() {
|
||||
c, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
if err := s.RefreshMetricsSnapshot(c); err != nil {
|
||||
log.Printf("pgmonitor: metrics refresh: %v", err)
|
||||
}
|
||||
if err := s.DetectAutovacuumLag(c); err != nil {
|
||||
log.Printf("pgmonitor: autovacuum lag: %v", err)
|
||||
}
|
||||
}
|
||||
runHeavy := func() {
|
||||
c, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||
defer cancel()
|
||||
if err := s.AggregateSlowQueries(c, 30); err != nil {
|
||||
log.Printf("pgmonitor: slow queries: %v", err)
|
||||
}
|
||||
if err := s.EstimateTableBloat(c); err != nil {
|
||||
log.Printf("pgmonitor: bloat: %v", err)
|
||||
}
|
||||
if err := s.AnalyzeIndexUsage(c); err != nil {
|
||||
log.Printf("pgmonitor: index usage: %v", err)
|
||||
}
|
||||
}
|
||||
runLight()
|
||||
runHeavy()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t5.C:
|
||||
runLight()
|
||||
case <-t15.C:
|
||||
runHeavy()
|
||||
}
|
||||
}
|
||||
}()
|
||||
log.Printf("pgmonitor: scheduler started (5m light / 15m heavy)")
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package pgmonitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// Service provides PostgreSQL observability and maintenance helpers (control plane instance scope).
|
||||
type Service struct {
|
||||
pool *pgxpool.Pool
|
||||
cache *ttlCache
|
||||
}
|
||||
|
||||
// NewService constructs a metrics service for the API PostgreSQL pool.
|
||||
func NewService(pool *pgxpool.Pool) *Service {
|
||||
if pool == nil {
|
||||
return nil
|
||||
}
|
||||
return &Service{
|
||||
pool: pool,
|
||||
cache: newTTLCache(10 * time.Second),
|
||||
}
|
||||
}
|
||||
|
||||
// Pool exposes the underlying pool for job workers.
|
||||
func (s *Service) Pool() *pgxpool.Pool {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
return s.pool
|
||||
}
|
||||
|
||||
// Overview returns cached instance-level stats.
|
||||
func (s *Service) Overview(ctx context.Context) (Overview, error) {
|
||||
if s == nil || s.pool == nil {
|
||||
return Overview{}, errors.New("pgmonitor: postgres not configured")
|
||||
}
|
||||
if v, ok := s.cache.get("overview"); ok {
|
||||
if o, ok := v.(Overview); ok {
|
||||
return o, nil
|
||||
}
|
||||
}
|
||||
o, err := s.fetchOverview(ctx)
|
||||
if err != nil {
|
||||
return Overview{}, err
|
||||
}
|
||||
s.cache.set("overview", o)
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// Locks returns active / blocking locks.
|
||||
func (s *Service) Locks(ctx context.Context) ([]LockRow, error) {
|
||||
if s == nil || s.pool == nil {
|
||||
return nil, errors.New("pgmonitor: postgres not configured")
|
||||
}
|
||||
if v, ok := s.cache.get("locks"); ok {
|
||||
if rows, ok := v.([]LockRow); ok {
|
||||
return rows, nil
|
||||
}
|
||||
}
|
||||
rows, err := queryLocks(ctx, s.pool)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.cache.set("locks", rows)
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// Tables returns top tables by size with I/O stats.
|
||||
func (s *Service) Tables(ctx context.Context, limit int) ([]TableStat, error) {
|
||||
if s == nil || s.pool == nil {
|
||||
return nil, errors.New("pgmonitor: postgres not configured")
|
||||
}
|
||||
key := fmt.Sprintf("tables:%d", limit)
|
||||
if v, ok := s.cache.get(key); ok {
|
||||
if rows, ok := v.([]TableStat); ok {
|
||||
return rows, nil
|
||||
}
|
||||
}
|
||||
rows, err := queryTables(ctx, s.pool, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.cache.set(key, rows)
|
||||
return rows, nil
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package pgmonitor
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestClampLimit(t *testing.T) {
|
||||
if clampLimit(0, 20, 100) != 20 {
|
||||
t.Fatal("default")
|
||||
}
|
||||
if clampLimit(200, 20, 100) != 100 {
|
||||
t.Fatal("max")
|
||||
}
|
||||
if clampLimit(5, 20, 100) != 5 {
|
||||
t.Fatal("value")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsSafeIdent(t *testing.T) {
|
||||
if !isSafeIdent("revision_materialized_prefix") {
|
||||
t.Fatal("valid")
|
||||
}
|
||||
if isSafeIdent("bad-name") {
|
||||
t.Fatal("invalid")
|
||||
}
|
||||
if !isSafeIdent("") {
|
||||
t.Fatal("empty ok")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewServiceNilPool(t *testing.T) {
|
||||
if NewService(nil) != nil {
|
||||
t.Fatal("expected nil service")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package pgmonitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type snapshotRow struct {
|
||||
ID string
|
||||
CollectedAt time.Time
|
||||
Payload json.RawMessage
|
||||
}
|
||||
|
||||
func (s *Service) loadSnapshot(ctx context.Context, id string, maxAge time.Duration) (snapshotRow, bool, error) {
|
||||
var row snapshotRow
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT id, collected_at, payload_json
|
||||
FROM postgres_monitor_snapshot
|
||||
WHERE id = $1 AND collected_at >= $2`,
|
||||
id, time.Now().UTC().Add(-maxAge)).Scan(&row.ID, &row.CollectedAt, &row.Payload)
|
||||
if err != nil {
|
||||
return snapshotRow{}, false, nil
|
||||
}
|
||||
return row, true, nil
|
||||
}
|
||||
|
||||
func (s *Service) UpsertSnapshot(ctx context.Context, id string, payload any) error {
|
||||
if s == nil || s.pool == nil {
|
||||
return fmt.Errorf("pgmonitor: postgres not configured")
|
||||
}
|
||||
b, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.pool.Exec(ctx, `
|
||||
INSERT INTO postgres_monitor_snapshot (id, collected_at, payload_json)
|
||||
VALUES ($1, now(), $2::jsonb)
|
||||
ON CONFLICT (id) DO UPDATE SET collected_at = EXCLUDED.collected_at, payload_json = EXCLUDED.payload_json`,
|
||||
id, string(b))
|
||||
return err
|
||||
}
|
||||
|
||||
func decodePayload(raw json.RawMessage, dest any) error {
|
||||
return json.Unmarshal(raw, dest)
|
||||
}
|
||||
|
||||
// RefreshMetricsSnapshot stores overview and tables for heavy reads.
|
||||
func (s *Service) RefreshMetricsSnapshot(ctx context.Context) error {
|
||||
ov, err := s.fetchOverview(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.UpsertSnapshot(ctx, "overview", ov); err != nil {
|
||||
return err
|
||||
}
|
||||
tables, err := queryTables(ctx, s.pool, 50)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.UpsertSnapshot(ctx, "tables", tables)
|
||||
}
|
||||
|
||||
// AggregateSlowQueries stores top statements snapshot.
|
||||
func (s *Service) AggregateSlowQueries(ctx context.Context, limit int) error {
|
||||
if !s.statementsEnabled(ctx) {
|
||||
return s.UpsertSnapshot(ctx, "slow_queries", []QueryStat{})
|
||||
}
|
||||
items, err := queryTopStatements(ctx, s.pool, clampLimit(limit, 20, 100))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.UpsertSnapshot(ctx, "slow_queries", items)
|
||||
}
|
||||
|
||||
// EstimateTableBloat refreshes bloat heuristics on tables snapshot.
|
||||
func (s *Service) EstimateTableBloat(ctx context.Context) error {
|
||||
tables, err := queryTables(ctx, s.pool, 100)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.UpsertSnapshot(ctx, "table_bloat", tables)
|
||||
}
|
||||
|
||||
// AnalyzeIndexUsage stores unused indexes.
|
||||
func (s *Service) AnalyzeIndexUsage(ctx context.Context) error {
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT indexrelname, idx_scan, pg_relation_size(indexrelid)
|
||||
FROM pg_stat_user_indexes
|
||||
WHERE schemaname = 'public' AND idx_scan = 0
|
||||
ORDER BY pg_relation_size(indexrelid) DESC
|
||||
LIMIT 50`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pgmonitor: index usage: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
type unused struct {
|
||||
Index string `json:"index"`
|
||||
IdxScan int64 `json:"idx_scan"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
}
|
||||
var items []unused
|
||||
for rows.Next() {
|
||||
var u unused
|
||||
if err := rows.Scan(&u.Index, &u.IdxScan, &u.SizeBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
items = append(items, u)
|
||||
}
|
||||
return s.UpsertSnapshot(ctx, "unused_indexes", items)
|
||||
}
|
||||
|
||||
// DetectAutovacuumLag stores tables with high dead tuple ratio.
|
||||
func (s *Service) DetectAutovacuumLag(ctx context.Context) error {
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT relname, n_dead_tup, last_autovacuum,
|
||||
CASE WHEN n_live_tup + n_dead_tup > 0
|
||||
THEN round(n_dead_tup::numeric / (n_live_tup + n_dead_tup), 4) ELSE 0 END
|
||||
FROM pg_stat_user_tables
|
||||
WHERE schemaname = 'public' AND n_dead_tup > 1000
|
||||
ORDER BY n_dead_tup DESC
|
||||
LIMIT 30`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pgmonitor: autovacuum lag: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
type lagRow struct {
|
||||
Relname string `json:"relname"`
|
||||
DeadTuples int64 `json:"n_dead_tup"`
|
||||
LastAutovacuum *time.Time `json:"last_autovacuum,omitempty"`
|
||||
Ratio float64 `json:"ratio"`
|
||||
}
|
||||
var items []lagRow
|
||||
for rows.Next() {
|
||||
var r lagRow
|
||||
if err := rows.Scan(&r.Relname, &r.DeadTuples, &r.LastAutovacuum, &r.Ratio); err != nil {
|
||||
return err
|
||||
}
|
||||
items = append(items, r)
|
||||
}
|
||||
return s.UpsertSnapshot(ctx, "autovacuum_lag", items)
|
||||
}
|
||||
|
||||
// RunPeriodicAnalyzerJobs runs all snapshot analyzers (for scheduler).
|
||||
func RunPeriodicAnalyzerJobs(ctx context.Context, pool *pgxpool.Pool) {
|
||||
s := NewService(pool)
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
_ = s.RefreshMetricsSnapshot(ctx)
|
||||
_ = s.AggregateSlowQueries(ctx, 30)
|
||||
_ = s.EstimateTableBloat(ctx)
|
||||
_ = s.AnalyzeIndexUsage(ctx)
|
||||
_ = s.DetectAutovacuumLag(ctx)
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package pgmonitor
|
||||
|
||||
import "time"
|
||||
|
||||
// Overview is instance-level PostgreSQL health snapshot.
|
||||
type Overview struct {
|
||||
CollectedAt time.Time `json:"collected_at"`
|
||||
Connections Connections `json:"connections"`
|
||||
Database DatabaseStats `json:"database"`
|
||||
Bgwriter BgwriterStats `json:"bgwriter"`
|
||||
SizeBytes int64 `json:"database_size_bytes"`
|
||||
MemorySettings MemorySettings `json:"memory_settings"`
|
||||
Replication []ReplicationPeer `json:"replication"`
|
||||
StatementsEnabled bool `json:"pg_stat_statements_enabled"`
|
||||
}
|
||||
|
||||
// Connections summarizes pg_stat_activity for current database.
|
||||
type Connections struct {
|
||||
Active int `json:"active"`
|
||||
Idle int `json:"idle"`
|
||||
Total int `json:"total"`
|
||||
MaxConnections int `json:"max_connections"`
|
||||
}
|
||||
|
||||
// DatabaseStats from pg_stat_database.
|
||||
type DatabaseStats struct {
|
||||
Backends int `json:"backends"`
|
||||
XactCommit int64 `json:"xact_commit"`
|
||||
XactRollback int64 `json:"xact_rollback"`
|
||||
Deadlocks int64 `json:"deadlocks"`
|
||||
BlksHit int64 `json:"blks_hit"`
|
||||
BlksRead int64 `json:"blks_read"`
|
||||
CacheHitPct float64 `json:"cache_hit_pct"`
|
||||
}
|
||||
|
||||
// BgwriterStats from pg_stat_bgwriter.
|
||||
type BgwriterStats struct {
|
||||
CheckpointsTimed int64 `json:"checkpoints_timed"`
|
||||
CheckpointsReq int64 `json:"checkpoints_req"`
|
||||
BuffersCheckpoint int64 `json:"buffers_checkpoint"`
|
||||
BuffersClean int64 `json:"buffers_clean"`
|
||||
MaxWrittenClean int64 `json:"maxwritten_clean"`
|
||||
BuffersBackend int64 `json:"buffers_backend"`
|
||||
BuffersAlloc int64 `json:"buffers_alloc"`
|
||||
}
|
||||
|
||||
// MemorySettings is best-effort from pg_settings (not RSS).
|
||||
type MemorySettings struct {
|
||||
SharedBuffers string `json:"shared_buffers"`
|
||||
WorkMem string `json:"work_mem"`
|
||||
EffectiveCacheSize string `json:"effective_cache_size"`
|
||||
}
|
||||
|
||||
// ReplicationPeer from pg_stat_replication.
|
||||
type ReplicationPeer struct {
|
||||
ClientAddr string `json:"client_addr,omitempty"`
|
||||
State string `json:"state"`
|
||||
SyncState string `json:"sync_state,omitempty"`
|
||||
LagMs *int64 `json:"lag_ms,omitempty"`
|
||||
}
|
||||
|
||||
// QueryStat is a row from pg_stat_statements or snapshot.
|
||||
type QueryStat struct {
|
||||
QueryID int64 `json:"queryid,omitempty"`
|
||||
Query string `json:"query"`
|
||||
Calls int64 `json:"calls"`
|
||||
TotalExecMs float64 `json:"total_exec_ms"`
|
||||
MeanExecMs float64 `json:"mean_exec_ms"`
|
||||
Rows int64 `json:"rows"`
|
||||
}
|
||||
|
||||
// QueriesResponse for GET /monitoring/postgres/queries.
|
||||
type QueriesResponse struct {
|
||||
CollectedAt time.Time `json:"collected_at"`
|
||||
Source string `json:"source"` // live | snapshot
|
||||
Items []QueryStat `json:"items"`
|
||||
}
|
||||
|
||||
// LockRow describes a lock / blocking session.
|
||||
type LockRow struct {
|
||||
Locktype string `json:"locktype"`
|
||||
Mode string `json:"mode"`
|
||||
Granted bool `json:"granted"`
|
||||
PID int32 `json:"pid"`
|
||||
User string `json:"usename,omitempty"`
|
||||
State string `json:"state,omitempty"`
|
||||
Query string `json:"query,omitempty"`
|
||||
Blocked bool `json:"blocked"`
|
||||
}
|
||||
|
||||
// TableStat combines size and scan stats for a user table.
|
||||
type TableStat struct {
|
||||
Relname string `json:"relname"`
|
||||
TotalBytes int64 `json:"total_bytes"`
|
||||
HeapBlksRead int64 `json:"heap_blks_read"`
|
||||
HeapBlksHit int64 `json:"heap_blks_hit"`
|
||||
IdxScan int64 `json:"idx_scan"`
|
||||
SeqScan int64 `json:"seq_scan"`
|
||||
DeadTuples int64 `json:"n_dead_tup"`
|
||||
LastAutovacuum *time.Time `json:"last_autovacuum,omitempty"`
|
||||
BloatRatio float64 `json:"bloat_ratio,omitempty"`
|
||||
}
|
||||
|
||||
// RecommendationItem is a heuristic ops hint.
|
||||
type RecommendationItem struct {
|
||||
Severity string `json:"severity"` // info | warn | critical
|
||||
Code string `json:"code"`
|
||||
Title string `json:"title"`
|
||||
Detail string `json:"detail"`
|
||||
Refs []string `json:"refs,omitempty"`
|
||||
}
|
||||
|
||||
// RecommendationsResponse for GET recommendations.
|
||||
type RecommendationsResponse struct {
|
||||
CollectedAt time.Time `json:"collected_at"`
|
||||
Items []RecommendationItem `json:"items"`
|
||||
}
|
||||
|
||||
// CorrelationPoint is one aligned sample for overlay charts.
|
||||
type CorrelationPoint struct {
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
PipelineRefreshP99Ms float64 `json:"pipeline_refresh_p99_ms,omitempty"`
|
||||
BirdScrapeOK *float64 `json:"bird_scrape_ok,omitempty"`
|
||||
HTTPRequestRate float64 `json:"http_request_rate,omitempty"`
|
||||
CacheHitPct float64 `json:"cache_hit_pct,omitempty"`
|
||||
}
|
||||
|
||||
// CorrelationResponse for GET /monitoring/correlation.
|
||||
type CorrelationResponse struct {
|
||||
WindowMinutes int `json:"window_minutes"`
|
||||
Points []CorrelationPoint `json:"points"`
|
||||
}
|
||||
|
||||
// MaintenanceLogRow is an audit entry.
|
||||
type MaintenanceLogRow struct {
|
||||
ID string `json:"id"`
|
||||
TenantID string `json:"tenant_id,omitempty"`
|
||||
ActorPrefix string `json:"actor_prefix,omitempty"`
|
||||
Kind string `json:"kind"`
|
||||
TargetTable string `json:"target_table,omitempty"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
Status string `json:"status"`
|
||||
Detail map[string]any `json:"detail,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
}
|
||||
Reference in New Issue
Block a user