feat(db): enhance PostgreSQL statistics monitoring and error handling
CI / changes (push) Successful in 8s
CI / commitlint (push) Has been skipped
CI / openapi (push) Has been skipped
CI / web (push) Successful in 30s
CI / go (push) Failing after 24s
CI / bird2 (push) Has been skipped
CI / release (push) Has been skipped
CI / changes (push) Successful in 8s
CI / commitlint (push) Has been skipped
CI / openapi (push) Has been skipped
CI / web (push) Successful in 30s
CI / go (push) Failing after 24s
CI / bird2 (push) Has been skipped
CI / release (push) Has been skipped
Updated the PostgreSQL monitoring service to improve handling of `pg_stat_statements` availability. Introduced a new method to check if the extension is queryable and updated the response structure to include availability status and hints. Enhanced the documentation to clarify the requirements for enabling `pg_stat_statements`. Adjusted related components to reflect these changes, ensuring better user feedback in the monitoring interface.
This commit is contained in:
@@ -21,7 +21,14 @@ psql "$EVOBGP_DATABASE_URL"
|
|||||||
|
|
||||||
CLI на CP: `evobgp-api db report|vacuum|analyze|cleanup` (см. `internal/dbcli`).
|
CLI на CP: `evobgp-api db report|vacuum|analyze|cleanup` (см. `internal/dbcli`).
|
||||||
|
|
||||||
Миграция `000023` создаёт `pg_stat_statements`; в production может потребоваться `shared_preload_libraries` и перезапуск Postgres.
|
Миграция `000023` создаёт `pg_stat_statements`; для сбора статистики **обязательно** preload и перезапуск Postgres:
|
||||||
|
|
||||||
|
```text
|
||||||
|
# postgresql.conf или command в compose
|
||||||
|
shared_preload_libraries = 'pg_stat_statements'
|
||||||
|
```
|
||||||
|
|
||||||
|
После изменения — restart контейнера/сервиса Postgres. Без этого API `/v1/monitoring/postgres/queries` вернёт пустой список (`statements_available: false`), без 5xx.
|
||||||
|
|
||||||
## 1. Размеры таблиц и индексов
|
## 1. Размеры таблиц и индексов
|
||||||
|
|
||||||
|
|||||||
@@ -106,11 +106,7 @@ func (s *Service) fetchOverview(ctx context.Context) (Overview, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var ext bool
|
out.StatementsEnabled = s.statementsQueryable(ctx)
|
||||||
_ = s.pool.QueryRow(ctx, `
|
|
||||||
SELECT EXISTS (
|
|
||||||
SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements')`).Scan(&ext)
|
|
||||||
out.StatementsEnabled = ext
|
|
||||||
|
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
@@ -185,27 +181,75 @@ func (s *Service) TopQueries(ctx context.Context, limit int) (QueriesResponse, e
|
|||||||
if snap, ok, err := s.loadSnapshot(ctx, "slow_queries", 15*time.Minute); err == nil && ok {
|
if snap, ok, err := s.loadSnapshot(ctx, "slow_queries", 15*time.Minute); err == nil && ok {
|
||||||
var items []QueryStat
|
var items []QueryStat
|
||||||
if err := decodePayload(snap.Payload, &items); err == nil {
|
if err := decodePayload(snap.Payload, &items); err == nil {
|
||||||
return QueriesResponse{CollectedAt: snap.CollectedAt, Source: "snapshot", Items: items}, nil
|
return QueriesResponse{
|
||||||
|
CollectedAt: snap.CollectedAt,
|
||||||
|
Source: "snapshot",
|
||||||
|
Items: items,
|
||||||
|
StatementsAvailable: true,
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if !s.statementsEnabled(ctx) {
|
if !s.statementsQueryable(ctx) {
|
||||||
return QueriesResponse{CollectedAt: now, Source: "live", Items: nil}, nil
|
return queriesUnavailable(now), nil
|
||||||
}
|
}
|
||||||
items, err := queryTopStatements(ctx, s.pool, limit)
|
items, err := queryTopStatements(ctx, s.pool, limit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if isPgStatStatementsUnavailable(err) {
|
||||||
|
s.markStatementsUnavailable()
|
||||||
|
return queriesUnavailable(now), nil
|
||||||
|
}
|
||||||
return QueriesResponse{}, err
|
return QueriesResponse{}, err
|
||||||
}
|
}
|
||||||
return QueriesResponse{CollectedAt: now, Source: "live", Items: items}, nil
|
return QueriesResponse{
|
||||||
|
CollectedAt: now,
|
||||||
|
Source: "live",
|
||||||
|
Items: items,
|
||||||
|
StatementsAvailable: true,
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) statementsEnabled(ctx context.Context) bool {
|
func queriesUnavailable(at time.Time) QueriesResponse {
|
||||||
var ok bool
|
return QueriesResponse{
|
||||||
_ = s.pool.QueryRow(ctx, `
|
CollectedAt: at,
|
||||||
SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements')`).Scan(&ok)
|
Source: "unavailable",
|
||||||
|
Items: nil,
|
||||||
|
StatementsAvailable: false,
|
||||||
|
StatementsHint: statementsUnavailableHint,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const statementsUnavailableHint = "pg_stat_statements requires shared_preload_libraries and PostgreSQL restart (see docs/db-diagnostics.md)"
|
||||||
|
|
||||||
|
// statementsQueryable returns true only when pg_stat_statements can be queried (not merely installed).
|
||||||
|
func (s *Service) statementsQueryable(ctx context.Context) bool {
|
||||||
|
if s == nil || s.pool == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if v, ok := s.cache.get("stmt_queryable"); ok {
|
||||||
|
if b, ok := v.(bool); ok {
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ok := probePgStatStatements(ctx, s.pool)
|
||||||
|
s.cache.set("stmt_queryable", ok)
|
||||||
return ok
|
return ok
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Service) markStatementsUnavailable() {
|
||||||
|
s.cache.set("stmt_queryable", false)
|
||||||
|
}
|
||||||
|
|
||||||
|
func probePgStatStatements(ctx context.Context, pool *pgxpool.Pool) bool {
|
||||||
|
var dummy int64
|
||||||
|
err := pool.QueryRow(ctx, `
|
||||||
|
SELECT COALESCE(SUM(calls), 0)::bigint FROM pg_stat_statements LIMIT 1`).Scan(&dummy)
|
||||||
|
if err == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return !isPgStatStatementsUnavailable(err)
|
||||||
|
}
|
||||||
|
|
||||||
func queryTopStatements(ctx context.Context, pool *pgxpool.Pool, limit int) ([]QueryStat, error) {
|
func queryTopStatements(ctx context.Context, pool *pgxpool.Pool, limit int) ([]QueryStat, error) {
|
||||||
rows, err := pool.Query(ctx, `
|
rows, err := pool.Query(ctx, `
|
||||||
SELECT queryid, left(query, 500), calls, total_exec_time, mean_exec_time, rows
|
SELECT queryid, left(query, 500), calls, total_exec_time, mean_exec_time, rows
|
||||||
@@ -214,7 +258,7 @@ func queryTopStatements(ctx context.Context, pool *pgxpool.Pool, limit int) ([]Q
|
|||||||
ORDER BY mean_exec_time DESC
|
ORDER BY mean_exec_time DESC
|
||||||
LIMIT $1`, limit)
|
LIMIT $1`, limit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if isUndefinedTable(err) {
|
if isPgStatStatementsUnavailable(err) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
return nil, fmt.Errorf("pgmonitor: pg_stat_statements: %w", err)
|
return nil, fmt.Errorf("pgmonitor: pg_stat_statements: %w", err)
|
||||||
@@ -231,12 +275,24 @@ func queryTopStatements(ctx context.Context, pool *pgxpool.Pool, limit int) ([]Q
|
|||||||
return out, rows.Err()
|
return out, rows.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
func isUndefinedTable(err error) bool {
|
// isPgStatStatementsUnavailable reports extension missing or not loaded via shared_preload_libraries.
|
||||||
|
func isPgStatStatementsUnavailable(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
var pgErr *pgconn.PgError
|
var pgErr *pgconn.PgError
|
||||||
if errors.As(err, &pgErr) {
|
if errors.As(err, &pgErr) {
|
||||||
return pgErr.Code == "42P01" || pgErr.Code == "42704"
|
switch pgErr.Code {
|
||||||
|
case "42P01", "42704", "55000":
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
msg := strings.ToLower(pgErr.Message)
|
||||||
|
if strings.Contains(msg, "shared_preload_libraries") || strings.Contains(msg, "pg_stat_statements") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return strings.Contains(err.Error(), "pg_stat_statements")
|
low := strings.ToLower(err.Error())
|
||||||
|
return strings.Contains(low, "shared_preload_libraries") || strings.Contains(low, "pg_stat_statements")
|
||||||
}
|
}
|
||||||
|
|
||||||
func isSafeIdent(name string) bool {
|
func isSafeIdent(name string) bool {
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ func StartScheduler(ctx context.Context, pool *pgxpool.Pool) {
|
|||||||
c, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
c, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
if err := s.AggregateSlowQueries(c, 30); err != nil {
|
if err := s.AggregateSlowQueries(c, 30); err != nil {
|
||||||
log.Printf("pgmonitor: slow queries: %v", err)
|
log.Printf("pgmonitor: slow queries snapshot: %v", err)
|
||||||
}
|
}
|
||||||
if err := s.EstimateTableBloat(c); err != nil {
|
if err := s.EstimateTableBloat(c); err != nil {
|
||||||
log.Printf("pgmonitor: bloat: %v", err)
|
log.Printf("pgmonitor: bloat: %v", err)
|
||||||
|
|||||||
@@ -1,6 +1,11 @@
|
|||||||
package pgmonitor
|
package pgmonitor
|
||||||
|
|
||||||
import "testing"
|
import (
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5/pgconn"
|
||||||
|
)
|
||||||
|
|
||||||
func TestClampLimit(t *testing.T) {
|
func TestClampLimit(t *testing.T) {
|
||||||
if clampLimit(0, 20, 100) != 20 {
|
if clampLimit(0, 20, 100) != 20 {
|
||||||
@@ -31,3 +36,13 @@ func TestNewServiceNilPool(t *testing.T) {
|
|||||||
t.Fatal("expected nil service")
|
t.Fatal("expected nil service")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestIsPgStatStatementsUnavailable(t *testing.T) {
|
||||||
|
err := &pgconn.PgError{Code: "55000", Message: "pg_stat_statements must be loaded via shared_preload_libraries"}
|
||||||
|
if !isPgStatStatementsUnavailable(err) {
|
||||||
|
t.Fatal("55000")
|
||||||
|
}
|
||||||
|
if isPgStatStatementsUnavailable(errors.New("other")) {
|
||||||
|
t.Fatal("unrelated")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -66,11 +66,15 @@ func (s *Service) RefreshMetricsSnapshot(ctx context.Context) error {
|
|||||||
|
|
||||||
// AggregateSlowQueries stores top statements snapshot.
|
// AggregateSlowQueries stores top statements snapshot.
|
||||||
func (s *Service) AggregateSlowQueries(ctx context.Context, limit int) error {
|
func (s *Service) AggregateSlowQueries(ctx context.Context, limit int) error {
|
||||||
if !s.statementsEnabled(ctx) {
|
if !s.statementsQueryable(ctx) {
|
||||||
return s.UpsertSnapshot(ctx, "slow_queries", []QueryStat{})
|
return s.UpsertSnapshot(ctx, "slow_queries", []QueryStat{})
|
||||||
}
|
}
|
||||||
items, err := queryTopStatements(ctx, s.pool, clampLimit(limit, 20, 100))
|
items, err := queryTopStatements(ctx, s.pool, clampLimit(limit, 20, 100))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if isPgStatStatementsUnavailable(err) {
|
||||||
|
s.markStatementsUnavailable()
|
||||||
|
return s.UpsertSnapshot(ctx, "slow_queries", []QueryStat{})
|
||||||
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return s.UpsertSnapshot(ctx, "slow_queries", items)
|
return s.UpsertSnapshot(ctx, "slow_queries", items)
|
||||||
|
|||||||
@@ -71,9 +71,11 @@ type QueryStat struct {
|
|||||||
|
|
||||||
// QueriesResponse for GET /monitoring/postgres/queries.
|
// QueriesResponse for GET /monitoring/postgres/queries.
|
||||||
type QueriesResponse struct {
|
type QueriesResponse struct {
|
||||||
CollectedAt time.Time `json:"collected_at"`
|
CollectedAt time.Time `json:"collected_at"`
|
||||||
Source string `json:"source"` // live | snapshot
|
Source string `json:"source"` // live | snapshot | unavailable
|
||||||
Items []QueryStat `json:"items"`
|
Items []QueryStat `json:"items"`
|
||||||
|
StatementsAvailable bool `json:"statements_available"`
|
||||||
|
StatementsHint string `json:"statements_hint,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// LockRow describes a lock / blocking session.
|
// LockRow describes a lock / blocking session.
|
||||||
|
|||||||
@@ -281,12 +281,18 @@
|
|||||||
<CardTitle>Медленные запросы</CardTitle>
|
<CardTitle>Медленные запросы</CardTitle>
|
||||||
<CardDescription>
|
<CardDescription>
|
||||||
Источник: {queries?.source ?? '—'}
|
Источник: {queries?.source ?? '—'}
|
||||||
{#if overview && !overview.pg_stat_statements_enabled}
|
{#if queries?.statements_available === false || (overview && !overview.pg_stat_statements_enabled)}
|
||||||
· pg_stat_statements не включён
|
· pg_stat_statements недоступен
|
||||||
{/if}
|
{/if}
|
||||||
</CardDescription>
|
</CardDescription>
|
||||||
</CardHeader>
|
</CardHeader>
|
||||||
<CardContent>
|
<CardContent>
|
||||||
|
{#if queries?.statements_hint}
|
||||||
|
<Alert class="mb-4">
|
||||||
|
<AlertTitle>Нет статистики запросов</AlertTitle>
|
||||||
|
<AlertDescription>{queries.statements_hint}</AlertDescription>
|
||||||
|
</Alert>
|
||||||
|
{/if}
|
||||||
<Table>
|
<Table>
|
||||||
<TableHeader>
|
<TableHeader>
|
||||||
<TableRow>
|
<TableRow>
|
||||||
|
|||||||
@@ -48,6 +48,8 @@ export type PostgresQueriesResponse = {
|
|||||||
collected_at: string;
|
collected_at: string;
|
||||||
source: string;
|
source: string;
|
||||||
items: PostgresQueryRow[];
|
items: PostgresQueryRow[];
|
||||||
|
statements_available?: boolean;
|
||||||
|
statements_hint?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type PostgresLockRow = {
|
export type PostgresLockRow = {
|
||||||
|
|||||||
Reference in New Issue
Block a user