Implement bad connections baseline reset functionality
- Added a new field `badConnBaseline` in the `Handler` struct to store the baseline for bad connections. - Updated the `BuildIncidents` function to utilize the effective bad connections count for incident generation. - Introduced a new API endpoint `/api/agg/bad-connections/reset` to reset the bad connections baseline. - Enhanced the frontend with a button to trigger the reset action, providing user feedback through toast notifications. - Updated Svelte components to handle the reset functionality and display appropriate messages based on the operation's success or failure.
This commit is contained in:
@@ -0,0 +1,69 @@
|
||||
package aggregate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// effectiveConnectionsBad возвращает значение для порогов инцидентов: прирост от последнего сброса.
|
||||
func (h *Handler) effectiveConnectionsBad(alias string, current uint64) uint64 {
|
||||
h.badConnBaselineMu.Lock()
|
||||
defer h.badConnBaselineMu.Unlock()
|
||||
b, ok := h.badConnBaseline[alias]
|
||||
if !ok {
|
||||
return current
|
||||
}
|
||||
if current < b {
|
||||
// Счётчик на ноде уменьшился (перезапуск и т.п.) — старая отметка неприменима.
|
||||
delete(h.badConnBaseline, alias)
|
||||
return current
|
||||
}
|
||||
return current - b
|
||||
}
|
||||
|
||||
func (h *Handler) setBadConnectionBaseline(alias string, snapshot uint64) {
|
||||
h.badConnBaselineMu.Lock()
|
||||
defer h.badConnBaselineMu.Unlock()
|
||||
if h.badConnBaseline == nil {
|
||||
h.badConnBaseline = make(map[string]uint64)
|
||||
}
|
||||
h.badConnBaseline[alias] = snapshot
|
||||
}
|
||||
|
||||
// HandleResetBadConnectionsBaseline — POST /api/agg/bad-connections/reset
|
||||
// Сохраняет текущий connections_bad_total по каждой выбранной ноде как нулевую отметку для алертов.
|
||||
func (h *Handler) HandleResetBadConnectionsBaseline(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
w.Header().Set("Allow", http.MethodPost)
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
aliases, err := h.resolveAliases(r)
|
||||
if err != nil {
|
||||
writeBadRequest(w, err)
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
type row struct {
|
||||
Alias string `json:"alias"`
|
||||
OK bool `json:"ok"`
|
||||
Baseline uint64 `json:"baseline,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
rows := make([]row, 0, len(aliases))
|
||||
partial := false
|
||||
for _, alias := range aliases {
|
||||
data, meta := FetchTelemtGET[statsSummaryLite](ctx, h.Client, h.Parsed, alias, "stats/summary")
|
||||
if !meta.OK {
|
||||
partial = true
|
||||
rows = append(rows, row{Alias: alias, OK: false, Error: meta.Error})
|
||||
continue
|
||||
}
|
||||
h.setBadConnectionBaseline(alias, data.ConnectionsBadTotal)
|
||||
rows = append(rows, row{Alias: alias, OK: true, Baseline: data.ConnectionsBadTotal})
|
||||
}
|
||||
writeAggOK(w, partial, map[string]any{"servers": rows})
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package aggregate
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestEffectiveConnectionsBad(t *testing.T) {
|
||||
h := &Handler{}
|
||||
|
||||
if got := h.effectiveConnectionsBad("a", 500); got != 500 {
|
||||
t.Fatalf("without baseline want 500, got %d", got)
|
||||
}
|
||||
|
||||
h.setBadConnectionBaseline("a", 400)
|
||||
if got := h.effectiveConnectionsBad("a", 900); got != 500 {
|
||||
t.Fatalf("with baseline 400 want delta 500, got %d", got)
|
||||
}
|
||||
|
||||
h.setBadConnectionBaseline("b", 1000)
|
||||
if got := h.effectiveConnectionsBad("b", 500); got != 500 {
|
||||
t.Fatalf("after counter drop want raw value 500, got %d", got)
|
||||
}
|
||||
if _, ok := h.badConnBaseline["b"]; ok {
|
||||
t.Fatal("baseline for b should be cleared when current < baseline")
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,11 @@ type Handler struct {
|
||||
|
||||
cacheMu sync.Mutex
|
||||
cache map[string]cacheEntry
|
||||
|
||||
// badConnBaseline хранит снимок connections_bad_total на момент сброса в UI;
|
||||
// инциденты bad_connections_* считаются по приросту от этой отметки (в памяти процесса шлюза).
|
||||
badConnBaselineMu sync.Mutex
|
||||
badConnBaseline map[string]uint64
|
||||
}
|
||||
|
||||
// ResolveAliasesForLive resolves aliases for external endpoints (SSE/live).
|
||||
|
||||
@@ -65,7 +65,8 @@ func BuildIncidents(ctx context.Context, h *Handler, aliases []string) (Incident
|
||||
partial = true
|
||||
continue
|
||||
}
|
||||
if data.ConnectionsBadTotal >= 10000 {
|
||||
badEff := h.effectiveConnectionsBad(alias, data.ConnectionsBadTotal)
|
||||
if badEff >= 10000 {
|
||||
out.Items = append(out.Items, IncidentItem{
|
||||
ID: fmt.Sprintf("bad_connections_high:%s", alias),
|
||||
Kind: "bad_connections_high",
|
||||
@@ -75,13 +76,13 @@ func BuildIncidents(ctx context.Context, h *Handler, aliases []string) (Incident
|
||||
Summary: "Резкий рост ошибок клиентских соединений",
|
||||
AffectedAliases: []string{alias},
|
||||
MetricName: "connections_bad_total",
|
||||
MetricValue: float64(data.ConnectionsBadTotal),
|
||||
MetricValue: float64(badEff),
|
||||
MetricThreshold: 10000,
|
||||
Actions: []IncidentAction{
|
||||
{Label: "Node dashboard", Href: "/servers/" + alias},
|
||||
},
|
||||
})
|
||||
} else if data.ConnectionsBadTotal >= 1000 {
|
||||
} else if badEff >= 1000 {
|
||||
out.Items = append(out.Items, IncidentItem{
|
||||
ID: fmt.Sprintf("bad_connections_warn:%s", alias),
|
||||
Kind: "bad_connections_warn",
|
||||
@@ -91,7 +92,7 @@ func BuildIncidents(ctx context.Context, h *Handler, aliases []string) (Incident
|
||||
Summary: "Наблюдается рост ошибок клиентских соединений",
|
||||
AffectedAliases: []string{alias},
|
||||
MetricName: "connections_bad_total",
|
||||
MetricValue: float64(data.ConnectionsBadTotal),
|
||||
MetricValue: float64(badEff),
|
||||
MetricThreshold: 1000,
|
||||
Actions: []IncidentAction{
|
||||
{Label: "Node dashboard", Href: "/servers/" + alias},
|
||||
|
||||
@@ -308,6 +308,10 @@ func (g *Gateway) serve(w http.ResponseWriter, r *http.Request) {
|
||||
proxy.NormalizeRequestURLPath(r)
|
||||
}
|
||||
const prefix = "/api/"
|
||||
if r.Method == http.MethodPost && r.URL.Path == "/api/agg/bad-connections/reset" {
|
||||
g.agg.HandleResetBadConnectionsBaseline(w, r)
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/api/agg" || strings.HasPrefix(r.URL.Path, "/api/agg/") {
|
||||
g.agg.ServeHTTP(w, r)
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user