Enhance API and UI for incident management and live updates
- Added a new endpoint `/api/agg/incidents` to provide a normalized snapshot of incidents for fleet triage, including severity and recommended actions. - Implemented live event streaming via `/api/live/events` for real-time updates on fleet status and incidents, enhancing observability. - Updated the Web UI to include dedicated sections for incidents and live updates, improving user navigation and access to critical information. - Enhanced API documentation to reflect new endpoints and their functionalities, ensuring clarity for developers and users.
This commit is contained in:
Denozordec
@@ -35,6 +35,11 @@ type Handler struct {
|
||||
cache map[string]cacheEntry
|
||||
}
|
||||
|
||||
// ResolveAliasesForLive resolves aliases for external endpoints (SSE/live).
|
||||
func (h *Handler) ResolveAliasesForLive(r *http.Request) ([]string, error) {
|
||||
return h.resolveAliases(r)
|
||||
}
|
||||
|
||||
// NewHandler builds an aggregate handler; client must use a non-nil Transport (e.g. gateway shared transport).
|
||||
// Geo may be nil (no GeoLite2 lookups). cacheTTL 0 disables response caching.
|
||||
func NewHandler(p *config.Parsed, client *http.Client, geo *geoip.Service, cacheTTL time.Duration) *Handler {
|
||||
@@ -108,6 +113,8 @@ func (h *Handler) dispatch(w http.ResponseWriter, r *http.Request, sub string) {
|
||||
h.handleUsers(w, r)
|
||||
case sub == "fleet-status":
|
||||
h.handleFleetStatus(w, r)
|
||||
case sub == "incidents":
|
||||
h.handleIncidents(w, r)
|
||||
case strings.HasPrefix(sub, "user/"):
|
||||
username := strings.TrimPrefix(sub, "user/")
|
||||
if username == "" {
|
||||
@@ -304,6 +311,18 @@ func (h *Handler) handleUserOne(w http.ResponseWriter, r *http.Request, username
|
||||
writeAggOK(w, partial, row)
|
||||
}
|
||||
|
||||
func (h *Handler) handleIncidents(w http.ResponseWriter, r *http.Request) {
|
||||
aliases, err := h.resolveAliases(r)
|
||||
if err != nil {
|
||||
writeBadRequest(w, err)
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
|
||||
defer cancel()
|
||||
data, partial := BuildIncidents(ctx, h, aliases)
|
||||
writeAggOK(w, partial, data)
|
||||
}
|
||||
|
||||
func writeBadRequest(w http.ResponseWriter, err error) {
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
package aggregate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type statsSummaryLite struct {
|
||||
ConnectionsBadTotal uint64 `json:"connections_bad_total"`
|
||||
ConnectionsTotal uint64 `json:"connections_total"`
|
||||
}
|
||||
|
||||
func BuildIncidents(ctx context.Context, h *Handler, aliases []string) (IncidentsData, bool) {
|
||||
out := IncidentsData{Items: make([]IncidentItem, 0)}
|
||||
partial := false
|
||||
|
||||
fleet := FetchFleetStatus(ctx, h.Client, h.Parsed, aliases)
|
||||
if fleet.ServersFailed > 0 {
|
||||
partial = true
|
||||
}
|
||||
|
||||
for _, s := range fleet.Servers {
|
||||
if !s.OK {
|
||||
severity := SeverityWarning
|
||||
if !s.HealthOK && !s.SystemInfoOK {
|
||||
severity = SeverityCritical
|
||||
}
|
||||
out.Items = append(out.Items, IncidentItem{
|
||||
ID: fmt.Sprintf("node_degraded:%s", s.Alias),
|
||||
Kind: "node_degraded",
|
||||
Severity: severity,
|
||||
Status: IncidentStatusFiring,
|
||||
Title: fmt.Sprintf("Нода %s degraded", s.Alias),
|
||||
Summary: "health/system_info не прошли полностью",
|
||||
AffectedAliases: []string{s.Alias},
|
||||
Actions: []IncidentAction{
|
||||
{Label: "Открыть ноду", Href: "/servers/" + s.Alias},
|
||||
{Label: "Runtime", Href: "/servers/" + s.Alias + "/runtime"},
|
||||
},
|
||||
})
|
||||
}
|
||||
if s.Health != nil && s.Health.ReadOnly {
|
||||
out.Items = append(out.Items, IncidentItem{
|
||||
ID: fmt.Sprintf("node_read_only:%s", s.Alias),
|
||||
Kind: "node_read_only",
|
||||
Severity: SeverityWarning,
|
||||
Status: IncidentStatusFiring,
|
||||
Title: fmt.Sprintf("Нода %s в read_only", s.Alias),
|
||||
Summary: "API ноды не принимает mutating операции",
|
||||
AffectedAliases: []string{s.Alias},
|
||||
Actions: []IncidentAction{
|
||||
{Label: "Users", Href: "/servers/" + s.Alias + "/users"},
|
||||
{Label: "Security", Href: "/servers/" + s.Alias + "/security"},
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for _, alias := range aliases {
|
||||
data, meta := FetchTelemtGET[statsSummaryLite](ctx, h.Client, h.Parsed, alias, "stats/summary")
|
||||
if !meta.OK {
|
||||
partial = true
|
||||
continue
|
||||
}
|
||||
if data.ConnectionsBadTotal >= 10000 {
|
||||
out.Items = append(out.Items, IncidentItem{
|
||||
ID: fmt.Sprintf("bad_connections_high:%s", alias),
|
||||
Kind: "bad_connections_high",
|
||||
Severity: SeverityCritical,
|
||||
Status: IncidentStatusFiring,
|
||||
Title: fmt.Sprintf("Высокий bad connections на %s", alias),
|
||||
Summary: "Резкий рост ошибок клиентских соединений",
|
||||
AffectedAliases: []string{alias},
|
||||
MetricName: "connections_bad_total",
|
||||
MetricValue: float64(data.ConnectionsBadTotal),
|
||||
MetricThreshold: 10000,
|
||||
Actions: []IncidentAction{
|
||||
{Label: "Node dashboard", Href: "/servers/" + alias},
|
||||
},
|
||||
})
|
||||
} else if data.ConnectionsBadTotal >= 1000 {
|
||||
out.Items = append(out.Items, IncidentItem{
|
||||
ID: fmt.Sprintf("bad_connections_warn:%s", alias),
|
||||
Kind: "bad_connections_warn",
|
||||
Severity: SeverityWarning,
|
||||
Status: IncidentStatusFiring,
|
||||
Title: fmt.Sprintf("Рост bad connections на %s", alias),
|
||||
Summary: "Наблюдается рост ошибок клиентских соединений",
|
||||
AffectedAliases: []string{alias},
|
||||
MetricName: "connections_bad_total",
|
||||
MetricValue: float64(data.ConnectionsBadTotal),
|
||||
MetricThreshold: 1000,
|
||||
Actions: []IncidentAction{
|
||||
{Label: "Node dashboard", Href: "/servers/" + alias},
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(out.Items, func(i, j int) bool {
|
||||
if severityWeight(out.Items[i].Severity) != severityWeight(out.Items[j].Severity) {
|
||||
return severityWeight(out.Items[i].Severity) > severityWeight(out.Items[j].Severity)
|
||||
}
|
||||
return strings.Compare(out.Items[i].ID, out.Items[j].ID) < 0
|
||||
})
|
||||
|
||||
out.Total = len(out.Items)
|
||||
for _, it := range out.Items {
|
||||
switch it.Severity {
|
||||
case SeverityCritical:
|
||||
out.CriticalTotal++
|
||||
case SeverityWarning:
|
||||
out.WarningTotal++
|
||||
default:
|
||||
out.InfoTotal++
|
||||
}
|
||||
}
|
||||
return out, partial
|
||||
}
|
||||
|
||||
func severityWeight(s IncidentSeverity) int {
|
||||
switch s {
|
||||
case SeverityCritical:
|
||||
return 3
|
||||
case SeverityWarning:
|
||||
return 2
|
||||
case SeverityInfo:
|
||||
return 1
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func BuildLiveEnvelope(ctx context.Context, h *Handler, aliases []string) map[string]any {
|
||||
inc, partial := BuildIncidents(ctx, h, aliases)
|
||||
status := "healthy"
|
||||
if inc.CriticalTotal > 0 {
|
||||
status = "critical"
|
||||
} else if inc.WarningTotal > 0 {
|
||||
status = "degraded"
|
||||
}
|
||||
return map[string]any{
|
||||
"type": "live_snapshot",
|
||||
"timestamp": time.Now().UTC().Format(time.RFC3339Nano),
|
||||
"status": status,
|
||||
"partial": partial,
|
||||
"incidents": inc.Items,
|
||||
"counts": map[string]int{"total": inc.Total, "critical": inc.CriticalTotal, "warning": inc.WarningTotal, "info": inc.InfoTotal},
|
||||
"aliases_used": aliases,
|
||||
}
|
||||
}
|
||||
@@ -149,6 +149,52 @@ type SummaryData struct {
|
||||
TopUsersByUniqueIPs []TopUserByUniqueIPs `json:"top_users_by_unique_ips"`
|
||||
}
|
||||
|
||||
// IncidentSeverity is normalized severity used by aggregate incidents.
|
||||
type IncidentSeverity string
|
||||
|
||||
const (
|
||||
SeverityInfo IncidentSeverity = "info"
|
||||
SeverityWarning IncidentSeverity = "warning"
|
||||
SeverityCritical IncidentSeverity = "critical"
|
||||
)
|
||||
|
||||
// IncidentStatus is current calculated incident state.
|
||||
type IncidentStatus string
|
||||
|
||||
const (
|
||||
IncidentStatusFiring IncidentStatus = "firing"
|
||||
)
|
||||
|
||||
// IncidentAction points to a UI route with details/runbook context.
|
||||
type IncidentAction struct {
|
||||
Label string `json:"label"`
|
||||
Href string `json:"href"`
|
||||
}
|
||||
|
||||
// IncidentItem is one normalized fleet incident.
|
||||
type IncidentItem struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Severity IncidentSeverity `json:"severity"`
|
||||
Status IncidentStatus `json:"status"`
|
||||
Title string `json:"title"`
|
||||
Summary string `json:"summary"`
|
||||
AffectedAliases []string `json:"affected_aliases,omitempty"`
|
||||
MetricName string `json:"metric_name,omitempty"`
|
||||
MetricValue float64 `json:"metric_value,omitempty"`
|
||||
MetricThreshold float64 `json:"metric_threshold,omitempty"`
|
||||
Actions []IncidentAction `json:"actions,omitempty"`
|
||||
}
|
||||
|
||||
// IncidentsData is aggregate response payload for /api/agg/incidents.
|
||||
type IncidentsData struct {
|
||||
Items []IncidentItem `json:"items"`
|
||||
Total int `json:"total"`
|
||||
CriticalTotal int `json:"critical_total"`
|
||||
WarningTotal int `json:"warning_total"`
|
||||
InfoTotal int `json:"info_total"`
|
||||
}
|
||||
|
||||
// TopUser by summed traffic across servers for one username (мегабайты).
|
||||
type TopUser struct {
|
||||
Username string `json:"username"`
|
||||
|
||||
Reference in New Issue
Block a user