Enhance API and UI for incident management and live updates

- Added a new endpoint `/api/agg/incidents` to provide a normalized snapshot of incidents for fleet triage, including severity and recommended actions. - Implemented live event streaming via `/api/live/events` for real-time updates on fleet status and incidents, enhancing observability. - Updated the Web UI to include dedicated sections for incidents and live updates, improving user navigation and access to critical information. - Enhanced API documentation to reflect new endpoints and their functionalities, ensuring clarity for developers and users.
2026-03-30 19:17:29 +07:00
parent af11a49c81
commit 8c8ccce6ee
18 changed files with 1655 additions and 28 deletions
@@ -0,0 +1,154 @@
+package aggregate
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+type statsSummaryLite struct {
+	ConnectionsBadTotal uint64 `json:"connections_bad_total"`
+	ConnectionsTotal    uint64 `json:"connections_total"`
+}
+
+func BuildIncidents(ctx context.Context, h *Handler, aliases []string) (IncidentsData, bool) {
+	out := IncidentsData{Items: make([]IncidentItem, 0)}
+	partial := false
+
+	fleet := FetchFleetStatus(ctx, h.Client, h.Parsed, aliases)
+	if fleet.ServersFailed > 0 {
+		partial = true
+	}
+
+	for _, s := range fleet.Servers {
+		if !s.OK {
+			severity := SeverityWarning
+			if !s.HealthOK && !s.SystemInfoOK {
+				severity = SeverityCritical
+			}
+			out.Items = append(out.Items, IncidentItem{
+				ID:              fmt.Sprintf("node_degraded:%s", s.Alias),
+				Kind:            "node_degraded",
+				Severity:        severity,
+				Status:          IncidentStatusFiring,
+				Title:           fmt.Sprintf("Нода %s degraded", s.Alias),
+				Summary:         "health/system_info не прошли полностью",
+				AffectedAliases: []string{s.Alias},
+				Actions: []IncidentAction{
+					{Label: "Открыть ноду", Href: "/servers/" + s.Alias},
+					{Label: "Runtime", Href: "/servers/" + s.Alias + "/runtime"},
+				},
+			})
+		}
+		if s.Health != nil && s.Health.ReadOnly {
+			out.Items = append(out.Items, IncidentItem{
+				ID:              fmt.Sprintf("node_read_only:%s", s.Alias),
+				Kind:            "node_read_only",
+				Severity:        SeverityWarning,
+				Status:          IncidentStatusFiring,
+				Title:           fmt.Sprintf("Нода %s в read_only", s.Alias),
+				Summary:         "API ноды не принимает mutating операции",
+				AffectedAliases: []string{s.Alias},
+				Actions: []IncidentAction{
+					{Label: "Users", Href: "/servers/" + s.Alias + "/users"},
+					{Label: "Security", Href: "/servers/" + s.Alias + "/security"},
+				},
+			})
+		}
+	}
+
+	for _, alias := range aliases {
+		data, meta := FetchTelemtGET[statsSummaryLite](ctx, h.Client, h.Parsed, alias, "stats/summary")
+		if !meta.OK {
+			partial = true
+			continue
+		}
+		if data.ConnectionsBadTotal >= 10000 {
+			out.Items = append(out.Items, IncidentItem{
+				ID:              fmt.Sprintf("bad_connections_high:%s", alias),
+				Kind:            "bad_connections_high",
+				Severity:        SeverityCritical,
+				Status:          IncidentStatusFiring,
+				Title:           fmt.Sprintf("Высокий bad connections на %s", alias),
+				Summary:         "Резкий рост ошибок клиентских соединений",
+				AffectedAliases: []string{alias},
+				MetricName:      "connections_bad_total",
+				MetricValue:     float64(data.ConnectionsBadTotal),
+				MetricThreshold: 10000,
+				Actions: []IncidentAction{
+					{Label: "Node dashboard", Href: "/servers/" + alias},
+				},
+			})
+		} else if data.ConnectionsBadTotal >= 1000 {
+			out.Items = append(out.Items, IncidentItem{
+				ID:              fmt.Sprintf("bad_connections_warn:%s", alias),
+				Kind:            "bad_connections_warn",
+				Severity:        SeverityWarning,
+				Status:          IncidentStatusFiring,
+				Title:           fmt.Sprintf("Рост bad connections на %s", alias),
+				Summary:         "Наблюдается рост ошибок клиентских соединений",
+				AffectedAliases: []string{alias},
+				MetricName:      "connections_bad_total",
+				MetricValue:     float64(data.ConnectionsBadTotal),
+				MetricThreshold: 1000,
+				Actions: []IncidentAction{
+					{Label: "Node dashboard", Href: "/servers/" + alias},
+				},
+			})
+		}
+	}
+
+	sort.Slice(out.Items, func(i, j int) bool {
+		if severityWeight(out.Items[i].Severity) != severityWeight(out.Items[j].Severity) {
+			return severityWeight(out.Items[i].Severity) > severityWeight(out.Items[j].Severity)
+		}
+		return strings.Compare(out.Items[i].ID, out.Items[j].ID) < 0
+	})
+
+	out.Total = len(out.Items)
+	for _, it := range out.Items {
+		switch it.Severity {
+		case SeverityCritical:
+			out.CriticalTotal++
+		case SeverityWarning:
+			out.WarningTotal++
+		default:
+			out.InfoTotal++
+		}
+	}
+	return out, partial
+}
+
+func severityWeight(s IncidentSeverity) int {
+	switch s {
+	case SeverityCritical:
+		return 3
+	case SeverityWarning:
+		return 2
+	case SeverityInfo:
+		return 1
+	default:
+		return 0
+	}
+}
+
+func BuildLiveEnvelope(ctx context.Context, h *Handler, aliases []string) map[string]any {
+	inc, partial := BuildIncidents(ctx, h, aliases)
+	status := "healthy"
+	if inc.CriticalTotal > 0 {
+		status = "critical"
+	} else if inc.WarningTotal > 0 {
+		status = "degraded"
+	}
+	return map[string]any{
+		"type":         "live_snapshot",
+		"timestamp":    time.Now().UTC().Format(time.RFC3339Nano),
+		"status":       status,
+		"partial":      partial,
+		"incidents":    inc.Items,
+		"counts":       map[string]int{"total": inc.Total, "critical": inc.CriticalTotal, "warning": inc.WarningTotal, "info": inc.InfoTotal},
+		"aliases_used": aliases,
+	}
+}