Enhance API and UI for incident management and live updates

- Added a new endpoint `/api/agg/incidents` to provide a normalized snapshot of incidents for fleet triage, including severity and recommended actions.
- Implemented live event streaming via `/api/live/events` for real-time updates on fleet status and incidents, enhancing observability.
- Updated the Web UI to include dedicated sections for incidents and live updates, improving user navigation and access to critical information.
- Enhanced API documentation to reflect new endpoints and their functionalities, ensuring clarity for developers and users.

internal/aggregate/handlers.go

@@ -35,6 +35,11 @@ type Handler struct {
 	cache   map[string]cacheEntry
 }
 
+// ResolveAliasesForLive resolves aliases for external endpoints (SSE/live).
+func (h *Handler) ResolveAliasesForLive(r *http.Request) ([]string, error) {
+	return h.resolveAliases(r)
+}
+
 // NewHandler builds an aggregate handler; client must use a non-nil Transport (e.g. gateway shared transport).
 // Geo may be nil (no GeoLite2 lookups). cacheTTL 0 disables response caching.
 func NewHandler(p *config.Parsed, client *http.Client, geo *geoip.Service, cacheTTL time.Duration) *Handler {
@@ -108,6 +113,8 @@ func (h *Handler) dispatch(w http.ResponseWriter, r *http.Request, sub string) {
 		h.handleUsers(w, r)
 	case sub == "fleet-status":
 		h.handleFleetStatus(w, r)
+	case sub == "incidents":
+		h.handleIncidents(w, r)
 	case strings.HasPrefix(sub, "user/"):
 		username := strings.TrimPrefix(sub, "user/")
 		if username == "" {
@@ -304,6 +311,18 @@ func (h *Handler) handleUserOne(w http.ResponseWriter, r *http.Request, username
 	writeAggOK(w, partial, row)
 }
 
+func (h *Handler) handleIncidents(w http.ResponseWriter, r *http.Request) {
+	aliases, err := h.resolveAliases(r)
+	if err != nil {
+		writeBadRequest(w, err)
+		return
+	}
+	ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
+	defer cancel()
+	data, partial := BuildIncidents(ctx, h, aliases)
+	writeAggOK(w, partial, data)
+}
+
 func writeBadRequest(w http.ResponseWriter, err error) {
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	w.WriteHeader(http.StatusBadRequest)

internal/aggregate/incidents.go

@@ -0,0 +1,154 @@
+package aggregate
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+type statsSummaryLite struct {
+	ConnectionsBadTotal uint64 `json:"connections_bad_total"`
+	ConnectionsTotal    uint64 `json:"connections_total"`
+}
+
+func BuildIncidents(ctx context.Context, h *Handler, aliases []string) (IncidentsData, bool) {
+	out := IncidentsData{Items: make([]IncidentItem, 0)}
+	partial := false
+
+	fleet := FetchFleetStatus(ctx, h.Client, h.Parsed, aliases)
+	if fleet.ServersFailed > 0 {
+		partial = true
+	}
+
+	for _, s := range fleet.Servers {
+		if !s.OK {
+			severity := SeverityWarning
+			if !s.HealthOK && !s.SystemInfoOK {
+				severity = SeverityCritical
+			}
+			out.Items = append(out.Items, IncidentItem{
+				ID:              fmt.Sprintf("node_degraded:%s", s.Alias),
+				Kind:            "node_degraded",
+				Severity:        severity,
+				Status:          IncidentStatusFiring,
+				Title:           fmt.Sprintf("Нода %s degraded", s.Alias),
+				Summary:         "health/system_info не прошли полностью",
+				AffectedAliases: []string{s.Alias},
+				Actions: []IncidentAction{
+					{Label: "Открыть ноду", Href: "/servers/" + s.Alias},
+					{Label: "Runtime", Href: "/servers/" + s.Alias + "/runtime"},
+				},
+			})
+		}
+		if s.Health != nil && s.Health.ReadOnly {
+			out.Items = append(out.Items, IncidentItem{
+				ID:              fmt.Sprintf("node_read_only:%s", s.Alias),
+				Kind:            "node_read_only",
+				Severity:        SeverityWarning,
+				Status:          IncidentStatusFiring,
+				Title:           fmt.Sprintf("Нода %s в read_only", s.Alias),
+				Summary:         "API ноды не принимает mutating операции",
+				AffectedAliases: []string{s.Alias},
+				Actions: []IncidentAction{
+					{Label: "Users", Href: "/servers/" + s.Alias + "/users"},
+					{Label: "Security", Href: "/servers/" + s.Alias + "/security"},
+				},
+			})
+		}
+	}
+
+	for _, alias := range aliases {
+		data, meta := FetchTelemtGET[statsSummaryLite](ctx, h.Client, h.Parsed, alias, "stats/summary")
+		if !meta.OK {
+			partial = true
+			continue
+		}
+		if data.ConnectionsBadTotal >= 10000 {
+			out.Items = append(out.Items, IncidentItem{
+				ID:              fmt.Sprintf("bad_connections_high:%s", alias),
+				Kind:            "bad_connections_high",
+				Severity:        SeverityCritical,
+				Status:          IncidentStatusFiring,
+				Title:           fmt.Sprintf("Высокий bad connections на %s", alias),
+				Summary:         "Резкий рост ошибок клиентских соединений",
+				AffectedAliases: []string{alias},
+				MetricName:      "connections_bad_total",
+				MetricValue:     float64(data.ConnectionsBadTotal),
+				MetricThreshold: 10000,
+				Actions: []IncidentAction{
+					{Label: "Node dashboard", Href: "/servers/" + alias},
+				},
+			})
+		} else if data.ConnectionsBadTotal >= 1000 {
+			out.Items = append(out.Items, IncidentItem{
+				ID:              fmt.Sprintf("bad_connections_warn:%s", alias),
+				Kind:            "bad_connections_warn",
+				Severity:        SeverityWarning,
+				Status:          IncidentStatusFiring,
+				Title:           fmt.Sprintf("Рост bad connections на %s", alias),
+				Summary:         "Наблюдается рост ошибок клиентских соединений",
+				AffectedAliases: []string{alias},
+				MetricName:      "connections_bad_total",
+				MetricValue:     float64(data.ConnectionsBadTotal),
+				MetricThreshold: 1000,
+				Actions: []IncidentAction{
+					{Label: "Node dashboard", Href: "/servers/" + alias},
+				},
+			})
+		}
+	}
+
+	sort.Slice(out.Items, func(i, j int) bool {
+		if severityWeight(out.Items[i].Severity) != severityWeight(out.Items[j].Severity) {
+			return severityWeight(out.Items[i].Severity) > severityWeight(out.Items[j].Severity)
+		}
+		return strings.Compare(out.Items[i].ID, out.Items[j].ID) < 0
+	})
+
+	out.Total = len(out.Items)
+	for _, it := range out.Items {
+		switch it.Severity {
+		case SeverityCritical:
+			out.CriticalTotal++
+		case SeverityWarning:
+			out.WarningTotal++
+		default:
+			out.InfoTotal++
+		}
+	}
+	return out, partial
+}
+
+func severityWeight(s IncidentSeverity) int {
+	switch s {
+	case SeverityCritical:
+		return 3
+	case SeverityWarning:
+		return 2
+	case SeverityInfo:
+		return 1
+	default:
+		return 0
+	}
+}
+
+func BuildLiveEnvelope(ctx context.Context, h *Handler, aliases []string) map[string]any {
+	inc, partial := BuildIncidents(ctx, h, aliases)
+	status := "healthy"
+	if inc.CriticalTotal > 0 {
+		status = "critical"
+	} else if inc.WarningTotal > 0 {
+		status = "degraded"
+	}
+	return map[string]any{
+		"type":         "live_snapshot",
+		"timestamp":    time.Now().UTC().Format(time.RFC3339Nano),
+		"status":       status,
+		"partial":      partial,
+		"incidents":    inc.Items,
+		"counts":       map[string]int{"total": inc.Total, "critical": inc.CriticalTotal, "warning": inc.WarningTotal, "info": inc.InfoTotal},
+		"aliases_used": aliases,
+	}
+}

internal/aggregate/types.go

@@ -149,6 +149,52 @@ type SummaryData struct {
 	TopUsersByUniqueIPs     []TopUserByUniqueIPs  `json:"top_users_by_unique_ips"`
 }
 
+// IncidentSeverity is normalized severity used by aggregate incidents.
+type IncidentSeverity string
+
+const (
+	SeverityInfo     IncidentSeverity = "info"
+	SeverityWarning  IncidentSeverity = "warning"
+	SeverityCritical IncidentSeverity = "critical"
+)
+
+// IncidentStatus is current calculated incident state.
+type IncidentStatus string
+
+const (
+	IncidentStatusFiring IncidentStatus = "firing"
+)
+
+// IncidentAction points to a UI route with details/runbook context.
+type IncidentAction struct {
+	Label string `json:"label"`
+	Href  string `json:"href"`
+}
+
+// IncidentItem is one normalized fleet incident.
+type IncidentItem struct {
+	ID              string           `json:"id"`
+	Kind            string           `json:"kind"`
+	Severity        IncidentSeverity `json:"severity"`
+	Status          IncidentStatus   `json:"status"`
+	Title           string           `json:"title"`
+	Summary         string           `json:"summary"`
+	AffectedAliases []string         `json:"affected_aliases,omitempty"`
+	MetricName      string           `json:"metric_name,omitempty"`
+	MetricValue     float64          `json:"metric_value,omitempty"`
+	MetricThreshold float64          `json:"metric_threshold,omitempty"`
+	Actions         []IncidentAction `json:"actions,omitempty"`
+}
+
+// IncidentsData is aggregate response payload for /api/agg/incidents.
+type IncidentsData struct {
+	Items         []IncidentItem `json:"items"`
+	Total         int            `json:"total"`
+	CriticalTotal int            `json:"critical_total"`
+	WarningTotal  int            `json:"warning_total"`
+	InfoTotal     int            `json:"info_total"`
+}
+
 // TopUser by summed traffic across servers for one username (мегабайты).
 type TopUser struct {
 	Username       string  `json:"username"`

internal/server/gateway.go

@@ -3,6 +3,7 @@ package server
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"log/slog"
 	"net"
 	"net/http"
@@ -181,8 +182,9 @@ func (g *Gateway) withMetrics(next http.Handler) http.Handler {
 		httpInFlight.Inc()
 		start := time.Now()
 		alias := routeAlias(r.URL.Path)
+		endpoint := routeEndpoint(r.URL.Path)
 		lw := &statusWriter{ResponseWriter: w, status: http.StatusOK}
-		defer observeRequest(r.Method, alias, lw.status, start)
+		defer observeRequest(r.Method, alias, endpoint, lw.status, start)
 		next.ServeHTTP(lw, r)
 	})
 }
@@ -206,6 +208,33 @@ func routeAlias(path string) string {
 	return rest[:i]
 }
 
+func routeEndpoint(path string) string {
+	if path == "" || path == "/" {
+		return "ui_root"
+	}
+	if path == "/health" || path == "/metrics" {
+		return strings.TrimPrefix(path, "/")
+	}
+	if strings.HasPrefix(path, "/api/agg/") {
+		return strings.TrimPrefix(path, "/api/agg/")
+	}
+	if path == "/api/agg" {
+		return "agg"
+	}
+	if strings.HasPrefix(path, "/api/live/events") {
+		return "live_events"
+	}
+	if strings.HasPrefix(path, "/api/") {
+		rest := strings.TrimPrefix(path, "/api/")
+		i := strings.IndexByte(rest, '/')
+		if i < 0 {
+			return "proxy_root"
+		}
+		return "proxy_" + rest[i+1:]
+	}
+	return "ui"
+}
+
 func (g *Gateway) serve(w http.ResponseWriter, r *http.Request) {
 	switch r.URL.Path {
 	case "/health":
@@ -233,6 +262,10 @@ func (g *Gateway) serve(w http.ResponseWriter, r *http.Request) {
 		g.agg.ServeHTTP(w, r)
 		return
 	}
+	if r.URL.Path == "/api/live/events" {
+		g.serveLiveEvents(w, r)
+		return
+	}
 	if !strings.HasPrefix(r.URL.Path, prefix) {
 		g.webUI.ServeHTTP(w, r)
 		return
@@ -265,6 +298,54 @@ func (g *Gateway) serve(w http.ResponseWriter, r *http.Request) {
 	rp.ServeHTTP(w, r)
 }
 
+func (g *Gateway) serveLiveEvents(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	flusher, ok := w.(http.Flusher)
+	if !ok {
+		http.Error(w, "stream unsupported", http.StatusInternalServerError)
+		return
+	}
+	aliases, err := g.agg.ResolveAliasesForLive(r)
+	if err != nil {
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.WriteHeader(http.StatusBadRequest)
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"ok":    false,
+			"error": map[string]string{"code": "bad_request", "message": err.Error()},
+		})
+		return
+	}
+	w.Header().Set("Content-Type", "text/event-stream")
+	w.Header().Set("Cache-Control", "no-cache")
+	w.Header().Set("Connection", "keep-alive")
+	w.WriteHeader(http.StatusOK)
+
+	ticker := time.NewTicker(5 * time.Second)
+	defer ticker.Stop()
+
+	push := func() {
+		ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
+		defer cancel()
+		payload := aggregate.BuildLiveEnvelope(ctx, g.agg, aliases)
+		b, _ := json.Marshal(payload)
+		_, _ = fmt.Fprintf(w, "event: snapshot\n")
+		_, _ = fmt.Fprintf(w, "data: %s\n\n", string(b))
+		flusher.Flush()
+	}
+	push()
+	for {
+		select {
+		case <-r.Context().Done():
+			return
+		case <-ticker.C:
+			push()
+		}
+	}
+}
+
 type statusWriter struct {
 	http.ResponseWriter
 	status int

internal/server/metrics.go

@@ -15,17 +15,17 @@ var (
 	})
 	httpRequests = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "telemt_gateway_http_requests_total",
-		Help: "HTTP requests by status, method, alias.",
-	}, []string{"code", "method", "alias"})
+		Help: "HTTP requests by status, method, alias, endpoint.",
+	}, []string{"code", "method", "alias", "endpoint"})
 	httpDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
 		Name:    "telemt_gateway_http_request_duration_seconds",
 		Help:    "Request duration in seconds.",
 		Buckets: prometheus.DefBuckets,
-	}, []string{"method", "alias"})
+	}, []string{"method", "alias", "endpoint"})
 )
 
-func observeRequest(method, alias string, status int, started time.Time) {
+func observeRequest(method, alias, endpoint string, status int, started time.Time) {
 	httpInFlight.Dec()
-	httpRequests.WithLabelValues(strconv.Itoa(status), method, alias).Inc()
-	httpDuration.WithLabelValues(method, alias).Observe(time.Since(started).Seconds())
+	httpRequests.WithLabelValues(strconv.Itoa(status), method, alias, endpoint).Inc()
+	httpDuration.WithLabelValues(method, alias, endpoint).Observe(time.Since(started).Seconds())
 }