Enhance API and UI for incident management and live updates

- Added a new endpoint `/api/agg/incidents` to provide a normalized snapshot of incidents for fleet triage, including severity and recommended actions.
- Implemented live event streaming via `/api/live/events` for real-time updates on fleet status and incidents, enhancing observability.
- Updated the Web UI to include dedicated sections for incidents and live updates, improving user navigation and access to critical information.
- Enhanced API documentation to reflect new endpoints and their functionalities, ensuring clarity for developers and users.

internal/server/gateway.go

@@ -3,6 +3,7 @@ package server
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"log/slog"
 	"net"
 	"net/http"
@@ -181,8 +182,9 @@ func (g *Gateway) withMetrics(next http.Handler) http.Handler {
 		httpInFlight.Inc()
 		start := time.Now()
 		alias := routeAlias(r.URL.Path)
+		endpoint := routeEndpoint(r.URL.Path)
 		lw := &statusWriter{ResponseWriter: w, status: http.StatusOK}
-		defer observeRequest(r.Method, alias, lw.status, start)
+		defer observeRequest(r.Method, alias, endpoint, lw.status, start)
 		next.ServeHTTP(lw, r)
 	})
 }
@@ -206,6 +208,33 @@ func routeAlias(path string) string {
 	return rest[:i]
 }
 
+func routeEndpoint(path string) string {
+	if path == "" || path == "/" {
+		return "ui_root"
+	}
+	if path == "/health" || path == "/metrics" {
+		return strings.TrimPrefix(path, "/")
+	}
+	if strings.HasPrefix(path, "/api/agg/") {
+		return strings.TrimPrefix(path, "/api/agg/")
+	}
+	if path == "/api/agg" {
+		return "agg"
+	}
+	if strings.HasPrefix(path, "/api/live/events") {
+		return "live_events"
+	}
+	if strings.HasPrefix(path, "/api/") {
+		rest := strings.TrimPrefix(path, "/api/")
+		i := strings.IndexByte(rest, '/')
+		if i < 0 {
+			return "proxy_root"
+		}
+		return "proxy_" + rest[i+1:]
+	}
+	return "ui"
+}
+
 func (g *Gateway) serve(w http.ResponseWriter, r *http.Request) {
 	switch r.URL.Path {
 	case "/health":
@@ -233,6 +262,10 @@ func (g *Gateway) serve(w http.ResponseWriter, r *http.Request) {
 		g.agg.ServeHTTP(w, r)
 		return
 	}
+	if r.URL.Path == "/api/live/events" {
+		g.serveLiveEvents(w, r)
+		return
+	}
 	if !strings.HasPrefix(r.URL.Path, prefix) {
 		g.webUI.ServeHTTP(w, r)
 		return
@@ -265,6 +298,54 @@ func (g *Gateway) serve(w http.ResponseWriter, r *http.Request) {
 	rp.ServeHTTP(w, r)
 }
 
+func (g *Gateway) serveLiveEvents(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	flusher, ok := w.(http.Flusher)
+	if !ok {
+		http.Error(w, "stream unsupported", http.StatusInternalServerError)
+		return
+	}
+	aliases, err := g.agg.ResolveAliasesForLive(r)
+	if err != nil {
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.WriteHeader(http.StatusBadRequest)
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"ok":    false,
+			"error": map[string]string{"code": "bad_request", "message": err.Error()},
+		})
+		return
+	}
+	w.Header().Set("Content-Type", "text/event-stream")
+	w.Header().Set("Cache-Control", "no-cache")
+	w.Header().Set("Connection", "keep-alive")
+	w.WriteHeader(http.StatusOK)
+
+	ticker := time.NewTicker(5 * time.Second)
+	defer ticker.Stop()
+
+	push := func() {
+		ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
+		defer cancel()
+		payload := aggregate.BuildLiveEnvelope(ctx, g.agg, aliases)
+		b, _ := json.Marshal(payload)
+		_, _ = fmt.Fprintf(w, "event: snapshot\n")
+		_, _ = fmt.Fprintf(w, "data: %s\n\n", string(b))
+		flusher.Flush()
+	}
+	push()
+	for {
+		select {
+		case <-r.Context().Done():
+			return
+		case <-ticker.C:
+			push()
+		}
+	}
+}
+
 type statusWriter struct {
 	http.ResponseWriter
 	status int

internal/server/metrics.go

@@ -15,17 +15,17 @@ var (
 	})
 	httpRequests = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "telemt_gateway_http_requests_total",
-		Help: "HTTP requests by status, method, alias.",
-	}, []string{"code", "method", "alias"})
+		Help: "HTTP requests by status, method, alias, endpoint.",
+	}, []string{"code", "method", "alias", "endpoint"})
 	httpDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
 		Name:    "telemt_gateway_http_request_duration_seconds",
 		Help:    "Request duration in seconds.",
 		Buckets: prometheus.DefBuckets,
-	}, []string{"method", "alias"})
+	}, []string{"method", "alias", "endpoint"})
 )
 
-func observeRequest(method, alias string, status int, started time.Time) {
+func observeRequest(method, alias, endpoint string, status int, started time.Time) {
 	httpInFlight.Dec()
-	httpRequests.WithLabelValues(strconv.Itoa(status), method, alias).Inc()
-	httpDuration.WithLabelValues(method, alias).Observe(time.Since(started).Seconds())
+	httpRequests.WithLabelValues(strconv.Itoa(status), method, alias, endpoint).Inc()
+	httpDuration.WithLabelValues(method, alias, endpoint).Observe(time.Since(started).Seconds())
 }