telemt-api/internal/aggregate/incidents.go

package aggregate

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"time"
)

type statsSummaryLite struct {
	ConnectionsBadTotal uint64 `json:"connections_bad_total"`
	ConnectionsTotal    uint64 `json:"connections_total"`
}

func BuildIncidents(ctx context.Context, h *Handler, aliases []string) (IncidentsData, bool) {
	out := IncidentsData{Items: make([]IncidentItem, 0)}
	partial := false

	fleet := FetchFleetStatus(ctx, h.Client, h.Parsed, aliases)
	if fleet.ServersFailed > 0 {
		partial = true
	}

	for _, s := range fleet.Servers {
		if !s.OK {
			severity := SeverityWarning
			if !s.HealthOK && !s.SystemInfoOK {
				severity = SeverityCritical
			}
			out.Items = append(out.Items, IncidentItem{
				ID:              fmt.Sprintf("node_degraded:%s", s.Alias),
				Kind:            "node_degraded",
				Severity:        severity,
				Status:          IncidentStatusFiring,
				Title:           fmt.Sprintf("Нода %s degraded", s.Alias),
				Summary:         "health/system_info не прошли полностью",
				AffectedAliases: []string{s.Alias},
				Actions: []IncidentAction{
					{Label: "Открыть ноду", Href: "/servers/" + s.Alias},
					{Label: "Runtime", Href: "/servers/" + s.Alias + "/runtime"},
				},
			})
		}
		if s.Health != nil && s.Health.ReadOnly {
			out.Items = append(out.Items, IncidentItem{
				ID:              fmt.Sprintf("node_read_only:%s", s.Alias),
				Kind:            "node_read_only",
				Severity:        SeverityWarning,
				Status:          IncidentStatusFiring,
				Title:           fmt.Sprintf("Нода %s в read_only", s.Alias),
				Summary:         "API ноды не принимает mutating операции",
				AffectedAliases: []string{s.Alias},
				Actions: []IncidentAction{
					{Label: "Users", Href: "/servers/" + s.Alias + "/users"},
					{Label: "Security", Href: "/servers/" + s.Alias + "/security"},
				},
			})
		}
	}

	for _, alias := range aliases {
		data, meta := FetchTelemtGET[statsSummaryLite](ctx, h.Client, h.Parsed, alias, "stats/summary")
		if !meta.OK {
			partial = true
			continue
		}
		if data.ConnectionsBadTotal >= 10000 {
			out.Items = append(out.Items, IncidentItem{
				ID:              fmt.Sprintf("bad_connections_high:%s", alias),
				Kind:            "bad_connections_high",
				Severity:        SeverityCritical,
				Status:          IncidentStatusFiring,
				Title:           fmt.Sprintf("Высокий bad connections на %s", alias),
				Summary:         "Резкий рост ошибок клиентских соединений",
				AffectedAliases: []string{alias},
				MetricName:      "connections_bad_total",
				MetricValue:     float64(data.ConnectionsBadTotal),
				MetricThreshold: 10000,
				Actions: []IncidentAction{
					{Label: "Node dashboard", Href: "/servers/" + alias},
				},
			})
		} else if data.ConnectionsBadTotal >= 1000 {
			out.Items = append(out.Items, IncidentItem{
				ID:              fmt.Sprintf("bad_connections_warn:%s", alias),
				Kind:            "bad_connections_warn",
				Severity:        SeverityWarning,
				Status:          IncidentStatusFiring,
				Title:           fmt.Sprintf("Рост bad connections на %s", alias),
				Summary:         "Наблюдается рост ошибок клиентских соединений",
				AffectedAliases: []string{alias},
				MetricName:      "connections_bad_total",
				MetricValue:     float64(data.ConnectionsBadTotal),
				MetricThreshold: 1000,
				Actions: []IncidentAction{
					{Label: "Node dashboard", Href: "/servers/" + alias},
				},
			})
		}
	}

	sort.Slice(out.Items, func(i, j int) bool {
		if severityWeight(out.Items[i].Severity) != severityWeight(out.Items[j].Severity) {
			return severityWeight(out.Items[i].Severity) > severityWeight(out.Items[j].Severity)
		}
		return strings.Compare(out.Items[i].ID, out.Items[j].ID) < 0
	})

	out.Total = len(out.Items)
	for _, it := range out.Items {
		switch it.Severity {
		case SeverityCritical:
			out.CriticalTotal++
		case SeverityWarning:
			out.WarningTotal++
		default:
			out.InfoTotal++
		}
	}
	return out, partial
}

func severityWeight(s IncidentSeverity) int {
	switch s {
	case SeverityCritical:
		return 3
	case SeverityWarning:
		return 2
	case SeverityInfo:
		return 1
	default:
		return 0
	}
}

func BuildLiveEnvelope(ctx context.Context, h *Handler, aliases []string) map[string]any {
	inc, partial := BuildIncidents(ctx, h, aliases)
	status := "healthy"
	if inc.CriticalTotal > 0 {
		status = "critical"
	} else if inc.WarningTotal > 0 {
		status = "degraded"
	}
	return map[string]any{
		"type":         "live_snapshot",
		"timestamp":    time.Now().UTC().Format(time.RFC3339Nano),
		"status":       status,
		"partial":      partial,
		"incidents":    inc.Items,
		"counts":       map[string]int{"total": inc.Total, "critical": inc.CriticalTotal, "warning": inc.WarningTotal, "info": inc.InfoTotal},
		"aliases_used": aliases,
	}
}