package aggregate import ( "context" "fmt" "sort" "strings" "time" ) type statsSummaryLite struct { ConnectionsBadTotal uint64 `json:"connections_bad_total"` ConnectionsTotal uint64 `json:"connections_total"` } func BuildIncidents(ctx context.Context, h *Handler, aliases []string) (IncidentsData, bool) { out := IncidentsData{Items: make([]IncidentItem, 0)} partial := false fleet := FetchFleetStatus(ctx, h.Client, h.Parsed, aliases) if fleet.ServersFailed > 0 { partial = true } for _, s := range fleet.Servers { if !s.OK { severity := SeverityWarning if !s.HealthOK && !s.SystemInfoOK { severity = SeverityCritical } out.Items = append(out.Items, IncidentItem{ ID: fmt.Sprintf("node_degraded:%s", s.Alias), Kind: "node_degraded", Severity: severity, Status: IncidentStatusFiring, Title: fmt.Sprintf("Нода %s degraded", s.Alias), Summary: "health/system_info не прошли полностью", AffectedAliases: []string{s.Alias}, Actions: []IncidentAction{ {Label: "Открыть ноду", Href: "/servers/" + s.Alias}, {Label: "Runtime", Href: "/servers/" + s.Alias + "/runtime"}, }, }) } if s.Health != nil && s.Health.ReadOnly { out.Items = append(out.Items, IncidentItem{ ID: fmt.Sprintf("node_read_only:%s", s.Alias), Kind: "node_read_only", Severity: SeverityWarning, Status: IncidentStatusFiring, Title: fmt.Sprintf("Нода %s в read_only", s.Alias), Summary: "API ноды не принимает mutating операции", AffectedAliases: []string{s.Alias}, Actions: []IncidentAction{ {Label: "Users", Href: "/servers/" + s.Alias + "/users"}, {Label: "Security", Href: "/servers/" + s.Alias + "/security"}, }, }) } } for _, alias := range aliases { data, meta := FetchTelemtGET[statsSummaryLite](ctx, h.Client, h.Parsed, alias, "stats/summary") if !meta.OK { partial = true continue } badEff := h.effectiveConnectionsBad(alias, data.ConnectionsBadTotal) if badEff >= 10000 { out.Items = append(out.Items, IncidentItem{ ID: fmt.Sprintf("bad_connections_high:%s", alias), Kind: "bad_connections_high", Severity: SeverityCritical, Status: IncidentStatusFiring, Title: fmt.Sprintf("Высокий bad connections на %s", alias), Summary: "Резкий рост ошибок клиентских соединений", AffectedAliases: []string{alias}, MetricName: "connections_bad_total", MetricValue: float64(badEff), MetricThreshold: 10000, Actions: []IncidentAction{ {Label: "Node dashboard", Href: "/servers/" + alias}, }, }) } else if badEff >= 1000 { out.Items = append(out.Items, IncidentItem{ ID: fmt.Sprintf("bad_connections_warn:%s", alias), Kind: "bad_connections_warn", Severity: SeverityWarning, Status: IncidentStatusFiring, Title: fmt.Sprintf("Рост bad connections на %s", alias), Summary: "Наблюдается рост ошибок клиентских соединений", AffectedAliases: []string{alias}, MetricName: "connections_bad_total", MetricValue: float64(badEff), MetricThreshold: 1000, Actions: []IncidentAction{ {Label: "Node dashboard", Href: "/servers/" + alias}, }, }) } } sort.Slice(out.Items, func(i, j int) bool { if severityWeight(out.Items[i].Severity) != severityWeight(out.Items[j].Severity) { return severityWeight(out.Items[i].Severity) > severityWeight(out.Items[j].Severity) } return strings.Compare(out.Items[i].ID, out.Items[j].ID) < 0 }) out.Total = len(out.Items) for _, it := range out.Items { switch it.Severity { case SeverityCritical: out.CriticalTotal++ case SeverityWarning: out.WarningTotal++ default: out.InfoTotal++ } } return out, partial } func severityWeight(s IncidentSeverity) int { switch s { case SeverityCritical: return 3 case SeverityWarning: return 2 case SeverityInfo: return 1 default: return 0 } } func BuildLiveEnvelope(ctx context.Context, h *Handler, aliases []string) map[string]any { inc, partial := BuildIncidents(ctx, h, aliases) status := "healthy" if inc.CriticalTotal > 0 { status = "critical" } else if inc.WarningTotal > 0 { status = "degraded" } return map[string]any{ "type": "live_snapshot", "timestamp": time.Now().UTC().Format(time.RFC3339Nano), "status": status, "partial": partial, "incidents": inc.Items, "counts": map[string]int{"total": inc.Total, "critical": inc.CriticalTotal, "warning": inc.WarningTotal, "info": inc.InfoTotal}, "aliases_used": aliases, } }