fix(services): показывать живой health и историю выхода из пула

Таблица IP брала hysteresis unknown вместо последней пробы. Failover теперь показывает выход и возврат в пул, а не только DNS add.

Co-authored-by: Cursor <[email protected]>

apps/web/src/lib/failover-events.test.ts

@@ -3,10 +3,14 @@ import { describe, expect, it } from 'vitest'
 import {
   failoverEventCopy,
   isFailoverEventStatus,
+  mergeFailoverHistory,
   toFailoverEvents,
+  toIpAliveTransitions,
   type FailoverBindingPool,
   type FailoverHealthInput,
 } from '@/lib/failover-events'
+import type { FailoverLogEntry } from '@/lib/schemas'
+import type { HealthLogProbe } from '@/lib/health-log'
 
 function row(
   overrides: Partial<FailoverHealthInput> & Pick<FailoverHealthInput, 'ip'>,
@@ -186,3 +190,105 @@ describe('toFailoverEvents', () => {
     ])
   })
 })
+
+function probe(
+  overrides: Partial<HealthLogProbe> & Pick<HealthLogProbe, 'id' | 'status' | 'checked_at'>,
+): HealthLogProbe {
+  return {
+    ip: '130.49.213.153',
+    provider: 'local',
+    ok: overrides.status === 'up',
+    latency_ms: 12,
+    colo: null,
+    error: null,
+    ...overrides,
+  }
+}
+
+function dns(
+  overrides: Partial<FailoverLogEntry> & Pick<FailoverLogEntry, 'id' | 'action' | 'created_at'>,
+): FailoverLogEntry {
+  return {
+    service_id: 13,
+    binding_id: 1,
+    fqdn: 'gt.rkns.top',
+    ip: '130.49.213.153',
+    ...overrides,
+  }
+}
+
+describe('toIpAliveTransitions', () => {
+  it('emits leave then return, not the initial up', () => {
+    const transitions = toIpAliveTransitions([
+      probe({ id: 1, status: 'up', checked_at: '2026-08-20T09:00:00Z' }),
+      probe({ id: 2, status: 'up', checked_at: '2026-08-20T09:01:00Z' }),
+      probe({ id: 3, status: 'down', checked_at: '2026-08-20T09:02:00Z' }),
+      probe({ id: 4, status: 'down', checked_at: '2026-08-20T09:03:00Z' }),
+      probe({ id: 5, status: 'up', checked_at: '2026-08-20T09:04:00Z' }),
+    ])
+    expect(transitions).toEqual([
+      { id: 'probe:3', ip: '130.49.213.153', alive: false, at: '2026-08-20T09:02:00Z' },
+      { id: 'probe:5', ip: '130.49.213.153', alive: true, at: '2026-08-20T09:04:00Z' },
+    ])
+  })
+
+  it('any-up across providers: leave only when every source is down', () => {
+    const transitions = toIpAliveTransitions([
+      probe({
+        id: 1,
+        provider: 'local',
+        status: 'up',
+        checked_at: '2026-08-20T09:00:00Z',
+      }),
+      probe({
+        id: 2,
+        provider: 'cloudflare',
+        status: 'down',
+        checked_at: '2026-08-20T09:01:00Z',
+      }),
+      probe({
+        id: 3,
+        provider: 'local',
+        status: 'down',
+        checked_at: '2026-08-20T09:02:00Z',
+      }),
+      probe({
+        id: 4,
+        provider: 'local',
+        status: 'up',
+        checked_at: '2026-08-20T09:03:00Z',
+      }),
+    ])
+    expect(transitions.map((item) => item.id)).toEqual(['probe:3', 'probe:4'])
+  })
+})
+
+describe('mergeFailoverHistory', () => {
+  it('fills leave/return from probes when DNS has only the add', () => {
+    const history = mergeFailoverHistory(
+      [dns({ id: 10, action: 'added', created_at: '2026-08-20 09:26:00' })],
+      [
+        probe({ id: 1, status: 'up', checked_at: '2026-08-20T09:00:00Z' }),
+        probe({ id: 2, status: 'down', checked_at: '2026-08-20T09:10:00Z' }),
+        probe({ id: 3, status: 'up', checked_at: '2026-08-20T09:26:30Z' }),
+      ],
+      mskHip,
+    )
+    expect(history.map((item) => `${item.action}:${item.source}`)).toEqual([
+      'added:dns',
+      'removed:probe',
+    ])
+    expect(history[0]?.copy).toBe('130.49.213.153 добавлена на gt.rkns.top')
+    expect(history[1]?.copy).toContain('вышла из пула')
+  })
+
+  it('keeps DNS over a probe return in the same 2-minute window', () => {
+    const history = mergeFailoverHistory(
+      [dns({ id: 10, action: 'added', created_at: '2026-08-20 09:26:00' })],
+      [probe({ id: 3, status: 'up', checked_at: '2026-08-20T09:26:30Z' })],
+      mskHip,
+    )
+    expect(history).toHaveLength(1)
+    expect(history[0]?.source).toBe('dns')
+  })
+})

apps/web/src/lib/failover-events.ts

@@ -1,5 +1,23 @@
+import {
+  bestAliveHealthStatus,
+  probeTime,
+  type HealthLogProbe,
+  type HealthLogStatus,
+} from '@/lib/health-log'
+import type { FailoverLogEntry } from '@/lib/schemas'
+
 export type FailoverEventKind = 'removed' | 'last-resort'
 
+export interface FailoverHistoryItem {
+  id: string
+  ip: string
+  fqdn: string
+  action: 'added' | 'removed'
+  created_at: string
+  copy: string
+  source: 'dns' | 'probe'
+}
+
 export interface FailoverEvent {
   id: string
   address: string
@@ -73,3 +91,141 @@ export function toFailoverEvents(
     }
   })
 }
+
+const DEDUPE_WINDOW_MS = 2 * 60 * 1000
+
+function fqdnsForIp(
+  ip: string,
+  bindings: readonly FailoverBindingPool[],
+): string[] {
+  return bindings.filter((binding) => binding.configured.includes(ip)).map((binding) => binding.fqdn)
+}
+
+function fqdnLabel(fqdns: string[]): string {
+  return fqdns.join(', ') || 'пул'
+}
+
+function isAliveStatus(status: HealthLogStatus): boolean {
+  return status === 'up' || status === 'degraded'
+}
+
+/**
+ * Per-IP any-up flips: down → вышла из пула, up после down → вернулась.
+ * Initial state is not an event.
+ */
+export function toIpAliveTransitions(
+  probes: readonly HealthLogProbe[],
+): Array<{ id: string; ip: string; alive: boolean; at: string }> {
+  const byIp = new Map<string, HealthLogProbe[]>()
+  for (const item of probes) {
+    const list = byIp.get(item.ip)
+    if (list) list.push(item)
+    else byIp.set(item.ip, [item])
+  }
+
+  const out: Array<{ id: string; ip: string; alive: boolean; at: string }> = []
+  for (const [ip, list] of byIp) {
+    list.sort(
+      (a, b) => probeTime(a.checked_at) - probeTime(b.checked_at) || a.id - b.id,
+    )
+    const latestByProvider = new Map<string, HealthLogProbe>()
+    let prevAlive: boolean | undefined
+    for (const probe of list) {
+      latestByProvider.set(probe.provider, probe)
+      const status = bestAliveHealthStatus(
+        [...latestByProvider.values()].map((item) => item.status),
+      )
+      if (status === 'unknown') continue
+      const alive = isAliveStatus(status)
+      if (prevAlive !== undefined && alive !== prevAlive) {
+        out.push({
+          id: `probe:${probe.id}`,
+          ip,
+          alive,
+          at: probe.checked_at,
+        })
+      }
+      prevAlive = alive
+    }
+  }
+  return out
+}
+
+function dnsHistoryItem(item: FailoverLogEntry): FailoverHistoryItem {
+  return {
+    id: `dns:${item.id}`,
+    ip: item.ip,
+    fqdn: item.fqdn,
+    action: item.action,
+    created_at: item.created_at,
+    copy:
+      item.action === 'removed'
+        ? `${item.ip} убрана с ${item.fqdn}`
+        : `${item.ip} добавлена на ${item.fqdn}`,
+    source: 'dns',
+  }
+}
+
+function probeHistoryItem(
+  transition: { id: string; ip: string; alive: boolean; at: string },
+  bindings: readonly FailoverBindingPool[],
+): FailoverHistoryItem {
+  const fqdns = fqdnsForIp(transition.ip, bindings)
+  const fqdn = fqdnLabel(fqdns)
+  const action = transition.alive ? 'added' : 'removed'
+  return {
+    id: transition.id,
+    ip: transition.ip,
+    fqdn,
+    action,
+    created_at: transition.at,
+    copy:
+      action === 'removed'
+        ? `${transition.ip} вышла из пула (${fqdn})`
+        : `${transition.ip} вернулась в пул (${fqdn})`,
+    source: 'probe',
+  }
+}
+
+function eventTime(value: string): number {
+  return probeTime(value)
+}
+
+/**
+ * DNS add/remove + health leave/return, newest first.
+ * Same IP+action within 2 minutes: keep the DNS row (it has a concrete FQDN).
+ */
+export function mergeFailoverHistory(
+  dns: readonly FailoverLogEntry[],
+  probes: readonly HealthLogProbe[],
+  bindings: readonly FailoverBindingPool[] = [],
+): FailoverHistoryItem[] {
+  const merged = [
+    ...dns.map(dnsHistoryItem),
+    ...toIpAliveTransitions(probes).map((transition) =>
+      probeHistoryItem(transition, bindings),
+    ),
+  ]
+  merged.sort(
+    (a, b) => eventTime(b.created_at) - eventTime(a.created_at) || a.id.localeCompare(b.id),
+  )
+
+  const kept: FailoverHistoryItem[] = []
+  for (const item of merged) {
+    const duplicate = kept.find(
+      (other) =>
+        other.ip === item.ip &&
+        other.action === item.action &&
+        Math.abs(eventTime(other.created_at) - eventTime(item.created_at)) <=
+          DEDUPE_WINDOW_MS,
+    )
+    if (!duplicate) {
+      kept.push(item)
+      continue
+    }
+    if (duplicate.source === 'probe' && item.source === 'dns') {
+      kept[kept.indexOf(duplicate)] = item
+    }
+  }
+  return kept
+}

apps/web/src/lib/health-log.test.ts

@@ -4,7 +4,9 @@ import {
   bestAliveHealthStatus,
   collapseStatusChanges,
   enabledHealthProviders,
+  latestHealthByIp,
   providerHealthStatuses,
+  resolveIpDisplayHealth,
   worstHealthStatus,
   type HealthLogProbe,
 } from '@/lib/health-log'
@@ -105,3 +107,42 @@ describe('bestAliveHealthStatus', () => {
     expect(bestAliveHealthStatus([])).toBe('unknown')
   })
 })
+
+describe('latestHealthByIp', () => {
+  it('any-up among latest-per-provider probes', () => {
+    const items = [
+      probe({
+        id: 1,
+        ip: '130.49.213.153',
+        provider: 'local',
+        status: 'up',
+        checked_at: '2026-08-20T09:00:00Z',
+      }),
+      probe({
+        id: 2,
+        ip: '130.49.213.153',
+        provider: 'cloudflare',
+        status: 'down',
+        checked_at: '2026-08-20T09:00:01Z',
+      }),
+      probe({
+        id: 3,
+        ip: '93.115.203.183',
+        status: 'unknown',
+        checked_at: '2026-08-20T08:59:00Z',
+      }),
+    ]
+    const byIp = latestHealthByIp(items)
+    expect(byIp.get('130.49.213.153')?.status).toBe('up')
+    expect(byIp.get('93.115.203.183')?.status).toBe('unknown')
+  })
+})
+
+describe('resolveIpDisplayHealth', () => {
+  it('prefers a live probe over stored unknown', () => {
+    expect(resolveIpDisplayHealth('unknown', 'up')).toBe('up')
+    expect(resolveIpDisplayHealth('down', 'up')).toBe('up')
+    expect(resolveIpDisplayHealth('up', 'unknown')).toBe('up')
+    expect(resolveIpDisplayHealth('unknown', undefined)).toBe('unknown')
+  })
+})

apps/web/src/lib/health-log.ts

@@ -136,3 +136,61 @@ export function providerHealthStatuses(
 
   return result
 }
+
+export interface IpDisplayHealth {
+  status: HealthLogStatus
+  latency_ms: number | null
+  last_checked_at: string
+  last_error: string | null
+  colo: string | null
+  provider: HealthCheckProvider
+}
+
+/**
+ * Latest probe per provider+IP, then any-up among those providers.
+ * Used by the IP table so hysteresis `unknown` in ip_health does not hide a live OK.
+ */
+export function latestHealthByIp(
+  items: readonly HealthLogProbe[],
+): Map<string, IpDisplayHealth> {
+  const latest = new Map<string, HealthLogProbe>()
+  const sorted = [...items].sort(
+    (a, b) => probeTime(b.checked_at) - probeTime(a.checked_at) || b.id - a.id,
+  )
+  for (const item of sorted) {
+    const key = `${item.provider}\0${item.ip}`
+    if (!latest.has(key)) latest.set(key, item)
+  }
+
+  const byIp = new Map<string, HealthLogProbe[]>()
+  for (const item of latest.values()) {
+    const list = byIp.get(item.ip)
+    if (list) list.push(item)
+    else byIp.set(item.ip, [item])
+  }
+
+  const result = new Map<string, IpDisplayHealth>()
+  for (const [ip, probes] of byIp) {
+    const status = bestAliveHealthStatus(probes.map((probe) => probe.status))
+    const preferred =
+      probes.find((probe) => probe.status === status) ?? probes[0]!
+    result.set(ip, {
+      status,
+      latency_ms: preferred.latency_ms,
+      last_checked_at: preferred.checked_at,
+      last_error: preferred.error,
+      colo: preferred.colo,
+      provider: preferred.provider,
+    })
+  }
+  return result
+}
+
+/** Prefer a concrete live probe over stored hysteresis `unknown`. */
+export function resolveIpDisplayHealth(
+  stored: HealthLogStatus | undefined,
+  live: HealthLogStatus | undefined,
+): HealthLogStatus {
+  if (live && live !== 'unknown') return live
+  return stored ?? live ?? 'unknown'
+}