fix(services): считать failover по IP Health а не по строке ноды

Панель брала service_nodes, которую затирал group apply.
Теперь тот же binding ip_health, что таблица активов; group не пишет в ноду.

Co-authored-by: Cursor <[email protected]>

apps/api/src/services/health-check-service.ts

@@ -386,7 +386,8 @@ function applyAggregatedStatus(
     { colo, provider: statusProvider },
   );
   const matchedNode = repos.findNodeByIp(db, target.ip);
-  if (matchedNode && matchedNode.enabled) {
+  // Binding-scope only: group apply must not clobber node with its own fetch failed.
+  if (matchedNode && matchedNode.enabled && target.scope === "binding") {
     repos.updateNode(db, matchedNode.id, {
       health_status: node,
       consecutive_failures: failures,

apps/api/test/health-check.test.ts

@@ -291,6 +291,77 @@ describe("health-check state derivation via runAllChecks", () => {
     expect(targets[0]?.ip).toBe("2.59.161.102");
     expect(targets[0]?.hostname).toBe("s.rkns.top");
   });
+
+  it("does not mark node unhealthy when binding majority is OK and group local fails", async () => {
+    const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db");
+    const { db, sqlite } = createMemoryDb();
+    runMigrations(sqlite);
+
+    const tcp = await startTcpServer();
+    try {
+      const domain = repos.createDomain(db, null, "example.com", "zone-id");
+      const group = repos.createServiceGroup(
+        db,
+        "VPN",
+        "vpn",
+        null,
+        "vpn.example.com",
+        {
+          health_check_enabled: true,
+          health_check_type: "http",
+          health_check_port: 1,
+          health_check_timeout_ms: 200,
+          health_check_path: "/",
+        },
+      );
+      const service = repos.createService(db, "Svc", "svc");
+      repos.setServiceGroup(db, service.id, group.id);
+      repos.setServiceEnabled(db, service.id, true);
+      const binding = repos.insertBinding(db, domain.id, service.id, "@", null);
+      repos.updateBindingLbConfig(db, binding.id, {
+        health_check_enabled: true,
+        health_check_type: "tcp",
+        health_check_port: tcp.port,
+        health_check_timeout_ms: 500,
+      });
+      repos.replaceBindingIpsWithMeta(db, binding.id, [
+        { ip: "127.0.0.1", weight: 1, priority: 1 },
+      ]);
+      const node = repos.findNodeByIp(db, "127.0.0.1");
+      expect(node).not.toBeNull();
+
+      await healthCheckService.runAllChecks(db, {
+        probeGapMs: 0,
+        thresholds: {
+          degradedFailures: 1,
+          downFailures: 1,
+          latencyWarnMs: 1000,
+        },
+      });
+
+      const bindingHealth = repos.getIpHealthStatusRow(
+        db,
+        "binding",
+        binding.id,
+        "127.0.0.1",
+      );
+      const groupHealth = repos.getIpHealthStatusRow(
+        db,
+        "group",
+        group.id,
+        "127.0.0.1",
+      );
+      const after = repos.getNode(db, node!.id);
+
+      expect(bindingHealth?.status).toBe("up");
+      expect(groupHealth?.status).toBe("down");
+      expect(after.health_status).toBe("healthy");
+      expect(after.consecutive_failures).toBe(0);
+      expect(after.last_failure_reason).toBeNull();
+    } finally {
+      await new Promise<void>((resolve) => tcp.server.close(() => resolve()));
+    }
+  });
 });
 
 describe("CNAME health mapped onto service IPs", () => {

apps/web/src/components/failover-timeline.tsx

@@ -1,40 +1,116 @@
+import { ShieldCheckIcon } from 'lucide-react'
+
 import {
   Timeline,
   TimelineContent,
+  TimelineDate,
   TimelineHeader,
   TimelineIndicator,
   TimelineItem,
   TimelineSeparator,
   TimelineTitle,
 } from '@/components/reui/timeline'
+import { HealthCheckBadge } from '@/components/health-check-badge'
+import { EmptyState } from '@/components/empty-state'
+import { formatDate, formatRelative, sqliteUtcToIso } from '@/lib/format'
+import type { FailoverEvent } from '@/lib/failover-events'
+import { cn } from '@cfdm/ui/lib/utils'
+import type { ComponentProps } from 'react'
 
-export interface FailoverEvent {
-  id: string
-  title: string
-  detail: string
+type HealthBadgeStatus = ComponentProps<typeof HealthCheckBadge>['status']
+
+function failStreakLabel(count: number): string {
+  const mod10 = count % 10
+  const mod100 = count % 100
+  if (mod10 === 1 && mod100 !== 11) return `${count} ошибка подряд`
+  if (mod10 >= 2 && mod10 <= 4 && (mod100 < 12 || mod100 > 14)) {
+    return `${count} ошибки подряд`
+  }
+  return `${count} ошибок подряд`
 }
 
+function indicatorClass(status: string): string {
+  if (status === 'checking') {
+    return 'border-warning bg-warning/15 group-data-completed/timeline-item:border-warning'
+  }
+  return 'border-destructive bg-destructive/15 group-data-completed/timeline-item:border-destructive'
+}
+
+function separatorClass(status: string): string {
+  if (status === 'checking') return 'bg-warning/25'
+  return 'bg-destructive/25'
+}
+
+/**
+ * Failover как sibling «Смены статуса»: ReUI Timeline + Badge, не степпер.
+ * Preview: https://reui.io/preview/base/components/c-timeline-10
+ * Preview: https://reui.io/preview/base/empty-state-12
+ * Docs: https://reui.io/docs/components/base/timeline
+ * Docs: https://reui.io/docs/components/base/badge
+ */
 export function FailoverTimeline({ events }: { events: FailoverEvent[] }) {
   if (events.length === 0) {
     return (
-      <p className="text-muted-foreground text-sm">
-        Событий failover пока нет.
-      </p>
+      <EmptyState
+        icon={ShieldCheckIcon}
+        title="Пул в DNS на месте"
+        description="Standby и last-resort не красные — только down, снятые с A-записей"
+        stackedIcon={false}
+        centered={false}
+      />
     )
   }
 
   return (
-    <Timeline defaultValue={events.length} className="w-full">
-      {events.map((event, index) => (
-        <TimelineItem key={event.id} step={index + 1}>
-          <TimelineSeparator />
-          <TimelineIndicator />
-          <TimelineHeader>
-            <TimelineTitle>{event.title}</TimelineTitle>
-          </TimelineHeader>
-          <TimelineContent>{event.detail}</TimelineContent>
-        </TimelineItem>
-      ))}
+    <Timeline defaultValue={0} className="gap-0">
+      {events.map((event, index) => {
+        const checkedIso = event.lastCheckAt
+          ? (sqliteUtcToIso(event.lastCheckAt) ?? event.lastCheckAt)
+          : null
+        const isChecking = event.status === 'checking'
+
+        return (
+          <TimelineItem key={event.id} step={index + 1}>
+            <TimelineSeparator className={separatorClass(event.status)} />
+            <TimelineIndicator className={indicatorClass(event.status)} />
+            <TimelineHeader>
+              <TimelineTitle className="flex flex-wrap items-center gap-2">
+                <span className="font-mono text-sm">{event.address}</span>
+                <HealthCheckBadge
+                  status={event.status as HealthBadgeStatus}
+                  lastError={event.lastFailureReason}
+                  lastCheckedAt={checkedIso}
+                  size="xs"
+                />
+              </TimelineTitle>
+              <TimelineDate>
+                {event.consecutiveFailures > 0
+                  ? failStreakLabel(event.consecutiveFailures)
+                  : null}
+                {checkedIso
+                  ? `${event.consecutiveFailures > 0 ? ' · ' : ''}${formatRelative(checkedIso)} · ${formatDate(checkedIso)}`
+                  : null}
+              </TimelineDate>
+            </TimelineHeader>
+            <TimelineContent className="flex flex-col gap-2">
+              <p className="text-foreground text-sm">
+                {isChecking
+                  ? 'Снята с DNS, идёт восстановление'
+                  : 'Снята с DNS — в A-записях этого адреса нет'}
+              </p>
+              {event.lastFailureReason ? (
+                <code
+                  className={cn(
+                    'bg-muted block overflow-x-auto rounded-md px-2 py-1.5 font-mono text-xs',
+                  )}
+                >
+                  {event.lastFailureReason}
+                </code>
+              ) : null}
+            </TimelineContent>
+          </TimelineItem>
+        )
+      })}
     </Timeline>
   )
 }

apps/web/src/components/reui-kit/index.ts

@@ -1,5 +1,6 @@
 export { UptimeChart, type UptimeProbe, type UptimePeriodKey, probeUptimePercent, lastProbeLatency } from './uptime-chart'
 export { ServiceHealthMonitor } from './service-health-monitor'
+export { ServiceFailoverPanel } from './service-failover-panel'
 export { applyFiltersToData, getActiveFilters, renderSingleSelectedLabel } from './filter-utils'
 export { CertStatusChart, GroupDomainsChart } from './dashboard-analytics'
 export { ResourcePage, type ResourcePageProps, type ResourcePageTab } from './resource-page'

apps/web/src/components/reui-kit/service-failover-panel.tsx

@@ -0,0 +1,85 @@
+import { UnplugIcon } from 'lucide-react'
+
+import { FailoverTimeline } from '@/components/failover-timeline'
+import {
+  toFailoverEvents,
+  type FailoverHealthInput,
+} from '@/lib/failover-events'
+import { Badge } from '@/components/reui/badge'
+import {
+  Frame,
+  FrameDescription,
+  FrameHeader,
+  FramePanel,
+  FrameTitle,
+} from '@/components/reui/frame'
+import {
+  Alert,
+  AlertDescription,
+  AlertTitle,
+} from '@/components/reui/alert'
+
+function failoverCountLabel(count: number): string {
+  const mod10 = count % 10
+  const mod100 = count % 100
+  if (mod10 === 1 && mod100 !== 11) return `${count} нода вне пула`
+  if (mod10 >= 2 && mod10 <= 4 && (mod100 < 12 || mod100 > 14)) {
+    return `${count} ноды вне пула`
+  }
+  return `${count} нод вне пула`
+}
+
+/**
+ * Failover — sibling ServiceHealthMonitor: Frame stacked + Alert + Timeline.
+ * Preview: https://reui.io/preview/base/components/c-timeline-10
+ * Preview: https://reui.io/preview/base/empty-state-12
+ * Docs: https://reui.io/docs/components/base/frame
+ * Docs: https://reui.io/docs/components/base/timeline
+ * Docs: https://reui.io/docs/components/base/badge
+ * Docs: https://reui.io/docs/components/base/alert
+ */
+export function ServiceFailoverPanel({
+  ipHealth,
+  activeAddresses,
+}: {
+  ipHealth: readonly FailoverHealthInput[]
+  activeAddresses: readonly string[]
+}) {
+  const events = toFailoverEvents(ipHealth, activeAddresses)
+
+  return (
+    <Frame stacked spacing="sm" className="min-w-0 w-full">
+      <FramePanel className="flex flex-col gap-3">
+        <FrameHeader className="gap-1 px-0 py-0">
+          <FrameTitle className="flex flex-wrap items-center gap-2">
+            Failover
+            {events.length > 0 ? (
+              <Badge variant="destructive-light" size="xs" radius="full">
+                {events.length}
+              </Badge>
+            ) : (
+              <Badge variant="success-light" size="xs" radius="full">
+                OK
+              </Badge>
+            )}
+          </FrameTitle>
+          <FrameDescription>
+            Только down из IP Health вне DNS-пула · как таблица активов
+          </FrameDescription>
+        </FrameHeader>
+
+        {events.length > 0 ? (
+          <Alert variant="destructive">
+            <UnplugIcon aria-hidden="true" />
+            <AlertTitle>{failoverCountLabel(events.length)}</AlertTitle>
+            <AlertDescription>
+              Эти IP сняты с A-записей
+            </AlertDescription>
+          </Alert>
+        ) : null}
+
+        <FailoverTimeline events={events} />
+      </FramePanel>
+    </Frame>
+  )
+}

apps/web/src/lib/failover-events.test.ts

@@ -0,0 +1,105 @@
+import { describe, expect, it } from 'vitest'
+
+import {
+  isFailoverEventStatus,
+  toFailoverEvents,
+  type FailoverHealthInput,
+} from '@/lib/failover-events'
+
+function row(
+  overrides: Partial<FailoverHealthInput> & Pick<FailoverHealthInput, 'ip'>,
+): FailoverHealthInput {
+  return {
+    status: 'up',
+    consecutive_failures: 0,
+    last_error: null,
+    last_checked_at: null,
+    ...overrides,
+  }
+}
+
+describe('toFailoverEvents', () => {
+  it('инцидент только при binding down, не при unhealthy ноды', () => {
+    expect(isFailoverEventStatus('down')).toBe(true)
+    expect(isFailoverEventStatus('up')).toBe(false)
+    expect(isFailoverEventStatus('degraded')).toBe(false)
+    expect(isFailoverEventStatus('unknown')).toBe(false)
+    expect(isFailoverEventStatus('unhealthy')).toBe(false)
+  })
+
+  it('без DNS-пула ничего не показывает — нельзя врать про вывод', () => {
+    const events = toFailoverEvents([
+      row({
+        ip: '130.49.213.153',
+        status: 'down',
+        consecutive_failures: 9,
+        last_error: 'fetch failed',
+      }),
+    ])
+    expect(events).toEqual([])
+  })
+
+  it('OK вне пула не инцидент (standby)', () => {
+    const events = toFailoverEvents(
+      [
+        row({ ip: '10.0.0.1', status: 'up' }),
+        row({ ip: '130.49.213.153', status: 'up' }),
+      ],
+      ['10.0.0.1'],
+    )
+    expect(events).toEqual([])
+  })
+
+  it('unknown и degraded вне пула не инцидент', () => {
+    const events = toFailoverEvents(
+      [
+        row({ ip: '10.0.0.1', status: 'up' }),
+        row({ ip: '10.0.0.2', status: 'unknown' }),
+        row({ ip: '10.0.0.3', status: 'degraded' }),
+      ],
+      ['10.0.0.1'],
+    )
+    expect(events).toEqual([])
+  })
+
+  it('down в DNS-пуле не инцидент (last-resort / ещё в A-записи)', () => {
+    const events = toFailoverEvents(
+      [
+        row({
+          ip: '130.49.213.153',
+          status: 'down',
+          consecutive_failures: 9,
+          last_error: 'fetch failed',
+        }),
+        row({ ip: '10.0.0.3', status: 'down' }),
+      ],
+      ['130.49.213.153', '10.0.0.2'],
+    )
+    expect(events.map((event) => event.address)).toEqual(['10.0.0.3'])
+  })
+
+  it('down без A-записи — реальный вывод из пула', () => {
+    const events = toFailoverEvents(
+      [
+        row({
+          ip: '130.49.213.153',
+          status: 'down',
+          consecutive_failures: 9,
+          last_error: 'fetch failed',
+          last_checked_at: '2026-08-20 07:00:00',
+        }),
+      ],
+      ['10.0.0.1'],
+    )
+    expect(events).toEqual([
+      {
+        id: '130.49.213.153',
+        address: '130.49.213.153',
+        status: 'down',
+        consecutiveFailures: 9,
+        lastFailureReason: 'fetch failed',
+        lastCheckAt: '2026-08-20 07:00:00',
+      },
+    ])
+  })
+})

apps/web/src/lib/failover-events.ts

@@ -0,0 +1,46 @@
+export interface FailoverEvent {
+  id: string
+  address: string
+  status: string
+  consecutiveFailures: number
+  lastFailureReason: string | null
+  lastCheckAt?: string | null
+}
+
+/** Binding IP health — тот же контур, что таблица активов. */
+export interface FailoverHealthInput {
+  ip: string
+  status: string
+  consecutive_failures?: number
+  last_error?: string | null
+  last_checked_at?: string | null
+}
+
+/** Инцидент только при down. up / degraded / unknown — не вывод из пула. */
+export function isFailoverEventStatus(status: string): boolean {
+  return status === 'down'
+}
+
+/**
+ * Инцидент failover = binding health down и адреса нет в DNS-пуле.
+ * OK / unknown / degraded вне пула — standby, не инцидент.
+ * Down в activeAddresses (last-resort) — не «снята с DNS».
+ */
+export function toFailoverEvents(
+  ipHealth: readonly FailoverHealthInput[],
+  activeAddresses: readonly string[] = [],
+): FailoverEvent[] {
+  const pool = new Set(activeAddresses)
+  if (pool.size === 0) return []
+
+  return ipHealth
+    .filter((row) => isFailoverEventStatus(row.status) && !pool.has(row.ip))
+    .map((row) => ({
+      id: row.ip,
+      address: row.ip,
+      status: row.status,
+      consecutiveFailures: row.consecutive_failures ?? 0,
+      lastFailureReason: row.last_error ?? null,
+      lastCheckAt: row.last_checked_at ?? null,
+    }))
+}

apps/web/src/routes/_auth/services/$serviceId/index.tsx

@@ -14,7 +14,6 @@ import {
 import { ChangeDomainSheet } from '@/components/change-domain-sheet'
 import { ChangeIpSheet } from '@/components/change-ip-sheet'
 import { EmptyState } from '@/components/empty-state'
-import { FailoverTimeline } from '@/components/failover-timeline'
 import { FormFieldSimple } from '@/components/form-field'
 import { FormSheet } from '@/components/form-sheet'
 import { HealthCheckBadge } from '@/components/health-check-badge'
@@ -29,15 +28,9 @@ import {
 import { LbModeTile } from '@/components/services/service-unit-card'
 import {
   KpiStatGrid,
+  ServiceFailoverPanel,
   ServiceHealthMonitor,
 } from '@/components/reui-kit'
-import {
-  Frame,
-  FrameDescription,
-  FrameHeader,
-  FramePanel,
-  FrameTitle,
-} from '@/components/reui/frame'
 import { api } from '@/lib/api-client'
 import {
   enabledHealthProviders,
@@ -84,6 +77,7 @@ interface OverviewPayload {
     priority: number
     consecutive_failures: number
     last_failure_reason: string | null
+    last_check_at?: string | null
   }>
 }
 
@@ -209,22 +203,6 @@ function ServiceDetailPage() {
   const isError = viewQuery.isError || overviewQuery.isError
   const error = viewQuery.error ?? overviewQuery.error
 
-  const failoverEvents =
-    (nodes.length > 0 ? nodes : (overview?.nodes ?? []))
-      .filter(
-        (node) =>
-          node.health_status === 'unhealthy' ||
-          node.health_status === 'down' ||
-          node.health_status === 'checking',
-      )
-      .map((node) => ({
-        id: node.address,
-        title: `${node.address}: ${node.health_status}`,
-        detail: node.last_failure_reason
-          ? `${node.last_failure_reason} · fail ${node.consecutive_failures}`
-          : `fail ${node.consecutive_failures}`,
-      }))
-
   const enabledProviders = useMemo(
     () => enabledHealthProviders(service?.domains ?? []),
     [service],
@@ -333,17 +311,10 @@ function ServiceDetailPage() {
               statuses={providerStatuses}
               isLoading={logQuery.isLoading}
             />
-            <Frame dense spacing="sm" className="min-w-0 w-full">
-              <FrameHeader>
-                <FrameTitle>Failover</FrameTitle>
-                <FrameDescription>
-                  Нездоровые ноды и причины последней ошибки
-                </FrameDescription>
-              </FrameHeader>
-              <FramePanel>
-                <FailoverTimeline events={failoverEvents} />
-              </FramePanel>
-            </Frame>
+            <ServiceFailoverPanel
+              ipHealth={service.ip_health}
+              activeAddresses={overview?.active_addresses ?? service.active_ips}
+            />
           </section>
 
           {service.ips.length === 0 && service.domains.length === 0 ? (