feat(services): считать сервис живым при одном OK и писать историю failover

Сервис up, если жив хотя бы один IP. Failover показывает Down по FQDN и журнал add/remove A-записей.

Co-authored-by: Cursor <[email protected]>

apps/api/src/routes/services.ts

@@ -78,6 +78,18 @@ export async function serviceRoutes(app: FastifyInstance) {
     };
   });
 
+  app.get("/services/:id/failover-log", async (request) => {
+    const { id } = request.params as { id: string };
+    repos.getService(request.server.db, Number(id));
+    return {
+      items: repos.listFailoverLogForService(
+        request.server.db,
+        Number(id),
+        200,
+      ),
+    };
+  });
+
   app.get("/services/:id/certificates", async (request) => {
     const { id } = request.params as { id: string };
     return certificateService.listServiceCertificates(

apps/api/src/services/service-config-service.ts

@@ -39,6 +39,43 @@ import {
 export type { LbIpRow, LbTargetConfig };
 export { selectActiveIpsByMode };
 
+export function failoverARecordDiff(
+  existingA: readonly string[],
+  desiredIps: readonly string[],
+): { added: string[]; removed: string[] } {
+  const before = new Set(existingA);
+  const after = new Set(desiredIps);
+  return {
+    added: desiredIps.filter((ip) => !before.has(ip)),
+    removed: existingA.filter((ip) => !after.has(ip)),
+  };
+}
+
+function recordFailoverDnsDiff(
+  db: Db,
+  bindingId: number,
+  hostname: string,
+  zoneName: string,
+  existingRecords: DnsRecord[],
+  desiredIps: string[],
+): void {
+  const existingA = existingRecords
+    .filter((record) => record.record_type.toUpperCase() === "A")
+    .map((record) => record.content);
+  const { added, removed } = failoverARecordDiff(existingA, desiredIps);
+  if (added.length === 0 && removed.length === 0) return;
+  const binding = repos.getBinding(db, bindingId);
+  repos.insertFailoverLog(db, {
+    serviceId: binding.service_id,
+    bindingId,
+    fqdn: fqdnToDisplay(hostname, zoneName),
+    entries: [
+      ...added.map((ip) => ({ ip, action: "added" as const })),
+      ...removed.map((ip) => ({ ip, action: "removed" as const })),
+    ],
+  });
+}
+
 export interface ServiceDomainInput {
   fqdn: string;
   target_ips?: string[];
@@ -285,6 +322,11 @@ async function buildView(db: Db, serviceId: number): Promise<ServiceView> {
       if (target_ip_priorities[ip] === undefined) target_ip_priorities[ip] = 1;
     }
 
+    const { config, rows } = getBindingLbState(db, binding.id);
+    const bindingActiveIps = targetCname
+      ? []
+      : selectActiveIpsByMode(config, rows);
+
     return {
       binding_id: binding.id,
       domain_id: binding.domain_id,
@@ -312,13 +354,13 @@ async function buildView(db: Db, serviceId: number): Promise<ServiceView> {
       health_check_aggregate: binding.health_check_aggregate ?? "majority",
       cert_monitoring: binding.cert_monitoring ?? "auto",
       sync_status: aggregateSyncStatus(statuses),
+      active_ips: bindingActiveIps,
     };
   });
 
   const activeIps = new Set<string>();
-  for (const binding of bindings) {
-    const { config, rows } = getBindingLbState(db, binding.id);
-    for (const ip of selectActiveIpsByMode(config, rows)) {
+  for (const domain of domainViews) {
+    for (const ip of domain.active_ips) {
       activeIps.add(ip);
     }
   }
@@ -677,6 +719,14 @@ async function syncBindingADns(
 
   if (desiredIps.length === 0) {
     repos.setBindingDnsRecordId(db, bindingId, null);
+    recordFailoverDnsDiff(
+      db,
+      bindingId,
+      hostname,
+      zoneName,
+      existingRecords,
+      desiredIps,
+    );
     return;
   }
 
@@ -725,6 +775,14 @@ async function syncBindingADns(
   }
 
   repos.setBindingDnsRecordId(db, bindingId, primaryId);
+  recordFailoverDnsDiff(
+    db,
+    bindingId,
+    hostname,
+    zoneName,
+    existingRecords,
+    desiredIps,
+  );
 }
 
 async function cleanupBindingDns(

apps/api/test/failover-log.test.ts

@@ -0,0 +1,112 @@
+import { describe, expect, it } from "vitest";
+import { createMemoryDb, repos, runMigrations } from "@cfdm/db";
+import { buildApp } from "../src/app.js";
+import { loadConfig } from "../src/config.js";
+import { failoverARecordDiff } from "../src/services/service-config-service.js";
+
+async function authHeaders(app: Awaited<ReturnType<typeof buildApp>>) {
+  const config = loadConfig();
+  const res = await app.inject({
+    method: "POST",
+    url: "/api/v1/auth/login",
+    payload: { username: config.adminUsername, password: "admin" },
+  });
+  expect(res.statusCode).toBe(200);
+  const { token } = res.json() as { token: string };
+  return { authorization: `Bearer ${token}` };
+}
+
+describe("failoverARecordDiff", () => {
+  it("diffs added and removed A contents", () => {
+    expect(
+      failoverARecordDiff(
+        ["130.49.213.153", "93.115.203.183"],
+        ["93.115.203.183"],
+      ),
+    ).toEqual({
+      added: [],
+      removed: ["130.49.213.153"],
+    });
+    expect(failoverARecordDiff(["10.0.0.1"], ["10.0.0.1", "10.0.0.2"])).toEqual({
+      added: ["10.0.0.2"],
+      removed: [],
+    });
+  });
+});
+
+describe("GET /services/:id/failover-log", () => {
+  it("returns add/remove rows for the service", async () => {
+    const app = await buildApp({
+      config: { ...loadConfig(), staticDir: null },
+      memory: true,
+    });
+    const headers = await authHeaders(app);
+
+    const domain = repos.createDomain(app.db, null, "rkns.top", "zone-1");
+    const service = repos.createService(app.db, "MSK Hip", "msk-hip");
+    const binding = repos.insertBinding(app.db, domain.id, service.id, "gt", null);
+
+    repos.insertFailoverLog(app.db, {
+      serviceId: service.id,
+      bindingId: binding.id,
+      fqdn: "gt.rkns.top",
+      entries: [
+        { ip: "130.49.213.153", action: "removed" },
+        { ip: "93.115.203.183", action: "added" },
+      ],
+    });
+
+    const res = await app.inject({
+      method: "GET",
+      url: `/api/v1/services/${service.id}/failover-log`,
+      headers,
+    });
+    expect(res.statusCode).toBe(200);
+    const body = res.json() as {
+      items: Array<{ ip: string; fqdn: string; action: string }>;
+    };
+    expect(body.items).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          ip: "130.49.213.153",
+          fqdn: "gt.rkns.top",
+          action: "removed",
+        }),
+        expect.objectContaining({
+          ip: "93.115.203.183",
+          fqdn: "gt.rkns.top",
+          action: "added",
+        }),
+      ]),
+    );
+
+    await app.close();
+  });
+});
+
+describe("failover_log table", () => {
+  it("lists newest first", () => {
+    const { db, sqlite } = createMemoryDb();
+    runMigrations(sqlite);
+
+    const domain = repos.createDomain(db, null, "example.com", "zone-1");
+    const service = repos.createService(db, "Panel", "panel");
+    const binding = repos.insertBinding(db, domain.id, service.id, "panel", null);
+
+    repos.insertFailoverLog(db, {
+      serviceId: service.id,
+      bindingId: binding.id,
+      fqdn: "panel.example.com",
+      entries: [{ ip: "1.1.1.1", action: "removed" }],
+    });
+    repos.insertFailoverLog(db, {
+      serviceId: service.id,
+      bindingId: binding.id,
+      fqdn: "panel.example.com",
+      entries: [{ ip: "1.1.1.1", action: "added" }],
+    });
+
+    const rows = repos.listFailoverLogForService(db, service.id);
+    expect(rows.map((row) => row.action)).toEqual(["added", "removed"]);
+  });
+});

apps/api/test/health-check.test.ts

@@ -397,4 +397,43 @@ describe("CNAME health mapped onto service IPs", () => {
       }),
     ]);
   });
+
+  it("getView is up when any binding IP is up", async () => {
+    const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db");
+    const { getView } = await import("../src/services/service-config-service.js");
+    const { db, sqlite } = createMemoryDb();
+    runMigrations(sqlite);
+
+    const domain = repos.createDomain(db, null, "rkns.top", "zone-id");
+    const service = repos.createService(db, "MSK Hip", "msk-hip");
+    repos.replaceServiceIps(db, service.id, ["10.0.0.1", "10.0.0.2"]);
+    const binding = repos.insertBinding(db, domain.id, service.id, "gt", null);
+    repos.replaceBindingIpsWithMeta(db, binding.id, [
+      { ip: "10.0.0.1", weight: 1, priority: 1 },
+      { ip: "10.0.0.2", weight: 1, priority: 1 },
+    ]);
+    repos.upsertIpHealthStatus(
+      db,
+      "binding",
+      binding.id,
+      "10.0.0.1",
+      "up",
+      12,
+      0,
+      null,
+    );
+    repos.upsertIpHealthStatus(
+      db,
+      "binding",
+      binding.id,
+      "10.0.0.2",
+      "down",
+      null,
+      5,
+      "timeout",
+    );
+
+    const view = await getView(db, service.id);
+    expect(view.health_status).toBe("up");
+  });
 });

apps/web/src/components/failover-timeline.tsx

@@ -13,7 +13,11 @@ import {
 import { HealthCheckBadge } from '@/components/health-check-badge'
 import { EmptyState } from '@/components/empty-state'
 import { formatDate, formatRelative, sqliteUtcToIso } from '@/lib/format'
-import type { FailoverEvent } from '@/lib/failover-events'
+import {
+  failoverEventCopy,
+  type FailoverEvent,
+} from '@/lib/failover-events'
+import type { FailoverLogEntry } from '@/lib/schemas'
 import { cn } from '@cfdm/ui/lib/utils'
 import type { ComponentProps } from 'react'
 
@@ -29,32 +33,44 @@ function failStreakLabel(count: number): string {
   return `${count} ошибок подряд`
 }
 
-function indicatorClass(status: string): string {
-  if (status === 'checking') {
-    return 'border-warning bg-warning/15 group-data-completed/timeline-item:border-warning'
+function indicatorClass(tone: 'down' | 'added' | 'removed'): string {
+  if (tone === 'added') {
+    return 'border-success bg-success/15 group-data-completed/timeline-item:border-success'
   }
   return 'border-destructive bg-destructive/15 group-data-completed/timeline-item:border-destructive'
 }
 
-function separatorClass(status: string): string {
-  if (status === 'checking') return 'bg-warning/25'
+function separatorClass(tone: 'down' | 'added' | 'removed'): string {
+  if (tone === 'added') return 'bg-success/25'
   return 'bg-destructive/25'
 }
 
+function failoverHistoryCopy(item: FailoverLogEntry): string {
+  return item.action === 'removed'
+    ? `${item.ip} убрана с ${item.fqdn}`
+    : `${item.ip} добавлена на ${item.fqdn}`
+}
+
 /**
- * Failover как sibling «Смены статуса»: ReUI Timeline + Badge, не степпер.
+ * Failover как sibling «Смены статуса»: ReUI Timeline + Badge.
  * Preview: https://reui.io/preview/base/components/c-timeline-10
  * Preview: https://reui.io/preview/base/empty-state-12
  * Docs: https://reui.io/docs/components/base/timeline
  * Docs: https://reui.io/docs/components/base/badge
  */
-export function FailoverTimeline({ events }: { events: FailoverEvent[] }) {
-  if (events.length === 0) {
+export function FailoverTimeline({
+  events,
+  history = [],
+}: {
+  events: FailoverEvent[]
+  history?: readonly FailoverLogEntry[]
+}) {
+  if (events.length === 0 && history.length === 0) {
     return (
       <EmptyState
         icon={ShieldCheckIcon}
-        title="Пул в DNS на месте"
-        description="Standby и last-resort не красные — только down, снятые с A-записей"
+        title="Нет инцидентов Failover"
+        description="Нет Down и нет смен A-записей по FQDN"
         stackedIcon={false}
         centered={false}
       />
@@ -62,55 +78,88 @@ export function FailoverTimeline({ events }: { events: FailoverEvent[] }) {
   }
 
   return (
-    <Timeline defaultValue={0} className="gap-0">
-      {events.map((event, index) => {
-        const checkedIso = event.lastCheckAt
-          ? (sqliteUtcToIso(event.lastCheckAt) ?? event.lastCheckAt)
-          : null
-        const isChecking = event.status === 'checking'
+    <div className="flex flex-col gap-4">
+      {events.length > 0 ? (
+        <Timeline defaultValue={0} className="gap-0">
+          {events.map((event, index) => {
+            const checkedIso = event.lastCheckAt
+              ? (sqliteUtcToIso(event.lastCheckAt) ?? event.lastCheckAt)
+              : null
 
-        return (
-          <TimelineItem key={event.id} step={index + 1}>
-            <TimelineSeparator className={separatorClass(event.status)} />
-            <TimelineIndicator className={indicatorClass(event.status)} />
-            <TimelineHeader>
-              <TimelineTitle className="flex flex-wrap items-center gap-2">
-                <span className="font-mono text-sm">{event.address}</span>
-                <HealthCheckBadge
-                  status={event.status as HealthBadgeStatus}
-                  lastError={event.lastFailureReason}
-                  lastCheckedAt={checkedIso}
-                  size="xs"
-                />
-              </TimelineTitle>
-              <TimelineDate>
-                {event.consecutiveFailures > 0
-                  ? failStreakLabel(event.consecutiveFailures)
-                  : null}
-                {checkedIso
-                  ? `${event.consecutiveFailures > 0 ? ' · ' : ''}${formatRelative(checkedIso)} · ${formatDate(checkedIso)}`
-                  : null}
-              </TimelineDate>
-            </TimelineHeader>
-            <TimelineContent className="flex flex-col gap-2">
-              <p className="text-foreground text-sm">
-                {isChecking
-                  ? 'Снята с DNS, идёт восстановление'
-                  : 'Снята с DNS — в A-записях этого адреса нет'}
-              </p>
-              {event.lastFailureReason ? (
-                <code
-                  className={cn(
-                    'bg-muted block overflow-x-auto rounded-md px-2 py-1.5 font-mono text-xs',
-                  )}
-                >
-                  {event.lastFailureReason}
-                </code>
-              ) : null}
-            </TimelineContent>
-          </TimelineItem>
-        )
-      })}
-    </Timeline>
+            return (
+              <TimelineItem key={event.id} step={index + 1}>
+                <TimelineSeparator className={separatorClass('down')} />
+                <TimelineIndicator className={indicatorClass('down')} />
+                <TimelineHeader>
+                  <TimelineTitle className="flex flex-wrap items-center gap-2">
+                    <span className="font-mono text-sm">{event.address}</span>
+                    <HealthCheckBadge
+                      status={event.status as HealthBadgeStatus}
+                      lastError={event.lastFailureReason}
+                      lastCheckedAt={checkedIso}
+                      size="xs"
+                    />
+                  </TimelineTitle>
+                  <TimelineDate>
+                    {event.consecutiveFailures > 0
+                      ? failStreakLabel(event.consecutiveFailures)
+                      : null}
+                    {checkedIso
+                      ? `${event.consecutiveFailures > 0 ? ' · ' : ''}${formatRelative(checkedIso)} · ${formatDate(checkedIso)}`
+                      : null}
+                  </TimelineDate>
+                </TimelineHeader>
+                <TimelineContent className="flex flex-col gap-2">
+                  <p className="text-foreground text-sm">
+                    {failoverEventCopy(event)}
+                  </p>
+                  {event.lastFailureReason ? (
+                    <code
+                      className={cn(
+                        'bg-muted block overflow-x-auto rounded-md px-2 py-1.5 font-mono text-xs',
+                      )}
+                    >
+                      {event.lastFailureReason}
+                    </code>
+                  ) : null}
+                </TimelineContent>
+              </TimelineItem>
+            )
+          })}
+        </Timeline>
+      ) : null}
+
+      {history.length > 0 ? (
+        <Timeline defaultValue={0} className="gap-0">
+          {history.map((item, index) => {
+            const checkedIso = sqliteUtcToIso(item.created_at) ?? item.created_at
+            const tone = item.action === 'added' ? 'added' : 'removed'
+
+            return (
+              <TimelineItem key={item.id} step={index + 1}>
+                <TimelineSeparator className={separatorClass(tone)} />
+                <TimelineIndicator className={indicatorClass(tone)} />
+                <TimelineHeader>
+                  <TimelineTitle className="flex flex-wrap items-center gap-2">
+                    <span className="font-mono text-sm">{item.ip}</span>
+                    <span className="text-muted-foreground text-xs">
+                      {item.fqdn}
+                    </span>
+                  </TimelineTitle>
+                  <TimelineDate>
+                    {formatRelative(checkedIso)} · {formatDate(checkedIso)}
+                  </TimelineDate>
+                </TimelineHeader>
+                <TimelineContent>
+                  <p className="text-foreground text-sm">
+                    {failoverHistoryCopy(item)}
+                  </p>
+                </TimelineContent>
+              </TimelineItem>
+            )
+          })}
+        </Timeline>
+      ) : null}
+    </div>
   )
 }

apps/web/src/components/reui-kit/service-failover-panel.tsx

@@ -3,8 +3,10 @@ import { UnplugIcon } from 'lucide-react'
 import { FailoverTimeline } from '@/components/failover-timeline'
 import {
   toFailoverEvents,
+  type FailoverBindingPool,
   type FailoverHealthInput,
 } from '@/lib/failover-events'
+import type { FailoverLogEntry } from '@/lib/schemas'
 import { Badge } from '@/components/reui/badge'
 import {
   Frame,
@@ -22,15 +24,15 @@ import {
 function failoverCountLabel(count: number): string {
   const mod10 = count % 10
   const mod100 = count % 100
-  if (mod10 === 1 && mod100 !== 11) return `${count} нода вне пула`
+  if (mod10 === 1 && mod100 !== 11) return `${count} адрес Down`
   if (mod10 >= 2 && mod10 <= 4 && (mod100 < 12 || mod100 > 14)) {
-    return `${count} ноды вне пула`
+    return `${count} адреса Down`
   }
-  return `${count} нод вне пула`
+  return `${count} адресов Down`
 }
 
 /**
- * Failover — sibling ServiceHealthMonitor: Frame stacked + Alert + Timeline.
+ * Failover — текущие Down + журнал add/remove по FQDN.
  * Preview: https://reui.io/preview/base/components/c-timeline-10
  * Preview: https://reui.io/preview/base/empty-state-12
  * Docs: https://reui.io/docs/components/base/frame
@@ -40,12 +42,15 @@ function failoverCountLabel(count: number): string {
  */
 export function ServiceFailoverPanel({
   ipHealth,
-  activeAddresses,
+  bindings,
+  history,
 }: {
   ipHealth: readonly FailoverHealthInput[]
-  activeAddresses: readonly string[]
+  bindings: readonly FailoverBindingPool[]
+  history: readonly FailoverLogEntry[]
 }) {
-  const events = toFailoverEvents(ipHealth, activeAddresses)
+  const events = toFailoverEvents(ipHealth, bindings)
+  const removedCount = events.filter((event) => event.kind === 'removed').length
 
   return (
     <Frame stacked spacing="sm" className="min-w-0 w-full">
@@ -64,7 +69,7 @@ export function ServiceFailoverPanel({
             )}
           </FrameTitle>
           <FrameDescription>
-            Только down из IP Health вне DNS-пула · как таблица активов
+            Текущие Down и история A-записей по FQDN
           </FrameDescription>
         </FrameHeader>
 
@@ -73,12 +78,14 @@ export function ServiceFailoverPanel({
             <UnplugIcon aria-hidden="true" />
             <AlertTitle>{failoverCountLabel(events.length)}</AlertTitle>
             <AlertDescription>
-              Эти IP сняты с A-записей
+              {removedCount > 0
+                ? 'Сняты с FQDN или остались last-resort'
+                : 'Остались в A-записях как last-resort'}
             </AlertDescription>
           </Alert>
         ) : null}
 
-        <FailoverTimeline events={events} />
+        <FailoverTimeline events={events} history={history} />
       </FramePanel>
     </Frame>
   )

apps/web/src/lib/failover-events.test.ts

@@ -1,8 +1,10 @@
 import { describe, expect, it } from 'vitest'
 
 import {
+  failoverEventCopy,
   isFailoverEventStatus,
   toFailoverEvents,
+  type FailoverBindingPool,
   type FailoverHealthInput,
 } from '@/lib/failover-events'
 
@@ -18,6 +20,24 @@ function row(
   }
 }
 
+const mskHip: FailoverBindingPool[] = [
+  {
+    fqdn: 'gt.rkns.top',
+    configured: ['130.49.213.153', '93.115.203.183'],
+    active: ['93.115.203.183'],
+  },
+  {
+    fqdn: 'nsgt.rkns.top',
+    configured: ['130.49.213.153'],
+    active: ['130.49.213.153'],
+  },
+  {
+    fqdn: 'rutg.rkns.top',
+    configured: ['93.115.203.183'],
+    active: ['93.115.203.183'],
+  },
+]
+
 describe('toFailoverEvents', () => {
   it('инцидент только при binding down, не при unhealthy ноды', () => {
     expect(isFailoverEventStatus('down')).toBe(true)
@@ -27,7 +47,7 @@ describe('toFailoverEvents', () => {
     expect(isFailoverEventStatus('unhealthy')).toBe(false)
   })
 
-  it('без DNS-пула ничего не показывает — нельзя врать про вывод', () => {
+  it('Down всегда виден, даже без A-записей', () => {
     const events = toFailoverEvents([
       row({
         ip: '130.49.213.153',
@@ -36,7 +56,19 @@ describe('toFailoverEvents', () => {
         last_error: 'fetch failed',
       }),
     ])
-    expect(events).toEqual([])
+    expect(events).toEqual([
+      {
+        id: '130.49.213.153',
+        address: '130.49.213.153',
+        status: 'down',
+        kind: 'last-resort',
+        fqdns: [],
+        consecutiveFailures: 9,
+        lastFailureReason: 'fetch failed',
+        lastCheckAt: null,
+      },
+    ])
+    expect(failoverEventCopy(events[0]!)).toBe('Down, в A-записях last-resort')
   })
 
   it('OK вне пула не инцидент (standby)', () => {
@@ -45,7 +77,13 @@ describe('toFailoverEvents', () => {
         row({ ip: '10.0.0.1', status: 'up' }),
         row({ ip: '130.49.213.153', status: 'up' }),
       ],
-      ['10.0.0.1'],
+      [
+        {
+          fqdn: 'pool.example.com',
+          configured: ['10.0.0.1', '130.49.213.153'],
+          active: ['10.0.0.1'],
+        },
+      ],
     )
     expect(events).toEqual([])
   })
@@ -57,12 +95,18 @@ describe('toFailoverEvents', () => {
         row({ ip: '10.0.0.2', status: 'unknown' }),
         row({ ip: '10.0.0.3', status: 'degraded' }),
       ],
-      ['10.0.0.1'],
+      [
+        {
+          fqdn: 'pool.example.com',
+          configured: ['10.0.0.1', '10.0.0.2', '10.0.0.3'],
+          active: ['10.0.0.1'],
+        },
+      ],
     )
     expect(events).toEqual([])
   })
 
-  it('down в DNS-пуле не инцидент (last-resort / ещё в A-записи)', () => {
+  it('Down на доп. FQDN last-resort, снятая с общего пула', () => {
     const events = toFailoverEvents(
       [
         row({
@@ -70,15 +114,46 @@ describe('toFailoverEvents', () => {
           status: 'down',
           consecutive_failures: 9,
           last_error: 'fetch failed',
+          last_checked_at: '2026-08-20 07:00:00',
         }),
-        row({ ip: '10.0.0.3', status: 'down' }),
+        row({ ip: '93.115.203.183', status: 'up' }),
       ],
-      ['130.49.213.153', '10.0.0.2'],
+      mskHip,
     )
-    expect(events.map((event) => event.address)).toEqual(['10.0.0.3'])
+    expect(events).toEqual([
+      {
+        id: '130.49.213.153',
+        address: '130.49.213.153',
+        status: 'down',
+        kind: 'removed',
+        fqdns: ['gt.rkns.top'],
+        consecutiveFailures: 9,
+        lastFailureReason: 'fetch failed',
+        lastCheckAt: '2026-08-20 07:00:00',
+      },
+    ])
+    expect(failoverEventCopy(events[0]!)).toBe('Снята с gt.rkns.top')
   })
 
-  it('down без A-записи — реальный вывод из пула', () => {
+  it('Down только last-resort на своём FQDN', () => {
+    const events = toFailoverEvents(
+      [row({ ip: '130.49.213.153', status: 'down' })],
+      [
+        {
+          fqdn: 'nsgt.rkns.top',
+          configured: ['130.49.213.153'],
+          active: ['130.49.213.153'],
+        },
+      ],
+    )
+    expect(events[0]?.kind).toBe('last-resort')
+    expect(events[0]?.fqdns).toEqual(['nsgt.rkns.top'])
+    expect(failoverEventCopy(events[0]!)).toBe(
+      'Down, в A-записях last-resort на nsgt.rkns.top',
+    )
+  })
+
+  it('down без A-записи на configured FQDN — снятие', () => {
     const events = toFailoverEvents(
       [
         row({
@@ -89,13 +164,21 @@ describe('toFailoverEvents', () => {
           last_checked_at: '2026-08-20 07:00:00',
         }),
       ],
-      ['10.0.0.1'],
+      [
+        {
+          fqdn: 'pool.example.com',
+          configured: ['10.0.0.1', '130.49.213.153'],
+          active: ['10.0.0.1'],
+        },
+      ],
     )
     expect(events).toEqual([
       {
         id: '130.49.213.153',
         address: '130.49.213.153',
         status: 'down',
+        kind: 'removed',
+        fqdns: ['pool.example.com'],
         consecutiveFailures: 9,
         lastFailureReason: 'fetch failed',
         lastCheckAt: '2026-08-20 07:00:00',

apps/web/src/lib/failover-events.ts

@@ -1,7 +1,11 @@
+export type FailoverEventKind = 'removed' | 'last-resort'
+
 export interface FailoverEvent {
   id: string
   address: string
   status: string
+  kind: FailoverEventKind
+  fqdns: string[]
   consecutiveFailures: number
   lastFailureReason: string | null
   lastCheckAt?: string | null
@@ -16,31 +20,56 @@ export interface FailoverHealthInput {
   last_checked_at?: string | null
 }
 
+export interface FailoverBindingPool {
+  fqdn: string
+  configured: readonly string[]
+  active: readonly string[]
+}
+
 /** Инцидент только при down. up / degraded / unknown — не вывод из пула. */
 export function isFailoverEventStatus(status: string): boolean {
   return status === 'down'
 }
 
+export function failoverEventCopy(event: FailoverEvent): string {
+  if (event.kind === 'removed') {
+    return event.fqdns.length > 0
+      ? `Снята с ${event.fqdns.join(', ')}`
+      : 'Снята с DNS'
+  }
+  return event.fqdns.length > 0
+    ? `Down, в A-записях last-resort на ${event.fqdns.join(', ')}`
+    : 'Down, в A-записях last-resort'
+}
+
 /**
- * Инцидент failover = binding health down и адреса нет в DNS-пуле.
- * OK / unknown / degraded вне пула — standby, не инцидент.
- * Down в activeAddresses (last-resort) — не «снята с DNS».
+ * Текущие Down всегда в панели. Per-FQDN: снята с hostname или last-resort.
+ * Standby (up / degraded / unknown) — не инцидент.
  */
 export function toFailoverEvents(
   ipHealth: readonly FailoverHealthInput[],
-  activeAddresses: readonly string[] = [],
+  bindings: readonly FailoverBindingPool[] = [],
 ): FailoverEvent[] {
-  const pool = new Set(activeAddresses)
-  if (pool.size === 0) return []
-
-  return ipHealth
-    .filter((row) => isFailoverEventStatus(row.status) && !pool.has(row.ip))
-    .map((row) => ({
+  return ipHealth.filter((row) => isFailoverEventStatus(row.status)).map((row) => {
+    const removedFqdns: string[] = []
+    const lastResortFqdns: string[] = []
+    for (const binding of bindings) {
+      const configured = binding.configured.includes(row.ip)
+      const active = binding.active.includes(row.ip)
+      if (configured && !active) removedFqdns.push(binding.fqdn)
+      else if (configured && active) lastResortFqdns.push(binding.fqdn)
+    }
+    const kind: FailoverEventKind =
+      removedFqdns.length > 0 ? 'removed' : 'last-resort'
+    return {
       id: row.ip,
       address: row.ip,
       status: row.status,
+      kind,
+      fqdns: kind === 'removed' ? removedFqdns : lastResortFqdns,
       consecutiveFailures: row.consecutive_failures ?? 0,
       lastFailureReason: row.last_error ?? null,
       lastCheckAt: row.last_checked_at ?? null,
-    }))
+    }
+  })
 }

apps/web/src/lib/health-log.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, it } from 'vitest'
 
 import {
+  bestAliveHealthStatus,
   collapseStatusChanges,
   enabledHealthProviders,
   providerHealthStatuses,
@@ -73,7 +74,7 @@ describe('enabledHealthProviders', () => {
 })
 
 describe('providerHealthStatuses', () => {
-  it('uses worst latest-per-ip status', () => {
+  it('uses any-up among latest-per-ip statuses', () => {
     const items = [
       probe({ id: 1, ip: '1.1.1.1', status: 'up', checked_at: '2026-01-01T00:02:00Z' }),
       probe({ id: 2, ip: '2.2.2.2', status: 'down', checked_at: '2026-01-01T00:01:00Z' }),
@@ -84,7 +85,7 @@ describe('providerHealthStatuses', () => {
         checked_at: '2026-01-01T00:00:00Z',
       }),
     ]
-    expect(providerHealthStatuses(items, ['local']).local).toBe('down')
+    expect(providerHealthStatuses(items, ['local']).local).toBe('up')
   })
 })
 
@@ -95,3 +96,12 @@ describe('worstHealthStatus', () => {
     expect(worstHealthStatus([])).toBe('unknown')
   })
 })
+
+describe('bestAliveHealthStatus', () => {
+  it('is up if any IP is up', () => {
+    expect(bestAliveHealthStatus(['up', 'down'])).toBe('up')
+    expect(bestAliveHealthStatus(['degraded', 'down'])).toBe('degraded')
+    expect(bestAliveHealthStatus(['down'])).toBe('down')
+    expect(bestAliveHealthStatus([])).toBe('unknown')
+  })
+})

apps/web/src/lib/health-log.ts

@@ -98,7 +98,18 @@ export function worstHealthStatus(statuses: readonly HealthLogStatus[]): HealthL
   )
 }
 
-/** Latest probe per IP for a provider, then worst among those IPs. */
+/** Any up → up; else degraded, then down, then unknown. */
+export function bestAliveHealthStatus(
+  statuses: readonly HealthLogStatus[],
+): HealthLogStatus {
+  if (statuses.length === 0) return 'unknown'
+  if (statuses.some((status) => status === 'up')) return 'up'
+  if (statuses.some((status) => status === 'degraded')) return 'degraded'
+  if (statuses.some((status) => status === 'down')) return 'down'
+  return 'unknown'
+}
+
+/** Latest probe per IP for a provider, then any-up among those IPs. */
 export function providerHealthStatuses(
   items: readonly HealthLogProbe[],
   providers: readonly HealthCheckProvider[],
@@ -120,7 +131,7 @@ export function providerHealthStatuses(
     const statuses = [...latestByIp.values()]
       .filter((item) => item.provider === provider)
       .map((item) => item.status)
-    result[provider] = worstHealthStatus(statuses)
+    result[provider] = bestAliveHealthStatus(statuses)
   }
 
   return result

apps/web/src/lib/schemas.ts

@@ -84,6 +84,7 @@ export const serviceDomainBindingSchema = z
     health_check_aggregate: z.enum(['any', 'all', 'majority']).catch('majority'),
     cert_monitoring: z.enum(['auto', 'required', 'skipped']).default('auto'),
     sync_status: z.string().nullable().default(null),
+    active_ips: z.array(z.string()).default([]),
   })
   .transform((binding) => ({
     ...binding,
@@ -125,6 +126,18 @@ export const healthProbeLogSchema = z.object({
   checked_at: z.string(),
 })
 
+export const failoverLogSchema = z.object({
+  id: z.number(),
+  service_id: z.number(),
+  binding_id: z.number(),
+  fqdn: z.string(),
+  ip: z.string(),
+  action: z.enum(['added', 'removed']),
+  created_at: z.string(),
+})
+
+export type FailoverLogEntry = z.infer<typeof failoverLogSchema>
+
 export const serviceViewSchema = serviceSchema.extend({
   subdomain: z.string().default(''),
   enabled: z.coerce.boolean().default(false),

apps/web/src/queries/services.ts

@@ -1,6 +1,7 @@
 import { queryOptions } from '@tanstack/react-query'
 import { api } from '@/lib/api-client'
 import {
+  failoverLogSchema,
   healthProbeLogSchema,
   serviceBindingSchema,
   serviceGroupsResponseSchema,
@@ -89,6 +90,7 @@ export const serviceDetailKeys = {
   overview: (id: number) => [...serviceKeys.all, id, 'overview'] as const,
   nodes: (id: number) => [...serviceKeys.all, id, 'nodes'] as const,
   healthLog: (id: number) => [...serviceKeys.all, id, 'health-log'] as const,
+  failoverLog: (id: number) => [...serviceKeys.all, id, 'failover-log'] as const,
   view: (id: number) => [...serviceKeys.all, id, 'view'] as const,
 }
 
@@ -110,6 +112,15 @@ export const serviceHealthLogQueryOptions = (id: number) =>
     },
   })
 
+export const serviceFailoverLogQueryOptions = (id: number) =>
+  queryOptions({
+    queryKey: serviceDetailKeys.failoverLog(id),
+    queryFn: async () => {
+      const data = await api.get<unknown>(`/api/v1/services/${id}/failover-log`)
+      return z.object({ items: z.array(failoverLogSchema) }).parse(data)
+    },
+  })
+
 export const serviceOverviewQueryOptions = (id: number) =>
   queryOptions({
     queryKey: serviceDetailKeys.overview(id),

apps/web/src/routes/_auth/services/$serviceId/index.tsx

@@ -44,6 +44,7 @@ import {
   domainsListQueryOptions,
   serviceBindingKeys,
   serviceDetailKeys,
+  serviceFailoverLogQueryOptions,
   serviceGroupKeys,
   serviceGroupsQueryOptions,
   serviceHealthLogQueryOptions,
@@ -90,6 +91,7 @@ function ServiceDetailPage() {
   const viewQuery = useQuery(serviceViewQueryOptions(id))
   const overviewQuery = useQuery(serviceOverviewQueryOptions(id))
   const logQuery = useQuery(serviceHealthLogQueryOptions(id))
+  const failoverLogQuery = useQuery(serviceFailoverLogQueryOptions(id))
   const nodesQuery = useQuery(serviceNodesQueryOptions(id))
   const groupsQuery = useQuery(serviceGroupsQueryOptions())
   const domainsQuery = useQuery(domainsListQueryOptions())
@@ -100,6 +102,19 @@ function ServiceDetailPage() {
     () => logQuery.data?.items ?? [],
     [logQuery.data?.items],
   )
+  const failoverHistory = useMemo(
+    () => failoverLogQuery.data?.items ?? [],
+    [failoverLogQuery.data?.items],
+  )
+  const failoverBindings = useMemo(
+    () =>
+      (service?.domains ?? []).map((domain) => ({
+        fqdn: domain.fqdn,
+        configured: domain.target_ips,
+        active: domain.active_ips ?? [],
+      })),
+    [service?.domains],
+  )
   const nodes = (nodesQuery.data as OverviewPayload['nodes']) ?? []
 
   const [editOpen, setEditOpen] = useState(false)
@@ -129,6 +144,7 @@ function ServiceDetailPage() {
       queryClient.invalidateQueries({ queryKey: serviceDetailKeys.overview(id) }),
       queryClient.invalidateQueries({ queryKey: serviceDetailKeys.nodes(id) }),
       queryClient.invalidateQueries({ queryKey: serviceDetailKeys.healthLog(id) }),
+      queryClient.invalidateQueries({ queryKey: serviceDetailKeys.failoverLog(id) }),
     ])
   }
 
@@ -313,7 +329,8 @@ function ServiceDetailPage() {
             />
             <ServiceFailoverPanel
               ipHealth={service.ip_health}
-              activeAddresses={overview?.active_addresses ?? service.active_ips}
+              bindings={failoverBindings}
+              history={failoverHistory}
             />
           </section>
 

apps/web/src/routes/_auth/services/$serviceId/route.tsx

@@ -1,6 +1,7 @@
 import { createFileRoute, Outlet } from '@tanstack/react-router'
 import { PageShell } from '@/components/page-shell'
 import {
+  serviceFailoverLogQueryOptions,
   serviceHealthLogQueryOptions,
   serviceNodesQueryOptions,
   serviceOverviewQueryOptions,
@@ -14,6 +15,7 @@ export const Route = createFileRoute('/_auth/services/$serviceId')({
       queryClient.ensureQueryData(serviceViewQueryOptions(id)),
       queryClient.ensureQueryData(serviceOverviewQueryOptions(id)),
       queryClient.ensureQueryData(serviceHealthLogQueryOptions(id)),
+      queryClient.ensureQueryData(serviceFailoverLogQueryOptions(id)),
       queryClient.ensureQueryData(serviceNodesQueryOptions(id)),
     ])
     return { breadcrumb: view.name }