feat(services): считать сервис живым при одном OK и писать историю failover

Сервис up, если жив хотя бы один IP. Failover показывает Down по FQDN и журнал add/remove A-записей.

Co-authored-by: Cursor <[email protected]>

apps/api/src/routes/services.ts

@@ -78,6 +78,18 @@ export async function serviceRoutes(app: FastifyInstance) {
     };
   });
 
+  app.get("/services/:id/failover-log", async (request) => {
+    const { id } = request.params as { id: string };
+    repos.getService(request.server.db, Number(id));
+    return {
+      items: repos.listFailoverLogForService(
+        request.server.db,
+        Number(id),
+        200,
+      ),
+    };
+  });
+
   app.get("/services/:id/certificates", async (request) => {
     const { id } = request.params as { id: string };
     return certificateService.listServiceCertificates(

apps/api/src/services/service-config-service.ts

@@ -39,6 +39,43 @@ import {
 export type { LbIpRow, LbTargetConfig };
 export { selectActiveIpsByMode };
 
+export function failoverARecordDiff(
+  existingA: readonly string[],
+  desiredIps: readonly string[],
+): { added: string[]; removed: string[] } {
+  const before = new Set(existingA);
+  const after = new Set(desiredIps);
+  return {
+    added: desiredIps.filter((ip) => !before.has(ip)),
+    removed: existingA.filter((ip) => !after.has(ip)),
+  };
+}
+
+function recordFailoverDnsDiff(
+  db: Db,
+  bindingId: number,
+  hostname: string,
+  zoneName: string,
+  existingRecords: DnsRecord[],
+  desiredIps: string[],
+): void {
+  const existingA = existingRecords
+    .filter((record) => record.record_type.toUpperCase() === "A")
+    .map((record) => record.content);
+  const { added, removed } = failoverARecordDiff(existingA, desiredIps);
+  if (added.length === 0 && removed.length === 0) return;
+  const binding = repos.getBinding(db, bindingId);
+  repos.insertFailoverLog(db, {
+    serviceId: binding.service_id,
+    bindingId,
+    fqdn: fqdnToDisplay(hostname, zoneName),
+    entries: [
+      ...added.map((ip) => ({ ip, action: "added" as const })),
+      ...removed.map((ip) => ({ ip, action: "removed" as const })),
+    ],
+  });
+}
+
 export interface ServiceDomainInput {
   fqdn: string;
   target_ips?: string[];
@@ -285,6 +322,11 @@ async function buildView(db: Db, serviceId: number): Promise<ServiceView> {
       if (target_ip_priorities[ip] === undefined) target_ip_priorities[ip] = 1;
     }
 
+    const { config, rows } = getBindingLbState(db, binding.id);
+    const bindingActiveIps = targetCname
+      ? []
+      : selectActiveIpsByMode(config, rows);
+
     return {
       binding_id: binding.id,
       domain_id: binding.domain_id,
@@ -312,13 +354,13 @@ async function buildView(db: Db, serviceId: number): Promise<ServiceView> {
       health_check_aggregate: binding.health_check_aggregate ?? "majority",
       cert_monitoring: binding.cert_monitoring ?? "auto",
       sync_status: aggregateSyncStatus(statuses),
+      active_ips: bindingActiveIps,
     };
   });
 
   const activeIps = new Set<string>();
-  for (const binding of bindings) {
-    const { config, rows } = getBindingLbState(db, binding.id);
-    for (const ip of selectActiveIpsByMode(config, rows)) {
+  for (const domain of domainViews) {
+    for (const ip of domain.active_ips) {
       activeIps.add(ip);
     }
   }
@@ -677,6 +719,14 @@ async function syncBindingADns(
 
   if (desiredIps.length === 0) {
     repos.setBindingDnsRecordId(db, bindingId, null);
+    recordFailoverDnsDiff(
+      db,
+      bindingId,
+      hostname,
+      zoneName,
+      existingRecords,
+      desiredIps,
+    );
     return;
   }
 
@@ -725,6 +775,14 @@ async function syncBindingADns(
   }
 
   repos.setBindingDnsRecordId(db, bindingId, primaryId);
+  recordFailoverDnsDiff(
+    db,
+    bindingId,
+    hostname,
+    zoneName,
+    existingRecords,
+    desiredIps,
+  );
 }
 
 async function cleanupBindingDns(

apps/api/test/failover-log.test.ts

@@ -0,0 +1,112 @@
+import { describe, expect, it } from "vitest";
+import { createMemoryDb, repos, runMigrations } from "@cfdm/db";
+import { buildApp } from "../src/app.js";
+import { loadConfig } from "../src/config.js";
+import { failoverARecordDiff } from "../src/services/service-config-service.js";
+
+async function authHeaders(app: Awaited<ReturnType<typeof buildApp>>) {
+  const config = loadConfig();
+  const res = await app.inject({
+    method: "POST",
+    url: "/api/v1/auth/login",
+    payload: { username: config.adminUsername, password: "admin" },
+  });
+  expect(res.statusCode).toBe(200);
+  const { token } = res.json() as { token: string };
+  return { authorization: `Bearer ${token}` };
+}
+
+describe("failoverARecordDiff", () => {
+  it("diffs added and removed A contents", () => {
+    expect(
+      failoverARecordDiff(
+        ["130.49.213.153", "93.115.203.183"],
+        ["93.115.203.183"],
+      ),
+    ).toEqual({
+      added: [],
+      removed: ["130.49.213.153"],
+    });
+    expect(failoverARecordDiff(["10.0.0.1"], ["10.0.0.1", "10.0.0.2"])).toEqual({
+      added: ["10.0.0.2"],
+      removed: [],
+    });
+  });
+});
+
+describe("GET /services/:id/failover-log", () => {
+  it("returns add/remove rows for the service", async () => {
+    const app = await buildApp({
+      config: { ...loadConfig(), staticDir: null },
+      memory: true,
+    });
+    const headers = await authHeaders(app);
+
+    const domain = repos.createDomain(app.db, null, "rkns.top", "zone-1");
+    const service = repos.createService(app.db, "MSK Hip", "msk-hip");
+    const binding = repos.insertBinding(app.db, domain.id, service.id, "gt", null);
+
+    repos.insertFailoverLog(app.db, {
+      serviceId: service.id,
+      bindingId: binding.id,
+      fqdn: "gt.rkns.top",
+      entries: [
+        { ip: "130.49.213.153", action: "removed" },
+        { ip: "93.115.203.183", action: "added" },
+      ],
+    });
+
+    const res = await app.inject({
+      method: "GET",
+      url: `/api/v1/services/${service.id}/failover-log`,
+      headers,
+    });
+    expect(res.statusCode).toBe(200);
+    const body = res.json() as {
+      items: Array<{ ip: string; fqdn: string; action: string }>;
+    };
+    expect(body.items).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          ip: "130.49.213.153",
+          fqdn: "gt.rkns.top",
+          action: "removed",
+        }),
+        expect.objectContaining({
+          ip: "93.115.203.183",
+          fqdn: "gt.rkns.top",
+          action: "added",
+        }),
+      ]),
+    );
+
+    await app.close();
+  });
+});
+
+describe("failover_log table", () => {
+  it("lists newest first", () => {
+    const { db, sqlite } = createMemoryDb();
+    runMigrations(sqlite);
+
+    const domain = repos.createDomain(db, null, "example.com", "zone-1");
+    const service = repos.createService(db, "Panel", "panel");
+    const binding = repos.insertBinding(db, domain.id, service.id, "panel", null);
+
+    repos.insertFailoverLog(db, {
+      serviceId: service.id,
+      bindingId: binding.id,
+      fqdn: "panel.example.com",
+      entries: [{ ip: "1.1.1.1", action: "removed" }],
+    });
+    repos.insertFailoverLog(db, {
+      serviceId: service.id,
+      bindingId: binding.id,
+      fqdn: "panel.example.com",
+      entries: [{ ip: "1.1.1.1", action: "added" }],
+    });
+
+    const rows = repos.listFailoverLogForService(db, service.id);
+    expect(rows.map((row) => row.action)).toEqual(["added", "removed"]);
+  });
+});

apps/api/test/health-check.test.ts

@@ -397,4 +397,43 @@ describe("CNAME health mapped onto service IPs", () => {
       }),
     ]);
   });
+
+  it("getView is up when any binding IP is up", async () => {
+    const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db");
+    const { getView } = await import("../src/services/service-config-service.js");
+    const { db, sqlite } = createMemoryDb();
+    runMigrations(sqlite);
+
+    const domain = repos.createDomain(db, null, "rkns.top", "zone-id");
+    const service = repos.createService(db, "MSK Hip", "msk-hip");
+    repos.replaceServiceIps(db, service.id, ["10.0.0.1", "10.0.0.2"]);
+    const binding = repos.insertBinding(db, domain.id, service.id, "gt", null);
+    repos.replaceBindingIpsWithMeta(db, binding.id, [
+      { ip: "10.0.0.1", weight: 1, priority: 1 },
+      { ip: "10.0.0.2", weight: 1, priority: 1 },
+    ]);
+    repos.upsertIpHealthStatus(
+      db,
+      "binding",
+      binding.id,
+      "10.0.0.1",
+      "up",
+      12,
+      0,
+      null,
+    );
+    repos.upsertIpHealthStatus(
+      db,
+      "binding",
+      binding.id,
+      "10.0.0.2",
+      "down",
+      null,
+      5,
+      "timeout",
+    );
+
+    const view = await getView(db, service.id);
+    expect(view.health_status).toBe("up");
+  });
 });