feat(services): считать сервис живым при одном OK и писать историю failover

Сервис up, если жив хотя бы один IP. Failover показывает Down по FQDN и журнал add/remove A-записей.

Co-authored-by: Cursor <[email protected]>

packages/db/dist/index.js

@@ -355,6 +355,15 @@ var notificationLog = sqliteTable("notification_log", {
   message: text("message").notNull(),
   created_at: text("created_at").notNull().default(sql`datetime('now')`)
 });
+var failoverLog = sqliteTable("failover_log", {
+  id: integer("id").primaryKey({ autoIncrement: true }),
+  service_id: integer("service_id").notNull().references(() => services.id, { onDelete: "cascade" }),
+  binding_id: integer("binding_id").notNull().references(() => serviceBindings.id, { onDelete: "cascade" }),
+  fqdn: text("fqdn").notNull(),
+  ip: text("ip").notNull(),
+  action: text("action").notNull(),
+  created_at: text("created_at").notNull().default(sql`datetime('now')`)
+});
 var auditLog = sqliteTable("audit_log", {
   id: text("id").primaryKey(),
   event_id: text("event_id"),
@@ -395,6 +404,7 @@ var schema = {
   domainMonitorResults,
   healthProbeLog,
   notificationLog,
+  failoverLog,
   auditLog
 };
 
@@ -715,6 +725,7 @@ __export(repos_exports, {
   getSyncJob: () => getSyncJob,
   insertBinding: () => insertBinding,
   insertDnsRecord: () => insertDnsRecord,
+  insertFailoverLog: () => insertFailoverLog,
   insertHealthProbeLog: () => insertHealthProbeLog,
   insertNotificationLog: () => insertNotificationLog,
   linkBindingRecord: () => linkBindingRecord,
@@ -738,6 +749,7 @@ __export(repos_exports, {
   listDomains: () => listDomains,
   listDomainsEnriched: () => listDomainsEnriched,
   listEnabledDomainMonitors: () => listEnabledDomainMonitors,
+  listFailoverLogForService: () => listFailoverLogForService,
   listGroupDnsRecords: () => listGroupDnsRecords,
   listGroups: () => listGroups,
   listHealthCheckTargets: () => listHealthCheckTargets,
@@ -1995,6 +2007,12 @@ var WORST_HEALTH_SQL = sql2.raw(`CASE
   END) = 1 THEN 'up'
   ELSE 'unknown'
 END`);
+var BEST_ALIVE_HEALTH_SQL = sql2.raw(`CASE
+  WHEN MAX(CASE WHEN status = 'up' THEN 1 ELSE 0 END) = 1 THEN 'up'
+  WHEN MAX(CASE WHEN status = 'degraded' THEN 1 ELSE 0 END) = 1 THEN 'degraded'
+  WHEN MAX(CASE WHEN status = 'down' THEN 1 ELSE 0 END) = 1 THEN 'down'
+  ELSE 'unknown'
+END`);
 function aggregateIpHealthByRefs(db, scope, refIds) {
   const result = /* @__PURE__ */ new Map();
   if (refIds.length === 0) return result;
@@ -2055,7 +2073,7 @@ function aggregateIpHealthByServiceIds(db, serviceIds) {
   );
   const rows = db.all(sql2`
     SELECT sb.service_id AS service_id,
-      ${WORST_HEALTH_SQL} AS health_status,
+      ${BEST_ALIVE_HEALTH_SQL} AS health_status,
       MAX(ihs.latency_ms) AS health_latency_ms
     FROM ip_health_status ihs
     INNER JOIN service_bindings sb
@@ -2545,6 +2563,26 @@ function listNotificationLog(db, limit = 50) {
     LIMIT ${limit}
   `);
 }
+function insertFailoverLog(db, input) {
+  for (const entry of input.entries) {
+    db.insert(failoverLog).values({
+      service_id: input.serviceId,
+      binding_id: input.bindingId,
+      fqdn: input.fqdn,
+      ip: entry.ip,
+      action: entry.action
+    }).run();
+  }
+}
+function listFailoverLogForService(db, serviceId, limit = 100) {
+  return db.all(sql2`
+    SELECT id, service_id, binding_id, fqdn, ip, action, created_at
+    FROM failover_log
+    WHERE service_id = ${serviceId}
+    ORDER BY created_at DESC, id DESC
+    LIMIT ${limit}
+  `);
+}
 export {
   ConflictError,
   NotFoundError,
@@ -2560,6 +2598,7 @@ export {
   domainMonitors,
   domainTags,
   domains,
+  failoverLog,
   getAppSettings,
   getAppSettingsSecrets,
   groups,

packages/db/migrations/025_failover_log.sql

@@ -0,0 +1,11 @@
+CREATE TABLE failover_log (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  service_id INTEGER NOT NULL REFERENCES services(id) ON DELETE CASCADE,
+  binding_id INTEGER NOT NULL REFERENCES service_bindings(id) ON DELETE CASCADE,
+  fqdn TEXT NOT NULL,
+  ip TEXT NOT NULL,
+  action TEXT NOT NULL CHECK (action IN ('added', 'removed')),
+  created_at TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+CREATE INDEX idx_failover_log_service ON failover_log(service_id, created_at DESC);

packages/db/src/repos.ts

@@ -49,6 +49,7 @@ import {
   nodes,
   bindingNodes,
   notificationLog,
+  failoverLog,
   serviceBindingIps,
   serviceBindingRecords,
   serviceBindings,
@@ -2163,6 +2164,14 @@ const WORST_HEALTH_SQL = sql.raw(`CASE
   ELSE 'unknown'
 END`);
 
+/** Service is up if any binding IP is up. */
+const BEST_ALIVE_HEALTH_SQL = sql.raw(`CASE
+  WHEN MAX(CASE WHEN status = 'up' THEN 1 ELSE 0 END) = 1 THEN 'up'
+  WHEN MAX(CASE WHEN status = 'degraded' THEN 1 ELSE 0 END) = 1 THEN 'degraded'
+  WHEN MAX(CASE WHEN status = 'down' THEN 1 ELSE 0 END) = 1 THEN 'down'
+  ELSE 'unknown'
+END`);
+
 /** Worst status across ip_health_status rows for each ref_id in a scope. */
 export function aggregateIpHealthByRefs(
   db: Db,
@@ -2236,7 +2245,7 @@ export function aggregateGroupScopeHealthByIds(
   return result;
 }
 
-/** Worst binding-scope health rolled up per service_id. */
+/** Binding-scope health rolled up per service_id: any up → service up. */
 export function aggregateIpHealthByServiceIds(
   db: Db,
   serviceIds: number[],
@@ -2253,7 +2262,7 @@ export function aggregateIpHealthByServiceIds(
     health_latency_ms: number | null;
   }>(sql`
     SELECT sb.service_id AS service_id,
-      ${WORST_HEALTH_SQL} AS health_status,
+      ${BEST_ALIVE_HEALTH_SQL} AS health_status,
       MAX(ihs.latency_ms) AS health_latency_ms
     FROM ip_health_status ihs
     INNER JOIN service_bindings sb
@@ -3055,3 +3064,51 @@ export function listNotificationLog(
   `);
 }
 
+export type FailoverLogAction = "added" | "removed";
+
+export type FailoverLogRow = {
+  id: number;
+  service_id: number;
+  binding_id: number;
+  fqdn: string;
+  ip: string;
+  action: FailoverLogAction;
+  created_at: string;
+};
+
+export function insertFailoverLog(
+  db: Db,
+  input: {
+    serviceId: number;
+    bindingId: number;
+    fqdn: string;
+    entries: Array<{ ip: string; action: FailoverLogAction }>;
+  },
+): void {
+  for (const entry of input.entries) {
+    db.insert(failoverLog)
+      .values({
+        service_id: input.serviceId,
+        binding_id: input.bindingId,
+        fqdn: input.fqdn,
+        ip: entry.ip,
+        action: entry.action,
+      })
+      .run();
+  }
+}
+
+export function listFailoverLogForService(
+  db: Db,
+  serviceId: number,
+  limit = 100,
+): FailoverLogRow[] {
+  return db.all<FailoverLogRow>(sql`
+    SELECT id, service_id, binding_id, fqdn, ip, action, created_at
+    FROM failover_log
+    WHERE service_id = ${serviceId}
+    ORDER BY created_at DESC, id DESC
+    LIMIT ${limit}
+  `);
+}
+

packages/db/src/schema.ts

@@ -497,6 +497,22 @@ export const notificationLog = sqliteTable("notification_log", {
     .default(sql`datetime('now')`),
 });
 
+export const failoverLog = sqliteTable("failover_log", {
+  id: integer("id").primaryKey({ autoIncrement: true }),
+  service_id: integer("service_id")
+    .notNull()
+    .references(() => services.id, { onDelete: "cascade" }),
+  binding_id: integer("binding_id")
+    .notNull()
+    .references(() => serviceBindings.id, { onDelete: "cascade" }),
+  fqdn: text("fqdn").notNull(),
+  ip: text("ip").notNull(),
+  action: text("action").notNull(),
+  created_at: text("created_at")
+    .notNull()
+    .default(sql`datetime('now')`),
+});
+
 export const auditLog = sqliteTable("audit_log", {
   id: text("id").primaryKey(),
   event_id: text("event_id"),
@@ -540,5 +556,6 @@ export const schema = {
   domainMonitorResults,
   healthProbeLog,
   notificationLog,
+  failoverLog,
   auditLog,
 };