feat(health-checks): implement health check IP toggling and configuration updates

- Added functionality to toggle individual IPs for services, allowing for dynamic management of IP health status.
- Enhanced the health check configuration in the UI, enabling users to set parameters directly from the settings page.
- Updated service views to include IP health tracking, improving visibility into the status of each IP associated with a service.
- Refactored relevant components to support the new IP toggling feature, ensuring a seamless user experience.

This commit significantly enhances the health management capabilities of services, providing users with more control over IP configurations and health monitoring.

packages/db/dist/index.js

@@ -187,6 +187,7 @@ var serviceIps = sqliteTable("service_ips", {
   id: integer("id").primaryKey({ autoIncrement: true }),
   service_id: integer("service_id").notNull().references(() => services.id, { onDelete: "cascade" }),
   ip: text("ip").notNull(),
+  enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
   created_at: text("created_at").notNull().default(sql`datetime('now')`)
 });
 var serviceBindingRecords = sqliteTable(
@@ -269,6 +270,11 @@ var appSettings = sqliteTable("app_settings", {
   show_quick_actions: integer("show_quick_actions", {
     mode: "boolean"
   }).notNull().default(true),
+  health_check_cron: text("health_check_cron"),
+  health_degraded_failures: integer("health_degraded_failures"),
+  health_down_failures: integer("health_down_failures"),
+  health_latency_warn_ms: integer("health_latency_warn_ms"),
+  health_success_recoveries: integer("health_success_recoveries"),
   created_at: text("created_at").notNull().default(sql`datetime('now')`),
   updated_at: text("updated_at").notNull().default(sql`datetime('now')`)
 });
@@ -493,7 +499,17 @@ function listAudit(db, opts = {}) {
 // src/settings-repo.ts
 import { eq as eq2 } from "drizzle-orm";
 var SETTINGS_ID = "settings-main";
-function toDto(row) {
+function coalesceInt(value, fallback) {
+  return value == null || Number.isNaN(value) || value < 1 ? fallback : value;
+}
+function toDto(row, fallbacks) {
+  const env = fallbacks ?? {
+    healthCheckCron: "0 */2 * * * *",
+    healthDegradedFailures: 1,
+    healthDownFailures: 2,
+    healthLatencyWarnMs: 1e3,
+    healthSuccessRecoveries: 2
+  };
   return {
     id: row.id,
     vpsTrackerUrl: row.vps_tracker_url?.trim() ?? "",
@@ -502,18 +518,36 @@ function toDto(row) {
     ),
     vpsTrackerSyncEnabled: Boolean(row.vps_tracker_sync_enabled),
     vpsTrackerLastSyncAt: row.vps_tracker_last_sync_at,
-    showQuickActions: row.show_quick_actions == null ? true : Boolean(row.show_quick_actions)
+    showQuickActions: row.show_quick_actions == null ? true : Boolean(row.show_quick_actions),
+    healthCheckCron: row.health_check_cron?.trim() || env.healthCheckCron,
+    healthDegradedFailures: coalesceInt(
+      row.health_degraded_failures,
+      env.healthDegradedFailures
+    ),
+    healthDownFailures: coalesceInt(
+      row.health_down_failures,
+      env.healthDownFailures
+    ),
+    healthLatencyWarnMs: coalesceInt(
+      row.health_latency_warn_ms,
+      env.healthLatencyWarnMs
+    ),
+    healthSuccessRecoveries: coalesceInt(
+      row.health_success_recoveries,
+      env.healthSuccessRecoveries
+    )
   };
 }
-function getAppSettings(db) {
+function getAppSettings(db, fallbacks) {
   const row = db.select().from(appSettings).where(eq2(appSettings.id, SETTINGS_ID)).get();
   if (!row) {
     db.insert(appSettings).values({ id: SETTINGS_ID }).run();
     return toDto(
-      db.select().from(appSettings).where(eq2(appSettings.id, SETTINGS_ID)).get()
+      db.select().from(appSettings).where(eq2(appSettings.id, SETTINGS_ID)).get(),
+      fallbacks
     );
   }
-  return toDto(row);
+  return toDto(row, fallbacks);
 }
 function getAppSettingsSecrets(db) {
   const row = db.select().from(appSettings).where(eq2(appSettings.id, SETTINGS_ID)).get();
@@ -523,7 +557,7 @@ function getAppSettingsSecrets(db) {
     vpsTrackerSyncEnabled: Boolean(row?.vps_tracker_sync_enabled)
   };
 }
-function updateAppSettings(db, patch) {
+function updateAppSettings(db, patch, fallbacks) {
   const existing = db.select().from(appSettings).where(eq2(appSettings.id, SETTINGS_ID)).get();
   if (!existing) {
     db.insert(appSettings).values({ id: SETTINGS_ID }).run();
@@ -535,9 +569,14 @@ function updateAppSettings(db, patch) {
     vps_tracker_integration_token: patch.vpsTrackerIntegrationToken !== void 0 && patch.vpsTrackerIntegrationToken.trim() !== "" ? patch.vpsTrackerIntegrationToken : current.vps_tracker_integration_token,
     vps_tracker_sync_enabled: patch.vpsTrackerSyncEnabled !== void 0 ? patch.vpsTrackerSyncEnabled : current.vps_tracker_sync_enabled,
     show_quick_actions: patch.showQuickActions !== void 0 ? patch.showQuickActions : current.show_quick_actions,
+    health_check_cron: patch.healthCheckCron !== void 0 ? patch.healthCheckCron.trim() : current.health_check_cron,
+    health_degraded_failures: patch.healthDegradedFailures !== void 0 ? patch.healthDegradedFailures : current.health_degraded_failures,
+    health_down_failures: patch.healthDownFailures !== void 0 ? patch.healthDownFailures : current.health_down_failures,
+    health_latency_warn_ms: patch.healthLatencyWarnMs !== void 0 ? patch.healthLatencyWarnMs : current.health_latency_warn_ms,
+    health_success_recoveries: patch.healthSuccessRecoveries !== void 0 ? patch.healthSuccessRecoveries : current.health_success_recoveries,
     updated_at: (/* @__PURE__ */ new Date()).toISOString()
   }).where(eq2(appSettings.id, SETTINGS_ID)).run();
-  return getAppSettings(db);
+  return getAppSettings(db, fallbacks);
 }
 function touchVpsTrackerSync(db) {
   db.update(appSettings).set({
@@ -638,6 +677,7 @@ __export(repos_exports, {
   listOriginIpsForFqdn: () => listOriginIpsForFqdn,
   listRecordsForBinding: () => listRecordsForBinding,
   listServiceGroups: () => listServiceGroups,
+  listServiceIpRows: () => listServiceIpRows,
   listServiceIps: () => listServiceIps,
   listServices: () => listServices,
   listServicesByGroup: () => listServicesByGroup,
@@ -659,6 +699,7 @@ __export(repos_exports, {
   setServiceEnabled: () => setServiceEnabled,
   setServiceGroup: () => setServiceGroup,
   setServiceGroupEnabled: () => setServiceGroupEnabled,
+  setServiceIpEnabled: () => setServiceIpEnabled,
   setServiceLb: () => setServiceLb,
   unlinkBindingRecord: () => unlinkBindingRecord,
   unlinkGroupDnsRecord: () => unlinkGroupDnsRecord,
@@ -1155,16 +1196,32 @@ function deleteServiceGroup(db, id) {
 function insertServiceIpIfMissing(db, serviceId, ip) {
   const existing = db.select({ ip: serviceIps.ip }).from(serviceIps).where(and2(eq3(serviceIps.service_id, serviceId), eq3(serviceIps.ip, ip))).get();
   if (!existing) {
-    db.insert(serviceIps).values({ service_id: serviceId, ip }).run();
+    db.insert(serviceIps).values({ service_id: serviceId, ip, enabled: true }).run();
   }
 }
+function listServiceIpRows(db, serviceId) {
+  return db.select({ ip: serviceIps.ip, enabled: serviceIps.enabled }).from(serviceIps).where(eq3(serviceIps.service_id, serviceId)).all().map((row) => ({ ip: row.ip, enabled: Boolean(row.enabled) }));
+}
 function listServiceIps(db, serviceId) {
-  return db.select({ ip: serviceIps.ip }).from(serviceIps).where(eq3(serviceIps.service_id, serviceId)).all().map((r) => r.ip);
+  return listServiceIpRows(db, serviceId).map((row) => row.ip);
+}
+function setServiceIpEnabled(db, serviceId, ip, enabled) {
+  const result = db.update(serviceIps).set({ enabled }).where(and2(eq3(serviceIps.service_id, serviceId), eq3(serviceIps.ip, ip))).run();
+  if (result.changes === 0) {
+    throw new NotFoundError(`service ip ${ip}`);
+  }
 }
 function replaceServiceIps(db, serviceId, ips) {
+  const previous = new Map(
+    listServiceIpRows(db, serviceId).map((row) => [row.ip, row.enabled])
+  );
   db.delete(serviceIps).where(eq3(serviceIps.service_id, serviceId)).run();
   for (const ip of ips) {
-    db.insert(serviceIps).values({ service_id: serviceId, ip }).run();
+    db.insert(serviceIps).values({
+      service_id: serviceId,
+      ip,
+      enabled: previous.get(ip) ?? true
+    }).run();
     ensureNode(db, serviceId, ip);
   }
   const keep = new Set(ips);

packages/db/migrations/019_service_ips_enabled.sql

@@ -0,0 +1 @@
+ALTER TABLE service_ips ADD COLUMN enabled INTEGER NOT NULL DEFAULT 1;

packages/db/migrations/020_health_engine_settings.sql

@@ -0,0 +1,7 @@
+-- Local health-check engine settings (cron + state-machine thresholds).
+-- NULL = inherit from process env (HEALTH_CHECK_CRON / HEALTH_*).
+ALTER TABLE app_settings ADD COLUMN health_check_cron TEXT;
+ALTER TABLE app_settings ADD COLUMN health_degraded_failures INTEGER;
+ALTER TABLE app_settings ADD COLUMN health_down_failures INTEGER;
+ALTER TABLE app_settings ADD COLUMN health_latency_warn_ms INTEGER;
+ALTER TABLE app_settings ADD COLUMN health_success_recoveries INTEGER;

packages/db/src/repos.ts

@@ -923,17 +923,42 @@ function insertServiceIpIfMissing(db: Db, serviceId: number, ip: string): void {
     .where(and(eq(serviceIps.service_id, serviceId), eq(serviceIps.ip, ip)))
     .get();
   if (!existing) {
-    db.insert(serviceIps).values({ service_id: serviceId, ip }).run();
+    db.insert(serviceIps).values({ service_id: serviceId, ip, enabled: true }).run();
   }
 }
 
-export function listServiceIps(db: Db, serviceId: number): string[] {
+export type ServiceIpRow = {
+  ip: string;
+  enabled: boolean;
+};
+
+export function listServiceIpRows(db: Db, serviceId: number): ServiceIpRow[] {
   return db
-    .select({ ip: serviceIps.ip })
+    .select({ ip: serviceIps.ip, enabled: serviceIps.enabled })
     .from(serviceIps)
     .where(eq(serviceIps.service_id, serviceId))
     .all()
-    .map((r) => r.ip);
+    .map((row) => ({ ip: row.ip, enabled: Boolean(row.enabled) }));
+}
+
+export function listServiceIps(db: Db, serviceId: number): string[] {
+  return listServiceIpRows(db, serviceId).map((row) => row.ip);
+}
+
+export function setServiceIpEnabled(
+  db: Db,
+  serviceId: number,
+  ip: string,
+  enabled: boolean,
+): void {
+  const result = db
+    .update(serviceIps)
+    .set({ enabled })
+    .where(and(eq(serviceIps.service_id, serviceId), eq(serviceIps.ip, ip)))
+    .run();
+  if (result.changes === 0) {
+    throw new NotFoundError(`service ip ${ip}`);
+  }
 }
 
 export function replaceServiceIps(
@@ -941,9 +966,18 @@ export function replaceServiceIps(
   serviceId: number,
   ips: string[],
 ): void {
+  const previous = new Map(
+    listServiceIpRows(db, serviceId).map((row) => [row.ip, row.enabled]),
+  );
   db.delete(serviceIps).where(eq(serviceIps.service_id, serviceId)).run();
   for (const ip of ips) {
-    db.insert(serviceIps).values({ service_id: serviceId, ip }).run();
+    db.insert(serviceIps)
+      .values({
+        service_id: serviceId,
+        ip,
+        enabled: previous.get(ip) ?? true,
+      })
+      .run();
     ensureNode(db, serviceId, ip);
   }
   const keep = new Set(ips);

packages/db/src/schema.ts

@@ -259,6 +259,7 @@ export const serviceIps = sqliteTable("service_ips", {
     .notNull()
     .references(() => services.id, { onDelete: "cascade" }),
   ip: text("ip").notNull(),
+  enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
   created_at: text("created_at")
     .notNull()
     .default(sql`datetime('now')`),
@@ -379,6 +380,11 @@ export const appSettings = sqliteTable("app_settings", {
   })
     .notNull()
     .default(true),
+  health_check_cron: text("health_check_cron"),
+  health_degraded_failures: integer("health_degraded_failures"),
+  health_down_failures: integer("health_down_failures"),
+  health_latency_warn_ms: integer("health_latency_warn_ms"),
+  health_success_recoveries: integer("health_success_recoveries"),
   created_at: text("created_at")
     .notNull()
     .default(sql`datetime('now')`),

packages/db/src/settings-repo.ts

@@ -4,6 +4,14 @@ import { appSettings } from "./schema.js";
 
 const SETTINGS_ID = "settings-main";
 
+export type HealthEngineSettings = {
+  healthCheckCron: string;
+  healthDegradedFailures: number;
+  healthDownFailures: number;
+  healthLatencyWarnMs: number;
+  healthSuccessRecoveries: number;
+};
+
 export type AppSettingsDto = {
   id: string;
   vpsTrackerUrl: string;
@@ -11,16 +19,37 @@ export type AppSettingsDto = {
   vpsTrackerSyncEnabled: boolean;
   vpsTrackerLastSyncAt: string | null;
   showQuickActions: boolean;
-};
+} & HealthEngineSettings;
 
 export type AppSettingsPatch = {
   vpsTrackerUrl?: string;
   vpsTrackerIntegrationToken?: string;
   vpsTrackerSyncEnabled?: boolean;
   showQuickActions?: boolean;
+  healthCheckCron?: string;
+  healthDegradedFailures?: number;
+  healthDownFailures?: number;
+  healthLatencyWarnMs?: number;
+  healthSuccessRecoveries?: number;
 };
 
-function toDto(row: typeof appSettings.$inferSelect): AppSettingsDto {
+export type HealthEngineFallbacks = HealthEngineSettings;
+
+function coalesceInt(value: number | null | undefined, fallback: number): number {
+  return value == null || Number.isNaN(value) || value < 1 ? fallback : value;
+}
+
+function toDto(
+  row: typeof appSettings.$inferSelect,
+  fallbacks?: HealthEngineFallbacks,
+): AppSettingsDto {
+  const env = fallbacks ?? {
+    healthCheckCron: "0 */2 * * * *",
+    healthDegradedFailures: 1,
+    healthDownFailures: 2,
+    healthLatencyWarnMs: 1000,
+    healthSuccessRecoveries: 2,
+  };
   return {
     id: row.id,
     vpsTrackerUrl: row.vps_tracker_url?.trim() ?? "",
@@ -31,10 +60,30 @@ function toDto(row: typeof appSettings.$inferSelect): AppSettingsDto {
     vpsTrackerLastSyncAt: row.vps_tracker_last_sync_at,
     showQuickActions:
       row.show_quick_actions == null ? true : Boolean(row.show_quick_actions),
+    healthCheckCron: row.health_check_cron?.trim() || env.healthCheckCron,
+    healthDegradedFailures: coalesceInt(
+      row.health_degraded_failures,
+      env.healthDegradedFailures,
+    ),
+    healthDownFailures: coalesceInt(
+      row.health_down_failures,
+      env.healthDownFailures,
+    ),
+    healthLatencyWarnMs: coalesceInt(
+      row.health_latency_warn_ms,
+      env.healthLatencyWarnMs,
+    ),
+    healthSuccessRecoveries: coalesceInt(
+      row.health_success_recoveries,
+      env.healthSuccessRecoveries,
+    ),
   };
 }
 
-export function getAppSettings(db: Db): AppSettingsDto {
+export function getAppSettings(
+  db: Db,
+  fallbacks?: HealthEngineFallbacks,
+): AppSettingsDto {
   const row = db
     .select()
     .from(appSettings)
@@ -44,9 +93,10 @@ export function getAppSettings(db: Db): AppSettingsDto {
     db.insert(appSettings).values({ id: SETTINGS_ID }).run();
     return toDto(
       db.select().from(appSettings).where(eq(appSettings.id, SETTINGS_ID)).get()!,
+      fallbacks,
     );
   }
-  return toDto(row);
+  return toDto(row, fallbacks);
 }
 
 export function getAppSettingsSecrets(db: Db): {
@@ -67,7 +117,11 @@ export function getAppSettingsSecrets(db: Db): {
   };
 }
 
-export function updateAppSettings(db: Db, patch: AppSettingsPatch): AppSettingsDto {
+export function updateAppSettings(
+  db: Db,
+  patch: AppSettingsPatch,
+  fallbacks?: HealthEngineFallbacks,
+): AppSettingsDto {
   const existing = db
     .select()
     .from(appSettings)
@@ -102,12 +156,32 @@ export function updateAppSettings(db: Db, patch: AppSettingsPatch): AppSettingsD
         patch.showQuickActions !== undefined
           ? patch.showQuickActions
           : current.show_quick_actions,
+      health_check_cron:
+        patch.healthCheckCron !== undefined
+          ? patch.healthCheckCron.trim()
+          : current.health_check_cron,
+      health_degraded_failures:
+        patch.healthDegradedFailures !== undefined
+          ? patch.healthDegradedFailures
+          : current.health_degraded_failures,
+      health_down_failures:
+        patch.healthDownFailures !== undefined
+          ? patch.healthDownFailures
+          : current.health_down_failures,
+      health_latency_warn_ms:
+        patch.healthLatencyWarnMs !== undefined
+          ? patch.healthLatencyWarnMs
+          : current.health_latency_warn_ms,
+      health_success_recoveries:
+        patch.healthSuccessRecoveries !== undefined
+          ? patch.healthSuccessRecoveries
+          : current.health_success_recoveries,
       updated_at: new Date().toISOString(),
     })
     .where(eq(appSettings.id, SETTINGS_ID))
     .run();
 
-  return getAppSettings(db);
+  return getAppSettings(db, fallbacks);
 }
 
 export function touchVpsTrackerSync(db: Db): void {