feat(health-checks): enhance health check configuration and logging

- Added new health worker URL and token fields to the AppConfig interface, allowing for Cloudflare Worker integration.
- Updated health check routes to utilize the new worker configuration, enabling dynamic health checks via Cloudflare Workers.
- Introduced a health log endpoint for services, providing detailed logs of health probe results.
- Enhanced health check service logic to support both local and Cloudflare Worker providers, improving flexibility in health monitoring.
- Updated UI components to reflect changes in health check provider settings and display relevant health information.

This commit significantly improves the health check management capabilities, allowing for better integration with Cloudflare Workers and enhanced logging features.

packages/db/dist/index.js

@@ -52,6 +52,7 @@ var serviceGroups = sqliteTable("service_groups", {
   health_check_interval_sec: integer("health_check_interval_sec").notNull().default(30),
   health_check_timeout_ms: integer("health_check_timeout_ms").notNull().default(3e3),
   health_check_verify_tls: integer("health_check_verify_tls", { mode: "boolean" }).notNull().default(false),
+  health_check_provider: text("health_check_provider").notNull().default("local"),
   created_at: text("created_at").notNull().default(sql`datetime('now')`),
   updated_at: text("updated_at").notNull().default(sql`datetime('now')`)
 });
@@ -117,6 +118,7 @@ var serviceBindings = sqliteTable(
     health_check_verify_tls: integer("health_check_verify_tls", {
       mode: "boolean"
     }).notNull().default(false),
+    health_check_provider: text("health_check_provider").notNull().default("local"),
     routing_strategy: text("routing_strategy").notNull().default("round_robin"),
     operation_version: integer("operation_version").notNull().default(0),
     created_at: text("created_at").notNull().default(sql`datetime('now')`),
@@ -252,6 +254,8 @@ var ipHealthStatus = sqliteTable(
     consecutive_successes: integer("consecutive_successes").notNull().default(0),
     last_checked_at: text("last_checked_at"),
     last_error: text("last_error"),
+    colo: text("colo"),
+    provider: text("provider").notNull().default("local"),
     created_at: text("created_at").notNull().default(sql`datetime('now')`),
     updated_at: text("updated_at").notNull().default(sql`datetime('now')`)
   },
@@ -275,6 +279,8 @@ var appSettings = sqliteTable("app_settings", {
   health_down_failures: integer("health_down_failures"),
   health_latency_warn_ms: integer("health_latency_warn_ms"),
   health_success_recoveries: integer("health_success_recoveries"),
+  health_worker_url: text("health_worker_url"),
+  health_worker_token: text("health_worker_token"),
   created_at: text("created_at").notNull().default(sql`datetime('now')`),
   updated_at: text("updated_at").notNull().default(sql`datetime('now')`)
 });
@@ -311,6 +317,19 @@ var domainMonitorResults = sqliteTable("domain_monitor_results", {
   error: text("error"),
   checked_at: text("checked_at").notNull().default(sql`datetime('now')`)
 });
+var healthProbeLog = sqliteTable("health_probe_log", {
+  id: integer("id").primaryKey({ autoIncrement: true }),
+  scope: text("scope").notNull(),
+  ref_id: integer("ref_id").notNull(),
+  ip: text("ip").notNull(),
+  provider: text("provider").notNull(),
+  status: text("status").notNull(),
+  ok: integer("ok", { mode: "boolean" }).notNull(),
+  latency_ms: integer("latency_ms"),
+  colo: text("colo"),
+  error: text("error"),
+  checked_at: text("checked_at").notNull().default(sql`datetime('now')`)
+});
 var notificationLog = sqliteTable("notification_log", {
   id: integer("id").primaryKey({ autoIncrement: true }),
   kind: text("kind").notNull(),
@@ -358,6 +377,7 @@ var schema = {
   domainTags,
   domainMonitors,
   domainMonitorResults,
+  healthProbeLog,
   notificationLog,
   auditLog
 };
@@ -508,7 +528,9 @@ function toDto(row, fallbacks) {
     healthDegradedFailures: 1,
     healthDownFailures: 2,
     healthLatencyWarnMs: 1e3,
-    healthSuccessRecoveries: 2
+    healthSuccessRecoveries: 2,
+    healthWorkerUrl: "",
+    healthWorkerTokenSet: false
   };
   return {
     id: row.id,
@@ -535,7 +557,9 @@ function toDto(row, fallbacks) {
     healthSuccessRecoveries: coalesceInt(
       row.health_success_recoveries,
       env.healthSuccessRecoveries
-    )
+    ),
+    healthWorkerUrl: row.health_worker_url?.trim() || env.healthWorkerUrl,
+    healthWorkerTokenSet: Boolean(row.health_worker_token?.trim()) || env.healthWorkerTokenSet
   };
 }
 function getAppSettings(db, fallbacks) {
@@ -554,7 +578,9 @@ function getAppSettingsSecrets(db) {
   return {
     vpsTrackerUrl: row?.vps_tracker_url?.trim() ?? "",
     vpsTrackerIntegrationToken: row?.vps_tracker_integration_token?.trim() ?? "",
-    vpsTrackerSyncEnabled: Boolean(row?.vps_tracker_sync_enabled)
+    vpsTrackerSyncEnabled: Boolean(row?.vps_tracker_sync_enabled),
+    healthWorkerUrl: row?.health_worker_url?.trim() ?? "",
+    healthWorkerToken: row?.health_worker_token?.trim() ?? ""
   };
 }
 function updateAppSettings(db, patch, fallbacks) {
@@ -574,6 +600,8 @@ function updateAppSettings(db, patch, fallbacks) {
     health_down_failures: patch.healthDownFailures !== void 0 ? patch.healthDownFailures : current.health_down_failures,
     health_latency_warn_ms: patch.healthLatencyWarnMs !== void 0 ? patch.healthLatencyWarnMs : current.health_latency_warn_ms,
     health_success_recoveries: patch.healthSuccessRecoveries !== void 0 ? patch.healthSuccessRecoveries : current.health_success_recoveries,
+    health_worker_url: patch.healthWorkerUrl !== void 0 ? patch.healthWorkerUrl.trim() || null : current.health_worker_url,
+    health_worker_token: patch.healthWorkerToken !== void 0 && patch.healthWorkerToken.trim() !== "" ? patch.healthWorkerToken : current.health_worker_token,
     updated_at: (/* @__PURE__ */ new Date()).toISOString()
   }).where(eq2(appSettings.id, SETTINGS_ID)).run();
   return getAppSettings(db, fallbacks);
@@ -644,6 +672,7 @@ __export(repos_exports, {
   getSyncJob: () => getSyncJob,
   insertBinding: () => insertBinding,
   insertDnsRecord: () => insertDnsRecord,
+  insertHealthProbeLog: () => insertHealthProbeLog,
   insertNotificationLog: () => insertNotificationLog,
   linkBindingRecord: () => linkBindingRecord,
   linkGroupDnsRecord: () => linkGroupDnsRecord,
@@ -670,6 +699,7 @@ __export(repos_exports, {
   listGroups: () => listGroups,
   listHealthCheckTargets: () => listHealthCheckTargets,
   listHealthChecks: () => listHealthChecks,
+  listHealthProbeLogForService: () => listHealthProbeLogForService,
   listIpHealthByServiceIds: () => listIpHealthByServiceIds,
   listIpHealthStatus: () => listIpHealthStatus,
   listNodes: () => listNodes,
@@ -1106,6 +1136,9 @@ function deleteService(db, id) {
   const result = db.delete(services).where(eq3(services.id, id)).run();
   if (result.changes === 0) throw new NotFoundError(`service ${id}`);
 }
+function normalizeHealthProvider(value) {
+  return value === "cloudflare" ? "cloudflare" : "local";
+}
 function mapServiceGroup(row) {
   return {
     id: row.id,
@@ -1123,6 +1156,7 @@ function mapServiceGroup(row) {
     health_check_interval_sec: row.health_check_interval_sec,
     health_check_timeout_ms: row.health_check_timeout_ms,
     health_check_verify_tls: row.health_check_verify_tls,
+    health_check_provider: normalizeHealthProvider(row.health_check_provider),
     created_at: row.created_at,
     updated_at: row.updated_at
   };
@@ -1149,7 +1183,8 @@ function createServiceGroup(db, name, groupType, icon, domain, lbPatch) {
     health_check_expected_status: lbPatch?.health_check_expected_status ?? null,
     health_check_interval_sec: lbPatch?.health_check_interval_sec ?? 30,
     health_check_timeout_ms: lbPatch?.health_check_timeout_ms ?? 3e3,
-    health_check_verify_tls: lbPatch?.health_check_verify_tls ?? false
+    health_check_verify_tls: lbPatch?.health_check_verify_tls ?? false,
+    health_check_provider: lbPatch?.health_check_provider ?? "local"
   }).returning({ id: serviceGroups.id }).get().id;
   return getServiceGroup(db, id);
 }
@@ -1179,6 +1214,8 @@ function updateServiceGroup(db, id, name, groupType, icon, domain, lbPatch) {
       update.health_check_timeout_ms = lbPatch.health_check_timeout_ms;
     if (lbPatch.health_check_verify_tls !== void 0)
       update.health_check_verify_tls = lbPatch.health_check_verify_tls;
+    if (lbPatch.health_check_provider !== void 0)
+      update.health_check_provider = lbPatch.health_check_provider;
   }
   const result = db.update(serviceGroups).set(update).where(eq3(serviceGroups.id, id)).run();
   if (result.changes === 0) throw new NotFoundError(`service group ${id}`);
@@ -1488,6 +1525,8 @@ function updateBindingLbConfig(db, bindingId, patch) {
     update.health_check_timeout_ms = patch.health_check_timeout_ms;
   if (patch.health_check_verify_tls !== void 0)
     update.health_check_verify_tls = patch.health_check_verify_tls;
+  if (patch.health_check_provider !== void 0)
+    update.health_check_provider = patch.health_check_provider;
   db.update(serviceBindings).set(update).where(eq3(serviceBindings.id, bindingId)).run();
 }
 function setBindingCnameTarget(db, bindingId, target) {
@@ -1546,7 +1585,7 @@ function dnsRecordMatchesHostname(recordName, hostname, zoneName) {
 var SERVICE_BINDING_SELECT_COLUMNS = `sb.id, sb.domain_id, sb.service_id, sb.hostname, sb.dns_record_id,
   sb.lb_mode, sb.health_check_enabled, sb.health_check_type, sb.health_check_port,
   sb.health_check_path, sb.health_check_expected_status, sb.health_check_interval_sec,
-  sb.health_check_timeout_ms, sb.health_check_verify_tls, sb.cname_target,
+  sb.health_check_timeout_ms, sb.health_check_verify_tls, sb.health_check_provider, sb.cname_target,
   d.zone_name, d.group_id, g.name AS group_name,
   s.name AS service_name, s.slug AS service_slug,
   dr.content AS target_ip, dr.sync_status,
@@ -1767,7 +1806,7 @@ function finishSyncJob(db, id, status, message) {
 function listIpHealthStatus(db, scope, refId) {
   return db.all(sql2`
     SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures,
-      last_checked_at, last_error
+      last_checked_at, last_error, colo, provider
     FROM ip_health_status
     WHERE scope = ${scope} AND ref_id = ${refId}
   `);
@@ -1891,7 +1930,11 @@ function listIpHealthByServiceIds(db, serviceIds) {
     SELECT sb.service_id AS service_id,
       ihs.ip AS ip,
       ${WORST_HEALTH_SQL} AS health_status,
-      MAX(ihs.latency_ms) AS health_latency_ms
+      MAX(ihs.latency_ms) AS health_latency_ms,
+      MAX(ihs.last_checked_at) AS last_checked_at,
+      MAX(ihs.last_error) AS last_error,
+      MAX(ihs.provider) AS provider,
+      MAX(ihs.colo) AS colo
     FROM ip_health_status ihs
     INNER JOIN service_bindings sb
       ON ihs.scope = 'binding' AND ihs.ref_id = sb.id
@@ -1907,7 +1950,11 @@ function listIpHealthByServiceIds(db, serviceIds) {
     list.push({
       ip: row.ip,
       status: parsed.health_status,
-      latency_ms: parsed.health_latency_ms
+      latency_ms: parsed.health_latency_ms,
+      last_checked_at: row.last_checked_at,
+      last_error: row.last_error,
+      provider: normalizeHealthProvider(row.provider),
+      colo: row.colo
     });
     result.set(row.service_id, list);
   }
@@ -1942,20 +1989,24 @@ function mergeHealthAggregates(parts) {
 function getIpHealthStatusRow(db, scope, refId, ip) {
   const rows = db.all(sql2`
     SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures,
-      consecutive_successes, last_checked_at, last_error
+      consecutive_successes, last_checked_at, last_error, colo, provider
     FROM ip_health_status
     WHERE scope = ${scope} AND ref_id = ${refId} AND ip = ${ip}
     LIMIT 1
   `);
   return rows[0] ?? null;
 }
-function upsertIpHealthStatus(db, scope, refId, ip, status, latencyMs, consecutiveFailures, lastError, consecutiveSuccesses = 0) {
+function upsertIpHealthStatus(db, scope, refId, ip, status, latencyMs, consecutiveFailures, lastError, consecutiveSuccesses = 0, extras) {
+  const colo = extras?.colo ?? null;
+  const provider = extras?.provider ?? "local";
   db.run(sql2`
     INSERT INTO ip_health_status
       (scope, ref_id, ip, status, latency_ms, consecutive_failures,
-       consecutive_successes, last_checked_at, last_error, created_at, updated_at)
+       consecutive_successes, last_checked_at, last_error, colo, provider,
+       created_at, updated_at)
     VALUES (${scope}, ${refId}, ${ip}, ${status}, ${latencyMs}, ${consecutiveFailures},
-      ${consecutiveSuccesses}, datetime('now'), ${lastError}, datetime('now'), datetime('now'))
+      ${consecutiveSuccesses}, datetime('now'), ${lastError}, ${colo}, ${provider},
+      datetime('now'), datetime('now'))
     ON CONFLICT(scope, ref_id, ip) DO UPDATE SET
       status = excluded.status,
       latency_ms = excluded.latency_ms,
@@ -1963,6 +2014,8 @@ function upsertIpHealthStatus(db, scope, refId, ip, status, latencyMs, consecuti
       consecutive_successes = excluded.consecutive_successes,
       last_checked_at = excluded.last_checked_at,
       last_error = excluded.last_error,
+      colo = excluded.colo,
+      provider = excluded.provider,
       updated_at = datetime('now')
   `);
 }
@@ -2025,7 +2078,8 @@ function listHealthCheckTargets(db) {
       sb.health_check_path AS path,
       sb.health_check_expected_status AS expected_status,
       sb.health_check_timeout_ms AS timeout_ms,
-      sb.health_check_verify_tls AS verify_tls
+      sb.health_check_verify_tls AS verify_tls,
+      COALESCE(sb.health_check_provider, 'local') AS provider
     FROM service_binding_ips sbi
     JOIN service_bindings sb ON sb.id = sbi.binding_id
     JOIN domains d ON d.id = sb.domain_id
@@ -2039,7 +2093,8 @@ function listHealthCheckTargets(db) {
       sg.health_check_path AS path,
       sg.health_check_expected_status AS expected_status,
       sg.health_check_timeout_ms AS timeout_ms,
-      sg.health_check_verify_tls AS verify_tls
+      sg.health_check_verify_tls AS verify_tls,
+      COALESCE(sg.health_check_provider, 'local') AS provider
     FROM service_binding_ips sbi
     JOIN service_bindings sb ON sb.id = sbi.binding_id
     JOIN services s ON s.id = sb.service_id
@@ -2059,7 +2114,8 @@ function listHealthCheckTargets(db) {
       sg.health_check_path AS path,
       sg.health_check_expected_status AS expected_status,
       sg.health_check_timeout_ms AS timeout_ms,
-      sg.health_check_verify_tls AS verify_tls
+      sg.health_check_verify_tls AS verify_tls,
+      COALESCE(sg.health_check_provider, 'local') AS provider
     FROM service_binding_ips sbi
     JOIN service_bindings sb ON sb.id = sbi.binding_id
     JOIN domains d ON d.id = sb.domain_id
@@ -2079,7 +2135,8 @@ function listHealthCheckTargets(db) {
       sb.health_check_path AS path,
       sb.health_check_expected_status AS expected_status,
       sb.health_check_timeout_ms AS timeout_ms,
-      sb.health_check_verify_tls AS verify_tls
+      sb.health_check_verify_tls AS verify_tls,
+      COALESCE(sb.health_check_provider, 'local') AS provider
     FROM service_bindings sb
     JOIN domains d ON d.id = sb.domain_id
     JOIN services s ON s.id = sb.service_id
@@ -2096,7 +2153,8 @@ function listHealthCheckTargets(db) {
       sg.health_check_path AS path,
       sg.health_check_expected_status AS expected_status,
       sg.health_check_timeout_ms AS timeout_ms,
-      sg.health_check_verify_tls AS verify_tls
+      sg.health_check_verify_tls AS verify_tls,
+      COALESCE(sg.health_check_provider, 'local') AS provider
     FROM service_bindings sb
     JOIN domains d ON d.id = sb.domain_id
     JOIN services s ON s.id = sb.service_id
@@ -2117,7 +2175,8 @@ function listHealthCheckTargets(db) {
     ...groupInheritedCnameBindingTargets
   ].map((t) => ({
     ...t,
-    verify_tls: Boolean(t.verify_tls)
+    verify_tls: Boolean(t.verify_tls),
+    provider: normalizeHealthProvider(t.provider)
   }));
 }
 function listDomainTags(db, domainId) {
@@ -2212,6 +2271,46 @@ function listDomainMonitorResultsForDomain(db, domainId, limit = 50) {
     LIMIT ${limit}
   `);
 }
+var HEALTH_PROBE_LOG_KEEP = 50;
+function insertHealthProbeLog(db, entry) {
+  db.insert(healthProbeLog).values({
+    scope: entry.scope,
+    ref_id: entry.refId,
+    ip: entry.ip,
+    provider: entry.provider,
+    status: entry.status,
+    ok: entry.ok,
+    latency_ms: entry.latencyMs,
+    colo: entry.colo,
+    error: entry.error
+  }).run();
+  db.run(sql2`
+    DELETE FROM health_probe_log
+    WHERE id IN (
+      SELECT id FROM health_probe_log
+      WHERE scope = ${entry.scope} AND ref_id = ${entry.refId} AND ip = ${entry.ip}
+      ORDER BY checked_at DESC, id DESC
+      LIMIT -1 OFFSET ${HEALTH_PROBE_LOG_KEEP}
+    )
+  `);
+}
+function listHealthProbeLogForService(db, serviceId, limit = 50) {
+  const rows = db.all(sql2`
+    SELECT l.id, l.scope, l.ref_id, l.ip, l.provider, l.status, l.ok,
+      l.latency_ms, l.colo, l.error, l.checked_at
+    FROM health_probe_log l
+    INNER JOIN service_bindings sb
+      ON l.scope = 'binding' AND l.ref_id = sb.id
+    WHERE sb.service_id = ${serviceId}
+    ORDER BY l.checked_at DESC, l.id DESC
+    LIMIT ${limit}
+  `);
+  return rows.map((row) => ({
+    ...row,
+    provider: normalizeHealthProvider(row.provider),
+    ok: Boolean(row.ok)
+  }));
+}
 function insertNotificationLog(db, kind, refType, refId, title, message) {
   db.insert(notificationLog).values({
     kind,
@@ -2249,6 +2348,7 @@ export {
   groups,
   healthCheck,
   healthChecks,
+  healthProbeLog,
   ipHealthStatus,
   listAudit,
   nodes,