feat(health-checks): enhance health check configuration and logging

- Added new health worker URL and token fields to the AppConfig interface, allowing for Cloudflare Worker integration.
- Updated health check routes to utilize the new worker configuration, enabling dynamic health checks via Cloudflare Workers.
- Introduced a health log endpoint for services, providing detailed logs of health probe results.
- Enhanced health check service logic to support both local and Cloudflare Worker providers, improving flexibility in health monitoring.
- Updated UI components to reflect changes in health check provider settings and display relevant health information.

This commit significantly improves the health check management capabilities, allowing for better integration with Cloudflare Workers and enhanced logging features.

packages/db/dist/index.js

@@ -52,6 +52,7 @@ var serviceGroups = sqliteTable("service_groups", {
   health_check_interval_sec: integer("health_check_interval_sec").notNull().default(30),
   health_check_timeout_ms: integer("health_check_timeout_ms").notNull().default(3e3),
   health_check_verify_tls: integer("health_check_verify_tls", { mode: "boolean" }).notNull().default(false),
+  health_check_provider: text("health_check_provider").notNull().default("local"),
   created_at: text("created_at").notNull().default(sql`datetime('now')`),
   updated_at: text("updated_at").notNull().default(sql`datetime('now')`)
 });
@@ -117,6 +118,7 @@ var serviceBindings = sqliteTable(
     health_check_verify_tls: integer("health_check_verify_tls", {
       mode: "boolean"
     }).notNull().default(false),
+    health_check_provider: text("health_check_provider").notNull().default("local"),
     routing_strategy: text("routing_strategy").notNull().default("round_robin"),
     operation_version: integer("operation_version").notNull().default(0),
     created_at: text("created_at").notNull().default(sql`datetime('now')`),
@@ -252,6 +254,8 @@ var ipHealthStatus = sqliteTable(
     consecutive_successes: integer("consecutive_successes").notNull().default(0),
     last_checked_at: text("last_checked_at"),
     last_error: text("last_error"),
+    colo: text("colo"),
+    provider: text("provider").notNull().default("local"),
     created_at: text("created_at").notNull().default(sql`datetime('now')`),
     updated_at: text("updated_at").notNull().default(sql`datetime('now')`)
   },
@@ -275,6 +279,8 @@ var appSettings = sqliteTable("app_settings", {
   health_down_failures: integer("health_down_failures"),
   health_latency_warn_ms: integer("health_latency_warn_ms"),
   health_success_recoveries: integer("health_success_recoveries"),
+  health_worker_url: text("health_worker_url"),
+  health_worker_token: text("health_worker_token"),
   created_at: text("created_at").notNull().default(sql`datetime('now')`),
   updated_at: text("updated_at").notNull().default(sql`datetime('now')`)
 });
@@ -311,6 +317,19 @@ var domainMonitorResults = sqliteTable("domain_monitor_results", {
   error: text("error"),
   checked_at: text("checked_at").notNull().default(sql`datetime('now')`)
 });
+var healthProbeLog = sqliteTable("health_probe_log", {
+  id: integer("id").primaryKey({ autoIncrement: true }),
+  scope: text("scope").notNull(),
+  ref_id: integer("ref_id").notNull(),
+  ip: text("ip").notNull(),
+  provider: text("provider").notNull(),
+  status: text("status").notNull(),
+  ok: integer("ok", { mode: "boolean" }).notNull(),
+  latency_ms: integer("latency_ms"),
+  colo: text("colo"),
+  error: text("error"),
+  checked_at: text("checked_at").notNull().default(sql`datetime('now')`)
+});
 var notificationLog = sqliteTable("notification_log", {
   id: integer("id").primaryKey({ autoIncrement: true }),
   kind: text("kind").notNull(),
@@ -358,6 +377,7 @@ var schema = {
   domainTags,
   domainMonitors,
   domainMonitorResults,
+  healthProbeLog,
   notificationLog,
   auditLog
 };
@@ -508,7 +528,9 @@ function toDto(row, fallbacks) {
     healthDegradedFailures: 1,
     healthDownFailures: 2,
     healthLatencyWarnMs: 1e3,
-    healthSuccessRecoveries: 2
+    healthSuccessRecoveries: 2,
+    healthWorkerUrl: "",
+    healthWorkerTokenSet: false
   };
   return {
     id: row.id,
@@ -535,7 +557,9 @@ function toDto(row, fallbacks) {
     healthSuccessRecoveries: coalesceInt(
       row.health_success_recoveries,
       env.healthSuccessRecoveries
-    )
+    ),
+    healthWorkerUrl: row.health_worker_url?.trim() || env.healthWorkerUrl,
+    healthWorkerTokenSet: Boolean(row.health_worker_token?.trim()) || env.healthWorkerTokenSet
   };
 }
 function getAppSettings(db, fallbacks) {
@@ -554,7 +578,9 @@ function getAppSettingsSecrets(db) {
   return {
     vpsTrackerUrl: row?.vps_tracker_url?.trim() ?? "",
     vpsTrackerIntegrationToken: row?.vps_tracker_integration_token?.trim() ?? "",
-    vpsTrackerSyncEnabled: Boolean(row?.vps_tracker_sync_enabled)
+    vpsTrackerSyncEnabled: Boolean(row?.vps_tracker_sync_enabled),
+    healthWorkerUrl: row?.health_worker_url?.trim() ?? "",
+    healthWorkerToken: row?.health_worker_token?.trim() ?? ""
   };
 }
 function updateAppSettings(db, patch, fallbacks) {
@@ -574,6 +600,8 @@ function updateAppSettings(db, patch, fallbacks) {
     health_down_failures: patch.healthDownFailures !== void 0 ? patch.healthDownFailures : current.health_down_failures,
     health_latency_warn_ms: patch.healthLatencyWarnMs !== void 0 ? patch.healthLatencyWarnMs : current.health_latency_warn_ms,
     health_success_recoveries: patch.healthSuccessRecoveries !== void 0 ? patch.healthSuccessRecoveries : current.health_success_recoveries,
+    health_worker_url: patch.healthWorkerUrl !== void 0 ? patch.healthWorkerUrl.trim() || null : current.health_worker_url,
+    health_worker_token: patch.healthWorkerToken !== void 0 && patch.healthWorkerToken.trim() !== "" ? patch.healthWorkerToken : current.health_worker_token,
     updated_at: (/* @__PURE__ */ new Date()).toISOString()
   }).where(eq2(appSettings.id, SETTINGS_ID)).run();
   return getAppSettings(db, fallbacks);
@@ -644,6 +672,7 @@ __export(repos_exports, {
   getSyncJob: () => getSyncJob,
   insertBinding: () => insertBinding,
   insertDnsRecord: () => insertDnsRecord,
+  insertHealthProbeLog: () => insertHealthProbeLog,
   insertNotificationLog: () => insertNotificationLog,
   linkBindingRecord: () => linkBindingRecord,
   linkGroupDnsRecord: () => linkGroupDnsRecord,
@@ -670,6 +699,7 @@ __export(repos_exports, {
   listGroups: () => listGroups,
   listHealthCheckTargets: () => listHealthCheckTargets,
   listHealthChecks: () => listHealthChecks,
+  listHealthProbeLogForService: () => listHealthProbeLogForService,
   listIpHealthByServiceIds: () => listIpHealthByServiceIds,
   listIpHealthStatus: () => listIpHealthStatus,
   listNodes: () => listNodes,
@@ -1106,6 +1136,9 @@ function deleteService(db, id) {
   const result = db.delete(services).where(eq3(services.id, id)).run();
   if (result.changes === 0) throw new NotFoundError(`service ${id}`);
 }
+function normalizeHealthProvider(value) {
+  return value === "cloudflare" ? "cloudflare" : "local";
+}
 function mapServiceGroup(row) {
   return {
     id: row.id,
@@ -1123,6 +1156,7 @@ function mapServiceGroup(row) {
     health_check_interval_sec: row.health_check_interval_sec,
     health_check_timeout_ms: row.health_check_timeout_ms,
     health_check_verify_tls: row.health_check_verify_tls,
+    health_check_provider: normalizeHealthProvider(row.health_check_provider),
     created_at: row.created_at,
     updated_at: row.updated_at
   };
@@ -1149,7 +1183,8 @@ function createServiceGroup(db, name, groupType, icon, domain, lbPatch) {
     health_check_expected_status: lbPatch?.health_check_expected_status ?? null,
     health_check_interval_sec: lbPatch?.health_check_interval_sec ?? 30,
     health_check_timeout_ms: lbPatch?.health_check_timeout_ms ?? 3e3,
-    health_check_verify_tls: lbPatch?.health_check_verify_tls ?? false
+    health_check_verify_tls: lbPatch?.health_check_verify_tls ?? false,
+    health_check_provider: lbPatch?.health_check_provider ?? "local"
   }).returning({ id: serviceGroups.id }).get().id;
   return getServiceGroup(db, id);
 }
@@ -1179,6 +1214,8 @@ function updateServiceGroup(db, id, name, groupType, icon, domain, lbPatch) {
       update.health_check_timeout_ms = lbPatch.health_check_timeout_ms;
     if (lbPatch.health_check_verify_tls !== void 0)
       update.health_check_verify_tls = lbPatch.health_check_verify_tls;
+    if (lbPatch.health_check_provider !== void 0)
+      update.health_check_provider = lbPatch.health_check_provider;
   }
   const result = db.update(serviceGroups).set(update).where(eq3(serviceGroups.id, id)).run();
   if (result.changes === 0) throw new NotFoundError(`service group ${id}`);
@@ -1488,6 +1525,8 @@ function updateBindingLbConfig(db, bindingId, patch) {
     update.health_check_timeout_ms = patch.health_check_timeout_ms;
   if (patch.health_check_verify_tls !== void 0)
     update.health_check_verify_tls = patch.health_check_verify_tls;
+  if (patch.health_check_provider !== void 0)
+    update.health_check_provider = patch.health_check_provider;
   db.update(serviceBindings).set(update).where(eq3(serviceBindings.id, bindingId)).run();
 }
 function setBindingCnameTarget(db, bindingId, target) {
@@ -1546,7 +1585,7 @@ function dnsRecordMatchesHostname(recordName, hostname, zoneName) {
 var SERVICE_BINDING_SELECT_COLUMNS = `sb.id, sb.domain_id, sb.service_id, sb.hostname, sb.dns_record_id,
   sb.lb_mode, sb.health_check_enabled, sb.health_check_type, sb.health_check_port,
   sb.health_check_path, sb.health_check_expected_status, sb.health_check_interval_sec,
-  sb.health_check_timeout_ms, sb.health_check_verify_tls, sb.cname_target,
+  sb.health_check_timeout_ms, sb.health_check_verify_tls, sb.health_check_provider, sb.cname_target,
   d.zone_name, d.group_id, g.name AS group_name,
   s.name AS service_name, s.slug AS service_slug,
   dr.content AS target_ip, dr.sync_status,
@@ -1767,7 +1806,7 @@ function finishSyncJob(db, id, status, message) {
 function listIpHealthStatus(db, scope, refId) {
   return db.all(sql2`
     SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures,
-      last_checked_at, last_error
+      last_checked_at, last_error, colo, provider
     FROM ip_health_status
     WHERE scope = ${scope} AND ref_id = ${refId}
   `);
@@ -1891,7 +1930,11 @@ function listIpHealthByServiceIds(db, serviceIds) {
     SELECT sb.service_id AS service_id,
       ihs.ip AS ip,
       ${WORST_HEALTH_SQL} AS health_status,
-      MAX(ihs.latency_ms) AS health_latency_ms
+      MAX(ihs.latency_ms) AS health_latency_ms,
+      MAX(ihs.last_checked_at) AS last_checked_at,
+      MAX(ihs.last_error) AS last_error,
+      MAX(ihs.provider) AS provider,
+      MAX(ihs.colo) AS colo
     FROM ip_health_status ihs
     INNER JOIN service_bindings sb
       ON ihs.scope = 'binding' AND ihs.ref_id = sb.id
@@ -1907,7 +1950,11 @@ function listIpHealthByServiceIds(db, serviceIds) {
     list.push({
       ip: row.ip,
       status: parsed.health_status,
-      latency_ms: parsed.health_latency_ms
+      latency_ms: parsed.health_latency_ms,
+      last_checked_at: row.last_checked_at,
+      last_error: row.last_error,
+      provider: normalizeHealthProvider(row.provider),
+      colo: row.colo
     });
     result.set(row.service_id, list);
   }
@@ -1942,20 +1989,24 @@ function mergeHealthAggregates(parts) {
 function getIpHealthStatusRow(db, scope, refId, ip) {
   const rows = db.all(sql2`
     SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures,
-      consecutive_successes, last_checked_at, last_error
+      consecutive_successes, last_checked_at, last_error, colo, provider
     FROM ip_health_status
     WHERE scope = ${scope} AND ref_id = ${refId} AND ip = ${ip}
     LIMIT 1
   `);
   return rows[0] ?? null;
 }
-function upsertIpHealthStatus(db, scope, refId, ip, status, latencyMs, consecutiveFailures, lastError, consecutiveSuccesses = 0) {
+function upsertIpHealthStatus(db, scope, refId, ip, status, latencyMs, consecutiveFailures, lastError, consecutiveSuccesses = 0, extras) {
+  const colo = extras?.colo ?? null;
+  const provider = extras?.provider ?? "local";
   db.run(sql2`
     INSERT INTO ip_health_status
       (scope, ref_id, ip, status, latency_ms, consecutive_failures,
-       consecutive_successes, last_checked_at, last_error, created_at, updated_at)
+       consecutive_successes, last_checked_at, last_error, colo, provider,
+       created_at, updated_at)
     VALUES (${scope}, ${refId}, ${ip}, ${status}, ${latencyMs}, ${consecutiveFailures},
-      ${consecutiveSuccesses}, datetime('now'), ${lastError}, datetime('now'), datetime('now'))
+      ${consecutiveSuccesses}, datetime('now'), ${lastError}, ${colo}, ${provider},
+      datetime('now'), datetime('now'))
     ON CONFLICT(scope, ref_id, ip) DO UPDATE SET
       status = excluded.status,
       latency_ms = excluded.latency_ms,
@@ -1963,6 +2014,8 @@ function upsertIpHealthStatus(db, scope, refId, ip, status, latencyMs, consecuti
       consecutive_successes = excluded.consecutive_successes,
       last_checked_at = excluded.last_checked_at,
       last_error = excluded.last_error,
+      colo = excluded.colo,
+      provider = excluded.provider,
       updated_at = datetime('now')
   `);
 }
@@ -2025,7 +2078,8 @@ function listHealthCheckTargets(db) {
       sb.health_check_path AS path,
       sb.health_check_expected_status AS expected_status,
       sb.health_check_timeout_ms AS timeout_ms,
-      sb.health_check_verify_tls AS verify_tls
+      sb.health_check_verify_tls AS verify_tls,
+      COALESCE(sb.health_check_provider, 'local') AS provider
     FROM service_binding_ips sbi
     JOIN service_bindings sb ON sb.id = sbi.binding_id
     JOIN domains d ON d.id = sb.domain_id
@@ -2039,7 +2093,8 @@ function listHealthCheckTargets(db) {
       sg.health_check_path AS path,
       sg.health_check_expected_status AS expected_status,
       sg.health_check_timeout_ms AS timeout_ms,
-      sg.health_check_verify_tls AS verify_tls
+      sg.health_check_verify_tls AS verify_tls,
+      COALESCE(sg.health_check_provider, 'local') AS provider
     FROM service_binding_ips sbi
     JOIN service_bindings sb ON sb.id = sbi.binding_id
     JOIN services s ON s.id = sb.service_id
@@ -2059,7 +2114,8 @@ function listHealthCheckTargets(db) {
       sg.health_check_path AS path,
       sg.health_check_expected_status AS expected_status,
       sg.health_check_timeout_ms AS timeout_ms,
-      sg.health_check_verify_tls AS verify_tls
+      sg.health_check_verify_tls AS verify_tls,
+      COALESCE(sg.health_check_provider, 'local') AS provider
     FROM service_binding_ips sbi
     JOIN service_bindings sb ON sb.id = sbi.binding_id
     JOIN domains d ON d.id = sb.domain_id
@@ -2079,7 +2135,8 @@ function listHealthCheckTargets(db) {
       sb.health_check_path AS path,
       sb.health_check_expected_status AS expected_status,
       sb.health_check_timeout_ms AS timeout_ms,
-      sb.health_check_verify_tls AS verify_tls
+      sb.health_check_verify_tls AS verify_tls,
+      COALESCE(sb.health_check_provider, 'local') AS provider
     FROM service_bindings sb
     JOIN domains d ON d.id = sb.domain_id
     JOIN services s ON s.id = sb.service_id
@@ -2096,7 +2153,8 @@ function listHealthCheckTargets(db) {
       sg.health_check_path AS path,
       sg.health_check_expected_status AS expected_status,
       sg.health_check_timeout_ms AS timeout_ms,
-      sg.health_check_verify_tls AS verify_tls
+      sg.health_check_verify_tls AS verify_tls,
+      COALESCE(sg.health_check_provider, 'local') AS provider
     FROM service_bindings sb
     JOIN domains d ON d.id = sb.domain_id
     JOIN services s ON s.id = sb.service_id
@@ -2117,7 +2175,8 @@ function listHealthCheckTargets(db) {
     ...groupInheritedCnameBindingTargets
   ].map((t) => ({
     ...t,
-    verify_tls: Boolean(t.verify_tls)
+    verify_tls: Boolean(t.verify_tls),
+    provider: normalizeHealthProvider(t.provider)
   }));
 }
 function listDomainTags(db, domainId) {
@@ -2212,6 +2271,46 @@ function listDomainMonitorResultsForDomain(db, domainId, limit = 50) {
     LIMIT ${limit}
   `);
 }
+var HEALTH_PROBE_LOG_KEEP = 50;
+function insertHealthProbeLog(db, entry) {
+  db.insert(healthProbeLog).values({
+    scope: entry.scope,
+    ref_id: entry.refId,
+    ip: entry.ip,
+    provider: entry.provider,
+    status: entry.status,
+    ok: entry.ok,
+    latency_ms: entry.latencyMs,
+    colo: entry.colo,
+    error: entry.error
+  }).run();
+  db.run(sql2`
+    DELETE FROM health_probe_log
+    WHERE id IN (
+      SELECT id FROM health_probe_log
+      WHERE scope = ${entry.scope} AND ref_id = ${entry.refId} AND ip = ${entry.ip}
+      ORDER BY checked_at DESC, id DESC
+      LIMIT -1 OFFSET ${HEALTH_PROBE_LOG_KEEP}
+    )
+  `);
+}
+function listHealthProbeLogForService(db, serviceId, limit = 50) {
+  const rows = db.all(sql2`
+    SELECT l.id, l.scope, l.ref_id, l.ip, l.provider, l.status, l.ok,
+      l.latency_ms, l.colo, l.error, l.checked_at
+    FROM health_probe_log l
+    INNER JOIN service_bindings sb
+      ON l.scope = 'binding' AND l.ref_id = sb.id
+    WHERE sb.service_id = ${serviceId}
+    ORDER BY l.checked_at DESC, l.id DESC
+    LIMIT ${limit}
+  `);
+  return rows.map((row) => ({
+    ...row,
+    provider: normalizeHealthProvider(row.provider),
+    ok: Boolean(row.ok)
+  }));
+}
 function insertNotificationLog(db, kind, refType, refId, title, message) {
   db.insert(notificationLog).values({
     kind,
@@ -2249,6 +2348,7 @@ export {
   groups,
   healthCheck,
   healthChecks,
+  healthProbeLog,
   ipHealthStatus,
   listAudit,
   nodes,

packages/db/migrations/021_health_worker_xor.sql

@@ -0,0 +1,27 @@
+-- XOR health-check: persist provider on bindings/groups; Worker URL/token;
+-- colo + probe journal. Cloudflare = Worker edge probe, not Health Checks API.
+ALTER TABLE service_bindings ADD COLUMN health_check_provider TEXT NOT NULL DEFAULT 'local';
+ALTER TABLE service_groups ADD COLUMN health_check_provider TEXT NOT NULL DEFAULT 'local';
+
+ALTER TABLE app_settings ADD COLUMN health_worker_url TEXT;
+ALTER TABLE app_settings ADD COLUMN health_worker_token TEXT;
+
+ALTER TABLE ip_health_status ADD COLUMN colo TEXT;
+ALTER TABLE ip_health_status ADD COLUMN provider TEXT NOT NULL DEFAULT 'local';
+
+CREATE TABLE IF NOT EXISTS health_probe_log (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  scope TEXT NOT NULL,
+  ref_id INTEGER NOT NULL,
+  ip TEXT NOT NULL,
+  provider TEXT NOT NULL,
+  status TEXT NOT NULL,
+  ok INTEGER NOT NULL,
+  latency_ms INTEGER,
+  colo TEXT,
+  error TEXT,
+  checked_at TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+CREATE INDEX IF NOT EXISTS idx_health_probe_log_target
+  ON health_probe_log(scope, ref_id, ip, checked_at DESC);

packages/db/src/repos.ts

@@ -5,6 +5,7 @@ import type {
   DomainListItem,
   Group,
   GroupWithStats,
+  HealthCheckProvider,
   HealthCheckScope,
   HealthCheckTarget,
   HealthCheckType,
@@ -33,6 +34,7 @@ import {
   domains,
   groups,
   healthChecks,
+  healthProbeLog,
   ipHealthStatus,
   nodes,
   bindingNodes,
@@ -768,6 +770,10 @@ export function deleteService(db: Db, id: number): void {
 
 // --- Service Groups ---
 
+function normalizeHealthProvider(value: unknown): HealthCheckProvider {
+  return value === "cloudflare" ? "cloudflare" : "local";
+}
+
 function mapServiceGroup(row: typeof serviceGroups.$inferSelect): ServiceGroup {
   return {
     id: row.id,
@@ -785,6 +791,7 @@ function mapServiceGroup(row: typeof serviceGroups.$inferSelect): ServiceGroup {
     health_check_interval_sec: row.health_check_interval_sec,
     health_check_timeout_ms: row.health_check_timeout_ms,
     health_check_verify_tls: row.health_check_verify_tls,
+    health_check_provider: normalizeHealthProvider(row.health_check_provider),
     created_at: row.created_at,
     updated_at: row.updated_at,
   };
@@ -819,6 +826,7 @@ export interface ServiceGroupLbPatch {
   health_check_interval_sec?: number;
   health_check_timeout_ms?: number;
   health_check_verify_tls?: boolean;
+  health_check_provider?: HealthCheckProvider;
 }
 
 export function createServiceGroup(
@@ -845,6 +853,7 @@ export function createServiceGroup(
       health_check_interval_sec: lbPatch?.health_check_interval_sec ?? 30,
       health_check_timeout_ms: lbPatch?.health_check_timeout_ms ?? 3000,
       health_check_verify_tls: lbPatch?.health_check_verify_tls ?? false,
+      health_check_provider: lbPatch?.health_check_provider ?? "local",
     })
     .returning({ id: serviceGroups.id })
     .get()!.id;
@@ -885,6 +894,8 @@ export function updateServiceGroup(
       update.health_check_timeout_ms = lbPatch.health_check_timeout_ms;
     if (lbPatch.health_check_verify_tls !== undefined)
       update.health_check_verify_tls = lbPatch.health_check_verify_tls;
+    if (lbPatch.health_check_provider !== undefined)
+      update.health_check_provider = lbPatch.health_check_provider;
   }
   const result = db
     .update(serviceGroups)
@@ -1427,6 +1438,7 @@ export interface BindingLbPatch {
   health_check_interval_sec?: number;
   health_check_timeout_ms?: number;
   health_check_verify_tls?: boolean;
+  health_check_provider?: HealthCheckProvider;
 }
 
 export function updateBindingLbConfig(
@@ -1457,6 +1469,8 @@ export function updateBindingLbConfig(
     update.health_check_timeout_ms = patch.health_check_timeout_ms;
   if (patch.health_check_verify_tls !== undefined)
     update.health_check_verify_tls = patch.health_check_verify_tls;
+  if (patch.health_check_provider !== undefined)
+    update.health_check_provider = patch.health_check_provider;
   db.update(serviceBindings)
     .set(update)
     .where(eq(serviceBindings.id, bindingId))
@@ -1564,7 +1578,7 @@ function dnsRecordMatchesHostname(
 const SERVICE_BINDING_SELECT_COLUMNS = `sb.id, sb.domain_id, sb.service_id, sb.hostname, sb.dns_record_id,
   sb.lb_mode, sb.health_check_enabled, sb.health_check_type, sb.health_check_port,
   sb.health_check_path, sb.health_check_expected_status, sb.health_check_interval_sec,
-  sb.health_check_timeout_ms, sb.health_check_verify_tls, sb.cname_target,
+  sb.health_check_timeout_ms, sb.health_check_verify_tls, sb.health_check_provider, sb.cname_target,
   d.zone_name, d.group_id, g.name AS group_name,
   s.name AS service_name, s.slug AS service_slug,
   dr.content AS target_ip, dr.sync_status,
@@ -1943,7 +1957,7 @@ export function listIpHealthStatus(
   return db
     .all<IpHealthStatus>(sql`
     SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures,
-      last_checked_at, last_error
+      last_checked_at, last_error, colo, provider
     FROM ip_health_status
     WHERE scope = ${scope} AND ref_id = ${refId}
   `);
@@ -2108,6 +2122,10 @@ export type ServiceIpHealthRow = {
   ip: string;
   status: IpHealthState;
   latency_ms: number | null;
+  last_checked_at: string | null;
+  last_error: string | null;
+  provider: HealthCheckProvider;
+  colo: string | null;
 };
 
 /** Per-IP binding-scope health, worst status if the same IP is on several bindings. */
@@ -2126,11 +2144,19 @@ export function listIpHealthByServiceIds(
     ip: string;
     health_status: string | null;
     health_latency_ms: number | null;
+    last_checked_at: string | null;
+    last_error: string | null;
+    provider: string | null;
+    colo: string | null;
   }>(sql`
     SELECT sb.service_id AS service_id,
       ihs.ip AS ip,
       ${WORST_HEALTH_SQL} AS health_status,
-      MAX(ihs.latency_ms) AS health_latency_ms
+      MAX(ihs.latency_ms) AS health_latency_ms,
+      MAX(ihs.last_checked_at) AS last_checked_at,
+      MAX(ihs.last_error) AS last_error,
+      MAX(ihs.provider) AS provider,
+      MAX(ihs.colo) AS colo
     FROM ip_health_status ihs
     INNER JOIN service_bindings sb
       ON ihs.scope = 'binding' AND ihs.ref_id = sb.id
@@ -2147,6 +2173,10 @@ export function listIpHealthByServiceIds(
       ip: row.ip,
       status: parsed.health_status,
       latency_ms: parsed.health_latency_ms,
+      last_checked_at: row.last_checked_at,
+      last_error: row.last_error,
+      provider: normalizeHealthProvider(row.provider),
+      colo: row.colo,
     });
     result.set(row.service_id, list);
   }
@@ -2195,7 +2225,7 @@ export function getIpHealthStatusRow(
 ): IpHealthStatus | null {
   const rows = db.all<IpHealthStatus>(sql`
     SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures,
-      consecutive_successes, last_checked_at, last_error
+      consecutive_successes, last_checked_at, last_error, colo, provider
     FROM ip_health_status
     WHERE scope = ${scope} AND ref_id = ${refId} AND ip = ${ip}
     LIMIT 1
@@ -2213,13 +2243,18 @@ export function upsertIpHealthStatus(
   consecutiveFailures: number,
   lastError: string | null,
   consecutiveSuccesses = 0,
+  extras?: { colo?: string | null; provider?: HealthCheckProvider },
 ): void {
+  const colo = extras?.colo ?? null;
+  const provider = extras?.provider ?? "local";
   db.run(sql`
     INSERT INTO ip_health_status
       (scope, ref_id, ip, status, latency_ms, consecutive_failures,
-       consecutive_successes, last_checked_at, last_error, created_at, updated_at)
+       consecutive_successes, last_checked_at, last_error, colo, provider,
+       created_at, updated_at)
     VALUES (${scope}, ${refId}, ${ip}, ${status}, ${latencyMs}, ${consecutiveFailures},
-      ${consecutiveSuccesses}, datetime('now'), ${lastError}, datetime('now'), datetime('now'))
+      ${consecutiveSuccesses}, datetime('now'), ${lastError}, ${colo}, ${provider},
+      datetime('now'), datetime('now'))
     ON CONFLICT(scope, ref_id, ip) DO UPDATE SET
       status = excluded.status,
       latency_ms = excluded.latency_ms,
@@ -2227,6 +2262,8 @@ export function upsertIpHealthStatus(
       consecutive_successes = excluded.consecutive_successes,
       last_checked_at = excluded.last_checked_at,
       last_error = excluded.last_error,
+      colo = excluded.colo,
+      provider = excluded.provider,
       updated_at = datetime('now')
   `);
 }
@@ -2321,7 +2358,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
       sb.health_check_path AS path,
       sb.health_check_expected_status AS expected_status,
       sb.health_check_timeout_ms AS timeout_ms,
-      sb.health_check_verify_tls AS verify_tls
+      sb.health_check_verify_tls AS verify_tls,
+      COALESCE(sb.health_check_provider, 'local') AS provider
     FROM service_binding_ips sbi
     JOIN service_bindings sb ON sb.id = sbi.binding_id
     JOIN domains d ON d.id = sb.domain_id
@@ -2339,7 +2377,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
       sg.health_check_path AS path,
       sg.health_check_expected_status AS expected_status,
       sg.health_check_timeout_ms AS timeout_ms,
-      sg.health_check_verify_tls AS verify_tls
+      sg.health_check_verify_tls AS verify_tls,
+      COALESCE(sg.health_check_provider, 'local') AS provider
     FROM service_binding_ips sbi
     JOIN service_bindings sb ON sb.id = sbi.binding_id
     JOIN services s ON s.id = sb.service_id
@@ -2364,7 +2403,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
       sg.health_check_path AS path,
       sg.health_check_expected_status AS expected_status,
       sg.health_check_timeout_ms AS timeout_ms,
-      sg.health_check_verify_tls AS verify_tls
+      sg.health_check_verify_tls AS verify_tls,
+      COALESCE(sg.health_check_provider, 'local') AS provider
     FROM service_binding_ips sbi
     JOIN service_bindings sb ON sb.id = sbi.binding_id
     JOIN domains d ON d.id = sb.domain_id
@@ -2386,7 +2426,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
       sb.health_check_path AS path,
       sb.health_check_expected_status AS expected_status,
       sb.health_check_timeout_ms AS timeout_ms,
-      sb.health_check_verify_tls AS verify_tls
+      sb.health_check_verify_tls AS verify_tls,
+      COALESCE(sb.health_check_provider, 'local') AS provider
     FROM service_bindings sb
     JOIN domains d ON d.id = sb.domain_id
     JOIN services s ON s.id = sb.service_id
@@ -2406,7 +2447,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
       sg.health_check_path AS path,
       sg.health_check_expected_status AS expected_status,
       sg.health_check_timeout_ms AS timeout_ms,
-      sg.health_check_verify_tls AS verify_tls
+      sg.health_check_verify_tls AS verify_tls,
+      COALESCE(sg.health_check_provider, 'local') AS provider
     FROM service_bindings sb
     JOIN domains d ON d.id = sb.domain_id
     JOIN services s ON s.id = sb.service_id
@@ -2429,6 +2471,7 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] {
   ].map((t) => ({
     ...t,
     verify_tls: Boolean(t.verify_tls),
+    provider: normalizeHealthProvider(t.provider),
   }));
 }
 
@@ -2633,6 +2676,94 @@ export function listDomainMonitorResultsForDomain(
   `);
 }
 
+// --- Health probe journal ---
+
+const HEALTH_PROBE_LOG_KEEP = 50;
+
+export function insertHealthProbeLog(
+  db: Db,
+  entry: {
+    scope: HealthCheckScope;
+    refId: number;
+    ip: string;
+    provider: HealthCheckProvider;
+    status: string;
+    ok: boolean;
+    latencyMs: number | null;
+    colo: string | null;
+    error: string | null;
+  },
+): void {
+  db.insert(healthProbeLog)
+    .values({
+      scope: entry.scope,
+      ref_id: entry.refId,
+      ip: entry.ip,
+      provider: entry.provider,
+      status: entry.status,
+      ok: entry.ok,
+      latency_ms: entry.latencyMs,
+      colo: entry.colo,
+      error: entry.error,
+    })
+    .run();
+  db.run(sql`
+    DELETE FROM health_probe_log
+    WHERE id IN (
+      SELECT id FROM health_probe_log
+      WHERE scope = ${entry.scope} AND ref_id = ${entry.refId} AND ip = ${entry.ip}
+      ORDER BY checked_at DESC, id DESC
+      LIMIT -1 OFFSET ${HEALTH_PROBE_LOG_KEEP}
+    )
+  `);
+}
+
+export function listHealthProbeLogForService(
+  db: Db,
+  serviceId: number,
+  limit = 50,
+): {
+  id: number;
+  scope: string;
+  ref_id: number;
+  ip: string;
+  provider: HealthCheckProvider;
+  status: string;
+  ok: boolean;
+  latency_ms: number | null;
+  colo: string | null;
+  error: string | null;
+  checked_at: string;
+}[] {
+  const rows = db.all<{
+    id: number;
+    scope: string;
+    ref_id: number;
+    ip: string;
+    provider: string;
+    status: string;
+    ok: number;
+    latency_ms: number | null;
+    colo: string | null;
+    error: string | null;
+    checked_at: string;
+  }>(sql`
+    SELECT l.id, l.scope, l.ref_id, l.ip, l.provider, l.status, l.ok,
+      l.latency_ms, l.colo, l.error, l.checked_at
+    FROM health_probe_log l
+    INNER JOIN service_bindings sb
+      ON l.scope = 'binding' AND l.ref_id = sb.id
+    WHERE sb.service_id = ${serviceId}
+    ORDER BY l.checked_at DESC, l.id DESC
+    LIMIT ${limit}
+  `);
+  return rows.map((row) => ({
+    ...row,
+    provider: normalizeHealthProvider(row.provider),
+    ok: Boolean(row.ok),
+  }));
+}
+
 // --- Notification log ---
 
 export function insertNotificationLog(

packages/db/src/schema.ts

@@ -64,6 +64,7 @@ export const serviceGroups = sqliteTable("service_groups", {
   health_check_verify_tls: integer("health_check_verify_tls", { mode: "boolean" })
     .notNull()
     .default(false),
+  health_check_provider: text("health_check_provider").notNull().default("local"),
   created_at: text("created_at")
     .notNull()
     .default(sql`datetime('now')`),
@@ -165,6 +166,9 @@ export const serviceBindings = sqliteTable(
     })
       .notNull()
       .default(false),
+    health_check_provider: text("health_check_provider")
+      .notNull()
+      .default("local"),
     routing_strategy: text("routing_strategy").notNull().default("round_robin"),
     operation_version: integer("operation_version").notNull().default(0),
     created_at: text("created_at")
@@ -354,6 +358,8 @@ export const ipHealthStatus = sqliteTable(
       .default(0),
     last_checked_at: text("last_checked_at"),
     last_error: text("last_error"),
+    colo: text("colo"),
+    provider: text("provider").notNull().default("local"),
     created_at: text("created_at")
       .notNull()
       .default(sql`datetime('now')`),
@@ -385,6 +391,8 @@ export const appSettings = sqliteTable("app_settings", {
   health_down_failures: integer("health_down_failures"),
   health_latency_warn_ms: integer("health_latency_warn_ms"),
   health_success_recoveries: integer("health_success_recoveries"),
+  health_worker_url: text("health_worker_url"),
+  health_worker_token: text("health_worker_token"),
   created_at: text("created_at")
     .notNull()
     .default(sql`datetime('now')`),
@@ -441,6 +449,22 @@ export const domainMonitorResults = sqliteTable("domain_monitor_results", {
     .default(sql`datetime('now')`),
 });
 
+export const healthProbeLog = sqliteTable("health_probe_log", {
+  id: integer("id").primaryKey({ autoIncrement: true }),
+  scope: text("scope").notNull(),
+  ref_id: integer("ref_id").notNull(),
+  ip: text("ip").notNull(),
+  provider: text("provider").notNull(),
+  status: text("status").notNull(),
+  ok: integer("ok", { mode: "boolean" }).notNull(),
+  latency_ms: integer("latency_ms"),
+  colo: text("colo"),
+  error: text("error"),
+  checked_at: text("checked_at")
+    .notNull()
+    .default(sql`datetime('now')`),
+});
+
 export const notificationLog = sqliteTable("notification_log", {
   id: integer("id").primaryKey({ autoIncrement: true }),
   kind: text("kind").notNull(),
@@ -494,6 +518,7 @@ export const schema = {
   domainTags,
   domainMonitors,
   domainMonitorResults,
+  healthProbeLog,
   notificationLog,
   auditLog,
 };

packages/db/src/settings-repo.ts

@@ -19,6 +19,8 @@ export type AppSettingsDto = {
   vpsTrackerSyncEnabled: boolean;
   vpsTrackerLastSyncAt: string | null;
   showQuickActions: boolean;
+  healthWorkerUrl: string;
+  healthWorkerTokenSet: boolean;
 } & HealthEngineSettings;
 
 export type AppSettingsPatch = {
@@ -31,9 +33,14 @@ export type AppSettingsPatch = {
   healthDownFailures?: number;
   healthLatencyWarnMs?: number;
   healthSuccessRecoveries?: number;
+  healthWorkerUrl?: string;
+  healthWorkerToken?: string;
 };
 
-export type HealthEngineFallbacks = HealthEngineSettings;
+export type HealthEngineFallbacks = HealthEngineSettings & {
+  healthWorkerUrl: string;
+  healthWorkerTokenSet: boolean;
+};
 
 function coalesceInt(value: number | null | undefined, fallback: number): number {
   return value == null || Number.isNaN(value) || value < 1 ? fallback : value;
@@ -49,6 +56,8 @@ function toDto(
     healthDownFailures: 2,
     healthLatencyWarnMs: 1000,
     healthSuccessRecoveries: 2,
+    healthWorkerUrl: "",
+    healthWorkerTokenSet: false,
   };
   return {
     id: row.id,
@@ -77,6 +86,9 @@ function toDto(
       row.health_success_recoveries,
       env.healthSuccessRecoveries,
     ),
+    healthWorkerUrl: row.health_worker_url?.trim() || env.healthWorkerUrl,
+    healthWorkerTokenSet:
+      Boolean(row.health_worker_token?.trim()) || env.healthWorkerTokenSet,
   };
 }
 
@@ -103,6 +115,8 @@ export function getAppSettingsSecrets(db: Db): {
   vpsTrackerUrl: string;
   vpsTrackerIntegrationToken: string;
   vpsTrackerSyncEnabled: boolean;
+  healthWorkerUrl: string;
+  healthWorkerToken: string;
 } {
   const row = db
     .select()
@@ -114,6 +128,8 @@ export function getAppSettingsSecrets(db: Db): {
     vpsTrackerIntegrationToken:
       row?.vps_tracker_integration_token?.trim() ?? "",
     vpsTrackerSyncEnabled: Boolean(row?.vps_tracker_sync_enabled),
+    healthWorkerUrl: row?.health_worker_url?.trim() ?? "",
+    healthWorkerToken: row?.health_worker_token?.trim() ?? "",
   };
 }
 
@@ -176,6 +192,15 @@ export function updateAppSettings(
         patch.healthSuccessRecoveries !== undefined
           ? patch.healthSuccessRecoveries
           : current.health_success_recoveries,
+      health_worker_url:
+        patch.healthWorkerUrl !== undefined
+          ? patch.healthWorkerUrl.trim() || null
+          : current.health_worker_url,
+      health_worker_token:
+        patch.healthWorkerToken !== undefined &&
+        patch.healthWorkerToken.trim() !== ""
+          ? patch.healthWorkerToken
+          : current.health_worker_token,
       updated_at: new Date().toISOString(),
     })
     .where(eq(appSettings.id, SETTINGS_ID))