feat(health-checks): enhance health check configuration and logging

- Added new health worker URL and token fields to the AppConfig interface, allowing for Cloudflare Worker integration.
- Updated health check routes to utilize the new worker configuration, enabling dynamic health checks via Cloudflare Workers.
- Introduced a health log endpoint for services, providing detailed logs of health probe results.
- Enhanced health check service logic to support both local and Cloudflare Worker providers, improving flexibility in health monitoring.
- Updated UI components to reflect changes in health check provider settings and display relevant health information.

This commit significantly improves the health check management capabilities, allowing for better integration with Cloudflare Workers and enhanced logging features.

apps/api/src/config.ts

@@ -17,6 +17,8 @@ export interface AppConfig {
   healthLatencyWarnMs: number;
   /** Min pause between probes to different physical targets (same IP is probed once). */
   healthProbeGapMs: number;
+  healthWorkerUrl: string;
+  healthWorkerToken: string;
   logLevel: string;
   /** Portal SSO — when true, require portal JWT with apps includes cfdm */
   authRequired: boolean;
@@ -61,6 +63,8 @@ export function loadConfig(): AppConfig {
     healthLatencyWarnMs:
       Number(process.env.HEALTH_LATENCY_WARN_MS ?? "1000") || 1000,
     healthProbeGapMs: Number(process.env.HEALTH_PROBE_GAP_MS ?? "2000") || 2000,
+    healthWorkerUrl: (process.env.HEALTH_WORKER_URL ?? "").trim(),
+    healthWorkerToken: (process.env.HEALTH_WORKER_TOKEN ?? "").trim(),
     logLevel: process.env.LOG_LEVEL ?? "info",
     authRequired: boolEnv(process.env.AUTH_REQUIRED, false),
     authIssuer:

apps/api/src/routes/health-check.ts

@@ -1,8 +1,12 @@
 import type { FastifyInstance } from "fastify";
 import { healthStatusQuerySchema } from "@cfdm/shared";
-import { repos } from "@cfdm/db";
+import { getAppSettings, repos } from "@cfdm/db";
 import * as healthCheckService from "../services/health-check-service.js";
 import * as serviceConfigService from "../services/service-config-service.js";
+import {
+  healthEngineFallbacksFromConfig,
+  resolveWorkerProbeConfig,
+} from "../services/health-check-scheduler.js";
 
 export async function healthCheckRoutes(app: FastifyInstance) {
   app.get("/health-status", async (request) => {
@@ -16,15 +20,20 @@ export async function healthCheckRoutes(app: FastifyInstance) {
 
   app.post("/health-check/run", async (request) => {
     const config = request.server.config;
+    const settings = getAppSettings(
+      request.server.db,
+      healthEngineFallbacksFromConfig(config),
+    );
     const thresholds = {
-      degradedFailures: config.healthDegradedFailures,
-      downFailures: config.healthDownFailures,
-      latencyWarnMs: config.healthLatencyWarnMs,
-      successRecoveries: config.healthSuccessRecoveries,
+      degradedFailures: settings.healthDegradedFailures,
+      downFailures: settings.healthDownFailures,
+      latencyWarnMs: settings.healthLatencyWarnMs,
+      successRecoveries: settings.healthSuccessRecoveries,
     };
     const checked = await healthCheckService.runAllChecks(request.server.db, {
       thresholds,
       probeGapMs: config.healthProbeGapMs,
+      worker: resolveWorkerProbeConfig(request.server.db, config),
       onStatusChange: async (target, prev, next) => {
         try {
           const label =

apps/api/src/routes/services.ts

@@ -65,6 +65,14 @@ export async function serviceRoutes(app: FastifyInstance) {
     return serviceConfig.getView(request.server.db, Number(id));
   });
 
+  app.get("/services/:id/health-log", async (request) => {
+    const { id } = request.params as { id: string };
+    repos.getService(request.server.db, Number(id));
+    return {
+      items: repos.listHealthProbeLogForService(request.server.db, Number(id)),
+    };
+  });
+
   app.get("/services/:id/overview", async (request) => {
     const { id } = request.params as { id: string };
     return nodeService.getOverview(request.server.db, Number(id));

apps/api/src/services/health-check-scheduler.ts

@@ -2,6 +2,7 @@ import type { FastifyInstance } from "fastify";
 import { AsyncTask, CronJob } from "toad-scheduler";
 import {
   getAppSettings,
+  getAppSettingsSecrets,
   type HealthEngineFallbacks,
 } from "@cfdm/db";
 import { repos } from "@cfdm/db";
@@ -27,9 +28,23 @@ export function healthEngineFallbacksFromConfig(
     healthDownFailures: config.healthDownFailures,
     healthLatencyWarnMs: config.healthLatencyWarnMs,
     healthSuccessRecoveries: config.healthSuccessRecoveries,
+    healthWorkerUrl: config.healthWorkerUrl,
+    healthWorkerTokenSet: Boolean(config.healthWorkerToken),
   };
 }
 
+export function resolveWorkerProbeConfig(
+  db: import("@cfdm/db").Db,
+  config: AppConfig,
+): { url: string; token: string } | null {
+  const settings = getAppSettings(db, healthEngineFallbacksFromConfig(config));
+  const secrets = getAppSettingsSecrets(db);
+  const url = settings.healthWorkerUrl.trim();
+  const token = (secrets.healthWorkerToken || config.healthWorkerToken).trim();
+  if (!url || !token) return null;
+  return { url, token };
+}
+
 export function assertValidHealthCron(expr: string): void {
   const cronExpression = expr.trim();
   const parts = cronExpression.split(/\s+/).filter(Boolean);
@@ -66,6 +81,7 @@ export function createHealthCheckTask(
       const n = await healthCheckService.runAllChecks(app.db, {
         thresholds,
         probeGapMs: config.healthProbeGapMs,
+        worker: resolveWorkerProbeConfig(app.db, config),
         onStatusChange: async (target, prev, next) => {
           try {
             const label =

apps/api/src/services/health-check-service.ts

@@ -6,6 +6,11 @@ import { repos } from "@cfdm/db";
 import type { HealthCheckTarget, IpHealthState } from "@cfdm/shared";
 import { AppError } from "../errors.js";
 import { nextHealthState } from "./health/state-machine.js";
+import { LocalHealthCheckProvider } from "./health/local.js";
+import {
+  CloudflareWorkerHealthCheckProvider,
+  workerNotConfiguredResult,
+} from "./health/worker.js";
 
 export interface HealthCheckThresholds {
   degradedFailures: number;
@@ -18,6 +23,7 @@ export interface ProbeResult {
   ok: boolean;
   latencyMs: number;
   error: string | null;
+  colo?: string | null;
 }
 
 /** Bracket IPv6 for URL authority; leave IPv4/hostname as-is. */
@@ -261,6 +267,8 @@ export interface RunAllChecksOptions {
   thresholds: HealthCheckThresholds;
   /** Pause between unique physical probes (default 2000). Same IP is only probed once. */
   probeGapMs?: number;
+  /** Cloudflare Worker URL+token. Missing → cloudflare targets fail, never Local fallback. */
+  worker?: { url: string; token: string } | null;
   onStatusChange?: (
     target: HealthCheckTarget,
     prevState: IpHealthState | null,
@@ -277,21 +285,34 @@ function sleep(ms: number): Promise<void> {
  * so anti-bot / rate-limit on the origin is not tripped by back-to-back checks.
  */
 export function physicalProbeKey(target: HealthCheckTarget): string {
+  const kind = target.provider === "cloudflare" ? "cloudflare" : "local";
   const port = target.port ?? (target.type === "http" ? 80 : 80);
   const ip = String(target.ip || "").trim().toLowerCase();
   if (target.type === "http") {
     const path = (target.path?.trim() || "/") || "/";
     const expected = target.expected_status ?? "";
-    return `http|${ip}|${port}|${path}|${expected}`;
+    return `${kind}|http|${ip}|${port}|${path}|${expected}`;
   }
-  if (target.type === "tcp") return `tcp|${ip}|${port}`;
+  if (target.type === "tcp") return `${kind}|tcp|${ip}|${port}`;
   if (target.type === "ping") {
-    return `ping|${String(target.hostname || target.ip || "").trim().toLowerCase()}`;
+    return `${kind}|ping|${String(target.hostname || target.ip || "").trim().toLowerCase()}`;
   }
   if (target.type === "dns") {
-    return `dns|${String(target.hostname || target.ip || "").trim().toLowerCase()}`;
+    return `${kind}|dns|${String(target.hostname || target.ip || "").trim().toLowerCase()}`;
   }
-  return `${target.type}|${ip}|${port}`;
+  return `${kind}|${target.type}|${ip}|${port}`;
+}
+
+async function executeProbe(
+  target: HealthCheckTarget,
+  local: LocalHealthCheckProvider,
+  worker: CloudflareWorkerHealthCheckProvider | null,
+): Promise<ProbeResult> {
+  if (target.provider === "cloudflare") {
+    if (!worker) return workerNotConfiguredResult();
+    return worker.probe(target);
+  }
+  return local.probe(target);
 }
 
 export async function runAllChecks(
@@ -300,6 +321,11 @@ export async function runAllChecks(
 ): Promise<number> {
   const targets = repos.listHealthCheckTargets(db);
   const gapMs = Math.max(0, options.probeGapMs ?? 2000);
+  const local = new LocalHealthCheckProvider();
+  const worker =
+    options.worker?.url && options.worker.token
+      ? new CloudflareWorkerHealthCheckProvider(options.worker)
+      : null;
 
   const byPhysical = new Map<string, HealthCheckTarget[]>();
   for (const target of targets) {
@@ -319,7 +345,7 @@ export async function runAllChecks(
     // Prefer binding hostname for SNI when several scopes share one IP.
     const representative =
       group.find((t) => t.scope === "binding") ?? group[0]!;
-    const result = await probeTarget(representative);
+    const result = await executeProbe(representative, local, worker);
 
     for (const target of group) {
       const prev = repos.getIpHealthStatusRow(
@@ -343,6 +369,7 @@ export async function runAllChecks(
       const prevState: IpHealthState | null = prev
         ? (prev.status as IpHealthState)
         : null;
+      const provider = target.provider === "cloudflare" ? "cloudflare" : "local";
       repos.upsertIpHealthStatus(
         db,
         target.scope,
@@ -353,7 +380,19 @@ export async function runAllChecks(
         failures,
         result.error,
         successes,
+        { colo: result.colo ?? null, provider },
       );
+      repos.insertHealthProbeLog(db, {
+        scope: target.scope,
+        refId: target.ref_id,
+        ip: target.ip,
+        provider,
+        status: state,
+        ok: result.ok,
+        latencyMs: result.latencyMs,
+        colo: result.colo ?? null,
+        error: result.error,
+      });
       const matchedNode = repos.findNodeByIp(db, target.ip);
       if (matchedNode && matchedNode.enabled) {
         repos.updateNode(db, matchedNode.id, {
@@ -392,6 +431,7 @@ export async function runDomainMonitors(
       expected_status: monitor.expected_status,
       timeout_ms: monitor.timeout_ms,
       verify_tls: false,
+      provider: "local",
     };
     let result: ProbeResult;
     if (monitor.type === "http") {

apps/api/src/services/health/worker.ts

@@ -0,0 +1,93 @@
+import type { HealthCheckTarget } from "@cfdm/shared";
+import type { ProbeResult } from "../health-check-service.js";
+import type { HealthCheckProvider } from "./provider.js";
+
+export interface WorkerProbeConfig {
+  url: string;
+  token: string;
+}
+
+const WORKER_NOT_CONFIGURED = "Cloudflare Worker не настроен (URL и токен)";
+
+export class CloudflareWorkerHealthCheckProvider implements HealthCheckProvider {
+  readonly kind = "cloudflare" as const;
+
+  constructor(private readonly config: WorkerProbeConfig) {}
+
+  async probe(target: HealthCheckTarget): Promise<ProbeResult> {
+    const base = this.config.url.replace(/\/$/, "");
+    if (!base || !this.config.token) {
+      return {
+        ok: false,
+        latencyMs: 0,
+        error: WORKER_NOT_CONFIGURED,
+        colo: null,
+      };
+    }
+    const timeoutMs = Math.max(100, target.timeout_ms ?? 3000);
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs + 2500);
+    try {
+      const res = await fetch(`${base}/probe`, {
+        method: "POST",
+        headers: {
+          Authorization: `Bearer ${this.config.token}`,
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({
+          type: target.type === "http" ? "http" : "tcp",
+          ip: target.ip,
+          hostname: target.hostname,
+          port: target.port ?? (target.type === "http" ? 80 : 80),
+          path: target.path ?? "/",
+          expected_status: target.expected_status,
+          timeout_ms: timeoutMs,
+          verify_tls: Boolean(target.verify_tls),
+          method: "GET",
+        }),
+        signal: controller.signal,
+      });
+      if (!res.ok) {
+        const text = await res.text().catch(() => "");
+        return {
+          ok: false,
+          latencyMs: 0,
+          error: `Worker HTTP ${res.status}${text ? `: ${text.slice(0, 180)}` : ""}`,
+          colo: null,
+        };
+      }
+      const body = (await res.json()) as {
+        ok?: boolean;
+        latencyMs?: number;
+        error?: string | null;
+        colo?: string | null;
+      };
+      const ok = Boolean(body.ok);
+      return {
+        ok,
+        latencyMs: typeof body.latencyMs === "number" ? body.latencyMs : 0,
+        error: ok ? null : (body.error ?? "probe failed"),
+        colo: body.colo ?? null,
+      };
+    } catch (err) {
+      const message =
+        err instanceof Error
+          ? err.name === "AbortError"
+            ? "Worker timeout"
+            : err.message
+          : "Worker probe failed";
+      return { ok: false, latencyMs: 0, error: message, colo: null };
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+}
+
+export function workerNotConfiguredResult(): ProbeResult {
+  return {
+    ok: false,
+    latencyMs: 0,
+    error: WORKER_NOT_CONFIGURED,
+    colo: null,
+  };
+}

apps/api/src/services/service-config-service.ts

@@ -50,6 +50,7 @@ export interface ServiceDomainInput {
   health_check_interval_sec?: number;
   health_check_timeout_ms?: number;
   health_check_verify_tls?: boolean;
+  health_check_provider?: "local" | "cloudflare";
 }
 
 export interface ToggleRequest {
@@ -70,6 +71,7 @@ export interface ServiceGroupBody {
   health_check_interval_sec?: number;
   health_check_timeout_ms?: number;
   health_check_verify_tls?: boolean;
+  health_check_provider?: "local" | "cloudflare";
 }
 
 export interface UpdateServiceGroupBody {
@@ -86,6 +88,7 @@ export interface UpdateServiceGroupBody {
   health_check_interval_sec?: number;
   health_check_timeout_ms?: number;
   health_check_verify_tls?: boolean;
+  health_check_provider?: "local" | "cloudflare";
 }
 
 export interface UpdateServiceConfigRequest {
@@ -291,6 +294,7 @@ async function buildView(db: Db, serviceId: number): Promise<ServiceView> {
       health_check_interval_sec: binding.health_check_interval_sec,
       health_check_timeout_ms: binding.health_check_timeout_ms,
       health_check_verify_tls: binding.health_check_verify_tls,
+      health_check_provider: binding.health_check_provider ?? "local",
       sync_status: aggregateSyncStatus(statuses),
     };
   });
@@ -334,6 +338,10 @@ function attachServiceHealth(
         ip,
         status: row?.status ?? ("unknown" as const),
         latency_ms: row?.latency_ms ?? null,
+        last_checked_at: row?.last_checked_at ?? null,
+        last_error: row?.last_error ?? null,
+        provider: row?.provider ?? "local",
+        colo: row?.colo ?? null,
       };
     });
     return {
@@ -1154,7 +1162,8 @@ export async function updateConfig(
         input.health_check_expected_status !== undefined ||
         input.health_check_interval_sec !== undefined ||
         input.health_check_timeout_ms !== undefined ||
-        input.health_check_verify_tls !== undefined
+        input.health_check_verify_tls !== undefined ||
+        input.health_check_provider !== undefined
       ) {
         repos.updateBindingLbConfig(db, binding.id, {
           lb_mode: input.lb_mode,
@@ -1166,6 +1175,7 @@ export async function updateConfig(
           health_check_interval_sec: input.health_check_interval_sec,
           health_check_timeout_ms: input.health_check_timeout_ms,
           health_check_verify_tls: input.health_check_verify_tls,
+          health_check_provider: input.health_check_provider,
         });
       }
 
@@ -1267,6 +1277,7 @@ export async function createGroup(
       health_check_interval_sec: body.health_check_interval_sec,
       health_check_timeout_ms: body.health_check_timeout_ms,
       health_check_verify_tls: body.health_check_verify_tls,
+      health_check_provider: body.health_check_provider,
     },
   );
 }
@@ -1303,6 +1314,7 @@ export async function updateGroup(
       health_check_interval_sec: body.health_check_interval_sec,
       health_check_timeout_ms: body.health_check_timeout_ms,
       health_check_verify_tls: body.health_check_verify_tls,
+      health_check_provider: body.health_check_provider,
     },
   );
   if (!domain && group.enabled) {