feat(health-checks): enhance health check configuration and logging

- Added new health worker URL and token fields to the AppConfig interface, allowing for Cloudflare Worker integration.
- Updated health check routes to utilize the new worker configuration, enabling dynamic health checks via Cloudflare Workers.
- Introduced a health log endpoint for services, providing detailed logs of health probe results.
- Enhanced health check service logic to support both local and Cloudflare Worker providers, improving flexibility in health monitoring.
- Updated UI components to reflect changes in health check provider settings and display relevant health information.

This commit significantly improves the health check management capabilities, allowing for better integration with Cloudflare Workers and enhanced logging features.

apps/api/test/health-check.test.ts

@@ -56,6 +56,7 @@ describe("health-check probeTarget", () => {
       expected_status: null,
       timeout_ms: 1000,
       verify_tls: false,
+      provider: "local",
     };
     const result = await healthCheckService.probeTarget(target);
     expect(result.ok).toBe(true);
@@ -75,6 +76,7 @@ describe("health-check probeTarget", () => {
       expected_status: null,
       timeout_ms: 500,
       verify_tls: false,
+      provider: "local",
     };
     const result = await healthCheckService.probeTarget(target);
     expect(result.ok).toBe(false);
@@ -107,6 +109,7 @@ describe("health-check probeTarget", () => {
         expected_status: 200,
         timeout_ms: 1000,
         verify_tls: false,
+        provider: "local",
       };
       const bindingTarget: HealthCheckTarget = {
         ...groupTarget,
@@ -142,6 +145,7 @@ describe("health-check probeTarget", () => {
       expected_status: null,
       timeout_ms: 3000,
       verify_tls: false,
+      provider: "local",
     };
     const binding: HealthCheckTarget = {
       ...group,

apps/api/test/health-worker.test.ts

@@ -0,0 +1,215 @@
+import { createServer, type Server as HttpServer } from "node:http";
+import { describe, expect, it } from "vitest";
+import { buildApp } from "../src/app.js";
+import { loadConfig } from "../src/config.js";
+import { repos, type Db } from "@cfdm/db";
+import * as healthCheckService from "../src/services/health-check-service.js";
+
+async function authHeaders(app: Awaited<ReturnType<typeof buildApp>>) {
+  const res = await app.inject({
+    method: "POST",
+    url: "/api/v1/auth/login",
+    payload: { username: "admin", password: "admin" },
+  });
+  expect(res.statusCode).toBe(200);
+  const { token } = res.json() as { token: string };
+  return { authorization: `Bearer ${token}` };
+}
+
+function startWorkerMock(handler: (req: {
+  url?: string;
+  headers: Record<string, string | string[] | undefined>;
+  body: string;
+}) => { status: number; json: unknown } | "hang"): Promise<{
+  server: HttpServer;
+  url: string;
+}> {
+  return new Promise((resolve) => {
+    const server = createServer((req, res) => {
+      const chunks: Buffer[] = [];
+      req.on("data", (chunk) => chunks.push(chunk as Buffer));
+      req.on("end", () => {
+        const result = handler({
+          url: req.url,
+          headers: req.headers,
+          body: Buffer.concat(chunks).toString("utf8"),
+        });
+        if (result === "hang") return;
+        res.writeHead(result.status, { "content-type": "application/json" });
+        res.end(JSON.stringify(result.json));
+      });
+    });
+    server.listen(0, "127.0.0.1", () => {
+      const address = server.address();
+      const port = typeof address === "object" && address ? address.port : 0;
+      resolve({ server, url: `http://127.0.0.1:${port}` });
+    });
+  });
+}
+
+async function seedBinding(
+  db: Db,
+  opts: { provider: "local" | "cloudflare"; ip: string },
+) {
+  const domain = repos.createDomain(db, null, "example.com", "zone-1");
+  const service = repos.createService(db, "Panel", "panel");
+  repos.setServiceEnabled(db, service.id, true);
+  repos.replaceServiceIps(db, service.id, [opts.ip]);
+  const binding = repos.insertBinding(db, domain.id, service.id, "panel", null);
+  repos.replaceBindingIpsWithMeta(db, binding.id, [
+    { ip: opts.ip, weight: 1, priority: 1 },
+  ]);
+  repos.updateBindingLbConfig(db, binding.id, {
+    health_check_enabled: true,
+    health_check_type: "tcp",
+    health_check_port: 1,
+    health_check_timeout_ms: 400,
+    health_check_provider: opts.provider,
+  });
+  return { service, binding, domain };
+}
+
+const thresholds = {
+  degradedFailures: 1,
+  downFailures: 2,
+  latencyWarnMs: 1000,
+  successRecoveries: 2,
+};
+
+describe("health-check XOR worker", () => {
+  it("lists only local providers when no cloudflare bindings", async () => {
+    const app = await buildApp({
+      config: { ...loadConfig(), staticDir: null },
+      memory: true,
+    });
+    await seedBinding(app.db, {
+      provider: "local",
+      ip: "10.0.0.1",
+    });
+    const targets = repos.listHealthCheckTargets(app.db);
+    expect(targets.length).toBeGreaterThan(0);
+    expect(targets.every((t) => t.provider === "local")).toBe(true);
+    expect(targets.some((t) => t.provider === "cloudflare")).toBe(false);
+    await app.close();
+  });
+
+  it("cloudflare without worker URL does not fall back to local", async () => {
+    const app = await buildApp({
+      config: { ...loadConfig(), staticDir: null },
+      memory: true,
+    });
+    const { binding } = await seedBinding(app.db, {
+      provider: "cloudflare",
+      ip: "127.0.0.1",
+    });
+    await healthCheckService.runAllChecks(app.db, {
+      thresholds,
+      probeGapMs: 0,
+      worker: null,
+    });
+    const row = repos.getIpHealthStatusRow(
+      app.db,
+      "binding",
+      binding.id,
+      "127.0.0.1",
+    );
+    expect(row?.last_error).toMatch(/Worker не настроен/i);
+    expect(row?.provider).toBe("cloudflare");
+    await app.close();
+  });
+
+  it("worker mock 200 writes colo and last_checked_at", async () => {
+    const mock = await startWorkerMock((req) => {
+      const auth = String(req.headers.authorization ?? "");
+      if (auth !== "Bearer secret") {
+        return { status: 401, json: { ok: false, error: "unauthorized" } };
+      }
+      return {
+        status: 200,
+        json: { ok: true, latencyMs: 42, error: null, colo: "AMS" },
+      };
+    });
+    const app = await buildApp({
+      config: { ...loadConfig(), staticDir: null },
+      memory: true,
+    });
+    const headers = await authHeaders(app);
+    const { service, binding } = await seedBinding(app.db, {
+      provider: "cloudflare",
+      ip: "203.0.113.10",
+    });
+    await healthCheckService.runAllChecks(app.db, {
+      thresholds,
+      probeGapMs: 0,
+      worker: { url: mock.url, token: "secret" },
+    });
+    const row = repos.getIpHealthStatusRow(
+      app.db,
+      "binding",
+      binding.id,
+      "203.0.113.10",
+    );
+    expect(row?.status).toBe("up");
+    expect(row?.colo).toBe("AMS");
+    expect(row?.last_checked_at).toBeTruthy();
+    expect(row?.provider).toBe("cloudflare");
+
+    const res = await app.inject({
+      method: "GET",
+      url: `/api/v1/services/${service.id}`,
+      headers,
+    });
+    expect(res.statusCode).toBe(200);
+    const body = res.json() as {
+      ip_health: Array<{
+        ip: string;
+        colo: string | null;
+        last_checked_at: string | null;
+        provider: string;
+      }>;
+    };
+    const ipRow = body.ip_health.find((item) => item.ip === "203.0.113.10");
+    expect(ipRow?.colo).toBe("AMS");
+    expect(ipRow?.last_checked_at).toBeTruthy();
+    expect(ipRow?.provider).toBe("cloudflare");
+
+    const logRes = await app.inject({
+      method: "GET",
+      url: `/api/v1/services/${service.id}/health-log`,
+      headers,
+    });
+    expect(logRes.statusCode).toBe(200);
+    const logBody = logRes.json() as { items: Array<{ colo: string | null }> };
+    expect(logBody.items[0]?.colo).toBe("AMS");
+
+    await new Promise<void>((resolve) => mock.server.close(() => resolve()));
+    await app.close();
+  });
+
+  it("worker timeout is recorded, not local probe", async () => {
+    const mock = await startWorkerMock(() => "hang");
+    const app = await buildApp({
+      config: { ...loadConfig(), staticDir: null },
+      memory: true,
+    });
+    const { binding } = await seedBinding(app.db, {
+      provider: "cloudflare",
+      ip: "203.0.113.20",
+    });
+    await healthCheckService.runAllChecks(app.db, {
+      thresholds,
+      probeGapMs: 0,
+      worker: { url: mock.url, token: "secret" },
+    });
+    const row = repos.getIpHealthStatusRow(
+      app.db,
+      "binding",
+      binding.id,
+      "203.0.113.20",
+    );
+    expect(row?.last_error).toMatch(/timeout|Worker/i);
+    expect(row?.provider).toBe("cloudflare");
+    await new Promise<void>((resolve) => mock.server.close(() => resolve()));
+    await app.close();
+  }, 15_000);
+});

apps/api/test/service-groups-health.test.ts

@@ -99,7 +99,15 @@ describe("service groups health enrichment", () => {
     expect(groupView!.services[0]?.health_status).toBe("degraded");
     expect(groupView!.services[0]?.health_latency_ms).toBe(120);
     expect(groupView!.services[0]?.ip_health).toEqual([
-      { ip: "1.2.3.4", status: "degraded", latency_ms: 120 },
+      {
+        ip: "1.2.3.4",
+        status: "degraded",
+        latency_ms: 120,
+        last_checked_at: expect.any(String),
+        last_error: null,
+        provider: "local",
+        colo: null,
+      },
     ]);
     // group worst = degraded (from service) over up (group scope)
     expect(groupView!.health_status).toBe("degraded");

apps/api/test/settings-health.test.ts

@@ -133,4 +133,33 @@ describe("settings health engine", () => {
     expect(res.statusCode).toBe(400);
     await app.close();
   });
+
+  it("PATCH worker URL; token is not returned in GET", async () => {
+    const app = await buildApp({
+      config: { ...loadConfig(), staticDir: null },
+      memory: true,
+    });
+    const headers = await authHeaders(app);
+    const res = await app.inject({
+      method: "PATCH",
+      url: "/api/v1/settings",
+      headers,
+      payload: {
+        healthWorkerUrl: "https://cfdm-health-probe.example.workers.dev",
+        healthWorkerToken: "super-secret",
+      },
+    });
+    expect(res.statusCode).toBe(200);
+    const body = res.json() as {
+      healthWorkerUrl: string;
+      healthWorkerTokenSet: boolean;
+      healthWorkerToken?: string;
+    };
+    expect(body.healthWorkerUrl).toBe(
+      "https://cfdm-health-probe.example.workers.dev",
+    );
+    expect(body.healthWorkerTokenSet).toBe(true);
+    expect(body.healthWorkerToken).toBeUndefined();
+    await app.close();
+  });
 });