feat(health-checks): enhance health check functionality and add new routes

- Introduced origin health check routes and integrated them into the application.
- Updated health check configuration to include success recovery thresholds.
- Expanded error handling with new error codes for health check failures.
- Added new service routes for managing health checks, including creation and listing.
- Improved health check service logic to track consecutive successes and failures.

This commit enhances the health check capabilities, providing better monitoring and management of service health.

apps/api/test/cf-healthcheck.test.ts

@@ -0,0 +1,38 @@
+import { describe, expect, it } from "vitest";
+import { repos } from "@cfdm/db";
+import { buildApp } from "../src/app.js";
+import { loadConfig } from "../src/config.js";
+
+async function authHeaders(app: Awaited<ReturnType<typeof buildApp>>) {
+  const config = loadConfig();
+  const res = await app.inject({
+    method: "POST",
+    url: "/api/v1/auth/login",
+    payload: { username: config.adminUsername, password: "admin" },
+  });
+  expect(res.statusCode).toBe(200);
+  const { token } = res.json() as { token: string };
+  return { authorization: `Bearer ${token}` };
+}
+
+describe("origin health checks", () => {
+  it("creates a local health check", async () => {
+    const app = await buildApp({
+      config: { ...loadConfig(), staticDir: null },
+      memory: true,
+    });
+    const headers = await authHeaders(app);
+    const res = await app.inject({
+      method: "POST",
+      url: "/api/v1/health-checks",
+      headers,
+      payload: { provider: "local", name: "origin-1", protocol: "tcp" },
+    });
+    expect(res.statusCode).toBe(200);
+    const body = res.json() as { provider: string; name: string };
+    expect(body.provider).toBe("local");
+    expect(body.name).toBe("origin-1");
+    expect(repos.listHealthChecks(app.db)).toHaveLength(1);
+    await app.close();
+  });
+});

apps/api/test/change-domain.test.ts

@@ -0,0 +1,76 @@
+import { describe, expect, it } from "vitest";
+import { repos } from "@cfdm/db";
+import type { CloudflareClient } from "../src/lib/cf-client.js";
+import { buildApp } from "../src/app.js";
+import { loadConfig } from "../src/config.js";
+import { changeServiceDomain } from "../src/services/change-domain-service.js";
+import { updateConfig } from "../src/services/service-config-service.js";
+
+function mockCf(): CloudflareClient {
+  return {
+    listDnsRecords: async () => [],
+    createDnsRecord: async (_zoneId: string, payload: { type: string; name: string; content: string }) => ({
+      id: `cf-${payload.name}-${payload.content}`,
+      type: payload.type,
+      name: payload.name,
+      content: payload.content,
+      ttl: 1,
+      proxied: false,
+    }),
+    updateDnsRecord: async (
+      _zoneId: string,
+      id: string,
+      payload: { type: string; name: string; content: string },
+    ) => ({
+      id,
+      type: payload.type,
+      name: payload.name,
+      content: payload.content,
+      ttl: 1,
+      proxied: false,
+    }),
+    patchDnsRecord: async (
+      _zoneId: string,
+      id: string,
+      payload: { content?: string },
+    ) => ({
+      id,
+      type: "A",
+      name: "app.example.com",
+      content: payload.content ?? "1.1.1.1",
+      ttl: 1,
+      proxied: false,
+    }),
+    deleteDnsRecord: async () => undefined,
+    listZones: async () => [
+      { id: "zone-1", name: "example.com", status: "active" },
+      { id: "zone-2", name: "other.com", status: "active" },
+    ],
+  } as unknown as CloudflareClient;
+}
+
+describe("change-domain", () => {
+  it("dry-run lists FQDN from → to", async () => {
+    const app = await buildApp({
+      config: { ...loadConfig(), staticDir: null },
+      memory: true,
+    });
+    const cf = mockCf();
+    const from = repos.createDomain(app.db, null, "example.com", "zone-1");
+    const to = repos.createDomain(app.db, null, "other.com", "zone-2");
+    const service = repos.createService(app.db, "App", "app");
+    await updateConfig(app.db, cf, service.id, {
+      ips: ["1.1.1.1"],
+      domains: [{ fqdn: "app.example.com", target_ips: ["1.1.1.1"] }],
+    });
+    const preview = await changeServiceDomain(app.db, cf, service.id, {
+      from_domain_id: from.id,
+      to_domain_id: to.id,
+      dry_run: true,
+    });
+    expect(preview.applied).toBe(false);
+    expect(preview.items[0]?.from_fqdn).toBe("app.example.com");
+    expect(preview.items[0]?.to_fqdn).toBe("app.other.com");
+    await app.close();
+  });
+});

apps/api/test/change-ip.test.ts

@@ -0,0 +1,110 @@
+import { describe, expect, it } from "vitest";
+import { repos } from "@cfdm/db";
+import type { CloudflareClient } from "../src/lib/cf-client.js";
+import { buildApp } from "../src/app.js";
+import { loadConfig } from "../src/config.js";
+import { changeBindingIp } from "../src/services/change-ip-service.js";
+import { withBindingLock } from "../src/services/routing/index.js";
+import { updateConfig } from "../src/services/service-config-service.js";
+
+function mockCf(): CloudflareClient {
+  return {
+    listDnsRecords: async () => [],
+    createDnsRecord: async (_zoneId: string, payload: { type: string; name: string; content: string }) => ({
+      id: `cf-${payload.name}-${payload.content}`,
+      type: payload.type,
+      name: payload.name,
+      content: payload.content,
+      ttl: 1,
+      proxied: false,
+    }),
+    updateDnsRecord: async (
+      _zoneId: string,
+      id: string,
+      payload: { type: string; name: string; content: string },
+    ) => ({
+      id,
+      type: payload.type,
+      name: payload.name,
+      content: payload.content,
+      ttl: 1,
+      proxied: false,
+    }),
+    patchDnsRecord: async (
+      _zoneId: string,
+      id: string,
+      payload: { content?: string },
+    ) => ({
+      id,
+      type: "A",
+      name: "panel.example.com",
+      content: payload.content ?? "0.0.0.0",
+      ttl: 1,
+      proxied: false,
+    }),
+    deleteDnsRecord: async () => undefined,
+    listZones: async () => [{ id: "zone-1", name: "example.com", status: "active" }],
+  } as unknown as CloudflareClient;
+}
+
+describe("change-ip", () => {
+  it("dry-run previews from → to without writing", async () => {
+    const app = await buildApp({
+      config: { ...loadConfig(), staticDir: null },
+      memory: true,
+    });
+    const cf = mockCf();
+    repos.createDomain(app.db, null, "example.com", "zone-1");
+    const service = repos.createService(app.db, "Panel", "panel");
+    await updateConfig(app.db, cf, service.id, {
+      ips: ["10.0.0.10"],
+      domains: [{ fqdn: "panel.example.com", target_ips: ["10.0.0.10"] }],
+    });
+    const bindings = repos.listBindingsByService(app.db, service.id);
+    const preview = await changeBindingIp(app.db, cf, bindings[0]!.id, {
+      from_ip: "10.0.0.10",
+      to_ip: "10.0.0.20",
+      dry_run: true,
+    });
+    expect(preview.applied).toBe(false);
+    expect(preview.message).toBe("10.0.0.10 → 10.0.0.20");
+    expect(repos.listBindingIps(app.db, bindings[0]!.id)).toEqual(["10.0.0.10"]);
+    await app.close();
+  });
+
+  it("apply patches binding IP", async () => {
+    const app = await buildApp({
+      config: { ...loadConfig(), staticDir: null },
+      memory: true,
+    });
+    const cf = mockCf();
+    repos.createDomain(app.db, null, "example.com", "zone-1");
+    const service = repos.createService(app.db, "Panel", "panel");
+    await updateConfig(app.db, cf, service.id, {
+      ips: ["10.0.0.10"],
+      domains: [{ fqdn: "panel.example.com", target_ips: ["10.0.0.10"] }],
+    });
+    const binding = repos.listBindingsByService(app.db, service.id)[0]!;
+    const result = await changeBindingIp(app.db, cf, binding.id, {
+      from_ip: "10.0.0.10",
+      to_ip: "10.0.0.20",
+    });
+    expect(result.applied).toBe(true);
+    expect(repos.listBindingIps(app.db, binding.id)).toEqual(["10.0.0.20"]);
+    await app.close();
+  });
+
+  it("serializes concurrent binding locks", async () => {
+    const order: number[] = [];
+    await Promise.all([
+      withBindingLock(1, async () => {
+        await new Promise((r) => setTimeout(r, 20));
+        order.push(1);
+      }),
+      withBindingLock(1, async () => {
+        order.push(2);
+      }),
+    ]);
+    expect(order).toEqual([1, 2]);
+  });
+});

apps/api/test/health-state-machine.test.ts

@@ -0,0 +1,62 @@
+import { describe, expect, it } from "vitest";
+import { nextHealthState } from "../src/services/health/state-machine.js";
+
+const thresholds = {
+  degradedFailures: 1,
+  downFailures: 2,
+  successRecoveries: 2,
+  latencyWarnMs: 1000,
+};
+
+describe("health state machine", () => {
+  it("first success from unknown is healthy immediately", () => {
+    const next = nextHealthState(true, 20, null, thresholds);
+    expect(next.legacy).toBe("up");
+    expect(next.node).toBe("healthy");
+  });
+
+  it("recovery from down goes checking until consecutive successes", () => {
+    const first = nextHealthState(
+      true,
+      10,
+      { status: "down", consecutive_failures: 3, consecutive_successes: 0 },
+      thresholds,
+    );
+    expect(first.node).toBe("checking");
+    expect(first.legacy).toBe("unknown");
+    const second = nextHealthState(
+      true,
+      10,
+      {
+        status: "checking",
+        consecutive_failures: 0,
+        consecutive_successes: first.successes,
+      },
+      thresholds,
+    );
+    expect(second.node).toBe("healthy");
+    expect(second.legacy).toBe("up");
+  });
+
+  it("two failures mark unhealthy", () => {
+    const first = nextHealthState(
+      false,
+      5,
+      { status: "up", consecutive_failures: 0, consecutive_successes: 1 },
+      thresholds,
+    );
+    expect(first.node).toBe("degraded");
+    const second = nextHealthState(
+      false,
+      5,
+      {
+        status: "degraded",
+        consecutive_failures: first.failures,
+        consecutive_successes: 0,
+      },
+      thresholds,
+    );
+    expect(second.node).toBe("unhealthy");
+    expect(second.legacy).toBe("down");
+  });
+});

apps/api/test/lb-reconcile.test.ts

@@ -39,7 +39,10 @@ describe("selectActiveIpsByMode", () => {
       lb_mode: "round_robin",
       health_check_enabled: true,
     };
-    const rows = [row("1.1.1.1", { health: "unknown" }), row("2.2.2.2")];
+    const rows = [
+      row("1.1.1.1", { health: "unknown" }),
+      row("2.2.2.2", { health: "unknown" }),
+    ];
     expect(selectActiveIpsByMode(config, rows).sort()).toEqual([
       "1.1.1.1",
       "2.2.2.2",
@@ -88,6 +91,18 @@ describe("selectActiveIpsByMode", () => {
     ]);
   });
 
+  it("round_robin excludes unknown when another ip is up", () => {
+    const config: LbTargetConfig = {
+      lb_mode: "round_robin",
+      health_check_enabled: true,
+    };
+    const rows = [
+      row("1.1.1.1", { health: "up" }),
+      row("2.2.2.2", { health: "unknown" }),
+    ];
+    expect(selectActiveIpsByMode(config, rows)).toEqual(["1.1.1.1"]);
+  });
+
   it("returns empty array for no rows", () => {
     const config: LbTargetConfig = {
       lb_mode: "round_robin",

apps/api/test/nodes-api.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it } from "vitest";
+import { repos } from "@cfdm/db";
+import { buildApp } from "../src/app.js";
+import { loadConfig } from "../src/config.js";
+
+async function authHeaders(app: Awaited<ReturnType<typeof buildApp>>) {
+  const config = loadConfig();
+  const res = await app.inject({
+    method: "POST",
+    url: "/api/v1/auth/login",
+    payload: { username: config.adminUsername, password: "admin" },
+  });
+  expect(res.statusCode).toBe(200);
+  const { token } = res.json() as { token: string };
+  return { authorization: `Bearer ${token}` };
+}
+
+describe("service nodes API", () => {
+  it("creates and lists nodes without changing empty DNS pool until bound", async () => {
+    const app = await buildApp({
+      config: { ...loadConfig(), staticDir: null },
+      memory: true,
+    });
+    const headers = await authHeaders(app);
+    const service = repos.createService(app.db, "Panel", "panel");
+
+    const created = await app.inject({
+      method: "POST",
+      url: `/api/v1/services/${service.id}/nodes`,
+      headers,
+      payload: { address: "10.0.0.8", protocol: "tcp" },
+    });
+    expect(created.statusCode).toBe(200);
+    const node = created.json() as { address: string };
+    expect(node.address).toBe("10.0.0.8");
+    expect(repos.listServiceIps(app.db, service.id)).toContain("10.0.0.8");
+
+    const listed = await app.inject({
+      method: "GET",
+      url: `/api/v1/services/${service.id}/nodes`,
+      headers,
+    });
+    expect(listed.statusCode).toBe(200);
+    expect(listed.json()).toHaveLength(1);
+
+    const overview = await app.inject({
+      method: "GET",
+      url: `/api/v1/services/${service.id}/overview`,
+      headers,
+    });
+    expect(overview.statusCode).toBe(200);
+    await app.close();
+  });
+});