import { describe, expect, it, beforeAll, afterAll } from "vitest"; import { createServer, type Server } from "node:net"; import { createServer as createHttpServer, type Server as HttpServer } from "node:http"; import * as healthCheckService from "../src/services/health-check-service.js"; import type { HealthCheckTarget } from "@cfdm/shared"; function startTcpServer(): Promise<{ server: Server; port: number }> { return new Promise((resolve) => { const server = createServer(); server.listen(0, "127.0.0.1", () => { const address = server.address(); const port = typeof address === "object" && address ? address.port : 0; resolve({ server, port }); }); }); } describe("health-check URL helpers", () => { it("buildHttpProbeUrl uses FQDN in URL (IP pinned via connector)", () => { expect( healthCheckService.buildHttpProbeUrl("gt.rkns.top", 443, "/", true), ).toBe("https://gt.rkns.top/"); expect( healthCheckService.buildHttpProbeUrl("gt.rkns.top", 8080, "/health", false), ).toBe("http://gt.rkns.top:8080/health"); expect( healthCheckService.buildHttpProbeUrl("2001:db8::1", 443, "/", true), ).toBe("https://[2001:db8::1]/"); }); }); describe("health-check probeTarget", () => { let server: Server; let port: number; beforeAll(async () => { const started = await startTcpServer(); server = started.server; port = started.port; }); afterAll(async () => { await new Promise((resolve) => server.close(() => resolve())); }); it("tcp probe succeeds for open port", async () => { const target: HealthCheckTarget = { scope: "binding", ref_id: 1, ip: "127.0.0.1", hostname: "test.local", type: "tcp", port, path: null, expected_status: null, timeout_ms: 1000, verify_tls: false, provider: "local", }; const result = await healthCheckService.probeTarget(target); expect(result.ok).toBe(true); expect(result.error).toBeNull(); expect(result.latencyMs).toBeGreaterThanOrEqual(0); }); it("tcp probe fails for closed port", async () => { const target: HealthCheckTarget = { scope: "binding", ref_id: 1, ip: "127.0.0.1", hostname: "test.local", type: "tcp", port: 1, path: null, expected_status: null, timeout_ms: 500, verify_tls: false, provider: "local", }; const result = await healthCheckService.probeTarget(target); expect(result.ok).toBe(false); expect(result.error).not.toBeNull(); }); it("http probe hits IP with Host=hostname (same IP, different FQDN)", async () => { let seenHost: string | undefined; const httpServer: HttpServer = createHttpServer((req, res) => { seenHost = req.headers.host; res.writeHead(200); res.end("ok"); }); const httpPort = await new Promise((resolve) => { httpServer.listen(0, "127.0.0.1", () => { const address = httpServer.address(); resolve(typeof address === "object" && address ? address.port : 0); }); }); try { const groupTarget: HealthCheckTarget = { scope: "group", ref_id: 1, ip: "127.0.0.1", hostname: "gt.rkns.top", type: "http", port: httpPort, path: "/", expected_status: 200, timeout_ms: 1000, verify_tls: false, provider: "local", }; const bindingTarget: HealthCheckTarget = { ...groupTarget, scope: "binding", hostname: "rutg.rkns.top", }; const groupResult = await healthCheckService.probeTarget(groupTarget); expect(groupResult.ok).toBe(true); expect(seenHost?.startsWith("gt.rkns.top")).toBe(true); const bindingResult = await healthCheckService.probeTarget(bindingTarget); expect(bindingResult.ok).toBe(true); expect(seenHost?.startsWith("rutg.rkns.top")).toBe(true); // Same loopback IP → latencies in the same ballpark (not ~1s DNS skew) expect(Math.abs(groupResult.latencyMs - bindingResult.latencyMs)).toBeLessThan(200); } finally { await new Promise((resolve) => httpServer.close(() => resolve())); } }); it("physicalProbeKey collapses group+binding on same IP for tcp", async () => { const { physicalProbeKey } = await import("../src/services/health-check-service.js"); const group: HealthCheckTarget = { scope: "group", ref_id: 1, ip: "93.115.203.183", hostname: "gt.rkns.top", type: "tcp", port: 443, path: null, expected_status: null, timeout_ms: 3000, verify_tls: false, provider: "local", }; const binding: HealthCheckTarget = { ...group, scope: "binding", ref_id: 2, hostname: "rutg.rkns.top", }; expect(physicalProbeKey(group)).toBe(physicalProbeKey(binding)); }); }); describe("health-check state derivation via runAllChecks", () => { it("marks ip down after threshold failures and up after recovery", async () => { const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db"); const { db, sqlite } = createMemoryDb(); runMigrations(sqlite); const domain = repos.createDomain(db, null, "example.com", "zone-id"); const service = repos.createService(db, "Svc", "svc"); const binding = repos.insertBinding( db, domain.id, service.id, "@", null, ); repos.updateBindingLbConfig(db, binding.id, { health_check_enabled: true, health_check_type: "tcp", health_check_port: 1, health_check_timeout_ms: 200, }); repos.replaceBindingIpsWithMeta(db, binding.id, [ { ip: "127.0.0.1", weight: 1, priority: 1 }, ]); await healthCheckService.runAllChecks(db, { thresholds: { degradedFailures: 1, downFailures: 2, latencyWarnMs: 1000, }, }); let status = repos.getIpHealthStatusRow( db, "binding", binding.id, "127.0.0.1", ); expect(status?.status).toBe("degraded"); await healthCheckService.runAllChecks(db, { thresholds: { degradedFailures: 1, downFailures: 2, latencyWarnMs: 1000, }, }); status = repos.getIpHealthStatusRow( db, "binding", binding.id, "127.0.0.1", ); expect(status?.status).toBe("down"); }); it("listHealthCheckTargets includes verify_tls from binding config", async () => { const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db"); const { db, sqlite } = createMemoryDb(); runMigrations(sqlite); const domain = repos.createDomain(db, null, "example.com", "zone-id"); const service = repos.createService(db, "Svc", "svc"); const binding = repos.insertBinding(db, domain.id, service.id, "@", null); repos.updateBindingLbConfig(db, binding.id, { health_check_enabled: true, health_check_type: "http", health_check_port: 443, health_check_verify_tls: true, }); repos.replaceBindingIpsWithMeta(db, binding.id, [ { ip: "10.0.0.1", weight: 1, priority: 1 }, ]); const targets = repos.listHealthCheckTargets(db); expect(targets).toHaveLength(1); expect(targets[0]?.verify_tls).toBe(true); }); it("unwraps CNAME target to origin A record IPs", async () => { const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db"); const { db, sqlite } = createMemoryDb(); runMigrations(sqlite); const domain = repos.createDomain(db, null, "rkns.top", "zone-id"); repos.insertDnsRecord( db, domain.id, "A", "ihome", "2.59.161.102", 1, false, null, "synced", "cf", null, ); const service = repos.createService(db, "RW Sub", "rw-sub"); const binding = repos.insertBinding(db, domain.id, service.id, "s", null); repos.setBindingCnameTarget(db, binding.id, "ihome.rkns.top"); repos.updateBindingLbConfig(db, binding.id, { health_check_enabled: true, health_check_type: "tcp", health_check_port: 443, }); const targets = repos.listHealthCheckTargets(db); expect(targets).toHaveLength(1); expect(targets[0]?.ip).toBe("2.59.161.102"); expect(targets[0]?.hostname).toBe("s.rkns.top"); }); it("unwraps CNAME target to service IP pool when origin DNS is empty", async () => { const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db"); const { db, sqlite } = createMemoryDb(); runMigrations(sqlite); const domain = repos.createDomain(db, null, "rkns.top", "zone-id"); const service = repos.createService(db, "RW Sub", "rw-sub"); repos.replaceServiceIps(db, service.id, ["2.59.161.102"]); const binding = repos.insertBinding(db, domain.id, service.id, "s", null); repos.setBindingCnameTarget(db, binding.id, "ihome.rkns.top"); repos.updateBindingLbConfig(db, binding.id, { health_check_enabled: true, health_check_type: "tcp", health_check_port: 443, }); const targets = repos.listHealthCheckTargets(db); expect(targets).toHaveLength(1); expect(targets[0]?.ip).toBe("2.59.161.102"); expect(targets[0]?.hostname).toBe("s.rkns.top"); }); it("does not mark node unhealthy when binding majority is OK and group local fails", async () => { const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db"); const { db, sqlite } = createMemoryDb(); runMigrations(sqlite); const tcp = await startTcpServer(); try { const domain = repos.createDomain(db, null, "example.com", "zone-id"); const group = repos.createServiceGroup( db, "VPN", "vpn", null, "vpn.example.com", { health_check_enabled: true, health_check_type: "http", health_check_port: 1, health_check_timeout_ms: 200, health_check_path: "/", }, ); const service = repos.createService(db, "Svc", "svc"); repos.setServiceGroup(db, service.id, group.id); repos.setServiceEnabled(db, service.id, true); const binding = repos.insertBinding(db, domain.id, service.id, "@", null); repos.updateBindingLbConfig(db, binding.id, { health_check_enabled: true, health_check_type: "tcp", health_check_port: tcp.port, health_check_timeout_ms: 500, }); repos.replaceBindingIpsWithMeta(db, binding.id, [ { ip: "127.0.0.1", weight: 1, priority: 1 }, ]); const node = repos.findNodeByIp(db, "127.0.0.1"); expect(node).not.toBeNull(); await healthCheckService.runAllChecks(db, { probeGapMs: 0, thresholds: { degradedFailures: 1, downFailures: 1, latencyWarnMs: 1000, }, }); const bindingHealth = repos.getIpHealthStatusRow( db, "binding", binding.id, "127.0.0.1", ); const groupHealth = repos.getIpHealthStatusRow( db, "group", group.id, "127.0.0.1", ); const after = repos.getNode(db, node!.id); expect(bindingHealth?.status).toBe("up"); expect(groupHealth?.status).toBe("down"); expect(after.health_status).toBe("healthy"); expect(after.consecutive_failures).toBe(0); expect(after.last_failure_reason).toBeNull(); } finally { await new Promise((resolve) => tcp.server.close(() => resolve())); } }); }); describe("CNAME health mapped onto service IPs", () => { it("getView copies CNAME-keyed health onto the service IP row", async () => { const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db"); const { getView } = await import("../src/services/service-config-service.js"); const { db, sqlite } = createMemoryDb(); runMigrations(sqlite); const domain = repos.createDomain(db, null, "rkns.top", "zone-id"); const service = repos.createService(db, "RW Sub", "rw-sub"); repos.replaceServiceIps(db, service.id, ["2.59.161.102"]); const binding = repos.insertBinding(db, domain.id, service.id, "s", null); repos.setBindingCnameTarget(db, binding.id, "ihome.rkns.top"); repos.updateBindingLbConfig(db, binding.id, { health_check_enabled: true }); repos.upsertIpHealthStatus( db, "binding", binding.id, "ihome.rkns.top", "up", 12, 0, null, ); const view = await getView(db, service.id); expect(view.health_status).toBe("up"); expect(view.ip_health).toEqual([ expect.objectContaining({ ip: "2.59.161.102", status: "up", latency_ms: 12, }), ]); }); it("getView is up when any binding IP is up", async () => { const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db"); const { getView } = await import("../src/services/service-config-service.js"); const { db, sqlite } = createMemoryDb(); runMigrations(sqlite); const domain = repos.createDomain(db, null, "rkns.top", "zone-id"); const service = repos.createService(db, "MSK Hip", "msk-hip"); repos.replaceServiceIps(db, service.id, ["10.0.0.1", "10.0.0.2"]); const binding = repos.insertBinding(db, domain.id, service.id, "gt", null); repos.replaceBindingIpsWithMeta(db, binding.id, [ { ip: "10.0.0.1", weight: 1, priority: 1 }, { ip: "10.0.0.2", weight: 1, priority: 1 }, ]); repos.updateBindingLbConfig(db, binding.id, { health_check_enabled: true }); repos.upsertIpHealthStatus( db, "binding", binding.id, "10.0.0.1", "up", 12, 0, null, ); repos.upsertIpHealthStatus( db, "binding", binding.id, "10.0.0.2", "down", null, 5, "timeout", ); const view = await getView(db, service.id); expect(view.health_status).toBe("up"); }); it("getView shows live OK over hysteresis unknown", async () => { const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db"); const { getView } = await import("../src/services/service-config-service.js"); const { db, sqlite } = createMemoryDb(); runMigrations(sqlite); const domain = repos.createDomain(db, null, "rkns.top", "zone-id"); const service = repos.createService(db, "RW Panel", "rw-panel"); repos.replaceServiceIps(db, service.id, ["2.59.161.102"]); const binding = repos.insertBinding(db, domain.id, service.id, "c", null); repos.replaceBindingIpsWithMeta(db, binding.id, [ { ip: "2.59.161.102", weight: 1, priority: 1 }, ]); repos.updateBindingLbConfig(db, binding.id, { health_check_enabled: true }); repos.upsertIpHealthStatus( db, "binding", binding.id, "2.59.161.102", "unknown", 63, 0, null, 1, ); repos.insertHealthProbeLog(db, { scope: "binding", refId: binding.id, ip: "2.59.161.102", provider: "local", status: "up", ok: true, latencyMs: 63, colo: null, error: null, }); const view = await getView(db, service.id); expect(view.health_status).toBe("up"); expect(view.ip_health[0]?.status).toBe("up"); expect(view.ip_health[0]?.latency_ms).toBe(63); }); it("getView masks stale down when health-check is disabled", async () => { const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db"); const { getView } = await import("../src/services/service-config-service.js"); const { db, sqlite } = createMemoryDb(); runMigrations(sqlite); const domain = repos.createDomain(db, null, "rkns.top", "zone-id"); const service = repos.createService(db, "Main TG", "main-tg"); repos.setServiceEnabled(db, service.id, true); repos.replaceServiceIps(db, service.id, ["130.49.213.176"]); const binding = repos.insertBinding(db, domain.id, service.id, "gt", null); repos.replaceBindingIpsWithMeta(db, binding.id, [ { ip: "130.49.213.176", weight: 1, priority: 1 }, ]); repos.updateBindingLbConfig(db, binding.id, { health_check_enabled: false }); repos.upsertIpHealthStatus( db, "binding", binding.id, "130.49.213.176", "down", null, 5, "timeout", ); const view = await getView(db, service.id); expect(view.health_status).toBe("unknown"); expect(view.ip_health).toEqual([ expect.objectContaining({ ip: "130.49.213.176", status: "unknown", latency_ms: null, last_error: null, }), ]); }); it("getView masks stale down when IP is disabled in pool", async () => { const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db"); const { getView } = await import("../src/services/service-config-service.js"); const { db, sqlite } = createMemoryDb(); runMigrations(sqlite); const domain = repos.createDomain(db, null, "rkns.top", "zone-id"); const service = repos.createService(db, "Main TG", "main-tg"); repos.setServiceEnabled(db, service.id, true); repos.replaceServiceIps(db, service.id, ["130.49.213.176"]); repos.setServiceIpEnabled(db, service.id, "130.49.213.176", false); const binding = repos.insertBinding(db, domain.id, service.id, "gt", null); repos.replaceBindingIpsWithMeta(db, binding.id, [ { ip: "130.49.213.176", weight: 1, priority: 1 }, ]); repos.updateBindingLbConfig(db, binding.id, { health_check_enabled: true }); repos.upsertIpHealthStatus( db, "binding", binding.id, "130.49.213.176", "down", null, 5, "timeout", ); const view = await getView(db, service.id); expect(view.health_status).toBe("unknown"); expect(view.ip_health[0]?.status).toBe("unknown"); }); });