feat(health-check): introduce health probe gap configuration and enhance health check logic

Added `healthProbeGapMs` configuration to control the minimum pause between probes to different physical targets. Updated health check service to utilize this configuration, ensuring efficient probing without overwhelming the targets. Enhanced the `runAllChecks` function to group probes by physical IP and implement the new gap logic. Updated related tests to validate the new functionality.

apps/api/src/app.ts

@@ -131,6 +131,7 @@ export async function buildApp(opts: BuildAppOptions = {}) {
         };
         const n = await healthCheckService.runAllChecks(app.db, {
           thresholds,
+          probeGapMs: config.healthProbeGapMs,
           onStatusChange: async (target, prev, next) => {
             try {
               const label =

apps/api/src/config.ts

@@ -14,6 +14,8 @@ export interface AppConfig {
   healthDegradedFailures: number;
   healthDownFailures: number;
   healthLatencyWarnMs: number;
+  /** Min pause between probes to different physical targets (same IP is probed once). */
+  healthProbeGapMs: number;
   logLevel: string;
   /** Portal SSO — when true, require portal JWT with apps includes cfdm */
   authRequired: boolean;
@@ -46,12 +48,14 @@ export function loadConfig(): AppConfig {
       ? resolve(process.env.STATIC_DIR)
       : null,
     certCheckCron: process.env.CERT_CHECK_CRON ?? "0 0 */6 * * *",
-    healthCheckCron: process.env.HEALTH_CHECK_CRON ?? "*/30 * * * * *",
+    // Default: every 2 minutes (was every 30s — hammered origins / anti-bot).
+    healthCheckCron: process.env.HEALTH_CHECK_CRON ?? "0 */2 * * * *",
     healthDegradedFailures:
       Number(process.env.HEALTH_DEGRADED_FAILURES ?? "1") || 1,
     healthDownFailures: Number(process.env.HEALTH_DOWN_FAILURES ?? "2") || 2,
     healthLatencyWarnMs:
       Number(process.env.HEALTH_LATENCY_WARN_MS ?? "1000") || 1000,
+    healthProbeGapMs: Number(process.env.HEALTH_PROBE_GAP_MS ?? "2000") || 2000,
     logLevel: process.env.LOG_LEVEL ?? "info",
     authRequired: boolEnv(process.env.AUTH_REQUIRED, false),
     authIssuer:

apps/api/src/routes/health-check.ts

@@ -23,6 +23,7 @@ export async function healthCheckRoutes(app: FastifyInstance) {
     };
     const checked = await healthCheckService.runAllChecks(request.server.db, {
       thresholds,
+      probeGapMs: config.healthProbeGapMs,
       onStatusChange: async (target, prev, next) => {
         try {
           const label =

apps/api/src/services/health-check-service.ts

@@ -1,6 +1,6 @@
 import { connect, isIP } from "node:net";
 import { resolve4, resolve6 } from "node:dns/promises";
-import { Agent, fetch as undiciFetch, interceptors } from "undici";
+import { Agent, buildConnector, fetch as undiciFetch } from "undici";
 import type { Db } from "@cfdm/db";
 import { repos } from "@cfdm/db";
 import type { HealthCheckTarget, IpHealthState } from "@cfdm/shared";
@@ -25,7 +25,7 @@ export function hostForUrl(ipOrHost: string): string {
 
 /**
  * Build http(s) URL authority for the probe.
- * Prefer FQDN in the URL (correct Host/SNI); IP is pinned via DNS interceptor.
+ * Prefer FQDN in the URL (correct Host/SNI); TCP dial goes to configured IP via custom connector.
  */
 export function buildHttpProbeUrl(
   urlHost: string,
@@ -79,6 +79,36 @@ function tcpProbe(
   });
 }
 
+/** Dial `connectAddr` for TCP/TLS while URL Host/SNI stay on the FQDN. */
+function createIpPinnedAgent(
+  connectAddr: string,
+  sniHost: string,
+  useTls: boolean,
+  timeoutMs: number,
+): Agent {
+  const connector = buildConnector({
+    rejectUnauthorized: false,
+    timeout: timeoutMs,
+  });
+  return new Agent({
+    connect(opts, callback) {
+      connector(
+        {
+          ...opts,
+          // Force socket to configured IP (or CNAME target), not public DNS of FQDN.
+          hostname: connectAddr,
+          host: connectAddr,
+          servername:
+            useTls && isIP(sniHost) === 0
+              ? sniHost
+              : (opts.servername as string | undefined),
+        },
+        callback,
+      );
+    },
+  });
+}
+
 /**
  * HTTP(S) probe: URL/Host/SNI use hostname (vhost), TCP connects to configured IP when numeric.
  * Avoids re-resolving FQDN via public DNS (which skewed group vs binding latency for the same IP).
@@ -95,37 +125,16 @@ async function httpProbe(
   const useTls = port === 443;
   const connectAddr = String(ip || "").trim();
   const headerHost = (target.hostname || "").trim() || connectAddr;
-  const urlHost = headerHost;
-  const url = buildHttpProbeUrl(urlHost, port, pathWithSlash, useTls);
+  const url = buildHttpProbeUrl(headerHost, port, pathWithSlash, useTls);
 
   const family = isIP(connectAddr);
   const pinToIp = family === 4 || family === 6;
 
-  let dispatcher: Agent | undefined;
-  if (pinToIp) {
-    // URL stays on FQDN (Host + SNI), lookup always returns the configured IP.
-    dispatcher = new Agent({
-      connect: {
-        ...(useTls && isIP(headerHost) === 0 ? { servername: headerHost } : {}),
-        rejectUnauthorized: false,
-      },
-    }).compose(
-      interceptors.dns({
-        dualStack: false,
-        affinity: family === 6 ? 6 : 4,
-        lookup: (_origin, _opts, cb) => {
-          cb(null, [{ address: connectAddr, family: family === 6 ? 6 : 4 }]);
-        },
-      }),
-    ) as Agent;
-  } else if (useTls) {
-    dispatcher = new Agent({
-      connect: {
-        ...(isIP(headerHost) === 0 ? { servername: headerHost } : {}),
-        rejectUnauthorized: false,
-      },
-    });
-  }
+  const dispatcher = pinToIp
+    ? createIpPinnedAgent(connectAddr, headerHost, useTls, timeoutMs)
+    : useTls
+      ? createIpPinnedAgent(connectAddr, headerHost, useTls, timeoutMs)
+      : undefined;
 
   try {
     const response = await undiciFetch(url, {
@@ -244,6 +253,8 @@ function deriveState(
 
 export interface RunAllChecksOptions {
   thresholds: HealthCheckThresholds;
+  /** Pause between unique physical probes (default 2000). Same IP is only probed once. */
+  probeGapMs?: number;
   onStatusChange?: (
     target: HealthCheckTarget,
     prevState: IpHealthState | null,
@@ -251,47 +262,97 @@ export interface RunAllChecksOptions {
   ) => void;
 }
 
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * One network hit per key. Group+binding on the same IP share a single TCP/HTTP probe
+ * so anti-bot / rate-limit on the origin is not tripped by back-to-back checks.
+ */
+export function physicalProbeKey(target: HealthCheckTarget): string {
+  const port = target.port ?? (target.type === "http" ? 80 : 80);
+  const ip = String(target.ip || "").trim().toLowerCase();
+  if (target.type === "http") {
+    const path = (target.path?.trim() || "/") || "/";
+    const expected = target.expected_status ?? "";
+    return `http|${ip}|${port}|${path}|${expected}`;
+  }
+  if (target.type === "tcp") return `tcp|${ip}|${port}`;
+  if (target.type === "ping") {
+    return `ping|${String(target.hostname || target.ip || "").trim().toLowerCase()}`;
+  }
+  if (target.type === "dns") {
+    return `dns|${String(target.hostname || target.ip || "").trim().toLowerCase()}`;
+  }
+  return `${target.type}|${ip}|${port}`;
+}
+
 export async function runAllChecks(
   db: Db,
   options: RunAllChecksOptions,
 ): Promise<number> {
   const targets = repos.listHealthCheckTargets(db);
+  const gapMs = Math.max(0, options.probeGapMs ?? 2000);
+
+  const byPhysical = new Map<string, HealthCheckTarget[]>();
   for (const target of targets) {
-    const prev = repos.getIpHealthStatusRow(
-      db,
-      target.scope,
-      target.ref_id,
-      target.ip,
-    );
-    const result = await probeTarget(target);
-    const { state, failures } = deriveState(
-      result.ok,
-      result.latencyMs,
-      prev
-        ? {
-            consecutive_failures: prev.consecutive_failures,
-            status: prev.status,
-          }
-        : null,
-      options.thresholds,
-    );
-    const prevState: IpHealthState | null = prev
-      ? (prev.status as IpHealthState)
-      : null;
-    repos.upsertIpHealthStatus(
-      db,
-      target.scope,
-      target.ref_id,
-      target.ip,
-      state,
-      result.latencyMs,
-      failures,
-      result.error,
-    );
-    if (prevState !== state) {
-      options.onStatusChange?.(target, prevState, state);
+    const key = physicalProbeKey(target);
+    const list = byPhysical.get(key);
+    if (list) list.push(target);
+    else byPhysical.set(key, [target]);
+  }
+
+  let probeIndex = 0;
+  for (const group of byPhysical.values()) {
+    if (probeIndex > 0 && gapMs > 0) {
+      await sleep(gapMs);
+    }
+    probeIndex += 1;
+
+    // Prefer binding hostname for SNI when several scopes share one IP.
+    const representative =
+      group.find((t) => t.scope === "binding") ?? group[0]!;
+    const result = await probeTarget(representative);
+
+    for (const target of group) {
+      const prev = repos.getIpHealthStatusRow(
+        db,
+        target.scope,
+        target.ref_id,
+        target.ip,
+      );
+      const { state, failures } = deriveState(
+        result.ok,
+        result.latencyMs,
+        prev
+          ? {
+              consecutive_failures: prev.consecutive_failures,
+              status: prev.status,
+            }
+          : null,
+        options.thresholds,
+      );
+      const prevState: IpHealthState | null = prev
+        ? (prev.status as IpHealthState)
+        : null;
+      repos.upsertIpHealthStatus(
+        db,
+        target.scope,
+        target.ref_id,
+        target.ip,
+        state,
+        result.latencyMs,
+        failures,
+        result.error,
+      );
+      if (prevState !== state) {
+        options.onStatusChange?.(target, prevState, state);
+      }
     }
   }
+  // Orphan rows (old IPs / hostname keys) still feed MAX latency on group badge.
+  repos.pruneStaleIpHealthStatus(db, targets);
   return targets.length;
 }