feat(health-check): introduce health probe gap configuration and enhance health check logic

Added `healthProbeGapMs` configuration to control the minimum pause between probes to different physical targets. Updated health check service to utilize this configuration, ensuring efficient probing without overwhelming the targets. Enhanced the `runAllChecks` function to group probes by physical IP and implement the new gap logic. Updated related tests to validate the new functionality.
2026-07-20 03:57:37 +07:00
parent a58d91ff5e
commit 9783974949
8 changed files with 270 additions and 66 deletions
@@ -1,6 +1,6 @@
 import { connect, isIP } from "node:net";
 import { resolve4, resolve6 } from "node:dns/promises";
-import { Agent, fetch as undiciFetch, interceptors } from "undici";
+import { Agent, buildConnector, fetch as undiciFetch } from "undici";
 import type { Db } from "@cfdm/db";
 import { repos } from "@cfdm/db";
 import type { HealthCheckTarget, IpHealthState } from "@cfdm/shared";
@@ -25,7 +25,7 @@ export function hostForUrl(ipOrHost: string): string {

 /**
 * Build http(s) URL authority for the probe.
- * Prefer FQDN in the URL (correct Host/SNI); IP is pinned via DNS interceptor.
+ * Prefer FQDN in the URL (correct Host/SNI); TCP dial goes to configured IP via custom connector.
 */
 export function buildHttpProbeUrl(
  urlHost: string,
@@ -79,6 +79,36 @@ function tcpProbe(
  });
 }

+/** Dial `connectAddr` for TCP/TLS while URL Host/SNI stay on the FQDN. */
+function createIpPinnedAgent(
+  connectAddr: string,
+  sniHost: string,
+  useTls: boolean,
+  timeoutMs: number,
+): Agent {
+  const connector = buildConnector({
+    rejectUnauthorized: false,
+    timeout: timeoutMs,
+  });
+  return new Agent({
+    connect(opts, callback) {
+      connector(
+        {
+          ...opts,
+          // Force socket to configured IP (or CNAME target), not public DNS of FQDN.
+          hostname: connectAddr,
+          host: connectAddr,
+          servername:
+            useTls && isIP(sniHost) === 0
+              ? sniHost
+              : (opts.servername as string | undefined),
+        },
+        callback,
+      );
+    },
+  });
+}
+
 /**
 * HTTP(S) probe: URL/Host/SNI use hostname (vhost), TCP connects to configured IP when numeric.
 * Avoids re-resolving FQDN via public DNS (which skewed group vs binding latency for the same IP).
@@ -95,37 +125,16 @@ async function httpProbe(
  const useTls = port === 443;
  const connectAddr = String(ip || "").trim();
  const headerHost = (target.hostname || "").trim() || connectAddr;
-  const urlHost = headerHost;
-  const url = buildHttpProbeUrl(urlHost, port, pathWithSlash, useTls);
+  const url = buildHttpProbeUrl(headerHost, port, pathWithSlash, useTls);

  const family = isIP(connectAddr);
  const pinToIp = family === 4 || family === 6;

-  let dispatcher: Agent | undefined;
-  if (pinToIp) {
-    // URL stays on FQDN (Host + SNI), lookup always returns the configured IP.
-    dispatcher = new Agent({
-      connect: {
-        ...(useTls && isIP(headerHost) === 0 ? { servername: headerHost } : {}),
-        rejectUnauthorized: false,
-      },
-    }).compose(
-      interceptors.dns({
-        dualStack: false,
-        affinity: family === 6 ? 6 : 4,
-        lookup: (_origin, _opts, cb) => {
-          cb(null, [{ address: connectAddr, family: family === 6 ? 6 : 4 }]);
-        },
-      }),
-    ) as Agent;
-  } else if (useTls) {
-    dispatcher = new Agent({
-      connect: {
-        ...(isIP(headerHost) === 0 ? { servername: headerHost } : {}),
-        rejectUnauthorized: false,
-      },
-    });
-  }
+  const dispatcher = pinToIp
+    ? createIpPinnedAgent(connectAddr, headerHost, useTls, timeoutMs)
+    : useTls
+      ? createIpPinnedAgent(connectAddr, headerHost, useTls, timeoutMs)
+      : undefined;

  try {
    const response = await undiciFetch(url, {
@@ -244,6 +253,8 @@ function deriveState(

 export interface RunAllChecksOptions {
  thresholds: HealthCheckThresholds;
+  /** Pause between unique physical probes (default 2000). Same IP is only probed once. */
+  probeGapMs?: number;
  onStatusChange?: (
    target: HealthCheckTarget,
    prevState: IpHealthState | null,
@@ -251,47 +262,97 @@ export interface RunAllChecksOptions {
  ) => void;
 }

+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * One network hit per key. Group+binding on the same IP share a single TCP/HTTP probe
+ * so anti-bot / rate-limit on the origin is not tripped by back-to-back checks.
+ */
+export function physicalProbeKey(target: HealthCheckTarget): string {
+  const port = target.port ?? (target.type === "http" ? 80 : 80);
+  const ip = String(target.ip || "").trim().toLowerCase();
+  if (target.type === "http") {
+    const path = (target.path?.trim() || "/") || "/";
+    const expected = target.expected_status ?? "";
+    return `http|${ip}|${port}|${path}|${expected}`;
+  }
+  if (target.type === "tcp") return `tcp|${ip}|${port}`;
+  if (target.type === "ping") {
+    return `ping|${String(target.hostname || target.ip || "").trim().toLowerCase()}`;
+  }
+  if (target.type === "dns") {
+    return `dns|${String(target.hostname || target.ip || "").trim().toLowerCase()}`;
+  }
+  return `${target.type}|${ip}|${port}`;
+}
+
 export async function runAllChecks(
  db: Db,
  options: RunAllChecksOptions,
 ): Promise<number> {
  const targets = repos.listHealthCheckTargets(db);
+  const gapMs = Math.max(0, options.probeGapMs ?? 2000);
+
+  const byPhysical = new Map<string, HealthCheckTarget[]>();
  for (const target of targets) {
-    const prev = repos.getIpHealthStatusRow(
-      db,
-      target.scope,
-      target.ref_id,
-      target.ip,
-    );
-    const result = await probeTarget(target);
-    const { state, failures } = deriveState(
-      result.ok,
-      result.latencyMs,
-      prev
-        ? {
-            consecutive_failures: prev.consecutive_failures,
-            status: prev.status,
-          }
-        : null,
-      options.thresholds,
-    );
-    const prevState: IpHealthState | null = prev
-      ? (prev.status as IpHealthState)
-      : null;
-    repos.upsertIpHealthStatus(
-      db,
-      target.scope,
-      target.ref_id,
-      target.ip,
-      state,
-      result.latencyMs,
-      failures,
-      result.error,
-    );
-    if (prevState !== state) {
-      options.onStatusChange?.(target, prevState, state);
+    const key = physicalProbeKey(target);
+    const list = byPhysical.get(key);
+    if (list) list.push(target);
+    else byPhysical.set(key, [target]);
+  }
+
+  let probeIndex = 0;
+  for (const group of byPhysical.values()) {
+    if (probeIndex > 0 && gapMs > 0) {
+      await sleep(gapMs);
+    }
+    probeIndex += 1;
+
+    // Prefer binding hostname for SNI when several scopes share one IP.
+    const representative =
+      group.find((t) => t.scope === "binding") ?? group[0]!;
+    const result = await probeTarget(representative);
+
+    for (const target of group) {
+      const prev = repos.getIpHealthStatusRow(
+        db,
+        target.scope,
+        target.ref_id,
+        target.ip,
+      );
+      const { state, failures } = deriveState(
+        result.ok,
+        result.latencyMs,
+        prev
+          ? {
+              consecutive_failures: prev.consecutive_failures,
+              status: prev.status,
+            }
+          : null,
+        options.thresholds,
+      );
+      const prevState: IpHealthState | null = prev
+        ? (prev.status as IpHealthState)
+        : null;
+      repos.upsertIpHealthStatus(
+        db,
+        target.scope,
+        target.ref_id,
+        target.ip,
+        state,
+        result.latencyMs,
+        failures,
+        result.error,
+      );
+      if (prevState !== state) {
+        options.onStatusChange?.(target, prevState, state);
+      }
    }
  }
+  // Orphan rows (old IPs / hostname keys) still feed MAX latency on group badge.
+  repos.pruneStaleIpHealthStatus(db, targets);
  return targets.length;
 }