feat(api, web): enhance health check and domain management features

- Integrated domain monitoring routes and bulk update functionality for domains in the API.
- Improved health check service to include domain monitoring and logging of health status changes.
- Updated web components to reflect health status with new HealthCheckBadge and enhanced domain filtering options.
- Refactored domain service to support bulk updates and improved domain management capabilities.

Co-authored-by: Cursor <[email protected]>

apps/api/src/app.ts

@@ -7,6 +7,7 @@ import {
 } from "@fastify/type-provider-zod";
 import type { AppConfig } from "./config.js";
 import { loadConfig } from "./config.js";
+import { repos } from "@cfdm/db";
 import authPlugin from "./plugins/auth.js";
 import cfClientPlugin from "./plugins/cf-client.js";
 import { requireAuth } from "./plugins/auth.js";
@@ -24,6 +25,10 @@ import { subdomainRoutes } from "./routes/subdomains.js";
 import { certificateRoutes } from "./routes/certificates.js";
 import { syncRoutes } from "./routes/sync.js";
 import { healthCheckRoutes } from "./routes/health-check.js";
+import {
+  domainMonitorRoutes,
+  notificationRoutes,
+} from "./routes/domain-monitors.js";
 import { settingsRoutes } from "./routes/settings.js";
 import { integrationsVpsTrackerRoutes } from "./routes/integrations-vps-tracker.js";
 import * as certificateService from "./services/certificate-service.js";
@@ -75,6 +80,8 @@ export async function buildApp(opts: BuildAppOptions = {}) {
       await protectedApi.register(certificateRoutes);
       await protectedApi.register(syncRoutes);
       await protectedApi.register(healthCheckRoutes);
+      await protectedApi.register(domainMonitorRoutes);
+      await protectedApi.register(notificationRoutes);
       await protectedApi.register(settingsRoutes);
     },
     { prefix: "/api/v1" },
@@ -117,14 +124,31 @@ export async function buildApp(opts: BuildAppOptions = {}) {
     const healthTask = new AsyncTask(
       "health-check",
       async () => {
+        const thresholds = {
+          degradedFailures: config.healthDegradedFailures,
+          downFailures: config.healthDownFailures,
+          latencyWarnMs: config.healthLatencyWarnMs,
+        };
         const n = await healthCheckService.runAllChecks(app.db, {
-          thresholds: {
-            degradedFailures: config.healthDegradedFailures,
-            downFailures: config.healthDownFailures,
-            latencyWarnMs: config.healthLatencyWarnMs,
-          },
-          onStatusChange: async (target, _prev, _next) => {
+          thresholds,
+          onStatusChange: async (target, prev, next) => {
             try {
+              const label =
+                next === "up"
+                  ? "OK"
+                  : next === "degraded"
+                    ? "Slow"
+                    : next === "down"
+                      ? "Down"
+                      : "—";
+              repos.insertNotificationLog(
+                app.db,
+                "ip_health",
+                target.scope,
+                target.ref_id,
+                `${target.hostname || target.ip}: ${label}`,
+                `IP ${target.ip}: ${prev ?? "—"} → ${label}`,
+              );
               await serviceConfigService.reconcileDnsForTarget(
                 app.db,
                 app.cf,
@@ -139,7 +163,14 @@ export async function buildApp(opts: BuildAppOptions = {}) {
             }
           },
         });
-        app.log.info({ checked: n }, "health check completed");
+        const monitors = await healthCheckService.runDomainMonitors(
+          app.db,
+          thresholds,
+        );
+        app.log.info(
+          { checked: n, monitors },
+          "health check completed",
+        );
       },
       (err) => {
         app.log.warn({ err }, "health check failed");

apps/api/src/routes/domain-monitors.ts

@@ -0,0 +1,55 @@
+import type { FastifyInstance } from "fastify";
+import { createDomainMonitorSchema } from "@cfdm/shared";
+import { repos } from "@cfdm/db";
+import * as healthCheckService from "../services/health-check-service.js";
+
+export async function domainMonitorRoutes(app: FastifyInstance) {
+  app.get("/domains/:id/monitors", async (request) => {
+    const { id } = request.params as { id: string };
+    return repos.listDomainMonitors(request.server.db, Number(id));
+  });
+
+  app.post("/domains/:id/monitors", async (request) => {
+    const { id } = request.params as { id: string };
+    const body = createDomainMonitorSchema.parse(request.body);
+    return repos.createDomainMonitor(request.server.db, Number(id), body);
+  });
+
+  app.delete("/domains/:domainId/monitors/:monitorId", async (request) => {
+    const { monitorId } = request.params as {
+      domainId: string;
+      monitorId: string;
+    };
+    repos.deleteDomainMonitor(request.server.db, Number(monitorId));
+    return { deleted: true };
+  });
+
+  app.get("/domains/:id/monitor-results", async (request) => {
+    const { id } = request.params as { id: string };
+    const query = request.query as { limit?: string };
+    const limit = query.limit ? Number(query.limit) : 50;
+    return repos.listDomainMonitorResultsForDomain(
+      request.server.db,
+      Number(id),
+      limit,
+    );
+  });
+
+  app.post("/domains/:id/monitors/run", async (request) => {
+    const config = request.server.config;
+    const checked = await healthCheckService.runDomainMonitors(request.server.db, {
+      degradedFailures: config.healthDegradedFailures,
+      downFailures: config.healthDownFailures,
+      latencyWarnMs: config.healthLatencyWarnMs,
+    });
+    return { checked };
+  });
+}
+
+export async function notificationRoutes(app: FastifyInstance) {
+  app.get("/notifications/log", async (request) => {
+    const query = request.query as { limit?: string };
+    const limit = query.limit ? Number(query.limit) : 50;
+    return repos.listNotificationLog(request.server.db, limit);
+  });
+}

apps/api/src/routes/domains.ts

@@ -1,5 +1,5 @@
 import type { FastifyInstance } from "fastify";
-import { updateDomainSchema } from "@cfdm/shared";
+import { bulkUpdateDomainsSchema, updateDomainSchema } from "@cfdm/shared";
 import { z } from "zod";
 import * as domainService from "../services/domain-service.js";
 
@@ -25,6 +25,20 @@ export async function domainRoutes(app: FastifyInstance) {
     );
   });
 
+  app.post("/domains/bulk", async (request) => {
+    const body = bulkUpdateDomainsSchema.parse(request.body);
+    const updated = domainService.bulkUpdateDomains(
+      request.server.db,
+      body.ids,
+      {
+        group_id: body.group_id,
+        environment: body.environment,
+        tags_add: body.tags_add,
+      },
+    );
+    return { updated };
+  });
+
   app.get("/domains/:id", async (request) => {
     const { id } = request.params as { id: string };
     return domainService.getDomain(request.server.db, Number(id));
@@ -33,14 +47,7 @@ export async function domainRoutes(app: FastifyInstance) {
   app.patch("/domains/:id", async (request) => {
     const { id } = request.params as { id: string };
     const body = updateDomainSchema.parse(request.body);
-    const existing = domainService.getDomain(request.server.db, Number(id));
-    return domainService.updateDomain(
-      request.server.db,
-      Number(id),
-      body.group_id !== undefined ? body.group_id : existing.group_id,
-      body.status ?? existing.status,
-      body.cert_monitoring,
-    );
+    return domainService.updateDomain(request.server.db, Number(id), body);
   });
 
   app.delete("/domains/:id", async (request) => {

apps/api/src/routes/health-check.ts

@@ -1,5 +1,6 @@
 import type { FastifyInstance } from "fastify";
 import { healthStatusQuerySchema } from "@cfdm/shared";
+import { repos } from "@cfdm/db";
 import * as healthCheckService from "../services/health-check-service.js";
 import * as serviceConfigService from "../services/service-config-service.js";
 
@@ -15,14 +16,31 @@ export async function healthCheckRoutes(app: FastifyInstance) {
 
   app.post("/health-check/run", async (request) => {
     const config = request.server.config;
+    const thresholds = {
+      degradedFailures: config.healthDegradedFailures,
+      downFailures: config.healthDownFailures,
+      latencyWarnMs: config.healthLatencyWarnMs,
+    };
     const checked = await healthCheckService.runAllChecks(request.server.db, {
-      thresholds: {
-        degradedFailures: config.healthDegradedFailures,
-        downFailures: config.healthDownFailures,
-        latencyWarnMs: config.healthLatencyWarnMs,
-      },
-      onStatusChange: async (target, _prev, _next) => {
+      thresholds,
+      onStatusChange: async (target, prev, next) => {
         try {
+          const label =
+            next === "up"
+              ? "OK"
+              : next === "degraded"
+                ? "Slow"
+                : next === "down"
+                  ? "Down"
+                  : "—";
+          repos.insertNotificationLog(
+            request.server.db,
+            "ip_health",
+            target.scope,
+            target.ref_id,
+            `${target.hostname || target.ip}: ${label}`,
+            `IP ${target.ip}: ${prev ?? "—"} → ${label}`,
+          );
           await serviceConfigService.reconcileDnsForTarget(
             request.server.db,
             request.server.cf,
@@ -30,10 +48,14 @@ export async function healthCheckRoutes(app: FastifyInstance) {
             target.ref_id,
           );
         } catch {
-          // best-effort reconcile; ошибки логируются cron-задачей
+          // best-effort
         }
       },
     });
-    return { checked };
+    const monitors = await healthCheckService.runDomainMonitors(
+      request.server.db,
+      thresholds,
+    );
+    return { checked, monitors };
   });
 }

apps/api/src/services/domain-service.ts

@@ -43,11 +43,51 @@ export async function createDomain(
 export function updateDomain(
   db: Db,
   id: number,
-  groupId: number | null,
-  status: string,
-  certMonitoring?: string,
+  patch: {
+    group_id?: number | null;
+    status?: string;
+    cert_monitoring?: string;
+    environment?: string | null;
+    tags?: string[];
+  },
 ): Domain {
-  return repos.updateDomain(db, id, groupId, status, certMonitoring);
+  const domain = repos.updateDomain(db, id, {
+    group_id: patch.group_id,
+    status: patch.status,
+    cert_monitoring: patch.cert_monitoring,
+    environment: patch.environment,
+  });
+  if (patch.tags !== undefined) {
+    repos.setDomainTags(db, id, patch.tags);
+  }
+  return domain;
+}
+
+export function bulkUpdateDomains(
+  db: Db,
+  ids: number[],
+  patch: {
+    group_id?: number | null;
+    environment?: string | null;
+    tags_add?: string[];
+  },
+): number {
+  let updated = 0;
+  for (const id of ids) {
+    try {
+      repos.updateDomain(db, id, {
+        group_id: patch.group_id,
+        environment: patch.environment,
+      });
+      if (patch.tags_add?.length) {
+        repos.addDomainTags(db, id, patch.tags_add);
+      }
+      updated += 1;
+    } catch {
+      // skip missing
+    }
+  }
+  return updated;
 }
 
 export function deleteDomain(db: Db, id: number): void {

apps/api/src/services/health-check-service.ts

@@ -1,4 +1,5 @@
 import { connect } from "node:net";
+import { resolve4, resolve6 } from "node:dns/promises";
 import { Agent, fetch as undiciFetch } from "undici";
 import type { Db } from "@cfdm/db";
 import { repos } from "@cfdm/db";
@@ -68,10 +69,6 @@ async function httpProbe(
   const pathWithSlash = path.startsWith("/") ? path : `/${path}`;
   const port = target.port ?? 80;
   const useTls = port === 443;
-  // For HTTPS on 443, probe via the hostname (URL host = hostname) so TLS SNI, Host header
-  // and any edge/vhost protection (e.g. Cloudflare origin 421 on direct-IP) all line up.
-  // For multi-A records this loses strict per-IP HTTPS granularity — use TCP probe for that.
-  // For plain HTTP we still hit the literal IP (per-record target).
   const urlHost = useTls ? target.hostname || ip : ip;
   const url = `${useTls ? "https" : "http"}://${urlHost}${pathWithSlash}`;
   const dispatcher =
@@ -119,6 +116,47 @@ async function httpProbe(
   }
 }
 
+/** Host reachability via TCP :443 then :80 (ICMP often unavailable in Node). */
+async function pingProbe(hostname: string, timeoutMs: number): Promise<ProbeResult> {
+  const ports = [443, 80];
+  let last: ProbeResult = {
+    ok: false,
+    latencyMs: 0,
+    error: "unreachable",
+  };
+  for (const port of ports) {
+    last = await tcpProbe(hostname, port, timeoutMs);
+    if (last.ok) return last;
+  }
+  return last;
+}
+
+async function dnsProbe(hostname: string): Promise<ProbeResult> {
+  const started = Date.now();
+  try {
+    const [v4, v6] = await Promise.allSettled([
+      resolve4(hostname),
+      resolve6(hostname),
+    ]);
+    const hasV4 = v4.status === "fulfilled" && v4.value.length > 0;
+    const hasV6 = v6.status === "fulfilled" && v6.value.length > 0;
+    if (!hasV4 && !hasV6) {
+      return {
+        ok: false,
+        latencyMs: Date.now() - started,
+        error: "no A/AAAA records",
+      };
+    }
+    return { ok: true, latencyMs: Date.now() - started, error: null };
+  } catch (err) {
+    return {
+      ok: false,
+      latencyMs: Date.now() - started,
+      error: err instanceof Error ? err.message : String(err),
+    };
+  }
+}
+
 export async function probeTarget(
   target: HealthCheckTarget,
 ): Promise<ProbeResult> {
@@ -127,6 +165,12 @@ export async function probeTarget(
   if (target.type === "http") {
     return httpProbe(target.ip, target, timeoutMs);
   }
+  if (target.type === "ping") {
+    return pingProbe(target.hostname || target.ip, timeoutMs);
+  }
+  if (target.type === "dns") {
+    return dnsProbe(target.hostname || target.ip);
+  }
   return tcpProbe(target.ip, port, timeoutMs);
 }
 
@@ -205,6 +249,79 @@ export async function runAllChecks(
   return targets.length;
 }
 
+export async function runDomainMonitors(
+  db: Db,
+  thresholds: HealthCheckThresholds,
+): Promise<number> {
+  const monitors = repos.listEnabledDomainMonitors(db);
+  let checked = 0;
+  for (const monitor of monitors) {
+    const target: HealthCheckTarget = {
+      scope: "binding",
+      ref_id: monitor.id,
+      ip: monitor.hostname,
+      hostname: monitor.hostname,
+      type: monitor.type as HealthCheckTarget["type"],
+      port: monitor.type === "http" ? (monitor.path?.includes("443") ? 443 : 80) : null,
+      path: monitor.path,
+      expected_status: monitor.expected_status,
+      timeout_ms: monitor.timeout_ms,
+    };
+    let result: ProbeResult;
+    if (monitor.type === "http") {
+      result = await httpProbe(monitor.hostname, {
+        ...target,
+        port: 443,
+        path: monitor.path ?? "/",
+      }, monitor.timeout_ms);
+      if (!result.ok) {
+        result = await httpProbe(monitor.hostname, {
+          ...target,
+          port: 80,
+          path: monitor.path ?? "/",
+        }, monitor.timeout_ms);
+      }
+    } else if (monitor.type === "ping") {
+      result = await pingProbe(monitor.hostname, monitor.timeout_ms);
+    } else {
+      result = await dnsProbe(monitor.hostname);
+    }
+    const prevStatus = monitor.last_status as IpHealthState;
+    const { state } = deriveState(
+      result.ok,
+      result.latencyMs,
+      {
+        consecutive_failures: result.ok ? 0 : 1,
+        status: prevStatus,
+      },
+      thresholds,
+    );
+    repos.updateDomainMonitorResult(
+      db,
+      monitor.id,
+      state,
+      result.latencyMs,
+      result.error,
+    );
+    if (prevStatus !== state && prevStatus !== "unknown") {
+      const label =
+        state === "up" ? "OK" : state === "degraded" ? "Slow" : state === "down" ? "Down" : "—";
+      repos.insertNotificationLog(
+        db,
+        "domain_monitor",
+        "domain_monitor",
+        monitor.id,
+        `${monitor.hostname}: ${label}`,
+        result.error
+          ? `${monitor.type.toUpperCase()} → ${label}. ${result.error}`
+          : `${monitor.type.toUpperCase()} → ${label}${result.latencyMs != null ? ` (${result.latencyMs} мс)` : ""}`,
+      );
+    }
+    checked += 1;
+  }
+  return checked;
+}
+
 export function listStatus(
   db: Db,
   scope: "binding" | "group",

apps/api/test/certificates.test.ts

@@ -141,13 +141,11 @@ describe("certificates", () => {
       "required.example.com",
       "cf-zone-req",
     );
-    repos.updateDomain(
-      testApp.db,
-      domain.id,
-      null,
-      "active",
-      CERT_MONITOR_REQUIRED,
-    );
+    repos.updateDomain(testApp.db, domain.id, {
+      group_id: null,
+      status: "active",
+      cert_monitoring: CERT_MONITOR_REQUIRED,
+    });
 
     vi.spyOn(certificateService, "checkHostname").mockResolvedValue({
       expiresAt: new Date(Date.now() + 90 * 24 * 60 * 60 * 1000),
@@ -191,13 +189,11 @@ describe("certificates", () => {
       CERT_ERROR,
       "stale",
     );
-    repos.updateDomain(
-      testApp.db,
-      domain.id,
-      null,
-      "active",
-      CERT_MONITOR_SKIPPED,
-    );
+    repos.updateDomain(testApp.db, domain.id, {
+      group_id: null,
+      status: "active",
+      cert_monitoring: CERT_MONITOR_SKIPPED,
+    });
 
     vi.spyOn(certificateService, "checkHostname").mockResolvedValue({
       expiresAt: null,
@@ -229,13 +225,11 @@ describe("certificates", () => {
       "broken.example.com",
       "cf-zone-broken",
     );
-    repos.updateDomain(
-      testApp.db,
-      domain.id,
-      null,
-      "active",
-      CERT_MONITOR_REQUIRED,
-    );
+    repos.updateDomain(testApp.db, domain.id, {
+      group_id: null,
+      status: "active",
+      cert_monitoring: CERT_MONITOR_REQUIRED,
+    });
 
     vi.spyOn(certificateService, "checkHostname").mockResolvedValue({
       expiresAt: null,