feat(health-checks): implement health check IP toggling and configuration updates

- Added functionality to toggle individual IPs for services, allowing for dynamic management of IP health status.
- Enhanced the health check configuration in the UI, enabling users to set parameters directly from the settings page.
- Updated service views to include IP health tracking, improving visibility into the status of each IP associated with a service.
- Refactored relevant components to support the new IP toggling feature, ensuring a seamless user experience.

This commit significantly enhances the health management capabilities of services, providing users with more control over IP configurations and health monitoring.

apps/api/src/app.ts

@@ -7,7 +7,6 @@ import {
 } from "@fastify/type-provider-zod";
 import type { AppConfig } from "./config.js";
 import { loadConfig } from "./config.js";
-import { repos } from "@cfdm/db";
 import authPlugin from "./plugins/auth.js";
 import cfClientPlugin from "./plugins/cf-client.js";
 import { requireAuth } from "./plugins/auth.js";
@@ -34,8 +33,10 @@ import { settingsRoutes } from "./routes/settings.js";
 import { integrationsVpsTrackerRoutes } from "./routes/integrations-vps-tracker.js";
 import { auditRoutes } from "./routes/audit.js";
 import * as certificateService from "./services/certificate-service.js";
-import * as healthCheckService from "./services/health-check-service.js";
-import * as serviceConfigService from "./services/service-config-service.js";
+import {
+  createHealthCheckTask,
+  scheduleHealthCheckJob,
+} from "./services/health-check-scheduler.js";
 import { AsyncTask, CronJob } from "toad-scheduler";
 
 export interface BuildAppOptions {
@@ -125,71 +126,11 @@ export async function buildApp(opts: BuildAppOptions = {}) {
       ),
     );
 
-    const healthTask = new AsyncTask(
-      "health-check",
-      async () => {
-        const thresholds = {
-          degradedFailures: config.healthDegradedFailures,
-          downFailures: config.healthDownFailures,
-          latencyWarnMs: config.healthLatencyWarnMs,
-          successRecoveries: config.healthSuccessRecoveries,
-        };
-        const n = await healthCheckService.runAllChecks(app.db, {
-          thresholds,
-          probeGapMs: config.healthProbeGapMs,
-          onStatusChange: async (target, prev, next) => {
-            try {
-              const label =
-                next === "up"
-                  ? "OK"
-                  : next === "degraded"
-                    ? "Slow"
-                    : next === "down"
-                      ? "Down"
-                      : "—";
-              repos.insertNotificationLog(
-                app.db,
-                "ip_health",
-                target.scope,
-                target.ref_id,
-                `${target.hostname || target.ip}: ${label}`,
-                `IP ${target.ip}: ${prev ?? "—"} → ${label}`,
-              );
-              await serviceConfigService.reconcileDnsForTarget(
-                app.db,
-                app.cf,
-                target.scope,
-                target.ref_id,
-              );
-            } catch (err) {
-              app.log.warn(
-                { err, scope: target.scope, refId: target.ref_id },
-                "health-check reconcile failed",
-              );
-            }
-          },
-        });
-        const monitors = await healthCheckService.runDomainMonitors(
-          app.db,
-          thresholds,
-        );
-        app.log.info(
-          { checked: n, monitors },
-          "health check completed",
-        );
-      },
-      (err) => {
-        app.log.warn({ err }, "health check failed");
-      },
-    );
-
-    app.scheduler.addCronJob(
-      new CronJob(
-        { cronExpression: config.healthCheckCron },
-        healthTask,
-        { preventOverrun: true },
-      ),
-    );
+    const healthTask = createHealthCheckTask(app, config);
+    scheduleHealthCheckJob(app, config, healthTask);
+    app.decorate("reloadHealthCheckJob", () => {
+      scheduleHealthCheckJob(app, config, healthTask);
+    });
   }
 
   return app;