feat(health-checks): implement health check IP toggling and configuration updates

- Added functionality to toggle individual IPs for services, allowing for dynamic management of IP health status.
- Enhanced the health check configuration in the UI, enabling users to set parameters directly from the settings page.
- Updated service views to include IP health tracking, improving visibility into the status of each IP associated with a service.
- Refactored relevant components to support the new IP toggling feature, ensuring a seamless user experience.

This commit significantly enhances the health management capabilities of services, providing users with more control over IP configurations and health monitoring.

apps/api/src/services/health-check-scheduler.ts

@@ -0,0 +1,134 @@
+import type { FastifyInstance } from "fastify";
+import { AsyncTask, CronJob } from "toad-scheduler";
+import {
+  getAppSettings,
+  type HealthEngineFallbacks,
+} from "@cfdm/db";
+import { repos } from "@cfdm/db";
+import type { AppConfig } from "../config.js";
+import { AppError } from "../errors.js";
+import * as healthCheckService from "./health-check-service.js";
+import * as serviceConfigService from "./service-config-service.js";
+
+declare module "fastify" {
+  interface FastifyInstance {
+    reloadHealthCheckJob?: () => void;
+  }
+}
+
+export const HEALTH_CHECK_JOB_ID = "health-check";
+
+export function healthEngineFallbacksFromConfig(
+  config: AppConfig,
+): HealthEngineFallbacks {
+  return {
+    healthCheckCron: config.healthCheckCron,
+    healthDegradedFailures: config.healthDegradedFailures,
+    healthDownFailures: config.healthDownFailures,
+    healthLatencyWarnMs: config.healthLatencyWarnMs,
+    healthSuccessRecoveries: config.healthSuccessRecoveries,
+  };
+}
+
+export function assertValidHealthCron(expr: string): void {
+  const cronExpression = expr.trim();
+  const parts = cronExpression.split(/\s+/).filter(Boolean);
+  if (parts.length < 5 || parts.length > 6) {
+    throw AppError.validation("некорректное cron-выражение");
+  }
+  try {
+    const job = new CronJob(
+      { cronExpression },
+      new AsyncTask("validate-cron", async () => undefined),
+      { id: "validate-cron" },
+    );
+    job.stop();
+  } catch {
+    throw AppError.validation("некорректное cron-выражение");
+  }
+}
+
+export function createHealthCheckTask(
+  app: FastifyInstance,
+  config: AppConfig,
+): AsyncTask {
+  const fallbacks = healthEngineFallbacksFromConfig(config);
+  return new AsyncTask(
+    HEALTH_CHECK_JOB_ID,
+    async () => {
+      const settings = getAppSettings(app.db, fallbacks);
+      const thresholds = {
+        degradedFailures: settings.healthDegradedFailures,
+        downFailures: settings.healthDownFailures,
+        latencyWarnMs: settings.healthLatencyWarnMs,
+        successRecoveries: settings.healthSuccessRecoveries,
+      };
+      const n = await healthCheckService.runAllChecks(app.db, {
+        thresholds,
+        probeGapMs: config.healthProbeGapMs,
+        onStatusChange: async (target, prev, next) => {
+          try {
+            const label =
+              next === "up"
+                ? "OK"
+                : next === "degraded"
+                  ? "Slow"
+                  : next === "down"
+                    ? "Down"
+                    : "—";
+            repos.insertNotificationLog(
+              app.db,
+              "ip_health",
+              target.scope,
+              target.ref_id,
+              `${target.hostname || target.ip}: ${label}`,
+              `IP ${target.ip}: ${prev ?? "—"} → ${label}`,
+            );
+            await serviceConfigService.reconcileDnsForTarget(
+              app.db,
+              app.cf,
+              target.scope,
+              target.ref_id,
+            );
+          } catch (err) {
+            app.log.warn(
+              { err, scope: target.scope, refId: target.ref_id },
+              "health-check reconcile failed",
+            );
+          }
+        },
+      });
+      const monitors = await healthCheckService.runDomainMonitors(
+        app.db,
+        thresholds,
+      );
+      app.log.info({ checked: n, monitors }, "health check completed");
+    },
+    (err) => {
+      app.log.warn({ err }, "health check failed");
+    },
+  );
+}
+
+export function scheduleHealthCheckJob(
+  app: FastifyInstance,
+  config: AppConfig,
+  task: AsyncTask,
+): void {
+  const scheduler = app.scheduler;
+  if (!scheduler) return;
+  if (scheduler.existsById(HEALTH_CHECK_JOB_ID)) {
+    scheduler.removeById(HEALTH_CHECK_JOB_ID);
+  }
+  const settings = getAppSettings(
+    app.db,
+    healthEngineFallbacksFromConfig(config),
+  );
+  scheduler.addCronJob(
+    new CronJob(
+      { cronExpression: settings.healthCheckCron },
+      task,
+      { preventOverrun: true, id: HEALTH_CHECK_JOB_ID },
+    ),
+  );
+}

apps/api/src/services/service-config-service.ts

@@ -242,7 +242,11 @@ async function collectKnownZones(
 
 async function buildView(db: Db, serviceId: number): Promise<ServiceView> {
   const service = repos.getService(db, serviceId);
-  const ips = repos.listServiceIps(db, serviceId);
+  const ipRows = repos.listServiceIpRows(db, serviceId);
+  const ips = ipRows.map((row) => row.ip);
+  const ip_enabled = Object.fromEntries(
+    ipRows.map((row) => [row.ip, row.enabled]),
+  );
   const bindings = repos.listBindingsByService(db, serviceId);
 
   const domainViews = bindings.map((binding) => {
@@ -304,6 +308,7 @@ async function buildView(db: Db, serviceId: number): Promise<ServiceView> {
     created_at: service.created_at,
     updated_at: service.updated_at,
     ips,
+    ip_enabled,
     domains: domainViews,
     health_status: "unknown",
     health_latency_ms: null,
@@ -384,7 +389,7 @@ export async function listGroupViews(db: Db): Promise<ServiceGroupsResponse> {
 
   const groupViews = groupViewsRaw.map((group) => {
     const services = group.services.map(
-      (s) => healthById.get(s.id) ?? { ...s, health_status: "unknown" as const, health_latency_ms: null, ip_health: [] },
+      (s) => healthById.get(s.id) ?? { ...s, health_status: "unknown" as const, health_latency_ms: null, ip_health: [], ip_enabled: {} },
     );
     const groupScopeHealth = groupHealthById.get(group.id);
     // Only enabled services feed the group badge — a disabled service with a
@@ -414,6 +419,7 @@ export async function listGroupViews(db: Db): Promise<ServiceGroupsResponse> {
         health_status: "unknown" as const,
         health_latency_ms: null,
         ip_health: [],
+        ip_enabled: {},
       },
   );
 
@@ -1345,6 +1351,59 @@ export async function toggleService(
   return enabledView!;
 }
 
+export async function toggleServiceIp(
+  db: Db,
+  cf: CloudflareClient,
+  serviceId: number,
+  ip: string,
+  enabled: boolean,
+): Promise<ServiceView> {
+  repos.getService(db, serviceId);
+  const pool = repos.listServiceIps(db, serviceId);
+  if (!pool.includes(ip)) {
+    throw AppError.validation(`IP ${ip} не входит в пул адресов сервиса`);
+  }
+
+  repos.setServiceIpEnabled(db, serviceId, ip, enabled);
+  const node = repos
+    .listNodes(db, serviceId)
+    .find((entry) => entry.address === ip);
+  if (node) {
+    repos.updateNode(db, node.id, { enabled });
+  }
+
+  const bindings = repos.listBindingsByService(db, serviceId);
+  for (const binding of bindings) {
+    if (binding.cname_target?.trim()) continue;
+    const current = repos.listBindingIpsWithMeta(db, binding.id);
+    const hasIp = current.some((entry) => entry.ip === ip);
+    if (enabled && !hasIp) {
+      repos.replaceBindingIpsWithMeta(db, binding.id, [
+        ...current,
+        { ip, weight: 1, priority: 1 },
+      ]);
+      continue;
+    }
+    if (!enabled && hasIp) {
+      repos.replaceBindingIpsWithMeta(
+        db,
+        binding.id,
+        current.filter((entry) => entry.ip !== ip),
+      );
+    }
+  }
+
+  const service = repos.getService(db, serviceId);
+  if (shouldPushDns(db, service)) {
+    await syncServiceBindingsToDns(db, cf, serviceId);
+    await syncGroupDomainForService(db, cf, serviceId);
+  }
+  void syncServiceToVpsTracker(db, serviceId);
+
+  const [view] = attachServiceHealth(db, [await buildView(db, serviceId)]);
+  return view!;
+}
+
 export async function toggleGroup(
   db: Db,
   cf: CloudflareClient,

apps/api/src/services/vps-tracker-sync.ts

@@ -43,8 +43,9 @@ function resolveIpsLocally(
   const binding = index.byFqdn.get(key);
   if (!binding) return [];
 
-  if (binding.target_ips.some(isIpLiteral)) {
-    return binding.target_ips.filter(isIpLiteral);
+  const ips = (binding.target_ips ?? []).filter(isIpLiteral);
+  if (ips.length > 0) {
+    return ips;
   }
 
   const cname = binding.cname_target?.trim();
@@ -79,7 +80,7 @@ export async function resolveBindingIpsForSync(
   index: BindingIpIndex,
   db?: Db,
 ): Promise<string[]> {
-  const directIps = binding.target_ips.filter(isIpLiteral);
+  const directIps = (binding.target_ips ?? []).filter(isIpLiteral);
   if (directIps.length > 0) {
     return [...directIps];
   }
@@ -124,7 +125,7 @@ export async function buildServiceSyncBindingsAsync(
   const serviceIps = repos.listServiceIps(db, serviceId);
   const allBindings = repos.listAllBindings(db);
   const index = buildBindingIndex(allBindings);
-  const bindings = repos.listBindingsByService(db, serviceId);
+  const bindings = allBindings.filter((row) => row.service_id === serviceId);
 
   const items: CfdmBindingSyncItem[] = [];
   for (const binding of bindings) {