feat: Implement load balancing and health check features for service groups, including DNS-based load balancing modes and health check configurations, enhancing service reliability and performance monitoring

apps/api/src/app.ts

@@ -23,7 +23,10 @@ import { dnsRoutes } from "./routes/dns.js";
 import { subdomainRoutes } from "./routes/subdomains.js";
 import { certificateRoutes } from "./routes/certificates.js";
 import { syncRoutes } from "./routes/sync.js";
+import { healthCheckRoutes } from "./routes/health-check.js";
 import * as certificateService from "./services/certificate-service.js";
+import * as healthCheckService from "./services/health-check-service.js";
+import * as serviceConfigService from "./services/service-config-service.js";
 import { AsyncTask, CronJob } from "toad-scheduler";
 
 export interface BuildAppOptions {
@@ -68,6 +71,7 @@ export async function buildApp(opts: BuildAppOptions = {}) {
       await protectedApi.register(subdomainRoutes);
       await protectedApi.register(certificateRoutes);
       await protectedApi.register(syncRoutes);
+      await protectedApi.register(healthCheckRoutes);
     },
     { prefix: "/api/v1" },
   );
@@ -105,6 +109,46 @@ export async function buildApp(opts: BuildAppOptions = {}) {
         { preventOverrun: true },
       ),
     );
+
+    const healthTask = new AsyncTask(
+      "health-check",
+      async () => {
+        const n = await healthCheckService.runAllChecks(app.db, {
+          thresholds: {
+            degradedFailures: config.healthDegradedFailures,
+            downFailures: config.healthDownFailures,
+            latencyWarnMs: config.healthLatencyWarnMs,
+          },
+          onStatusChange: async (target, _prev, _next) => {
+            try {
+              await serviceConfigService.reconcileDnsForTarget(
+                app.db,
+                app.cf,
+                target.scope,
+                target.ref_id,
+              );
+            } catch (err) {
+              app.log.warn(
+                { err, scope: target.scope, refId: target.ref_id },
+                "health-check reconcile failed",
+              );
+            }
+          },
+        });
+        app.log.info({ checked: n }, "health check completed");
+      },
+      (err) => {
+        app.log.warn({ err }, "health check failed");
+      },
+    );
+
+    app.scheduler.addCronJob(
+      new CronJob(
+        { cronExpression: config.healthCheckCron },
+        healthTask,
+        { preventOverrun: true },
+      ),
+    );
   }
 
   return app;