feat(health-checks): enhance health check functionality and add new routes

- Introduced origin health check routes and integrated them into the application.
- Updated health check configuration to include success recovery thresholds.
- Expanded error handling with new error codes for health check failures.
- Added new service routes for managing health checks, including creation and listing.
- Improved health check service logic to track consecutive successes and failures.

This commit enhances the health check capabilities, providing better monitoring and management of service health.

packages/db/dist/index.js

@@ -117,6 +117,8 @@ var serviceBindings = sqliteTable(
     health_check_verify_tls: integer("health_check_verify_tls", {
       mode: "boolean"
     }).notNull().default(false),
+    routing_strategy: text("routing_strategy").notNull().default("round_robin"),
+    operation_version: integer("operation_version").notNull().default(0),
     created_at: text("created_at").notNull().default(sql`datetime('now')`),
     updated_at: text("updated_at").notNull().default(sql`datetime('now')`)
   },
@@ -128,6 +130,59 @@ var serviceBindings = sqliteTable(
     )
   ]
 );
+var healthChecks = sqliteTable("health_checks", {
+  id: integer("id").primaryKey({ autoIncrement: true }),
+  provider: text("provider").notNull().default("local"),
+  cf_healthcheck_id: text("cf_healthcheck_id"),
+  cf_zone_id: text("cf_zone_id"),
+  name: text("name").notNull(),
+  protocol: text("protocol").notNull().default("tcp"),
+  path: text("path"),
+  method: text("method"),
+  timeout: integer("timeout").notNull().default(5),
+  interval_sec: integer("interval_sec").notNull().default(30),
+  retries: integer("retries").notNull().default(2),
+  expected_status: integer("expected_status"),
+  consecutive_fails: integer("consecutive_fails").notNull().default(2),
+  consecutive_successes: integer("consecutive_successes").notNull().default(2),
+  suspended: integer("suspended", { mode: "boolean" }).notNull().default(false),
+  created_at: text("created_at").notNull().default(sql`datetime('now')`),
+  updated_at: text("updated_at").notNull().default(sql`datetime('now')`)
+});
+var nodes = sqliteTable(
+  "nodes",
+  {
+    id: integer("id").primaryKey({ autoIncrement: true }),
+    service_id: integer("service_id").notNull().references(() => services.id, { onDelete: "cascade" }),
+    address: text("address").notNull(),
+    protocol: text("protocol").notNull().default("tcp"),
+    port: integer("port"),
+    enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
+    priority: integer("priority").notNull().default(1),
+    weight: integer("weight").notNull().default(1),
+    health_status: text("health_status").notNull().default("unknown"),
+    health_check_id: integer("health_check_id").references(() => healthChecks.id, {
+      onDelete: "set null"
+    }),
+    consecutive_failures: integer("consecutive_failures").notNull().default(0),
+    consecutive_successes: integer("consecutive_successes").notNull().default(0),
+    last_check_at: text("last_check_at"),
+    last_failure_reason: text("last_failure_reason"),
+    created_at: text("created_at").notNull().default(sql`datetime('now')`),
+    updated_at: text("updated_at").notNull().default(sql`datetime('now')`)
+  },
+  (t) => [unique("nodes_service_address").on(t.service_id, t.address)]
+);
+var bindingNodes = sqliteTable(
+  "binding_nodes",
+  {
+    binding_id: integer("binding_id").notNull().references(() => serviceBindings.id, { onDelete: "cascade" }),
+    node_id: integer("node_id").notNull().references(() => nodes.id, { onDelete: "cascade" }),
+    weight: integer("weight").notNull().default(1),
+    priority: integer("priority").notNull().default(1)
+  },
+  (t) => [primaryKey({ columns: [t.binding_id, t.node_id] })]
+);
 var serviceIps = sqliteTable("service_ips", {
   id: integer("id").primaryKey({ autoIncrement: true }),
   service_id: integer("service_id").notNull().references(() => services.id, { onDelete: "cascade" }),
@@ -193,6 +248,7 @@ var ipHealthStatus = sqliteTable(
     status: text("status").notNull().default("unknown"),
     latency_ms: integer("latency_ms"),
     consecutive_failures: integer("consecutive_failures").notNull().default(0),
+    consecutive_successes: integer("consecutive_successes").notNull().default(0),
     last_checked_at: text("last_checked_at"),
     last_error: text("last_error"),
     created_at: text("created_at").notNull().default(sql`datetime('now')`),
@@ -282,6 +338,9 @@ var schema = {
   subdomains,
   dnsRecords,
   serviceBindings,
+  healthChecks,
+  nodes,
+  bindingNodes,
   serviceIps,
   serviceBindingRecords,
   serviceBindingIps,
@@ -495,10 +554,13 @@ __export(repos_exports, {
   aggregateIpHealthByRefs: () => aggregateIpHealthByRefs,
   aggregateIpHealthByServiceIds: () => aggregateIpHealthByServiceIds,
   bindingsToRemove: () => bindingsToRemove,
+  bumpBindingVersion: () => bumpBindingVersion,
   countCertificatesByStatus: () => countCertificatesByStatus,
   createDomain: () => createDomain,
   createDomainMonitor: () => createDomainMonitor,
   createGroup: () => createGroup,
+  createHealthCheck: () => createHealthCheck,
+  createNode: () => createNode,
   createService: () => createService,
   createServiceGroup: () => createServiceGroup,
   createSubdomain: () => createSubdomain,
@@ -510,14 +572,20 @@ __export(repos_exports, {
   deleteDomain: () => deleteDomain,
   deleteDomainMonitor: () => deleteDomainMonitor,
   deleteGroup: () => deleteGroup,
+  deleteHealthCheck: () => deleteHealthCheck,
   deleteIpHealthStatusForIp: () => deleteIpHealthStatusForIp,
   deleteIpHealthStatusForRef: () => deleteIpHealthStatusForRef,
+  deleteNode: () => deleteNode,
   deleteService: () => deleteService,
   deleteServiceGroup: () => deleteServiceGroup,
   deleteSubdomain: () => deleteSubdomain,
+  ensureNode: () => ensureNode,
   findBinding: () => findBinding,
   findDnsByCfId: () => findDnsByCfId,
   findDomainByZoneName: () => findDomainByZoneName,
+  findHealthCheckByCfId: () => findHealthCheckByCfId,
+  findNodeByAddress: () => findNodeByAddress,
+  findNodeByIp: () => findNodeByIp,
   findSubdomainByDomainAndName: () => findSubdomainByDomainAndName,
   finishSyncJob: () => finishSyncJob,
   getBinding: () => getBinding,
@@ -528,7 +596,9 @@ __export(repos_exports, {
   getDomainMonitor: () => getDomainMonitor,
   getGroup: () => getGroup,
   getGroupWithStats: () => getGroupWithStats,
+  getHealthCheck: () => getHealthCheck,
   getIpHealthStatusRow: () => getIpHealthStatusRow,
+  getNode: () => getNode,
   getService: () => getService,
   getServiceGroup: () => getServiceGroup,
   getSubdomain: () => getSubdomain,
@@ -540,9 +610,11 @@ __export(repos_exports, {
   linkGroupDnsRecord: () => linkGroupDnsRecord,
   listAllBindings: () => listAllBindings,
   listAllDomains: () => listAllDomains,
+  listAllNodes: () => listAllNodes,
   listAllSubdomains: () => listAllSubdomains,
   listBindingIps: () => listBindingIps,
   listBindingIpsWithMeta: () => listBindingIpsWithMeta,
+  listBindingNodes: () => listBindingNodes,
   listBindingsByDomain: () => listBindingsByDomain,
   listBindingsByService: () => listBindingsByService,
   listCertificates: () => listCertificates,
@@ -558,7 +630,9 @@ __export(repos_exports, {
   listGroupDnsRecords: () => listGroupDnsRecords,
   listGroups: () => listGroups,
   listHealthCheckTargets: () => listHealthCheckTargets,
+  listHealthChecks: () => listHealthChecks,
   listIpHealthStatus: () => listIpHealthStatus,
+  listNodes: () => listNodes,
   listNotificationLog: () => listNotificationLog,
   listOriginIpsForFqdn: () => listOriginIpsForFqdn,
   listRecordsForBinding: () => listRecordsForBinding,
@@ -577,6 +651,7 @@ __export(repos_exports, {
   replaceServiceIps: () => replaceServiceIps,
   setBindingCnameTarget: () => setBindingCnameTarget,
   setBindingDnsRecordId: () => setBindingDnsRecordId,
+  setBindingRoutingStrategy: () => setBindingRoutingStrategy,
   setDnsSyncStatus: () => setDnsSyncStatus,
   setDomainLastSynced: () => setDomainLastSynced,
   setDomainTags: () => setDomainTags,
@@ -586,12 +661,15 @@ __export(repos_exports, {
   setServiceLb: () => setServiceLb,
   unlinkBindingRecord: () => unlinkBindingRecord,
   unlinkGroupDnsRecord: () => unlinkGroupDnsRecord,
+  updateBindingDomain: () => updateBindingDomain,
   updateBindingFields: () => updateBindingFields,
   updateBindingLbConfig: () => updateBindingLbConfig,
   updateDnsFields: () => updateDnsFields,
   updateDomain: () => updateDomain,
   updateDomainMonitorResult: () => updateDomainMonitorResult,
   updateGroup: () => updateGroup,
+  updateHealthCheck: () => updateHealthCheck,
+  updateNode: () => updateNode,
   updateService: () => updateService,
   updateServiceGroup: () => updateServiceGroup,
   updateSubdomain: () => updateSubdomain,
@@ -1073,6 +1151,12 @@ function deleteServiceGroup(db, id) {
   const result = db.delete(serviceGroups).where(eq3(serviceGroups.id, id)).run();
   if (result.changes === 0) throw new NotFoundError(`service group ${id}`);
 }
+function insertServiceIpIfMissing(db, serviceId, ip) {
+  const existing = db.select({ ip: serviceIps.ip }).from(serviceIps).where(and2(eq3(serviceIps.service_id, serviceId), eq3(serviceIps.ip, ip))).get();
+  if (!existing) {
+    db.insert(serviceIps).values({ service_id: serviceId, ip }).run();
+  }
+}
 function listServiceIps(db, serviceId) {
   return db.select({ ip: serviceIps.ip }).from(serviceIps).where(eq3(serviceIps.service_id, serviceId)).all().map((r) => r.ip);
 }
@@ -1080,7 +1164,207 @@ function replaceServiceIps(db, serviceId, ips) {
   db.delete(serviceIps).where(eq3(serviceIps.service_id, serviceId)).run();
   for (const ip of ips) {
     db.insert(serviceIps).values({ service_id: serviceId, ip }).run();
+    ensureNode(db, serviceId, ip);
   }
+  const keep = new Set(ips);
+  for (const node of listNodes(db, serviceId)) {
+    if (keep.has(node.address)) continue;
+    const bound = db.select({ node_id: bindingNodes.node_id }).from(bindingNodes).where(eq3(bindingNodes.node_id, node.id)).get();
+    if (!bound) deleteNode(db, node.id);
+  }
+}
+function mapNode(row) {
+  return {
+    id: row.id,
+    service_id: row.service_id,
+    address: row.address,
+    protocol: row.protocol,
+    port: row.port,
+    enabled: Boolean(row.enabled),
+    priority: row.priority,
+    weight: row.weight,
+    health_status: row.health_status,
+    health_check_id: row.health_check_id,
+    consecutive_failures: row.consecutive_failures,
+    consecutive_successes: row.consecutive_successes,
+    last_check_at: row.last_check_at,
+    last_failure_reason: row.last_failure_reason,
+    created_at: row.created_at,
+    updated_at: row.updated_at
+  };
+}
+function listNodes(db, serviceId) {
+  return db.select().from(nodes).where(eq3(nodes.service_id, serviceId)).all().map(mapNode);
+}
+function getNode(db, id) {
+  const row = db.select().from(nodes).where(eq3(nodes.id, id)).get();
+  if (!row) throw new NotFoundError(`node ${id}`);
+  return mapNode(row);
+}
+function findNodeByAddress(db, serviceId, address) {
+  const row = db.select().from(nodes).where(and2(eq3(nodes.service_id, serviceId), eq3(nodes.address, address))).get();
+  return row ? mapNode(row) : null;
+}
+function findNodeByIp(db, address) {
+  const row = db.select().from(nodes).where(eq3(nodes.address, address)).get();
+  return row ? mapNode(row) : null;
+}
+function ensureNode(db, serviceId, address, meta) {
+  const existing = findNodeByAddress(db, serviceId, address);
+  if (existing) return existing;
+  const id = db.insert(nodes).values({
+    service_id: serviceId,
+    address,
+    protocol: meta?.protocol ?? "tcp",
+    port: meta?.port ?? null,
+    weight: meta?.weight ?? 1,
+    priority: meta?.priority ?? 1
+  }).returning({ id: nodes.id }).get().id;
+  return getNode(db, id);
+}
+function createNode(db, serviceId, input) {
+  getService(db, serviceId);
+  const existing = findNodeByAddress(db, serviceId, input.address);
+  if (existing) {
+    throw new ConflictError(`node ${input.address} already exists`);
+  }
+  const id = db.insert(nodes).values({
+    service_id: serviceId,
+    address: input.address,
+    protocol: input.protocol ?? "tcp",
+    port: input.port ?? null,
+    enabled: input.enabled ?? true,
+    priority: input.priority ?? 1,
+    weight: input.weight ?? 1,
+    health_check_id: input.health_check_id ?? null
+  }).returning({ id: nodes.id }).get().id;
+  insertServiceIpIfMissing(db, serviceId, input.address);
+  return getNode(db, id);
+}
+function updateNode(db, id, patch) {
+  const current = getNode(db, id);
+  const update = { updated_at: sql2`datetime('now')` };
+  for (const [key, value] of Object.entries(patch)) {
+    if (value !== void 0) update[key] = value;
+  }
+  db.update(nodes).set(update).where(eq3(nodes.id, id)).run();
+  if (patch.address && patch.address !== current.address) {
+    db.delete(serviceIps).where(
+      and2(
+        eq3(serviceIps.service_id, current.service_id),
+        eq3(serviceIps.ip, current.address)
+      )
+    ).run();
+    insertServiceIpIfMissing(db, current.service_id, patch.address);
+  }
+  return getNode(db, id);
+}
+function deleteNode(db, id) {
+  const current = getNode(db, id);
+  db.delete(nodes).where(eq3(nodes.id, id)).run();
+  db.delete(serviceIps).where(
+    and2(
+      eq3(serviceIps.service_id, current.service_id),
+      eq3(serviceIps.ip, current.address)
+    )
+  ).run();
+}
+function mapHealthCheck(row) {
+  return {
+    id: row.id,
+    provider: row.provider,
+    cf_healthcheck_id: row.cf_healthcheck_id,
+    cf_zone_id: row.cf_zone_id,
+    name: row.name,
+    protocol: row.protocol,
+    path: row.path,
+    method: row.method,
+    timeout: row.timeout,
+    interval_sec: row.interval_sec,
+    retries: row.retries,
+    expected_status: row.expected_status,
+    consecutive_fails: row.consecutive_fails,
+    consecutive_successes: row.consecutive_successes,
+    suspended: Boolean(row.suspended),
+    created_at: row.created_at,
+    updated_at: row.updated_at
+  };
+}
+function listHealthChecks(db) {
+  return db.select().from(healthChecks).all().map(mapHealthCheck);
+}
+function getHealthCheck(db, id) {
+  const row = db.select().from(healthChecks).where(eq3(healthChecks.id, id)).get();
+  if (!row) throw new NotFoundError(`health check ${id}`);
+  return mapHealthCheck(row);
+}
+function findHealthCheckByCfId(db, cfId) {
+  const row = db.select().from(healthChecks).where(eq3(healthChecks.cf_healthcheck_id, cfId)).get();
+  return row ? mapHealthCheck(row) : null;
+}
+function createHealthCheck(db, input) {
+  const id = db.insert(healthChecks).values({
+    provider: input.provider,
+    name: input.name,
+    cf_healthcheck_id: input.cf_healthcheck_id ?? null,
+    cf_zone_id: input.cf_zone_id ?? null,
+    protocol: input.protocol ?? "tcp",
+    path: input.path ?? null,
+    method: input.method ?? null,
+    timeout: input.timeout ?? 5,
+    interval_sec: input.interval_sec ?? 30,
+    retries: input.retries ?? 2,
+    expected_status: input.expected_status ?? null,
+    consecutive_fails: input.consecutive_fails ?? 2,
+    consecutive_successes: input.consecutive_successes ?? 2,
+    suspended: input.suspended ?? false
+  }).returning({ id: healthChecks.id }).get().id;
+  return getHealthCheck(db, id);
+}
+function updateHealthCheck(db, id, patch) {
+  getHealthCheck(db, id);
+  const update = { updated_at: sql2`datetime('now')` };
+  for (const [key, value] of Object.entries(patch)) {
+    if (value !== void 0) update[key] = value;
+  }
+  db.update(healthChecks).set(update).where(eq3(healthChecks.id, id)).run();
+  return getHealthCheck(db, id);
+}
+function deleteHealthCheck(db, id) {
+  const result = db.delete(healthChecks).where(eq3(healthChecks.id, id)).run();
+  if (result.changes === 0) throw new NotFoundError(`health check ${id}`);
+}
+function bumpBindingVersion(db, bindingId, expected) {
+  const binding = getBinding(db, bindingId);
+  if (expected != null && binding.operation_version !== expected) {
+    throw new ConflictError(`binding ${bindingId} version conflict`);
+  }
+  const next = (binding.operation_version ?? 0) + 1;
+  db.update(serviceBindings).set({
+    operation_version: next,
+    updated_at: sql2`datetime('now')`
+  }).where(eq3(serviceBindings.id, bindingId)).run();
+  return next;
+}
+function setBindingRoutingStrategy(db, bindingId, strategy) {
+  db.update(serviceBindings).set({
+    routing_strategy: strategy,
+    lb_mode: strategy,
+    updated_at: sql2`datetime('now')`
+  }).where(eq3(serviceBindings.id, bindingId)).run();
+}
+function listAllNodes(db) {
+  return db.select().from(nodes).all().map(mapNode);
+}
+function updateBindingDomain(db, bindingId, domainId, hostname) {
+  db.update(serviceBindings).set({
+    domain_id: domainId,
+    hostname,
+    updated_at: sql2`datetime('now')`
+  }).where(eq3(serviceBindings.id, bindingId)).run();
+}
+function listBindingNodes(db, bindingId) {
+  return db.select({ node: nodes }).from(bindingNodes).innerJoin(nodes, eq3(bindingNodes.node_id, nodes.id)).where(eq3(bindingNodes.binding_id, bindingId)).all().map((row) => mapNode(row.node));
 }
 function listBindingIps(db, bindingId) {
   return db.select({ ip: serviceBindingIps.ip }).from(serviceBindingIps).where(eq3(serviceBindingIps.binding_id, bindingId)).all().map((r) => r.ip);
@@ -1100,7 +1384,9 @@ function replaceBindingIps(db, bindingId, ips) {
   );
 }
 function replaceBindingIpsWithMeta(db, bindingId, entries) {
+  const binding = getBinding(db, bindingId);
   db.delete(serviceBindingIps).where(eq3(serviceBindingIps.binding_id, bindingId)).run();
+  db.delete(bindingNodes).where(eq3(bindingNodes.binding_id, bindingId)).run();
   for (const entry of entries) {
     db.insert(serviceBindingIps).values({
       binding_id: bindingId,
@@ -1108,13 +1394,26 @@ function replaceBindingIpsWithMeta(db, bindingId, entries) {
       weight: entry.weight,
       priority: entry.priority
     }).run();
+    const node = ensureNode(db, binding.service_id, entry.ip, {
+      weight: entry.weight,
+      priority: entry.priority
+    });
+    db.insert(bindingNodes).values({
+      binding_id: bindingId,
+      node_id: node.id,
+      weight: entry.weight,
+      priority: entry.priority
+    }).run();
   }
 }
 function updateBindingLbConfig(db, bindingId, patch) {
   const update = {
     updated_at: sql2`datetime('now')`
   };
-  if (patch.lb_mode !== void 0) update.lb_mode = patch.lb_mode;
+  if (patch.lb_mode !== void 0) {
+    update.lb_mode = patch.lb_mode;
+    update.routing_strategy = patch.lb_mode;
+  }
   if (patch.health_check_enabled !== void 0)
     update.health_check_enabled = patch.health_check_enabled;
   if (patch.health_check_type !== void 0)
@@ -1552,24 +1851,25 @@ function mergeHealthAggregates(parts) {
 function getIpHealthStatusRow(db, scope, refId, ip) {
   const rows = db.all(sql2`
     SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures,
-      last_checked_at, last_error
+      consecutive_successes, last_checked_at, last_error
     FROM ip_health_status
     WHERE scope = ${scope} AND ref_id = ${refId} AND ip = ${ip}
     LIMIT 1
   `);
   return rows[0] ?? null;
 }
-function upsertIpHealthStatus(db, scope, refId, ip, status, latencyMs, consecutiveFailures, lastError) {
+function upsertIpHealthStatus(db, scope, refId, ip, status, latencyMs, consecutiveFailures, lastError, consecutiveSuccesses = 0) {
   db.run(sql2`
     INSERT INTO ip_health_status
       (scope, ref_id, ip, status, latency_ms, consecutive_failures,
-       last_checked_at, last_error, created_at, updated_at)
+       consecutive_successes, last_checked_at, last_error, created_at, updated_at)
     VALUES (${scope}, ${refId}, ${ip}, ${status}, ${latencyMs}, ${consecutiveFailures},
-      datetime('now'), ${lastError}, datetime('now'), datetime('now'))
+      ${consecutiveSuccesses}, datetime('now'), ${lastError}, datetime('now'), datetime('now'))
     ON CONFLICT(scope, ref_id, ip) DO UPDATE SET
       status = excluded.status,
       latency_ms = excluded.latency_ms,
       consecutive_failures = excluded.consecutive_failures,
+      consecutive_successes = excluded.consecutive_successes,
       last_checked_at = excluded.last_checked_at,
       last_error = excluded.last_error,
       updated_at = datetime('now')
@@ -1844,6 +2144,7 @@ export {
   appSettings,
   appendAudit,
   auditLog,
+  bindingNodes,
   certificates,
   createDb,
   createMemoryDb,
@@ -1856,8 +2157,10 @@ export {
   getAppSettingsSecrets,
   groups,
   healthCheck,
+  healthChecks,
   ipHealthStatus,
   listAudit,
+  nodes,
   notificationLog,
   repos_exports as repos,
   resolveDatabasePath,