feat: Implement load balancing and health check features for service groups, including DNS-based load balancing modes and health check configurations, enhancing service reliability and performance monitoring

packages/db/dist/index.js

@@ -30,6 +30,8 @@ var services = sqliteTable("services", {
   subdomain: text("subdomain"),
   enabled: integer("enabled", { mode: "boolean" }).notNull().default(false),
   sort_order: integer("sort_order").notNull().default(0),
+  lb_weight: integer("lb_weight").notNull().default(1),
+  lb_priority: integer("lb_priority").notNull().default(1),
   created_at: text("created_at").notNull().default(sql`datetime('now')`),
   updated_at: text("updated_at").notNull().default(sql`datetime('now')`)
 });
@@ -40,6 +42,14 @@ var serviceGroups = sqliteTable("service_groups", {
   icon: text("icon"),
   domain: text("domain"),
   enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
+  lb_mode: text("lb_mode").notNull().default("round_robin"),
+  health_check_enabled: integer("health_check_enabled", { mode: "boolean" }).notNull().default(false),
+  health_check_type: text("health_check_type").notNull().default("tcp"),
+  health_check_port: integer("health_check_port"),
+  health_check_path: text("health_check_path"),
+  health_check_expected_status: integer("health_check_expected_status"),
+  health_check_interval_sec: integer("health_check_interval_sec").notNull().default(30),
+  health_check_timeout_ms: integer("health_check_timeout_ms").notNull().default(3e3),
   created_at: text("created_at").notNull().default(sql`datetime('now')`),
   updated_at: text("updated_at").notNull().default(sql`datetime('now')`)
 });
@@ -91,6 +101,14 @@ var serviceBindings = sqliteTable("service_bindings", {
   dns_record_id: integer("dns_record_id").references(() => dnsRecords.id, {
     onDelete: "set null"
   }),
+  lb_mode: text("lb_mode").notNull().default("round_robin"),
+  health_check_enabled: integer("health_check_enabled", { mode: "boolean" }).notNull().default(false),
+  health_check_type: text("health_check_type").notNull().default("tcp"),
+  health_check_port: integer("health_check_port"),
+  health_check_path: text("health_check_path"),
+  health_check_expected_status: integer("health_check_expected_status"),
+  health_check_interval_sec: integer("health_check_interval_sec").notNull().default(30),
+  health_check_timeout_ms: integer("health_check_timeout_ms").notNull().default(3e3),
   created_at: text("created_at").notNull().default(sql`datetime('now')`),
   updated_at: text("updated_at").notNull().default(sql`datetime('now')`)
 });
@@ -112,7 +130,9 @@ var serviceBindingIps = sqliteTable(
   "service_binding_ips",
   {
     binding_id: integer("binding_id").notNull().references(() => serviceBindings.id, { onDelete: "cascade" }),
-    ip: text("ip").notNull()
+    ip: text("ip").notNull(),
+    weight: integer("weight").notNull().default(1),
+    priority: integer("priority").notNull().default(1)
   },
   (t) => [primaryKey({ columns: [t.binding_id, t.ip] })]
 );
@@ -148,6 +168,22 @@ var syncJobs = sqliteTable("sync_jobs", {
   created_at: text("created_at").notNull().default(sql`datetime('now')`),
   finished_at: text("finished_at")
 });
+var ipHealthStatus = sqliteTable(
+  "ip_health_status",
+  {
+    scope: text("scope").notNull(),
+    ref_id: integer("ref_id").notNull(),
+    ip: text("ip").notNull(),
+    status: text("status").notNull().default("unknown"),
+    latency_ms: integer("latency_ms"),
+    consecutive_failures: integer("consecutive_failures").notNull().default(0),
+    last_checked_at: text("last_checked_at"),
+    last_error: text("last_error"),
+    created_at: text("created_at").notNull().default(sql`datetime('now')`),
+    updated_at: text("updated_at").notNull().default(sql`datetime('now')`)
+  },
+  (t) => [primaryKey({ columns: [t.scope, t.ref_id, t.ip] })]
+);
 var schema = {
   groups,
   services,
@@ -161,7 +197,8 @@ var schema = {
   serviceBindingIps,
   serviceGroupDnsRecords,
   certificates,
-  syncJobs
+  syncJobs,
+  ipHealthStatus
 };
 
 // src/client.ts
@@ -243,6 +280,8 @@ __export(repos_exports, {
   deleteDnsRecord: () => deleteDnsRecord,
   deleteDomain: () => deleteDomain,
   deleteGroup: () => deleteGroup,
+  deleteIpHealthStatusForIp: () => deleteIpHealthStatusForIp,
+  deleteIpHealthStatusForRef: () => deleteIpHealthStatusForRef,
   deleteService: () => deleteService,
   deleteServiceGroup: () => deleteServiceGroup,
   deleteSubdomain: () => deleteSubdomain,
@@ -258,6 +297,7 @@ __export(repos_exports, {
   getDomain: () => getDomain,
   getGroup: () => getGroup,
   getGroupWithStats: () => getGroupWithStats,
+  getIpHealthStatusRow: () => getIpHealthStatusRow,
   getService: () => getService,
   getServiceGroup: () => getServiceGroup,
   getSubdomain: () => getSubdomain,
@@ -270,6 +310,7 @@ __export(repos_exports, {
   listAllDomains: () => listAllDomains,
   listAllSubdomains: () => listAllSubdomains,
   listBindingIps: () => listBindingIps,
+  listBindingIpsWithMeta: () => listBindingIpsWithMeta,
   listBindingsByDomain: () => listBindingsByDomain,
   listBindingsByService: () => listBindingsByService,
   listCertificates: () => listCertificates,
@@ -279,6 +320,8 @@ __export(repos_exports, {
   listDomainsEnriched: () => listDomainsEnriched,
   listGroupDnsRecords: () => listGroupDnsRecords,
   listGroups: () => listGroups,
+  listHealthCheckTargets: () => listHealthCheckTargets,
+  listIpHealthStatus: () => listIpHealthStatus,
   listRecordsForBinding: () => listRecordsForBinding,
   listServiceGroups: () => listServiceGroups,
   listServiceIps: () => listServiceIps,
@@ -289,6 +332,7 @@ __export(repos_exports, {
   markDnsPendingDelete: () => markDnsPendingDelete,
   reorderServices: () => reorderServices,
   replaceBindingIps: () => replaceBindingIps,
+  replaceBindingIpsWithMeta: () => replaceBindingIpsWithMeta,
   replaceServiceIps: () => replaceServiceIps,
   setBindingCnameTarget: () => setBindingCnameTarget,
   setBindingDnsRecordId: () => setBindingDnsRecordId,
@@ -297,9 +341,11 @@ __export(repos_exports, {
   setServiceEnabled: () => setServiceEnabled,
   setServiceGroup: () => setServiceGroup,
   setServiceGroupEnabled: () => setServiceGroupEnabled,
+  setServiceLb: () => setServiceLb,
   unlinkBindingRecord: () => unlinkBindingRecord,
   unlinkGroupDnsRecord: () => unlinkGroupDnsRecord,
   updateBindingFields: () => updateBindingFields,
+  updateBindingLbConfig: () => updateBindingLbConfig,
   updateDnsFields: () => updateDnsFields,
   updateDomain: () => updateDomain,
   updateGroup: () => updateGroup,
@@ -307,6 +353,7 @@ __export(repos_exports, {
   updateServiceGroup: () => updateServiceGroup,
   updateSubdomain: () => updateSubdomain,
   upsertCertificateCheck: () => upsertCertificateCheck,
+  upsertIpHealthStatus: () => upsertIpHealthStatus,
   upsertSubdomain: () => upsertSubdomain
 });
 import { dnsRecordNamesMatch } from "@cfdm/shared";
@@ -580,6 +627,13 @@ function setServiceEnabled(db, id, enabled) {
   db.update(services).set({ enabled, updated_at: sql2`datetime('now')` }).where(eq(services.id, id)).run();
   return getService(db, id);
 }
+function setServiceLb(db, id, weight, priority) {
+  db.update(services).set({
+    lb_weight: weight,
+    lb_priority: priority,
+    updated_at: sql2`datetime('now')`
+  }).where(eq(services.id, id)).run();
+}
 function setServiceGroup(db, id, groupId) {
   const sortOrder = maxSortOrderInGroup(db, groupId) + 1;
   db.update(services).set({
@@ -619,6 +673,14 @@ function mapServiceGroup(row) {
     icon: row.icon,
     domain: row.domain,
     enabled: row.enabled,
+    lb_mode: row.lb_mode,
+    health_check_enabled: row.health_check_enabled,
+    health_check_type: row.health_check_type,
+    health_check_port: row.health_check_port,
+    health_check_path: row.health_check_path,
+    health_check_expected_status: row.health_check_expected_status,
+    health_check_interval_sec: row.health_check_interval_sec,
+    health_check_timeout_ms: row.health_check_timeout_ms,
     created_at: row.created_at,
     updated_at: row.updated_at
   };
@@ -631,18 +693,49 @@ function getServiceGroup(db, id) {
   if (!row) throw new NotFoundError(`service group ${id}`);
   return mapServiceGroup(row);
 }
-function createServiceGroup(db, name, groupType, icon, domain) {
-  const id = db.insert(serviceGroups).values({ name, type: groupType, icon, domain }).returning({ id: serviceGroups.id }).get().id;
+function createServiceGroup(db, name, groupType, icon, domain, lbPatch) {
+  const id = db.insert(serviceGroups).values({
+    name,
+    type: groupType,
+    icon,
+    domain,
+    lb_mode: lbPatch?.lb_mode ?? "round_robin",
+    health_check_enabled: lbPatch?.health_check_enabled ?? false,
+    health_check_type: lbPatch?.health_check_type ?? "tcp",
+    health_check_port: lbPatch?.health_check_port ?? null,
+    health_check_path: lbPatch?.health_check_path ?? null,
+    health_check_expected_status: lbPatch?.health_check_expected_status ?? null,
+    health_check_interval_sec: lbPatch?.health_check_interval_sec ?? 30,
+    health_check_timeout_ms: lbPatch?.health_check_timeout_ms ?? 3e3
+  }).returning({ id: serviceGroups.id }).get().id;
   return getServiceGroup(db, id);
 }
-function updateServiceGroup(db, id, name, groupType, icon, domain) {
-  const result = db.update(serviceGroups).set({
+function updateServiceGroup(db, id, name, groupType, icon, domain, lbPatch) {
+  const update = {
     name,
     type: groupType,
     icon,
     domain,
     updated_at: sql2`datetime('now')`
-  }).where(eq(serviceGroups.id, id)).run();
+  };
+  if (lbPatch) {
+    if (lbPatch.lb_mode !== void 0) update.lb_mode = lbPatch.lb_mode;
+    if (lbPatch.health_check_enabled !== void 0)
+      update.health_check_enabled = lbPatch.health_check_enabled;
+    if (lbPatch.health_check_type !== void 0)
+      update.health_check_type = lbPatch.health_check_type;
+    if (lbPatch.health_check_port !== void 0)
+      update.health_check_port = lbPatch.health_check_port;
+    if (lbPatch.health_check_path !== void 0)
+      update.health_check_path = lbPatch.health_check_path;
+    if (lbPatch.health_check_expected_status !== void 0)
+      update.health_check_expected_status = lbPatch.health_check_expected_status;
+    if (lbPatch.health_check_interval_sec !== void 0)
+      update.health_check_interval_sec = lbPatch.health_check_interval_sec;
+    if (lbPatch.health_check_timeout_ms !== void 0)
+      update.health_check_timeout_ms = lbPatch.health_check_timeout_ms;
+  }
+  const result = db.update(serviceGroups).set(update).where(eq(serviceGroups.id, id)).run();
   if (result.changes === 0) throw new NotFoundError(`service group ${id}`);
   return getServiceGroup(db, id);
 }
@@ -667,12 +760,52 @@ function replaceServiceIps(db, serviceId, ips) {
 function listBindingIps(db, bindingId) {
   return db.select({ ip: serviceBindingIps.ip }).from(serviceBindingIps).where(eq(serviceBindingIps.binding_id, bindingId)).all().map((r) => r.ip);
 }
+function listBindingIpsWithMeta(db, bindingId) {
+  return db.select({
+    ip: serviceBindingIps.ip,
+    weight: serviceBindingIps.weight,
+    priority: serviceBindingIps.priority
+  }).from(serviceBindingIps).where(eq(serviceBindingIps.binding_id, bindingId)).all();
+}
 function replaceBindingIps(db, bindingId, ips) {
+  replaceBindingIpsWithMeta(
+    db,
+    bindingId,
+    ips.map((ip) => ({ ip, weight: 1, priority: 1 }))
+  );
+}
+function replaceBindingIpsWithMeta(db, bindingId, entries) {
   db.delete(serviceBindingIps).where(eq(serviceBindingIps.binding_id, bindingId)).run();
-  for (const ip of ips) {
-    db.insert(serviceBindingIps).values({ binding_id: bindingId, ip }).run();
+  for (const entry of entries) {
+    db.insert(serviceBindingIps).values({
+      binding_id: bindingId,
+      ip: entry.ip,
+      weight: entry.weight,
+      priority: entry.priority
+    }).run();
   }
 }
+function updateBindingLbConfig(db, bindingId, patch) {
+  const update = {
+    updated_at: sql2`datetime('now')`
+  };
+  if (patch.lb_mode !== void 0) update.lb_mode = patch.lb_mode;
+  if (patch.health_check_enabled !== void 0)
+    update.health_check_enabled = patch.health_check_enabled;
+  if (patch.health_check_type !== void 0)
+    update.health_check_type = patch.health_check_type;
+  if (patch.health_check_port !== void 0)
+    update.health_check_port = patch.health_check_port;
+  if (patch.health_check_path !== void 0)
+    update.health_check_path = patch.health_check_path;
+  if (patch.health_check_expected_status !== void 0)
+    update.health_check_expected_status = patch.health_check_expected_status;
+  if (patch.health_check_interval_sec !== void 0)
+    update.health_check_interval_sec = patch.health_check_interval_sec;
+  if (patch.health_check_timeout_ms !== void 0)
+    update.health_check_timeout_ms = patch.health_check_timeout_ms;
+  db.update(serviceBindings).set(update).where(eq(serviceBindings.id, bindingId)).run();
+}
 function setBindingCnameTarget(db, bindingId, target) {
   db.update(serviceBindings).set({
     cname_target: target,
@@ -726,13 +859,22 @@ function unlinkGroupDnsRecord(db, groupId, dnsRecordId) {
 function dnsRecordMatchesHostname(recordName, hostname, zoneName) {
   return dnsRecordNamesMatch(recordName, hostname, zoneName);
 }
+var SERVICE_BINDING_SELECT_COLUMNS = `sb.id, sb.domain_id, sb.service_id, sb.hostname, sb.dns_record_id,
+  sb.lb_mode, sb.health_check_enabled, sb.health_check_type, sb.health_check_port,
+  sb.health_check_path, sb.health_check_expected_status, sb.health_check_interval_sec,
+  sb.health_check_timeout_ms, sb.cname_target,
+  d.zone_name, d.group_id, g.name AS group_name,
+  s.name AS service_name, s.slug AS service_slug,
+  dr.content AS target_ip, dr.sync_status,
+  sb.created_at, sb.updated_at`;
 function enrichServiceBindingView(db, row) {
-  const configured = listBindingIps(db, row.id);
+  const configured = listBindingIpsWithMeta(db, row.id);
+  const configuredIps = configured.map((c) => c.ip);
   const linkedRecords = listRecordsForBinding(db, row.id);
   const linkedIps = linkedRecords.filter((record) => record.record_type.toUpperCase() === "A").map((record) => record.content);
   const target_ips = [
     .../* @__PURE__ */ new Set([
-      ...configured,
+      ...configuredIps,
       ...linkedIps,
       ...row.target_ip ? [row.target_ip] : []
     ])
@@ -749,21 +891,29 @@ function enrichServiceBindingView(db, row) {
     }
     target_ips.sort();
   }
+  const target_ip_weights = {};
+  const target_ip_priorities = {};
+  for (const entry of configured) {
+    target_ip_weights[entry.ip] = entry.weight;
+    target_ip_priorities[entry.ip] = entry.priority;
+  }
+  for (const ip of target_ips) {
+    if (target_ip_weights[ip] === void 0) target_ip_weights[ip] = 1;
+    if (target_ip_priorities[ip] === void 0) target_ip_priorities[ip] = 1;
+  }
   const sync_status = row.sync_status ?? linkedRecords.find((record) => record.sync_status)?.sync_status ?? null;
   return {
     ...row,
     target_ips,
     target_ip: target_ips[0] ?? null,
+    target_ip_weights,
+    target_ip_priorities,
     sync_status
   };
 }
 function listAllBindings(db) {
   return db.all(sql2`
-    SELECT sb.id, sb.domain_id, sb.service_id, sb.hostname, sb.dns_record_id,
-      d.zone_name, d.group_id, g.name AS group_name,
-      s.name AS service_name, s.slug AS service_slug,
-      dr.content AS target_ip, dr.sync_status,
-      sb.created_at, sb.updated_at
+    SELECT ${sql2.raw(SERVICE_BINDING_SELECT_COLUMNS)}
     FROM service_bindings sb
     JOIN domains d ON d.id = sb.domain_id
     LEFT JOIN groups g ON g.id = d.group_id
@@ -774,11 +924,7 @@ function listAllBindings(db) {
 }
 function listBindingsByDomain(db, domainId) {
   return db.all(sql2`
-    SELECT sb.id, sb.domain_id, sb.service_id, sb.hostname, sb.dns_record_id,
-      d.zone_name, d.group_id, g.name AS group_name,
-      s.name AS service_name, s.slug AS service_slug,
-      dr.content AS target_ip, dr.sync_status,
-      sb.created_at, sb.updated_at
+    SELECT ${sql2.raw(SERVICE_BINDING_SELECT_COLUMNS)}
     FROM service_bindings sb
     JOIN domains d ON d.id = sb.domain_id
     LEFT JOIN groups g ON g.id = d.group_id
@@ -802,11 +948,7 @@ function getBinding(db, id) {
 }
 function getBindingView(db, id) {
   const rows = db.all(sql2`
-    SELECT sb.id, sb.domain_id, sb.service_id, sb.hostname, sb.dns_record_id,
-      d.zone_name, d.group_id, g.name AS group_name,
-      s.name AS service_name, s.slug AS service_slug,
-      dr.content AS target_ip, dr.sync_status,
-      sb.created_at, sb.updated_at
+    SELECT ${sql2.raw(SERVICE_BINDING_SELECT_COLUMNS)}
     FROM service_bindings sb
     JOIN domains d ON d.id = sb.domain_id
     LEFT JOIN groups g ON g.id = d.group_id
@@ -932,6 +1074,88 @@ function finishSyncJob(db, id, status, message) {
     finished_at: sql2`datetime('now')`
   }).where(eq(syncJobs.id, id)).run();
 }
+function listIpHealthStatus(db, scope, refId) {
+  return db.all(sql2`
+    SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures,
+      last_checked_at, last_error
+    FROM ip_health_status
+    WHERE scope = ${scope} AND ref_id = ${refId}
+  `);
+}
+function getIpHealthStatusRow(db, scope, refId, ip) {
+  const rows = db.all(sql2`
+    SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures,
+      last_checked_at, last_error
+    FROM ip_health_status
+    WHERE scope = ${scope} AND ref_id = ${refId} AND ip = ${ip}
+    LIMIT 1
+  `);
+  return rows[0] ?? null;
+}
+function upsertIpHealthStatus(db, scope, refId, ip, status, latencyMs, consecutiveFailures, lastError) {
+  db.run(sql2`
+    INSERT INTO ip_health_status
+      (scope, ref_id, ip, status, latency_ms, consecutive_failures,
+       last_checked_at, last_error, created_at, updated_at)
+    VALUES (${scope}, ${refId}, ${ip}, ${status}, ${latencyMs}, ${consecutiveFailures},
+      datetime('now'), ${lastError}, datetime('now'), datetime('now'))
+    ON CONFLICT(scope, ref_id, ip) DO UPDATE SET
+      status = excluded.status,
+      latency_ms = excluded.latency_ms,
+      consecutive_failures = excluded.consecutive_failures,
+      last_checked_at = excluded.last_checked_at,
+      last_error = excluded.last_error,
+      updated_at = datetime('now')
+  `);
+}
+function deleteIpHealthStatusForRef(db, scope, refId) {
+  db.delete(ipHealthStatus).where(
+    and(
+      eq(ipHealthStatus.scope, scope),
+      eq(ipHealthStatus.ref_id, refId)
+    )
+  ).run();
+}
+function deleteIpHealthStatusForIp(db, scope, refId, ip) {
+  db.delete(ipHealthStatus).where(
+    and(
+      eq(ipHealthStatus.scope, scope),
+      eq(ipHealthStatus.ref_id, refId),
+      eq(ipHealthStatus.ip, ip)
+    )
+  ).run();
+}
+function listHealthCheckTargets(db) {
+  const bindingTargets = db.all(sql2`
+    SELECT 'binding' AS scope, sb.id AS ref_id, sbi.ip,
+      sb.hostname AS hostname,
+      sb.health_check_type AS type,
+      sb.health_check_port AS port,
+      sb.health_check_path AS path,
+      sb.health_check_expected_status AS expected_status,
+      sb.health_check_timeout_ms AS timeout_ms
+    FROM service_binding_ips sbi
+    JOIN service_bindings sb ON sb.id = sbi.binding_id
+    WHERE sb.health_check_enabled = 1
+  `);
+  const groupTargets = db.all(sql2`
+    SELECT 'group' AS scope, sg.id AS ref_id, sip.ip,
+      sg.domain AS hostname,
+      sg.health_check_type AS type,
+      sg.health_check_port AS port,
+      sg.health_check_path AS path,
+      sg.health_check_expected_status AS expected_status,
+      sg.health_check_timeout_ms AS timeout_ms
+    FROM services s
+    JOIN service_ips sip ON sip.service_id = s.id
+    JOIN service_groups sg ON sg.id = s.service_group_id
+    WHERE sg.health_check_enabled = 1
+      AND sg.domain IS NOT NULL
+      AND s.enabled = 1
+      AND (sg.enabled = 1)
+  `);
+  return [...bindingTargets, ...groupTargets];
+}
 export {
   ConflictError,
   NotFoundError,
@@ -942,6 +1166,7 @@ export {
   domains,
   groups,
   healthCheck,
+  ipHealthStatus,
   repos_exports as repos,
   resolveDatabasePath,
   runMigrations,