feat(health-checks): enhance health check functionality and add new routes

- Introduced origin health check routes and integrated them into the application.
- Updated health check configuration to include success recovery thresholds.
- Expanded error handling with new error codes for health check failures.
- Added new service routes for managing health checks, including creation and listing.
- Improved health check service logic to track consecutive successes and failures.

This commit enhances the health check capabilities, providing better monitoring and management of service health.

apps/api/src/lib/cf-client.ts

@@ -1,152 +1,84 @@
 import type {
   CfDnsRecord,
+  CfHealthCheck,
   CfZone,
   CreateDnsRecordPayload,
+  PatchDnsRecordPayload,
 } from "@cfdm/shared";
-import { AppError } from "../errors.js";
-import { withRetry, parseRetryAfter } from "./cf-retry.js";
+import { createDnsAdapter } from "./cloudflare/dns-service.js";
+import { createHealthCheckAdapter, type CfHealthCheckPayload } from "./cloudflare/healthcheck-service.js";
+import { createZoneAdapter } from "./cloudflare/zone-service.js";
 
-const BASE_URL = "https://api.cloudflare.com/client/v4";
-
-interface CfResponse<T> {
-  success: boolean;
-  result?: T;
-  errors?: Array<{ code: number; message: string }>;
-}
+export type { CfHealthCheckPayload };
 
 export class CloudflareClient {
-  constructor(private readonly token: string) {}
+  private readonly zones;
+  private readonly dns;
+  private readonly healthchecks;
 
-  private async handleResponse<T>(
-    response: Response,
-    operation: string,
-  ): Promise<T> {
-    if (response.status === 429) {
-      const wait = parseRetryAfter(response.headers) ?? 5000;
-      throw AppError.cloudflare(`rate limited, retry after ${wait}ms`);
-    }
-
-    const body = (await response.json()) as CfResponse<T>;
-    if (!body.success) {
-      const msg =
-        body.errors?.map((e) => e.message).join("; ") ??
-        "unknown cloudflare error";
-      throw AppError.cloudflare(`${operation}: ${msg}`);
-    }
-    if (body.result === undefined) {
-      throw AppError.cloudflare(`${operation}: empty result`);
-    }
-    return body.result;
+  constructor(token: string) {
+    this.zones = createZoneAdapter(token);
+    this.dns = createDnsAdapter(token);
+    this.healthchecks = createHealthCheckAdapter(token);
   }
 
-  async listZones(): Promise<CfZone[]> {
-    return withRetry(async () => {
-      const all: CfZone[] = [];
-      let page = 1;
-      while (true) {
-        const url = new URL(`${BASE_URL}/zones`);
-        url.searchParams.set("per_page", "50");
-        url.searchParams.set("page", String(page));
-        const response = await fetch(url, {
-          headers: { Authorization: `Bearer ${this.token}` },
-          signal: AbortSignal.timeout(30_000),
-        });
-        if (response.status >= 500 || response.status === 429) {
-          throw AppError.cloudflare(String(response.status));
-        }
-        const batch = await this.handleResponse<CfZone[]>(
-          response,
-          "list_zones",
-        );
-        if (batch.length === 0) break;
-        all.push(...batch);
-        if (batch.length < 50) break;
-        page += 1;
-      }
-      return all;
-    });
+  listZones(): Promise<CfZone[]> {
+    return this.zones.listZones();
   }
 
-  async getZone(zoneId: string): Promise<CfZone> {
-    const response = await fetch(`${BASE_URL}/zones/${zoneId}`, {
-      headers: { Authorization: `Bearer ${this.token}` },
-      signal: AbortSignal.timeout(30_000),
-    });
-    return this.handleResponse(response, "get_zone");
+  getZone(zoneId: string): Promise<CfZone> {
+    return this.zones.getZone(zoneId);
   }
 
-  async listDnsRecords(zoneId: string): Promise<CfDnsRecord[]> {
-    return withRetry(async () => {
-      const all: CfDnsRecord[] = [];
-      let page = 1;
-      while (page <= 50) {
-        const url = new URL(`${BASE_URL}/zones/${zoneId}/dns_records`);
-        url.searchParams.set("per_page", "100");
-        url.searchParams.set("page", String(page));
-        const response = await fetch(url, {
-          headers: { Authorization: `Bearer ${this.token}` },
-          signal: AbortSignal.timeout(30_000),
-        });
-        if (response.status >= 500 || response.status === 429) {
-          throw AppError.cloudflare(String(response.status));
-        }
-        const batch = await this.handleResponse<CfDnsRecord[]>(
-          response,
-          "list_dns_records",
-        );
-        if (batch.length === 0) break;
-        all.push(...batch);
-        page += 1;
-      }
-      return all;
-    });
+  listDnsRecords(zoneId: string): Promise<CfDnsRecord[]> {
+    return this.dns.listDnsRecords(zoneId);
   }
 
-  async createDnsRecord(
-    zoneId: string,
-    payload: CreateDnsRecordPayload,
-  ): Promise<CfDnsRecord> {
-    const response = await fetch(`${BASE_URL}/zones/${zoneId}/dns_records`, {
-      method: "POST",
-      headers: {
-        Authorization: `Bearer ${this.token}`,
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify(payload),
-      signal: AbortSignal.timeout(30_000),
-    });
-    return this.handleResponse(response, "create_dns_record");
+  createDnsRecord(zoneId: string, payload: CreateDnsRecordPayload): Promise<CfDnsRecord> {
+    return this.dns.createDnsRecord(zoneId, payload);
   }
 
-  async updateDnsRecord(
+  updateDnsRecord(
     zoneId: string,
     recordId: string,
     payload: CreateDnsRecordPayload,
   ): Promise<CfDnsRecord> {
-    const response = await fetch(
-      `${BASE_URL}/zones/${zoneId}/dns_records/${recordId}`,
-      {
-        method: "PUT",
-        headers: {
-          Authorization: `Bearer ${this.token}`,
-          "Content-Type": "application/json",
-        },
-        body: JSON.stringify(payload),
-        signal: AbortSignal.timeout(30_000),
-      },
-    );
-    return this.handleResponse(response, "update_dns_record");
+    return this.dns.updateDnsRecord(zoneId, recordId, payload);
   }
 
-  async deleteDnsRecord(zoneId: string, recordId: string): Promise<void> {
-    const response = await fetch(
-      `${BASE_URL}/zones/${zoneId}/dns_records/${recordId}`,
-      {
-        method: "DELETE",
-        headers: { Authorization: `Bearer ${this.token}` },
-        signal: AbortSignal.timeout(30_000),
-      },
-    );
-    await this.handleResponse(response, "delete_dns_record");
+  patchDnsRecord(
+    zoneId: string,
+    recordId: string,
+    payload: PatchDnsRecordPayload,
+  ): Promise<CfDnsRecord> {
+    return this.dns.patchDnsRecord(zoneId, recordId, payload);
+  }
+
+  deleteDnsRecord(zoneId: string, recordId: string): Promise<void> {
+    return this.dns.deleteDnsRecord(zoneId, recordId);
+  }
+
+  listHealthChecks(zoneId: string): Promise<CfHealthCheck[]> {
+    return this.healthchecks.listHealthChecks(zoneId);
+  }
+
+  getHealthCheck(zoneId: string, id: string): Promise<CfHealthCheck> {
+    return this.healthchecks.getHealthCheck(zoneId, id);
+  }
+
+  createHealthCheck(zoneId: string, payload: CfHealthCheckPayload): Promise<CfHealthCheck> {
+    return this.healthchecks.createHealthCheck(zoneId, payload);
+  }
+
+  updateHealthCheck(
+    zoneId: string,
+    id: string,
+    payload: CfHealthCheckPayload,
+  ): Promise<CfHealthCheck> {
+    return this.healthchecks.updateHealthCheck(zoneId, id, payload);
+  }
+
+  deleteHealthCheck(zoneId: string, id: string): Promise<void> {
+    return this.healthchecks.deleteHealthCheck(zoneId, id);
   }
 }

apps/api/src/lib/cloudflare/dns-service.ts

@@ -0,0 +1,106 @@
+import type { CfDnsRecord, CreateDnsRecordPayload, PatchDnsRecordPayload } from "@cfdm/shared";
+import { withRetry } from "../cf-retry.js";
+import { CF_API_BASE, handleCfResponse, mapCloudflareFailure } from "./http.js";
+
+export function createDnsAdapter(token: string) {
+  return {
+    async listDnsRecords(zoneId: string): Promise<CfDnsRecord[]> {
+      return withRetry(async () => {
+        const all: CfDnsRecord[] = [];
+        let page = 1;
+        while (page <= 50) {
+          const url = new URL(`${CF_API_BASE}/zones/${zoneId}/dns_records`);
+          url.searchParams.set("per_page", "100");
+          url.searchParams.set("page", String(page));
+          const response = await fetch(url, {
+            headers: { Authorization: `Bearer ${token}` },
+            signal: AbortSignal.timeout(30_000),
+          });
+          if (response.status >= 500 || response.status === 429) {
+            throw mapCloudflareFailure(
+              "list_dns_records",
+              response.status,
+              String(response.status),
+            );
+          }
+          const batch = await handleCfResponse<CfDnsRecord[]>(
+            response,
+            "list_dns_records",
+          );
+          if (batch.length === 0) break;
+          all.push(...batch);
+          page += 1;
+        }
+        return all;
+      });
+    },
+
+    async createDnsRecord(
+      zoneId: string,
+      payload: CreateDnsRecordPayload,
+    ): Promise<CfDnsRecord> {
+      const response = await fetch(`${CF_API_BASE}/zones/${zoneId}/dns_records`, {
+        method: "POST",
+        headers: {
+          Authorization: `Bearer ${token}`,
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify(payload),
+        signal: AbortSignal.timeout(30_000),
+      });
+      return handleCfResponse(response, "create_dns_record");
+    },
+
+    async updateDnsRecord(
+      zoneId: string,
+      recordId: string,
+      payload: CreateDnsRecordPayload,
+    ): Promise<CfDnsRecord> {
+      const response = await fetch(
+        `${CF_API_BASE}/zones/${zoneId}/dns_records/${recordId}`,
+        {
+          method: "PUT",
+          headers: {
+            Authorization: `Bearer ${token}`,
+            "Content-Type": "application/json",
+          },
+          body: JSON.stringify(payload),
+          signal: AbortSignal.timeout(30_000),
+        },
+      );
+      return handleCfResponse(response, "update_dns_record");
+    },
+
+    async patchDnsRecord(
+      zoneId: string,
+      recordId: string,
+      payload: PatchDnsRecordPayload,
+    ): Promise<CfDnsRecord> {
+      const response = await fetch(
+        `${CF_API_BASE}/zones/${zoneId}/dns_records/${recordId}`,
+        {
+          method: "PATCH",
+          headers: {
+            Authorization: `Bearer ${token}`,
+            "Content-Type": "application/json",
+          },
+          body: JSON.stringify(payload),
+          signal: AbortSignal.timeout(30_000),
+        },
+      );
+      return handleCfResponse(response, "patch_dns_record");
+    },
+
+    async deleteDnsRecord(zoneId: string, recordId: string): Promise<void> {
+      const response = await fetch(
+        `${CF_API_BASE}/zones/${zoneId}/dns_records/${recordId}`,
+        {
+          method: "DELETE",
+          headers: { Authorization: `Bearer ${token}` },
+          signal: AbortSignal.timeout(30_000),
+        },
+      );
+      await handleCfResponse(response, "delete_dns_record");
+    },
+  };
+}

apps/api/src/lib/cloudflare/healthcheck-service.ts

@@ -0,0 +1,110 @@
+import type { CfHealthCheck } from "@cfdm/shared";
+import { CF_API_BASE, handleCfResponse } from "./http.js";
+
+export interface CfHealthCheckHttpConfig {
+  method?: string;
+  path?: string;
+  expected_codes?: string[];
+  header?: Record<string, string[]>;
+  port?: number;
+  follow_redirects?: boolean;
+  allow_insecure?: boolean;
+}
+
+export interface CfHealthCheckTcpConfig {
+  method?: "connection_established";
+  port?: number;
+}
+
+export interface CfHealthCheckPayload {
+  address: string;
+  name: string;
+  type?: "HTTP" | "HTTPS" | "TCP";
+  description?: string;
+  interval?: number;
+  timeout?: number;
+  retries?: number;
+  consecutive_fails?: number;
+  consecutive_successes?: number;
+  suspended?: boolean;
+  check_regions?: string[];
+  http_config?: CfHealthCheckHttpConfig;
+  tcp_config?: CfHealthCheckTcpConfig;
+}
+
+export function createHealthCheckAdapter(token: string) {
+  return {
+    async listHealthChecks(zoneId: string): Promise<CfHealthCheck[]> {
+      const response = await fetch(
+        `${CF_API_BASE}/zones/${zoneId}/healthchecks`,
+        {
+          headers: { Authorization: `Bearer ${token}` },
+          signal: AbortSignal.timeout(30_000),
+        },
+      );
+      return handleCfResponse(response, "list_healthchecks");
+    },
+
+    async getHealthCheck(zoneId: string, id: string): Promise<CfHealthCheck> {
+      const response = await fetch(
+        `${CF_API_BASE}/zones/${zoneId}/healthchecks/${id}`,
+        {
+          headers: { Authorization: `Bearer ${token}` },
+          signal: AbortSignal.timeout(30_000),
+        },
+      );
+      return handleCfResponse(response, "get_healthcheck");
+    },
+
+    async createHealthCheck(
+      zoneId: string,
+      payload: CfHealthCheckPayload,
+    ): Promise<CfHealthCheck> {
+      const response = await fetch(
+        `${CF_API_BASE}/zones/${zoneId}/healthchecks`,
+        {
+          method: "POST",
+          headers: {
+            Authorization: `Bearer ${token}`,
+            "Content-Type": "application/json",
+          },
+          body: JSON.stringify(payload),
+          signal: AbortSignal.timeout(30_000),
+        },
+      );
+      return handleCfResponse(response, "create_healthcheck");
+    },
+
+    async updateHealthCheck(
+      zoneId: string,
+      id: string,
+      payload: CfHealthCheckPayload,
+    ): Promise<CfHealthCheck> {
+      const response = await fetch(
+        `${CF_API_BASE}/zones/${zoneId}/healthchecks/${id}`,
+        {
+          method: "PUT",
+          headers: {
+            Authorization: `Bearer ${token}`,
+            "Content-Type": "application/json",
+          },
+          body: JSON.stringify(payload),
+          signal: AbortSignal.timeout(30_000),
+        },
+      );
+      return handleCfResponse(response, "update_healthcheck");
+    },
+
+    async deleteHealthCheck(zoneId: string, id: string): Promise<void> {
+      const response = await fetch(
+        `${CF_API_BASE}/zones/${zoneId}/healthchecks/${id}`,
+        {
+          method: "DELETE",
+          headers: { Authorization: `Bearer ${token}` },
+          signal: AbortSignal.timeout(30_000),
+        },
+      );
+      await handleCfResponse(response, "delete_healthcheck");
+    },
+  };
+}

apps/api/src/lib/cloudflare/http.ts

@@ -0,0 +1,91 @@
+import type { CfDnsRecord } from "@cfdm/shared";
+import { AppError } from "../../errors.js";
+import { parseRetryAfter } from "../cf-retry.js";
+
+export const CF_API_BASE = "https://api.cloudflare.com/client/v4";
+
+export interface CfResponse<T> {
+  success: boolean;
+  result?: T;
+  errors?: Array<{ code: number; message: string }>;
+}
+
+export function mapCloudflareFailure(
+  operation: string,
+  status: number,
+  message: string,
+): AppError {
+  const lower = message.toLowerCase();
+  if (status === 401 || status === 403 || lower.includes("authentication")) {
+    return AppError.cloudflareAuthFailed(
+      "Cloudflare отклонил токен. Проверьте CLOUDFLARE_API_TOKEN.",
+    );
+  }
+  if (status === 429 || lower.includes("rate limit")) {
+    return AppError.rateLimited();
+  }
+  if (lower.includes("zone") && (lower.includes("not found") || status === 404)) {
+    return AppError.zoneNotFound();
+  }
+  if (
+    operation.includes("healthcheck") &&
+    (lower.includes("plan") ||
+      lower.includes("not entitled") ||
+      lower.includes("not allowed") ||
+      lower.includes("permission"))
+  ) {
+    return AppError.healthcheckCreateFailed(
+      "Cloudflare Health Checks недоступны для этой зоны. Используйте локальные проверки.",
+    );
+  }
+  if (operation.includes("dns") || operation.includes("dns_record")) {
+    return AppError.dnsUpdateFailed(`Не удалось обновить DNS в Cloudflare: ${message}`);
+  }
+  return AppError.cloudflare(`${operation}: ${message}`);
+}
+
+export async function handleCfResponse<T>(
+  response: Response,
+  operation: string,
+): Promise<T> {
+  if (response.status === 429) {
+    const wait = parseRetryAfter(response.headers) ?? 5000;
+    throw AppError.rateLimited(
+      `Cloudflare временно ограничил запросы. Повторите через ${Math.ceil(wait / 1000)} с.`,
+    );
+  }
+
+  const body = (await response.json()) as CfResponse<T>;
+  if (!body.success) {
+    const msg =
+      body.errors?.map((e) => e.message).join("; ") ?? "unknown cloudflare error";
+    throw mapCloudflareFailure(operation, response.status, msg);
+  }
+  if (body.result === undefined) {
+    throw mapCloudflareFailure(operation, response.status, "empty result");
+  }
+  return body.result;
+}
+
+export async function cfRequest<T>(
+  token: string,
+  path: string,
+  operation: string,
+  init: RequestInit = {},
+): Promise<T> {
+  const response = await fetch(`${CF_API_BASE}${path}`, {
+    ...init,
+    headers: {
+      Authorization: `Bearer ${token}`,
+      ...(init.body ? { "Content-Type": "application/json" } : {}),
+      ...init.headers,
+    },
+    signal: init.signal ?? AbortSignal.timeout(30_000),
+  });
+  if (response.status >= 500 || response.status === 429) {
+    throw mapCloudflareFailure(operation, response.status, String(response.status));
+  }
+  return handleCfResponse<T>(response, operation);
+}
+
+export type DnsRecordResult = CfDnsRecord;

apps/api/src/lib/cloudflare/zone-service.ts

@@ -0,0 +1,40 @@
+import type { CfZone } from "@cfdm/shared";
+import { withRetry } from "../cf-retry.js";
+import { CF_API_BASE, handleCfResponse, mapCloudflareFailure } from "./http.js";
+
+export function createZoneAdapter(token: string) {
+  return {
+    async listZones(): Promise<CfZone[]> {
+      return withRetry(async () => {
+        const all: CfZone[] = [];
+        let page = 1;
+        while (true) {
+          const url = new URL(`${CF_API_BASE}/zones`);
+          url.searchParams.set("per_page", "50");
+          url.searchParams.set("page", String(page));
+          const response = await fetch(url, {
+            headers: { Authorization: `Bearer ${token}` },
+            signal: AbortSignal.timeout(30_000),
+          });
+          if (response.status >= 500 || response.status === 429) {
+            throw mapCloudflareFailure("list_zones", response.status, String(response.status));
+          }
+          const batch = await handleCfResponse<CfZone[]>(response, "list_zones");
+          if (batch.length === 0) break;
+          all.push(...batch);
+          if (batch.length < 50) break;
+          page += 1;
+        }
+        return all;
+      });
+    },
+
+    async getZone(zoneId: string): Promise<CfZone> {
+      const response = await fetch(`${CF_API_BASE}/zones/${zoneId}`, {
+        headers: { Authorization: `Bearer ${token}` },
+        signal: AbortSignal.timeout(30_000),
+      });
+      return handleCfResponse(response, "get_zone");
+    },
+  };
+}

apps/api/src/lib/permissions.ts

@@ -94,14 +94,17 @@ const RULES: Rule[] = [
     methods: ["GET"],
     match: (p) =>
       p.startsWith("/api/v1/services") ||
-      p.startsWith("/api/v1/service-bindings"),
+      p.startsWith("/api/v1/service-bindings") ||
+      p.startsWith("/api/v1/health-checks") ||
+      p === "/api/v1/ops-summary",
     permission: "cfdm:services:read",
   },
   {
     methods: ["POST", "PUT", "PATCH", "DELETE"],
     match: (p) =>
       p.startsWith("/api/v1/services") ||
-      p.startsWith("/api/v1/service-bindings"),
+      p.startsWith("/api/v1/service-bindings") ||
+      p.startsWith("/api/v1/health-checks"),
     permission: "cfdm:services:write",
   },
   {
@@ -114,8 +117,7 @@ const RULES: Rule[] = [
     match: (p) =>
       p.startsWith("/api/v1/settings") ||
       p.startsWith("/api/v1/notifications") ||
-      p.startsWith("/api/v1/health-check") ||
-      p.startsWith("/api/v1/health-checks"),
+      p.startsWith("/api/v1/health-check"),
     permission: "cfdm:settings:admin",
   },
   {