From 2c92e78b241bf58aad7402fb175c6cbdf564588b Mon Sep 17 00:00:00 2001 From: Denozordec Date: Wed, 19 Aug 2026 16:27:34 +0700 Subject: [PATCH] feat(health-checks): enhance health check configuration and logging - Added new health worker URL and token fields to the AppConfig interface, allowing for Cloudflare Worker integration. - Updated health check routes to utilize the new worker configuration, enabling dynamic health checks via Cloudflare Workers. - Introduced a health log endpoint for services, providing detailed logs of health probe results. - Enhanced health check service logic to support both local and Cloudflare Worker providers, improving flexibility in health monitoring. - Updated UI components to reflect changes in health check provider settings and display relevant health information. This commit significantly improves the health check management capabilities, allowing for better integration with Cloudflare Workers and enhanced logging features. --- apps/api/src/config.ts | 4 + apps/api/src/routes/health-check.ts | 19 +- apps/api/src/routes/services.ts | 8 + .../src/services/health-check-scheduler.ts | 16 + apps/api/src/services/health-check-service.ts | 52 +- apps/api/src/services/health/worker.ts | 93 +++ .../src/services/service-config-service.ts | 14 +- apps/api/test/health-check.test.ts | 4 + apps/api/test/health-worker.test.ts | 215 ++++++ apps/api/test/service-groups-health.test.ts | 10 +- apps/api/test/settings-health.test.ts | 29 + .../web/src/components/health-check-badge.tsx | 7 + .../components/health-check-config-fields.tsx | 106 +-- .../src/components/health/health-timeline.tsx | 4 + .../web/src/components/service-edit-sheet.tsx | 4 +- .../components/services/service-fqdn-list.tsx | 4 + apps/web/src/lib/schemas.ts | 22 + apps/web/src/queries/services.ts | 21 + .../_auth/services/$serviceId/health.tsx | 199 ++--- apps/web/src/routes/_auth/settings/health.tsx | 77 +- docs/Home.md | 28 +- packages/db/dist/index.d.ts | 698 +++++++++++++++++- packages/db/dist/index.js | 136 +++- .../db/migrations/021_health_worker_xor.sql | 27 + packages/db/src/repos.ts | 153 +++- packages/db/src/schema.ts | 25 + packages/db/src/settings-repo.ts | 27 +- packages/shared/dist/index.d.ts | 139 +++- packages/shared/dist/index.js | 33 +- .../shared/src/integration-vps-tracker.ts | 2 + packages/shared/src/schemas.ts | 25 + packages/shared/src/types.ts | 11 + workers/health-probe/README.md | 37 + workers/health-probe/package.json | 11 + workers/health-probe/src/index.ts | 181 +++++ workers/health-probe/wrangler.toml | 6 + 36 files changed, 2184 insertions(+), 263 deletions(-) create mode 100644 apps/api/src/services/health/worker.ts create mode 100644 apps/api/test/health-worker.test.ts create mode 100644 packages/db/migrations/021_health_worker_xor.sql create mode 100644 workers/health-probe/README.md create mode 100644 workers/health-probe/package.json create mode 100644 workers/health-probe/src/index.ts create mode 100644 workers/health-probe/wrangler.toml diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts index 215590b..357e334 100644 --- a/apps/api/src/config.ts +++ b/apps/api/src/config.ts @@ -17,6 +17,8 @@ export interface AppConfig { healthLatencyWarnMs: number; /** Min pause between probes to different physical targets (same IP is probed once). */ healthProbeGapMs: number; + healthWorkerUrl: string; + healthWorkerToken: string; logLevel: string; /** Portal SSO — when true, require portal JWT with apps includes cfdm */ authRequired: boolean; @@ -61,6 +63,8 @@ export function loadConfig(): AppConfig { healthLatencyWarnMs: Number(process.env.HEALTH_LATENCY_WARN_MS ?? "1000") || 1000, healthProbeGapMs: Number(process.env.HEALTH_PROBE_GAP_MS ?? "2000") || 2000, + healthWorkerUrl: (process.env.HEALTH_WORKER_URL ?? "").trim(), + healthWorkerToken: (process.env.HEALTH_WORKER_TOKEN ?? "").trim(), logLevel: process.env.LOG_LEVEL ?? "info", authRequired: boolEnv(process.env.AUTH_REQUIRED, false), authIssuer: diff --git a/apps/api/src/routes/health-check.ts b/apps/api/src/routes/health-check.ts index 0270aeb..cd1a948 100644 --- a/apps/api/src/routes/health-check.ts +++ b/apps/api/src/routes/health-check.ts @@ -1,8 +1,12 @@ import type { FastifyInstance } from "fastify"; import { healthStatusQuerySchema } from "@cfdm/shared"; -import { repos } from "@cfdm/db"; +import { getAppSettings, repos } from "@cfdm/db"; import * as healthCheckService from "../services/health-check-service.js"; import * as serviceConfigService from "../services/service-config-service.js"; +import { + healthEngineFallbacksFromConfig, + resolveWorkerProbeConfig, +} from "../services/health-check-scheduler.js"; export async function healthCheckRoutes(app: FastifyInstance) { app.get("/health-status", async (request) => { @@ -16,15 +20,20 @@ export async function healthCheckRoutes(app: FastifyInstance) { app.post("/health-check/run", async (request) => { const config = request.server.config; + const settings = getAppSettings( + request.server.db, + healthEngineFallbacksFromConfig(config), + ); const thresholds = { - degradedFailures: config.healthDegradedFailures, - downFailures: config.healthDownFailures, - latencyWarnMs: config.healthLatencyWarnMs, - successRecoveries: config.healthSuccessRecoveries, + degradedFailures: settings.healthDegradedFailures, + downFailures: settings.healthDownFailures, + latencyWarnMs: settings.healthLatencyWarnMs, + successRecoveries: settings.healthSuccessRecoveries, }; const checked = await healthCheckService.runAllChecks(request.server.db, { thresholds, probeGapMs: config.healthProbeGapMs, + worker: resolveWorkerProbeConfig(request.server.db, config), onStatusChange: async (target, prev, next) => { try { const label = diff --git a/apps/api/src/routes/services.ts b/apps/api/src/routes/services.ts index 4caed84..e9b60bf 100644 --- a/apps/api/src/routes/services.ts +++ b/apps/api/src/routes/services.ts @@ -65,6 +65,14 @@ export async function serviceRoutes(app: FastifyInstance) { return serviceConfig.getView(request.server.db, Number(id)); }); + app.get("/services/:id/health-log", async (request) => { + const { id } = request.params as { id: string }; + repos.getService(request.server.db, Number(id)); + return { + items: repos.listHealthProbeLogForService(request.server.db, Number(id)), + }; + }); + app.get("/services/:id/overview", async (request) => { const { id } = request.params as { id: string }; return nodeService.getOverview(request.server.db, Number(id)); diff --git a/apps/api/src/services/health-check-scheduler.ts b/apps/api/src/services/health-check-scheduler.ts index de253ad..bfc01f4 100644 --- a/apps/api/src/services/health-check-scheduler.ts +++ b/apps/api/src/services/health-check-scheduler.ts @@ -2,6 +2,7 @@ import type { FastifyInstance } from "fastify"; import { AsyncTask, CronJob } from "toad-scheduler"; import { getAppSettings, + getAppSettingsSecrets, type HealthEngineFallbacks, } from "@cfdm/db"; import { repos } from "@cfdm/db"; @@ -27,9 +28,23 @@ export function healthEngineFallbacksFromConfig( healthDownFailures: config.healthDownFailures, healthLatencyWarnMs: config.healthLatencyWarnMs, healthSuccessRecoveries: config.healthSuccessRecoveries, + healthWorkerUrl: config.healthWorkerUrl, + healthWorkerTokenSet: Boolean(config.healthWorkerToken), }; } +export function resolveWorkerProbeConfig( + db: import("@cfdm/db").Db, + config: AppConfig, +): { url: string; token: string } | null { + const settings = getAppSettings(db, healthEngineFallbacksFromConfig(config)); + const secrets = getAppSettingsSecrets(db); + const url = settings.healthWorkerUrl.trim(); + const token = (secrets.healthWorkerToken || config.healthWorkerToken).trim(); + if (!url || !token) return null; + return { url, token }; +} + export function assertValidHealthCron(expr: string): void { const cronExpression = expr.trim(); const parts = cronExpression.split(/\s+/).filter(Boolean); @@ -66,6 +81,7 @@ export function createHealthCheckTask( const n = await healthCheckService.runAllChecks(app.db, { thresholds, probeGapMs: config.healthProbeGapMs, + worker: resolveWorkerProbeConfig(app.db, config), onStatusChange: async (target, prev, next) => { try { const label = diff --git a/apps/api/src/services/health-check-service.ts b/apps/api/src/services/health-check-service.ts index 6be5e09..dfbd74c 100644 --- a/apps/api/src/services/health-check-service.ts +++ b/apps/api/src/services/health-check-service.ts @@ -6,6 +6,11 @@ import { repos } from "@cfdm/db"; import type { HealthCheckTarget, IpHealthState } from "@cfdm/shared"; import { AppError } from "../errors.js"; import { nextHealthState } from "./health/state-machine.js"; +import { LocalHealthCheckProvider } from "./health/local.js"; +import { + CloudflareWorkerHealthCheckProvider, + workerNotConfiguredResult, +} from "./health/worker.js"; export interface HealthCheckThresholds { degradedFailures: number; @@ -18,6 +23,7 @@ export interface ProbeResult { ok: boolean; latencyMs: number; error: string | null; + colo?: string | null; } /** Bracket IPv6 for URL authority; leave IPv4/hostname as-is. */ @@ -261,6 +267,8 @@ export interface RunAllChecksOptions { thresholds: HealthCheckThresholds; /** Pause between unique physical probes (default 2000). Same IP is only probed once. */ probeGapMs?: number; + /** Cloudflare Worker URL+token. Missing → cloudflare targets fail, never Local fallback. */ + worker?: { url: string; token: string } | null; onStatusChange?: ( target: HealthCheckTarget, prevState: IpHealthState | null, @@ -277,21 +285,34 @@ function sleep(ms: number): Promise { * so anti-bot / rate-limit on the origin is not tripped by back-to-back checks. */ export function physicalProbeKey(target: HealthCheckTarget): string { + const kind = target.provider === "cloudflare" ? "cloudflare" : "local"; const port = target.port ?? (target.type === "http" ? 80 : 80); const ip = String(target.ip || "").trim().toLowerCase(); if (target.type === "http") { const path = (target.path?.trim() || "/") || "/"; const expected = target.expected_status ?? ""; - return `http|${ip}|${port}|${path}|${expected}`; + return `${kind}|http|${ip}|${port}|${path}|${expected}`; } - if (target.type === "tcp") return `tcp|${ip}|${port}`; + if (target.type === "tcp") return `${kind}|tcp|${ip}|${port}`; if (target.type === "ping") { - return `ping|${String(target.hostname || target.ip || "").trim().toLowerCase()}`; + return `${kind}|ping|${String(target.hostname || target.ip || "").trim().toLowerCase()}`; } if (target.type === "dns") { - return `dns|${String(target.hostname || target.ip || "").trim().toLowerCase()}`; + return `${kind}|dns|${String(target.hostname || target.ip || "").trim().toLowerCase()}`; } - return `${target.type}|${ip}|${port}`; + return `${kind}|${target.type}|${ip}|${port}`; +} + +async function executeProbe( + target: HealthCheckTarget, + local: LocalHealthCheckProvider, + worker: CloudflareWorkerHealthCheckProvider | null, +): Promise { + if (target.provider === "cloudflare") { + if (!worker) return workerNotConfiguredResult(); + return worker.probe(target); + } + return local.probe(target); } export async function runAllChecks( @@ -300,6 +321,11 @@ export async function runAllChecks( ): Promise { const targets = repos.listHealthCheckTargets(db); const gapMs = Math.max(0, options.probeGapMs ?? 2000); + const local = new LocalHealthCheckProvider(); + const worker = + options.worker?.url && options.worker.token + ? new CloudflareWorkerHealthCheckProvider(options.worker) + : null; const byPhysical = new Map(); for (const target of targets) { @@ -319,7 +345,7 @@ export async function runAllChecks( // Prefer binding hostname for SNI when several scopes share one IP. const representative = group.find((t) => t.scope === "binding") ?? group[0]!; - const result = await probeTarget(representative); + const result = await executeProbe(representative, local, worker); for (const target of group) { const prev = repos.getIpHealthStatusRow( @@ -343,6 +369,7 @@ export async function runAllChecks( const prevState: IpHealthState | null = prev ? (prev.status as IpHealthState) : null; + const provider = target.provider === "cloudflare" ? "cloudflare" : "local"; repos.upsertIpHealthStatus( db, target.scope, @@ -353,7 +380,19 @@ export async function runAllChecks( failures, result.error, successes, + { colo: result.colo ?? null, provider }, ); + repos.insertHealthProbeLog(db, { + scope: target.scope, + refId: target.ref_id, + ip: target.ip, + provider, + status: state, + ok: result.ok, + latencyMs: result.latencyMs, + colo: result.colo ?? null, + error: result.error, + }); const matchedNode = repos.findNodeByIp(db, target.ip); if (matchedNode && matchedNode.enabled) { repos.updateNode(db, matchedNode.id, { @@ -392,6 +431,7 @@ export async function runDomainMonitors( expected_status: monitor.expected_status, timeout_ms: monitor.timeout_ms, verify_tls: false, + provider: "local", }; let result: ProbeResult; if (monitor.type === "http") { diff --git a/apps/api/src/services/health/worker.ts b/apps/api/src/services/health/worker.ts new file mode 100644 index 0000000..35e9c42 --- /dev/null +++ b/apps/api/src/services/health/worker.ts @@ -0,0 +1,93 @@ +import type { HealthCheckTarget } from "@cfdm/shared"; +import type { ProbeResult } from "../health-check-service.js"; +import type { HealthCheckProvider } from "./provider.js"; + +export interface WorkerProbeConfig { + url: string; + token: string; +} + +const WORKER_NOT_CONFIGURED = "Cloudflare Worker не настроен (URL и токен)"; + +export class CloudflareWorkerHealthCheckProvider implements HealthCheckProvider { + readonly kind = "cloudflare" as const; + + constructor(private readonly config: WorkerProbeConfig) {} + + async probe(target: HealthCheckTarget): Promise { + const base = this.config.url.replace(/\/$/, ""); + if (!base || !this.config.token) { + return { + ok: false, + latencyMs: 0, + error: WORKER_NOT_CONFIGURED, + colo: null, + }; + } + const timeoutMs = Math.max(100, target.timeout_ms ?? 3000); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs + 2500); + try { + const res = await fetch(`${base}/probe`, { + method: "POST", + headers: { + Authorization: `Bearer ${this.config.token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + type: target.type === "http" ? "http" : "tcp", + ip: target.ip, + hostname: target.hostname, + port: target.port ?? (target.type === "http" ? 80 : 80), + path: target.path ?? "/", + expected_status: target.expected_status, + timeout_ms: timeoutMs, + verify_tls: Boolean(target.verify_tls), + method: "GET", + }), + signal: controller.signal, + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + return { + ok: false, + latencyMs: 0, + error: `Worker HTTP ${res.status}${text ? `: ${text.slice(0, 180)}` : ""}`, + colo: null, + }; + } + const body = (await res.json()) as { + ok?: boolean; + latencyMs?: number; + error?: string | null; + colo?: string | null; + }; + const ok = Boolean(body.ok); + return { + ok, + latencyMs: typeof body.latencyMs === "number" ? body.latencyMs : 0, + error: ok ? null : (body.error ?? "probe failed"), + colo: body.colo ?? null, + }; + } catch (err) { + const message = + err instanceof Error + ? err.name === "AbortError" + ? "Worker timeout" + : err.message + : "Worker probe failed"; + return { ok: false, latencyMs: 0, error: message, colo: null }; + } finally { + clearTimeout(timer); + } + } +} + +export function workerNotConfiguredResult(): ProbeResult { + return { + ok: false, + latencyMs: 0, + error: WORKER_NOT_CONFIGURED, + colo: null, + }; +} diff --git a/apps/api/src/services/service-config-service.ts b/apps/api/src/services/service-config-service.ts index a73c781..50ed287 100644 --- a/apps/api/src/services/service-config-service.ts +++ b/apps/api/src/services/service-config-service.ts @@ -50,6 +50,7 @@ export interface ServiceDomainInput { health_check_interval_sec?: number; health_check_timeout_ms?: number; health_check_verify_tls?: boolean; + health_check_provider?: "local" | "cloudflare"; } export interface ToggleRequest { @@ -70,6 +71,7 @@ export interface ServiceGroupBody { health_check_interval_sec?: number; health_check_timeout_ms?: number; health_check_verify_tls?: boolean; + health_check_provider?: "local" | "cloudflare"; } export interface UpdateServiceGroupBody { @@ -86,6 +88,7 @@ export interface UpdateServiceGroupBody { health_check_interval_sec?: number; health_check_timeout_ms?: number; health_check_verify_tls?: boolean; + health_check_provider?: "local" | "cloudflare"; } export interface UpdateServiceConfigRequest { @@ -291,6 +294,7 @@ async function buildView(db: Db, serviceId: number): Promise { health_check_interval_sec: binding.health_check_interval_sec, health_check_timeout_ms: binding.health_check_timeout_ms, health_check_verify_tls: binding.health_check_verify_tls, + health_check_provider: binding.health_check_provider ?? "local", sync_status: aggregateSyncStatus(statuses), }; }); @@ -334,6 +338,10 @@ function attachServiceHealth( ip, status: row?.status ?? ("unknown" as const), latency_ms: row?.latency_ms ?? null, + last_checked_at: row?.last_checked_at ?? null, + last_error: row?.last_error ?? null, + provider: row?.provider ?? "local", + colo: row?.colo ?? null, }; }); return { @@ -1154,7 +1162,8 @@ export async function updateConfig( input.health_check_expected_status !== undefined || input.health_check_interval_sec !== undefined || input.health_check_timeout_ms !== undefined || - input.health_check_verify_tls !== undefined + input.health_check_verify_tls !== undefined || + input.health_check_provider !== undefined ) { repos.updateBindingLbConfig(db, binding.id, { lb_mode: input.lb_mode, @@ -1166,6 +1175,7 @@ export async function updateConfig( health_check_interval_sec: input.health_check_interval_sec, health_check_timeout_ms: input.health_check_timeout_ms, health_check_verify_tls: input.health_check_verify_tls, + health_check_provider: input.health_check_provider, }); } @@ -1267,6 +1277,7 @@ export async function createGroup( health_check_interval_sec: body.health_check_interval_sec, health_check_timeout_ms: body.health_check_timeout_ms, health_check_verify_tls: body.health_check_verify_tls, + health_check_provider: body.health_check_provider, }, ); } @@ -1303,6 +1314,7 @@ export async function updateGroup( health_check_interval_sec: body.health_check_interval_sec, health_check_timeout_ms: body.health_check_timeout_ms, health_check_verify_tls: body.health_check_verify_tls, + health_check_provider: body.health_check_provider, }, ); if (!domain && group.enabled) { diff --git a/apps/api/test/health-check.test.ts b/apps/api/test/health-check.test.ts index ec5763e..f365309 100644 --- a/apps/api/test/health-check.test.ts +++ b/apps/api/test/health-check.test.ts @@ -56,6 +56,7 @@ describe("health-check probeTarget", () => { expected_status: null, timeout_ms: 1000, verify_tls: false, + provider: "local", }; const result = await healthCheckService.probeTarget(target); expect(result.ok).toBe(true); @@ -75,6 +76,7 @@ describe("health-check probeTarget", () => { expected_status: null, timeout_ms: 500, verify_tls: false, + provider: "local", }; const result = await healthCheckService.probeTarget(target); expect(result.ok).toBe(false); @@ -107,6 +109,7 @@ describe("health-check probeTarget", () => { expected_status: 200, timeout_ms: 1000, verify_tls: false, + provider: "local", }; const bindingTarget: HealthCheckTarget = { ...groupTarget, @@ -142,6 +145,7 @@ describe("health-check probeTarget", () => { expected_status: null, timeout_ms: 3000, verify_tls: false, + provider: "local", }; const binding: HealthCheckTarget = { ...group, diff --git a/apps/api/test/health-worker.test.ts b/apps/api/test/health-worker.test.ts new file mode 100644 index 0000000..091516e --- /dev/null +++ b/apps/api/test/health-worker.test.ts @@ -0,0 +1,215 @@ +import { createServer, type Server as HttpServer } from "node:http"; +import { describe, expect, it } from "vitest"; +import { buildApp } from "../src/app.js"; +import { loadConfig } from "../src/config.js"; +import { repos, type Db } from "@cfdm/db"; +import * as healthCheckService from "../src/services/health-check-service.js"; + +async function authHeaders(app: Awaited>) { + const res = await app.inject({ + method: "POST", + url: "/api/v1/auth/login", + payload: { username: "admin", password: "admin" }, + }); + expect(res.statusCode).toBe(200); + const { token } = res.json() as { token: string }; + return { authorization: `Bearer ${token}` }; +} + +function startWorkerMock(handler: (req: { + url?: string; + headers: Record; + body: string; +}) => { status: number; json: unknown } | "hang"): Promise<{ + server: HttpServer; + url: string; +}> { + return new Promise((resolve) => { + const server = createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (chunk) => chunks.push(chunk as Buffer)); + req.on("end", () => { + const result = handler({ + url: req.url, + headers: req.headers, + body: Buffer.concat(chunks).toString("utf8"), + }); + if (result === "hang") return; + res.writeHead(result.status, { "content-type": "application/json" }); + res.end(JSON.stringify(result.json)); + }); + }); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address ? address.port : 0; + resolve({ server, url: `http://127.0.0.1:${port}` }); + }); + }); +} + +async function seedBinding( + db: Db, + opts: { provider: "local" | "cloudflare"; ip: string }, +) { + const domain = repos.createDomain(db, null, "example.com", "zone-1"); + const service = repos.createService(db, "Panel", "panel"); + repos.setServiceEnabled(db, service.id, true); + repos.replaceServiceIps(db, service.id, [opts.ip]); + const binding = repos.insertBinding(db, domain.id, service.id, "panel", null); + repos.replaceBindingIpsWithMeta(db, binding.id, [ + { ip: opts.ip, weight: 1, priority: 1 }, + ]); + repos.updateBindingLbConfig(db, binding.id, { + health_check_enabled: true, + health_check_type: "tcp", + health_check_port: 1, + health_check_timeout_ms: 400, + health_check_provider: opts.provider, + }); + return { service, binding, domain }; +} + +const thresholds = { + degradedFailures: 1, + downFailures: 2, + latencyWarnMs: 1000, + successRecoveries: 2, +}; + +describe("health-check XOR worker", () => { + it("lists only local providers when no cloudflare bindings", async () => { + const app = await buildApp({ + config: { ...loadConfig(), staticDir: null }, + memory: true, + }); + await seedBinding(app.db, { + provider: "local", + ip: "10.0.0.1", + }); + const targets = repos.listHealthCheckTargets(app.db); + expect(targets.length).toBeGreaterThan(0); + expect(targets.every((t) => t.provider === "local")).toBe(true); + expect(targets.some((t) => t.provider === "cloudflare")).toBe(false); + await app.close(); + }); + + it("cloudflare without worker URL does not fall back to local", async () => { + const app = await buildApp({ + config: { ...loadConfig(), staticDir: null }, + memory: true, + }); + const { binding } = await seedBinding(app.db, { + provider: "cloudflare", + ip: "127.0.0.1", + }); + await healthCheckService.runAllChecks(app.db, { + thresholds, + probeGapMs: 0, + worker: null, + }); + const row = repos.getIpHealthStatusRow( + app.db, + "binding", + binding.id, + "127.0.0.1", + ); + expect(row?.last_error).toMatch(/Worker не настроен/i); + expect(row?.provider).toBe("cloudflare"); + await app.close(); + }); + + it("worker mock 200 writes colo and last_checked_at", async () => { + const mock = await startWorkerMock((req) => { + const auth = String(req.headers.authorization ?? ""); + if (auth !== "Bearer secret") { + return { status: 401, json: { ok: false, error: "unauthorized" } }; + } + return { + status: 200, + json: { ok: true, latencyMs: 42, error: null, colo: "AMS" }, + }; + }); + const app = await buildApp({ + config: { ...loadConfig(), staticDir: null }, + memory: true, + }); + const headers = await authHeaders(app); + const { service, binding } = await seedBinding(app.db, { + provider: "cloudflare", + ip: "203.0.113.10", + }); + await healthCheckService.runAllChecks(app.db, { + thresholds, + probeGapMs: 0, + worker: { url: mock.url, token: "secret" }, + }); + const row = repos.getIpHealthStatusRow( + app.db, + "binding", + binding.id, + "203.0.113.10", + ); + expect(row?.status).toBe("up"); + expect(row?.colo).toBe("AMS"); + expect(row?.last_checked_at).toBeTruthy(); + expect(row?.provider).toBe("cloudflare"); + + const res = await app.inject({ + method: "GET", + url: `/api/v1/services/${service.id}`, + headers, + }); + expect(res.statusCode).toBe(200); + const body = res.json() as { + ip_health: Array<{ + ip: string; + colo: string | null; + last_checked_at: string | null; + provider: string; + }>; + }; + const ipRow = body.ip_health.find((item) => item.ip === "203.0.113.10"); + expect(ipRow?.colo).toBe("AMS"); + expect(ipRow?.last_checked_at).toBeTruthy(); + expect(ipRow?.provider).toBe("cloudflare"); + + const logRes = await app.inject({ + method: "GET", + url: `/api/v1/services/${service.id}/health-log`, + headers, + }); + expect(logRes.statusCode).toBe(200); + const logBody = logRes.json() as { items: Array<{ colo: string | null }> }; + expect(logBody.items[0]?.colo).toBe("AMS"); + + await new Promise((resolve) => mock.server.close(() => resolve())); + await app.close(); + }); + + it("worker timeout is recorded, not local probe", async () => { + const mock = await startWorkerMock(() => "hang"); + const app = await buildApp({ + config: { ...loadConfig(), staticDir: null }, + memory: true, + }); + const { binding } = await seedBinding(app.db, { + provider: "cloudflare", + ip: "203.0.113.20", + }); + await healthCheckService.runAllChecks(app.db, { + thresholds, + probeGapMs: 0, + worker: { url: mock.url, token: "secret" }, + }); + const row = repos.getIpHealthStatusRow( + app.db, + "binding", + binding.id, + "203.0.113.20", + ); + expect(row?.last_error).toMatch(/timeout|Worker/i); + expect(row?.provider).toBe("cloudflare"); + await new Promise((resolve) => mock.server.close(() => resolve())); + await app.close(); + }, 15_000); +}); diff --git a/apps/api/test/service-groups-health.test.ts b/apps/api/test/service-groups-health.test.ts index fa44ac7..49b9243 100644 --- a/apps/api/test/service-groups-health.test.ts +++ b/apps/api/test/service-groups-health.test.ts @@ -99,7 +99,15 @@ describe("service groups health enrichment", () => { expect(groupView!.services[0]?.health_status).toBe("degraded"); expect(groupView!.services[0]?.health_latency_ms).toBe(120); expect(groupView!.services[0]?.ip_health).toEqual([ - { ip: "1.2.3.4", status: "degraded", latency_ms: 120 }, + { + ip: "1.2.3.4", + status: "degraded", + latency_ms: 120, + last_checked_at: expect.any(String), + last_error: null, + provider: "local", + colo: null, + }, ]); // group worst = degraded (from service) over up (group scope) expect(groupView!.health_status).toBe("degraded"); diff --git a/apps/api/test/settings-health.test.ts b/apps/api/test/settings-health.test.ts index b46ceb2..12b50fe 100644 --- a/apps/api/test/settings-health.test.ts +++ b/apps/api/test/settings-health.test.ts @@ -133,4 +133,33 @@ describe("settings health engine", () => { expect(res.statusCode).toBe(400); await app.close(); }); + + it("PATCH worker URL; token is not returned in GET", async () => { + const app = await buildApp({ + config: { ...loadConfig(), staticDir: null }, + memory: true, + }); + const headers = await authHeaders(app); + const res = await app.inject({ + method: "PATCH", + url: "/api/v1/settings", + headers, + payload: { + healthWorkerUrl: "https://cfdm-health-probe.example.workers.dev", + healthWorkerToken: "super-secret", + }, + }); + expect(res.statusCode).toBe(200); + const body = res.json() as { + healthWorkerUrl: string; + healthWorkerTokenSet: boolean; + healthWorkerToken?: string; + }; + expect(body.healthWorkerUrl).toBe( + "https://cfdm-health-probe.example.workers.dev", + ); + expect(body.healthWorkerTokenSet).toBe(true); + expect(body.healthWorkerToken).toBeUndefined(); + await app.close(); + }); }); diff --git a/apps/web/src/components/health-check-badge.tsx b/apps/web/src/components/health-check-badge.tsx index 41619ea..cd8a3ee 100644 --- a/apps/web/src/components/health-check-badge.tsx +++ b/apps/web/src/components/health-check-badge.tsx @@ -54,6 +54,8 @@ interface HealthCheckBadgeProps { latencyMs?: number | null lastCheckedAt?: string | null lastError?: string | null + colo?: string | null + provider?: 'local' | 'cloudflare' | string | null title?: string showLatency?: boolean size?: 'xs' | 'sm' @@ -65,6 +67,8 @@ export function HealthCheckBadge({ latencyMs, lastCheckedAt, lastError, + colo, + provider, title, showLatency = false, size = 'sm', @@ -79,6 +83,9 @@ export function HealthCheckBadge({ tooltipParts.push(`Статус: ${label}`) if (latencyMs != null) tooltipParts.push(`Задержка: ${latencyMs} мс`) if (lastCheckedAt) tooltipParts.push(`Проверка: ${formatDate(lastCheckedAt)}`) + if (colo) tooltipParts.push(`Colo: ${colo}`) + if (provider === 'cloudflare') tooltipParts.push('Провайдер: Cloudflare Worker') + if (provider === 'local') tooltipParts.push('Провайдер: Local') if (lastError) tooltipParts.push(`Ошибка: ${lastError}`) return ( diff --git a/apps/web/src/components/health-check-config-fields.tsx b/apps/web/src/components/health-check-config-fields.tsx index d5ecc8b..ccbfd54 100644 --- a/apps/web/src/components/health-check-config-fields.tsx +++ b/apps/web/src/components/health-check-config-fields.tsx @@ -190,7 +190,7 @@ export function HealthCheckConfigFields({ {value.provider === 'cloudflare' ? ( - Cloudflare Health Checks + Cloudflare Worker - Поля соответствуют официальному API зоны. Если план не позволяет Health - Checks, API вернёт ошибку — останется Local. Workers не используются. + Проба с edge Cloudflare, не продукт Health Checks API (на Free его нет). + Регионы WNAM/WEU недоступны — в результате будет colo ближайшего POP + (например AMS). URL и токен Worker — в{' '} + + Настройках → Health-check + + . Если Worker не задан, цель не пробируется как Local. ) : ( Local health-check - Интервал и таймаут пробы — ниже. Cron и пороги Slow/Down задаются в{' '} + Проба TCP/HTTP с сервера API. Cron и пороги Slow/Down — в{' '} Настройках → Health-check - , как параметры Cloudflare Health Checks в этой форме. + . Интервал в карточке не используется. )} @@ -334,83 +339,18 @@ export function HealthCheckConfigFields({ ) : null} -
- - - patch({ interval_sec: next ?? 30 }) - } - /> - - - - patch({ timeout_ms: next ?? 3000 }) - } - /> - -
- {value.provider === 'cloudflare' ? ( -
- - patch({ retries: retries ?? 2 })} - /> - - - - patch({ consecutive_successes: consecutive_successes ?? 2 }) - } - /> - -
- ) : null} - {value.provider === 'cloudflare' && isHttp ? ( - - - - ) : null} + + + patch({ timeout_ms: next ?? 3000 }) + } + /> + ) : null} diff --git a/apps/web/src/components/health/health-timeline.tsx b/apps/web/src/components/health/health-timeline.tsx index 0a35290..2a552ed 100644 --- a/apps/web/src/components/health/health-timeline.tsx +++ b/apps/web/src/components/health/health-timeline.tsx @@ -23,6 +23,8 @@ export interface HealthTimelineEvent { latency_ms?: number | null error?: string | null checked_at: string + colo?: string | null + provider?: string | null } interface HealthTimelineProps { @@ -63,6 +65,8 @@ export function HealthTimeline({ events }: HealthTimelineProps) { diff --git a/apps/web/src/components/service-edit-sheet.tsx b/apps/web/src/components/service-edit-sheet.tsx index 99c5a07..d875779 100644 --- a/apps/web/src/components/service-edit-sheet.tsx +++ b/apps/web/src/components/service-edit-sheet.tsx @@ -110,7 +110,7 @@ function toBindingDrafts(service: ServiceView): ServiceBindingDraft[] { interval_sec: binding.health_check_interval_sec, timeout_ms: binding.health_check_timeout_ms, verify_tls: binding.health_check_verify_tls ?? false, - provider: 'local', + provider: binding.health_check_provider === 'cloudflare' ? 'cloudflare' : 'local', }, target_ip_weights: binding.target_ip_weights ?? {}, target_ip_priorities: binding.target_ip_priorities ?? {}, @@ -138,6 +138,7 @@ function buildDomainsPayload(bindings: ServiceBindingDraft[]) { health_check_interval_sec: binding.health.interval_sec, health_check_timeout_ms: binding.health.timeout_ms, health_check_verify_tls: binding.health.verify_tls, + health_check_provider: binding.health.provider, } : { fqdn: binding.fqdn.trim(), @@ -153,6 +154,7 @@ function buildDomainsPayload(bindings: ServiceBindingDraft[]) { health_check_interval_sec: binding.health.interval_sec, health_check_timeout_ms: binding.health.timeout_ms, health_check_verify_tls: binding.health.verify_tls, + health_check_provider: binding.health.provider, }, ) } diff --git a/apps/web/src/components/services/service-fqdn-list.tsx b/apps/web/src/components/services/service-fqdn-list.tsx index 1ae9cd5..dba74f2 100644 --- a/apps/web/src/components/services/service-fqdn-list.tsx +++ b/apps/web/src/components/services/service-fqdn-list.tsx @@ -166,6 +166,10 @@ export function ServiceIpList({ ({ @@ -98,6 +100,24 @@ export const serviceIpHealthSchema = z.object({ ip: z.string(), status: z.enum(['up', 'down', 'degraded', 'unknown']), latency_ms: z.number().nullable(), + last_checked_at: z.string().nullable().optional(), + last_error: z.string().nullable().optional(), + provider: z.enum(['local', 'cloudflare']).optional(), + colo: z.string().nullable().optional(), +}) + +export const healthProbeLogSchema = z.object({ + id: z.number(), + scope: z.string(), + ref_id: z.number(), + ip: z.string(), + provider: z.enum(['local', 'cloudflare']), + status: z.enum(['up', 'down', 'degraded', 'unknown']), + ok: z.coerce.boolean(), + latency_ms: z.number().nullable(), + colo: z.string().nullable(), + error: z.string().nullable(), + checked_at: z.string(), }) export const serviceViewSchema = serviceSchema.extend({ @@ -168,6 +188,7 @@ export const serviceBindingSchema = z health_check_interval_sec: z.number().default(30), health_check_timeout_ms: z.number().default(3000), health_check_verify_tls: z.coerce.boolean().default(false), + health_check_provider: z.enum(['local', 'cloudflare']).catch('local'), sync_status: z.string().nullable().default(null), created_at: z.string(), updated_at: z.string(), @@ -252,6 +273,7 @@ const healthCheckConfigFields = { health_check_interval_sec: z.number().int().min(5).max(3600).optional(), health_check_timeout_ms: z.number().int().min(100).max(30000).optional(), health_check_verify_tls: z.boolean().optional(), + health_check_provider: z.enum(['local', 'cloudflare']).optional(), } const serviceDomainInputSchema = z diff --git a/apps/web/src/queries/services.ts b/apps/web/src/queries/services.ts index a5e3ca7..5439731 100644 --- a/apps/web/src/queries/services.ts +++ b/apps/web/src/queries/services.ts @@ -1,6 +1,7 @@ import { queryOptions } from '@tanstack/react-query' import { api } from '@/lib/api-client' import { + healthProbeLogSchema, serviceBindingSchema, serviceGroupsResponseSchema, serviceViewSchema, @@ -87,8 +88,28 @@ export async function deleteServiceBinding(id: number) { export const serviceDetailKeys = { overview: (id: number) => [...serviceKeys.all, id, 'overview'] as const, nodes: (id: number) => [...serviceKeys.all, id, 'nodes'] as const, + healthLog: (id: number) => [...serviceKeys.all, id, 'health-log'] as const, + view: (id: number) => [...serviceKeys.all, id, 'view'] as const, } +export const serviceViewQueryOptions = (id: number) => + queryOptions({ + queryKey: serviceDetailKeys.view(id), + queryFn: async () => { + const data = await api.get(`/api/v1/services/${id}`) + return serviceViewSchema.parse(data) + }, + }) + +export const serviceHealthLogQueryOptions = (id: number) => + queryOptions({ + queryKey: serviceDetailKeys.healthLog(id), + queryFn: async () => { + const data = await api.get(`/api/v1/services/${id}/health-log`) + return z.object({ items: z.array(healthProbeLogSchema) }).parse(data) + }, + }) + export const serviceOverviewQueryOptions = (id: number) => queryOptions({ queryKey: serviceDetailKeys.overview(id), diff --git a/apps/web/src/routes/_auth/services/$serviceId/health.tsx b/apps/web/src/routes/_auth/services/$serviceId/health.tsx index 5cb7fdf..6fa95c8 100644 --- a/apps/web/src/routes/_auth/services/$serviceId/health.tsx +++ b/apps/web/src/routes/_auth/services/$serviceId/health.tsx @@ -1,22 +1,16 @@ -import { createFileRoute } from '@tanstack/react-router' -import { useState } from 'react' -import { useForm } from 'react-hook-form' -import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' -import { toast } from 'sonner' -import { DetailPanel } from '@/components/reui-kit' +import { createFileRoute, Link } from '@tanstack/react-router' +import { useQuery } from '@tanstack/react-query' +import { ActivityIcon, GlobeIcon, ServerIcon } from 'lucide-react' +import { DetailPanel, KpiStatGrid } from '@/components/reui-kit' import { EmptyState } from '@/components/empty-state' -import { FormSheet } from '@/components/form-sheet' -import { FormFieldSimple } from '@/components/form-field' -import { LoadingButton } from '@/components/loading-button' -import { Button } from '@cfdm/ui/components/button' -import { Input } from '@cfdm/ui/components/input' import { Alert, AlertDescription, AlertTitle } from '@/components/reui/alert' -import { HealthProviderToggle } from '@/components/health-check-config-fields' +import { HealthTimeline } from '@/components/health/health-timeline' +import { HealthCheckBadge } from '@/components/health-check-badge' import { - createOriginHealthCheck, - listOriginHealthChecks, - serviceOverviewQueryOptions, + serviceHealthLogQueryOptions, + serviceViewQueryOptions, } from '@/queries' +import { formatDate } from '@/lib/format' export const Route = createFileRoute('/_auth/services/$serviceId/health')({ component: ServiceHealthPage, @@ -25,119 +19,86 @@ export const Route = createFileRoute('/_auth/services/$serviceId/health')({ export function ServiceHealthPage() { const { serviceId } = Route.useParams() const id = Number(serviceId) - const queryClient = useQueryClient() - const overview = useQuery(serviceOverviewQueryOptions(id)) - const checksQuery = useQuery({ - queryKey: ['health-checks'], - queryFn: listOriginHealthChecks, - }) - const [open, setOpen] = useState(false) - const form = useForm<{ - name: string - provider: 'local' | 'cloudflare' - protocol: string - }>({ - defaultValues: { name: '', provider: 'local', protocol: 'tcp' }, - }) - const provider = form.watch('provider') - const checks = (checksQuery.data ?? []) as Array<{ - id: number - name: string - provider: string - protocol: string - }> + const serviceQuery = useQuery(serviceViewQueryOptions(id)) + const logQuery = useQuery(serviceHealthLogQueryOptions(id)) + const service = serviceQuery.data + const items = logQuery.data?.items ?? [] + const ipHealth = service?.ip_health ?? [] - const createMut = useMutation({ - mutationFn: (values: { name: string; provider: 'local' | 'cloudflare'; protocol: string }) => - createOriginHealthCheck({ - name: values.name, - provider: values.provider, - protocol: values.protocol, - }), - onSuccess: async () => { - toast.success('Health check сохранён') - await queryClient.invalidateQueries({ queryKey: ['health-checks'] }) - setOpen(false) - }, - onError: (e: unknown) => - toast.error( - e instanceof Error - ? e.message - : 'Cloudflare Health Checks недоступны для этой зоны', + const kpiCards = ipHealth.map((row) => { + const variant = + row.status === 'down' + ? ('destructive' as const) + : row.status === 'degraded' + ? ('warning' as const) + : ('default' as const) + return { + id: row.ip, + label: row.ip, + value: row.latency_ms != null ? `${row.latency_ms} мс` : '—', + hint: row.colo ? `colo ${row.colo}` : row.provider === 'cloudflare' ? 'Worker' : 'Local', + icon: row.provider === 'cloudflare' ? : , + variant, + footer: ( + ), + } }) - void overview - return ( setOpen(true)}> - Добавить проверку - + title="Health" + description="Снимок проб этого сервиса. Cloudflare = Worker с edge, не Health Checks API." + /> + + XOR провайдеров + + Local ходит с API CFDM; Cloudflare — через Worker. Cron и пороги Slow/Down общие, в{' '} + + Настройках → Health-check + + . Если Worker не задан, цель не пробируется как Local. + + + {kpiCards.length > 0 ? ( + + ) : ( + + )} + - {checks.length === 0 ? ( - - ) : ( -
- {checks.map((check) => ( -
-
- {check.name} - - {check.provider} · {check.protocol} - -
-
- ))} -
- )} - createMut.mutate(values)} - footer={ - - Сохранить - - } - > - - - - - form.setValue('provider', next)} - /> - - {provider === 'cloudflare' ? ( - - Cloudflare Health Checks - - Если зона не поддерживает Health Checks, вернётся ошибка плана — останется - Local. Workers не используются. - - - ) : null} - - - - + ({ + id: row.id, + hostname: row.ip, + type: row.provider, + status: row.status, + latency_ms: row.latency_ms, + error: row.error, + checked_at: row.checked_at, + colo: row.colo, + provider: row.provider, + }))} + />
) } diff --git a/apps/web/src/routes/_auth/settings/health.tsx b/apps/web/src/routes/_auth/settings/health.tsx index dac0823..3806bca 100644 --- a/apps/web/src/routes/_auth/settings/health.tsx +++ b/apps/web/src/routes/_auth/settings/health.tsx @@ -27,6 +27,7 @@ import { } from '@/components/reui/number-field' import { FieldGroup } from '@cfdm/ui/components/field' import { Input } from '@cfdm/ui/components/input' +import { Alert, AlertDescription, AlertTitle } from '@/components/reui/alert' const formSchema = z.object({ healthCheckCron: z.string().trim().min(1, 'Укажите cron').max(64), @@ -34,6 +35,8 @@ const formSchema = z.object({ healthDownFailures: z.number().int().min(1).max(50), healthLatencyWarnMs: z.number().int().min(50).max(60_000), healthSuccessRecoveries: z.number().int().min(1).max(20), + healthWorkerUrl: z.string().trim().url('Некорректный URL').or(z.literal('')), + healthWorkerToken: z.string().optional(), }).superRefine((data, ctx) => { if (data.healthDownFailures < data.healthDegradedFailures) { ctx.addIssue({ @@ -48,6 +51,7 @@ type FormValues = z.infer type SettingsResponse = FormValues & { id: string + healthWorkerTokenSet?: boolean } export const Route = createFileRoute('/_auth/settings/health')({ @@ -105,6 +109,8 @@ function HealthSettingsPage() { healthDownFailures: 2, healthLatencyWarnMs: 1000, healthSuccessRecoveries: 2, + healthWorkerUrl: '', + healthWorkerToken: '', }, }) @@ -116,12 +122,26 @@ function HealthSettingsPage() { healthDownFailures: data.healthDownFailures, healthLatencyWarnMs: data.healthLatencyWarnMs, healthSuccessRecoveries: data.healthSuccessRecoveries, + healthWorkerUrl: data.healthWorkerUrl ?? '', + healthWorkerToken: '', }) }, [data, form]) const saveMut = useMutation({ - mutationFn: (values: FormValues) => - api.patch('/api/v1/settings', values), + mutationFn: (values: FormValues) => { + const payload: Record = { + healthCheckCron: values.healthCheckCron, + healthDegradedFailures: values.healthDegradedFailures, + healthDownFailures: values.healthDownFailures, + healthLatencyWarnMs: values.healthLatencyWarnMs, + healthSuccessRecoveries: values.healthSuccessRecoveries, + healthWorkerUrl: values.healthWorkerUrl, + } + if (values.healthWorkerToken?.trim()) { + payload.healthWorkerToken = values.healthWorkerToken.trim() + } + return api.patch('/api/v1/settings', payload) + }, onSuccess: () => { void queryClient.invalidateQueries({ queryKey: ['app-settings'] }) toast.success('Настройки health-check сохранены') @@ -144,8 +164,8 @@ function HealthSettingsPage() { Local health-check - Расписание и пороги движка. Параметры Cloudflare Health Checks - задаются в карточке сервиса. + Расписание и пороги движка — общие для Local и Cloudflare Worker. + Тип/порт/path задаются в карточке сервиса. @@ -247,7 +267,6 @@ function HealthSettingsPage() { description="Подряд успешных проб, чтобы выйти из Checking в Healthy. Env: HEALTH_SUCCESS_RECOVERIES." labelFor="health-recoveries" compact - last >
+ + + + + {form.formState.errors.healthWorkerUrl ? ( +

+ {form.formState.errors.healthWorkerUrl.message} +

+ ) : null} + + + + + + Cloudflare Worker, не Health Checks API + + На Free-плане продукта Health Checks нет. CFDM вызывает Worker с edge; + cron остаётся здесь. Лимит Free Workers ≈ 100k запросов/сутки (cron × число IP). + + ; + health_check_provider: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "health_check_provider"; + tableName: "service_groups"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: true; + hasDefault: true; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; created_at: drizzle_orm_sqlite_core.SQLiteColumn<{ name: "created_at"; tableName: "service_groups"; @@ -1499,6 +1518,25 @@ declare const serviceBindings: drizzle_orm_sqlite_core.SQLiteTableWithColumns<{ identity: undefined; generated: undefined; }, {}, {}>; + health_check_provider: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "health_check_provider"; + tableName: "service_bindings"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: true; + hasDefault: true; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; routing_strategy: drizzle_orm_sqlite_core.SQLiteColumn<{ name: "routing_strategy"; tableName: "service_bindings"; @@ -2990,6 +3028,44 @@ declare const ipHealthStatus: drizzle_orm_sqlite_core.SQLiteTableWithColumns<{ }, {}, { length: number | undefined; }>; + colo: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "colo"; + tableName: "ip_health_status"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: false; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; + provider: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "provider"; + tableName: "ip_health_status"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: true; + hasDefault: true; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; created_at: drizzle_orm_sqlite_core.SQLiteColumn<{ name: "created_at"; tableName: "ip_health_status"; @@ -3251,6 +3327,44 @@ declare const appSettings: drizzle_orm_sqlite_core.SQLiteTableWithColumns<{ identity: undefined; generated: undefined; }, {}, {}>; + health_worker_url: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "health_worker_url"; + tableName: "app_settings"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: false; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; + health_worker_token: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "health_worker_token"; + tableName: "app_settings"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: false; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; created_at: drizzle_orm_sqlite_core.SQLiteColumn<{ name: "created_at"; tableName: "app_settings"; @@ -3728,6 +3842,214 @@ declare const domainMonitorResults: drizzle_orm_sqlite_core.SQLiteTableWithColum }; dialect: "sqlite"; }>; +declare const healthProbeLog: drizzle_orm_sqlite_core.SQLiteTableWithColumns<{ + name: "health_probe_log"; + schema: undefined; + columns: { + id: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "id"; + tableName: "health_probe_log"; + dataType: "number"; + columnType: "SQLiteInteger"; + data: number; + driverParam: number; + notNull: true; + hasDefault: true; + isPrimaryKey: true; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: undefined; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, {}>; + scope: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "scope"; + tableName: "health_probe_log"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: true; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; + ref_id: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "ref_id"; + tableName: "health_probe_log"; + dataType: "number"; + columnType: "SQLiteInteger"; + data: number; + driverParam: number; + notNull: true; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: undefined; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, {}>; + ip: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "ip"; + tableName: "health_probe_log"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: true; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; + provider: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "provider"; + tableName: "health_probe_log"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: true; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; + status: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "status"; + tableName: "health_probe_log"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: true; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; + ok: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "ok"; + tableName: "health_probe_log"; + dataType: "boolean"; + columnType: "SQLiteBoolean"; + data: boolean; + driverParam: number; + notNull: true; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: undefined; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, {}>; + latency_ms: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "latency_ms"; + tableName: "health_probe_log"; + dataType: "number"; + columnType: "SQLiteInteger"; + data: number; + driverParam: number; + notNull: false; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: undefined; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, {}>; + colo: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "colo"; + tableName: "health_probe_log"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: false; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; + error: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "error"; + tableName: "health_probe_log"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: false; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; + checked_at: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "checked_at"; + tableName: "health_probe_log"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: true; + hasDefault: true; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; + }; + dialect: "sqlite"; +}>; declare const notificationLog: drizzle_orm_sqlite_core.SQLiteTableWithColumns<{ name: "notification_log"; schema: undefined; @@ -4715,6 +5037,25 @@ declare const schema: { identity: undefined; generated: undefined; }, {}, {}>; + health_check_provider: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "health_check_provider"; + tableName: "service_groups"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: true; + hasDefault: true; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; created_at: drizzle_orm_sqlite_core.SQLiteColumn<{ name: "created_at"; tableName: "service_groups"; @@ -5634,6 +5975,25 @@ declare const schema: { identity: undefined; generated: undefined; }, {}, {}>; + health_check_provider: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "health_check_provider"; + tableName: "service_bindings"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: true; + hasDefault: true; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; routing_strategy: drizzle_orm_sqlite_core.SQLiteColumn<{ name: "routing_strategy"; tableName: "service_bindings"; @@ -7125,6 +7485,44 @@ declare const schema: { }, {}, { length: number | undefined; }>; + colo: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "colo"; + tableName: "ip_health_status"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: false; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; + provider: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "provider"; + tableName: "ip_health_status"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: true; + hasDefault: true; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; created_at: drizzle_orm_sqlite_core.SQLiteColumn<{ name: "created_at"; tableName: "ip_health_status"; @@ -7386,6 +7784,44 @@ declare const schema: { identity: undefined; generated: undefined; }, {}, {}>; + health_worker_url: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "health_worker_url"; + tableName: "app_settings"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: false; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; + health_worker_token: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "health_worker_token"; + tableName: "app_settings"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: false; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; created_at: drizzle_orm_sqlite_core.SQLiteColumn<{ name: "created_at"; tableName: "app_settings"; @@ -7863,6 +8299,214 @@ declare const schema: { }; dialect: "sqlite"; }>; + healthProbeLog: drizzle_orm_sqlite_core.SQLiteTableWithColumns<{ + name: "health_probe_log"; + schema: undefined; + columns: { + id: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "id"; + tableName: "health_probe_log"; + dataType: "number"; + columnType: "SQLiteInteger"; + data: number; + driverParam: number; + notNull: true; + hasDefault: true; + isPrimaryKey: true; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: undefined; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, {}>; + scope: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "scope"; + tableName: "health_probe_log"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: true; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; + ref_id: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "ref_id"; + tableName: "health_probe_log"; + dataType: "number"; + columnType: "SQLiteInteger"; + data: number; + driverParam: number; + notNull: true; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: undefined; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, {}>; + ip: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "ip"; + tableName: "health_probe_log"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: true; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; + provider: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "provider"; + tableName: "health_probe_log"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: true; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; + status: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "status"; + tableName: "health_probe_log"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: true; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; + ok: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "ok"; + tableName: "health_probe_log"; + dataType: "boolean"; + columnType: "SQLiteBoolean"; + data: boolean; + driverParam: number; + notNull: true; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: undefined; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, {}>; + latency_ms: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "latency_ms"; + tableName: "health_probe_log"; + dataType: "number"; + columnType: "SQLiteInteger"; + data: number; + driverParam: number; + notNull: false; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: undefined; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, {}>; + colo: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "colo"; + tableName: "health_probe_log"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: false; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; + error: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "error"; + tableName: "health_probe_log"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: false; + hasDefault: false; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; + checked_at: drizzle_orm_sqlite_core.SQLiteColumn<{ + name: "checked_at"; + tableName: "health_probe_log"; + dataType: "string"; + columnType: "SQLiteText"; + data: string; + driverParam: string; + notNull: true; + hasDefault: true; + isPrimaryKey: false; + isAutoincrement: false; + hasRuntimeDefault: false; + enumValues: [string, ...string[]]; + baseColumn: never; + identity: undefined; + generated: undefined; + }, {}, { + length: number | undefined; + }>; + }; + dialect: "sqlite"; + }>; notificationLog: drizzle_orm_sqlite_core.SQLiteTableWithColumns<{ name: "notification_log"; schema: undefined; @@ -8334,6 +8978,8 @@ type AppSettingsDto = { vpsTrackerSyncEnabled: boolean; vpsTrackerLastSyncAt: string | null; showQuickActions: boolean; + healthWorkerUrl: string; + healthWorkerTokenSet: boolean; } & HealthEngineSettings; type AppSettingsPatch = { vpsTrackerUrl?: string; @@ -8345,13 +8991,20 @@ type AppSettingsPatch = { healthDownFailures?: number; healthLatencyWarnMs?: number; healthSuccessRecoveries?: number; + healthWorkerUrl?: string; + healthWorkerToken?: string; +}; +type HealthEngineFallbacks = HealthEngineSettings & { + healthWorkerUrl: string; + healthWorkerTokenSet: boolean; }; -type HealthEngineFallbacks = HealthEngineSettings; declare function getAppSettings(db: Db, fallbacks?: HealthEngineFallbacks): AppSettingsDto; declare function getAppSettingsSecrets(db: Db): { vpsTrackerUrl: string; vpsTrackerIntegrationToken: string; vpsTrackerSyncEnabled: boolean; + healthWorkerUrl: string; + healthWorkerToken: string; }; declare function updateAppSettings(db: Db, patch: AppSettingsPatch, fallbacks?: HealthEngineFallbacks): AppSettingsDto; declare function touchVpsTrackerSync(db: Db): void; @@ -8439,6 +9092,7 @@ interface ServiceGroupLbPatch { health_check_interval_sec?: number; health_check_timeout_ms?: number; health_check_verify_tls?: boolean; + health_check_provider?: HealthCheckProvider; } declare function createServiceGroup(db: Db, name: string, groupType: string, icon: string | null, domain: string | null, lbPatch?: ServiceGroupLbPatch): ServiceGroup; declare function updateServiceGroup(db: Db, id: number, name: string, groupType: string, icon: string | null, domain: string | null, lbPatch?: ServiceGroupLbPatch): ServiceGroup; @@ -8546,6 +9200,7 @@ interface BindingLbPatch { health_check_interval_sec?: number; health_check_timeout_ms?: number; health_check_verify_tls?: boolean; + health_check_provider?: HealthCheckProvider; } declare function updateBindingLbConfig(db: Db, bindingId: number, patch: BindingLbPatch): void; declare function setBindingCnameTarget(db: Db, bindingId: number, target: string | null): void; @@ -8595,12 +9250,19 @@ type ServiceIpHealthRow = { ip: string; status: IpHealthState; latency_ms: number | null; + last_checked_at: string | null; + last_error: string | null; + provider: HealthCheckProvider; + colo: string | null; }; /** Per-IP binding-scope health, worst status if the same IP is on several bindings. */ declare function listIpHealthByServiceIds(db: Db, serviceIds: number[]): Map; declare function mergeHealthAggregates(parts: Array): HealthAggregate; declare function getIpHealthStatusRow(db: Db, scope: HealthCheckScope, refId: number, ip: string): IpHealthStatus | null; -declare function upsertIpHealthStatus(db: Db, scope: HealthCheckScope, refId: number, ip: string, status: string, latencyMs: number | null, consecutiveFailures: number, lastError: string | null, consecutiveSuccesses?: number): void; +declare function upsertIpHealthStatus(db: Db, scope: HealthCheckScope, refId: number, ip: string, status: string, latencyMs: number | null, consecutiveFailures: number, lastError: string | null, consecutiveSuccesses?: number, extras?: { + colo?: string | null; + provider?: HealthCheckProvider; +}): void; declare function deleteIpHealthStatusForRef(db: Db, scope: HealthCheckScope, refId: number): void; declare function deleteIpHealthStatusForIp(db: Db, scope: HealthCheckScope, refId: number, ip: string): void; /** Drop health rows whose IP is no longer a live probe target for that scope/ref. @@ -8665,6 +9327,30 @@ declare function listDomainMonitorResultsForDomain(db: Db, domainId: number, lim error: string | null; checked_at: string; }[]; +declare function insertHealthProbeLog(db: Db, entry: { + scope: HealthCheckScope; + refId: number; + ip: string; + provider: HealthCheckProvider; + status: string; + ok: boolean; + latencyMs: number | null; + colo: string | null; + error: string | null; +}): void; +declare function listHealthProbeLogForService(db: Db, serviceId: number, limit?: number): { + id: number; + scope: string; + ref_id: number; + ip: string; + provider: HealthCheckProvider; + status: string; + ok: boolean; + latency_ms: number | null; + colo: string | null; + error: string | null; + checked_at: string; +}[]; declare function insertNotificationLog(db: Db, kind: string, refType: string, refId: number | null, title: string, message: string): void; declare function listNotificationLog(db: Db, limit?: number): { id: number; @@ -8741,6 +9427,7 @@ declare const repos_getSubdomain: typeof getSubdomain; declare const repos_getSyncJob: typeof getSyncJob; declare const repos_insertBinding: typeof insertBinding; declare const repos_insertDnsRecord: typeof insertDnsRecord; +declare const repos_insertHealthProbeLog: typeof insertHealthProbeLog; declare const repos_insertNotificationLog: typeof insertNotificationLog; declare const repos_linkBindingRecord: typeof linkBindingRecord; declare const repos_linkGroupDnsRecord: typeof linkGroupDnsRecord; @@ -8767,6 +9454,7 @@ declare const repos_listGroupDnsRecords: typeof listGroupDnsRecords; declare const repos_listGroups: typeof listGroups; declare const repos_listHealthCheckTargets: typeof listHealthCheckTargets; declare const repos_listHealthChecks: typeof listHealthChecks; +declare const repos_listHealthProbeLogForService: typeof listHealthProbeLogForService; declare const repos_listIpHealthByServiceIds: typeof listIpHealthByServiceIds; declare const repos_listIpHealthStatus: typeof listIpHealthStatus; declare const repos_listNodes: typeof listNodes; @@ -8816,7 +9504,7 @@ declare const repos_upsertCertificateCheck: typeof upsertCertificateCheck; declare const repos_upsertIpHealthStatus: typeof upsertIpHealthStatus; declare const repos_upsertSubdomain: typeof upsertSubdomain; declare namespace repos { - export { type repos_BindingIpMeta as BindingIpMeta, type repos_BindingLbPatch as BindingLbPatch, type repos_DnsListFilter as DnsListFilter, type repos_DomainMonitorRow as DomainMonitorRow, type repos_HealthAggregate as HealthAggregate, type repos_ServiceGroupLbPatch as ServiceGroupLbPatch, type repos_ServiceIpHealthRow as ServiceIpHealthRow, type repos_ServiceIpRow as ServiceIpRow, type repos_UpdateSubdomainPatch as UpdateSubdomainPatch, repos_addDomainTags as addDomainTags, repos_aggregateGroupScopeHealthByIds as aggregateGroupScopeHealthByIds, repos_aggregateIpHealthByRefs as aggregateIpHealthByRefs, repos_aggregateIpHealthByServiceIds as aggregateIpHealthByServiceIds, repos_bindingsToRemove as bindingsToRemove, repos_bumpBindingVersion as bumpBindingVersion, repos_countCertificatesByStatus as countCertificatesByStatus, repos_createDomain as createDomain, repos_createDomainMonitor as createDomainMonitor, repos_createGroup as createGroup, repos_createHealthCheck as createHealthCheck, repos_createNode as createNode, repos_createService as createService, repos_createServiceGroup as createServiceGroup, repos_createSubdomain as createSubdomain, repos_createSyncJob as createSyncJob, repos_deleteBinding as deleteBinding, repos_deleteBindingsExcept as deleteBindingsExcept, repos_deleteCertificatesNotIn as deleteCertificatesNotIn, repos_deleteDnsRecord as deleteDnsRecord, repos_deleteDomain as deleteDomain, repos_deleteDomainMonitor as deleteDomainMonitor, repos_deleteGroup as deleteGroup, repos_deleteHealthCheck as deleteHealthCheck, repos_deleteIpHealthStatusForIp as deleteIpHealthStatusForIp, repos_deleteIpHealthStatusForRef as deleteIpHealthStatusForRef, repos_deleteNode as deleteNode, repos_deleteService as deleteService, repos_deleteServiceGroup as deleteServiceGroup, repos_deleteSubdomain as deleteSubdomain, repos_ensureNode as ensureNode, repos_findBinding as findBinding, repos_findDnsByCfId as findDnsByCfId, repos_findDomainByZoneName as findDomainByZoneName, repos_findHealthCheckByCfId as findHealthCheckByCfId, repos_findNodeByAddress as findNodeByAddress, repos_findNodeByIp as findNodeByIp, repos_findSubdomainByDomainAndName as findSubdomainByDomainAndName, repos_finishSyncJob as finishSyncJob, repos_getBinding as getBinding, repos_getBindingView as getBindingView, repos_getCertificate as getCertificate, repos_getDnsRecord as getDnsRecord, repos_getDomain as getDomain, repos_getDomainMonitor as getDomainMonitor, repos_getGroup as getGroup, repos_getGroupWithStats as getGroupWithStats, repos_getHealthCheck as getHealthCheck, repos_getIpHealthStatusRow as getIpHealthStatusRow, repos_getNode as getNode, repos_getService as getService, repos_getServiceGroup as getServiceGroup, repos_getSubdomain as getSubdomain, repos_getSyncJob as getSyncJob, repos_insertBinding as insertBinding, repos_insertDnsRecord as insertDnsRecord, repos_insertNotificationLog as insertNotificationLog, repos_linkBindingRecord as linkBindingRecord, repos_linkGroupDnsRecord as linkGroupDnsRecord, repos_listAllBindings as listAllBindings, repos_listAllDomains as listAllDomains, repos_listAllNodes as listAllNodes, repos_listAllSubdomains as listAllSubdomains, repos_listBindingIps as listBindingIps, repos_listBindingIpsWithMeta as listBindingIpsWithMeta, repos_listBindingNodes as listBindingNodes, repos_listBindingsByDomain as listBindingsByDomain, repos_listBindingsByService as listBindingsByService, repos_listCertificates as listCertificates, repos_listDnsByDomain as listDnsByDomain, repos_listDnsRecords as listDnsRecords, repos_listDomainMonitorResults as listDomainMonitorResults, repos_listDomainMonitorResultsForDomain as listDomainMonitorResultsForDomain, repos_listDomainMonitors as listDomainMonitors, repos_listDomainTags as listDomainTags, repos_listDomains as listDomains, repos_listDomainsEnriched as listDomainsEnriched, repos_listEnabledDomainMonitors as listEnabledDomainMonitors, repos_listGroupDnsRecords as listGroupDnsRecords, repos_listGroups as listGroups, repos_listHealthCheckTargets as listHealthCheckTargets, repos_listHealthChecks as listHealthChecks, repos_listIpHealthByServiceIds as listIpHealthByServiceIds, repos_listIpHealthStatus as listIpHealthStatus, repos_listNodes as listNodes, repos_listNotificationLog as listNotificationLog, repos_listOriginIpsForFqdn as listOriginIpsForFqdn, repos_listRecordsForBinding as listRecordsForBinding, repos_listServiceGroups as listServiceGroups, repos_listServiceIpRows as listServiceIpRows, repos_listServiceIps as listServiceIps, repos_listServices as listServices, repos_listServicesByGroup as listServicesByGroup, repos_listSubdomainsByDomain as listSubdomainsByDomain, repos_listUngroupedServices as listUngroupedServices, repos_markDnsPendingDelete as markDnsPendingDelete, repos_mergeHealthAggregates as mergeHealthAggregates, repos_pruneStaleIpHealthStatus as pruneStaleIpHealthStatus, repos_reorderServices as reorderServices, repos_replaceBindingIps as replaceBindingIps, repos_replaceBindingIpsWithMeta as replaceBindingIpsWithMeta, repos_replaceServiceIps as replaceServiceIps, repos_setBindingCnameTarget as setBindingCnameTarget, repos_setBindingDnsRecordId as setBindingDnsRecordId, repos_setBindingRoutingStrategy as setBindingRoutingStrategy, repos_setDnsSyncStatus as setDnsSyncStatus, repos_setDomainLastSynced as setDomainLastSynced, repos_setDomainTags as setDomainTags, repos_setServiceEnabled as setServiceEnabled, repos_setServiceGroup as setServiceGroup, repos_setServiceGroupEnabled as setServiceGroupEnabled, repos_setServiceIpEnabled as setServiceIpEnabled, repos_setServiceLb as setServiceLb, repos_unlinkBindingRecord as unlinkBindingRecord, repos_unlinkGroupDnsRecord as unlinkGroupDnsRecord, repos_updateBindingDomain as updateBindingDomain, repos_updateBindingFields as updateBindingFields, repos_updateBindingLbConfig as updateBindingLbConfig, repos_updateDnsFields as updateDnsFields, repos_updateDomain as updateDomain, repos_updateDomainMonitorResult as updateDomainMonitorResult, repos_updateGroup as updateGroup, repos_updateHealthCheck as updateHealthCheck, repos_updateNode as updateNode, repos_updateService as updateService, repos_updateServiceGroup as updateServiceGroup, repos_updateSubdomain as updateSubdomain, repos_upsertCertificateCheck as upsertCertificateCheck, repos_upsertIpHealthStatus as upsertIpHealthStatus, repos_upsertSubdomain as upsertSubdomain }; + export { type repos_BindingIpMeta as BindingIpMeta, type repos_BindingLbPatch as BindingLbPatch, type repos_DnsListFilter as DnsListFilter, type repos_DomainMonitorRow as DomainMonitorRow, type repos_HealthAggregate as HealthAggregate, type repos_ServiceGroupLbPatch as ServiceGroupLbPatch, type repos_ServiceIpHealthRow as ServiceIpHealthRow, type repos_ServiceIpRow as ServiceIpRow, type repos_UpdateSubdomainPatch as UpdateSubdomainPatch, repos_addDomainTags as addDomainTags, repos_aggregateGroupScopeHealthByIds as aggregateGroupScopeHealthByIds, repos_aggregateIpHealthByRefs as aggregateIpHealthByRefs, repos_aggregateIpHealthByServiceIds as aggregateIpHealthByServiceIds, repos_bindingsToRemove as bindingsToRemove, repos_bumpBindingVersion as bumpBindingVersion, repos_countCertificatesByStatus as countCertificatesByStatus, repos_createDomain as createDomain, repos_createDomainMonitor as createDomainMonitor, repos_createGroup as createGroup, repos_createHealthCheck as createHealthCheck, repos_createNode as createNode, repos_createService as createService, repos_createServiceGroup as createServiceGroup, repos_createSubdomain as createSubdomain, repos_createSyncJob as createSyncJob, repos_deleteBinding as deleteBinding, repos_deleteBindingsExcept as deleteBindingsExcept, repos_deleteCertificatesNotIn as deleteCertificatesNotIn, repos_deleteDnsRecord as deleteDnsRecord, repos_deleteDomain as deleteDomain, repos_deleteDomainMonitor as deleteDomainMonitor, repos_deleteGroup as deleteGroup, repos_deleteHealthCheck as deleteHealthCheck, repos_deleteIpHealthStatusForIp as deleteIpHealthStatusForIp, repos_deleteIpHealthStatusForRef as deleteIpHealthStatusForRef, repos_deleteNode as deleteNode, repos_deleteService as deleteService, repos_deleteServiceGroup as deleteServiceGroup, repos_deleteSubdomain as deleteSubdomain, repos_ensureNode as ensureNode, repos_findBinding as findBinding, repos_findDnsByCfId as findDnsByCfId, repos_findDomainByZoneName as findDomainByZoneName, repos_findHealthCheckByCfId as findHealthCheckByCfId, repos_findNodeByAddress as findNodeByAddress, repos_findNodeByIp as findNodeByIp, repos_findSubdomainByDomainAndName as findSubdomainByDomainAndName, repos_finishSyncJob as finishSyncJob, repos_getBinding as getBinding, repos_getBindingView as getBindingView, repos_getCertificate as getCertificate, repos_getDnsRecord as getDnsRecord, repos_getDomain as getDomain, repos_getDomainMonitor as getDomainMonitor, repos_getGroup as getGroup, repos_getGroupWithStats as getGroupWithStats, repos_getHealthCheck as getHealthCheck, repos_getIpHealthStatusRow as getIpHealthStatusRow, repos_getNode as getNode, repos_getService as getService, repos_getServiceGroup as getServiceGroup, repos_getSubdomain as getSubdomain, repos_getSyncJob as getSyncJob, repos_insertBinding as insertBinding, repos_insertDnsRecord as insertDnsRecord, repos_insertHealthProbeLog as insertHealthProbeLog, repos_insertNotificationLog as insertNotificationLog, repos_linkBindingRecord as linkBindingRecord, repos_linkGroupDnsRecord as linkGroupDnsRecord, repos_listAllBindings as listAllBindings, repos_listAllDomains as listAllDomains, repos_listAllNodes as listAllNodes, repos_listAllSubdomains as listAllSubdomains, repos_listBindingIps as listBindingIps, repos_listBindingIpsWithMeta as listBindingIpsWithMeta, repos_listBindingNodes as listBindingNodes, repos_listBindingsByDomain as listBindingsByDomain, repos_listBindingsByService as listBindingsByService, repos_listCertificates as listCertificates, repos_listDnsByDomain as listDnsByDomain, repos_listDnsRecords as listDnsRecords, repos_listDomainMonitorResults as listDomainMonitorResults, repos_listDomainMonitorResultsForDomain as listDomainMonitorResultsForDomain, repos_listDomainMonitors as listDomainMonitors, repos_listDomainTags as listDomainTags, repos_listDomains as listDomains, repos_listDomainsEnriched as listDomainsEnriched, repos_listEnabledDomainMonitors as listEnabledDomainMonitors, repos_listGroupDnsRecords as listGroupDnsRecords, repos_listGroups as listGroups, repos_listHealthCheckTargets as listHealthCheckTargets, repos_listHealthChecks as listHealthChecks, repos_listHealthProbeLogForService as listHealthProbeLogForService, repos_listIpHealthByServiceIds as listIpHealthByServiceIds, repos_listIpHealthStatus as listIpHealthStatus, repos_listNodes as listNodes, repos_listNotificationLog as listNotificationLog, repos_listOriginIpsForFqdn as listOriginIpsForFqdn, repos_listRecordsForBinding as listRecordsForBinding, repos_listServiceGroups as listServiceGroups, repos_listServiceIpRows as listServiceIpRows, repos_listServiceIps as listServiceIps, repos_listServices as listServices, repos_listServicesByGroup as listServicesByGroup, repos_listSubdomainsByDomain as listSubdomainsByDomain, repos_listUngroupedServices as listUngroupedServices, repos_markDnsPendingDelete as markDnsPendingDelete, repos_mergeHealthAggregates as mergeHealthAggregates, repos_pruneStaleIpHealthStatus as pruneStaleIpHealthStatus, repos_reorderServices as reorderServices, repos_replaceBindingIps as replaceBindingIps, repos_replaceBindingIpsWithMeta as replaceBindingIpsWithMeta, repos_replaceServiceIps as replaceServiceIps, repos_setBindingCnameTarget as setBindingCnameTarget, repos_setBindingDnsRecordId as setBindingDnsRecordId, repos_setBindingRoutingStrategy as setBindingRoutingStrategy, repos_setDnsSyncStatus as setDnsSyncStatus, repos_setDomainLastSynced as setDomainLastSynced, repos_setDomainTags as setDomainTags, repos_setServiceEnabled as setServiceEnabled, repos_setServiceGroup as setServiceGroup, repos_setServiceGroupEnabled as setServiceGroupEnabled, repos_setServiceIpEnabled as setServiceIpEnabled, repos_setServiceLb as setServiceLb, repos_unlinkBindingRecord as unlinkBindingRecord, repos_unlinkGroupDnsRecord as unlinkGroupDnsRecord, repos_updateBindingDomain as updateBindingDomain, repos_updateBindingFields as updateBindingFields, repos_updateBindingLbConfig as updateBindingLbConfig, repos_updateDnsFields as updateDnsFields, repos_updateDomain as updateDomain, repos_updateDomainMonitorResult as updateDomainMonitorResult, repos_updateGroup as updateGroup, repos_updateHealthCheck as updateHealthCheck, repos_updateNode as updateNode, repos_updateService as updateService, repos_updateServiceGroup as updateServiceGroup, repos_updateSubdomain as updateSubdomain, repos_upsertCertificateCheck as upsertCertificateCheck, repos_upsertIpHealthStatus as upsertIpHealthStatus, repos_upsertSubdomain as upsertSubdomain }; } -export { type AppSettingsDto, type AppSettingsPatch, type AppendAuditInput, ConflictError, type Db, type DnsListFilter, type HealthEngineFallbacks, type HealthEngineSettings, NotFoundError, type Sqlite, type UpdateSubdomainPatch, appSettings, appendAudit, auditLog, bindingNodes, certificates, createDb, createMemoryDb, dnsRecords, domainMonitorResults, domainMonitors, domainTags, domains, getAppSettings, getAppSettingsSecrets, groups, healthCheck, healthChecks, ipHealthStatus, listAudit, nodes, notificationLog, repos, resolveDatabasePath, runMigrations, schema, serviceBindingIps, serviceBindingRecords, serviceBindings, serviceGroupDnsRecords, serviceGroups, serviceIps, services, subdomains, syncJobs, touchVpsTrackerSync, updateAppSettings }; +export { type AppSettingsDto, type AppSettingsPatch, type AppendAuditInput, ConflictError, type Db, type DnsListFilter, type HealthEngineFallbacks, type HealthEngineSettings, NotFoundError, type Sqlite, type UpdateSubdomainPatch, appSettings, appendAudit, auditLog, bindingNodes, certificates, createDb, createMemoryDb, dnsRecords, domainMonitorResults, domainMonitors, domainTags, domains, getAppSettings, getAppSettingsSecrets, groups, healthCheck, healthChecks, healthProbeLog, ipHealthStatus, listAudit, nodes, notificationLog, repos, resolveDatabasePath, runMigrations, schema, serviceBindingIps, serviceBindingRecords, serviceBindings, serviceGroupDnsRecords, serviceGroups, serviceIps, services, subdomains, syncJobs, touchVpsTrackerSync, updateAppSettings }; diff --git a/packages/db/dist/index.js b/packages/db/dist/index.js index d7eb7e8..bf678b5 100644 --- a/packages/db/dist/index.js +++ b/packages/db/dist/index.js @@ -52,6 +52,7 @@ var serviceGroups = sqliteTable("service_groups", { health_check_interval_sec: integer("health_check_interval_sec").notNull().default(30), health_check_timeout_ms: integer("health_check_timeout_ms").notNull().default(3e3), health_check_verify_tls: integer("health_check_verify_tls", { mode: "boolean" }).notNull().default(false), + health_check_provider: text("health_check_provider").notNull().default("local"), created_at: text("created_at").notNull().default(sql`datetime('now')`), updated_at: text("updated_at").notNull().default(sql`datetime('now')`) }); @@ -117,6 +118,7 @@ var serviceBindings = sqliteTable( health_check_verify_tls: integer("health_check_verify_tls", { mode: "boolean" }).notNull().default(false), + health_check_provider: text("health_check_provider").notNull().default("local"), routing_strategy: text("routing_strategy").notNull().default("round_robin"), operation_version: integer("operation_version").notNull().default(0), created_at: text("created_at").notNull().default(sql`datetime('now')`), @@ -252,6 +254,8 @@ var ipHealthStatus = sqliteTable( consecutive_successes: integer("consecutive_successes").notNull().default(0), last_checked_at: text("last_checked_at"), last_error: text("last_error"), + colo: text("colo"), + provider: text("provider").notNull().default("local"), created_at: text("created_at").notNull().default(sql`datetime('now')`), updated_at: text("updated_at").notNull().default(sql`datetime('now')`) }, @@ -275,6 +279,8 @@ var appSettings = sqliteTable("app_settings", { health_down_failures: integer("health_down_failures"), health_latency_warn_ms: integer("health_latency_warn_ms"), health_success_recoveries: integer("health_success_recoveries"), + health_worker_url: text("health_worker_url"), + health_worker_token: text("health_worker_token"), created_at: text("created_at").notNull().default(sql`datetime('now')`), updated_at: text("updated_at").notNull().default(sql`datetime('now')`) }); @@ -311,6 +317,19 @@ var domainMonitorResults = sqliteTable("domain_monitor_results", { error: text("error"), checked_at: text("checked_at").notNull().default(sql`datetime('now')`) }); +var healthProbeLog = sqliteTable("health_probe_log", { + id: integer("id").primaryKey({ autoIncrement: true }), + scope: text("scope").notNull(), + ref_id: integer("ref_id").notNull(), + ip: text("ip").notNull(), + provider: text("provider").notNull(), + status: text("status").notNull(), + ok: integer("ok", { mode: "boolean" }).notNull(), + latency_ms: integer("latency_ms"), + colo: text("colo"), + error: text("error"), + checked_at: text("checked_at").notNull().default(sql`datetime('now')`) +}); var notificationLog = sqliteTable("notification_log", { id: integer("id").primaryKey({ autoIncrement: true }), kind: text("kind").notNull(), @@ -358,6 +377,7 @@ var schema = { domainTags, domainMonitors, domainMonitorResults, + healthProbeLog, notificationLog, auditLog }; @@ -508,7 +528,9 @@ function toDto(row, fallbacks) { healthDegradedFailures: 1, healthDownFailures: 2, healthLatencyWarnMs: 1e3, - healthSuccessRecoveries: 2 + healthSuccessRecoveries: 2, + healthWorkerUrl: "", + healthWorkerTokenSet: false }; return { id: row.id, @@ -535,7 +557,9 @@ function toDto(row, fallbacks) { healthSuccessRecoveries: coalesceInt( row.health_success_recoveries, env.healthSuccessRecoveries - ) + ), + healthWorkerUrl: row.health_worker_url?.trim() || env.healthWorkerUrl, + healthWorkerTokenSet: Boolean(row.health_worker_token?.trim()) || env.healthWorkerTokenSet }; } function getAppSettings(db, fallbacks) { @@ -554,7 +578,9 @@ function getAppSettingsSecrets(db) { return { vpsTrackerUrl: row?.vps_tracker_url?.trim() ?? "", vpsTrackerIntegrationToken: row?.vps_tracker_integration_token?.trim() ?? "", - vpsTrackerSyncEnabled: Boolean(row?.vps_tracker_sync_enabled) + vpsTrackerSyncEnabled: Boolean(row?.vps_tracker_sync_enabled), + healthWorkerUrl: row?.health_worker_url?.trim() ?? "", + healthWorkerToken: row?.health_worker_token?.trim() ?? "" }; } function updateAppSettings(db, patch, fallbacks) { @@ -574,6 +600,8 @@ function updateAppSettings(db, patch, fallbacks) { health_down_failures: patch.healthDownFailures !== void 0 ? patch.healthDownFailures : current.health_down_failures, health_latency_warn_ms: patch.healthLatencyWarnMs !== void 0 ? patch.healthLatencyWarnMs : current.health_latency_warn_ms, health_success_recoveries: patch.healthSuccessRecoveries !== void 0 ? patch.healthSuccessRecoveries : current.health_success_recoveries, + health_worker_url: patch.healthWorkerUrl !== void 0 ? patch.healthWorkerUrl.trim() || null : current.health_worker_url, + health_worker_token: patch.healthWorkerToken !== void 0 && patch.healthWorkerToken.trim() !== "" ? patch.healthWorkerToken : current.health_worker_token, updated_at: (/* @__PURE__ */ new Date()).toISOString() }).where(eq2(appSettings.id, SETTINGS_ID)).run(); return getAppSettings(db, fallbacks); @@ -644,6 +672,7 @@ __export(repos_exports, { getSyncJob: () => getSyncJob, insertBinding: () => insertBinding, insertDnsRecord: () => insertDnsRecord, + insertHealthProbeLog: () => insertHealthProbeLog, insertNotificationLog: () => insertNotificationLog, linkBindingRecord: () => linkBindingRecord, linkGroupDnsRecord: () => linkGroupDnsRecord, @@ -670,6 +699,7 @@ __export(repos_exports, { listGroups: () => listGroups, listHealthCheckTargets: () => listHealthCheckTargets, listHealthChecks: () => listHealthChecks, + listHealthProbeLogForService: () => listHealthProbeLogForService, listIpHealthByServiceIds: () => listIpHealthByServiceIds, listIpHealthStatus: () => listIpHealthStatus, listNodes: () => listNodes, @@ -1106,6 +1136,9 @@ function deleteService(db, id) { const result = db.delete(services).where(eq3(services.id, id)).run(); if (result.changes === 0) throw new NotFoundError(`service ${id}`); } +function normalizeHealthProvider(value) { + return value === "cloudflare" ? "cloudflare" : "local"; +} function mapServiceGroup(row) { return { id: row.id, @@ -1123,6 +1156,7 @@ function mapServiceGroup(row) { health_check_interval_sec: row.health_check_interval_sec, health_check_timeout_ms: row.health_check_timeout_ms, health_check_verify_tls: row.health_check_verify_tls, + health_check_provider: normalizeHealthProvider(row.health_check_provider), created_at: row.created_at, updated_at: row.updated_at }; @@ -1149,7 +1183,8 @@ function createServiceGroup(db, name, groupType, icon, domain, lbPatch) { health_check_expected_status: lbPatch?.health_check_expected_status ?? null, health_check_interval_sec: lbPatch?.health_check_interval_sec ?? 30, health_check_timeout_ms: lbPatch?.health_check_timeout_ms ?? 3e3, - health_check_verify_tls: lbPatch?.health_check_verify_tls ?? false + health_check_verify_tls: lbPatch?.health_check_verify_tls ?? false, + health_check_provider: lbPatch?.health_check_provider ?? "local" }).returning({ id: serviceGroups.id }).get().id; return getServiceGroup(db, id); } @@ -1179,6 +1214,8 @@ function updateServiceGroup(db, id, name, groupType, icon, domain, lbPatch) { update.health_check_timeout_ms = lbPatch.health_check_timeout_ms; if (lbPatch.health_check_verify_tls !== void 0) update.health_check_verify_tls = lbPatch.health_check_verify_tls; + if (lbPatch.health_check_provider !== void 0) + update.health_check_provider = lbPatch.health_check_provider; } const result = db.update(serviceGroups).set(update).where(eq3(serviceGroups.id, id)).run(); if (result.changes === 0) throw new NotFoundError(`service group ${id}`); @@ -1488,6 +1525,8 @@ function updateBindingLbConfig(db, bindingId, patch) { update.health_check_timeout_ms = patch.health_check_timeout_ms; if (patch.health_check_verify_tls !== void 0) update.health_check_verify_tls = patch.health_check_verify_tls; + if (patch.health_check_provider !== void 0) + update.health_check_provider = patch.health_check_provider; db.update(serviceBindings).set(update).where(eq3(serviceBindings.id, bindingId)).run(); } function setBindingCnameTarget(db, bindingId, target) { @@ -1546,7 +1585,7 @@ function dnsRecordMatchesHostname(recordName, hostname, zoneName) { var SERVICE_BINDING_SELECT_COLUMNS = `sb.id, sb.domain_id, sb.service_id, sb.hostname, sb.dns_record_id, sb.lb_mode, sb.health_check_enabled, sb.health_check_type, sb.health_check_port, sb.health_check_path, sb.health_check_expected_status, sb.health_check_interval_sec, - sb.health_check_timeout_ms, sb.health_check_verify_tls, sb.cname_target, + sb.health_check_timeout_ms, sb.health_check_verify_tls, sb.health_check_provider, sb.cname_target, d.zone_name, d.group_id, g.name AS group_name, s.name AS service_name, s.slug AS service_slug, dr.content AS target_ip, dr.sync_status, @@ -1767,7 +1806,7 @@ function finishSyncJob(db, id, status, message) { function listIpHealthStatus(db, scope, refId) { return db.all(sql2` SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures, - last_checked_at, last_error + last_checked_at, last_error, colo, provider FROM ip_health_status WHERE scope = ${scope} AND ref_id = ${refId} `); @@ -1891,7 +1930,11 @@ function listIpHealthByServiceIds(db, serviceIds) { SELECT sb.service_id AS service_id, ihs.ip AS ip, ${WORST_HEALTH_SQL} AS health_status, - MAX(ihs.latency_ms) AS health_latency_ms + MAX(ihs.latency_ms) AS health_latency_ms, + MAX(ihs.last_checked_at) AS last_checked_at, + MAX(ihs.last_error) AS last_error, + MAX(ihs.provider) AS provider, + MAX(ihs.colo) AS colo FROM ip_health_status ihs INNER JOIN service_bindings sb ON ihs.scope = 'binding' AND ihs.ref_id = sb.id @@ -1907,7 +1950,11 @@ function listIpHealthByServiceIds(db, serviceIds) { list.push({ ip: row.ip, status: parsed.health_status, - latency_ms: parsed.health_latency_ms + latency_ms: parsed.health_latency_ms, + last_checked_at: row.last_checked_at, + last_error: row.last_error, + provider: normalizeHealthProvider(row.provider), + colo: row.colo }); result.set(row.service_id, list); } @@ -1942,20 +1989,24 @@ function mergeHealthAggregates(parts) { function getIpHealthStatusRow(db, scope, refId, ip) { const rows = db.all(sql2` SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures, - consecutive_successes, last_checked_at, last_error + consecutive_successes, last_checked_at, last_error, colo, provider FROM ip_health_status WHERE scope = ${scope} AND ref_id = ${refId} AND ip = ${ip} LIMIT 1 `); return rows[0] ?? null; } -function upsertIpHealthStatus(db, scope, refId, ip, status, latencyMs, consecutiveFailures, lastError, consecutiveSuccesses = 0) { +function upsertIpHealthStatus(db, scope, refId, ip, status, latencyMs, consecutiveFailures, lastError, consecutiveSuccesses = 0, extras) { + const colo = extras?.colo ?? null; + const provider = extras?.provider ?? "local"; db.run(sql2` INSERT INTO ip_health_status (scope, ref_id, ip, status, latency_ms, consecutive_failures, - consecutive_successes, last_checked_at, last_error, created_at, updated_at) + consecutive_successes, last_checked_at, last_error, colo, provider, + created_at, updated_at) VALUES (${scope}, ${refId}, ${ip}, ${status}, ${latencyMs}, ${consecutiveFailures}, - ${consecutiveSuccesses}, datetime('now'), ${lastError}, datetime('now'), datetime('now')) + ${consecutiveSuccesses}, datetime('now'), ${lastError}, ${colo}, ${provider}, + datetime('now'), datetime('now')) ON CONFLICT(scope, ref_id, ip) DO UPDATE SET status = excluded.status, latency_ms = excluded.latency_ms, @@ -1963,6 +2014,8 @@ function upsertIpHealthStatus(db, scope, refId, ip, status, latencyMs, consecuti consecutive_successes = excluded.consecutive_successes, last_checked_at = excluded.last_checked_at, last_error = excluded.last_error, + colo = excluded.colo, + provider = excluded.provider, updated_at = datetime('now') `); } @@ -2025,7 +2078,8 @@ function listHealthCheckTargets(db) { sb.health_check_path AS path, sb.health_check_expected_status AS expected_status, sb.health_check_timeout_ms AS timeout_ms, - sb.health_check_verify_tls AS verify_tls + sb.health_check_verify_tls AS verify_tls, + COALESCE(sb.health_check_provider, 'local') AS provider FROM service_binding_ips sbi JOIN service_bindings sb ON sb.id = sbi.binding_id JOIN domains d ON d.id = sb.domain_id @@ -2039,7 +2093,8 @@ function listHealthCheckTargets(db) { sg.health_check_path AS path, sg.health_check_expected_status AS expected_status, sg.health_check_timeout_ms AS timeout_ms, - sg.health_check_verify_tls AS verify_tls + sg.health_check_verify_tls AS verify_tls, + COALESCE(sg.health_check_provider, 'local') AS provider FROM service_binding_ips sbi JOIN service_bindings sb ON sb.id = sbi.binding_id JOIN services s ON s.id = sb.service_id @@ -2059,7 +2114,8 @@ function listHealthCheckTargets(db) { sg.health_check_path AS path, sg.health_check_expected_status AS expected_status, sg.health_check_timeout_ms AS timeout_ms, - sg.health_check_verify_tls AS verify_tls + sg.health_check_verify_tls AS verify_tls, + COALESCE(sg.health_check_provider, 'local') AS provider FROM service_binding_ips sbi JOIN service_bindings sb ON sb.id = sbi.binding_id JOIN domains d ON d.id = sb.domain_id @@ -2079,7 +2135,8 @@ function listHealthCheckTargets(db) { sb.health_check_path AS path, sb.health_check_expected_status AS expected_status, sb.health_check_timeout_ms AS timeout_ms, - sb.health_check_verify_tls AS verify_tls + sb.health_check_verify_tls AS verify_tls, + COALESCE(sb.health_check_provider, 'local') AS provider FROM service_bindings sb JOIN domains d ON d.id = sb.domain_id JOIN services s ON s.id = sb.service_id @@ -2096,7 +2153,8 @@ function listHealthCheckTargets(db) { sg.health_check_path AS path, sg.health_check_expected_status AS expected_status, sg.health_check_timeout_ms AS timeout_ms, - sg.health_check_verify_tls AS verify_tls + sg.health_check_verify_tls AS verify_tls, + COALESCE(sg.health_check_provider, 'local') AS provider FROM service_bindings sb JOIN domains d ON d.id = sb.domain_id JOIN services s ON s.id = sb.service_id @@ -2117,7 +2175,8 @@ function listHealthCheckTargets(db) { ...groupInheritedCnameBindingTargets ].map((t) => ({ ...t, - verify_tls: Boolean(t.verify_tls) + verify_tls: Boolean(t.verify_tls), + provider: normalizeHealthProvider(t.provider) })); } function listDomainTags(db, domainId) { @@ -2212,6 +2271,46 @@ function listDomainMonitorResultsForDomain(db, domainId, limit = 50) { LIMIT ${limit} `); } +var HEALTH_PROBE_LOG_KEEP = 50; +function insertHealthProbeLog(db, entry) { + db.insert(healthProbeLog).values({ + scope: entry.scope, + ref_id: entry.refId, + ip: entry.ip, + provider: entry.provider, + status: entry.status, + ok: entry.ok, + latency_ms: entry.latencyMs, + colo: entry.colo, + error: entry.error + }).run(); + db.run(sql2` + DELETE FROM health_probe_log + WHERE id IN ( + SELECT id FROM health_probe_log + WHERE scope = ${entry.scope} AND ref_id = ${entry.refId} AND ip = ${entry.ip} + ORDER BY checked_at DESC, id DESC + LIMIT -1 OFFSET ${HEALTH_PROBE_LOG_KEEP} + ) + `); +} +function listHealthProbeLogForService(db, serviceId, limit = 50) { + const rows = db.all(sql2` + SELECT l.id, l.scope, l.ref_id, l.ip, l.provider, l.status, l.ok, + l.latency_ms, l.colo, l.error, l.checked_at + FROM health_probe_log l + INNER JOIN service_bindings sb + ON l.scope = 'binding' AND l.ref_id = sb.id + WHERE sb.service_id = ${serviceId} + ORDER BY l.checked_at DESC, l.id DESC + LIMIT ${limit} + `); + return rows.map((row) => ({ + ...row, + provider: normalizeHealthProvider(row.provider), + ok: Boolean(row.ok) + })); +} function insertNotificationLog(db, kind, refType, refId, title, message) { db.insert(notificationLog).values({ kind, @@ -2249,6 +2348,7 @@ export { groups, healthCheck, healthChecks, + healthProbeLog, ipHealthStatus, listAudit, nodes, diff --git a/packages/db/migrations/021_health_worker_xor.sql b/packages/db/migrations/021_health_worker_xor.sql new file mode 100644 index 0000000..1db4cc1 --- /dev/null +++ b/packages/db/migrations/021_health_worker_xor.sql @@ -0,0 +1,27 @@ +-- XOR health-check: persist provider on bindings/groups; Worker URL/token; +-- colo + probe journal. Cloudflare = Worker edge probe, not Health Checks API. +ALTER TABLE service_bindings ADD COLUMN health_check_provider TEXT NOT NULL DEFAULT 'local'; +ALTER TABLE service_groups ADD COLUMN health_check_provider TEXT NOT NULL DEFAULT 'local'; + +ALTER TABLE app_settings ADD COLUMN health_worker_url TEXT; +ALTER TABLE app_settings ADD COLUMN health_worker_token TEXT; + +ALTER TABLE ip_health_status ADD COLUMN colo TEXT; +ALTER TABLE ip_health_status ADD COLUMN provider TEXT NOT NULL DEFAULT 'local'; + +CREATE TABLE IF NOT EXISTS health_probe_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + scope TEXT NOT NULL, + ref_id INTEGER NOT NULL, + ip TEXT NOT NULL, + provider TEXT NOT NULL, + status TEXT NOT NULL, + ok INTEGER NOT NULL, + latency_ms INTEGER, + colo TEXT, + error TEXT, + checked_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_health_probe_log_target + ON health_probe_log(scope, ref_id, ip, checked_at DESC); diff --git a/packages/db/src/repos.ts b/packages/db/src/repos.ts index cb5f8e4..7eaee10 100644 --- a/packages/db/src/repos.ts +++ b/packages/db/src/repos.ts @@ -5,6 +5,7 @@ import type { DomainListItem, Group, GroupWithStats, + HealthCheckProvider, HealthCheckScope, HealthCheckTarget, HealthCheckType, @@ -33,6 +34,7 @@ import { domains, groups, healthChecks, + healthProbeLog, ipHealthStatus, nodes, bindingNodes, @@ -768,6 +770,10 @@ export function deleteService(db: Db, id: number): void { // --- Service Groups --- +function normalizeHealthProvider(value: unknown): HealthCheckProvider { + return value === "cloudflare" ? "cloudflare" : "local"; +} + function mapServiceGroup(row: typeof serviceGroups.$inferSelect): ServiceGroup { return { id: row.id, @@ -785,6 +791,7 @@ function mapServiceGroup(row: typeof serviceGroups.$inferSelect): ServiceGroup { health_check_interval_sec: row.health_check_interval_sec, health_check_timeout_ms: row.health_check_timeout_ms, health_check_verify_tls: row.health_check_verify_tls, + health_check_provider: normalizeHealthProvider(row.health_check_provider), created_at: row.created_at, updated_at: row.updated_at, }; @@ -819,6 +826,7 @@ export interface ServiceGroupLbPatch { health_check_interval_sec?: number; health_check_timeout_ms?: number; health_check_verify_tls?: boolean; + health_check_provider?: HealthCheckProvider; } export function createServiceGroup( @@ -845,6 +853,7 @@ export function createServiceGroup( health_check_interval_sec: lbPatch?.health_check_interval_sec ?? 30, health_check_timeout_ms: lbPatch?.health_check_timeout_ms ?? 3000, health_check_verify_tls: lbPatch?.health_check_verify_tls ?? false, + health_check_provider: lbPatch?.health_check_provider ?? "local", }) .returning({ id: serviceGroups.id }) .get()!.id; @@ -885,6 +894,8 @@ export function updateServiceGroup( update.health_check_timeout_ms = lbPatch.health_check_timeout_ms; if (lbPatch.health_check_verify_tls !== undefined) update.health_check_verify_tls = lbPatch.health_check_verify_tls; + if (lbPatch.health_check_provider !== undefined) + update.health_check_provider = lbPatch.health_check_provider; } const result = db .update(serviceGroups) @@ -1427,6 +1438,7 @@ export interface BindingLbPatch { health_check_interval_sec?: number; health_check_timeout_ms?: number; health_check_verify_tls?: boolean; + health_check_provider?: HealthCheckProvider; } export function updateBindingLbConfig( @@ -1457,6 +1469,8 @@ export function updateBindingLbConfig( update.health_check_timeout_ms = patch.health_check_timeout_ms; if (patch.health_check_verify_tls !== undefined) update.health_check_verify_tls = patch.health_check_verify_tls; + if (patch.health_check_provider !== undefined) + update.health_check_provider = patch.health_check_provider; db.update(serviceBindings) .set(update) .where(eq(serviceBindings.id, bindingId)) @@ -1564,7 +1578,7 @@ function dnsRecordMatchesHostname( const SERVICE_BINDING_SELECT_COLUMNS = `sb.id, sb.domain_id, sb.service_id, sb.hostname, sb.dns_record_id, sb.lb_mode, sb.health_check_enabled, sb.health_check_type, sb.health_check_port, sb.health_check_path, sb.health_check_expected_status, sb.health_check_interval_sec, - sb.health_check_timeout_ms, sb.health_check_verify_tls, sb.cname_target, + sb.health_check_timeout_ms, sb.health_check_verify_tls, sb.health_check_provider, sb.cname_target, d.zone_name, d.group_id, g.name AS group_name, s.name AS service_name, s.slug AS service_slug, dr.content AS target_ip, dr.sync_status, @@ -1943,7 +1957,7 @@ export function listIpHealthStatus( return db .all(sql` SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures, - last_checked_at, last_error + last_checked_at, last_error, colo, provider FROM ip_health_status WHERE scope = ${scope} AND ref_id = ${refId} `); @@ -2108,6 +2122,10 @@ export type ServiceIpHealthRow = { ip: string; status: IpHealthState; latency_ms: number | null; + last_checked_at: string | null; + last_error: string | null; + provider: HealthCheckProvider; + colo: string | null; }; /** Per-IP binding-scope health, worst status if the same IP is on several bindings. */ @@ -2126,11 +2144,19 @@ export function listIpHealthByServiceIds( ip: string; health_status: string | null; health_latency_ms: number | null; + last_checked_at: string | null; + last_error: string | null; + provider: string | null; + colo: string | null; }>(sql` SELECT sb.service_id AS service_id, ihs.ip AS ip, ${WORST_HEALTH_SQL} AS health_status, - MAX(ihs.latency_ms) AS health_latency_ms + MAX(ihs.latency_ms) AS health_latency_ms, + MAX(ihs.last_checked_at) AS last_checked_at, + MAX(ihs.last_error) AS last_error, + MAX(ihs.provider) AS provider, + MAX(ihs.colo) AS colo FROM ip_health_status ihs INNER JOIN service_bindings sb ON ihs.scope = 'binding' AND ihs.ref_id = sb.id @@ -2147,6 +2173,10 @@ export function listIpHealthByServiceIds( ip: row.ip, status: parsed.health_status, latency_ms: parsed.health_latency_ms, + last_checked_at: row.last_checked_at, + last_error: row.last_error, + provider: normalizeHealthProvider(row.provider), + colo: row.colo, }); result.set(row.service_id, list); } @@ -2195,7 +2225,7 @@ export function getIpHealthStatusRow( ): IpHealthStatus | null { const rows = db.all(sql` SELECT scope, ref_id, ip, status, latency_ms, consecutive_failures, - consecutive_successes, last_checked_at, last_error + consecutive_successes, last_checked_at, last_error, colo, provider FROM ip_health_status WHERE scope = ${scope} AND ref_id = ${refId} AND ip = ${ip} LIMIT 1 @@ -2213,13 +2243,18 @@ export function upsertIpHealthStatus( consecutiveFailures: number, lastError: string | null, consecutiveSuccesses = 0, + extras?: { colo?: string | null; provider?: HealthCheckProvider }, ): void { + const colo = extras?.colo ?? null; + const provider = extras?.provider ?? "local"; db.run(sql` INSERT INTO ip_health_status (scope, ref_id, ip, status, latency_ms, consecutive_failures, - consecutive_successes, last_checked_at, last_error, created_at, updated_at) + consecutive_successes, last_checked_at, last_error, colo, provider, + created_at, updated_at) VALUES (${scope}, ${refId}, ${ip}, ${status}, ${latencyMs}, ${consecutiveFailures}, - ${consecutiveSuccesses}, datetime('now'), ${lastError}, datetime('now'), datetime('now')) + ${consecutiveSuccesses}, datetime('now'), ${lastError}, ${colo}, ${provider}, + datetime('now'), datetime('now')) ON CONFLICT(scope, ref_id, ip) DO UPDATE SET status = excluded.status, latency_ms = excluded.latency_ms, @@ -2227,6 +2262,8 @@ export function upsertIpHealthStatus( consecutive_successes = excluded.consecutive_successes, last_checked_at = excluded.last_checked_at, last_error = excluded.last_error, + colo = excluded.colo, + provider = excluded.provider, updated_at = datetime('now') `); } @@ -2321,7 +2358,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] { sb.health_check_path AS path, sb.health_check_expected_status AS expected_status, sb.health_check_timeout_ms AS timeout_ms, - sb.health_check_verify_tls AS verify_tls + sb.health_check_verify_tls AS verify_tls, + COALESCE(sb.health_check_provider, 'local') AS provider FROM service_binding_ips sbi JOIN service_bindings sb ON sb.id = sbi.binding_id JOIN domains d ON d.id = sb.domain_id @@ -2339,7 +2377,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] { sg.health_check_path AS path, sg.health_check_expected_status AS expected_status, sg.health_check_timeout_ms AS timeout_ms, - sg.health_check_verify_tls AS verify_tls + sg.health_check_verify_tls AS verify_tls, + COALESCE(sg.health_check_provider, 'local') AS provider FROM service_binding_ips sbi JOIN service_bindings sb ON sb.id = sbi.binding_id JOIN services s ON s.id = sb.service_id @@ -2364,7 +2403,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] { sg.health_check_path AS path, sg.health_check_expected_status AS expected_status, sg.health_check_timeout_ms AS timeout_ms, - sg.health_check_verify_tls AS verify_tls + sg.health_check_verify_tls AS verify_tls, + COALESCE(sg.health_check_provider, 'local') AS provider FROM service_binding_ips sbi JOIN service_bindings sb ON sb.id = sbi.binding_id JOIN domains d ON d.id = sb.domain_id @@ -2386,7 +2426,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] { sb.health_check_path AS path, sb.health_check_expected_status AS expected_status, sb.health_check_timeout_ms AS timeout_ms, - sb.health_check_verify_tls AS verify_tls + sb.health_check_verify_tls AS verify_tls, + COALESCE(sb.health_check_provider, 'local') AS provider FROM service_bindings sb JOIN domains d ON d.id = sb.domain_id JOIN services s ON s.id = sb.service_id @@ -2406,7 +2447,8 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] { sg.health_check_path AS path, sg.health_check_expected_status AS expected_status, sg.health_check_timeout_ms AS timeout_ms, - sg.health_check_verify_tls AS verify_tls + sg.health_check_verify_tls AS verify_tls, + COALESCE(sg.health_check_provider, 'local') AS provider FROM service_bindings sb JOIN domains d ON d.id = sb.domain_id JOIN services s ON s.id = sb.service_id @@ -2429,6 +2471,7 @@ export function listHealthCheckTargets(db: Db): HealthCheckTarget[] { ].map((t) => ({ ...t, verify_tls: Boolean(t.verify_tls), + provider: normalizeHealthProvider(t.provider), })); } @@ -2633,6 +2676,94 @@ export function listDomainMonitorResultsForDomain( `); } +// --- Health probe journal --- + +const HEALTH_PROBE_LOG_KEEP = 50; + +export function insertHealthProbeLog( + db: Db, + entry: { + scope: HealthCheckScope; + refId: number; + ip: string; + provider: HealthCheckProvider; + status: string; + ok: boolean; + latencyMs: number | null; + colo: string | null; + error: string | null; + }, +): void { + db.insert(healthProbeLog) + .values({ + scope: entry.scope, + ref_id: entry.refId, + ip: entry.ip, + provider: entry.provider, + status: entry.status, + ok: entry.ok, + latency_ms: entry.latencyMs, + colo: entry.colo, + error: entry.error, + }) + .run(); + db.run(sql` + DELETE FROM health_probe_log + WHERE id IN ( + SELECT id FROM health_probe_log + WHERE scope = ${entry.scope} AND ref_id = ${entry.refId} AND ip = ${entry.ip} + ORDER BY checked_at DESC, id DESC + LIMIT -1 OFFSET ${HEALTH_PROBE_LOG_KEEP} + ) + `); +} + +export function listHealthProbeLogForService( + db: Db, + serviceId: number, + limit = 50, +): { + id: number; + scope: string; + ref_id: number; + ip: string; + provider: HealthCheckProvider; + status: string; + ok: boolean; + latency_ms: number | null; + colo: string | null; + error: string | null; + checked_at: string; +}[] { + const rows = db.all<{ + id: number; + scope: string; + ref_id: number; + ip: string; + provider: string; + status: string; + ok: number; + latency_ms: number | null; + colo: string | null; + error: string | null; + checked_at: string; + }>(sql` + SELECT l.id, l.scope, l.ref_id, l.ip, l.provider, l.status, l.ok, + l.latency_ms, l.colo, l.error, l.checked_at + FROM health_probe_log l + INNER JOIN service_bindings sb + ON l.scope = 'binding' AND l.ref_id = sb.id + WHERE sb.service_id = ${serviceId} + ORDER BY l.checked_at DESC, l.id DESC + LIMIT ${limit} + `); + return rows.map((row) => ({ + ...row, + provider: normalizeHealthProvider(row.provider), + ok: Boolean(row.ok), + })); +} + // --- Notification log --- export function insertNotificationLog( diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 32b8184..bfc50d7 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -64,6 +64,7 @@ export const serviceGroups = sqliteTable("service_groups", { health_check_verify_tls: integer("health_check_verify_tls", { mode: "boolean" }) .notNull() .default(false), + health_check_provider: text("health_check_provider").notNull().default("local"), created_at: text("created_at") .notNull() .default(sql`datetime('now')`), @@ -165,6 +166,9 @@ export const serviceBindings = sqliteTable( }) .notNull() .default(false), + health_check_provider: text("health_check_provider") + .notNull() + .default("local"), routing_strategy: text("routing_strategy").notNull().default("round_robin"), operation_version: integer("operation_version").notNull().default(0), created_at: text("created_at") @@ -354,6 +358,8 @@ export const ipHealthStatus = sqliteTable( .default(0), last_checked_at: text("last_checked_at"), last_error: text("last_error"), + colo: text("colo"), + provider: text("provider").notNull().default("local"), created_at: text("created_at") .notNull() .default(sql`datetime('now')`), @@ -385,6 +391,8 @@ export const appSettings = sqliteTable("app_settings", { health_down_failures: integer("health_down_failures"), health_latency_warn_ms: integer("health_latency_warn_ms"), health_success_recoveries: integer("health_success_recoveries"), + health_worker_url: text("health_worker_url"), + health_worker_token: text("health_worker_token"), created_at: text("created_at") .notNull() .default(sql`datetime('now')`), @@ -441,6 +449,22 @@ export const domainMonitorResults = sqliteTable("domain_monitor_results", { .default(sql`datetime('now')`), }); +export const healthProbeLog = sqliteTable("health_probe_log", { + id: integer("id").primaryKey({ autoIncrement: true }), + scope: text("scope").notNull(), + ref_id: integer("ref_id").notNull(), + ip: text("ip").notNull(), + provider: text("provider").notNull(), + status: text("status").notNull(), + ok: integer("ok", { mode: "boolean" }).notNull(), + latency_ms: integer("latency_ms"), + colo: text("colo"), + error: text("error"), + checked_at: text("checked_at") + .notNull() + .default(sql`datetime('now')`), +}); + export const notificationLog = sqliteTable("notification_log", { id: integer("id").primaryKey({ autoIncrement: true }), kind: text("kind").notNull(), @@ -494,6 +518,7 @@ export const schema = { domainTags, domainMonitors, domainMonitorResults, + healthProbeLog, notificationLog, auditLog, }; diff --git a/packages/db/src/settings-repo.ts b/packages/db/src/settings-repo.ts index 3afac87..8aa8e29 100644 --- a/packages/db/src/settings-repo.ts +++ b/packages/db/src/settings-repo.ts @@ -19,6 +19,8 @@ export type AppSettingsDto = { vpsTrackerSyncEnabled: boolean; vpsTrackerLastSyncAt: string | null; showQuickActions: boolean; + healthWorkerUrl: string; + healthWorkerTokenSet: boolean; } & HealthEngineSettings; export type AppSettingsPatch = { @@ -31,9 +33,14 @@ export type AppSettingsPatch = { healthDownFailures?: number; healthLatencyWarnMs?: number; healthSuccessRecoveries?: number; + healthWorkerUrl?: string; + healthWorkerToken?: string; }; -export type HealthEngineFallbacks = HealthEngineSettings; +export type HealthEngineFallbacks = HealthEngineSettings & { + healthWorkerUrl: string; + healthWorkerTokenSet: boolean; +}; function coalesceInt(value: number | null | undefined, fallback: number): number { return value == null || Number.isNaN(value) || value < 1 ? fallback : value; @@ -49,6 +56,8 @@ function toDto( healthDownFailures: 2, healthLatencyWarnMs: 1000, healthSuccessRecoveries: 2, + healthWorkerUrl: "", + healthWorkerTokenSet: false, }; return { id: row.id, @@ -77,6 +86,9 @@ function toDto( row.health_success_recoveries, env.healthSuccessRecoveries, ), + healthWorkerUrl: row.health_worker_url?.trim() || env.healthWorkerUrl, + healthWorkerTokenSet: + Boolean(row.health_worker_token?.trim()) || env.healthWorkerTokenSet, }; } @@ -103,6 +115,8 @@ export function getAppSettingsSecrets(db: Db): { vpsTrackerUrl: string; vpsTrackerIntegrationToken: string; vpsTrackerSyncEnabled: boolean; + healthWorkerUrl: string; + healthWorkerToken: string; } { const row = db .select() @@ -114,6 +128,8 @@ export function getAppSettingsSecrets(db: Db): { vpsTrackerIntegrationToken: row?.vps_tracker_integration_token?.trim() ?? "", vpsTrackerSyncEnabled: Boolean(row?.vps_tracker_sync_enabled), + healthWorkerUrl: row?.health_worker_url?.trim() ?? "", + healthWorkerToken: row?.health_worker_token?.trim() ?? "", }; } @@ -176,6 +192,15 @@ export function updateAppSettings( patch.healthSuccessRecoveries !== undefined ? patch.healthSuccessRecoveries : current.health_success_recoveries, + health_worker_url: + patch.healthWorkerUrl !== undefined + ? patch.healthWorkerUrl.trim() || null + : current.health_worker_url, + health_worker_token: + patch.healthWorkerToken !== undefined && + patch.healthWorkerToken.trim() !== "" + ? patch.healthWorkerToken + : current.health_worker_token, updated_at: new Date().toISOString(), }) .where(eq(appSettings.id, SETTINGS_ID)) diff --git a/packages/shared/dist/index.d.ts b/packages/shared/dist/index.d.ts index ae6fe3a..f01f8c5 100644 --- a/packages/shared/dist/index.d.ts +++ b/packages/shared/dist/index.d.ts @@ -31,6 +31,7 @@ interface ServiceGroup$1 { health_check_interval_sec: number; health_check_timeout_ms: number; health_check_verify_tls: boolean; + health_check_provider: HealthCheckProvider; created_at: string; updated_at: string; } @@ -60,6 +61,7 @@ interface ServiceBinding { health_check_interval_sec: number; health_check_timeout_ms: number; health_check_verify_tls: boolean; + health_check_provider: HealthCheckProvider; routing_strategy: LbMode; operation_version: number; created_at: string; @@ -90,6 +92,7 @@ interface ServiceBindingView { health_check_interval_sec: number; health_check_timeout_ms: number; health_check_verify_tls: boolean; + health_check_provider: HealthCheckProvider; sync_status: string | null; created_at: string; updated_at: string; @@ -114,6 +117,7 @@ interface ServiceDomainBindingView { health_check_interval_sec: number; health_check_timeout_ms: number; health_check_verify_tls: boolean; + health_check_provider: HealthCheckProvider; sync_status: string | null; } interface ServiceView$1 { @@ -193,11 +197,17 @@ interface IpHealthStatus { consecutive_successes?: number; last_checked_at: string | null; last_error: string | null; + colo?: string | null; + provider?: HealthCheckProvider; } interface ServiceIpHealth$1 { ip: string; status: IpHealthState; latency_ms: number | null; + last_checked_at?: string | null; + last_error?: string | null; + provider?: HealthCheckProvider; + colo?: string | null; } interface ServiceNode { id: number; @@ -275,6 +285,7 @@ interface HealthCheckTarget { expected_status: number | null; timeout_ms: number; verify_tls: boolean; + provider: HealthCheckProvider; } declare class ValidationError extends Error { @@ -378,6 +389,11 @@ declare const ipHealthStatusSchema: z.ZodObject<{ consecutive_successes: z.ZodDefault>; last_checked_at: z.ZodNullable; last_error: z.ZodNullable; + colo: z.ZodOptional>; + provider: z.ZodOptional>; }, z.core.$strip>; declare const serviceIpHealthSchema: z.ZodObject<{ ip: z.ZodString; @@ -388,8 +404,37 @@ declare const serviceIpHealthSchema: z.ZodObject<{ degraded: "degraded"; }>; latency_ms: z.ZodNullable; + last_checked_at: z.ZodOptional>; + last_error: z.ZodOptional>; + provider: z.ZodOptional>; + colo: z.ZodOptional>; }, z.core.$strip>; type ServiceIpHealth = z.infer; +declare const healthProbeLogSchema: z.ZodObject<{ + id: z.ZodNumber; + scope: z.ZodString; + ref_id: z.ZodNumber; + ip: z.ZodString; + provider: z.ZodEnum<{ + local: "local"; + cloudflare: "cloudflare"; + }>; + status: z.ZodEnum<{ + unknown: "unknown"; + up: "up"; + down: "down"; + degraded: "degraded"; + }>; + ok: z.ZodCoercedBoolean; + latency_ms: z.ZodNullable; + colo: z.ZodNullable; + error: z.ZodNullable; + checked_at: z.ZodString; +}, z.core.$strip>; +type HealthProbeLog = z.infer; declare const groupSchema: z.ZodObject<{ id: z.ZodNumber; name: z.ZodString; @@ -443,6 +488,10 @@ declare const serviceGroupSchema: z.ZodObject<{ health_check_interval_sec: z.ZodDefault; health_check_timeout_ms: z.ZodDefault; health_check_verify_tls: z.ZodDefault>; + health_check_provider: z.ZodCatch>; created_at: z.ZodString; updated_at: z.ZodString; }, z.core.$strip>; @@ -492,6 +541,10 @@ declare const serviceDomainBindingSchema: z.ZodPipe; health_check_timeout_ms: z.ZodDefault; health_check_verify_tls: z.ZodDefault>; + health_check_provider: z.ZodCatch>; sync_status: z.ZodDefault>; }, z.core.$strip>, z.ZodTransform<{ target_ips: string[]; @@ -513,6 +566,7 @@ declare const serviceDomainBindingSchema: z.ZodPipe; health_check_timeout_ms: z.ZodDefault; health_check_verify_tls: z.ZodDefault>; + health_check_provider: z.ZodCatch>; sync_status: z.ZodDefault>; }, z.core.$strip>, z.ZodTransform<{ target_ips: string[]; @@ -605,6 +664,7 @@ declare const serviceViewSchema: z.ZodObject<{ health_check_interval_sec: number; health_check_timeout_ms: number; health_check_verify_tls: boolean; + health_check_provider: "local" | "cloudflare"; sync_status: string | null; target_ip?: string | null | undefined; }, { @@ -623,6 +683,7 @@ declare const serviceViewSchema: z.ZodObject<{ health_check_interval_sec: number; health_check_timeout_ms: number; health_check_verify_tls: boolean; + health_check_provider: "local" | "cloudflare"; sync_status: string | null; target_ips?: string[] | undefined; target_ip?: string | null | undefined; @@ -646,6 +707,13 @@ declare const serviceViewSchema: z.ZodObject<{ degraded: "degraded"; }>; latency_ms: z.ZodNullable; + last_checked_at: z.ZodOptional>; + last_error: z.ZodOptional>; + provider: z.ZodOptional>; + colo: z.ZodOptional>; }, z.core.$strip>>>; ip_enabled: z.ZodDefault>; }, z.core.$strip>; @@ -680,6 +748,10 @@ declare const serviceGroupViewSchema: z.ZodObject<{ health_check_interval_sec: z.ZodDefault; health_check_timeout_ms: z.ZodDefault; health_check_verify_tls: z.ZodDefault>; + health_check_provider: z.ZodCatch>; created_at: z.ZodString; updated_at: z.ZodString; services: z.ZodDefault; health_check_timeout_ms: z.ZodDefault; health_check_verify_tls: z.ZodDefault>; + health_check_provider: z.ZodCatch>; sync_status: z.ZodDefault>; }, z.core.$strip>, z.ZodTransform<{ target_ips: string[]; @@ -749,6 +825,7 @@ declare const serviceGroupViewSchema: z.ZodObject<{ health_check_interval_sec: number; health_check_timeout_ms: number; health_check_verify_tls: boolean; + health_check_provider: "local" | "cloudflare"; sync_status: string | null; target_ip?: string | null | undefined; }, { @@ -767,6 +844,7 @@ declare const serviceGroupViewSchema: z.ZodObject<{ health_check_interval_sec: number; health_check_timeout_ms: number; health_check_verify_tls: boolean; + health_check_provider: "local" | "cloudflare"; sync_status: string | null; target_ips?: string[] | undefined; target_ip?: string | null | undefined; @@ -790,6 +868,13 @@ declare const serviceGroupViewSchema: z.ZodObject<{ degraded: "degraded"; }>; latency_ms: z.ZodNullable; + last_checked_at: z.ZodOptional>; + last_error: z.ZodOptional>; + provider: z.ZodOptional>; + colo: z.ZodOptional>; }, z.core.$strip>>>; ip_enabled: z.ZodDefault>; }, z.core.$strip>>>; @@ -833,6 +918,10 @@ declare const serviceGroupsResponseSchema: z.ZodObject<{ health_check_interval_sec: z.ZodDefault; health_check_timeout_ms: z.ZodDefault; health_check_verify_tls: z.ZodDefault>; + health_check_provider: z.ZodCatch>; created_at: z.ZodString; updated_at: z.ZodString; services: z.ZodDefault; health_check_timeout_ms: z.ZodDefault; health_check_verify_tls: z.ZodDefault>; + health_check_provider: z.ZodCatch>; sync_status: z.ZodDefault>; }, z.core.$strip>, z.ZodTransform<{ target_ips: string[]; @@ -902,6 +995,7 @@ declare const serviceGroupsResponseSchema: z.ZodObject<{ health_check_interval_sec: number; health_check_timeout_ms: number; health_check_verify_tls: boolean; + health_check_provider: "local" | "cloudflare"; sync_status: string | null; target_ip?: string | null | undefined; }, { @@ -920,6 +1014,7 @@ declare const serviceGroupsResponseSchema: z.ZodObject<{ health_check_interval_sec: number; health_check_timeout_ms: number; health_check_verify_tls: boolean; + health_check_provider: "local" | "cloudflare"; sync_status: string | null; target_ips?: string[] | undefined; target_ip?: string | null | undefined; @@ -943,6 +1038,13 @@ declare const serviceGroupsResponseSchema: z.ZodObject<{ degraded: "degraded"; }>; latency_ms: z.ZodNullable; + last_checked_at: z.ZodOptional>; + last_error: z.ZodOptional>; + provider: z.ZodOptional>; + colo: z.ZodOptional>; }, z.core.$strip>>>; ip_enabled: z.ZodDefault>; }, z.core.$strip>>>; @@ -1000,6 +1102,10 @@ declare const serviceGroupsResponseSchema: z.ZodObject<{ health_check_interval_sec: z.ZodDefault; health_check_timeout_ms: z.ZodDefault; health_check_verify_tls: z.ZodDefault>; + health_check_provider: z.ZodCatch>; sync_status: z.ZodDefault>; }, z.core.$strip>, z.ZodTransform<{ target_ips: string[]; @@ -1021,6 +1127,7 @@ declare const serviceGroupsResponseSchema: z.ZodObject<{ health_check_interval_sec: number; health_check_timeout_ms: number; health_check_verify_tls: boolean; + health_check_provider: "local" | "cloudflare"; sync_status: string | null; target_ip?: string | null | undefined; }, { @@ -1039,6 +1146,7 @@ declare const serviceGroupsResponseSchema: z.ZodObject<{ health_check_interval_sec: number; health_check_timeout_ms: number; health_check_verify_tls: boolean; + health_check_provider: "local" | "cloudflare"; sync_status: string | null; target_ips?: string[] | undefined; target_ip?: string | null | undefined; @@ -1062,6 +1170,13 @@ declare const serviceGroupsResponseSchema: z.ZodObject<{ degraded: "degraded"; }>; latency_ms: z.ZodNullable; + last_checked_at: z.ZodOptional>; + last_error: z.ZodOptional>; + provider: z.ZodOptional>; + colo: z.ZodOptional>; }, z.core.$strip>>>; ip_enabled: z.ZodDefault>; }, z.core.$strip>>>; @@ -1265,6 +1380,10 @@ declare const healthCheckConfigSchema: z.ZodObject<{ health_check_interval_sec: z.ZodOptional; health_check_timeout_ms: z.ZodOptional; health_check_verify_tls: z.ZodOptional; + health_check_provider: z.ZodOptional>; }, z.core.$strip>; type HealthCheckConfig = z.infer; declare const createServiceSchema: z.ZodObject<{ @@ -1292,6 +1411,10 @@ declare const createServiceWithConfigSchema: z.ZodObject<{ health_check_interval_sec: z.ZodOptional; health_check_timeout_ms: z.ZodOptional; health_check_verify_tls: z.ZodOptional; + health_check_provider: z.ZodOptional>; fqdn: z.ZodString; target_ips: z.ZodOptional>; target_cname: z.ZodOptional; @@ -1480,6 +1603,10 @@ declare const updateServiceConfigSchema: z.ZodObject<{ health_check_interval_sec: z.ZodOptional; health_check_timeout_ms: z.ZodOptional; health_check_verify_tls: z.ZodOptional; + health_check_provider: z.ZodOptional>; fqdn: z.ZodString; target_ips: z.ZodOptional>; target_cname: z.ZodOptional; @@ -1507,6 +1634,10 @@ declare const createServiceGroupSchema: z.ZodObject<{ health_check_interval_sec: z.ZodOptional; health_check_timeout_ms: z.ZodOptional; health_check_verify_tls: z.ZodOptional; + health_check_provider: z.ZodOptional>; name: z.ZodString; type: z.ZodDefault; health_check_timeout_ms: z.ZodOptional; health_check_verify_tls: z.ZodOptional; + health_check_provider: z.ZodOptional>; name: z.ZodOptional; type: z.ZodOptional; healthLatencyWarnMs: z.ZodOptional; healthSuccessRecoveries: z.ZodOptional; + healthWorkerUrl: z.ZodOptional]>>; + healthWorkerToken: z.ZodOptional; }, z.core.$strip>; type AppSettingsPatch = z.infer; declare const vpsTrackerEventSchema: z.ZodObject<{ @@ -1907,4 +2044,4 @@ declare const ingestAuditEventSchema: z.ZodObject<{ }, z.core.$strip>; type IngestAuditEvent = z.infer; -export { AUDIT_SEVERITIES, AUDIT_SOURCE_APPS, AUDIT_TARGET_TYPES, type AppSettingsPatch, type AppSwitcherConfig, type AppSwitcherEntry, type AuditListQuery, type AuditLogEntry, type AuditSeverity, type AuditSourceApp, type AuditTargetType, type BulkUpdateDomainsInput, CERT_ERROR, CERT_EXPIRED, CERT_MONITORING_VALUES, CERT_MONITOR_AUTO, CERT_MONITOR_REQUIRED, CERT_MONITOR_SKIPPED, CERT_OK, CERT_UNKNOWN, CERT_WARNING, type CertMonitoring, type Certificate, type CfDnsRecord, type CfHealthCheck, type CfZone, type CfdmBindingSyncItem, type ChangeDomainInput, type ChangeIpInput, type CreateDnsRecordInput, type CreateDnsRecordPayload, type CreateDomainInput, type CreateDomainMonitorInput, type CreateGroupInput, type CreateOriginHealthCheckInput, type CreateServiceBindingInput, type CreateServiceGroupInput, type CreateServiceInput, type CreateServiceNodeInput, type CreateServiceWithConfigInput, type CreateSubdomainInput, type DnsRecord, type Domain, type DomainEnvironment, type DomainListItem, type DomainMonitor, type DomainMonitorResult, type DomainMonitorType, type Group, type GroupWithStats, type HealthCheckConfig, type HealthCheckProvider, type HealthCheckScope, type HealthCheckTarget, type HealthCheckType, type HealthStatusQuery, type IngestAuditEvent, type IpHealthState, type IpHealthStatus, type JwtClaims, type LbMode, type LoginInput, type LoginRequest, type LoginResponse, type NodeHealthState, type NotificationLog, type OriginHealthCheck, type OriginHealthCheckRecord, type ParsedFqdn, type PatchDnsRecordPayload, type ReorderServicesInput, SYNC_CONFLICT, SYNC_ERROR, SYNC_PENDING_DELETE, SYNC_PENDING_PUSH, SYNC_SYNCED, type Service, type ServiceBinding, type ServiceBindingView, type ServiceDomainBinding, type ServiceDomainBindingView, type ServiceGroup, type ServiceGroupView, type ServiceGroupsResponse, type ServiceIpHealth, type ServiceNode, type ServiceNodeRecord, type ServiceOverview, type ServiceView, type Subdomain, type SubdomainRecord, type SyncJob, type ToggleEnabledInput, type ToggleServiceIpInput, type UpdateDomainInput, type UpdateServiceConfigInput, type UpdateServiceGroupInput, type UpdateServiceNodeInput, type UpdateSubdomainInput, ValidationError, type VpsTrackerEvent, appSettingsPatchSchema, appSwitcherConfigSchema, appSwitcherEntrySchema, appSwitcherIconSchema, auditListQuerySchema, auditLogEntrySchema, auditSeveritySchema, auditSourceAppSchema, auditTargetTypeSchema, bindingToFqdn, bulkUpdateDomainsSchema, certMonitoringSchema, certStatusFromExpiry, certificateSchema, cfdmBindingSyncItemSchema, cfdmSyncBindingsBodySchema, changeDomainSchema, changeIpSchema, createDnsRecordSchema, createDomainMonitorSchema, createDomainSchema, createGroupSchema, createOriginHealthCheckSchema, createServiceBindingSchema, createServiceGroupSchema, createServiceNodeSchema, createServiceSchema, createServiceWithConfigSchema, createSubdomainSchema, dnsNameToSubdomainLabel, dnsRecordNamesMatch, dnsRecordSchema, domainEnvironmentSchema, domainListItemSchema, domainMonitorResultSchema, domainMonitorSchema, domainMonitorTypeSchema, domainSchema, fqdnToDisplay, groupSchema, groupWithStatsSchema, healthCheckConfigSchema, healthCheckProviderSchema, healthCheckScopeSchema, healthCheckTypeSchema, healthStatusQuerySchema, ingestAuditEventSchema, ipHealthStateSchema, ipHealthStatusSchema, isIpLiteral, isValidIpv4, lbModeSchema, loginSchema, nodeHealthStateSchema, normalizeDnsRecordName, notificationLogSchema, originHealthCheckSchema, parseFqdn, reorderServicesSchema, serviceBindingSchema, serviceDomainBindingSchema, serviceGroupSchema, serviceGroupTypeSchema, serviceGroupViewSchema, serviceGroupsResponseSchema, serviceIpHealthSchema, serviceNodeSchema, serviceSchema, serviceViewSchema, shouldMonitorService, subdomainLabelToFqdn, subdomainSchema, toggleEnabledSchema, toggleServiceIpSchema, updateDomainGroupSchema, updateDomainSchema, updateServiceConfigSchema, updateServiceGroupSchema, updateServiceNodeSchema, updateSubdomainSchema, validateDnsRecord, vpsTrackerEventSchema }; +export { AUDIT_SEVERITIES, AUDIT_SOURCE_APPS, AUDIT_TARGET_TYPES, type AppSettingsPatch, type AppSwitcherConfig, type AppSwitcherEntry, type AuditListQuery, type AuditLogEntry, type AuditSeverity, type AuditSourceApp, type AuditTargetType, type BulkUpdateDomainsInput, CERT_ERROR, CERT_EXPIRED, CERT_MONITORING_VALUES, CERT_MONITOR_AUTO, CERT_MONITOR_REQUIRED, CERT_MONITOR_SKIPPED, CERT_OK, CERT_UNKNOWN, CERT_WARNING, type CertMonitoring, type Certificate, type CfDnsRecord, type CfHealthCheck, type CfZone, type CfdmBindingSyncItem, type ChangeDomainInput, type ChangeIpInput, type CreateDnsRecordInput, type CreateDnsRecordPayload, type CreateDomainInput, type CreateDomainMonitorInput, type CreateGroupInput, type CreateOriginHealthCheckInput, type CreateServiceBindingInput, type CreateServiceGroupInput, type CreateServiceInput, type CreateServiceNodeInput, type CreateServiceWithConfigInput, type CreateSubdomainInput, type DnsRecord, type Domain, type DomainEnvironment, type DomainListItem, type DomainMonitor, type DomainMonitorResult, type DomainMonitorType, type Group, type GroupWithStats, type HealthCheckConfig, type HealthCheckProvider, type HealthCheckScope, type HealthCheckTarget, type HealthCheckType, type HealthProbeLog, type HealthStatusQuery, type IngestAuditEvent, type IpHealthState, type IpHealthStatus, type JwtClaims, type LbMode, type LoginInput, type LoginRequest, type LoginResponse, type NodeHealthState, type NotificationLog, type OriginHealthCheck, type OriginHealthCheckRecord, type ParsedFqdn, type PatchDnsRecordPayload, type ReorderServicesInput, SYNC_CONFLICT, SYNC_ERROR, SYNC_PENDING_DELETE, SYNC_PENDING_PUSH, SYNC_SYNCED, type Service, type ServiceBinding, type ServiceBindingView, type ServiceDomainBinding, type ServiceDomainBindingView, type ServiceGroup, type ServiceGroupView, type ServiceGroupsResponse, type ServiceIpHealth, type ServiceNode, type ServiceNodeRecord, type ServiceOverview, type ServiceView, type Subdomain, type SubdomainRecord, type SyncJob, type ToggleEnabledInput, type ToggleServiceIpInput, type UpdateDomainInput, type UpdateServiceConfigInput, type UpdateServiceGroupInput, type UpdateServiceNodeInput, type UpdateSubdomainInput, ValidationError, type VpsTrackerEvent, appSettingsPatchSchema, appSwitcherConfigSchema, appSwitcherEntrySchema, appSwitcherIconSchema, auditListQuerySchema, auditLogEntrySchema, auditSeveritySchema, auditSourceAppSchema, auditTargetTypeSchema, bindingToFqdn, bulkUpdateDomainsSchema, certMonitoringSchema, certStatusFromExpiry, certificateSchema, cfdmBindingSyncItemSchema, cfdmSyncBindingsBodySchema, changeDomainSchema, changeIpSchema, createDnsRecordSchema, createDomainMonitorSchema, createDomainSchema, createGroupSchema, createOriginHealthCheckSchema, createServiceBindingSchema, createServiceGroupSchema, createServiceNodeSchema, createServiceSchema, createServiceWithConfigSchema, createSubdomainSchema, dnsNameToSubdomainLabel, dnsRecordNamesMatch, dnsRecordSchema, domainEnvironmentSchema, domainListItemSchema, domainMonitorResultSchema, domainMonitorSchema, domainMonitorTypeSchema, domainSchema, fqdnToDisplay, groupSchema, groupWithStatsSchema, healthCheckConfigSchema, healthCheckProviderSchema, healthCheckScopeSchema, healthCheckTypeSchema, healthProbeLogSchema, healthStatusQuerySchema, ingestAuditEventSchema, ipHealthStateSchema, ipHealthStatusSchema, isIpLiteral, isValidIpv4, lbModeSchema, loginSchema, nodeHealthStateSchema, normalizeDnsRecordName, notificationLogSchema, originHealthCheckSchema, parseFqdn, reorderServicesSchema, serviceBindingSchema, serviceDomainBindingSchema, serviceGroupSchema, serviceGroupTypeSchema, serviceGroupViewSchema, serviceGroupsResponseSchema, serviceIpHealthSchema, serviceNodeSchema, serviceSchema, serviceViewSchema, shouldMonitorService, subdomainLabelToFqdn, subdomainSchema, toggleEnabledSchema, toggleServiceIpSchema, updateDomainGroupSchema, updateDomainSchema, updateServiceConfigSchema, updateServiceGroupSchema, updateServiceNodeSchema, updateSubdomainSchema, validateDnsRecord, vpsTrackerEventSchema }; diff --git a/packages/shared/dist/index.js b/packages/shared/dist/index.js index 68e796d..b91a7cc 100644 --- a/packages/shared/dist/index.js +++ b/packages/shared/dist/index.js @@ -190,12 +190,31 @@ var ipHealthStatusSchema = z.object({ consecutive_failures: z.number(), consecutive_successes: z.number().optional().default(0), last_checked_at: z.string().nullable(), - last_error: z.string().nullable() + last_error: z.string().nullable(), + colo: z.string().nullable().optional(), + provider: healthCheckProviderSchema.optional() }); var serviceIpHealthSchema = z.object({ ip: z.string(), status: ipHealthStateSchema, - latency_ms: z.number().nullable() + latency_ms: z.number().nullable(), + last_checked_at: z.string().nullable().optional(), + last_error: z.string().nullable().optional(), + provider: healthCheckProviderSchema.optional(), + colo: z.string().nullable().optional() +}); +var healthProbeLogSchema = z.object({ + id: z.number(), + scope: z.string(), + ref_id: z.number(), + ip: z.string(), + provider: healthCheckProviderSchema, + status: ipHealthStateSchema, + ok: z.coerce.boolean(), + latency_ms: z.number().nullable(), + colo: z.string().nullable(), + error: z.string().nullable(), + checked_at: z.string() }); var groupSchema = z.object({ id: z.number(), @@ -230,6 +249,7 @@ var serviceGroupSchema = z.object({ health_check_interval_sec: z.number().default(30), health_check_timeout_ms: z.number().default(3e3), health_check_verify_tls: z.coerce.boolean().default(false), + health_check_provider: healthCheckProviderSchema.catch("local"), created_at: z.string(), updated_at: z.string() }); @@ -267,6 +287,7 @@ var serviceDomainBindingSchema = z.object({ health_check_interval_sec: z.number().default(30), health_check_timeout_ms: z.number().default(3e3), health_check_verify_tls: z.coerce.boolean().default(false), + health_check_provider: healthCheckProviderSchema.catch("local"), sync_status: z.string().nullable().default(null) }).transform((binding) => ({ ...binding, @@ -392,7 +413,8 @@ var healthCheckConfigFields = { health_check_expected_status: z.number().int().min(100).max(599).nullable().optional(), health_check_interval_sec: z.number().int().min(5).max(3600).optional(), health_check_timeout_ms: z.number().int().min(100).max(3e4).optional(), - health_check_verify_tls: z.boolean().optional() + health_check_verify_tls: z.boolean().optional(), + health_check_provider: healthCheckProviderSchema.optional() }; var healthCheckConfigSchema = z.object(healthCheckConfigFields); var serviceDomainInputSchema = z.object({ @@ -703,7 +725,9 @@ var appSettingsPatchSchema = z3.object({ healthDegradedFailures: z3.number().int().min(1).max(20).optional(), healthDownFailures: z3.number().int().min(1).max(50).optional(), healthLatencyWarnMs: z3.number().int().min(50).max(6e4).optional(), - healthSuccessRecoveries: z3.number().int().min(1).max(20).optional() + healthSuccessRecoveries: z3.number().int().min(1).max(20).optional(), + healthWorkerUrl: z3.string().url().or(z3.literal("")).optional(), + healthWorkerToken: z3.string().optional() }).superRefine((data, ctx) => { if (data.healthDegradedFailures != null && data.healthDownFailures != null && data.healthDownFailures < data.healthDegradedFailures) { ctx.addIssue({ @@ -847,6 +871,7 @@ export { healthCheckProviderSchema, healthCheckScopeSchema, healthCheckTypeSchema, + healthProbeLogSchema, healthStatusQuerySchema, ingestAuditEventSchema, ipHealthStateSchema, diff --git a/packages/shared/src/integration-vps-tracker.ts b/packages/shared/src/integration-vps-tracker.ts index a08b61c..7943a91 100644 --- a/packages/shared/src/integration-vps-tracker.ts +++ b/packages/shared/src/integration-vps-tracker.ts @@ -35,6 +35,8 @@ export const appSettingsPatchSchema = z.object({ healthDownFailures: z.number().int().min(1).max(50).optional(), healthLatencyWarnMs: z.number().int().min(50).max(60_000).optional(), healthSuccessRecoveries: z.number().int().min(1).max(20).optional(), + healthWorkerUrl: z.string().url().or(z.literal("")).optional(), + healthWorkerToken: z.string().optional(), }).superRefine((data, ctx) => { if ( data.healthDegradedFailures != null && diff --git a/packages/shared/src/schemas.ts b/packages/shared/src/schemas.ts index 740a704..3b475ea 100644 --- a/packages/shared/src/schemas.ts +++ b/packages/shared/src/schemas.ts @@ -45,6 +45,8 @@ export const ipHealthStatusSchema = z.object({ consecutive_successes: z.number().optional().default(0), last_checked_at: z.string().nullable(), last_error: z.string().nullable(), + colo: z.string().nullable().optional(), + provider: healthCheckProviderSchema.optional(), }) export type IpHealthStatus = z.infer @@ -53,10 +55,30 @@ export const serviceIpHealthSchema = z.object({ ip: z.string(), status: ipHealthStateSchema, latency_ms: z.number().nullable(), + last_checked_at: z.string().nullable().optional(), + last_error: z.string().nullable().optional(), + provider: healthCheckProviderSchema.optional(), + colo: z.string().nullable().optional(), }) export type ServiceIpHealth = z.infer +export const healthProbeLogSchema = z.object({ + id: z.number(), + scope: z.string(), + ref_id: z.number(), + ip: z.string(), + provider: healthCheckProviderSchema, + status: ipHealthStateSchema, + ok: z.coerce.boolean(), + latency_ms: z.number().nullable(), + colo: z.string().nullable(), + error: z.string().nullable(), + checked_at: z.string(), +}) + +export type HealthProbeLog = z.infer + export const groupSchema = z.object({ id: z.number(), name: z.string(), @@ -93,6 +115,7 @@ export const serviceGroupSchema = z.object({ health_check_interval_sec: z.number().default(30), health_check_timeout_ms: z.number().default(3000), health_check_verify_tls: z.coerce.boolean().default(false), + health_check_provider: healthCheckProviderSchema.catch('local'), created_at: z.string(), updated_at: z.string(), }) @@ -133,6 +156,7 @@ export const serviceDomainBindingSchema = z health_check_interval_sec: z.number().default(30), health_check_timeout_ms: z.number().default(3000), health_check_verify_tls: z.coerce.boolean().default(false), + health_check_provider: healthCheckProviderSchema.catch('local'), sync_status: z.string().nullable().default(null), }) .transform((binding) => ({ @@ -304,6 +328,7 @@ const healthCheckConfigFields = { health_check_interval_sec: z.number().int().min(5).max(3600).optional(), health_check_timeout_ms: z.number().int().min(100).max(30000).optional(), health_check_verify_tls: z.boolean().optional(), + health_check_provider: healthCheckProviderSchema.optional(), } export const healthCheckConfigSchema = z.object(healthCheckConfigFields) diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index ecfa731..12cf1b4 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -22,6 +22,7 @@ export interface ServiceGroup { health_check_interval_sec: number; health_check_timeout_ms: number; health_check_verify_tls: boolean; + health_check_provider: HealthCheckProvider; created_at: string; updated_at: string; } @@ -121,6 +122,7 @@ export interface ServiceBinding { health_check_interval_sec: number; health_check_timeout_ms: number; health_check_verify_tls: boolean; + health_check_provider: HealthCheckProvider; routing_strategy: LbMode; operation_version: number; created_at: string; @@ -152,6 +154,7 @@ export interface ServiceBindingView { health_check_interval_sec: number; health_check_timeout_ms: number; health_check_verify_tls: boolean; + health_check_provider: HealthCheckProvider; sync_status: string | null; created_at: string; updated_at: string; @@ -177,6 +180,7 @@ export interface ServiceDomainBindingView { health_check_interval_sec: number; health_check_timeout_ms: number; health_check_verify_tls: boolean; + health_check_provider: HealthCheckProvider; sync_status: string | null; } @@ -293,12 +297,18 @@ export interface IpHealthStatus { consecutive_successes?: number; last_checked_at: string | null; last_error: string | null; + colo?: string | null; + provider?: HealthCheckProvider; } export interface ServiceIpHealth { ip: string; status: IpHealthState; latency_ms: number | null; + last_checked_at?: string | null; + last_error?: string | null; + provider?: HealthCheckProvider; + colo?: string | null; } export interface ServiceNode { @@ -382,4 +392,5 @@ export interface HealthCheckTarget { expected_status: number | null; timeout_ms: number; verify_tls: boolean; + provider: HealthCheckProvider; } diff --git a/workers/health-probe/README.md b/workers/health-probe/README.md new file mode 100644 index 0000000..a183ed0 --- /dev/null +++ b/workers/health-probe/README.md @@ -0,0 +1,37 @@ +# CFDM health-probe Worker + +Stateless edge probe for CFDM. **Not** Cloudflare Health Checks API (unavailable on Free). +Cron stays in CFDM — this Worker has no Cron Trigger. + +## Deploy + +```powershell +cd workers/health-probe +npx wrangler login +npx wrangler secret put PROBE_TOKEN +npx wrangler deploy +``` + +Paste the Worker URL (`https://cfdm-health-probe..workers.dev`) and the same token into **Настройки → Health-check**. + +Free Workers ≈ 100k requests/day. CFDM cron every 2 minutes × number of IPs must fit. + +## API + +`POST /probe` + `Authorization: Bearer ` + +```json +{ + "type": "tcp", + "ip": "1.2.3.4", + "hostname": "app.example.com", + "port": 443, + "path": "/", + "expected_status": 200, + "timeout_ms": 3000, + "verify_tls": true, + "method": "GET" +} +``` + +Response: `{ "ok": true, "latencyMs": 42, "error": null, "colo": "AMS" }`. diff --git a/workers/health-probe/package.json b/workers/health-probe/package.json new file mode 100644 index 0000000..fc5f458 --- /dev/null +++ b/workers/health-probe/package.json @@ -0,0 +1,11 @@ +{ + "name": "cfdm-health-probe", + "private": true, + "scripts": { + "dev": "wrangler dev", + "deploy": "wrangler deploy" + }, + "devDependencies": { + "wrangler": "^4.20.0" + } +} diff --git a/workers/health-probe/src/index.ts b/workers/health-probe/src/index.ts new file mode 100644 index 0000000..2829bd3 --- /dev/null +++ b/workers/health-probe/src/index.ts @@ -0,0 +1,181 @@ +/** + * Stateless CFDM health probe. Cron lives in CFDM API — this Worker only + * answers POST /probe. Deploy: wrangler deploy; paste URL + token into + * Настройки → Health-check. + */ + +export interface Env { + PROBE_TOKEN: string; +} + +type ProbeType = "tcp" | "http"; + +interface ProbeRequest { + type?: ProbeType; + ip?: string; + hostname?: string; + port?: number; + path?: string; + expected_status?: number | null; + timeout_ms?: number; + verify_tls?: boolean; + method?: string; +} + +interface ProbeResponse { + ok: boolean; + latencyMs: number; + error: string | null; + colo: string | null; +} + +export default { + async fetch(request: Request, env: Env): Promise { + const colo = + (request as Request & { cf?: { colo?: string } }).cf?.colo ?? null; + if (request.method !== "POST") { + return json({ ok: false, latencyMs: 0, error: "method not allowed", colo }, 405); + } + const pathname = new URL(request.url).pathname.replace(/\/$/, "") || "/"; + if (pathname !== "/probe") { + return json({ ok: false, latencyMs: 0, error: "not found", colo }, 404); + } + const token = bearer(request); + if (!env.PROBE_TOKEN || token !== env.PROBE_TOKEN) { + return json({ ok: false, latencyMs: 0, error: "unauthorized", colo }, 401); + } + + let body: ProbeRequest; + try { + body = (await request.json()) as ProbeRequest; + } catch { + return json({ ok: false, latencyMs: 0, error: "invalid json", colo }, 400); + } + + const ip = String(body.ip ?? "").trim(); + if (!ip) { + return json({ ok: false, latencyMs: 0, error: "ip required", colo }, 400); + } + const type: ProbeType = body.type === "http" ? "http" : "tcp"; + const port = Number(body.port) || (type === "http" ? 80 : 80); + const timeoutMs = Math.min(Math.max(Number(body.timeout_ms) || 3000, 100), 25_000); + const hostname = String(body.hostname ?? "").trim() || ip; + + try { + const result = + type === "http" + ? await httpProbe({ + ip, + hostname, + port, + path: body.path || "/", + expectedStatus: body.expected_status ?? 200, + timeoutMs, + verifyTls: Boolean(body.verify_tls), + method: (body.method || "GET").toUpperCase(), + }) + : await tcpProbe(ip, port, timeoutMs); + return json({ ...result, colo }); + } catch (err) { + const message = err instanceof Error ? err.message : "probe failed"; + return json({ ok: false, latencyMs: 0, error: message, colo }); + } + }, +}; + +function bearer(request: Request): string { + const header = request.headers.get("Authorization") ?? ""; + return header.startsWith("Bearer ") ? header.slice(7) : ""; +} + +function json(body: ProbeResponse, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); +} + +function withTimeout(promise: Promise, timeoutMs: number, label: string): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`${label} timeout`)), timeoutMs); + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (err) => { + clearTimeout(timer); + reject(err); + }, + ); + }); +} + +async function tcpProbe( + ip: string, + port: number, + timeoutMs: number, +): Promise> { + const started = Date.now(); + const { connect } = await import("cloudflare:sockets"); + const socket = connect({ hostname: ip, port }); + try { + await withTimeout(socket.opened, timeoutMs, "tcp"); + return { ok: true, latencyMs: Date.now() - started, error: null }; + } catch (err) { + const message = err instanceof Error ? err.message : "tcp failed"; + return { ok: false, latencyMs: Date.now() - started, error: message }; + } finally { + try { + socket.close(); + } catch { + // ignore + } + } +} + +async function httpProbe(opts: { + ip: string; + hostname: string; + port: number; + path: string; + expectedStatus: number; + timeoutMs: number; + verifyTls: boolean; + method: string; +}): Promise> { + const started = Date.now(); + const useTls = opts.verifyTls || opts.port === 443; + const host = opts.ip.includes(":") ? `[${opts.ip}]` : opts.ip; + const path = opts.path.startsWith("/") ? opts.path : `/${opts.path}`; + const url = `${useTls ? "https" : "http"}://${host}:${opts.port}${path}`; + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), opts.timeoutMs); + try { + const res = await fetch(url, { + method: opts.method === "HEAD" ? "HEAD" : "GET", + headers: { Host: opts.hostname }, + signal: controller.signal, + redirect: "manual", + }); + const latencyMs = Date.now() - started; + if (res.status !== opts.expectedStatus) { + return { + ok: false, + latencyMs, + error: `HTTP ${res.status} (ожидали ${opts.expectedStatus})`, + }; + } + return { ok: true, latencyMs, error: null }; + } catch (err) { + const message = + err instanceof Error + ? err.name === "AbortError" + ? "http timeout" + : err.message + : "http failed"; + return { ok: false, latencyMs: Date.now() - started, error: message }; + } finally { + clearTimeout(timer); + } +} diff --git a/workers/health-probe/wrangler.toml b/workers/health-probe/wrangler.toml new file mode 100644 index 0000000..fe8d9da --- /dev/null +++ b/workers/health-probe/wrangler.toml @@ -0,0 +1,6 @@ +name = "cfdm-health-probe" +main = "src/index.ts" +compatibility_date = "2025-04-01" + +# Set the shared secret: wrangler secret put PROBE_TOKEN +# Then paste the Worker URL + token into CFDM → Настройки → Health-check.