feat(health-checks): implement health check IP toggling and configuration updates
quality / commitlint (push) Skipped
CD / update-wiki (push) Successful in 8s
quality / changes (push) Successful in 5s
quality / docker-check (push) Skipped
quality / web (push) Successful in 51s
quality / api (push) Successful in 43s
CD / quality (push) Successful in 1m43s
CD / publish (push) Successful in 1m38s

- Added functionality to toggle individual IPs for services, allowing for dynamic management of IP health status.
- Enhanced the health check configuration in the UI, enabling users to set parameters directly from the settings page.
- Updated service views to include IP health tracking, improving visibility into the status of each IP associated with a service.
- Refactored relevant components to support the new IP toggling feature, ensuring a seamless user experience.

This commit significantly enhances the health management capabilities of services, providing users with more control over IP configurations and health monitoring.
This commit is contained in:
Denozordec
2026-08-19 15:33:47 +07:00
parent 4224db8eb3
commit d63c86065c
35 changed files with 1538 additions and 123 deletions
+9 -68
View File
@@ -7,7 +7,6 @@ import {
} from "@fastify/type-provider-zod";
import type { AppConfig } from "./config.js";
import { loadConfig } from "./config.js";
import { repos } from "@cfdm/db";
import authPlugin from "./plugins/auth.js";
import cfClientPlugin from "./plugins/cf-client.js";
import { requireAuth } from "./plugins/auth.js";
@@ -34,8 +33,10 @@ import { settingsRoutes } from "./routes/settings.js";
import { integrationsVpsTrackerRoutes } from "./routes/integrations-vps-tracker.js";
import { auditRoutes } from "./routes/audit.js";
import * as certificateService from "./services/certificate-service.js";
import * as healthCheckService from "./services/health-check-service.js";
import * as serviceConfigService from "./services/service-config-service.js";
import {
createHealthCheckTask,
scheduleHealthCheckJob,
} from "./services/health-check-scheduler.js";
import { AsyncTask, CronJob } from "toad-scheduler";
export interface BuildAppOptions {
@@ -125,71 +126,11 @@ export async function buildApp(opts: BuildAppOptions = {}) {
),
);
const healthTask = new AsyncTask(
"health-check",
async () => {
const thresholds = {
degradedFailures: config.healthDegradedFailures,
downFailures: config.healthDownFailures,
latencyWarnMs: config.healthLatencyWarnMs,
successRecoveries: config.healthSuccessRecoveries,
};
const n = await healthCheckService.runAllChecks(app.db, {
thresholds,
probeGapMs: config.healthProbeGapMs,
onStatusChange: async (target, prev, next) => {
try {
const label =
next === "up"
? "OK"
: next === "degraded"
? "Slow"
: next === "down"
? "Down"
: "—";
repos.insertNotificationLog(
app.db,
"ip_health",
target.scope,
target.ref_id,
`${target.hostname || target.ip}: ${label}`,
`IP ${target.ip}: ${prev ?? "—"}${label}`,
);
await serviceConfigService.reconcileDnsForTarget(
app.db,
app.cf,
target.scope,
target.ref_id,
);
} catch (err) {
app.log.warn(
{ err, scope: target.scope, refId: target.ref_id },
"health-check reconcile failed",
);
}
},
});
const monitors = await healthCheckService.runDomainMonitors(
app.db,
thresholds,
);
app.log.info(
{ checked: n, monitors },
"health check completed",
);
},
(err) => {
app.log.warn({ err }, "health check failed");
},
);
app.scheduler.addCronJob(
new CronJob(
{ cronExpression: config.healthCheckCron },
healthTask,
{ preventOverrun: true },
),
);
const healthTask = createHealthCheckTask(app, config);
scheduleHealthCheckJob(app, config, healthTask);
app.decorate("reloadHealthCheckJob", () => {
scheduleHealthCheckJob(app, config, healthTask);
});
}
return app;
+23
View File
@@ -4,6 +4,7 @@ import {
changeDomainSchema,
createServiceNodeSchema,
reorderServicesSchema,
toggleServiceIpSchema,
updateServiceConfigSchema,
updateServiceNodeSchema,
} from "@cfdm/shared";
@@ -191,4 +192,26 @@ export async function serviceRoutes(app: FastifyInstance) {
body.enabled,
);
});
app.patch("/services/:id/ips/toggle", async (request) => {
const { id } = request.params as { id: string };
const body = toggleServiceIpSchema.parse(request.body);
const view = await serviceConfig.toggleServiceIp(
request.server.db,
request.server.cf,
Number(id),
body.ip,
body.enabled,
);
recordAudit(request.server, request, {
action: "service.ip.toggle",
targetType: "app_resource",
targetId: String(id),
summary: body.enabled
? `Включён IP ${body.ip} сервиса «${view.name}»`
: `Выключен IP ${body.ip} сервиса «${view.name}»`,
details: { ip: body.ip, enabled: body.enabled },
});
return view;
});
}
+34 -3
View File
@@ -5,15 +5,46 @@ import {
updateAppSettings,
} from "@cfdm/db";
import { pingVpsTracker } from "../services/vps-tracker-sync.js";
import { AppError } from "../errors.js";
import {
assertValidHealthCron,
healthEngineFallbacksFromConfig,
} from "../services/health-check-scheduler.js";
export async function settingsRoutes(app: FastifyInstance) {
app.get("/settings", async (request) => {
return getAppSettings(request.server.db);
return getAppSettings(
request.server.db,
healthEngineFallbacksFromConfig(request.server.config),
);
});
app.patch("/settings", async (request) => {
const body = appSettingsPatchSchema.parse(request.body);
return updateAppSettings(request.server.db, body);
const parsed = appSettingsPatchSchema.safeParse(request.body);
if (!parsed.success) {
throw AppError.validation(
parsed.error.issues[0]?.message ?? "некорректные настройки",
);
}
const body = parsed.data;
if (body.healthCheckCron) {
assertValidHealthCron(body.healthCheckCron);
}
const fallbacks = healthEngineFallbacksFromConfig(request.server.config);
const current = getAppSettings(request.server.db, fallbacks);
const nextDegraded =
body.healthDegradedFailures ?? current.healthDegradedFailures;
const nextDown = body.healthDownFailures ?? current.healthDownFailures;
if (nextDown < nextDegraded) {
throw AppError.validation(
"ошибок до down не меньше, чем до degraded",
);
}
const next = updateAppSettings(request.server.db, body, fallbacks);
if (body.healthCheckCron !== undefined) {
request.server.reloadHealthCheckJob?.();
}
return next;
});
app.post("/settings/vps-tracker/test", async (request) => {
@@ -0,0 +1,134 @@
import type { FastifyInstance } from "fastify";
import { AsyncTask, CronJob } from "toad-scheduler";
import {
getAppSettings,
type HealthEngineFallbacks,
} from "@cfdm/db";
import { repos } from "@cfdm/db";
import type { AppConfig } from "../config.js";
import { AppError } from "../errors.js";
import * as healthCheckService from "./health-check-service.js";
import * as serviceConfigService from "./service-config-service.js";
declare module "fastify" {
interface FastifyInstance {
reloadHealthCheckJob?: () => void;
}
}
export const HEALTH_CHECK_JOB_ID = "health-check";
export function healthEngineFallbacksFromConfig(
config: AppConfig,
): HealthEngineFallbacks {
return {
healthCheckCron: config.healthCheckCron,
healthDegradedFailures: config.healthDegradedFailures,
healthDownFailures: config.healthDownFailures,
healthLatencyWarnMs: config.healthLatencyWarnMs,
healthSuccessRecoveries: config.healthSuccessRecoveries,
};
}
export function assertValidHealthCron(expr: string): void {
const cronExpression = expr.trim();
const parts = cronExpression.split(/\s+/).filter(Boolean);
if (parts.length < 5 || parts.length > 6) {
throw AppError.validation("некорректное cron-выражение");
}
try {
const job = new CronJob(
{ cronExpression },
new AsyncTask("validate-cron", async () => undefined),
{ id: "validate-cron" },
);
job.stop();
} catch {
throw AppError.validation("некорректное cron-выражение");
}
}
export function createHealthCheckTask(
app: FastifyInstance,
config: AppConfig,
): AsyncTask {
const fallbacks = healthEngineFallbacksFromConfig(config);
return new AsyncTask(
HEALTH_CHECK_JOB_ID,
async () => {
const settings = getAppSettings(app.db, fallbacks);
const thresholds = {
degradedFailures: settings.healthDegradedFailures,
downFailures: settings.healthDownFailures,
latencyWarnMs: settings.healthLatencyWarnMs,
successRecoveries: settings.healthSuccessRecoveries,
};
const n = await healthCheckService.runAllChecks(app.db, {
thresholds,
probeGapMs: config.healthProbeGapMs,
onStatusChange: async (target, prev, next) => {
try {
const label =
next === "up"
? "OK"
: next === "degraded"
? "Slow"
: next === "down"
? "Down"
: "—";
repos.insertNotificationLog(
app.db,
"ip_health",
target.scope,
target.ref_id,
`${target.hostname || target.ip}: ${label}`,
`IP ${target.ip}: ${prev ?? "—"}${label}`,
);
await serviceConfigService.reconcileDnsForTarget(
app.db,
app.cf,
target.scope,
target.ref_id,
);
} catch (err) {
app.log.warn(
{ err, scope: target.scope, refId: target.ref_id },
"health-check reconcile failed",
);
}
},
});
const monitors = await healthCheckService.runDomainMonitors(
app.db,
thresholds,
);
app.log.info({ checked: n, monitors }, "health check completed");
},
(err) => {
app.log.warn({ err }, "health check failed");
},
);
}
export function scheduleHealthCheckJob(
app: FastifyInstance,
config: AppConfig,
task: AsyncTask,
): void {
const scheduler = app.scheduler;
if (!scheduler) return;
if (scheduler.existsById(HEALTH_CHECK_JOB_ID)) {
scheduler.removeById(HEALTH_CHECK_JOB_ID);
}
const settings = getAppSettings(
app.db,
healthEngineFallbacksFromConfig(config),
);
scheduler.addCronJob(
new CronJob(
{ cronExpression: settings.healthCheckCron },
task,
{ preventOverrun: true, id: HEALTH_CHECK_JOB_ID },
),
);
}
@@ -242,7 +242,11 @@ async function collectKnownZones(
async function buildView(db: Db, serviceId: number): Promise<ServiceView> {
const service = repos.getService(db, serviceId);
const ips = repos.listServiceIps(db, serviceId);
const ipRows = repos.listServiceIpRows(db, serviceId);
const ips = ipRows.map((row) => row.ip);
const ip_enabled = Object.fromEntries(
ipRows.map((row) => [row.ip, row.enabled]),
);
const bindings = repos.listBindingsByService(db, serviceId);
const domainViews = bindings.map((binding) => {
@@ -304,6 +308,7 @@ async function buildView(db: Db, serviceId: number): Promise<ServiceView> {
created_at: service.created_at,
updated_at: service.updated_at,
ips,
ip_enabled,
domains: domainViews,
health_status: "unknown",
health_latency_ms: null,
@@ -384,7 +389,7 @@ export async function listGroupViews(db: Db): Promise<ServiceGroupsResponse> {
const groupViews = groupViewsRaw.map((group) => {
const services = group.services.map(
(s) => healthById.get(s.id) ?? { ...s, health_status: "unknown" as const, health_latency_ms: null, ip_health: [] },
(s) => healthById.get(s.id) ?? { ...s, health_status: "unknown" as const, health_latency_ms: null, ip_health: [], ip_enabled: {} },
);
const groupScopeHealth = groupHealthById.get(group.id);
// Only enabled services feed the group badge — a disabled service with a
@@ -414,6 +419,7 @@ export async function listGroupViews(db: Db): Promise<ServiceGroupsResponse> {
health_status: "unknown" as const,
health_latency_ms: null,
ip_health: [],
ip_enabled: {},
},
);
@@ -1345,6 +1351,59 @@ export async function toggleService(
return enabledView!;
}
export async function toggleServiceIp(
db: Db,
cf: CloudflareClient,
serviceId: number,
ip: string,
enabled: boolean,
): Promise<ServiceView> {
repos.getService(db, serviceId);
const pool = repos.listServiceIps(db, serviceId);
if (!pool.includes(ip)) {
throw AppError.validation(`IP ${ip} не входит в пул адресов сервиса`);
}
repos.setServiceIpEnabled(db, serviceId, ip, enabled);
const node = repos
.listNodes(db, serviceId)
.find((entry) => entry.address === ip);
if (node) {
repos.updateNode(db, node.id, { enabled });
}
const bindings = repos.listBindingsByService(db, serviceId);
for (const binding of bindings) {
if (binding.cname_target?.trim()) continue;
const current = repos.listBindingIpsWithMeta(db, binding.id);
const hasIp = current.some((entry) => entry.ip === ip);
if (enabled && !hasIp) {
repos.replaceBindingIpsWithMeta(db, binding.id, [
...current,
{ ip, weight: 1, priority: 1 },
]);
continue;
}
if (!enabled && hasIp) {
repos.replaceBindingIpsWithMeta(
db,
binding.id,
current.filter((entry) => entry.ip !== ip),
);
}
}
const service = repos.getService(db, serviceId);
if (shouldPushDns(db, service)) {
await syncServiceBindingsToDns(db, cf, serviceId);
await syncGroupDomainForService(db, cf, serviceId);
}
void syncServiceToVpsTracker(db, serviceId);
const [view] = attachServiceHealth(db, [await buildView(db, serviceId)]);
return view!;
}
export async function toggleGroup(
db: Db,
cf: CloudflareClient,
+5 -4
View File
@@ -43,8 +43,9 @@ function resolveIpsLocally(
const binding = index.byFqdn.get(key);
if (!binding) return [];
if (binding.target_ips.some(isIpLiteral)) {
return binding.target_ips.filter(isIpLiteral);
const ips = (binding.target_ips ?? []).filter(isIpLiteral);
if (ips.length > 0) {
return ips;
}
const cname = binding.cname_target?.trim();
@@ -79,7 +80,7 @@ export async function resolveBindingIpsForSync(
index: BindingIpIndex,
db?: Db,
): Promise<string[]> {
const directIps = binding.target_ips.filter(isIpLiteral);
const directIps = (binding.target_ips ?? []).filter(isIpLiteral);
if (directIps.length > 0) {
return [...directIps];
}
@@ -124,7 +125,7 @@ export async function buildServiceSyncBindingsAsync(
const serviceIps = repos.listServiceIps(db, serviceId);
const allBindings = repos.listAllBindings(db);
const index = buildBindingIndex(allBindings);
const bindings = repos.listBindingsByService(db, serviceId);
const bindings = allBindings.filter((row) => row.service_id === serviceId);
const items: CfdmBindingSyncItem[] = [];
for (const binding of bindings) {
@@ -165,4 +165,94 @@ describe("create service then list groups", () => {
await app.close();
});
it("PATCH /services/:id/ips/toggle keeps IP in pool and removes it from A-binding", async () => {
const app = await buildApp({
config: { ...loadConfig(), staticDir: null },
memory: true,
});
const headers = await authHeaders(app);
const cf = mockCf();
repos.createDomain(app.db, null, "example.com", "zone-1");
const group = repos.createServiceGroup(
app.db,
"VPN",
"vpn",
null,
"vpn.example.com",
);
const createRes = await app.inject({
method: "POST",
url: "/api/v1/services",
headers,
payload: {
name: "Panel",
slug: "panel-ip-toggle",
service_group_id: group.id,
},
});
expect(createRes.statusCode).toBe(200);
const created = createRes.json() as { id: number };
await updateConfig(app.db, cf, created.id, {
ips: ["1.2.3.4", "5.6.7.8"],
service_group_id: group.id,
domains: [
{
fqdn: "panel.example.com",
target_ips: ["1.2.3.4", "5.6.7.8"],
target_ip_weights: { "1.2.3.4": 1, "5.6.7.8": 1 },
target_ip_priorities: { "1.2.3.4": 1, "5.6.7.8": 1 },
lb_mode: "round_robin",
health_check_enabled: false,
health_check_type: "tcp",
health_check_port: 443,
health_check_path: null,
health_check_expected_status: null,
health_check_interval_sec: 30,
health_check_timeout_ms: 3000,
health_check_verify_tls: false,
},
],
});
// HTTP toggle uses request.server.cf; disable DNS push so the test
// does not call the real Cloudflare client.
repos.setServiceEnabled(app.db, created.id, false);
const offRes = await app.inject({
method: "PATCH",
url: `/api/v1/services/${created.id}/ips/toggle`,
headers,
payload: { ip: "1.2.3.4", enabled: false },
});
expect(offRes.statusCode, JSON.stringify(offRes.json())).toBe(200);
const offView = offRes.json() as {
ips: string[];
ip_enabled: Record<string, boolean>;
};
expect(offView.ips).toEqual(expect.arrayContaining(["1.2.3.4", "5.6.7.8"]));
expect(offView.ip_enabled["1.2.3.4"]).toBe(false);
expect(offView.ip_enabled["5.6.7.8"]).toBe(true);
const binding = repos.listBindingsByService(app.db, created.id)[0]!;
expect(repos.listBindingIps(app.db, binding.id)).toEqual(["5.6.7.8"]);
const onRes = await app.inject({
method: "PATCH",
url: `/api/v1/services/${created.id}/ips/toggle`,
headers,
payload: { ip: "1.2.3.4", enabled: true },
});
expect(onRes.statusCode).toBe(200);
const onView = onRes.json() as { ip_enabled: Record<string, boolean> };
expect(onView.ip_enabled["1.2.3.4"]).toBe(true);
expect(repos.listBindingIps(app.db, binding.id)).toEqual(
expect.arrayContaining(["1.2.3.4", "5.6.7.8"]),
);
await app.close();
});
});
+136
View File
@@ -0,0 +1,136 @@
import { describe, expect, it } from "vitest";
import { buildApp } from "../src/app.js";
import { loadConfig } from "../src/config.js";
async function authHeaders(app: Awaited<ReturnType<typeof buildApp>>) {
const res = await app.inject({
method: "POST",
url: "/api/v1/auth/login",
payload: { username: "admin", password: "admin" },
});
expect(res.statusCode).toBe(200);
const { token } = res.json() as { token: string };
return { authorization: `Bearer ${token}` };
}
describe("settings health engine", () => {
it("GET /api/v1/settings returns env fallbacks for health fields", async () => {
const app = await buildApp({
config: {
...loadConfig(),
staticDir: null,
healthCheckCron: "*/30 * * * * *",
healthDegradedFailures: 3,
healthDownFailures: 4,
healthLatencyWarnMs: 1500,
healthSuccessRecoveries: 5,
},
memory: true,
});
const headers = await authHeaders(app);
const res = await app.inject({
method: "GET",
url: "/api/v1/settings",
headers,
});
expect(res.statusCode).toBe(200);
const body = res.json() as {
healthCheckCron: string;
healthDegradedFailures: number;
healthDownFailures: number;
healthLatencyWarnMs: number;
healthSuccessRecoveries: number;
};
expect(body.healthCheckCron).toBe("*/30 * * * * *");
expect(body.healthDegradedFailures).toBe(3);
expect(body.healthDownFailures).toBe(4);
expect(body.healthLatencyWarnMs).toBe(1500);
expect(body.healthSuccessRecoveries).toBe(5);
await app.close();
});
it("PATCH persists health engine settings", async () => {
const app = await buildApp({
config: { ...loadConfig(), staticDir: null },
memory: true,
});
const headers = await authHeaders(app);
const res = await app.inject({
method: "PATCH",
url: "/api/v1/settings",
headers,
payload: {
healthCheckCron: "0 */5 * * * *",
healthDegradedFailures: 2,
healthDownFailures: 4,
healthLatencyWarnMs: 800,
healthSuccessRecoveries: 3,
},
});
expect(res.statusCode).toBe(200);
const body = res.json() as {
healthCheckCron: string;
healthDegradedFailures: number;
healthDownFailures: number;
healthLatencyWarnMs: number;
healthSuccessRecoveries: number;
};
expect(body.healthCheckCron).toBe("0 */5 * * * *");
expect(body.healthDegradedFailures).toBe(2);
expect(body.healthDownFailures).toBe(4);
expect(body.healthLatencyWarnMs).toBe(800);
expect(body.healthSuccessRecoveries).toBe(3);
const again = await app.inject({
method: "GET",
url: "/api/v1/settings",
headers,
});
expect(again.json()).toMatchObject({
healthCheckCron: "0 */5 * * * *",
healthDegradedFailures: 2,
healthDownFailures: 4,
healthLatencyWarnMs: 800,
healthSuccessRecoveries: 3,
});
await app.close();
});
it("PATCH rejects invalid cron", async () => {
const app = await buildApp({
config: { ...loadConfig(), staticDir: null },
memory: true,
});
const headers = await authHeaders(app);
const res = await app.inject({
method: "PATCH",
url: "/api/v1/settings",
headers,
payload: { healthCheckCron: "not-a-cron" },
});
expect(res.statusCode).toBe(400);
expect(res.json()).toMatchObject({
error: { code: "VALIDATION_ERROR" },
});
await app.close();
});
it("PATCH rejects down < degraded", async () => {
const app = await buildApp({
config: { ...loadConfig(), staticDir: null },
memory: true,
});
const headers = await authHeaders(app);
const res = await app.inject({
method: "PATCH",
url: "/api/v1/settings",
headers,
payload: {
healthDegradedFailures: 5,
healthDownFailures: 2,
},
});
expect(res.statusCode).toBe(400);
await app.close();
});
});
+13
View File
@@ -134,6 +134,19 @@ describe("resolveBindingIpsForSync", () => {
expect(ips).toEqual(["203.0.113.10"]);
});
it("treats missing target_ips as empty instead of throwing", async () => {
const cname = binding({
id: 2,
hostname: "imsk",
zone_name: "rkns.top",
cname_target: "ihome.rkns.top",
});
delete (cname as { target_ips?: string[] }).target_ips;
const index = { byFqdn: new Map([["imsk.rkns.top", cname]]) };
const ips = await resolveBindingIpsForSync(cname, ["198.51.100.9"], index);
expect(ips).toEqual(["198.51.100.9"]);
});
it("prefers service IPs over empty CNAME resolution chain", async () => {
const cname = binding({
id: 2,