fix(services): не балансировать сервис с одним IP
quality / commitlint (push) Skipped
CD / update-wiki (push) Successful in 5s
quality / changes (push) Successful in 8s
quality / docker-check (push) Skipped
quality / web (push) Successful in 55s
quality / api (push) Successful in 52s
CD / quality (push) Successful in 2m4s
CD / publish (push) Successful in 1m51s

KPI и карточка показывают живой OK, а не гистерезис unknown; DNS по-прежнему ждёт повторные успехи.

Co-authored-by: Cursor <[email protected]>
This commit is contained in:
Denozordec
2026-08-20 18:13:06 +07:00
co-authored by Cursor
parent a9a84fabac
commit f1443f1db5
13 changed files with 464 additions and 72 deletions
+19 -6
View File
@@ -9,7 +9,10 @@ import type {
import { AppError } from "../errors.js"; import { AppError } from "../errors.js";
import { isValidIpv4 } from "../lib/validators.js"; import { isValidIpv4 } from "../lib/validators.js";
import { getView } from "./service-config-service.js"; import { getView } from "./service-config-service.js";
import { selectActiveIpsByMode } from "./routing/index.js"; import {
isSharedPool,
resolveDesiredAIps,
} from "./routing/index.js";
function assertAddress(address: string): void { function assertAddress(address: string): void {
if (!isValidIpv4(address)) { if (!isValidIpv4(address)) {
@@ -89,8 +92,12 @@ export async function getOverview(
: null; : null;
const active = new Set<string>(); const active = new Set<string>();
const serviceIps = service.ips.filter(
(ip) => service.ip_enabled[ip] !== false,
);
for (const binding of bindings) { for (const binding of bindings) {
const metas = repos.listBindingIpsWithMeta(db, binding.id); const metas = repos.listBindingIpsWithMeta(db, binding.id);
const targetIps = metas.map((entry) => entry.ip);
const rows = metas.map((entry) => { const rows = metas.map((entry) => {
const status = repos.getIpHealthStatusRow(db, "binding", binding.id, entry.ip); const status = repos.getIpHealthStatusRow(db, "binding", binding.id, entry.ip);
return { return {
@@ -100,12 +107,15 @@ export async function getOverview(
health: status ? status.status : ("unknown" as const), health: status ? status.status : ("unknown" as const),
}; };
}); });
for (const ip of selectActiveIpsByMode( for (const ip of resolveDesiredAIps(
{ {
lb_mode: binding.lb_mode, lb_mode: binding.lb_mode,
health_check_enabled: binding.health_check_enabled, health_check_enabled: binding.health_check_enabled,
}, },
rows, rows,
targetIps,
Date.now(),
serviceIps,
)) { )) {
active.add(ip); active.add(ip);
} }
@@ -134,10 +144,13 @@ export function opsSummary(db: Db) {
if (binding.lb_mode !== "failover" || !binding.health_check_enabled) { if (binding.lb_mode !== "failover" || !binding.health_check_enabled) {
return false; return false;
} }
return binding.target_ips.some((ip) => { return (
const row = repos.getIpHealthStatusRow(db, "binding", binding.id, ip); isSharedPool(binding.target_ips) &&
return row?.status === "down"; binding.target_ips.some((ip) => {
}); const row = repos.getIpHealthStatusRow(db, "binding", binding.id, ip);
return row?.status === "down";
})
);
}).length; }).length;
return { return {
domains: domains.length, domains: domains.length,
+10 -3
View File
@@ -1,6 +1,6 @@
import type { LbMode } from "@cfdm/shared"; import type { LbMode } from "@cfdm/shared";
import { failoverDesired } from "./failover.js"; import { failoverDesired } from "./failover.js";
import { isSharedPool } from "./pool.js"; import { canApplyLb, isSharedPool } from "./pool.js";
import { roundRobinDesired } from "./round-robin.js"; import { roundRobinDesired } from "./round-robin.js";
import type { LbIpRow, LbTargetConfig } from "./types.js"; import type { LbIpRow, LbTargetConfig } from "./types.js";
import { weightedDesired } from "./weighted.js"; import { weightedDesired } from "./weighted.js";
@@ -8,7 +8,12 @@ import { weightedDesired } from "./weighted.js";
export type { LbIpRow, LbTargetConfig } from "./types.js"; export type { LbIpRow, LbTargetConfig } from "./types.js";
export { isHealthy } from "./health.js"; export { isHealthy } from "./health.js";
export { withBindingLock } from "./binding-lock.js"; export { withBindingLock } from "./binding-lock.js";
export { isSharedPool, shouldRecordFailoverDnsDiff } from "./pool.js"; export {
canApplyLb,
isSharedPool,
shouldRecordFailoverDnsDiff,
uniqueIpCount,
} from "./pool.js";
export { WEIGHTED_DNS_TTL, WEIGHTED_SLOT_MS, weightedDesired } from "./weighted.js"; export { WEIGHTED_DNS_TTL, WEIGHTED_SLOT_MS, weightedDesired } from "./weighted.js";
export function selectActiveIpsByMode( export function selectActiveIpsByMode(
@@ -31,9 +36,11 @@ export function resolveDesiredAIps(
rows: LbIpRow[], rows: LbIpRow[],
fallbackIps: readonly string[], fallbackIps: readonly string[],
nowMs = Date.now(), nowMs = Date.now(),
serviceIps: readonly string[] = fallbackIps,
): string[] { ): string[] {
const fallback = [...fallbackIps]; const fallback = [...fallbackIps];
if (!isSharedPool(fallback)) return fallback; if (!canApplyLb(serviceIps, fallback)) return fallback;
if (!isSharedPool(rows.map((row) => row.ip))) return fallback;
if (config.lb_mode === "weighted" || config.health_check_enabled) { if (config.lb_mode === "weighted" || config.health_check_enabled) {
const activeIps = selectActiveIpsByMode(config, rows, nowMs); const activeIps = selectActiveIpsByMode(config, rows, nowMs);
if (activeIps.length > 0) return activeIps; if (activeIps.length > 0) return activeIps;
+14 -2
View File
@@ -1,8 +1,20 @@
import type { LbMode } from "@cfdm/shared"; import type { LbMode } from "@cfdm/shared";
/** Shared pool FQDN — two or more A targets. Dedicated extra-FQDN has one IP. */ export function uniqueIpCount(ips: readonly string[]): number {
return new Set(ips.filter(Boolean)).size;
}
/** Shared pool — two or more unique IPs. One IP (even duplicated) is not a pool. */
export function isSharedPool(ips: readonly string[]): boolean { export function isSharedPool(ips: readonly string[]): boolean {
return ips.length >= 2; return uniqueIpCount(ips) >= 2;
}
/** LB / drain only when the service itself has a pool AND this FQDN is shared. */
export function canApplyLb(
serviceIps: readonly string[],
bindingIps: readonly string[],
): boolean {
return isSharedPool(serviceIps) && isSharedPool(bindingIps);
} }
export function shouldRecordFailoverDnsDiff(input: { export function shouldRecordFailoverDnsDiff(input: {
+75 -18
View File
@@ -29,6 +29,7 @@ import * as domainService from "./domain-service.js";
import { syncServiceToVpsTracker } from "./vps-tracker-sync.js"; import { syncServiceToVpsTracker } from "./vps-tracker-sync.js";
import { fireEnsureHealthWorker, DEFAULT_HEALTH_FALLBACKS } from "./health/health-worker-deploy.js"; import { fireEnsureHealthWorker, DEFAULT_HEALTH_FALLBACKS } from "./health/health-worker-deploy.js";
import { import {
canApplyLb,
isHealthy, isHealthy,
isSharedPool, isSharedPool,
resolveDesiredAIps, resolveDesiredAIps,
@@ -41,12 +42,24 @@ import {
} from "./routing/index.js"; } from "./routing/index.js";
export type { LbIpRow, LbTargetConfig }; export type { LbIpRow, LbTargetConfig };
export { resolveDesiredAIps, selectActiveIpsByMode, shouldRecordFailoverDnsDiff }; export {
canApplyLb,
resolveDesiredAIps,
selectActiveIpsByMode,
shouldRecordFailoverDnsDiff,
};
const AUTO_DNS_TTL = 1; const AUTO_DNS_TTL = 1;
function ttlForBinding(mode: LbMode, ipCount: number): number { function ttlForBinding(mode: LbMode, ips: readonly string[]): number {
return mode === "weighted" && ipCount >= 2 ? WEIGHTED_DNS_TTL : AUTO_DNS_TTL; return mode === "weighted" && isSharedPool(ips) ? WEIGHTED_DNS_TTL : AUTO_DNS_TTL;
}
function enabledServiceIps(db: Db, serviceId: number): string[] {
return repos
.listServiceIpRows(db, serviceId)
.filter((row) => row.enabled)
.map((row) => row.ip);
} }
export function failoverARecordDiff( export function failoverARecordDiff(
@@ -300,7 +313,17 @@ function desiredAIps(
scope === "binding" scope === "binding"
? getBindingLbState(db, refId) ? getBindingLbState(db, refId)
: getGroupLbState(db, refId); : getGroupLbState(db, refId);
return resolveDesiredAIps(state.config, state.rows, fallbackIps); const serviceIps =
scope === "binding"
? enabledServiceIps(db, repos.getBinding(db, refId).service_id)
: fallbackIps;
return resolveDesiredAIps(
state.config,
state.rows,
fallbackIps,
Date.now(),
serviceIps,
);
} }
async function collectKnownZones( async function collectKnownZones(
@@ -352,7 +375,7 @@ async function buildView(db: Db, serviceId: number): Promise<ServiceView> {
const { config, rows } = getBindingLbState(db, binding.id); const { config, rows } = getBindingLbState(db, binding.id);
const bindingActiveIps = targetCname const bindingActiveIps = targetCname
? [] ? []
: resolveDesiredAIps(config, rows, targetIps); : resolveDesiredAIps(config, rows, targetIps, Date.now(), ips);
return { return {
binding_id: binding.id, binding_id: binding.id,
@@ -473,6 +496,21 @@ function fallbackCnameHealth(
); );
} }
function overlayLiveHealth(
stored: IpHealthState | undefined,
live: IpHealthState | undefined,
): IpHealthState {
if (live && live !== "unknown") return live;
return stored ?? live ?? "unknown";
}
function bestAliveDisplayStatus(statuses: readonly string[]): IpHealthState {
if (statuses.some((status) => status === "up")) return "up";
if (statuses.some((status) => status === "degraded")) return "degraded";
if (statuses.some((status) => status === "down")) return "down";
return "unknown";
}
function attachServiceHealth( function attachServiceHealth(
db: Db, db: Db,
views: ServiceView[], views: ServiceView[],
@@ -480,10 +518,13 @@ function attachServiceHealth(
const ids = views.map((v) => v.id); const ids = views.map((v) => v.id);
const healthByService = repos.aggregateIpHealthByServiceIds(db, ids); const healthByService = repos.aggregateIpHealthByServiceIds(db, ids);
const ipHealthByService = repos.listIpHealthByServiceIds(db, ids); const ipHealthByService = repos.listIpHealthByServiceIds(db, ids);
const liveByService = repos.listLatestLiveHealthByServiceIds(db, ids);
return views.map((view) => { return views.map((view) => {
const health = healthByService.get(view.id); const health = healthByService.get(view.id);
const rows = ipHealthByService.get(view.id) ?? []; const rows = ipHealthByService.get(view.id) ?? [];
const liveRows = liveByService.get(view.id) ?? [];
const byIp = new Map(rows.map((row) => [row.ip, row])); const byIp = new Map(rows.map((row) => [row.ip, row]));
const liveByIp = new Map(liveRows.map((row) => [row.ip, row]));
const cnameFallback = fallbackCnameHealth(rows, view); const cnameFallback = fallbackCnameHealth(rows, view);
const aRecordIps = new Set( const aRecordIps = new Set(
(view.domains ?? []).flatMap((domain) => (view.domains ?? []).flatMap((domain) =>
@@ -492,20 +533,33 @@ function attachServiceHealth(
); );
const ip_health = (view.ips ?? []).map((ip) => { const ip_health = (view.ips ?? []).map((ip) => {
const row = byIp.get(ip) ?? (aRecordIps.has(ip) ? undefined : cnameFallback); const row = byIp.get(ip) ?? (aRecordIps.has(ip) ? undefined : cnameFallback);
const live = liveByIp.get(ip);
const status = overlayLiveHealth(row?.status, live?.status);
const extras = live && live.status !== "unknown" ? live : row;
return { return {
ip, ip,
status: row?.status ?? ("unknown" as const), status,
latency_ms: row?.latency_ms ?? null, latency_ms: extras?.latency_ms ?? null,
last_checked_at: row?.last_checked_at ?? null, last_checked_at: extras?.last_checked_at ?? null,
last_error: row?.last_error ?? null, last_error:
provider: row?.provider ?? "local", live && live.status !== "unknown"
colo: row?.colo ?? null, ? live.last_error
: (row?.last_error ?? null),
provider: extras?.provider ?? "local",
colo: extras?.colo ?? null,
}; };
}); });
const displayStatus = bestAliveDisplayStatus(ip_health.map((row) => row.status));
const latencyRow =
ip_health.find((row) => row.status === displayStatus && row.latency_ms != null) ??
ip_health.find((row) => row.latency_ms != null);
return { return {
...view, ...view,
health_status: health?.health_status ?? "unknown", health_status: overlayLiveHealth(health?.health_status, displayStatus),
health_latency_ms: health?.health_latency_ms ?? null, health_latency_ms:
displayStatus !== "unknown"
? (latencyRow?.latency_ms ?? null)
: (health?.health_latency_ms ?? null),
ip_health, ip_health,
}; };
}); });
@@ -649,7 +703,7 @@ async function syncBindingDns(
domainId, domainId,
hostname, hostname,
desiredIps, desiredIps,
ttlForBinding(binding.lb_mode, configuredIps.length), ttlForBinding(binding.lb_mode, configuredIps),
); );
} }
@@ -1212,7 +1266,7 @@ async function syncGroupDomainDns(
domainId, domainId,
hostname, hostname,
desiredIps, desiredIps,
ttlForBinding(group.lb_mode, fallbackIps.length), ttlForBinding(group.lb_mode, fallbackIps),
); );
} }
@@ -1705,10 +1759,12 @@ export async function reconcileDnsForTarget(
const cnameTarget = binding.cname_target?.trim() || null; const cnameTarget = binding.cname_target?.trim() || null;
if (cnameTarget) return; if (cnameTarget) return;
const ips = repos.listServiceIps(db, service.id); const ips = repos.listServiceIps(db, service.id);
const poolIps = enabledServiceIps(db, service.id);
const targetIps = repos.listBindingIps(db, binding.id); const targetIps = repos.listBindingIps(db, binding.id);
if (!isSharedPool(targetIps)) return;
validateTargetIpsInPool(targetIps, ips); validateTargetIpsInPool(targetIps, ips);
const desiredIps = desiredAIps(db, "binding", refId, targetIps); const desiredIps = canApplyLb(poolIps, targetIps)
? desiredAIps(db, "binding", refId, targetIps)
: targetIps;
await syncBindingDns( await syncBindingDns(
db, db,
cf, cf,
@@ -1749,7 +1805,8 @@ export async function reconcileWeightedDns(
const service = repos.getService(db, latest.service_id); const service = repos.getService(db, latest.service_id);
if (!shouldPushDns(db, service)) return; if (!shouldPushDns(db, service)) return;
const targetIps = repos.listBindingIps(db, latest.id); const targetIps = repos.listBindingIps(db, latest.id);
if (!isSharedPool(targetIps)) return; const poolIps = enabledServiceIps(db, service.id);
if (!canApplyLb(poolIps, targetIps)) return;
const ips = repos.listServiceIps(db, service.id); const ips = repos.listServiceIps(db, service.id);
validateTargetIpsInPool(targetIps, ips); validateTargetIpsInPool(targetIps, ips);
const desiredIps = desiredAIps(db, "binding", latest.id, targetIps); const desiredIps = desiredAIps(db, "binding", latest.id, targetIps);
+42
View File
@@ -436,4 +436,46 @@ describe("CNAME health mapped onto service IPs", () => {
const view = await getView(db, service.id); const view = await getView(db, service.id);
expect(view.health_status).toBe("up"); expect(view.health_status).toBe("up");
}); });
it("getView shows live OK over hysteresis unknown", async () => {
const { createMemoryDb, repos, runMigrations } = await import("@cfdm/db");
const { getView } = await import("../src/services/service-config-service.js");
const { db, sqlite } = createMemoryDb();
runMigrations(sqlite);
const domain = repos.createDomain(db, null, "rkns.top", "zone-id");
const service = repos.createService(db, "RW Panel", "rw-panel");
repos.replaceServiceIps(db, service.id, ["2.59.161.102"]);
const binding = repos.insertBinding(db, domain.id, service.id, "c", null);
repos.replaceBindingIpsWithMeta(db, binding.id, [
{ ip: "2.59.161.102", weight: 1, priority: 1 },
]);
repos.upsertIpHealthStatus(
db,
"binding",
binding.id,
"2.59.161.102",
"unknown",
63,
0,
null,
1,
);
repos.insertHealthProbeLog(db, {
scope: "binding",
refId: binding.id,
ip: "2.59.161.102",
provider: "local",
status: "up",
ok: true,
latencyMs: 63,
colo: null,
error: null,
});
const view = await getView(db, service.id);
expect(view.health_status).toBe("up");
expect(view.ip_health[0]?.status).toBe("up");
expect(view.ip_health[0]?.latency_ms).toBe(63);
});
}); });
+40
View File
@@ -158,6 +158,34 @@ describe("resolveDesiredAIps", () => {
).toEqual(["1.1.1.1"]); ).toEqual(["1.1.1.1"]);
}); });
it("does not drain when the service has only one unique IP", () => {
expect(
resolveDesiredAIps(
weightedConfig,
[row("1.1.1.1", { health: "down" })],
["1.1.1.1", "1.1.1.1"],
0,
["1.1.1.1"],
),
).toEqual(["1.1.1.1", "1.1.1.1"]);
});
it("does not apply overlay when service pool is a single IP", () => {
const rows = [
row("1.1.1.1", { health: "up" }),
row("2.2.2.2", { health: "down" }),
];
expect(
resolveDesiredAIps(
weightedConfig,
rows,
["1.1.1.1", "2.2.2.2"],
0,
["1.1.1.1"],
),
).toEqual(["1.1.1.1", "2.2.2.2"]);
});
it("applies weighted overlay on a shared pool", () => { it("applies weighted overlay on a shared pool", () => {
const rows = [ const rows = [
row("1.1.1.1", { weight: 1, health: "up" }), row("1.1.1.1", { weight: 1, health: "up" }),
@@ -170,6 +198,18 @@ describe("resolveDesiredAIps", () => {
}); });
describe("shouldRecordFailoverDnsDiff", () => { describe("shouldRecordFailoverDnsDiff", () => {
it("skips duplicate listings of the same IP", () => {
expect(
shouldRecordFailoverDnsDiff({
configuredIps: ["1.1.1.1", "1.1.1.1"],
lbMode: "failover",
added: [],
removed: ["1.1.1.1"],
downIps: new Set(["1.1.1.1"]),
}),
).toBe(false);
});
it("skips dedicated extra-FQDN", () => { it("skips dedicated extra-FQDN", () => {
expect( expect(
shouldRecordFailoverDnsDiff({ shouldRecordFailoverDnsDiff({
+42 -17
View File
@@ -51,7 +51,28 @@ describe('toFailoverEvents', () => {
expect(isFailoverEventStatus('unhealthy')).toBe(false) expect(isFailoverEventStatus('unhealthy')).toBe(false)
}) })
it('Down всегда виден, даже без A-записей', () => { it('один IP у сервиса — не инцидент пула', () => {
const events = toFailoverEvents(
[
row({
ip: '10.0.0.1',
status: 'down',
consecutive_failures: 9,
last_error: 'fetch failed',
}),
],
[
{
fqdn: 'solo.example.com',
configured: ['10.0.0.1'],
active: [],
},
],
)
expect(events).toEqual([])
})
it('Down без shared pool не событие балансировки', () => {
const events = toFailoverEvents([ const events = toFailoverEvents([
row({ row({
ip: '130.49.213.153', ip: '130.49.213.153',
@@ -60,19 +81,7 @@ describe('toFailoverEvents', () => {
last_error: 'fetch failed', last_error: 'fetch failed',
}), }),
]) ])
expect(events).toEqual([ expect(events).toEqual([])
{
id: '130.49.213.153',
address: '130.49.213.153',
status: 'down',
kind: 'last-resort',
fqdns: [],
consecutiveFailures: 9,
lastFailureReason: 'fetch failed',
lastCheckAt: null,
},
])
expect(failoverEventCopy(events[0]!)).toBe('Down, в A-записях last-resort')
}) })
it('OK вне пула не инцидент (standby)', () => { it('OK вне пула не инцидент (standby)', () => {
@@ -151,9 +160,7 @@ describe('toFailoverEvents', () => {
}, },
], ],
) )
expect(events[0]?.kind).toBe('last-resort') expect(events).toEqual([])
expect(events[0]?.fqdns).toEqual([])
expect(failoverEventCopy(events[0]!)).toBe('Down, в A-записях last-resort')
}) })
it('down без A-записи на configured FQDN — снятие', () => { it('down без A-записи на configured FQDN — снятие', () => {
@@ -293,6 +300,24 @@ describe('mergeFailoverHistory', () => {
expect(history[0]?.source).toBe('dns') expect(history[0]?.source).toBe('dns')
}) })
it('drops probe leave/return when the service has no shared pool', () => {
const history = mergeFailoverHistory(
[],
[
probe({ id: 1, status: 'up', checked_at: '2026-08-20T09:00:00Z' }),
probe({ id: 2, status: 'down', checked_at: '2026-08-20T09:10:00Z' }),
],
[
{
fqdn: 'solo.example.com',
configured: ['130.49.213.153'],
active: ['130.49.213.153'],
},
],
)
expect(history).toEqual([])
})
it('drops dedicated extra-FQDN DNS rows', () => { it('drops dedicated extra-FQDN DNS rows', () => {
const history = mergeFailoverHistory( const history = mergeFailoverHistory(
[ [
+19 -7
View File
@@ -44,9 +44,20 @@ export interface FailoverBindingPool {
active: readonly string[] active: readonly string[]
} }
/** Shared pool FQDN — two or more A targets. Dedicated extra-FQDN has one IP. */ /** Unique A targets. Duplicates of the same IP are not a pool. */
export function uniqueIpCount(ips: readonly string[]): number {
return new Set(ips.filter(Boolean)).size
}
/** Shared pool FQDN — two or more unique A targets. */
export function isSharedPoolBinding(binding: FailoverBindingPool): boolean { export function isSharedPoolBinding(binding: FailoverBindingPool): boolean {
return binding.configured.length >= 2 return uniqueIpCount(binding.configured) >= 2
}
export function hasSharedPool(
bindings: readonly FailoverBindingPool[],
): boolean {
return bindings.some(isSharedPoolBinding)
} }
/** Инцидент только при down. up / degraded / unknown — не вывод из пула. */ /** Инцидент только при down. up / degraded / unknown — не вывод из пула. */
@@ -66,13 +77,14 @@ export function failoverEventCopy(event: FailoverEvent): string {
} }
/** /**
* Текущие Down всегда в панели. Per-FQDN: снята с hostname или last-resort. * Инциденты пула только если у сервиса есть shared FQDN (2+ уникальных IP).
* Standby (up / degraded / unknown) — не инцидент. * Один IP — не балансировка и не вывод из пула; Down смотрит health-монитор.
*/ */
export function toFailoverEvents( export function toFailoverEvents(
ipHealth: readonly FailoverHealthInput[], ipHealth: readonly FailoverHealthInput[],
bindings: readonly FailoverBindingPool[] = [], bindings: readonly FailoverBindingPool[] = [],
): FailoverEvent[] { ): FailoverEvent[] {
if (!hasSharedPool(bindings)) return []
return ipHealth.filter((row) => isFailoverEventStatus(row.status)).map((row) => { return ipHealth.filter((row) => isFailoverEventStatus(row.status)).map((row) => {
const removedFqdns: string[] = [] const removedFqdns: string[] = []
const lastResortFqdns: string[] = [] const lastResortFqdns: string[] = []
@@ -221,9 +233,9 @@ export function mergeFailoverHistory(
...dns ...dns
.filter((item) => isPoolFqdn(item.fqdn, bindings)) .filter((item) => isPoolFqdn(item.fqdn, bindings))
.map(dnsHistoryItem), .map(dnsHistoryItem),
...toIpAliveTransitions(probes).map((transition) => ...toIpAliveTransitions(probes)
probeHistoryItem(transition, bindings), .filter((transition) => fqdnsForIp(transition.ip, bindings).length > 0)
), .map((transition) => probeHistoryItem(transition, bindings)),
] ]
merged.sort( merged.sort(
(a, b) => eventTime(b.created_at) - eventTime(a.created_at) || a.id.localeCompare(b.id), (a, b) => eventTime(b.created_at) - eventTime(a.created_at) || a.id.localeCompare(b.id),
+20
View File
@@ -7,6 +7,7 @@ import {
latestHealthByIp, latestHealthByIp,
providerHealthStatuses, providerHealthStatuses,
resolveIpDisplayHealth, resolveIpDisplayHealth,
resolveServiceDisplayHealth,
worstHealthStatus, worstHealthStatus,
type HealthLogProbe, type HealthLogProbe,
} from '@/lib/health-log' } from '@/lib/health-log'
@@ -146,3 +147,22 @@ describe('resolveIpDisplayHealth', () => {
expect(resolveIpDisplayHealth('unknown', undefined)).toBe('unknown') expect(resolveIpDisplayHealth('unknown', undefined)).toBe('unknown')
}) })
}) })
describe('resolveServiceDisplayHealth', () => {
it('shows OK when live probes recovered and stored is still unknown', () => {
expect(
resolveServiceDisplayHealth(
'unknown',
[{ ip: '2.59.161.102', status: 'unknown' }],
[
probe({
id: 1,
ip: '2.59.161.102',
status: 'up',
checked_at: '2026-08-20T18:00:00Z',
}),
],
),
).toBe('up')
})
})
+19
View File
@@ -194,3 +194,22 @@ export function resolveIpDisplayHealth(
if (live && live !== 'unknown') return live if (live && live !== 'unknown') return live
return stored ?? live ?? 'unknown' return stored ?? live ?? 'unknown'
} }
/** Service KPI / card: any-up of per-IP overlay (live probes beat hysteresis). */
export function resolveServiceDisplayHealth(
stored: HealthLogStatus | undefined,
ipHealth: readonly { ip: string; status: string }[],
probes: readonly HealthLogProbe[] = [],
): HealthLogStatus {
const liveByIp = latestHealthByIp(probes)
const statuses =
ipHealth.length > 0
? ipHealth.map((row) =>
resolveIpDisplayHealth(
row.status as HealthLogStatus,
liveByIp.get(row.ip)?.status,
),
)
: [...liveByIp.values()].map((row) => row.status)
return resolveIpDisplayHealth(stored, bestAliveHealthStatus(statuses))
}
+6
View File
@@ -36,6 +36,8 @@ export const serviceGroupsQueryOptions = () =>
} }
return parsed.data return parsed.data
}, },
refetchInterval: 10_000,
staleTime: 5_000,
}) })
export const servicesQueryOptions = () => export const servicesQueryOptions = () =>
@@ -101,6 +103,8 @@ export const serviceViewQueryOptions = (id: number) =>
const data = await api.get<unknown>(`/api/v1/services/${id}`) const data = await api.get<unknown>(`/api/v1/services/${id}`)
return serviceViewSchema.parse(data) return serviceViewSchema.parse(data)
}, },
refetchInterval: 10_000,
staleTime: 5_000,
}) })
export const serviceHealthLogQueryOptions = (id: number) => export const serviceHealthLogQueryOptions = (id: number) =>
@@ -110,6 +114,8 @@ export const serviceHealthLogQueryOptions = (id: number) =>
const data = await api.get<unknown>(`/api/v1/services/${id}/health-log`) const data = await api.get<unknown>(`/api/v1/services/${id}/health-log`)
return z.object({ items: z.array(healthProbeLogSchema) }).parse(data) return z.object({ items: z.array(healthProbeLogSchema) }).parse(data)
}, },
refetchInterval: 10_000,
staleTime: 5_000,
}) })
export const serviceFailoverLogQueryOptions = (id: number) => export const serviceFailoverLogQueryOptions = (id: number) =>
@@ -32,9 +32,11 @@ import {
ServiceHealthMonitor, ServiceHealthMonitor,
} from '@/components/reui-kit' } from '@/components/reui-kit'
import { api } from '@/lib/api-client' import { api } from '@/lib/api-client'
import { hasSharedPool } from '@/lib/failover-events'
import { import {
enabledHealthProviders, enabledHealthProviders,
providerHealthStatuses, providerHealthStatuses,
resolveServiceDisplayHealth,
} from '@/lib/health-log' } from '@/lib/health-log'
import type { ServiceView, UpdateServiceConfigInput } from '@/lib/schemas' import type { ServiceView, UpdateServiceConfigInput } from '@/lib/schemas'
import { import {
@@ -115,6 +117,22 @@ function ServiceDetailPage() {
})), })),
[service?.domains], [service?.domains],
) )
const uniqueEnabledIps = useMemo(
() =>
(service?.ips ?? []).filter((ip) => service?.ip_enabled[ip] !== false),
[service?.ips, service?.ip_enabled],
)
const displayHealth = useMemo(
() =>
resolveServiceDisplayHealth(
service?.health_status,
service?.ip_health ?? [],
logItems,
),
[service?.health_status, service?.ip_health, logItems],
)
const showPoolPanel =
uniqueEnabledIps.length >= 2 && hasSharedPool(failoverBindings)
const nodes = (nodesQuery.data as OverviewPayload['nodes']) ?? [] const nodes = (nodesQuery.data as OverviewPayload['nodes']) ?? []
const [editOpen, setEditOpen] = useState(false) const [editOpen, setEditOpen] = useState(false)
@@ -251,7 +269,7 @@ function ServiceDetailPage() {
actions={ actions={
<> <>
<LbModeTile mode={service.lb_mode} /> <LbModeTile mode={service.lb_mode} />
<HealthCheckBadge status={service.health_status} /> <HealthCheckBadge status={displayHealth} />
<Tooltip> <Tooltip>
<TooltipTrigger <TooltipTrigger
render={ render={
@@ -278,20 +296,20 @@ function ServiceDetailPage() {
id: 'status', id: 'status',
icon: <ActivityIcon />, icon: <ActivityIcon />,
label: 'Статус', label: 'Статус',
value: service.health_status === 'up' ? 'OK' : service.health_status, value: displayHealth === 'up' ? 'OK' : displayHealth,
variant: variant:
service.health_status === 'down' displayHealth === 'down'
? 'destructive' ? 'destructive'
: service.health_status === 'degraded' : displayHealth === 'degraded'
? 'warning' ? 'warning'
: 'default', : 'default',
iconClassName: iconClassName:
service.health_status === 'down' displayHealth === 'down'
? 'text-destructive' ? 'text-destructive'
: service.health_status === 'degraded' : displayHealth === 'degraded'
? 'text-warning' ? 'text-warning'
: 'text-success', : 'text-success',
hint: <HealthCheckBadge status={service.health_status} size="xs" />, hint: <HealthCheckBadge status={displayHealth} size="xs" />,
}, },
{ {
id: 'fqdn', id: 'fqdn',
@@ -305,21 +323,33 @@ function ServiceDetailPage() {
icon: <NetworkIcon />, icon: <NetworkIcon />,
label: 'IP', label: 'IP',
value: String(service.ips.length), value: String(service.ips.length),
hint: `${service.active_ips.length} в пуле`, hint: showPoolPanel
? `${service.active_ips.length} в пуле`
: uniqueEnabledIps.length <= 1
? 'без балансировки'
: service.ips.join(', ') || 'нет',
}, },
{ {
id: 'pool', id: 'pool',
icon: <ServerIcon />, icon: <ServerIcon />,
label: 'Активный пул', label: showPoolPanel ? 'Активный пул' : 'Адреса',
value: String((overview?.active_addresses ?? service.active_ips).length), value: String(
hint: (overview?.active_addresses ?? service.active_ips).join(', ') || 'нет', (overview?.active_addresses ?? service.active_ips).length,
),
hint:
(overview?.active_addresses ?? service.active_ips).join(', ') ||
'нет',
}, },
]} ]}
/> />
<section <section
aria-label="Мониторинг" aria-label="Мониторинг"
className="grid min-w-0 items-start gap-2 @3xl:grid-cols-2" className={
showPoolPanel
? 'grid min-w-0 items-start gap-2 @3xl:grid-cols-2'
: 'grid min-w-0 items-start gap-2'
}
> >
<ServiceHealthMonitor <ServiceHealthMonitor
items={logItems} items={logItems}
@@ -327,13 +357,15 @@ function ServiceDetailPage() {
statuses={providerStatuses} statuses={providerStatuses}
isLoading={logQuery.isLoading} isLoading={logQuery.isLoading}
/> />
<ServiceFailoverPanel {showPoolPanel ? (
lbMode={service.lb_mode} <ServiceFailoverPanel
ipHealth={service.ip_health} lbMode={service.lb_mode}
bindings={failoverBindings} ipHealth={service.ip_health}
history={failoverHistory} bindings={failoverBindings}
probes={logItems} history={failoverHistory}
/> probes={logItems}
/>
) : null}
</section> </section>
{service.ips.length === 0 && service.domains.length === 0 ? ( {service.ips.length === 0 && service.domains.length === 0 ? (
+107
View File
@@ -2341,6 +2341,113 @@ export function listIpHealthByServiceIds(
return result; return result;
} }
function parseIpHealthState(status: string | null): IpHealthState | undefined {
if (
status === "up" ||
status === "down" ||
status === "degraded" ||
status === "unknown"
) {
return status;
}
return undefined;
}
function bestAliveIpState(statuses: readonly IpHealthState[]): IpHealthState {
if (statuses.some((status) => status === "up")) return "up";
if (statuses.some((status) => status === "degraded")) return "degraded";
if (statuses.some((status) => status === "down")) return "down";
return "unknown";
}
/**
* Latest probe per service+IP+provider from health_probe_log, then any-up per IP.
* Display overlay — hysteresis in ip_health_status is unchanged (DNS).
*/
export function listLatestLiveHealthByServiceIds(
db: Db,
serviceIds: number[],
): Map<number, ServiceIpHealthRow[]> {
const result = new Map<number, ServiceIpHealthRow[]>();
if (serviceIds.length === 0) return result;
const idList = sql.join(
serviceIds.map((id) => sql`${id}`),
sql`, `,
);
const rows = db.all<{
service_id: number;
ip: string;
provider: string | null;
status: string | null;
latency_ms: number | null;
last_error: string | null;
colo: string | null;
checked_at: string | null;
}>(sql`
SELECT sb.service_id AS service_id,
l.ip AS ip,
l.provider AS provider,
l.status AS status,
l.latency_ms AS latency_ms,
l.error AS last_error,
l.colo AS colo,
l.checked_at AS checked_at
FROM health_probe_log l
INNER JOIN service_bindings sb
ON l.scope = 'binding' AND l.ref_id = sb.id
INNER JOIN (
SELECT sb2.service_id AS service_id,
l2.ip AS ip,
l2.provider AS provider,
MAX(l2.id) AS max_id
FROM health_probe_log l2
INNER JOIN service_bindings sb2
ON l2.scope = 'binding' AND l2.ref_id = sb2.id
WHERE sb2.service_id IN (${idList})
GROUP BY sb2.service_id, l2.ip, l2.provider
) latest
ON latest.max_id = l.id
`);
const byServiceIp = new Map<
string,
{ serviceId: number; ip: string; probes: ServiceIpHealthRow[] }
>();
for (const row of rows) {
const status = parseIpHealthState(row.status);
if (!status) continue;
const key = `${row.service_id}\0${row.ip}`;
const probe: ServiceIpHealthRow = {
ip: row.ip,
status,
latency_ms: row.latency_ms,
last_checked_at: row.checked_at,
last_error: row.last_error,
provider: normalizeStatusProvider(row.provider),
colo: row.colo,
};
const bucket = byServiceIp.get(key);
if (bucket) bucket.probes.push(probe);
else {
byServiceIp.set(key, {
serviceId: row.service_id,
ip: row.ip,
probes: [probe],
});
}
}
for (const { serviceId, ip, probes } of byServiceIp.values()) {
const status = bestAliveIpState(probes.map((probe) => probe.status));
const preferred =
probes.find((probe) => probe.status === status) ?? probes[0]!;
const list = result.get(serviceId) ?? [];
list.push({ ...preferred, ip, status });
result.set(serviceId, list);
}
return result;
}
export function mergeHealthAggregates( export function mergeHealthAggregates(
parts: Array<HealthAggregate | undefined | null>, parts: Array<HealthAggregate | undefined | null>,
): HealthAggregate { ): HealthAggregate {