fix(services): считать failover по DNS-пулу а не по статусу ноды
quality / commitlint (push) Skipped
CD / update-wiki (push) Successful in 7s
quality / changes (push) Successful in 9s
quality / api (push) Skipped
quality / docker-check (push) Skipped
CD / quality (push) Canceled after 34s
CD / publish (push) Canceled after 0s
quality / web (push) Canceled after 22s

Красный инцидент только если адреса нет в active_addresses, даже при unhealthy.

Co-authored-by: Cursor <[email protected]>
This commit is contained in:
Denozordec
2026-08-20 14:53:44 +07:00
co-authored by Cursor
parent 87b0f1a894
commit 994e79e118
5 changed files with 88 additions and 35 deletions
@@ -53,8 +53,8 @@ export function FailoverTimeline({ events }: { events: FailoverEvent[] }) {
return ( return (
<EmptyState <EmptyState
icon={ShieldCheckIcon} icon={ShieldCheckIcon}
title="Пул стабилен" title="Пул в DNS на месте"
description="События появятся, когда нода станет unhealthy или down и будет выведена из пула" description="Красным только адреса, которых нет в активном пуле A-записей"
stackedIcon={false} stackedIcon={false}
centered={false} centered={false}
/> />
@@ -93,8 +93,8 @@ export function FailoverTimeline({ events }: { events: FailoverEvent[] }) {
<TimelineContent className="flex flex-col gap-2"> <TimelineContent className="flex flex-col gap-2">
<p className="text-foreground text-sm"> <p className="text-foreground text-sm">
{isChecking {isChecking
? 'Health-check ещё не завершён — нода не в активном пуле' ? 'Снята с DNS, идёт восстановление'
: 'Нода выведена из активного пула'} : 'Снята с DNS — в A-записях этого адреса нет'}
</p> </p>
{event.lastFailureReason ? ( {event.lastFailureReason ? (
<code <code
@@ -40,10 +40,12 @@ function failoverCountLabel(count: number): string {
*/ */
export function ServiceFailoverPanel({ export function ServiceFailoverPanel({
nodes, nodes,
activeAddresses,
}: { }: {
nodes: readonly FailoverNodeInput[] nodes: readonly FailoverNodeInput[]
activeAddresses: readonly string[]
}) { }) {
const events = toFailoverEvents(nodes) const events = toFailoverEvents(nodes, activeAddresses)
return ( return (
<Frame stacked spacing="sm" className="min-w-0 w-full"> <Frame stacked spacing="sm" className="min-w-0 w-full">
@@ -62,7 +64,7 @@ export function ServiceFailoverPanel({
)} )}
</FrameTitle> </FrameTitle>
<FrameDescription> <FrameDescription>
Нездоровые ноды выведены из пула · причина последней ошибки Только адреса, которых нет в DNS-пуле · не статус строки ноды
</FrameDescription> </FrameDescription>
</FrameHeader> </FrameHeader>
@@ -71,7 +73,7 @@ export function ServiceFailoverPanel({
<UnplugIcon aria-hidden="true" /> <UnplugIcon aria-hidden="true" />
<AlertTitle>{failoverCountLabel(events.length)}</AlertTitle> <AlertTitle>{failoverCountLabel(events.length)}</AlertTitle>
<AlertDescription> <AlertDescription>
Трафик на эти адреса не идёт, пока health не восстановится Эти IP сняты с A-записей
</AlertDescription> </AlertDescription>
</Alert> </Alert>
) : null} ) : null}
+57 -19
View File
@@ -20,34 +20,72 @@ function node(
} }
describe('toFailoverEvents', () => { describe('toFailoverEvents', () => {
it('оставляет только unhealthy / down / checking', () => { it('не считает healthy failover-событием', () => {
expect(isFailoverEventStatus('healthy')).toBe(false)
expect(isFailoverEventStatus('unhealthy')).toBe(true)
})
it('без DNS-пула ничего не показывает — нельзя врать про вывод', () => {
const events = toFailoverEvents([ const events = toFailoverEvents([
node({ address: '10.0.0.1', health_status: 'healthy' }),
node({ node({
address: '130.49.213.153', address: '130.49.213.153',
health_status: 'unhealthy', health_status: 'unhealthy',
consecutive_failures: 9, consecutive_failures: 9,
last_failure_reason: 'fetch failed', last_failure_reason: 'fetch failed',
last_check_at: '2026-08-20 07:00:00',
}), }),
node({ address: '10.0.0.3', health_status: 'checking' }),
node({ address: '10.0.0.4', health_status: 'disabled' }),
]) ])
expect(events).toEqual([])
expect(events.map((event) => event.address)).toEqual([
'130.49.213.153',
'10.0.0.3',
])
expect(events[0]).toMatchObject({
consecutiveFailures: 9,
lastFailureReason: 'fetch failed',
lastCheckAt: '2026-08-20 07:00:00',
status: 'unhealthy',
})
}) })
it('не считает healthy failover-событием', () => { it('нездоровый адрес в DNS-пуле не инцидент (last-resort / ещё в A-записи)', () => {
expect(isFailoverEventStatus('healthy')).toBe(false) const events = toFailoverEvents(
expect(isFailoverEventStatus('unhealthy')).toBe(true) [
node({
address: '130.49.213.153',
health_status: 'unhealthy',
consecutive_failures: 9,
last_failure_reason: 'fetch failed',
}),
node({ address: '10.0.0.3', health_status: 'checking' }),
],
['130.49.213.153', '10.0.0.2'],
)
expect(events.map((event) => event.address)).toEqual(['10.0.0.3'])
})
it('здоровый standby вне пула не инцидент', () => {
const events = toFailoverEvents(
[
node({ address: '10.0.0.1', health_status: 'healthy' }),
node({ address: '10.0.0.2', health_status: 'healthy' }),
],
['10.0.0.1'],
)
expect(events).toEqual([])
})
it('нездоровый адрес без A-записи — реальный вывод из пула', () => {
const events = toFailoverEvents(
[
node({
address: '130.49.213.153',
health_status: 'unhealthy',
consecutive_failures: 9,
last_failure_reason: 'fetch failed',
last_check_at: '2026-08-20 07:00:00',
}),
],
['10.0.0.1'],
)
expect(events).toEqual([
{
id: '130.49.213.153',
address: '130.49.213.153',
status: 'unhealthy',
consecutiveFailures: 9,
lastFailureReason: 'fetch failed',
lastCheckAt: '2026-08-20 07:00:00',
},
])
}) })
}) })
+18 -8
View File
@@ -22,15 +22,25 @@ export function isFailoverEventStatus(status: string): boolean {
return FAILOVER_STATUSES.has(status) return FAILOVER_STATUSES.has(status)
} }
/**
* Инцидент failover = нездоровый адрес, которого нет в DNS-пуле.
* Нода в activeAddresses (в т.ч. last-resort) — не «выведена».
*/
export function toFailoverEvents( export function toFailoverEvents(
nodes: readonly FailoverNodeInput[], nodes: readonly FailoverNodeInput[],
activeAddresses: readonly string[] = [],
): FailoverEvent[] { ): FailoverEvent[] {
return nodes.filter((node) => isFailoverEventStatus(node.health_status)).map((node) => ({ const pool = new Set(activeAddresses)
id: String(node.id ?? node.address), if (pool.size === 0) return []
address: node.address,
status: node.health_status, return nodes
consecutiveFailures: node.consecutive_failures, .filter((node) => isFailoverEventStatus(node.health_status) && !pool.has(node.address))
lastFailureReason: node.last_failure_reason, .map((node) => ({
lastCheckAt: node.last_check_at ?? null, id: String(node.id ?? node.address),
})) address: node.address,
status: node.health_status,
consecutiveFailures: node.consecutive_failures,
lastFailureReason: node.last_failure_reason,
lastCheckAt: node.last_check_at ?? null,
}))
} }
@@ -313,7 +313,10 @@ function ServiceDetailPage() {
statuses={providerStatuses} statuses={providerStatuses}
isLoading={logQuery.isLoading} isLoading={logQuery.isLoading}
/> />
<ServiceFailoverPanel nodes={failoverNodes} /> <ServiceFailoverPanel
nodes={failoverNodes}
activeAddresses={overview?.active_addresses ?? service.active_ips}
/>
</section> </section>
{service.ips.length === 0 && service.domains.length === 0 ? ( {service.ips.length === 0 && service.domains.length === 0 ? (