Files
cloudflare-domain-manager/apps/web/src/lib/failover-events.ts
T
DenozordecandCursor f1443f1db5
quality / commitlint (push) Skipped
CD / update-wiki (push) Successful in 5s
quality / changes (push) Successful in 8s
quality / docker-check (push) Skipped
quality / web (push) Successful in 55s
quality / api (push) Successful in 52s
CD / quality (push) Successful in 2m4s
CD / publish (push) Successful in 1m51s
fix(services): не балансировать сервис с одним IP
KPI и карточка показывают живой OK, а не гистерезис unknown; DNS по-прежнему ждёт повторные успехи.

Co-authored-by: Cursor <[email protected]>
2026-08-20 18:13:06 +07:00

263 lines
7.6 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import {
bestAliveHealthStatus,
probeTime,
type HealthLogProbe,
type HealthLogStatus,
} from '@/lib/health-log'
import type { FailoverLogEntry } from '@/lib/schemas'
export type FailoverEventKind = 'removed' | 'last-resort'
export interface FailoverHistoryItem {
id: string
ip: string
fqdn: string
action: 'added' | 'removed'
created_at: string
copy: string
source: 'dns' | 'probe'
}
export interface FailoverEvent {
id: string
address: string
status: string
kind: FailoverEventKind
fqdns: string[]
consecutiveFailures: number
lastFailureReason: string | null
lastCheckAt?: string | null
}
/** Binding IP health — тот же контур, что таблица активов. */
export interface FailoverHealthInput {
ip: string
status: string
consecutive_failures?: number
last_error?: string | null
last_checked_at?: string | null
}
export interface FailoverBindingPool {
fqdn: string
configured: readonly string[]
active: readonly string[]
}
/** Unique A targets. Duplicates of the same IP are not a pool. */
export function uniqueIpCount(ips: readonly string[]): number {
return new Set(ips.filter(Boolean)).size
}
/** Shared pool FQDN — two or more unique A targets. */
export function isSharedPoolBinding(binding: FailoverBindingPool): boolean {
return uniqueIpCount(binding.configured) >= 2
}
export function hasSharedPool(
bindings: readonly FailoverBindingPool[],
): boolean {
return bindings.some(isSharedPoolBinding)
}
/** Инцидент только при down. up / degraded / unknown — не вывод из пула. */
export function isFailoverEventStatus(status: string): boolean {
return status === 'down'
}
export function failoverEventCopy(event: FailoverEvent): string {
if (event.kind === 'removed') {
return event.fqdns.length > 0
? `Снята с ${event.fqdns.join(', ')}`
: 'Снята с DNS'
}
return event.fqdns.length > 0
? `Down, в A-записях last-resort на ${event.fqdns.join(', ')}`
: 'Down, в A-записях last-resort'
}
/**
* Инциденты пула только если у сервиса есть shared FQDN (2+ уникальных IP).
* Один IP — не балансировка и не вывод из пула; Down смотрит health-монитор.
*/
export function toFailoverEvents(
ipHealth: readonly FailoverHealthInput[],
bindings: readonly FailoverBindingPool[] = [],
): FailoverEvent[] {
if (!hasSharedPool(bindings)) return []
return ipHealth.filter((row) => isFailoverEventStatus(row.status)).map((row) => {
const removedFqdns: string[] = []
const lastResortFqdns: string[] = []
for (const binding of bindings) {
if (!isSharedPoolBinding(binding)) continue
const configured = binding.configured.includes(row.ip)
const active = binding.active.includes(row.ip)
if (configured && !active) removedFqdns.push(binding.fqdn)
else if (configured && active) lastResortFqdns.push(binding.fqdn)
}
const kind: FailoverEventKind =
removedFqdns.length > 0 ? 'removed' : 'last-resort'
return {
id: row.ip,
address: row.ip,
status: row.status,
kind,
fqdns: kind === 'removed' ? removedFqdns : lastResortFqdns,
consecutiveFailures: row.consecutive_failures ?? 0,
lastFailureReason: row.last_error ?? null,
lastCheckAt: row.last_checked_at ?? null,
}
})
}
const DEDUPE_WINDOW_MS = 2 * 60 * 1000
function fqdnsForIp(
ip: string,
bindings: readonly FailoverBindingPool[],
): string[] {
return bindings
.filter((binding) => isSharedPoolBinding(binding) && binding.configured.includes(ip))
.map((binding) => binding.fqdn)
}
function isPoolFqdn(
fqdn: string,
bindings: readonly FailoverBindingPool[],
): boolean {
const binding = bindings.find((item) => item.fqdn === fqdn)
if (!binding) return true
return isSharedPoolBinding(binding)
}
function fqdnLabel(fqdns: string[]): string {
return fqdns.join(', ') || 'пул'
}
function isAliveStatus(status: HealthLogStatus): boolean {
return status === 'up' || status === 'degraded'
}
/**
* Per-IP any-up flips: down → вышла из пула, up после down → вернулась.
* Initial state is not an event.
*/
export function toIpAliveTransitions(
probes: readonly HealthLogProbe[],
): Array<{ id: string; ip: string; alive: boolean; at: string }> {
const byIp = new Map<string, HealthLogProbe[]>()
for (const item of probes) {
const list = byIp.get(item.ip)
if (list) list.push(item)
else byIp.set(item.ip, [item])
}
const out: Array<{ id: string; ip: string; alive: boolean; at: string }> = []
for (const [ip, list] of byIp) {
list.sort(
(a, b) => probeTime(a.checked_at) - probeTime(b.checked_at) || a.id - b.id,
)
const latestByProvider = new Map<string, HealthLogProbe>()
let prevAlive: boolean | undefined
for (const probe of list) {
latestByProvider.set(probe.provider, probe)
const status = bestAliveHealthStatus(
[...latestByProvider.values()].map((item) => item.status),
)
if (status === 'unknown') continue
const alive = isAliveStatus(status)
if (prevAlive !== undefined && alive !== prevAlive) {
out.push({
id: `probe:${probe.id}`,
ip,
alive,
at: probe.checked_at,
})
}
prevAlive = alive
}
}
return out
}
function dnsHistoryItem(item: FailoverLogEntry): FailoverHistoryItem {
return {
id: `dns:${item.id}`,
ip: item.ip,
fqdn: item.fqdn,
action: item.action,
created_at: item.created_at,
copy:
item.action === 'removed'
? `${item.ip} убрана с ${item.fqdn}`
: `${item.ip} добавлена на ${item.fqdn}`,
source: 'dns',
}
}
function probeHistoryItem(
transition: { id: string; ip: string; alive: boolean; at: string },
bindings: readonly FailoverBindingPool[],
): FailoverHistoryItem {
const fqdns = fqdnsForIp(transition.ip, bindings)
const fqdn = fqdnLabel(fqdns)
const action = transition.alive ? 'added' : 'removed'
return {
id: transition.id,
ip: transition.ip,
fqdn,
action,
created_at: transition.at,
copy:
action === 'removed'
? `${transition.ip} вышла из пула (${fqdn})`
: `${transition.ip} вернулась в пул (${fqdn})`,
source: 'probe',
}
}
function eventTime(value: string): number {
return probeTime(value)
}
/**
* DNS add/remove + health leave/return, newest first.
* Same IP+action within 2 minutes: keep the DNS row (it has a concrete FQDN).
*/
export function mergeFailoverHistory(
dns: readonly FailoverLogEntry[],
probes: readonly HealthLogProbe[],
bindings: readonly FailoverBindingPool[] = [],
): FailoverHistoryItem[] {
const merged = [
...dns
.filter((item) => isPoolFqdn(item.fqdn, bindings))
.map(dnsHistoryItem),
...toIpAliveTransitions(probes)
.filter((transition) => fqdnsForIp(transition.ip, bindings).length > 0)
.map((transition) => probeHistoryItem(transition, bindings)),
]
merged.sort(
(a, b) => eventTime(b.created_at) - eventTime(a.created_at) || a.id.localeCompare(b.id),
)
const kept: FailoverHistoryItem[] = []
for (const item of merged) {
const duplicate = kept.find(
(other) =>
other.ip === item.ip &&
other.action === item.action &&
Math.abs(eventTime(other.created_at) - eventTime(item.created_at)) <=
DEDUPE_WINDOW_MS,
)
if (!duplicate) {
kept.push(item)
continue
}
if (duplicate.source === 'probe' && item.source === 'dns') {
kept[kept.indexOf(duplicate)] = item
}
}
return kept
}