quality / commitlint (push) Skipped
CD / update-wiki (push) Successful in 5s
quality / changes (push) Successful in 8s
quality / docker-check (push) Skipped
quality / web (push) Successful in 55s
quality / api (push) Successful in 52s
CD / quality (push) Successful in 2m4s
CD / publish (push) Successful in 1m51s
KPI и карточка показывают живой OK, а не гистерезис unknown; DNS по-прежнему ждёт повторные успехи. Co-authored-by: Cursor <[email protected]>
263 lines
7.6 KiB
TypeScript
263 lines
7.6 KiB
TypeScript
import {
|
||
bestAliveHealthStatus,
|
||
probeTime,
|
||
type HealthLogProbe,
|
||
type HealthLogStatus,
|
||
} from '@/lib/health-log'
|
||
import type { FailoverLogEntry } from '@/lib/schemas'
|
||
|
||
export type FailoverEventKind = 'removed' | 'last-resort'
|
||
|
||
export interface FailoverHistoryItem {
|
||
id: string
|
||
ip: string
|
||
fqdn: string
|
||
action: 'added' | 'removed'
|
||
created_at: string
|
||
copy: string
|
||
source: 'dns' | 'probe'
|
||
}
|
||
|
||
export interface FailoverEvent {
|
||
id: string
|
||
address: string
|
||
status: string
|
||
kind: FailoverEventKind
|
||
fqdns: string[]
|
||
consecutiveFailures: number
|
||
lastFailureReason: string | null
|
||
lastCheckAt?: string | null
|
||
}
|
||
|
||
/** Binding IP health — тот же контур, что таблица активов. */
|
||
export interface FailoverHealthInput {
|
||
ip: string
|
||
status: string
|
||
consecutive_failures?: number
|
||
last_error?: string | null
|
||
last_checked_at?: string | null
|
||
}
|
||
|
||
export interface FailoverBindingPool {
|
||
fqdn: string
|
||
configured: readonly string[]
|
||
active: readonly string[]
|
||
}
|
||
|
||
/** Unique A targets. Duplicates of the same IP are not a pool. */
|
||
export function uniqueIpCount(ips: readonly string[]): number {
|
||
return new Set(ips.filter(Boolean)).size
|
||
}
|
||
|
||
/** Shared pool FQDN — two or more unique A targets. */
|
||
export function isSharedPoolBinding(binding: FailoverBindingPool): boolean {
|
||
return uniqueIpCount(binding.configured) >= 2
|
||
}
|
||
|
||
export function hasSharedPool(
|
||
bindings: readonly FailoverBindingPool[],
|
||
): boolean {
|
||
return bindings.some(isSharedPoolBinding)
|
||
}
|
||
|
||
/** Инцидент только при down. up / degraded / unknown — не вывод из пула. */
|
||
export function isFailoverEventStatus(status: string): boolean {
|
||
return status === 'down'
|
||
}
|
||
|
||
export function failoverEventCopy(event: FailoverEvent): string {
|
||
if (event.kind === 'removed') {
|
||
return event.fqdns.length > 0
|
||
? `Снята с ${event.fqdns.join(', ')}`
|
||
: 'Снята с DNS'
|
||
}
|
||
return event.fqdns.length > 0
|
||
? `Down, в A-записях last-resort на ${event.fqdns.join(', ')}`
|
||
: 'Down, в A-записях last-resort'
|
||
}
|
||
|
||
/**
|
||
* Инциденты пула только если у сервиса есть shared FQDN (2+ уникальных IP).
|
||
* Один IP — не балансировка и не вывод из пула; Down смотрит health-монитор.
|
||
*/
|
||
export function toFailoverEvents(
|
||
ipHealth: readonly FailoverHealthInput[],
|
||
bindings: readonly FailoverBindingPool[] = [],
|
||
): FailoverEvent[] {
|
||
if (!hasSharedPool(bindings)) return []
|
||
return ipHealth.filter((row) => isFailoverEventStatus(row.status)).map((row) => {
|
||
const removedFqdns: string[] = []
|
||
const lastResortFqdns: string[] = []
|
||
for (const binding of bindings) {
|
||
if (!isSharedPoolBinding(binding)) continue
|
||
const configured = binding.configured.includes(row.ip)
|
||
const active = binding.active.includes(row.ip)
|
||
if (configured && !active) removedFqdns.push(binding.fqdn)
|
||
else if (configured && active) lastResortFqdns.push(binding.fqdn)
|
||
}
|
||
const kind: FailoverEventKind =
|
||
removedFqdns.length > 0 ? 'removed' : 'last-resort'
|
||
return {
|
||
id: row.ip,
|
||
address: row.ip,
|
||
status: row.status,
|
||
kind,
|
||
fqdns: kind === 'removed' ? removedFqdns : lastResortFqdns,
|
||
consecutiveFailures: row.consecutive_failures ?? 0,
|
||
lastFailureReason: row.last_error ?? null,
|
||
lastCheckAt: row.last_checked_at ?? null,
|
||
}
|
||
})
|
||
}
|
||
|
||
const DEDUPE_WINDOW_MS = 2 * 60 * 1000
|
||
|
||
function fqdnsForIp(
|
||
ip: string,
|
||
bindings: readonly FailoverBindingPool[],
|
||
): string[] {
|
||
return bindings
|
||
.filter((binding) => isSharedPoolBinding(binding) && binding.configured.includes(ip))
|
||
.map((binding) => binding.fqdn)
|
||
}
|
||
|
||
function isPoolFqdn(
|
||
fqdn: string,
|
||
bindings: readonly FailoverBindingPool[],
|
||
): boolean {
|
||
const binding = bindings.find((item) => item.fqdn === fqdn)
|
||
if (!binding) return true
|
||
return isSharedPoolBinding(binding)
|
||
}
|
||
|
||
function fqdnLabel(fqdns: string[]): string {
|
||
return fqdns.join(', ') || 'пул'
|
||
}
|
||
|
||
function isAliveStatus(status: HealthLogStatus): boolean {
|
||
return status === 'up' || status === 'degraded'
|
||
}
|
||
|
||
/**
|
||
* Per-IP any-up flips: down → вышла из пула, up после down → вернулась.
|
||
* Initial state is not an event.
|
||
*/
|
||
export function toIpAliveTransitions(
|
||
probes: readonly HealthLogProbe[],
|
||
): Array<{ id: string; ip: string; alive: boolean; at: string }> {
|
||
const byIp = new Map<string, HealthLogProbe[]>()
|
||
for (const item of probes) {
|
||
const list = byIp.get(item.ip)
|
||
if (list) list.push(item)
|
||
else byIp.set(item.ip, [item])
|
||
}
|
||
|
||
const out: Array<{ id: string; ip: string; alive: boolean; at: string }> = []
|
||
for (const [ip, list] of byIp) {
|
||
list.sort(
|
||
(a, b) => probeTime(a.checked_at) - probeTime(b.checked_at) || a.id - b.id,
|
||
)
|
||
const latestByProvider = new Map<string, HealthLogProbe>()
|
||
let prevAlive: boolean | undefined
|
||
for (const probe of list) {
|
||
latestByProvider.set(probe.provider, probe)
|
||
const status = bestAliveHealthStatus(
|
||
[...latestByProvider.values()].map((item) => item.status),
|
||
)
|
||
if (status === 'unknown') continue
|
||
const alive = isAliveStatus(status)
|
||
if (prevAlive !== undefined && alive !== prevAlive) {
|
||
out.push({
|
||
id: `probe:${probe.id}`,
|
||
ip,
|
||
alive,
|
||
at: probe.checked_at,
|
||
})
|
||
}
|
||
prevAlive = alive
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
function dnsHistoryItem(item: FailoverLogEntry): FailoverHistoryItem {
|
||
return {
|
||
id: `dns:${item.id}`,
|
||
ip: item.ip,
|
||
fqdn: item.fqdn,
|
||
action: item.action,
|
||
created_at: item.created_at,
|
||
copy:
|
||
item.action === 'removed'
|
||
? `${item.ip} убрана с ${item.fqdn}`
|
||
: `${item.ip} добавлена на ${item.fqdn}`,
|
||
source: 'dns',
|
||
}
|
||
}
|
||
|
||
function probeHistoryItem(
|
||
transition: { id: string; ip: string; alive: boolean; at: string },
|
||
bindings: readonly FailoverBindingPool[],
|
||
): FailoverHistoryItem {
|
||
const fqdns = fqdnsForIp(transition.ip, bindings)
|
||
const fqdn = fqdnLabel(fqdns)
|
||
const action = transition.alive ? 'added' : 'removed'
|
||
return {
|
||
id: transition.id,
|
||
ip: transition.ip,
|
||
fqdn,
|
||
action,
|
||
created_at: transition.at,
|
||
copy:
|
||
action === 'removed'
|
||
? `${transition.ip} вышла из пула (${fqdn})`
|
||
: `${transition.ip} вернулась в пул (${fqdn})`,
|
||
source: 'probe',
|
||
}
|
||
}
|
||
|
||
function eventTime(value: string): number {
|
||
return probeTime(value)
|
||
}
|
||
|
||
/**
|
||
* DNS add/remove + health leave/return, newest first.
|
||
* Same IP+action within 2 minutes: keep the DNS row (it has a concrete FQDN).
|
||
*/
|
||
export function mergeFailoverHistory(
|
||
dns: readonly FailoverLogEntry[],
|
||
probes: readonly HealthLogProbe[],
|
||
bindings: readonly FailoverBindingPool[] = [],
|
||
): FailoverHistoryItem[] {
|
||
const merged = [
|
||
...dns
|
||
.filter((item) => isPoolFqdn(item.fqdn, bindings))
|
||
.map(dnsHistoryItem),
|
||
...toIpAliveTransitions(probes)
|
||
.filter((transition) => fqdnsForIp(transition.ip, bindings).length > 0)
|
||
.map((transition) => probeHistoryItem(transition, bindings)),
|
||
]
|
||
merged.sort(
|
||
(a, b) => eventTime(b.created_at) - eventTime(a.created_at) || a.id.localeCompare(b.id),
|
||
)
|
||
|
||
const kept: FailoverHistoryItem[] = []
|
||
for (const item of merged) {
|
||
const duplicate = kept.find(
|
||
(other) =>
|
||
other.ip === item.ip &&
|
||
other.action === item.action &&
|
||
Math.abs(eventTime(other.created_at) - eventTime(item.created_at)) <=
|
||
DEDUPE_WINDOW_MS,
|
||
)
|
||
if (!duplicate) {
|
||
kept.push(item)
|
||
continue
|
||
}
|
||
if (duplicate.source === 'probe' && item.source === 'dns') {
|
||
kept[kept.indexOf(duplicate)] = item
|
||
}
|
||
}
|
||
return kept
|
||
}
|