326 lines
11 KiB
TypeScript
326 lines
11 KiB
TypeScript
import { createHash } from "node:crypto"
|
||
import type { ApiAlertRule } from "../alerts-service.js"
|
||
import type { RuleEvalHit, SignalSnapshot } from "./types.js"
|
||
import { isKnownBgpState } from "./bgp-signals.js"
|
||
|
||
function hashPayload(parts: string[]): string {
|
||
return createHash("sha256").update(parts.join("\0")).digest("hex").slice(0, 24)
|
||
}
|
||
|
||
function firstNumberInString(s: string): number | null {
|
||
const m = s.replace(",", ".").match(/(\d+(?:\.\d+)?)/)
|
||
if (!m) return null
|
||
const n = Number.parseFloat(m[1] ?? "")
|
||
return Number.isFinite(n) ? n : null
|
||
}
|
||
|
||
function maxSeverity(
|
||
a: ApiAlertRule["severity"],
|
||
b: ApiAlertRule["severity"],
|
||
): ApiAlertRule["severity"] {
|
||
const o = { critical: 3, warning: 2, info: 1 } as const
|
||
return o[a] >= o[b] ? a : b
|
||
}
|
||
|
||
function findServer(snap: SignalSnapshot, name: string) {
|
||
const n = name.trim().toLowerCase()
|
||
return snap.servers.find((s) => s.name.trim().toLowerCase() === n)
|
||
}
|
||
|
||
function findProbe(snap: SignalSnapshot, targetKey: string) {
|
||
const k = targetKey.trim()
|
||
return snap.probes.find((p) => p.key.trim() === k)
|
||
}
|
||
|
||
function findTraffic(snap: SignalSnapshot, serverName: string) {
|
||
const n = serverName.trim().toLowerCase()
|
||
return snap.trafficByServer.find((t) => t.serverName.trim().toLowerCase() === n)
|
||
}
|
||
|
||
function findGreTunnel(snap: SignalSnapshot, targetLabel: string) {
|
||
const n = targetLabel.trim()
|
||
return snap.greTunnels.find((g) => g.targetLabel.trim() === n)
|
||
}
|
||
|
||
function findBgpPeer(snap: SignalSnapshot, key: string) {
|
||
const n = key.trim()
|
||
return snap.bgpSessions.find((b) => b.key.trim() === n)
|
||
}
|
||
|
||
function isBgpEstablished(state: string): boolean {
|
||
return state.trim().toLowerCase() === "established"
|
||
}
|
||
|
||
/** Заголовок для Telegram: не «…недоступен: … — восстановился». */
|
||
function formatServerAlertTitle(ruleName: string, wantRecovery: boolean): string {
|
||
if (!wantRecovery) return ruleName
|
||
if (!/недоступен/i.test(ruleName)) return ruleName
|
||
const trimmed = ruleName.replace(/\s*,\s*недоступен\s*$/i, "").replace(/\s+недоступен\s*$/i, "").trim()
|
||
if (!trimmed || trimmed === ruleName) return ruleName
|
||
return `${trimmed} — снова в сети`
|
||
}
|
||
|
||
function evalServerLike(
|
||
rule: ApiAlertRule,
|
||
snap: SignalSnapshot,
|
||
cond: string,
|
||
): RuleEvalHit | null {
|
||
const c = cond.toLowerCase()
|
||
const isOtkluchilsya = /отключился|отключилась|отключилось/.test(c)
|
||
/** «Перешёл в …» — только на границе, без спама пока сэмплы остаются offline. */
|
||
const impliesOfflineEdge = /переш(ё|е)л/.test(c) && (/offline|офлайн/.test(c) || /недоступен/.test(c))
|
||
const impliesDegradedEdge = /переш(ё|е)л/.test(c) && c.includes("degraded")
|
||
const wantOffline =
|
||
c.includes("offline") || c.includes("офлайн") || c.includes("отключ")
|
||
const wantDegraded = c.includes("degraded")
|
||
/** «Восстановился» / «подключился»: двойной online в сэмплах (см. тесты). Не отсекаем по слову «offline» в фразе вроде «после offline». */
|
||
const wantRecovery =
|
||
(c.includes("восстанов") || c.includes("подключ")) && !isOtkluchilsya
|
||
const hits: string[] = []
|
||
for (const tgt of rule.targets) {
|
||
const s = findServer(snap, tgt)
|
||
if (!s) continue
|
||
if (wantRecovery) {
|
||
const p2 = s.prev2Status
|
||
if (
|
||
s.status === "online" &&
|
||
s.prevStatus === "online" &&
|
||
p2 != null &&
|
||
(p2 === "offline" || p2 === "degraded")
|
||
) {
|
||
hits.push(`${tgt}`)
|
||
} else if (
|
||
s.status === "online" &&
|
||
(s.prevStatus === "offline" || s.prevStatus === "degraded")
|
||
) {
|
||
/** Один свежий online после offline (например ресурс уже online, REST только подтвердил). */
|
||
hits.push(`${tgt}`)
|
||
}
|
||
} else if ((impliesOfflineEdge || isOtkluchilsya) && s.status === "offline") {
|
||
if (s.prevStatus === null || s.prevStatus === "online") hits.push(tgt)
|
||
} else if (impliesDegradedEdge && s.status === "degraded") {
|
||
if (s.prevStatus === null || s.prevStatus === "online") hits.push(tgt)
|
||
} else if (wantDegraded && s.status === "degraded" && !impliesDegradedEdge) {
|
||
hits.push(tgt)
|
||
} else if (
|
||
wantOffline &&
|
||
s.status === "offline" &&
|
||
!impliesOfflineEdge &&
|
||
!isOtkluchilsya
|
||
) {
|
||
hits.push(tgt)
|
||
}
|
||
}
|
||
if (hits.length === 0) return null
|
||
const title = formatServerAlertTitle(rule.name, wantRecovery)
|
||
const msg = `${title}: ${hits.join(", ")} — ${cond}`
|
||
return {
|
||
ruleId: rule.id,
|
||
ruleName: rule.name,
|
||
severity: rule.severity,
|
||
message: msg,
|
||
transition: wantRecovery ? "recovery" : "problem",
|
||
payloadHash: hashPayload([rule.id, "server", hits.sort().join(","), cond]),
|
||
}
|
||
}
|
||
|
||
function evalRtt(rule: ApiAlertRule, snap: SignalSnapshot, cond: string): RuleEvalHit | null {
|
||
const threshold = firstNumberInString(cond)
|
||
const wantLow = /<|ниже|меньше/i.test(cond) && !/>|выше|больше/i.test(cond)
|
||
const wantHigh = />|выше|больше/i.test(cond)
|
||
const hits: string[] = []
|
||
for (const tgt of rule.targets) {
|
||
const p = findProbe(snap, tgt)
|
||
if (!p) continue
|
||
if (p.status === "down" || p.status === "warn") {
|
||
hits.push(`${tgt} (${p.status}, RTT ${p.rttMs ?? "—"} мс)`)
|
||
continue
|
||
}
|
||
if (p.rttMs == null) continue
|
||
if (wantHigh && threshold != null && p.rttMs > threshold) hits.push(`${tgt} (${p.rttMs} мс)`)
|
||
if (wantLow && threshold != null && p.rttMs < threshold) hits.push(`${tgt} (${p.rttMs} мс)`)
|
||
}
|
||
if (hits.length === 0) return null
|
||
const msg = `${rule.name}: RTT ${cond} — ${hits.join("; ")}`
|
||
return {
|
||
ruleId: rule.id,
|
||
ruleName: rule.name,
|
||
severity: rule.severity,
|
||
message: msg,
|
||
transition: "neutral",
|
||
payloadHash: hashPayload([rule.id, "rtt", hits.sort().join(";"), cond]),
|
||
}
|
||
}
|
||
|
||
function evalLoss(rule: ApiAlertRule, snap: SignalSnapshot, cond: string): RuleEvalHit | null {
|
||
const threshold = firstNumberInString(cond)
|
||
if (threshold == null) return null
|
||
const hits: string[] = []
|
||
for (const tgt of rule.targets) {
|
||
const p = findProbe(snap, tgt)
|
||
if (!p || p.lossPct == null) continue
|
||
if (p.lossPct > threshold) hits.push(`${tgt} (${p.lossPct}%)`)
|
||
}
|
||
if (hits.length === 0) return null
|
||
const msg = `${rule.name}: потери ${cond} — ${hits.join("; ")}`
|
||
return {
|
||
ruleId: rule.id,
|
||
ruleName: rule.name,
|
||
severity: rule.severity,
|
||
message: msg,
|
||
transition: "neutral",
|
||
payloadHash: hashPayload([rule.id, "loss", hits.sort().join(";"), cond]),
|
||
}
|
||
}
|
||
|
||
function evalGreTunnel(rule: ApiAlertRule, snap: SignalSnapshot, cond: string): RuleEvalHit | null {
|
||
const lc = cond.toLowerCase()
|
||
const wantOffline = lc.includes("offline") && !lc.includes("восстанов")
|
||
const wantRecovery =
|
||
lc.includes("восстанов") && !/offline|отключ/i.test(lc)
|
||
const hits: string[] = []
|
||
for (const tgt of rule.targets) {
|
||
const g = findGreTunnel(snap, tgt)
|
||
if (!g) continue
|
||
if (wantRecovery) {
|
||
const prev = g.prevStatus
|
||
if (
|
||
g.status === "up" &&
|
||
prev != null &&
|
||
(prev === "down" || prev === "degraded")
|
||
) {
|
||
hits.push(tgt)
|
||
}
|
||
} else if (wantOffline && g.status === "down") {
|
||
hits.push(tgt)
|
||
}
|
||
}
|
||
if (hits.length === 0) return null
|
||
const msg = `${rule.name}: ${hits.join(", ")} — ${cond}`
|
||
return {
|
||
ruleId: rule.id,
|
||
ruleName: rule.name,
|
||
severity: rule.severity,
|
||
message: msg,
|
||
transition: wantRecovery ? "recovery" : "problem",
|
||
payloadHash: hashPayload([rule.id, "gre-tunnel", hits.sort().join(","), cond]),
|
||
}
|
||
}
|
||
|
||
function evalBgpPeer(rule: ApiAlertRule, snap: SignalSnapshot, cond: string): RuleEvalHit | null {
|
||
const lc = cond.toLowerCase()
|
||
const wantDown = lc.includes("разорвал")
|
||
const wantUp = lc.includes("восстановил")
|
||
const hits: string[] = []
|
||
for (const tgt of rule.targets) {
|
||
const b = findBgpPeer(snap, tgt)
|
||
if (!b || !isKnownBgpState(b.state)) continue
|
||
if (wantDown && !isBgpEstablished(b.state)) hits.push(tgt)
|
||
if (
|
||
wantUp &&
|
||
isBgpEstablished(b.state) &&
|
||
b.prevState != null &&
|
||
isKnownBgpState(b.prevState) &&
|
||
String(b.prevState).trim() !== "" &&
|
||
!isBgpEstablished(b.prevState)
|
||
) {
|
||
hits.push(tgt)
|
||
}
|
||
}
|
||
if (hits.length === 0) return null
|
||
const msg = `${rule.name}: ${hits.join(", ")} — ${cond}`
|
||
return {
|
||
ruleId: rule.id,
|
||
ruleName: rule.name,
|
||
severity: rule.severity,
|
||
message: msg,
|
||
transition: wantUp ? "recovery" : "problem",
|
||
payloadHash: hashPayload([rule.id, "bgp-peer", hits.sort().join(","), cond]),
|
||
}
|
||
}
|
||
|
||
function evalTraffic(rule: ApiAlertRule, snap: SignalSnapshot, cond: string): RuleEvalHit | null {
|
||
const threshold = firstNumberInString(cond)
|
||
if (threshold == null) return null
|
||
const rxLow = /RX\s*</i.test(cond)
|
||
const txLow = /TX\s*</i.test(cond)
|
||
const hits: string[] = []
|
||
for (const tgt of rule.targets) {
|
||
const t = findTraffic(snap, tgt)
|
||
if (!t) continue
|
||
if (rxLow && t.rxMbps < threshold) hits.push(`${tgt} RX ${t.rxMbps.toFixed(2)} Мбит/с`)
|
||
if (txLow && t.txMbps < threshold) hits.push(`${tgt} TX ${t.txMbps.toFixed(2)} Мбит/с`)
|
||
}
|
||
if (hits.length === 0) return null
|
||
const msg = `${rule.name}: трафик ${cond} — ${hits.join("; ")}`
|
||
return {
|
||
ruleId: rule.id,
|
||
ruleName: rule.name,
|
||
severity: rule.severity,
|
||
message: msg,
|
||
transition: "neutral",
|
||
payloadHash: hashPayload([rule.id, "traffic", hits.sort().join(";"), cond]),
|
||
}
|
||
}
|
||
|
||
function evaluateRuleWithCondition(
|
||
rule: ApiAlertRule,
|
||
snap: SignalSnapshot,
|
||
cond: string,
|
||
): RuleEvalHit | null {
|
||
const c = cond.trim()
|
||
if (!c) return null
|
||
switch (rule.type) {
|
||
case "server":
|
||
case "gre-client":
|
||
return evalServerLike(rule, snap, c)
|
||
case "rtt":
|
||
return evalRtt(rule, snap, c)
|
||
case "loss":
|
||
return evalLoss(rule, snap, c)
|
||
case "traffic":
|
||
return evalTraffic(rule, snap, c)
|
||
case "gre-tunnel":
|
||
return evalGreTunnel(rule, snap, c)
|
||
case "bgp-peer":
|
||
return evalBgpPeer(rule, snap, c)
|
||
case "bgp-prefix":
|
||
return null
|
||
default:
|
||
return null
|
||
}
|
||
}
|
||
|
||
/** Одно правило: OR по targets; OR по conditions (любое из выбранных). */
|
||
export function evaluateRule(rule: ApiAlertRule, snap: SignalSnapshot): RuleEvalHit | null {
|
||
if (!rule.enabled || rule.targets.length === 0) return null
|
||
const conds =
|
||
rule.conditions?.length > 0
|
||
? rule.conditions.map((c) => c.trim()).filter(Boolean)
|
||
: [rule.condition.trim()].filter(Boolean)
|
||
if (conds.length === 0) return null
|
||
|
||
const hits: RuleEvalHit[] = []
|
||
for (const cond of conds) {
|
||
const h = evaluateRuleWithCondition(rule, snap, cond)
|
||
if (h) hits.push(h)
|
||
}
|
||
if (hits.length === 0) return null
|
||
if (hits.length === 1) return hits[0]!
|
||
const message = hits.map((h) => h.message).join("\n")
|
||
const severity = hits.reduce((a, h) => maxSeverity(a, h.severity), hits[0]!.severity)
|
||
const payloadHash = hashPayload([rule.id, "or-conds", ...hits.map((h) => h.payloadHash).sort()])
|
||
return {
|
||
ruleId: rule.id,
|
||
ruleName: rule.name,
|
||
severity,
|
||
message,
|
||
transition: hits.some((h) => h.transition === "problem")
|
||
? "problem"
|
||
: hits.some((h) => h.transition === "recovery")
|
||
? "recovery"
|
||
: "neutral",
|
||
payloadHash,
|
||
}
|
||
}
|