266 lines
7.6 KiB
TypeScript
266 lines
7.6 KiB
TypeScript
import assert from "node:assert/strict"
|
|
import type { ApiAlertRule } from "../alerts-service.js"
|
|
import { evaluateRule } from "./evaluate-rule.js"
|
|
import type { SignalSnapshot } from "./types.js"
|
|
import { collectBgpSignalsFromRuns } from "./bgp-signals.js"
|
|
import type { GreBgpSnapshotRunSnapshot } from "../../types/scheduler-run-snapshot.js"
|
|
|
|
function emptySnap(over: Partial<SignalSnapshot> = {}): SignalSnapshot {
|
|
return {
|
|
sampledAt: "t",
|
|
servers: [],
|
|
probes: [],
|
|
trafficByServer: [],
|
|
greTunnels: [],
|
|
bgpSessions: [],
|
|
...over,
|
|
}
|
|
}
|
|
|
|
function rule(p: Partial<ApiAlertRule> & Pick<ApiAlertRule, "id" | "name" | "type" | "targets">): ApiAlertRule {
|
|
return {
|
|
target: p.targets[0] ?? "",
|
|
groupId: null,
|
|
condition: p.conditions?.[0] ?? p.condition ?? "",
|
|
conditions: p.conditions ?? (p.condition ? [p.condition] : []),
|
|
severity: p.severity ?? "warning",
|
|
enabled: p.enabled ?? true,
|
|
cooldown: p.cooldown ?? "5м",
|
|
confirmStabilitySec: null,
|
|
recoveryMode: "always",
|
|
recoveryStabilitySec: null,
|
|
chatId: "",
|
|
lastFiredAt: null,
|
|
...p,
|
|
} as ApiAlertRule
|
|
}
|
|
|
|
{
|
|
const r = rule({
|
|
id: "g1",
|
|
name: "GRE",
|
|
type: "gre-tunnel",
|
|
targets: ["t1 / srv"],
|
|
conditions: ["перешёл в offline"],
|
|
})
|
|
const hit = evaluateRule(
|
|
r,
|
|
emptySnap({
|
|
greTunnels: [{ targetLabel: "t1 / srv", status: "down", prevStatus: "up" }],
|
|
}),
|
|
)
|
|
assert.ok(hit)
|
|
}
|
|
|
|
{
|
|
const r = rule({
|
|
id: "g2",
|
|
name: "GRE rec",
|
|
type: "gre-tunnel",
|
|
targets: ["t1 / srv"],
|
|
conditions: ["восстановился"],
|
|
})
|
|
const hit = evaluateRule(
|
|
r,
|
|
emptySnap({
|
|
greTunnels: [{ targetLabel: "t1 / srv", status: "up", prevStatus: "down" }],
|
|
}),
|
|
)
|
|
assert.ok(hit)
|
|
}
|
|
|
|
{
|
|
const r = rule({
|
|
id: "b1",
|
|
name: "BGP",
|
|
type: "bgp-peer",
|
|
targets: ["srv: peer1"],
|
|
conditions: ["разорвал сессию"],
|
|
})
|
|
const hit = evaluateRule(
|
|
r,
|
|
emptySnap({
|
|
bgpSessions: [{ key: "srv: peer1", state: "Idle", prevState: "Established" }],
|
|
}),
|
|
)
|
|
assert.ok(hit)
|
|
}
|
|
|
|
{
|
|
const r = rule({
|
|
id: "b2",
|
|
name: "BGP up",
|
|
type: "bgp-peer",
|
|
targets: ["srv: peer1"],
|
|
conditions: ["восстановил сессию"],
|
|
})
|
|
const hit = evaluateRule(
|
|
r,
|
|
emptySnap({
|
|
bgpSessions: [{ key: "srv: peer1", state: "Established", prevState: "Active" }],
|
|
}),
|
|
)
|
|
assert.ok(hit)
|
|
}
|
|
|
|
{
|
|
const r = rule({
|
|
id: "pfx",
|
|
name: "pfx",
|
|
type: "bgp-prefix",
|
|
targets: ["любой префикс"],
|
|
conditions: ["был отозван"],
|
|
})
|
|
assert.equal(evaluateRule(r, emptySnap()), null)
|
|
}
|
|
|
|
for (const state of ["Unknown", "", " ", "FutureState"]) {
|
|
const problem = rule({ id: "unknown-down", name: "BGP", type: "bgp-peer", targets: ["srv: peer1"], conditions: ["разорвал сессию"] })
|
|
const recovery = rule({ id: "unknown-up", name: "BGP", type: "bgp-peer", targets: ["srv: peer1"], conditions: ["восстановил сессию"] })
|
|
assert.equal(evaluateRule(problem, emptySnap({ bgpSessions: [{ key: "srv: peer1", state, prevState: "Established" }] })), null)
|
|
assert.equal(evaluateRule(recovery, emptySnap({ bgpSessions: [{ key: "srv: peer1", state: "Established", prevState: state }] })), null)
|
|
}
|
|
|
|
for (const [history, expectedPrevious, recoveryExpected] of [
|
|
[["Established", "Unknown", "Idle"], "Idle", true],
|
|
[["Established", "Unknown", "Established"], "Established", false],
|
|
[["Established", "Unknown", ""], null, false],
|
|
[["Unknown", "Idle", "Established"], "Idle", false],
|
|
] as const) {
|
|
const runs = history.map(state => ({ bgpPeers: [{ key: "srv: peer1", state }] })) as GreBgpSnapshotRunSnapshot[]
|
|
const signals = collectBgpSignalsFromRuns(runs)
|
|
assert.equal(signals[0].state, history[0])
|
|
assert.equal(signals[0].prevState, expectedPrevious)
|
|
const recovery = rule({ id: "gap-up", name: "BGP", type: "bgp-peer", targets: ["srv: peer1"], conditions: ["восстановил сессию"] })
|
|
assert.equal(evaluateRule(recovery, emptySnap({ bgpSessions: signals })) !== null, recoveryExpected)
|
|
}
|
|
|
|
for (const pair of [["Established", "Idle"], ["Established", "Established"], ["Unknown", "Idle"]]) {
|
|
const runs = [{ bgpPeers: pair.map(state => ({ key: "srv: peer1", state })) }] as GreBgpSnapshotRunSnapshot[]
|
|
assert.deepEqual(collectBgpSignalsFromRuns(runs), [{ key: "srv: peer1", state: "Unknown", prevState: null }])
|
|
}
|
|
assert.deepEqual(collectBgpSignalsFromRuns([{ bgpPeers: [] }, { bgpPeers: [{ key: "removed-peer", state: "Established" }] }] as GreBgpSnapshotRunSnapshot[]), [])
|
|
assert.deepEqual(collectBgpSignalsFromRuns([{ bgpPeers: [{ key: "current-peer", state: "Established" }] }, { bgpPeers: [{ key: "removed-peer", state: "Idle" }] }] as GreBgpSnapshotRunSnapshot[]), [{ key: "current-peer", state: "Established", prevState: null }])
|
|
{
|
|
const runs = [
|
|
{ bgpPeers: [{ key: "srv: peer1", state: "Established" }] },
|
|
{ bgpPeers: [{ key: "srv: peer1", state: "Established" }, { key: "srv: peer1", state: "Idle" }] },
|
|
{ bgpPeers: [{ key: "srv: peer1", state: "Idle" }] },
|
|
] as GreBgpSnapshotRunSnapshot[]
|
|
assert.deepEqual(collectBgpSignalsFromRuns(runs), [{ key: "srv: peer1", state: "Established", prevState: null }])
|
|
}
|
|
|
|
{
|
|
const r = rule({
|
|
id: "srv-rec",
|
|
name: "rec",
|
|
type: "server",
|
|
targets: ["core"],
|
|
conditions: ["восстановился"],
|
|
})
|
|
assert.ok(
|
|
evaluateRule(
|
|
r,
|
|
emptySnap({
|
|
servers: [
|
|
{
|
|
name: "core",
|
|
status: "online",
|
|
prevStatus: "offline",
|
|
prev2Status: "offline",
|
|
sampledAt: "t1",
|
|
},
|
|
],
|
|
}),
|
|
),
|
|
"один online после offline — тоже «восстановился» (ручной REST / ресурс уже online)",
|
|
)
|
|
const hit = evaluateRule(
|
|
r,
|
|
emptySnap({
|
|
servers: [
|
|
{
|
|
name: "core",
|
|
status: "online",
|
|
prevStatus: "online",
|
|
prev2Status: "offline",
|
|
sampledAt: "t2",
|
|
},
|
|
],
|
|
}),
|
|
)
|
|
assert.ok(hit, "два подряд online после offline — одно срабатывание")
|
|
assert.equal(hit?.transition, "recovery")
|
|
}
|
|
|
|
{
|
|
const r = rule({
|
|
id: "srv-rec-offline-phrase",
|
|
name: "rec-offline-phrase",
|
|
type: "server",
|
|
targets: ["core"],
|
|
conditions: ["восстановился после offline"],
|
|
})
|
|
const hit = evaluateRule(
|
|
r,
|
|
emptySnap({
|
|
servers: [
|
|
{
|
|
name: "core",
|
|
status: "online",
|
|
prevStatus: "online",
|
|
prev2Status: "offline",
|
|
sampledAt: "t2",
|
|
},
|
|
],
|
|
}),
|
|
)
|
|
assert.ok(hit, "в тексте условия есть «offline» — всё равно восстановление по сэмплам")
|
|
}
|
|
|
|
{
|
|
const r = rule({
|
|
id: "srv-off",
|
|
name: "off",
|
|
type: "server",
|
|
targets: ["core"],
|
|
conditions: ["перешёл в offline"],
|
|
})
|
|
const edgeHit = evaluateRule(
|
|
r,
|
|
emptySnap({
|
|
servers: [
|
|
{
|
|
name: "core",
|
|
status: "offline",
|
|
prevStatus: "online",
|
|
prev2Status: "online",
|
|
sampledAt: "t",
|
|
},
|
|
],
|
|
}),
|
|
)
|
|
assert.ok(edgeHit)
|
|
assert.equal(edgeHit?.transition, "problem")
|
|
assert.equal(
|
|
evaluateRule(
|
|
r,
|
|
emptySnap({
|
|
servers: [
|
|
{
|
|
name: "core",
|
|
status: "offline",
|
|
prevStatus: "offline",
|
|
prev2Status: "online",
|
|
sampledAt: "t",
|
|
},
|
|
],
|
|
}),
|
|
),
|
|
null,
|
|
"устойчивый offline — без повторного «перешёл» на каждом тике",
|
|
)
|
|
}
|
|
|
|
console.log("alert-engine evaluate-rule tests ok")
|