chore: synchronize pending app/backend updates and repository hygiene
Includes current frontend and backend work in progress and removes generated artifacts from tracking to keep the repository clean for дальнейшая разработка. Co-authored-by: Cursor <[email protected]>
This commit is contained in:
Denozordec
@@ -0,0 +1,48 @@
|
||||
/**
|
||||
* Внутренние хуки: после записи сигналов в SQLite (коллекторы) — отложенный прогон `alert_engine`.
|
||||
* Не импортирует `scheduler.ts` (избегаем цикла); runner задаётся через `wireAlertEngineRunner`.
|
||||
*
|
||||
* Любой код, пишущий таблицы для `buildSignalSnapshot`, после успешного коммита должен вызывать
|
||||
* `scheduleAlertEngineAfterDataCollectors()`.
|
||||
*/
|
||||
import { isSchedulerJobRunning } from "./scheduler-running.js"
|
||||
|
||||
const ALERT_AFTER_COLLECT_DEBOUNCE_MS = 250
|
||||
const ALERT_AFTER_COLLECT_WAIT_MS = 100
|
||||
|
||||
let alertAfterCollectTimer: ReturnType<typeof setTimeout> | null = null
|
||||
let alertAfterCollectChain: Promise<void> = Promise.resolve()
|
||||
|
||||
let alertEngineRunner: (() => Promise<void>) | null = null
|
||||
|
||||
/** Задать runner один раз при старте приложения (обычно `() => executeSchedulerJob("alert_engine")`). */
|
||||
export function wireAlertEngineRunner(run: () => Promise<void>): void {
|
||||
alertEngineRunner = run
|
||||
}
|
||||
|
||||
export function clearAlertCollectorHooksTimer(): void {
|
||||
if (alertAfterCollectTimer != null) {
|
||||
clearTimeout(alertAfterCollectTimer)
|
||||
alertAfterCollectTimer = null
|
||||
}
|
||||
}
|
||||
|
||||
async function runAlertEngineWhenIdle(): Promise<void> {
|
||||
const run = alertEngineRunner
|
||||
if (!run) return
|
||||
while (isSchedulerJobRunning("alert_engine")) {
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, ALERT_AFTER_COLLECT_WAIT_MS))
|
||||
}
|
||||
await run()
|
||||
}
|
||||
|
||||
/** После успешной записи сигналов в БД коллекторами (не из `alert_engine`). */
|
||||
export function scheduleAlertEngineAfterDataCollectors(): void {
|
||||
if (alertAfterCollectTimer != null) clearTimeout(alertAfterCollectTimer)
|
||||
alertAfterCollectTimer = setTimeout(() => {
|
||||
alertAfterCollectTimer = null
|
||||
alertAfterCollectChain = alertAfterCollectChain
|
||||
.then(() => runAlertEngineWhenIdle())
|
||||
.catch(() => {})
|
||||
}, ALERT_AFTER_COLLECT_DEBOUNCE_MS)
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { eq } from "drizzle-orm"
|
||||
import { db } from "../../db/index.js"
|
||||
import { alertEngineConfirmPending } from "../../db/schema.js"
|
||||
import type { ApiAlertRule } from "../alerts-service.js"
|
||||
import { computeStabilityReadyMap, deleteConfirmPending } from "./confirm-stability.js"
|
||||
|
||||
const ruleId = "test-confirm-stability-recovery-bypass"
|
||||
|
||||
deleteConfirmPending(ruleId)
|
||||
|
||||
const baseRule: ApiAlertRule = {
|
||||
id: ruleId,
|
||||
name: "Server transition",
|
||||
type: "server",
|
||||
target: "core-1",
|
||||
targets: ["core-1"],
|
||||
groupId: null,
|
||||
condition: "перешёл в offline · восстановился",
|
||||
conditions: ["перешёл в offline", "восстановился"],
|
||||
severity: "critical",
|
||||
enabled: true,
|
||||
cooldown: "5м",
|
||||
confirmStabilitySec: 120,
|
||||
recoveryMode: "always",
|
||||
recoveryStabilitySec: null,
|
||||
chatId: "",
|
||||
lastFiredAt: null,
|
||||
}
|
||||
|
||||
const problemHit = {
|
||||
ruleId,
|
||||
ruleName: baseRule.name,
|
||||
severity: baseRule.severity,
|
||||
message: "Сервер: core-1 — перешёл в offline",
|
||||
transition: "problem" as const,
|
||||
payloadHash: "problem-1",
|
||||
}
|
||||
|
||||
const recoveryHit = {
|
||||
ruleId,
|
||||
ruleName: baseRule.name,
|
||||
severity: baseRule.severity,
|
||||
message: "Сервер: core-1 — восстановился",
|
||||
transition: "recovery" as const,
|
||||
payloadHash: "recovery-1",
|
||||
}
|
||||
|
||||
{
|
||||
const m = computeStabilityReadyMap([baseRule], new Map([[ruleId, problemHit]]))
|
||||
assert.equal(m.get(ruleId), false, "проблемный переход должен ждать confirmStabilitySec")
|
||||
}
|
||||
|
||||
{
|
||||
const m = computeStabilityReadyMap([baseRule], new Map([[ruleId, recoveryHit]]))
|
||||
assert.equal(m.get(ruleId), true, "восстановление должно отправляться без задержки")
|
||||
}
|
||||
|
||||
{
|
||||
const m = computeStabilityReadyMap([baseRule], new Map([[ruleId, null]]))
|
||||
assert.equal(m.get(ruleId), false, "без hit отправка не готова")
|
||||
const left = db
|
||||
.select()
|
||||
.from(alertEngineConfirmPending)
|
||||
.where(eq(alertEngineConfirmPending.ruleId, ruleId))
|
||||
.limit(1)
|
||||
.all()[0]
|
||||
assert.equal(left, undefined, "pending должен очищаться после отсутствия hit")
|
||||
}
|
||||
|
||||
console.log("alert-engine confirm-stability tests ok")
|
||||
@@ -0,0 +1,82 @@
|
||||
import { eq } from "drizzle-orm"
|
||||
import { db } from "../../db/index.js"
|
||||
import { alertEngineConfirmPending } from "../../db/schema.js"
|
||||
import type { ApiAlertRule } from "../alerts-service.js"
|
||||
import type { RuleEvalHit } from "./types.js"
|
||||
|
||||
export function deleteConfirmPending(ruleId: string) {
|
||||
db.delete(alertEngineConfirmPending).where(eq(alertEngineConfirmPending.ruleId, ruleId)).run()
|
||||
}
|
||||
|
||||
/**
|
||||
* Обновляет состояние ожидания стабильности и возвращает, можно ли отправлять уведомление в этом тике.
|
||||
* При отсутствии срабатывания (`hit == null`) ожидание сбрасывается.
|
||||
*/
|
||||
export function stabilityReadyForSend(
|
||||
ruleId: string,
|
||||
hit: RuleEvalHit | null,
|
||||
delaySec: number | null | undefined,
|
||||
): boolean {
|
||||
if (!hit) {
|
||||
deleteConfirmPending(ruleId)
|
||||
return false
|
||||
}
|
||||
const sec = delaySec != null && delaySec > 0 ? Math.floor(delaySec) : 0
|
||||
if (sec <= 0) {
|
||||
deleteConfirmPending(ruleId)
|
||||
return true
|
||||
}
|
||||
|
||||
const now = Date.now()
|
||||
const nowIso = new Date().toISOString()
|
||||
const existing = db
|
||||
.select()
|
||||
.from(alertEngineConfirmPending)
|
||||
.where(eq(alertEngineConfirmPending.ruleId, ruleId))
|
||||
.limit(1)
|
||||
.all()[0]
|
||||
|
||||
if (!existing || existing.payloadHash !== hit.payloadHash) {
|
||||
if (existing) {
|
||||
db.update(alertEngineConfirmPending)
|
||||
.set({ payloadHash: hit.payloadHash, sinceAt: nowIso })
|
||||
.where(eq(alertEngineConfirmPending.ruleId, ruleId))
|
||||
.run()
|
||||
} else {
|
||||
db.insert(alertEngineConfirmPending).values({ ruleId, payloadHash: hit.payloadHash, sinceAt: nowIso }).run()
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
const t = Date.parse(existing.sinceAt)
|
||||
if (!Number.isFinite(t)) return false
|
||||
return now - t >= sec * 1000
|
||||
}
|
||||
|
||||
/** После успешной отправки — сбросить таймер стабильности для следующего цикла. */
|
||||
export function clearConfirmPendingAfterSend(ruleIds: string[]) {
|
||||
for (const id of ruleIds) deleteConfirmPending(id)
|
||||
}
|
||||
|
||||
export function computeStabilityReadyMap(
|
||||
rules: ApiAlertRule[],
|
||||
hitByRule: Map<string, RuleEvalHit | null>,
|
||||
): Map<string, boolean> {
|
||||
const m = new Map<string, boolean>()
|
||||
for (const rule of rules) {
|
||||
if (!rule.enabled) {
|
||||
deleteConfirmPending(rule.id)
|
||||
m.set(rule.id, false)
|
||||
continue
|
||||
}
|
||||
const hit = hitByRule.get(rule.id) ?? null
|
||||
const delaySec =
|
||||
hit?.transition === "recovery"
|
||||
? rule.recoveryMode === "conditional"
|
||||
? rule.recoveryStabilitySec
|
||||
: 0
|
||||
: rule.confirmStabilitySec
|
||||
m.set(rule.id, stabilityReadyForSend(rule.id, hit, delaySec))
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import type { ApiAlertRule } from "../alerts-service.js"
|
||||
|
||||
const COOLDOWN_MS: Record<ApiAlertRule["cooldown"], number> = {
|
||||
"1м": 60_000,
|
||||
"5м": 5 * 60_000,
|
||||
"15м": 15 * 60_000,
|
||||
"1ч": 60 * 60_000,
|
||||
"4ч": 4 * 60 * 60_000,
|
||||
"24ч": 24 * 60 * 60_000,
|
||||
}
|
||||
|
||||
export function cooldownToMs(cd: ApiAlertRule["cooldown"]): number {
|
||||
return COOLDOWN_MS[cd] ?? COOLDOWN_MS["5м"]
|
||||
}
|
||||
|
||||
/** Можно ли отправить снова после lastIso (ISO) */
|
||||
export function isCooldownElapsed(lastFiredIso: string | null | undefined, cooldown: ApiAlertRule["cooldown"]): boolean {
|
||||
if (!lastFiredIso?.trim()) return true
|
||||
const t = Date.parse(lastFiredIso)
|
||||
if (!Number.isFinite(t)) return true
|
||||
return Date.now() - t >= cooldownToMs(cooldown)
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import type { ApiAlertGroup, ApiAlertRule } from "../alerts-service.js"
|
||||
import { computeStabilityReadyMap } from "./confirm-stability.js"
|
||||
import { isCooldownElapsed } from "./cooldown.js"
|
||||
import { groupShouldFire } from "./groups.js"
|
||||
import { isPositiveRecoveryTelegramText } from "./telegram-emoji.js"
|
||||
import { shouldAllowRecoveryByPolicy } from "./state-machine.js"
|
||||
import type { AlertEngineRuleDiag, RuleEvalHit } from "./types.js"
|
||||
|
||||
export type EngineStateMap = Map<string, { lastFiredAt: string; lastPayloadHash: string | null }>
|
||||
|
||||
export type StandaloneDecision = {
|
||||
kind: "rule"
|
||||
rule: ApiAlertRule
|
||||
hit: RuleEvalHit
|
||||
}
|
||||
|
||||
export type GroupDecision = {
|
||||
kind: "group"
|
||||
group: ApiAlertGroup
|
||||
members: ApiAlertRule[]
|
||||
hits: RuleEvalHit[]
|
||||
}
|
||||
|
||||
function ruleScopeKey(ruleId: string, transition?: RuleEvalHit["transition"]): string {
|
||||
const suffix = transition ?? "neutral"
|
||||
return `rule:${ruleId}:${suffix}`
|
||||
}
|
||||
|
||||
function textAfterGroupAlertHeader(fullBody: string): string {
|
||||
const segs = fullBody.split(/\n\n/)
|
||||
if (segs.length >= 2 && /^Группа\s*«/i.test(segs[0] ?? "")) return segs.slice(1).join("\n\n").trim()
|
||||
return fullBody
|
||||
}
|
||||
|
||||
export function buildRuleDiagnostics(
|
||||
rules: ApiAlertRule[],
|
||||
hitByRule: Map<string, RuleEvalHit | null>,
|
||||
stabilityReady: Map<string, boolean>,
|
||||
state: EngineStateMap,
|
||||
canSend: boolean,
|
||||
): AlertEngineRuleDiag[] {
|
||||
const out: AlertEngineRuleDiag[] = []
|
||||
for (const rule of rules) {
|
||||
if (!rule.enabled) continue
|
||||
const hit = hitByRule.get(rule.id) ?? null
|
||||
const inGroup = Boolean(rule.groupId)
|
||||
const evalHit = Boolean(hit)
|
||||
const stab = stabilityReady.get(rule.id) ?? false
|
||||
const key = ruleScopeKey(rule.id, hit?.transition)
|
||||
const prev = state.get(key)
|
||||
const cooldownOk = isCooldownElapsed(prev?.lastFiredAt, rule.cooldown)
|
||||
let blocked: AlertEngineRuleDiag["blocked"]
|
||||
if (!evalHit) blocked = "no_hit"
|
||||
else if (!shouldAllowRecoveryByPolicy(hit, rule.recoveryMode)) blocked = "stability"
|
||||
else if (inGroup) blocked = "in_group"
|
||||
else if (!stab) blocked = "stability"
|
||||
else if (!cooldownOk) blocked = "cooldown"
|
||||
else if (!canSend) blocked = "no_telegram"
|
||||
else if (hit && isPositiveRecoveryTelegramText(hit.message) && prev?.lastPayloadHash === hit.payloadHash) blocked = "dedupe_positive"
|
||||
else blocked = undefined
|
||||
out.push({
|
||||
ruleId: rule.id,
|
||||
inGroup,
|
||||
evalHit,
|
||||
hitTransition: hit?.transition,
|
||||
hitMessage: hit?.message,
|
||||
stabilityOk: stab,
|
||||
cooldownOk,
|
||||
telegramOk: canSend,
|
||||
blocked,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
export function computeDecisions(args: {
|
||||
rules: ApiAlertRule[]
|
||||
groups: ApiAlertGroup[]
|
||||
hitByRule: Map<string, RuleEvalHit | null>
|
||||
state: EngineStateMap
|
||||
canSend: boolean
|
||||
}): {
|
||||
stabilityReady: Map<string, boolean>
|
||||
standalone: StandaloneDecision[]
|
||||
grouped: GroupDecision[]
|
||||
ruleDiag: AlertEngineRuleDiag[]
|
||||
} {
|
||||
const { rules, groups, hitByRule, state, canSend } = args
|
||||
const stabilityReady = computeStabilityReadyMap(rules, hitByRule)
|
||||
const ruleDiag = buildRuleDiagnostics(rules, hitByRule, stabilityReady, state, canSend)
|
||||
const standalone: StandaloneDecision[] = []
|
||||
const grouped: GroupDecision[] = []
|
||||
|
||||
for (const rule of rules) {
|
||||
if (!rule.enabled || rule.groupId) continue
|
||||
const hit = hitByRule.get(rule.id)
|
||||
if (!hit) continue
|
||||
if (!shouldAllowRecoveryByPolicy(hit, rule.recoveryMode)) continue
|
||||
if (!stabilityReady.get(rule.id)) continue
|
||||
const key = ruleScopeKey(rule.id, hit.transition)
|
||||
const prev = state.get(key)
|
||||
if (!isCooldownElapsed(prev?.lastFiredAt, rule.cooldown)) continue
|
||||
if (isPositiveRecoveryTelegramText(hit.message) && prev?.lastPayloadHash === hit.payloadHash) continue
|
||||
standalone.push({ kind: "rule", rule, hit })
|
||||
}
|
||||
|
||||
for (const g of groups) {
|
||||
if (!g.enabled) continue
|
||||
const members = rules.filter((r) => r.groupId === g.id && r.enabled)
|
||||
if (members.length === 0) continue
|
||||
const firing = members.map((r) => {
|
||||
const h = hitByRule.get(r.id)
|
||||
if (!shouldAllowRecoveryByPolicy(h ?? null, r.recoveryMode)) return false
|
||||
return Boolean(h) && Boolean(stabilityReady.get(r.id))
|
||||
})
|
||||
if (!groupShouldFire(g.combineMode, firing)) continue
|
||||
const hits: RuleEvalHit[] = []
|
||||
for (const member of members) {
|
||||
const hit = hitByRule.get(member.id)
|
||||
if (!hit) continue
|
||||
if (!shouldAllowRecoveryByPolicy(hit, member.recoveryMode)) continue
|
||||
hits.push(hit)
|
||||
}
|
||||
if (hits.length === 0) continue
|
||||
const key = `group:${g.id}`
|
||||
const prev = state.get(key)
|
||||
const cd = g.cooldownOverride ?? "5м"
|
||||
if (!isCooldownElapsed(prev?.lastFiredAt, cd)) continue
|
||||
const body = `Группа «${g.name}» (${g.combineMode === "any" ? "ANY" : "ALL"})\n\n${hits.map((h) => h.message).join("\n")}`
|
||||
const hash = hits.map((h) => h.payloadHash).sort().join("|")
|
||||
if (isPositiveRecoveryTelegramText(textAfterGroupAlertHeader(body)) && prev?.lastPayloadHash === hash) continue
|
||||
grouped.push({ kind: "group", group: g, members, hits })
|
||||
}
|
||||
|
||||
return { stabilityReady, standalone, grouped, ruleDiag }
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
import assert from "node:assert/strict"
|
||||
import type { ApiAlertRule } from "../alerts-service.js"
|
||||
import { evaluateRule } from "./evaluate-rule.js"
|
||||
import type { SignalSnapshot } from "./types.js"
|
||||
|
||||
function emptySnap(over: Partial<SignalSnapshot> = {}): SignalSnapshot {
|
||||
return {
|
||||
sampledAt: "t",
|
||||
servers: [],
|
||||
probes: [],
|
||||
trafficByServer: [],
|
||||
greTunnels: [],
|
||||
bgpSessions: [],
|
||||
...over,
|
||||
}
|
||||
}
|
||||
|
||||
function rule(p: Partial<ApiAlertRule> & Pick<ApiAlertRule, "id" | "name" | "type" | "targets">): ApiAlertRule {
|
||||
return {
|
||||
target: p.targets[0] ?? "",
|
||||
groupId: null,
|
||||
condition: p.conditions?.[0] ?? p.condition ?? "",
|
||||
conditions: p.conditions ?? (p.condition ? [p.condition] : []),
|
||||
severity: p.severity ?? "warning",
|
||||
enabled: p.enabled ?? true,
|
||||
cooldown: p.cooldown ?? "5м",
|
||||
confirmStabilitySec: null,
|
||||
recoveryMode: "always",
|
||||
recoveryStabilitySec: null,
|
||||
chatId: "",
|
||||
lastFiredAt: null,
|
||||
...p,
|
||||
} as ApiAlertRule
|
||||
}
|
||||
|
||||
{
|
||||
const r = rule({
|
||||
id: "g1",
|
||||
name: "GRE",
|
||||
type: "gre-tunnel",
|
||||
targets: ["t1 / srv"],
|
||||
conditions: ["перешёл в offline"],
|
||||
})
|
||||
const hit = evaluateRule(
|
||||
r,
|
||||
emptySnap({
|
||||
greTunnels: [{ targetLabel: "t1 / srv", status: "down", prevStatus: "up" }],
|
||||
}),
|
||||
)
|
||||
assert.ok(hit)
|
||||
}
|
||||
|
||||
{
|
||||
const r = rule({
|
||||
id: "g2",
|
||||
name: "GRE rec",
|
||||
type: "gre-tunnel",
|
||||
targets: ["t1 / srv"],
|
||||
conditions: ["восстановился"],
|
||||
})
|
||||
const hit = evaluateRule(
|
||||
r,
|
||||
emptySnap({
|
||||
greTunnels: [{ targetLabel: "t1 / srv", status: "up", prevStatus: "down" }],
|
||||
}),
|
||||
)
|
||||
assert.ok(hit)
|
||||
}
|
||||
|
||||
{
|
||||
const r = rule({
|
||||
id: "b1",
|
||||
name: "BGP",
|
||||
type: "bgp-peer",
|
||||
targets: ["srv: peer1"],
|
||||
conditions: ["разорвал сессию"],
|
||||
})
|
||||
const hit = evaluateRule(
|
||||
r,
|
||||
emptySnap({
|
||||
bgpSessions: [{ key: "srv: peer1", state: "Idle", prevState: "Established" }],
|
||||
}),
|
||||
)
|
||||
assert.ok(hit)
|
||||
}
|
||||
|
||||
{
|
||||
const r = rule({
|
||||
id: "b2",
|
||||
name: "BGP up",
|
||||
type: "bgp-peer",
|
||||
targets: ["srv: peer1"],
|
||||
conditions: ["восстановил сессию"],
|
||||
})
|
||||
const hit = evaluateRule(
|
||||
r,
|
||||
emptySnap({
|
||||
bgpSessions: [{ key: "srv: peer1", state: "Established", prevState: "Active" }],
|
||||
}),
|
||||
)
|
||||
assert.ok(hit)
|
||||
}
|
||||
|
||||
{
|
||||
const r = rule({
|
||||
id: "pfx",
|
||||
name: "pfx",
|
||||
type: "bgp-prefix",
|
||||
targets: ["любой префикс"],
|
||||
conditions: ["был отозван"],
|
||||
})
|
||||
assert.equal(evaluateRule(r, emptySnap()), null)
|
||||
}
|
||||
|
||||
{
|
||||
const r = rule({
|
||||
id: "srv-rec",
|
||||
name: "rec",
|
||||
type: "server",
|
||||
targets: ["core"],
|
||||
conditions: ["восстановился"],
|
||||
})
|
||||
assert.ok(
|
||||
evaluateRule(
|
||||
r,
|
||||
emptySnap({
|
||||
servers: [
|
||||
{
|
||||
name: "core",
|
||||
status: "online",
|
||||
prevStatus: "offline",
|
||||
prev2Status: "offline",
|
||||
sampledAt: "t1",
|
||||
},
|
||||
],
|
||||
}),
|
||||
),
|
||||
"один online после offline — тоже «восстановился» (ручной REST / ресурс уже online)",
|
||||
)
|
||||
const hit = evaluateRule(
|
||||
r,
|
||||
emptySnap({
|
||||
servers: [
|
||||
{
|
||||
name: "core",
|
||||
status: "online",
|
||||
prevStatus: "online",
|
||||
prev2Status: "offline",
|
||||
sampledAt: "t2",
|
||||
},
|
||||
],
|
||||
}),
|
||||
)
|
||||
assert.ok(hit, "два подряд online после offline — одно срабатывание")
|
||||
assert.equal(hit?.transition, "recovery")
|
||||
}
|
||||
|
||||
{
|
||||
const r = rule({
|
||||
id: "srv-rec-offline-phrase",
|
||||
name: "rec-offline-phrase",
|
||||
type: "server",
|
||||
targets: ["core"],
|
||||
conditions: ["восстановился после offline"],
|
||||
})
|
||||
const hit = evaluateRule(
|
||||
r,
|
||||
emptySnap({
|
||||
servers: [
|
||||
{
|
||||
name: "core",
|
||||
status: "online",
|
||||
prevStatus: "online",
|
||||
prev2Status: "offline",
|
||||
sampledAt: "t2",
|
||||
},
|
||||
],
|
||||
}),
|
||||
)
|
||||
assert.ok(hit, "в тексте условия есть «offline» — всё равно восстановление по сэмплам")
|
||||
}
|
||||
|
||||
{
|
||||
const r = rule({
|
||||
id: "srv-off",
|
||||
name: "off",
|
||||
type: "server",
|
||||
targets: ["core"],
|
||||
conditions: ["перешёл в offline"],
|
||||
})
|
||||
const edgeHit = evaluateRule(
|
||||
r,
|
||||
emptySnap({
|
||||
servers: [
|
||||
{
|
||||
name: "core",
|
||||
status: "offline",
|
||||
prevStatus: "online",
|
||||
prev2Status: "online",
|
||||
sampledAt: "t",
|
||||
},
|
||||
],
|
||||
}),
|
||||
)
|
||||
assert.ok(edgeHit)
|
||||
assert.equal(edgeHit?.transition, "problem")
|
||||
assert.equal(
|
||||
evaluateRule(
|
||||
r,
|
||||
emptySnap({
|
||||
servers: [
|
||||
{
|
||||
name: "core",
|
||||
status: "offline",
|
||||
prevStatus: "offline",
|
||||
prev2Status: "online",
|
||||
sampledAt: "t",
|
||||
},
|
||||
],
|
||||
}),
|
||||
),
|
||||
null,
|
||||
"устойчивый offline — без повторного «перешёл» на каждом тике",
|
||||
)
|
||||
}
|
||||
|
||||
console.log("alert-engine evaluate-rule tests ok")
|
||||
@@ -0,0 +1,323 @@
|
||||
import { createHash } from "node:crypto"
|
||||
import type { ApiAlertRule } from "../alerts-service.js"
|
||||
import type { RuleEvalHit, SignalSnapshot } from "./types.js"
|
||||
|
||||
function hashPayload(parts: string[]): string {
|
||||
return createHash("sha256").update(parts.join("\0")).digest("hex").slice(0, 24)
|
||||
}
|
||||
|
||||
function firstNumberInString(s: string): number | null {
|
||||
const m = s.replace(",", ".").match(/(\d+(?:\.\d+)?)/)
|
||||
if (!m) return null
|
||||
const n = Number.parseFloat(m[1] ?? "")
|
||||
return Number.isFinite(n) ? n : null
|
||||
}
|
||||
|
||||
function maxSeverity(
|
||||
a: ApiAlertRule["severity"],
|
||||
b: ApiAlertRule["severity"],
|
||||
): ApiAlertRule["severity"] {
|
||||
const o = { critical: 3, warning: 2, info: 1 } as const
|
||||
return o[a] >= o[b] ? a : b
|
||||
}
|
||||
|
||||
function findServer(snap: SignalSnapshot, name: string) {
|
||||
const n = name.trim().toLowerCase()
|
||||
return snap.servers.find((s) => s.name.trim().toLowerCase() === n)
|
||||
}
|
||||
|
||||
function findProbe(snap: SignalSnapshot, targetKey: string) {
|
||||
const k = targetKey.trim()
|
||||
return snap.probes.find((p) => p.key.trim() === k)
|
||||
}
|
||||
|
||||
function findTraffic(snap: SignalSnapshot, serverName: string) {
|
||||
const n = serverName.trim().toLowerCase()
|
||||
return snap.trafficByServer.find((t) => t.serverName.trim().toLowerCase() === n)
|
||||
}
|
||||
|
||||
function findGreTunnel(snap: SignalSnapshot, targetLabel: string) {
|
||||
const n = targetLabel.trim()
|
||||
return snap.greTunnels.find((g) => g.targetLabel.trim() === n)
|
||||
}
|
||||
|
||||
function findBgpPeer(snap: SignalSnapshot, key: string) {
|
||||
const n = key.trim()
|
||||
return snap.bgpSessions.find((b) => b.key.trim() === n)
|
||||
}
|
||||
|
||||
function isBgpEstablished(state: string): boolean {
|
||||
return state.trim().toLowerCase() === "established"
|
||||
}
|
||||
|
||||
/** Заголовок для Telegram: не «…недоступен: … — восстановился». */
|
||||
function formatServerAlertTitle(ruleName: string, wantRecovery: boolean): string {
|
||||
if (!wantRecovery) return ruleName
|
||||
if (!/недоступен/i.test(ruleName)) return ruleName
|
||||
const trimmed = ruleName.replace(/\s*,\s*недоступен\s*$/i, "").replace(/\s+недоступен\s*$/i, "").trim()
|
||||
if (!trimmed || trimmed === ruleName) return ruleName
|
||||
return `${trimmed} — снова в сети`
|
||||
}
|
||||
|
||||
function evalServerLike(
|
||||
rule: ApiAlertRule,
|
||||
snap: SignalSnapshot,
|
||||
cond: string,
|
||||
): RuleEvalHit | null {
|
||||
const c = cond.toLowerCase()
|
||||
const isOtkluchilsya = /отключился|отключилась|отключилось/.test(c)
|
||||
/** «Перешёл в …» — только на границе, без спама пока сэмплы остаются offline. */
|
||||
const impliesOfflineEdge = /переш(ё|е)л/.test(c) && (/offline|офлайн/.test(c) || /недоступен/.test(c))
|
||||
const impliesDegradedEdge = /переш(ё|е)л/.test(c) && c.includes("degraded")
|
||||
const wantOffline =
|
||||
c.includes("offline") || c.includes("офлайн") || c.includes("отключ")
|
||||
const wantDegraded = c.includes("degraded")
|
||||
/** «Восстановился» / «подключился»: двойной online в сэмплах (см. тесты). Не отсекаем по слову «offline» в фразе вроде «после offline». */
|
||||
const wantRecovery =
|
||||
(c.includes("восстанов") || c.includes("подключ")) && !isOtkluchilsya
|
||||
const hits: string[] = []
|
||||
for (const tgt of rule.targets) {
|
||||
const s = findServer(snap, tgt)
|
||||
if (!s) continue
|
||||
if (wantRecovery) {
|
||||
const p2 = s.prev2Status
|
||||
if (
|
||||
s.status === "online" &&
|
||||
s.prevStatus === "online" &&
|
||||
p2 != null &&
|
||||
(p2 === "offline" || p2 === "degraded")
|
||||
) {
|
||||
hits.push(`${tgt}`)
|
||||
} else if (
|
||||
s.status === "online" &&
|
||||
(s.prevStatus === "offline" || s.prevStatus === "degraded")
|
||||
) {
|
||||
/** Один свежий online после offline (например ресурс уже online, REST только подтвердил). */
|
||||
hits.push(`${tgt}`)
|
||||
}
|
||||
} else if ((impliesOfflineEdge || isOtkluchilsya) && s.status === "offline") {
|
||||
if (s.prevStatus === null || s.prevStatus === "online") hits.push(tgt)
|
||||
} else if (impliesDegradedEdge && s.status === "degraded") {
|
||||
if (s.prevStatus === null || s.prevStatus === "online") hits.push(tgt)
|
||||
} else if (wantDegraded && s.status === "degraded" && !impliesDegradedEdge) {
|
||||
hits.push(tgt)
|
||||
} else if (
|
||||
wantOffline &&
|
||||
s.status === "offline" &&
|
||||
!impliesOfflineEdge &&
|
||||
!isOtkluchilsya
|
||||
) {
|
||||
hits.push(tgt)
|
||||
}
|
||||
}
|
||||
if (hits.length === 0) return null
|
||||
const title = formatServerAlertTitle(rule.name, wantRecovery)
|
||||
const msg = `${title}: ${hits.join(", ")} — ${cond}`
|
||||
return {
|
||||
ruleId: rule.id,
|
||||
ruleName: rule.name,
|
||||
severity: rule.severity,
|
||||
message: msg,
|
||||
transition: wantRecovery ? "recovery" : "problem",
|
||||
payloadHash: hashPayload([rule.id, "server", hits.sort().join(","), cond]),
|
||||
}
|
||||
}
|
||||
|
||||
function evalRtt(rule: ApiAlertRule, snap: SignalSnapshot, cond: string): RuleEvalHit | null {
|
||||
const threshold = firstNumberInString(cond)
|
||||
const wantLow = /<|ниже|меньше/i.test(cond) && !/>|выше|больше/i.test(cond)
|
||||
const wantHigh = />|выше|больше/i.test(cond)
|
||||
const hits: string[] = []
|
||||
for (const tgt of rule.targets) {
|
||||
const p = findProbe(snap, tgt)
|
||||
if (!p) continue
|
||||
if (p.status === "down" || p.status === "warn") {
|
||||
hits.push(`${tgt} (${p.status}, RTT ${p.rttMs ?? "—"} мс)`)
|
||||
continue
|
||||
}
|
||||
if (p.rttMs == null) continue
|
||||
if (wantHigh && threshold != null && p.rttMs > threshold) hits.push(`${tgt} (${p.rttMs} мс)`)
|
||||
if (wantLow && threshold != null && p.rttMs < threshold) hits.push(`${tgt} (${p.rttMs} мс)`)
|
||||
}
|
||||
if (hits.length === 0) return null
|
||||
const msg = `${rule.name}: RTT ${cond} — ${hits.join("; ")}`
|
||||
return {
|
||||
ruleId: rule.id,
|
||||
ruleName: rule.name,
|
||||
severity: rule.severity,
|
||||
message: msg,
|
||||
transition: "neutral",
|
||||
payloadHash: hashPayload([rule.id, "rtt", hits.sort().join(";"), cond]),
|
||||
}
|
||||
}
|
||||
|
||||
function evalLoss(rule: ApiAlertRule, snap: SignalSnapshot, cond: string): RuleEvalHit | null {
|
||||
const threshold = firstNumberInString(cond)
|
||||
if (threshold == null) return null
|
||||
const hits: string[] = []
|
||||
for (const tgt of rule.targets) {
|
||||
const p = findProbe(snap, tgt)
|
||||
if (!p || p.lossPct == null) continue
|
||||
if (p.lossPct > threshold) hits.push(`${tgt} (${p.lossPct}%)`)
|
||||
}
|
||||
if (hits.length === 0) return null
|
||||
const msg = `${rule.name}: потери ${cond} — ${hits.join("; ")}`
|
||||
return {
|
||||
ruleId: rule.id,
|
||||
ruleName: rule.name,
|
||||
severity: rule.severity,
|
||||
message: msg,
|
||||
transition: "neutral",
|
||||
payloadHash: hashPayload([rule.id, "loss", hits.sort().join(";"), cond]),
|
||||
}
|
||||
}
|
||||
|
||||
function evalGreTunnel(rule: ApiAlertRule, snap: SignalSnapshot, cond: string): RuleEvalHit | null {
|
||||
const lc = cond.toLowerCase()
|
||||
const wantOffline = lc.includes("offline") && !lc.includes("восстанов")
|
||||
const wantRecovery =
|
||||
lc.includes("восстанов") && !/offline|отключ/i.test(lc)
|
||||
const hits: string[] = []
|
||||
for (const tgt of rule.targets) {
|
||||
const g = findGreTunnel(snap, tgt)
|
||||
if (!g) continue
|
||||
if (wantRecovery) {
|
||||
const prev = g.prevStatus
|
||||
if (
|
||||
g.status === "up" &&
|
||||
prev != null &&
|
||||
(prev === "down" || prev === "degraded")
|
||||
) {
|
||||
hits.push(tgt)
|
||||
}
|
||||
} else if (wantOffline && g.status === "down") {
|
||||
hits.push(tgt)
|
||||
}
|
||||
}
|
||||
if (hits.length === 0) return null
|
||||
const msg = `${rule.name}: ${hits.join(", ")} — ${cond}`
|
||||
return {
|
||||
ruleId: rule.id,
|
||||
ruleName: rule.name,
|
||||
severity: rule.severity,
|
||||
message: msg,
|
||||
transition: wantRecovery ? "recovery" : "problem",
|
||||
payloadHash: hashPayload([rule.id, "gre-tunnel", hits.sort().join(","), cond]),
|
||||
}
|
||||
}
|
||||
|
||||
function evalBgpPeer(rule: ApiAlertRule, snap: SignalSnapshot, cond: string): RuleEvalHit | null {
|
||||
const lc = cond.toLowerCase()
|
||||
const wantDown = lc.includes("разорвал")
|
||||
const wantUp = lc.includes("восстановил")
|
||||
const hits: string[] = []
|
||||
for (const tgt of rule.targets) {
|
||||
const b = findBgpPeer(snap, tgt)
|
||||
if (!b) continue
|
||||
if (wantDown && !isBgpEstablished(b.state)) hits.push(tgt)
|
||||
if (
|
||||
wantUp &&
|
||||
isBgpEstablished(b.state) &&
|
||||
b.prevState != null &&
|
||||
String(b.prevState).trim() !== "" &&
|
||||
!isBgpEstablished(b.prevState)
|
||||
) {
|
||||
hits.push(tgt)
|
||||
}
|
||||
}
|
||||
if (hits.length === 0) return null
|
||||
const msg = `${rule.name}: ${hits.join(", ")} — ${cond}`
|
||||
return {
|
||||
ruleId: rule.id,
|
||||
ruleName: rule.name,
|
||||
severity: rule.severity,
|
||||
message: msg,
|
||||
transition: wantUp ? "recovery" : "problem",
|
||||
payloadHash: hashPayload([rule.id, "bgp-peer", hits.sort().join(","), cond]),
|
||||
}
|
||||
}
|
||||
|
||||
function evalTraffic(rule: ApiAlertRule, snap: SignalSnapshot, cond: string): RuleEvalHit | null {
|
||||
const threshold = firstNumberInString(cond)
|
||||
if (threshold == null) return null
|
||||
const rxLow = /RX\s*</i.test(cond)
|
||||
const txLow = /TX\s*</i.test(cond)
|
||||
const hits: string[] = []
|
||||
for (const tgt of rule.targets) {
|
||||
const t = findTraffic(snap, tgt)
|
||||
if (!t) continue
|
||||
if (rxLow && t.rxMbps < threshold) hits.push(`${tgt} RX ${t.rxMbps.toFixed(2)} Мбит/с`)
|
||||
if (txLow && t.txMbps < threshold) hits.push(`${tgt} TX ${t.txMbps.toFixed(2)} Мбит/с`)
|
||||
}
|
||||
if (hits.length === 0) return null
|
||||
const msg = `${rule.name}: трафик ${cond} — ${hits.join("; ")}`
|
||||
return {
|
||||
ruleId: rule.id,
|
||||
ruleName: rule.name,
|
||||
severity: rule.severity,
|
||||
message: msg,
|
||||
transition: "neutral",
|
||||
payloadHash: hashPayload([rule.id, "traffic", hits.sort().join(";"), cond]),
|
||||
}
|
||||
}
|
||||
|
||||
function evaluateRuleWithCondition(
|
||||
rule: ApiAlertRule,
|
||||
snap: SignalSnapshot,
|
||||
cond: string,
|
||||
): RuleEvalHit | null {
|
||||
const c = cond.trim()
|
||||
if (!c) return null
|
||||
switch (rule.type) {
|
||||
case "server":
|
||||
case "gre-client":
|
||||
return evalServerLike(rule, snap, c)
|
||||
case "rtt":
|
||||
return evalRtt(rule, snap, c)
|
||||
case "loss":
|
||||
return evalLoss(rule, snap, c)
|
||||
case "traffic":
|
||||
return evalTraffic(rule, snap, c)
|
||||
case "gre-tunnel":
|
||||
return evalGreTunnel(rule, snap, c)
|
||||
case "bgp-peer":
|
||||
return evalBgpPeer(rule, snap, c)
|
||||
case "bgp-prefix":
|
||||
return null
|
||||
default:
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/** Одно правило: OR по targets; OR по conditions (любое из выбранных). */
|
||||
export function evaluateRule(rule: ApiAlertRule, snap: SignalSnapshot): RuleEvalHit | null {
|
||||
if (!rule.enabled || rule.targets.length === 0) return null
|
||||
const conds =
|
||||
rule.conditions?.length > 0
|
||||
? rule.conditions.map((c) => c.trim()).filter(Boolean)
|
||||
: [rule.condition.trim()].filter(Boolean)
|
||||
if (conds.length === 0) return null
|
||||
|
||||
const hits: RuleEvalHit[] = []
|
||||
for (const cond of conds) {
|
||||
const h = evaluateRuleWithCondition(rule, snap, cond)
|
||||
if (h) hits.push(h)
|
||||
}
|
||||
if (hits.length === 0) return null
|
||||
if (hits.length === 1) return hits[0]!
|
||||
const message = hits.map((h) => h.message).join("\n")
|
||||
const severity = hits.reduce((a, h) => maxSeverity(a, h.severity), hits[0]!.severity)
|
||||
const payloadHash = hashPayload([rule.id, "or-conds", ...hits.map((h) => h.payloadHash).sort()])
|
||||
return {
|
||||
ruleId: rule.id,
|
||||
ruleName: rule.name,
|
||||
severity,
|
||||
message,
|
||||
transition: hits.some((h) => h.transition === "problem")
|
||||
? "problem"
|
||||
: hits.some((h) => h.transition === "recovery")
|
||||
? "recovery"
|
||||
: "neutral",
|
||||
payloadHash,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { groupShouldFire } from "./groups.js"
|
||||
import { cooldownToMs, isCooldownElapsed } from "./cooldown.js"
|
||||
|
||||
assert.equal(groupShouldFire("any", []), false)
|
||||
assert.equal(groupShouldFire("any", [false, false]), false)
|
||||
assert.equal(groupShouldFire("any", [false, true]), true)
|
||||
assert.equal(groupShouldFire("all", [true, true]), true)
|
||||
assert.equal(groupShouldFire("all", [true, false]), false)
|
||||
assert.equal(groupShouldFire("all", [false, false]), false)
|
||||
|
||||
assert.equal(cooldownToMs("1м"), 60_000)
|
||||
assert.equal(isCooldownElapsed(null, "5м"), true)
|
||||
assert.equal(isCooldownElapsed(new Date(Date.now() - 6 * 60_000).toISOString(), "5м"), true)
|
||||
assert.equal(isCooldownElapsed(new Date(Date.now() - 2 * 60_000).toISOString(), "5м"), false)
|
||||
|
||||
console.log("alert-engine groups/cooldown tests ok")
|
||||
@@ -0,0 +1,9 @@
|
||||
/**
|
||||
* Логика группы: ANY = хотя бы один член «в огне»; ALL = все члены «в огне».
|
||||
* `memberFiring` — только включённые правила группы, по порядку.
|
||||
*/
|
||||
export function groupShouldFire(combineMode: "any" | "all", memberFiring: boolean[]): boolean {
|
||||
if (memberFiring.length === 0) return false
|
||||
if (combineMode === "any") return memberFiring.some(Boolean)
|
||||
return memberFiring.every(Boolean)
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
import { and, asc, eq, lte } from "drizzle-orm"
|
||||
import { db } from "../../db/index.js"
|
||||
import { alertOutbox } from "../../db/schema.js"
|
||||
import { appendAlertHistory, sendTelegramAlertMessage } from "../alerts-service.js"
|
||||
|
||||
export type AlertOutboxPayload = {
|
||||
text: string
|
||||
chatId?: string
|
||||
history: {
|
||||
id: string
|
||||
ruleId?: string | null
|
||||
groupId?: string | null
|
||||
ruleName: string
|
||||
severity: "critical" | "warning" | "info"
|
||||
message: string
|
||||
firedAt: string
|
||||
}
|
||||
}
|
||||
|
||||
function safeParsePayload(raw: string): AlertOutboxPayload | null {
|
||||
try {
|
||||
return JSON.parse(raw) as AlertOutboxPayload
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
export function enqueueTelegramOutbox(input: {
|
||||
id: string
|
||||
dedupeKey: string
|
||||
payload: AlertOutboxPayload
|
||||
maxRetries?: number
|
||||
}): boolean {
|
||||
const existing = db
|
||||
.select()
|
||||
.from(alertOutbox)
|
||||
.where(eq(alertOutbox.dedupeKey, input.dedupeKey))
|
||||
.limit(1)
|
||||
.all()[0]
|
||||
if (existing && (existing.status === "pending" || existing.status === "sent")) return false
|
||||
const nowIso = new Date().toISOString()
|
||||
db.insert(alertOutbox)
|
||||
.values({
|
||||
id: input.id,
|
||||
dedupeKey: input.dedupeKey,
|
||||
channel: "telegram",
|
||||
status: "pending",
|
||||
retryCount: 0,
|
||||
maxRetries: Math.max(1, Math.floor(input.maxRetries ?? 3)),
|
||||
nextAttemptAt: nowIso,
|
||||
payloadJson: JSON.stringify(input.payload),
|
||||
})
|
||||
.run()
|
||||
return true
|
||||
}
|
||||
|
||||
export function dispatchPendingOutbox(limit = 25): Promise<{ sent: number; failed: number; errors: string[] }> {
|
||||
return (async () => {
|
||||
const nowIso = new Date().toISOString()
|
||||
const rows = db
|
||||
.select()
|
||||
.from(alertOutbox)
|
||||
.where(and(eq(alertOutbox.status, "pending"), lte(alertOutbox.nextAttemptAt, nowIso)))
|
||||
.orderBy(asc(alertOutbox.createdAt))
|
||||
.limit(Math.max(1, limit))
|
||||
.all()
|
||||
|
||||
let sent = 0
|
||||
let failed = 0
|
||||
const errors: string[] = []
|
||||
|
||||
for (const row of rows) {
|
||||
const payload = safeParsePayload(row.payloadJson)
|
||||
if (!payload) {
|
||||
db.update(alertOutbox)
|
||||
.set({
|
||||
status: "failed",
|
||||
lastError: "invalid payload_json",
|
||||
})
|
||||
.where(eq(alertOutbox.id, row.id))
|
||||
.run()
|
||||
failed += 1
|
||||
continue
|
||||
}
|
||||
const send = await sendTelegramAlertMessage({ text: payload.text, chatId: payload.chatId })
|
||||
if (send.ok) {
|
||||
db.update(alertOutbox)
|
||||
.set({
|
||||
status: "sent",
|
||||
sentAt: new Date().toISOString(),
|
||||
lastError: null,
|
||||
})
|
||||
.where(eq(alertOutbox.id, row.id))
|
||||
.run()
|
||||
appendAlertHistory({
|
||||
id: payload.history.id,
|
||||
ruleId: payload.history.ruleId ?? null,
|
||||
groupId: payload.history.groupId ?? null,
|
||||
ruleName: payload.history.ruleName,
|
||||
severity: payload.history.severity,
|
||||
message: payload.history.message,
|
||||
sentOk: true,
|
||||
firedAt: payload.history.firedAt,
|
||||
})
|
||||
sent += 1
|
||||
continue
|
||||
}
|
||||
const nextRetry = row.retryCount + 1
|
||||
const exhausted = nextRetry >= row.maxRetries
|
||||
db.update(alertOutbox)
|
||||
.set({
|
||||
retryCount: nextRetry,
|
||||
status: exhausted ? "failed" : "pending",
|
||||
nextAttemptAt: new Date(Date.now() + Math.min(300_000, nextRetry * 15_000)).toISOString(),
|
||||
lastError: send.error,
|
||||
})
|
||||
.where(eq(alertOutbox.id, row.id))
|
||||
.run()
|
||||
if (exhausted) {
|
||||
appendAlertHistory({
|
||||
id: payload.history.id,
|
||||
ruleId: payload.history.ruleId ?? null,
|
||||
groupId: payload.history.groupId ?? null,
|
||||
ruleName: payload.history.ruleName,
|
||||
severity: payload.history.severity,
|
||||
message: payload.history.message,
|
||||
sentOk: false,
|
||||
firedAt: payload.history.firedAt,
|
||||
})
|
||||
failed += 1
|
||||
} else {
|
||||
errors.push(`outbox ${row.id}: ${send.error}`)
|
||||
}
|
||||
}
|
||||
|
||||
return { sent, failed, errors }
|
||||
})()
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { eq } from "drizzle-orm"
|
||||
import { db } from "../../db/index.js"
|
||||
import { alertEnginePrevLive } from "../../db/schema.js"
|
||||
|
||||
export type PrevLiveKind = "gre" | "bgp"
|
||||
|
||||
/** Предыдущие строковые состояния (GRE: up|down|degraded; BGP: state как с роутера). */
|
||||
export type PrevLiveStringMap = Record<string, string>
|
||||
|
||||
function safeParseMap(json: string | null | undefined): PrevLiveStringMap {
|
||||
if (!json || json.trim() === "") return {}
|
||||
try {
|
||||
const v = JSON.parse(json) as unknown
|
||||
if (v == null || typeof v !== "object" || Array.isArray(v)) return {}
|
||||
return v as PrevLiveStringMap
|
||||
} catch {
|
||||
return {}
|
||||
}
|
||||
}
|
||||
|
||||
export function loadPrevLiveMap(kind: PrevLiveKind): PrevLiveStringMap {
|
||||
const row = db
|
||||
.select()
|
||||
.from(alertEnginePrevLive)
|
||||
.where(eq(alertEnginePrevLive.kind, kind))
|
||||
.limit(1)
|
||||
.all()[0]
|
||||
return safeParseMap(row?.payloadJson)
|
||||
}
|
||||
|
||||
export function savePrevLiveMap(kind: PrevLiveKind, map: PrevLiveStringMap) {
|
||||
const payloadJson = JSON.stringify(map)
|
||||
const existing = db
|
||||
.select()
|
||||
.from(alertEnginePrevLive)
|
||||
.where(eq(alertEnginePrevLive.kind, kind))
|
||||
.limit(1)
|
||||
.all()[0]
|
||||
if (existing) {
|
||||
db.update(alertEnginePrevLive)
|
||||
.set({ payloadJson, updatedAt: new Date().toISOString() })
|
||||
.where(eq(alertEnginePrevLive.kind, kind))
|
||||
.run()
|
||||
} else {
|
||||
db.insert(alertEnginePrevLive).values({ kind, payloadJson, updatedAt: new Date().toISOString() }).run()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import type { ApiAlertRule } from "../alerts-service.js"
|
||||
import { evaluateRule } from "./evaluate-rule.js"
|
||||
import type { RuleEvalHit, SignalSnapshot } from "./types.js"
|
||||
|
||||
export function matchRules(rules: ApiAlertRule[], snap: SignalSnapshot): {
|
||||
hitByRule: Map<string, RuleEvalHit | null>
|
||||
errors: string[]
|
||||
} {
|
||||
const errors: string[] = []
|
||||
const hitByRule = new Map<string, RuleEvalHit | null>()
|
||||
for (const rule of rules) {
|
||||
try {
|
||||
hitByRule.set(rule.id, evaluateRule(rule, snap))
|
||||
} catch (e) {
|
||||
errors.push(`rule ${rule.id}: ${e instanceof Error ? e.message : String(e)}`)
|
||||
hitByRule.set(rule.id, null)
|
||||
}
|
||||
}
|
||||
return { hitByRule, errors }
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
import { db } from "../../db/index.js"
|
||||
import { alertEngineState } from "../../db/schema.js"
|
||||
import { eq } from "drizzle-orm"
|
||||
import {
|
||||
getTelegramPublic,
|
||||
listAlertGroups,
|
||||
listAlertRules,
|
||||
type ApiAlertRule,
|
||||
} from "../alerts-service.js"
|
||||
import { getGreBgpSnapshotCollectorState } from "../gre-bgp-snapshot-collector.js"
|
||||
import { isAnyAlertSnapshotSourceJobRunning } from "../scheduler-running.js"
|
||||
import { getServersRestPingCollectorState } from "../servers-rest-ping-collector.js"
|
||||
import { isTrafficCollecting } from "../traffic-collector.js"
|
||||
import { getUptimeCollectorsState } from "../uptime-collector.js"
|
||||
import { computeDecisions } from "./decision-engine.js"
|
||||
import { clearConfirmPendingAfterSend } from "./confirm-stability.js"
|
||||
import { dispatchPendingOutbox, enqueueTelegramOutbox } from "./outbox.js"
|
||||
import { matchRules } from "./rule-matcher.js"
|
||||
import { ingestAlertSignals } from "./signal-ingestor.js"
|
||||
import { getLatestSourceFinishedAt, getSourceWatermark, updateSourceWatermark } from "./source-watermark.js"
|
||||
import { pickTelegramAlertEmoji } from "./telegram-emoji.js"
|
||||
import type { AlertEngineRunResult, RuleEvalHit } from "./types.js"
|
||||
|
||||
const SNAPSHOT_SOURCE_WAIT_MS = 30_000
|
||||
const SNAPSHOT_SOURCE_POLL_MS = 20
|
||||
|
||||
/**
|
||||
* Не строить снимок, пока коллекторы пишут в SQLite — иначе гонка с `alert_engine` по таймеру
|
||||
* (в т.ч. слот планировщика занят до первого `await` в коллекторе, когда внутренний `collecting` ещё false).
|
||||
*/
|
||||
async function awaitSnapshotSourcesIdle(): Promise<void> {
|
||||
const t0 = Date.now()
|
||||
while (Date.now() - t0 < SNAPSHOT_SOURCE_WAIT_MS) {
|
||||
const busy =
|
||||
isAnyAlertSnapshotSourceJobRunning() ||
|
||||
getServersRestPingCollectorState().running ||
|
||||
getGreBgpSnapshotCollectorState().running ||
|
||||
getUptimeCollectorsState().resources ||
|
||||
getUptimeCollectorsState().ping ||
|
||||
isTrafficCollecting()
|
||||
if (!busy) return
|
||||
await new Promise<void>((r) => setTimeout(r, SNAPSHOT_SOURCE_POLL_MS))
|
||||
}
|
||||
}
|
||||
|
||||
function newHistoryId(): string {
|
||||
return `ah-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`
|
||||
}
|
||||
|
||||
function newOutboxId(): string {
|
||||
return `ao-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`
|
||||
}
|
||||
|
||||
function loadEngineStateMap(): Map<string, { lastFiredAt: string; lastPayloadHash: string | null }> {
|
||||
const rows = db.select().from(alertEngineState).all()
|
||||
const m = new Map<string, { lastFiredAt: string; lastPayloadHash: string | null }>()
|
||||
for (const r of rows) {
|
||||
m.set(r.scopeKey, { lastFiredAt: r.lastFiredAt, lastPayloadHash: r.lastPayloadHash ?? null })
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
function upsertEngineState(scopeKey: string, lastFiredAt: string, lastPayloadHash: string | null) {
|
||||
const existing = db.select().from(alertEngineState).where(eq(alertEngineState.scopeKey, scopeKey)).limit(1).all()[0]
|
||||
if (existing) {
|
||||
db.update(alertEngineState)
|
||||
.set({ lastFiredAt, lastPayloadHash })
|
||||
.where(eq(alertEngineState.scopeKey, scopeKey))
|
||||
.run()
|
||||
} else {
|
||||
db.insert(alertEngineState).values({ scopeKey, lastFiredAt, lastPayloadHash }).run()
|
||||
}
|
||||
}
|
||||
|
||||
function severityRank(s: ApiAlertRule["severity"]): number {
|
||||
if (s === "critical") return 3
|
||||
if (s === "warning") return 2
|
||||
return 1
|
||||
}
|
||||
|
||||
function maxSeverity(a: ApiAlertRule["severity"], b: ApiAlertRule["severity"]): ApiAlertRule["severity"] {
|
||||
return severityRank(a) >= severityRank(b) ? a : b
|
||||
}
|
||||
function ruleScopeKey(ruleId: string, transition?: RuleEvalHit["transition"]): string {
|
||||
return `rule:${ruleId}:${transition ?? "neutral"}`
|
||||
}
|
||||
|
||||
/** Один проход движка: оценка правил, группы, Telegram, история, состояние cooldown. */
|
||||
export async function runAlertEngineOnce(): Promise<AlertEngineRunResult> {
|
||||
const errors: string[] = []
|
||||
const sampledAt = new Date().toISOString()
|
||||
const latestSourceFinishedAt = getLatestSourceFinishedAt()
|
||||
const prevSourceFinishedAt = getSourceWatermark()
|
||||
const hasNewSources =
|
||||
latestSourceFinishedAt != null &&
|
||||
(prevSourceFinishedAt == null || latestSourceFinishedAt > prevSourceFinishedAt)
|
||||
|
||||
const rules = listAlertRules()
|
||||
const groups = listAlertGroups()
|
||||
const state = loadEngineStateMap()
|
||||
const pub = getTelegramPublic()
|
||||
const canSend = pub.tokenConfigured && Boolean(pub.chatId?.trim())
|
||||
let standaloneFires = 0
|
||||
let groupFires = 0
|
||||
|
||||
let ruleDiag: AlertEngineRunResult["ruleDiag"] = []
|
||||
if (hasNewSources) {
|
||||
await awaitSnapshotSourcesIdle()
|
||||
const snap = ingestAlertSignals()
|
||||
const matched = matchRules(rules, snap)
|
||||
errors.push(...matched.errors)
|
||||
const { standalone, grouped, ruleDiag: diag } = computeDecisions({
|
||||
rules,
|
||||
groups,
|
||||
hitByRule: matched.hitByRule,
|
||||
state,
|
||||
canSend,
|
||||
})
|
||||
ruleDiag = diag
|
||||
|
||||
for (const d of standalone) {
|
||||
if (!canSend) break
|
||||
const firedAt = new Date().toISOString()
|
||||
const key = ruleScopeKey(d.rule.id, d.hit.transition)
|
||||
const queued = enqueueTelegramOutbox({
|
||||
id: newOutboxId(),
|
||||
dedupeKey: `${key}:${d.hit.payloadHash}`,
|
||||
payload: {
|
||||
text: `${pickTelegramAlertEmoji(d.hit.message)} ${d.hit.message}`,
|
||||
chatId: d.rule.chatId?.trim() || undefined,
|
||||
history: {
|
||||
id: newHistoryId(),
|
||||
ruleId: d.rule.id,
|
||||
groupId: null,
|
||||
ruleName: d.hit.ruleName,
|
||||
severity: d.hit.severity,
|
||||
message: d.hit.message,
|
||||
firedAt,
|
||||
},
|
||||
},
|
||||
})
|
||||
if (!queued) continue
|
||||
standaloneFires += 1
|
||||
clearConfirmPendingAfterSend([d.rule.id])
|
||||
upsertEngineState(key, firedAt, d.hit.payloadHash)
|
||||
state.set(key, { lastFiredAt: firedAt, lastPayloadHash: d.hit.payloadHash })
|
||||
}
|
||||
|
||||
for (const d of grouped) {
|
||||
if (!canSend) break
|
||||
const firedAt = new Date().toISOString()
|
||||
const lines = d.hits.map((h) => h.message)
|
||||
const sev = d.hits.reduce((a, h) => maxSeverity(a, h.severity), d.hits[0]!.severity)
|
||||
const body = `Группа «${d.group.name}» (${d.group.combineMode === "any" ? "ANY" : "ALL"})\n\n${lines.join("\n")}`
|
||||
const hash = d.hits.map((h) => h.payloadHash).sort().join("|")
|
||||
const key = `group:${d.group.id}`
|
||||
const queued = enqueueTelegramOutbox({
|
||||
id: newOutboxId(),
|
||||
dedupeKey: `${key}:${hash}`,
|
||||
payload: {
|
||||
text: `${pickTelegramAlertEmoji(body)} ${body}`,
|
||||
history: {
|
||||
id: newHistoryId(),
|
||||
ruleId: null,
|
||||
groupId: d.group.id,
|
||||
ruleName: `Группа: ${d.group.name}`,
|
||||
severity: sev,
|
||||
message: body,
|
||||
firedAt,
|
||||
},
|
||||
},
|
||||
})
|
||||
if (!queued) continue
|
||||
groupFires += 1
|
||||
upsertEngineState(key, firedAt, hash)
|
||||
state.set(key, { lastFiredAt: firedAt, lastPayloadHash: hash })
|
||||
clearConfirmPendingAfterSend(d.members.map((r) => r.id))
|
||||
}
|
||||
updateSourceWatermark(latestSourceFinishedAt)
|
||||
}
|
||||
|
||||
const outbox = await dispatchPendingOutbox()
|
||||
errors.push(...outbox.errors)
|
||||
|
||||
return {
|
||||
sampledAt,
|
||||
rulesChecked: hasNewSources ? rules.length : 0,
|
||||
standaloneFires,
|
||||
groupFires,
|
||||
skippedNoTelegram: !canSend,
|
||||
errors,
|
||||
ruleDiag: ruleDiag ?? [],
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
import { buildSignalSnapshotFromCollectors } from "./signals-from-collectors.js"
|
||||
|
||||
/** Единая точка входа для подготовки сигналов к rule matching. */
|
||||
export function ingestAlertSignals() {
|
||||
return buildSignalSnapshotFromCollectors()
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
import { sqliteDatabase } from "../../db/index.js"
|
||||
import type {
|
||||
GreBgpSnapshotRunSnapshot,
|
||||
PingRunSnapshot,
|
||||
ResourcesRunSnapshot,
|
||||
SchedulerRunSnapshot,
|
||||
ServersRestPingRunSnapshot,
|
||||
TrafficRunSnapshot,
|
||||
} from "../../types/scheduler-run-snapshot.js"
|
||||
import type {
|
||||
BgpPeerSignal,
|
||||
GreTunnelSignal,
|
||||
ProbeSignal,
|
||||
ServerSignal,
|
||||
SignalSnapshot,
|
||||
TrafficServerSignal,
|
||||
} from "./types.js"
|
||||
|
||||
function normalizeNameKey(v: string): string {
|
||||
return v.trim().toLowerCase()
|
||||
}
|
||||
|
||||
function safeParseRunSnapshot(raw: string | null): SchedulerRunSnapshot | null {
|
||||
if (!raw) return null
|
||||
try {
|
||||
return JSON.parse(raw) as SchedulerRunSnapshot
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function readLatestOkRunSnapshots(jobKey: string, limit = 32): SchedulerRunSnapshot[] {
|
||||
const rows = sqliteDatabase
|
||||
.prepare(
|
||||
`SELECT result_json AS resultJson
|
||||
FROM scheduler_runs
|
||||
WHERE job_key = ? AND status = 'ok' AND result_json IS NOT NULL
|
||||
ORDER BY finished_at DESC
|
||||
LIMIT ?`,
|
||||
)
|
||||
.all(jobKey, limit) as { resultJson: string | null }[]
|
||||
const out: SchedulerRunSnapshot[] = []
|
||||
for (const row of rows) {
|
||||
const parsed = safeParseRunSnapshot(row.resultJson)
|
||||
if (!parsed || parsed.job !== jobKey) continue
|
||||
out.push(parsed)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type TriState = "online" | "offline" | "degraded"
|
||||
|
||||
function timelineToServerSignal(name: string, sampledAt: string, statuses: TriState[]): ServerSignal {
|
||||
return {
|
||||
name,
|
||||
status: statuses[0] ?? "offline",
|
||||
prevStatus: statuses[1] ?? null,
|
||||
prev2Status: statuses[2] ?? null,
|
||||
sampledAt,
|
||||
}
|
||||
}
|
||||
|
||||
function collectServerSignalsFromResourceRuns(runs: ResourcesRunSnapshot[]): Map<string, ServerSignal> {
|
||||
const byKey = new Map<
|
||||
string,
|
||||
{ name: string; sampledAt: string; statuses: TriState[] }
|
||||
>()
|
||||
for (const run of runs) {
|
||||
for (const s of run.servers ?? []) {
|
||||
const name = String(s.name ?? "").trim()
|
||||
if (!name) continue
|
||||
const key = normalizeNameKey(name)
|
||||
const rec = byKey.get(key) ?? { name, sampledAt: run.sampledAt, statuses: [] }
|
||||
if (rec.statuses.length < 3) rec.statuses.push(s.status === "online" ? "online" : "offline")
|
||||
if (!byKey.has(key)) rec.sampledAt = run.sampledAt
|
||||
byKey.set(key, rec)
|
||||
}
|
||||
}
|
||||
const out = new Map<string, ServerSignal>()
|
||||
for (const [key, rec] of byKey) out.set(key, timelineToServerSignal(rec.name, rec.sampledAt, rec.statuses))
|
||||
return out
|
||||
}
|
||||
|
||||
function collectServerSignalsFromRestRuns(runs: ServersRestPingRunSnapshot[]): Map<string, ServerSignal> {
|
||||
const byKey = new Map<
|
||||
string,
|
||||
{ name: string; sampledAt: string; statuses: TriState[] }
|
||||
>()
|
||||
for (const run of runs) {
|
||||
for (const s of run.servers ?? []) {
|
||||
const name = String(s.name ?? "").trim()
|
||||
if (!name) continue
|
||||
const key = normalizeNameKey(name)
|
||||
const rec = byKey.get(key) ?? { name, sampledAt: run.sampledAt, statuses: [] }
|
||||
if (rec.statuses.length < 3) rec.statuses.push(s.ok ? "online" : "offline")
|
||||
if (!byKey.has(key)) rec.sampledAt = run.sampledAt
|
||||
byKey.set(key, rec)
|
||||
}
|
||||
}
|
||||
const out = new Map<string, ServerSignal>()
|
||||
for (const [key, rec] of byKey) out.set(key, timelineToServerSignal(rec.name, rec.sampledAt, rec.statuses))
|
||||
return out
|
||||
}
|
||||
|
||||
function mergeResourceAndRestServer(resource: ServerSignal, rest: ServerSignal): ServerSignal {
|
||||
const rUp = resource.status === "online"
|
||||
const tUp = rest.status === "online"
|
||||
if (tUp && !rUp) {
|
||||
const rBad = resource.status === "offline" || resource.status === "degraded"
|
||||
if (rBad) {
|
||||
return {
|
||||
name: rest.name,
|
||||
status: "online",
|
||||
prevStatus: "online",
|
||||
prev2Status: resource.status,
|
||||
sampledAt: rest.sampledAt,
|
||||
}
|
||||
}
|
||||
return rest
|
||||
}
|
||||
if (rUp && !tUp) return resource
|
||||
if (tUp && rUp) {
|
||||
if (rest.sampledAt >= resource.sampledAt) {
|
||||
if (resource.prevStatus === "offline" || resource.prevStatus === "degraded") {
|
||||
return {
|
||||
name: rest.name,
|
||||
status: "online",
|
||||
prevStatus: "online",
|
||||
prev2Status: resource.prevStatus === "degraded" ? "degraded" : "offline",
|
||||
sampledAt: rest.sampledAt,
|
||||
}
|
||||
}
|
||||
if (
|
||||
resource.prevStatus === "online" &&
|
||||
resource.prev2Status != null &&
|
||||
(resource.prev2Status === "offline" || resource.prev2Status === "degraded")
|
||||
) {
|
||||
return {
|
||||
name: rest.name,
|
||||
status: "online",
|
||||
prevStatus: "online",
|
||||
prev2Status: resource.prev2Status,
|
||||
sampledAt: rest.sampledAt,
|
||||
}
|
||||
}
|
||||
}
|
||||
return rest.sampledAt >= resource.sampledAt ? rest : resource
|
||||
}
|
||||
return rest.sampledAt >= resource.sampledAt ? rest : resource
|
||||
}
|
||||
|
||||
function collectProbeSignalsFromRuns(runs: PingRunSnapshot[]): ProbeSignal[] {
|
||||
const out = new Map<string, ProbeSignal>()
|
||||
for (const run of runs) {
|
||||
for (const p of run.probes ?? []) {
|
||||
const key = `${String(p.name ?? "").trim()} → ${String(p.target ?? "").trim()}`
|
||||
if (!key.trim() || out.has(key)) continue
|
||||
out.set(key, {
|
||||
key,
|
||||
status: p.status,
|
||||
rttMs: p.rttMs ?? null,
|
||||
lossPct: p.lossPct ?? null,
|
||||
sampledAt: run.sampledAt,
|
||||
})
|
||||
}
|
||||
}
|
||||
return [...out.values()]
|
||||
}
|
||||
|
||||
function collectTrafficSignalsFromRuns(runs: TrafficRunSnapshot[]): TrafficServerSignal[] {
|
||||
const out = new Map<string, TrafficServerSignal>()
|
||||
for (const run of runs) {
|
||||
for (const s of run.servers ?? []) {
|
||||
const name = String(s.name ?? "").trim()
|
||||
if (!name) continue
|
||||
const key = normalizeNameKey(name)
|
||||
if (out.has(key)) continue
|
||||
out.set(key, {
|
||||
serverName: name,
|
||||
rxMbps: Number(s.sumRxMbps ?? 0),
|
||||
txMbps: Number(s.sumTxMbps ?? 0),
|
||||
sampledAt: run.sampledAt,
|
||||
})
|
||||
}
|
||||
}
|
||||
return [...out.values()]
|
||||
}
|
||||
|
||||
function collectGreSignalsFromRuns(runs: GreBgpSnapshotRunSnapshot[]): GreTunnelSignal[] {
|
||||
const byTarget = new Map<string, GreTunnelSignal>()
|
||||
for (const run of runs) {
|
||||
const greRows = run.greTunnels ?? []
|
||||
for (const row of greRows) {
|
||||
const label = String(row.targetLabel ?? "").trim()
|
||||
if (!label) continue
|
||||
const prev = byTarget.get(label)
|
||||
if (!prev) {
|
||||
byTarget.set(label, {
|
||||
targetLabel: label,
|
||||
status: row.status,
|
||||
prevStatus: null,
|
||||
})
|
||||
continue
|
||||
}
|
||||
if (prev.prevStatus == null) prev.prevStatus = row.status
|
||||
}
|
||||
}
|
||||
return [...byTarget.values()]
|
||||
}
|
||||
|
||||
function collectBgpSignalsFromRuns(runs: GreBgpSnapshotRunSnapshot[]): BgpPeerSignal[] {
|
||||
const byKey = new Map<string, BgpPeerSignal>()
|
||||
for (const run of runs) {
|
||||
const peers = run.bgpPeers ?? []
|
||||
for (const row of peers) {
|
||||
const key = String(row.key ?? "").trim()
|
||||
if (!key) continue
|
||||
const prev = byKey.get(key)
|
||||
if (!prev) {
|
||||
byKey.set(key, { key, state: String(row.state ?? "").trim(), prevState: null })
|
||||
continue
|
||||
}
|
||||
if (prev.prevState == null) prev.prevState = String(row.state ?? "").trim()
|
||||
}
|
||||
}
|
||||
return [...byKey.values()]
|
||||
}
|
||||
|
||||
/** Собирает сигналы alert_engine напрямую из результатов collector jobs (`scheduler_runs.result_json`). */
|
||||
export function buildSignalSnapshotFromCollectors(): SignalSnapshot {
|
||||
const sampledAt = new Date().toISOString()
|
||||
const resourceRuns = readLatestOkRunSnapshots("uptime_resources", 16) as ResourcesRunSnapshot[]
|
||||
const restRuns = readLatestOkRunSnapshots("servers_rest_ping", 16) as ServersRestPingRunSnapshot[]
|
||||
const pingRuns = readLatestOkRunSnapshots("uptime_ping", 24) as PingRunSnapshot[]
|
||||
const trafficRuns = readLatestOkRunSnapshots("traffic", 8) as TrafficRunSnapshot[]
|
||||
const greBgpRuns = readLatestOkRunSnapshots("gre_bgp", 8) as GreBgpSnapshotRunSnapshot[]
|
||||
|
||||
const resourceByKey = collectServerSignalsFromResourceRuns(resourceRuns)
|
||||
const restByKey = collectServerSignalsFromRestRuns(restRuns)
|
||||
const serverKeys = new Set([...resourceByKey.keys(), ...restByKey.keys()])
|
||||
const servers: ServerSignal[] = []
|
||||
for (const key of serverKeys) {
|
||||
const resource = resourceByKey.get(key)
|
||||
const rest = restByKey.get(key)
|
||||
if (!rest) {
|
||||
if (resource) servers.push(resource)
|
||||
continue
|
||||
}
|
||||
if (!resource) {
|
||||
servers.push(rest)
|
||||
continue
|
||||
}
|
||||
servers.push(mergeResourceAndRestServer(resource, rest))
|
||||
}
|
||||
servers.sort((a, b) => a.name.localeCompare(b.name, undefined, { sensitivity: "base" }))
|
||||
|
||||
return {
|
||||
sampledAt,
|
||||
servers,
|
||||
probes: collectProbeSignalsFromRuns(pingRuns),
|
||||
trafficByServer: collectTrafficSignalsFromRuns(trafficRuns),
|
||||
greTunnels: collectGreSignalsFromRuns(greBgpRuns),
|
||||
bgpSessions: collectBgpSignalsFromRuns(greBgpRuns),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import { buildSignalSnapshotFromCollectors } from "./signals-from-collectors.js"
|
||||
import type { SignalSnapshot } from "./types.js"
|
||||
|
||||
/** Legacy-совместимость: единый источник сигналов — collector snapshots из scheduler_runs. */
|
||||
export function buildSignalSnapshot(): SignalSnapshot {
|
||||
return buildSignalSnapshotFromCollectors()
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import { desc, eq, inArray } from "drizzle-orm"
|
||||
import { db } from "../../db/index.js"
|
||||
import { alertEngineCursor, schedulerRuns } from "../../db/schema.js"
|
||||
|
||||
const SOURCE_JOBS = [
|
||||
"traffic",
|
||||
"servers_rest_ping",
|
||||
"uptime_resources",
|
||||
"uptime_ping",
|
||||
"uptime_speed",
|
||||
"gre_bgp",
|
||||
] as const
|
||||
|
||||
export function getLatestSourceFinishedAt(): string | null {
|
||||
const row = db
|
||||
.select({ finishedAt: schedulerRuns.finishedAt })
|
||||
.from(schedulerRuns)
|
||||
.where(inArray(schedulerRuns.jobKey, [...SOURCE_JOBS]))
|
||||
.orderBy(desc(schedulerRuns.finishedAt))
|
||||
.limit(1)
|
||||
.all()[0]
|
||||
return row?.finishedAt ?? null
|
||||
}
|
||||
|
||||
export function getSourceWatermark(): string | null {
|
||||
const row = db.select().from(alertEngineCursor).where(eq(alertEngineCursor.id, 1)).limit(1).all()[0]
|
||||
return row?.lastSourceFinishedAt ?? null
|
||||
}
|
||||
|
||||
export function updateSourceWatermark(lastSourceFinishedAt: string | null): void {
|
||||
const existing = db
|
||||
.select()
|
||||
.from(alertEngineCursor)
|
||||
.where(eq(alertEngineCursor.id, 1))
|
||||
.limit(1)
|
||||
.all()[0]
|
||||
if (existing) {
|
||||
db.update(alertEngineCursor)
|
||||
.set({
|
||||
lastSourceFinishedAt,
|
||||
updatedAt: new Date().toISOString(),
|
||||
})
|
||||
.where(eq(alertEngineCursor.id, 1))
|
||||
.run()
|
||||
return
|
||||
}
|
||||
db.insert(alertEngineCursor)
|
||||
.values({
|
||||
id: 1,
|
||||
lastSourceFinishedAt,
|
||||
updatedAt: new Date().toISOString(),
|
||||
})
|
||||
.run()
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { shouldAllowRecoveryByPolicy } from "./state-machine.js"
|
||||
|
||||
const recoveryHit = {
|
||||
ruleId: "r",
|
||||
ruleName: "r",
|
||||
severity: "warning" as const,
|
||||
message: "восстановился",
|
||||
transition: "recovery" as const,
|
||||
payloadHash: "h1",
|
||||
}
|
||||
|
||||
const problemHit = {
|
||||
...recoveryHit,
|
||||
transition: "problem" as const,
|
||||
}
|
||||
|
||||
assert.equal(shouldAllowRecoveryByPolicy(recoveryHit, "always"), true)
|
||||
assert.equal(shouldAllowRecoveryByPolicy(recoveryHit, "conditional"), true)
|
||||
assert.equal(shouldAllowRecoveryByPolicy(recoveryHit, "never"), false)
|
||||
assert.equal(shouldAllowRecoveryByPolicy(problemHit, "never"), true)
|
||||
|
||||
console.log("alert-engine state-machine tests ok")
|
||||
@@ -0,0 +1,16 @@
|
||||
import type { RuleEvalHit } from "./types.js"
|
||||
|
||||
export type RecoveryMode = "always" | "never" | "conditional"
|
||||
|
||||
export function isRecoveryTransition(hit: RuleEvalHit | null): boolean {
|
||||
return hit?.transition === "recovery"
|
||||
}
|
||||
|
||||
export function shouldAllowRecoveryByPolicy(
|
||||
hit: RuleEvalHit | null,
|
||||
policy: RecoveryMode,
|
||||
): boolean {
|
||||
if (!isRecoveryTransition(hit)) return true
|
||||
if (policy === "never") return false
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { isPositiveRecoveryTelegramText, pickTelegramAlertEmoji } from "./telegram-emoji.js"
|
||||
|
||||
assert.equal(pickTelegramAlertEmoji("Сервер: x — снова в сети: x — восстановился"), "✅")
|
||||
assert.equal(pickTelegramAlertEmoji("Сервер: x недоступен: x — перешёл в offline"), "🚨")
|
||||
assert.equal(isPositiveRecoveryTelegramText("A — восстановился"), true)
|
||||
assert.equal(isPositiveRecoveryTelegramText("A — перешёл в offline"), false)
|
||||
assert.equal(isPositiveRecoveryTelegramText("BGP: p — восстановил сессию"), true)
|
||||
|
||||
console.log("telegram-emoji tests ok")
|
||||
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* Эмодзи в Telegram: восстановление / «подключился» / BGP session up — не сирена.
|
||||
* Текст совпадает с форматом `evaluate-rule` (`… — условие`, несколько строк через \\n).
|
||||
*/
|
||||
function isRecoveryLine(line: string): boolean {
|
||||
const t = line.toLowerCase()
|
||||
return (
|
||||
/[—\-]\s*(восстанов|подключ)/i.test(t) ||
|
||||
/восстановил[аи]?\s+сессию/i.test(t) ||
|
||||
/снова\s+в\s+сети/i.test(t)
|
||||
)
|
||||
}
|
||||
|
||||
/** «Хорошие» переходы: не слать дубль при том же payloadHash (см. `run-once`). */
|
||||
export function isPositiveRecoveryTelegramText(fullText: string): boolean {
|
||||
const lines = fullText
|
||||
.split(/\n+/)
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean)
|
||||
if (lines.length === 0) return false
|
||||
if (lines.length === 1) return isRecoveryLine(lines[0]!)
|
||||
return lines.every(isRecoveryLine)
|
||||
}
|
||||
|
||||
export function pickTelegramAlertEmoji(fullText: string): string {
|
||||
return isPositiveRecoveryTelegramText(fullText) ? "✅" : "🚨"
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import type { ApiAlertRule } from "../alerts-service.js"
|
||||
|
||||
export type ServerSignal = {
|
||||
name: string
|
||||
status: "online" | "offline" | "degraded"
|
||||
/** Предпоследний сэмпл (для условий «восстановился» / «подключился») */
|
||||
prevStatus: "online" | "offline" | "degraded" | null
|
||||
/** Третий с конца сэмпл — чтобы «восстановился» не висел истинным до второго online-сэмпла в БД. */
|
||||
prev2Status: "online" | "offline" | "degraded" | null
|
||||
sampledAt: string
|
||||
}
|
||||
|
||||
export type ProbeSignal = {
|
||||
/** Как в UI: `имя пробы → target` */
|
||||
key: string
|
||||
status: "up" | "warn" | "down"
|
||||
rttMs: number | null
|
||||
lossPct: number | null
|
||||
sampledAt: string
|
||||
}
|
||||
|
||||
export type TrafficServerSignal = {
|
||||
serverName: string
|
||||
rxMbps: number
|
||||
txMbps: number
|
||||
sampledAt: string
|
||||
}
|
||||
|
||||
export type GreTunnelSignal = {
|
||||
/** Как в UI: `tunnelName / serverName` */
|
||||
targetLabel: string
|
||||
status: "up" | "down" | "degraded"
|
||||
/** Состояние на прошлом тике движка (из БД), для «восстановился». */
|
||||
prevStatus: "up" | "down" | "degraded" | null
|
||||
}
|
||||
|
||||
export type BgpPeerSignal = {
|
||||
/** Как в UI: `serverName: peerName` */
|
||||
key: string
|
||||
state: string
|
||||
/** Состояние на прошлом тике (строка state с роутера). */
|
||||
prevState: string | null
|
||||
}
|
||||
|
||||
export type SignalSnapshot = {
|
||||
sampledAt: string
|
||||
servers: ServerSignal[]
|
||||
probes: ProbeSignal[]
|
||||
trafficByServer: TrafficServerSignal[]
|
||||
greTunnels: GreTunnelSignal[]
|
||||
bgpSessions: BgpPeerSignal[]
|
||||
}
|
||||
|
||||
export type RuleEvalHit = {
|
||||
ruleId: string
|
||||
ruleName: string
|
||||
severity: ApiAlertRule["severity"]
|
||||
message: string
|
||||
/** Тип перехода для диагностики и логики задержки отправки. */
|
||||
transition: "problem" | "recovery" | "neutral"
|
||||
/** Для дедупа */
|
||||
payloadHash: string
|
||||
}
|
||||
|
||||
import type { AlertEngineRuleDiagSnapshot } from "../../types/scheduler-run-snapshot.js"
|
||||
|
||||
export type AlertEngineRuleDiag = AlertEngineRuleDiagSnapshot
|
||||
|
||||
export type AlertEngineRunResult = {
|
||||
sampledAt: string
|
||||
rulesChecked: number
|
||||
standaloneFires: number
|
||||
groupFires: number
|
||||
skippedNoTelegram: boolean
|
||||
errors: string[]
|
||||
ruleDiag: AlertEngineRuleDiag[]
|
||||
}
|
||||
@@ -0,0 +1,593 @@
|
||||
import { asc, desc, eq, sql } from "drizzle-orm"
|
||||
import { db, sqliteDatabase } from "../db/index.js"
|
||||
import {
|
||||
alertGroups,
|
||||
alertHistory,
|
||||
alertRuleConditions,
|
||||
alertRuleTargets,
|
||||
alertRules,
|
||||
alertTelegramSettings,
|
||||
servers,
|
||||
uptimeProbes,
|
||||
} from "../db/schema.js"
|
||||
|
||||
export type ApiAlertGroup = {
|
||||
id: string
|
||||
name: string
|
||||
combineMode: "any" | "all"
|
||||
enabled: boolean
|
||||
cooldownOverride: (typeof alertRules.$inferSelect)["cooldown"] | null
|
||||
}
|
||||
|
||||
export type ApiAlertRule = {
|
||||
id: string
|
||||
name: string
|
||||
type: (typeof alertRules.$inferSelect)["type"]
|
||||
/** Сводная строка для списка / поиска */
|
||||
target: string
|
||||
/** Все объекты правила (OR) */
|
||||
targets: string[]
|
||||
groupId: string | null
|
||||
/** Сводная строка условий для списка */
|
||||
condition: string
|
||||
/** Все условия (OR); если пусто в БД — используется `condition` */
|
||||
conditions: string[]
|
||||
severity: (typeof alertRules.$inferSelect)["severity"]
|
||||
enabled: boolean
|
||||
cooldown: (typeof alertRules.$inferSelect)["cooldown"]
|
||||
/** Секунды стабильности перед отправкой; null — выключено */
|
||||
confirmStabilitySec: number | null
|
||||
/** Политика отправки recovery для этого правила. */
|
||||
recoveryMode: "always" | "never" | "conditional"
|
||||
/** Доп. задержка подтверждения recovery при `conditional`. */
|
||||
recoveryStabilitySec: number | null
|
||||
chatId: string
|
||||
lastFiredAt: string | null
|
||||
}
|
||||
|
||||
export type ApiAlertHistoryEntry = {
|
||||
id: string
|
||||
ruleName: string
|
||||
severity: (typeof alertHistory.$inferSelect)["severity"]
|
||||
message: string
|
||||
sent: boolean
|
||||
firedAt: string
|
||||
source: "db" | "live"
|
||||
}
|
||||
|
||||
function ensureTelegramRow() {
|
||||
let row = db.select().from(alertTelegramSettings).where(eq(alertTelegramSettings.id, 1)).limit(1).all()[0]
|
||||
if (!row) {
|
||||
db.insert(alertTelegramSettings).values({ id: 1 }).run()
|
||||
row = db.select().from(alertTelegramSettings).where(eq(alertTelegramSettings.id, 1)).limit(1).all()[0]
|
||||
}
|
||||
return row!
|
||||
}
|
||||
|
||||
function lastFiredByRuleId(): Map<string, string> {
|
||||
const rows = sqliteDatabase
|
||||
.prepare(
|
||||
`SELECT rule_id AS ruleId, MAX(fired_at) AS mx
|
||||
FROM alert_history
|
||||
WHERE rule_id IS NOT NULL AND rule_id != ''
|
||||
GROUP BY rule_id`,
|
||||
)
|
||||
.all() as { ruleId: string; mx: string }[]
|
||||
const m = new Map<string, string>()
|
||||
for (const r of rows) {
|
||||
if (r.ruleId && r.mx) m.set(r.ruleId, r.mx)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
function targetsByRuleId(): Map<string, string[]> {
|
||||
const rows = db
|
||||
.select()
|
||||
.from(alertRuleTargets)
|
||||
.orderBy(asc(alertRuleTargets.sortIndex), asc(alertRuleTargets.id))
|
||||
.all()
|
||||
const m = new Map<string, string[]>()
|
||||
for (const t of rows) {
|
||||
const rid = t.ruleId
|
||||
const arr = m.get(rid) ?? []
|
||||
arr.push(t.target)
|
||||
m.set(rid, arr)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
function conditionsByRuleId(): Map<string, string[]> {
|
||||
const rows = db
|
||||
.select()
|
||||
.from(alertRuleConditions)
|
||||
.orderBy(asc(alertRuleConditions.sortIndex), asc(alertRuleConditions.id))
|
||||
.all()
|
||||
const m = new Map<string, string[]>()
|
||||
for (const t of rows) {
|
||||
const rid = t.ruleId
|
||||
const arr = m.get(rid) ?? []
|
||||
arr.push(t.conditionLine)
|
||||
m.set(rid, arr)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
export function listAlertGroups(): ApiAlertGroup[] {
|
||||
return db
|
||||
.select()
|
||||
.from(alertGroups)
|
||||
.orderBy(asc(alertGroups.name))
|
||||
.all()
|
||||
.map((g) => ({
|
||||
id: g.id,
|
||||
name: g.name,
|
||||
combineMode: g.combineMode,
|
||||
enabled: g.enabled,
|
||||
cooldownOverride: g.cooldownOverride ?? null,
|
||||
}))
|
||||
}
|
||||
|
||||
export function listAlertRules(): ApiAlertRule[] {
|
||||
const lf = lastFiredByRuleId()
|
||||
const tgtMap = targetsByRuleId()
|
||||
const condMap = conditionsByRuleId()
|
||||
const rows = db.select().from(alertRules).all()
|
||||
return rows.map((r) => {
|
||||
const targets = tgtMap.get(r.id)
|
||||
const ts = targets?.length ? targets : [r.target]
|
||||
const condLines = condMap.get(r.id)
|
||||
const cs = condLines?.length ? condLines : [r.condition]
|
||||
return {
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
type: r.type,
|
||||
target: r.target,
|
||||
targets: ts,
|
||||
groupId: r.groupId ?? null,
|
||||
condition: r.condition,
|
||||
conditions: cs,
|
||||
severity: r.severity,
|
||||
enabled: r.enabled,
|
||||
cooldown: r.cooldown,
|
||||
confirmStabilitySec:
|
||||
r.confirmStabilitySec != null && r.confirmStabilitySec > 0 ? r.confirmStabilitySec : null,
|
||||
recoveryMode:
|
||||
r.recoveryMode === "never" || r.recoveryMode === "conditional" ? r.recoveryMode : "always",
|
||||
recoveryStabilitySec:
|
||||
r.recoveryStabilitySec != null && r.recoveryStabilitySec > 0 ? r.recoveryStabilitySec : null,
|
||||
chatId: r.chatId ?? "",
|
||||
lastFiredAt: lf.get(r.id) ?? null,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function summarizeTargets(targets: string[]): string {
|
||||
const u = targets.map((t) => t.trim()).filter(Boolean)
|
||||
if (u.length === 0) return ""
|
||||
if (u.length <= 2) return u.join(", ")
|
||||
return `${u[0]}, ${u[1]} +${u.length - 2}`
|
||||
}
|
||||
|
||||
function summarizeConditionLines(lines: string[]): string {
|
||||
const u = lines.map((t) => t.trim()).filter(Boolean)
|
||||
if (u.length === 0) return "—"
|
||||
if (u.length === 1) return u[0]!
|
||||
if (u.length <= 3) return u.join(" · ")
|
||||
return `${u[0]} · ${u[1]} · +${u.length - 2}`
|
||||
}
|
||||
|
||||
export type ReplaceAlertRuleInput = {
|
||||
id: string
|
||||
name: string
|
||||
type: ApiAlertRule["type"]
|
||||
/** @deprecated используйте targets; если нет targets — берётся target */
|
||||
target?: string
|
||||
targets?: string[]
|
||||
/** Сводная строка; если не передана — из conditions */
|
||||
condition?: string
|
||||
/** Несколько условий (OR); если пусто — используется `condition` */
|
||||
conditions?: string[]
|
||||
severity: ApiAlertRule["severity"]
|
||||
enabled: boolean
|
||||
cooldown: ApiAlertRule["cooldown"]
|
||||
/** Секунды; 0 / undefined / null — без задержки подтверждения */
|
||||
confirmStabilitySec?: number | null
|
||||
recoveryMode?: "always" | "never" | "conditional"
|
||||
recoveryStabilitySec?: number | null
|
||||
chatId: string
|
||||
groupId?: string | null
|
||||
}
|
||||
|
||||
export type ReplaceAlertGroupInput = {
|
||||
id: string
|
||||
name: string
|
||||
combineMode: "any" | "all"
|
||||
enabled: boolean
|
||||
cooldownOverride: ApiAlertRule["cooldown"] | null
|
||||
}
|
||||
|
||||
/** Атомарно заменяет группы, правила и строки targets. */
|
||||
export function replaceAlertsConfig(payload: { groups: ReplaceAlertGroupInput[]; rules: ReplaceAlertRuleInput[] }) {
|
||||
const now = sql`(datetime('now'))`
|
||||
const { groups, rules } = payload
|
||||
db.transaction((tx) => {
|
||||
tx.delete(alertRuleConditions).run()
|
||||
tx.delete(alertRuleTargets).run()
|
||||
tx.delete(alertRules).run()
|
||||
tx.delete(alertGroups).run()
|
||||
for (const g of groups) {
|
||||
tx.insert(alertGroups).values({
|
||||
id: g.id,
|
||||
name: g.name.trim() || g.id,
|
||||
combineMode: g.combineMode,
|
||||
enabled: g.enabled,
|
||||
cooldownOverride: g.cooldownOverride ?? null,
|
||||
updatedAt: now,
|
||||
}).run()
|
||||
}
|
||||
for (const r of rules) {
|
||||
const rawTargets =
|
||||
r.targets && r.targets.length > 0
|
||||
? r.targets.map((t) => t.trim()).filter(Boolean)
|
||||
: [String(r.target ?? "").trim()].filter(Boolean)
|
||||
const summary = summarizeTargets(rawTargets.length ? rawTargets : ["—"])
|
||||
const rawConds =
|
||||
r.conditions && r.conditions.length > 0
|
||||
? r.conditions.map((c) => c.trim()).filter(Boolean)
|
||||
: [String(r.condition ?? "").trim()].filter(Boolean)
|
||||
const condSummary = summarizeConditionLines(rawConds.length ? rawConds : ["—"])
|
||||
const stab =
|
||||
r.confirmStabilitySec != null &&
|
||||
Number.isFinite(r.confirmStabilitySec) &&
|
||||
r.confirmStabilitySec > 0
|
||||
? Math.min(86400, Math.max(1, Math.floor(r.confirmStabilitySec)))
|
||||
: null
|
||||
const recoveryMode =
|
||||
r.recoveryMode === "never" || r.recoveryMode === "conditional" ? r.recoveryMode : "always"
|
||||
const recoveryStabilitySec =
|
||||
r.recoveryStabilitySec != null &&
|
||||
Number.isFinite(r.recoveryStabilitySec) &&
|
||||
r.recoveryStabilitySec > 0
|
||||
? Math.min(86400, Math.max(1, Math.floor(r.recoveryStabilitySec)))
|
||||
: null
|
||||
tx.insert(alertRules).values({
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
type: r.type,
|
||||
target: summary || "—",
|
||||
condition: condSummary,
|
||||
severity: r.severity,
|
||||
enabled: r.enabled,
|
||||
cooldown: r.cooldown,
|
||||
confirmStabilitySec: stab,
|
||||
recoveryMode,
|
||||
recoveryStabilitySec,
|
||||
chatId: r.chatId ?? "",
|
||||
groupId: r.groupId && r.groupId.trim() ? r.groupId.trim() : null,
|
||||
updatedAt: now,
|
||||
}).run()
|
||||
rawTargets.forEach((t, i) => {
|
||||
tx.insert(alertRuleTargets).values({
|
||||
id: `rt-${r.id}-${i}`,
|
||||
ruleId: r.id,
|
||||
target: t,
|
||||
sortIndex: i,
|
||||
}).run()
|
||||
})
|
||||
rawConds.forEach((c, i) => {
|
||||
tx.insert(alertRuleConditions).values({
|
||||
id: `rc-${r.id}-${i}`,
|
||||
ruleId: r.id,
|
||||
conditionLine: c,
|
||||
sortIndex: i,
|
||||
}).run()
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/** Совместимость: только правила, группы не трогаем. */
|
||||
export function replaceAlertRules(rules: ReplaceAlertRuleInput[]) {
|
||||
const curGroups = listAlertGroups()
|
||||
replaceAlertsConfig({ groups: curGroups.map((g) => ({ ...g })), rules })
|
||||
}
|
||||
|
||||
export function getTelegramSettingsRow() {
|
||||
return ensureTelegramRow()
|
||||
}
|
||||
|
||||
export function getTelegramPublic() {
|
||||
const row = ensureTelegramRow()
|
||||
const tid = row.messageThreadId
|
||||
const messageThreadId =
|
||||
typeof tid === "number" && Number.isFinite(tid) && Math.floor(tid) >= 1 ? Math.floor(tid) : null
|
||||
return {
|
||||
chatId: row.chatId ?? "",
|
||||
tokenConfigured: Boolean(row.botToken?.trim()),
|
||||
messageThreadId,
|
||||
}
|
||||
}
|
||||
|
||||
export function updateTelegramSettings(patch: {
|
||||
token?: string | null
|
||||
chatId?: string | undefined
|
||||
/** undefined — не менять; null — сбросить тему */
|
||||
messageThreadId?: number | null
|
||||
}) {
|
||||
const cur = ensureTelegramRow()
|
||||
let nextToken = cur.botToken
|
||||
let nextChat = cur.chatId ?? ""
|
||||
let nextThread: number | null | undefined = undefined
|
||||
|
||||
if (patch.token !== undefined) {
|
||||
if (patch.token === null || patch.token === "") nextToken = ""
|
||||
else nextToken = patch.token.trim()
|
||||
}
|
||||
if (patch.chatId !== undefined) nextChat = patch.chatId.trim()
|
||||
if (patch.messageThreadId !== undefined) {
|
||||
if (patch.messageThreadId === null) nextThread = null
|
||||
else {
|
||||
const n = Math.floor(patch.messageThreadId)
|
||||
nextThread = Number.isFinite(n) && n >= 1 ? n : null
|
||||
}
|
||||
}
|
||||
|
||||
const base = {
|
||||
botToken: nextToken,
|
||||
chatId: nextChat,
|
||||
updatedAt: sql`(datetime('now'))`,
|
||||
}
|
||||
if (nextThread !== undefined) {
|
||||
db.update(alertTelegramSettings)
|
||||
.set({ ...base, messageThreadId: nextThread })
|
||||
.where(eq(alertTelegramSettings.id, 1))
|
||||
.run()
|
||||
} else {
|
||||
db.update(alertTelegramSettings).set(base).where(eq(alertTelegramSettings.id, 1)).run()
|
||||
}
|
||||
return getTelegramPublic()
|
||||
}
|
||||
|
||||
/** Токен только для внутреннего вызова Telegram API (не отдаётся клиенту). */
|
||||
export function getTelegramBotToken(): string {
|
||||
return ensureTelegramRow().botToken?.trim() ?? ""
|
||||
}
|
||||
|
||||
/** Для `sendMessage`: только если в БД задана валидная тема. */
|
||||
export function getTelegramMessageThreadIdForApi(): number | undefined {
|
||||
const { messageThreadId } = getTelegramPublic()
|
||||
return messageThreadId ?? undefined
|
||||
}
|
||||
|
||||
/** Текст тестового сообщения по полям правила (как в UI предпросмотра, без записи в историю). */
|
||||
export function formatAlertRuleTestTelegramText(opts: {
|
||||
name: string
|
||||
targets: string[]
|
||||
conditionLine: string
|
||||
severity: "critical" | "warning" | "info"
|
||||
cooldown: string
|
||||
}): string {
|
||||
const emoji =
|
||||
opts.severity === "critical" ? "🔴" : opts.severity === "warning" ? "🟡" : "🔵"
|
||||
const obj = opts.targets.length ? opts.targets.join(", ") : "—"
|
||||
return [
|
||||
"🧪 MikroTik Manager — тестовая отправка по полям формы правила (в историю срабатываний не записывается).",
|
||||
"",
|
||||
`${emoji} ${opts.name}`,
|
||||
`Объект: ${obj}`,
|
||||
`Событие: ${opts.conditionLine}`,
|
||||
`Cooldown: ${opts.cooldown}`,
|
||||
].join("\n")
|
||||
}
|
||||
|
||||
/** Отправка текста в Telegram (движок алертов и тест). */
|
||||
export async function sendTelegramAlertMessage(opts: {
|
||||
text: string
|
||||
/** Если задан — вместо токена из БД (ручной тест) */
|
||||
token?: string
|
||||
/** Пусто — глобальный chat из БД */
|
||||
chatId?: string
|
||||
/** Переопределение темы; undefined — как в настройках */
|
||||
messageThreadId?: number | null
|
||||
}): Promise<{ ok: true } | { ok: false; error: string }> {
|
||||
const token = (opts.token?.trim() || getTelegramBotToken()).trim()
|
||||
if (!token) return { ok: false, error: "Не задан Bot Token" }
|
||||
const pub = getTelegramPublic()
|
||||
const chatId = (opts.chatId?.trim() || pub.chatId || "").trim()
|
||||
if (!chatId) return { ok: false, error: "Не задан Chat ID" }
|
||||
const threadId =
|
||||
opts.messageThreadId !== undefined && opts.messageThreadId !== null
|
||||
? opts.messageThreadId >= 1
|
||||
? Math.floor(opts.messageThreadId)
|
||||
: undefined
|
||||
: getTelegramMessageThreadIdForApi()
|
||||
const url = `https://api.telegram.org/bot${encodeURIComponent(token)}/sendMessage`
|
||||
try {
|
||||
const payload: { chat_id: string; text: string; message_thread_id?: number } = {
|
||||
chat_id: chatId,
|
||||
text: opts.text,
|
||||
}
|
||||
if (threadId != null) payload.message_thread_id = threadId
|
||||
const res = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
})
|
||||
const data = (await res.json().catch(() => ({}))) as { ok?: boolean; description?: string }
|
||||
if (!res.ok || data.ok === false) {
|
||||
return { ok: false, error: data.description ?? res.statusText ?? "Telegram API error" }
|
||||
}
|
||||
return { ok: true }
|
||||
} catch (e) {
|
||||
return { ok: false, error: e instanceof Error ? e.message : "Ошибка сети" }
|
||||
}
|
||||
}
|
||||
|
||||
export function appendAlertHistory(entry: {
|
||||
id: string
|
||||
ruleId?: string | null
|
||||
groupId?: string | null
|
||||
ruleName: string
|
||||
severity: (typeof alertHistory.$inferSelect)["severity"]
|
||||
message: string
|
||||
sentOk: boolean
|
||||
firedAt?: string
|
||||
}) {
|
||||
const firedAt = entry.firedAt ?? new Date().toISOString()
|
||||
db.insert(alertHistory)
|
||||
.values({
|
||||
id: entry.id,
|
||||
ruleId: entry.ruleId ?? null,
|
||||
groupId: entry.groupId ?? null,
|
||||
ruleName: entry.ruleName,
|
||||
severity: entry.severity,
|
||||
message: entry.message,
|
||||
sentOk: entry.sentOk,
|
||||
firedAt,
|
||||
})
|
||||
.run()
|
||||
}
|
||||
|
||||
interface LiveResRow {
|
||||
serverName: string
|
||||
sampledAt: string
|
||||
}
|
||||
|
||||
interface LiveProbeRow {
|
||||
probeName: string
|
||||
target: string
|
||||
sampledAt: string
|
||||
status: string
|
||||
rttMs: number | null
|
||||
lossPct: number | null
|
||||
}
|
||||
|
||||
function loadLiveResourceIssues(): LiveResRow[] {
|
||||
return sqliteDatabase
|
||||
.prepare(
|
||||
`SELECT s.name AS serverName, urs.sampled_at AS sampledAt
|
||||
FROM uptime_resource_samples urs
|
||||
INNER JOIN (
|
||||
SELECT server_id, MAX(id) AS mid FROM uptime_resource_samples GROUP BY server_id
|
||||
) latest ON latest.mid = urs.id
|
||||
INNER JOIN servers s ON s.id = urs.server_id
|
||||
WHERE urs.status = 'offline'`,
|
||||
)
|
||||
.all() as LiveResRow[]
|
||||
}
|
||||
|
||||
function loadLiveProbeIssues(): LiveProbeRow[] {
|
||||
return sqliteDatabase
|
||||
.prepare(
|
||||
`SELECT p.name AS probeName, p.target AS target, ups.sampled_at AS sampledAt,
|
||||
ups.status AS status, ups.rtt_ms AS rttMs, ups.loss_pct AS lossPct
|
||||
FROM uptime_probe_samples ups
|
||||
INNER JOIN (
|
||||
SELECT probe_id, MAX(id) AS mid FROM uptime_probe_samples GROUP BY probe_id
|
||||
) latest ON latest.mid = ups.id
|
||||
INNER JOIN uptime_probes p ON p.id = ups.probe_id
|
||||
WHERE ups.status IN ('down', 'warn')`,
|
||||
)
|
||||
.all() as LiveProbeRow[]
|
||||
}
|
||||
|
||||
function toLiveHistoryEntries(): ApiAlertHistoryEntry[] {
|
||||
const out: ApiAlertHistoryEntry[] = []
|
||||
for (const r of loadLiveResourceIssues()) {
|
||||
const safeId = `live-res-${r.serverName}-${r.sampledAt}`.replace(/[^a-zA-Z0-9._-]+/g, "_")
|
||||
out.push({
|
||||
id: safeId,
|
||||
ruleName: "Сервер (ресурсы)",
|
||||
severity: "critical",
|
||||
message: `${r.serverName}: статус offline по данным мониторинга`,
|
||||
sent: true,
|
||||
firedAt: r.sampledAt,
|
||||
source: "live",
|
||||
})
|
||||
}
|
||||
for (const p of loadLiveProbeIssues()) {
|
||||
const sev = p.status === "down" ? "critical" : "warning"
|
||||
const loss = p.lossPct != null ? `${p.lossPct}%` : "—"
|
||||
const rtt = p.rttMs != null ? `${p.rttMs} мс` : "—"
|
||||
const safeId = `live-prb-${p.probeName}-${p.target}-${p.sampledAt}`.replace(/[^a-zA-Z0-9._-]+/g, "_")
|
||||
out.push({
|
||||
id: safeId,
|
||||
ruleName: "Ping-проба",
|
||||
severity: sev,
|
||||
message: `${p.probeName} → ${p.target}: ${p.status} (RTT ${rtt}, потери ${loss})`,
|
||||
sent: true,
|
||||
firedAt: p.sampledAt,
|
||||
source: "live",
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
export function listPersistedHistory(limit: number): ApiAlertHistoryEntry[] {
|
||||
const rows = db
|
||||
.select()
|
||||
.from(alertHistory)
|
||||
.orderBy(desc(alertHistory.firedAt))
|
||||
.limit(limit)
|
||||
.all()
|
||||
return rows.map((h) => ({
|
||||
id: h.id,
|
||||
ruleName: h.ruleName,
|
||||
severity: h.severity,
|
||||
message: h.message,
|
||||
sent: h.sentOk,
|
||||
firedAt: h.firedAt,
|
||||
source: "db" as const,
|
||||
}))
|
||||
}
|
||||
|
||||
export function listMergedHistory(persistLimit = 40, liveCap = 25): ApiAlertHistoryEntry[] {
|
||||
const persisted = listPersistedHistory(persistLimit)
|
||||
const live = toLiveHistoryEntries().slice(0, liveCap)
|
||||
const merged = [...persisted, ...live].sort((a, b) => (a.firedAt < b.firedAt ? 1 : a.firedAt > b.firedAt ? -1 : 0))
|
||||
const seen = new Set<string>()
|
||||
const dedup: ApiAlertHistoryEntry[] = []
|
||||
for (const e of merged) {
|
||||
if (seen.has(e.id)) continue
|
||||
seen.add(e.id)
|
||||
dedup.push(e)
|
||||
if (dedup.length >= 50) break
|
||||
}
|
||||
return dedup
|
||||
}
|
||||
|
||||
export function getAlertsMeta() {
|
||||
const srvRows = db
|
||||
.select({
|
||||
name: servers.name,
|
||||
host: servers.host,
|
||||
site: servers.site,
|
||||
country: servers.country,
|
||||
type: servers.type,
|
||||
})
|
||||
.from(servers)
|
||||
.all()
|
||||
const labels = srvRows.map((s) => (s.name?.trim() ? s.name.trim() : s.host))
|
||||
const serversDetail = srvRows.map((r) => ({
|
||||
name: r.name?.trim() || "",
|
||||
host: r.host?.trim() || "",
|
||||
site: r.site?.trim() || "",
|
||||
country: r.country?.trim() || "",
|
||||
type: (r.type ?? "home-router") as string,
|
||||
}))
|
||||
const probes = db.select({ name: uptimeProbes.name, target: uptimeProbes.target }).from(uptimeProbes).all()
|
||||
const probeTargets = probes.map((p) => `${p.name} → ${p.target}`)
|
||||
const defaults = ["8.8.8.8", "1.1.1.1"]
|
||||
const rttLossTargets: string[] = []
|
||||
for (const L of labels.slice(0, 12)) {
|
||||
for (const d of defaults) rttLossTargets.push(`${L} → ${d}`)
|
||||
}
|
||||
return {
|
||||
servers: labels,
|
||||
greClients: labels,
|
||||
probeTargets: probeTargets.length ? probeTargets : rttLossTargets,
|
||||
rttLossTargets: rttLossTargets.length ? rttLossTargets : labels.map((L) => `${L} → 8.8.8.8`),
|
||||
trafficServers: labels,
|
||||
serversDetail,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
import { servers } from "../db/schema.js"
|
||||
import type { BgpSessionRead, RosBgpSession } from "../types/server.js"
|
||||
|
||||
type ServerRow = typeof servers.$inferSelect
|
||||
|
||||
/** Parse RouterOS duration like "1m30s", "1d5h13m23s580ms", "30s" → seconds */
|
||||
function parseDuration(s: string | undefined): number {
|
||||
if (!s) return 0
|
||||
let total = 0
|
||||
const matches = s.matchAll(/(\d+)(w|d|h|m(?!s)|s|ms)/g)
|
||||
for (const m of matches) {
|
||||
const n = parseInt(m[1])
|
||||
switch (m[2]) {
|
||||
case "w":
|
||||
total += n * 604800
|
||||
break
|
||||
case "d":
|
||||
total += n * 86400
|
||||
break
|
||||
case "h":
|
||||
total += n * 3600
|
||||
break
|
||||
case "m":
|
||||
total += n * 60
|
||||
break
|
||||
case "s":
|
||||
total += n
|
||||
break
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
function parseCapabilities(capStr: string | undefined): string[] {
|
||||
if (!capStr) return []
|
||||
const codeMap: Record<string, string> = {
|
||||
mp: "MP-BGP",
|
||||
rr: "Route Refresh",
|
||||
gr: "Graceful Restart",
|
||||
as4: "4-byte-AS",
|
||||
enhe: "Extended Next-Hop",
|
||||
role: "BGP Role",
|
||||
err: "Extended Route Refresh",
|
||||
llgr: "Long-Lived GR",
|
||||
"add-path": "ADD-PATH",
|
||||
}
|
||||
return capStr
|
||||
.split(",")
|
||||
.map((c) => c.trim().toLowerCase())
|
||||
.filter(Boolean)
|
||||
.map((c) => codeMap[c] ?? c)
|
||||
.filter((v, i, a) => a.indexOf(v) === i)
|
||||
}
|
||||
|
||||
function detectState(s: RosBgpSession): string {
|
||||
if (s["established"] === "true") return "Established"
|
||||
|
||||
if (s["state"]) {
|
||||
const map: Record<string, string> = {
|
||||
established: "Established",
|
||||
active: "Active",
|
||||
idle: "Idle",
|
||||
connect: "Connect",
|
||||
opensent: "OpenSent",
|
||||
openconfirm: "OpenConfirm",
|
||||
"open sent": "OpenSent",
|
||||
"open confirm": "OpenConfirm",
|
||||
}
|
||||
const key = s["state"].trim().toLowerCase()
|
||||
if (map[key]) return map[key]
|
||||
for (const [k, v] of Object.entries(map)) {
|
||||
if (key.startsWith(k)) return v
|
||||
}
|
||||
return key.charAt(0).toUpperCase() + key.slice(1)
|
||||
}
|
||||
|
||||
if (s["uptime"]) return "Active"
|
||||
|
||||
return "Idle"
|
||||
}
|
||||
|
||||
/** Парсинг BGP-сессий с роутера — общая логика для HTTP-маршрута и alert-engine. */
|
||||
export function parseBgpSessions(server: ServerRow, raw: RosBgpSession[]): BgpSessionRead[] {
|
||||
return raw.map((s, idx) => {
|
||||
const remoteAs = parseInt(s["remote.as"] ?? "0") || 0
|
||||
const localAs = parseInt(s["local.as"] ?? "0") || 0
|
||||
const type: "eBGP" | "iBGP" = localAs > 0 && localAs === remoteAs ? "iBGP" : "eBGP"
|
||||
|
||||
const localCaps = parseCapabilities(s["local.capabilities"])
|
||||
const remoteCaps = parseCapabilities(s["remote.capabilities"])
|
||||
const legacyCaps: string[] = []
|
||||
if (s["4-octet-as-capability"] === "true") legacyCaps.push("4-byte-AS")
|
||||
if (s["as4-capability"] === "true") legacyCaps.push("4-byte-AS")
|
||||
if (s["refresh-capability"] === "true") legacyCaps.push("Route Refresh")
|
||||
if (s["add-path-capability"] === "true") legacyCaps.push("ADD-PATH")
|
||||
if (s["graceful-restart-capability"] === "true") legacyCaps.push("Graceful Restart")
|
||||
if (s["extended-message-capability"] === "true") legacyCaps.push("Extended Messages")
|
||||
const caps = [...new Set([...localCaps, ...remoteCaps, ...legacyCaps])]
|
||||
|
||||
const holdTime = parseDuration(s["hold-time"]) || parseInt(s["active-holdtime"] ?? "0") || 90
|
||||
|
||||
const keepalive = parseDuration(s["keepalive-time"]) || 30
|
||||
|
||||
const inputMessages =
|
||||
parseInt(s["remote.messages"] ?? s["total-messages-received"] ?? "0") || 0
|
||||
const outputMessages = parseInt(s["local.messages"] ?? s["total-messages-sent"] ?? "0") || 0
|
||||
|
||||
const peerIp = (s["remote.address"] ?? "").replace(/\/\d+$/, "")
|
||||
|
||||
return {
|
||||
id: s[".id"] ?? String(idx),
|
||||
serverId: server.id,
|
||||
serverName: server.name || server.host,
|
||||
serverSite: server.site,
|
||||
serverCountry: server.country,
|
||||
name: s["name"] ?? peerIp,
|
||||
peerIp,
|
||||
remoteAs,
|
||||
localAs,
|
||||
localId: s["local.id"] ?? "",
|
||||
remoteId: s["remote.id"] ?? "",
|
||||
state: detectState(s),
|
||||
type,
|
||||
uptime: s["uptime"] || null,
|
||||
holdTime,
|
||||
keepalive,
|
||||
prefixesRx: parseInt(s["prefix-count"] ?? s["total-updates-received"] ?? "0") || 0,
|
||||
prefixesTx: parseInt(s["total-updates-sent"] ?? "0") || 0,
|
||||
inputMessages,
|
||||
outputMessages,
|
||||
capabilities: caps,
|
||||
lastError: s["last-notification"] || null,
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { eq } from "drizzle-orm"
|
||||
import { db } from "../db/index.js"
|
||||
import { servers } from "../db/schema.js"
|
||||
import type { BgpSessionRead } from "../types/server.js"
|
||||
import { parseBgpSessions } from "./bgp-parse-sessions.js"
|
||||
import { MikrotikClient } from "./mikrotik.js"
|
||||
|
||||
/** Как в UI `/alerts`: `serverName: peerName`. */
|
||||
export function bgpPeerAlertKey(s: BgpSessionRead): string {
|
||||
const sn = String(s.serverName ?? "").trim()
|
||||
const nm = String(s.name ?? "").trim()
|
||||
return `${sn}: ${nm}`.trim()
|
||||
}
|
||||
|
||||
/** Агрегат BGP-сессий с включённых серверов — как `/api/bgp/sessions`. */
|
||||
export async function fetchBgpSessionsForAlerts(): Promise<BgpSessionRead[]> {
|
||||
const allServers = db.select().from(servers).where(eq(servers.enabled, true)).all()
|
||||
const results = await Promise.all(
|
||||
allServers.map(async (server) => {
|
||||
try {
|
||||
const client = MikrotikClient.fromServer(server)
|
||||
const raw = await client.getBgpSessions()
|
||||
return parseBgpSessions(server, raw)
|
||||
} catch {
|
||||
return [] as BgpSessionRead[]
|
||||
}
|
||||
}),
|
||||
)
|
||||
return results.flat()
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
/** Серийный доступ к bandwidth-test по node id (пара узлов не пересекаются). */
|
||||
const btestNodeLocks = new Map<number, Promise<void>>()
|
||||
|
||||
export async function withBtestNodeLocks<T>(nodeIds: number[], fn: () => Promise<T>): Promise<T> {
|
||||
const ids = [...new Set(nodeIds.filter((id) => Number.isFinite(id)))].sort((a, b) => a - b)
|
||||
const releases: Array<() => void> = []
|
||||
try {
|
||||
for (const id of ids) {
|
||||
const prev = btestNodeLocks.get(id)
|
||||
let resolveCurrent!: () => void
|
||||
const current = new Promise<void>((resolve) => {
|
||||
resolveCurrent = resolve
|
||||
})
|
||||
btestNodeLocks.set(id, current)
|
||||
if (prev) await prev
|
||||
releases.push(() => {
|
||||
if (btestNodeLocks.get(id) === current) btestNodeLocks.delete(id)
|
||||
resolveCurrent()
|
||||
})
|
||||
}
|
||||
return await fn()
|
||||
} finally {
|
||||
for (let i = releases.length - 1; i >= 0; i -= 1) releases[i]()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import { db, sqliteDatabase } from "../db/index.js"
|
||||
import { alertBgpPeerSamples, alertGreTunnelSamples } from "../db/schema.js"
|
||||
import { bgpPeerAlertKey, fetchBgpSessionsForAlerts } from "./bgp-peers-live.js"
|
||||
import { fetchGreTunnelLiveRows } from "./gre-tunnels-live.js"
|
||||
import type { GreBgpSnapshotRunSnapshot } from "../types/scheduler-run-snapshot.js"
|
||||
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
|
||||
|
||||
let collecting = false
|
||||
|
||||
export function getGreBgpSnapshotCollectorState(): { running: boolean } {
|
||||
return { running: collecting }
|
||||
}
|
||||
|
||||
/**
|
||||
* Один опрос GRE + BGP по включённым серверам и запись строк в SQLite для `buildSignalSnapshot`.
|
||||
* Движок оповещений больше не дублирует эти REST-запросы.
|
||||
*/
|
||||
export async function collectGreBgpSnapshotOnce(): Promise<GreBgpSnapshotRunSnapshot> {
|
||||
const sampledAt = new Date().toISOString()
|
||||
if (collecting) {
|
||||
return {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "gre_bgp",
|
||||
sampledAt,
|
||||
skipped: true,
|
||||
greWritten: 0,
|
||||
bgpWritten: 0,
|
||||
errors: [],
|
||||
}
|
||||
}
|
||||
collecting = true
|
||||
const errors: string[] = []
|
||||
const snapshot: GreBgpSnapshotRunSnapshot = {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "gre_bgp",
|
||||
sampledAt,
|
||||
greWritten: 0,
|
||||
bgpWritten: 0,
|
||||
greTunnels: [],
|
||||
bgpPeers: [],
|
||||
errors: [],
|
||||
}
|
||||
|
||||
try {
|
||||
const [greSettled, bgpSettled] = await Promise.allSettled([
|
||||
fetchGreTunnelLiveRows(),
|
||||
fetchBgpSessionsForAlerts(),
|
||||
])
|
||||
|
||||
if (greSettled.status === "rejected") {
|
||||
const msg =
|
||||
greSettled.reason instanceof Error ? greSettled.reason.message : String(greSettled.reason)
|
||||
errors.push(`GRE: ${msg}`)
|
||||
}
|
||||
if (bgpSettled.status === "rejected") {
|
||||
const msg =
|
||||
bgpSettled.reason instanceof Error ? bgpSettled.reason.message : String(bgpSettled.reason)
|
||||
errors.push(`BGP: ${msg}`)
|
||||
}
|
||||
|
||||
const greRows = greSettled.status === "fulfilled" ? greSettled.value : []
|
||||
const bgpRows = bgpSettled.status === "fulfilled" ? bgpSettled.value : []
|
||||
snapshot.greTunnels = greRows.map((r) => ({ targetLabel: r.targetLabel, status: r.status }))
|
||||
snapshot.bgpPeers = bgpRows.map((s) => ({ key: bgpPeerAlertKey(s), state: s.state }))
|
||||
|
||||
db.transaction((tx) => {
|
||||
for (const r of greRows) {
|
||||
tx.insert(alertGreTunnelSamples).values({
|
||||
sampledAt,
|
||||
targetLabel: r.targetLabel,
|
||||
status: r.status,
|
||||
}).run()
|
||||
snapshot.greWritten += 1
|
||||
}
|
||||
for (const s of bgpRows) {
|
||||
tx.insert(alertBgpPeerSamples).values({
|
||||
sampledAt,
|
||||
peerKey: bgpPeerAlertKey(s),
|
||||
state: s.state,
|
||||
}).run()
|
||||
snapshot.bgpWritten += 1
|
||||
}
|
||||
})
|
||||
|
||||
sqliteDatabase
|
||||
.prepare(`DELETE FROM alert_gre_tunnel_samples WHERE sampled_at < datetime('now', '-30 days')`)
|
||||
.run()
|
||||
sqliteDatabase
|
||||
.prepare(`DELETE FROM alert_bgp_peer_samples WHERE sampled_at < datetime('now', '-30 days')`)
|
||||
.run()
|
||||
|
||||
if (errors.length) snapshot.errors = errors
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e)
|
||||
snapshot.fatalError = msg
|
||||
} finally {
|
||||
collecting = false
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import { eq } from "drizzle-orm"
|
||||
import { db } from "../db/index.js"
|
||||
import { servers } from "../db/schema.js"
|
||||
import { MikrotikClient } from "./mikrotik.js"
|
||||
|
||||
type ServerRow = typeof servers.$inferSelect
|
||||
|
||||
interface RosGre {
|
||||
".id"?: string
|
||||
name?: string
|
||||
"local-address"?: string
|
||||
"remote-address"?: string
|
||||
"allow-fast-path"?: string
|
||||
"clamp-tcp-mss"?: string
|
||||
mtu?: string
|
||||
"keepalive"?: string
|
||||
dscp?: string
|
||||
running?: string
|
||||
disabled?: string
|
||||
comment?: string
|
||||
}
|
||||
|
||||
export type GreTunnelLiveStatus = "up" | "down" | "degraded"
|
||||
|
||||
/** Как в UI `/alerts`: `${tunnelName} / ${serverName}`. */
|
||||
export type GreTunnelLiveRow = {
|
||||
targetLabel: string
|
||||
status: GreTunnelLiveStatus
|
||||
}
|
||||
|
||||
function mapGreRow(server: ServerRow, g: RosGre, idx: number): GreTunnelLiveRow {
|
||||
const running = g.running === "true"
|
||||
const disabled = g.disabled === "true"
|
||||
const status: GreTunnelLiveStatus = disabled ? "down" : running ? "up" : "degraded"
|
||||
const sn = String(server.name ?? "").trim() || String(server.host ?? "").trim() || String(server.id)
|
||||
const tn = (String(g.name ?? "").trim() || String(g[".id"] ?? "").trim() || `gre-${idx + 1}`).trim()
|
||||
const label = `${tn} / ${sn}`.trim()
|
||||
return { targetLabel: label, status }
|
||||
}
|
||||
|
||||
async function fetchGreForServer(server: ServerRow): Promise<GreTunnelLiveRow[]> {
|
||||
const client = MikrotikClient.fromServer(server)
|
||||
const greRaw = await client.get<RosGre[]>("/interface/gre")
|
||||
return greRaw.map((g, idx) => mapGreRow(server, g, idx))
|
||||
}
|
||||
|
||||
/** GRE с включённых серверов — тот же смысл, что `/api/filters/gre-tunnels` (без правил фильтров). */
|
||||
export async function fetchGreTunnelLiveRows(): Promise<GreTunnelLiveRow[]> {
|
||||
const enabledServers = db.select().from(servers).where(eq(servers.enabled, true)).all()
|
||||
const results = await Promise.all(
|
||||
enabledServers.map(async (server) => {
|
||||
try {
|
||||
return await fetchGreForServer(server)
|
||||
} catch {
|
||||
return [] as GreTunnelLiveRow[]
|
||||
}
|
||||
}),
|
||||
)
|
||||
return results.flat()
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import http from "node:http"
|
||||
import https from "node:https"
|
||||
import type { Server } from "../db/schema.js"
|
||||
import { parseRosDataSizeBytes } from "./ros-metric-parse.js"
|
||||
import type {
|
||||
RosIdentity, RosInterface, RosIpAddress, RosResource,
|
||||
RosBgpSession,
|
||||
@@ -199,6 +200,76 @@ function rosDelete(
|
||||
})
|
||||
}
|
||||
|
||||
function rosPatch(
|
||||
params: MikrotikConnectParams,
|
||||
path: string,
|
||||
body: Record<string, string>,
|
||||
timeoutMs: number,
|
||||
): Promise<unknown> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const basePath = params.apiPath ?? "/rest"
|
||||
const authHeader = "Basic " + Buffer.from(`${params.username}:${params.password}`).toString("base64")
|
||||
const payload = JSON.stringify(body)
|
||||
|
||||
const options: https.RequestOptions = {
|
||||
hostname: params.host,
|
||||
port: params.port,
|
||||
path: basePath + path,
|
||||
method: "PATCH",
|
||||
headers: {
|
||||
Authorization: authHeader,
|
||||
"Content-Type": "application/json",
|
||||
"Content-Length": Buffer.byteLength(payload),
|
||||
},
|
||||
rejectUnauthorized: params.useSsl ? params.verifySsl : undefined,
|
||||
}
|
||||
|
||||
const lib = params.useSsl ? https : http
|
||||
const timer = setTimeout(() => {
|
||||
req.destroy(new Error(`Connection to ${params.host}:${params.port} timed out after ${timeoutMs / 1000}s`))
|
||||
}, timeoutMs)
|
||||
|
||||
const req = lib.request(options, (res) => {
|
||||
let buf = ""
|
||||
res.setEncoding("utf8")
|
||||
res.on("data", (chunk: string) => { buf += chunk })
|
||||
res.on("end", () => {
|
||||
clearTimeout(timer)
|
||||
if (!res.statusCode || res.statusCode < 200 || res.statusCode >= 300) {
|
||||
reject(new MikrotikError(res.statusCode ?? 0, path, buf)); return
|
||||
}
|
||||
if (buf.trim() === "") {
|
||||
resolve({})
|
||||
return
|
||||
}
|
||||
try { resolve(JSON.parse(buf)) } catch { resolve({}) }
|
||||
})
|
||||
})
|
||||
|
||||
req.on("error", (err) => {
|
||||
clearTimeout(timer)
|
||||
reject(err)
|
||||
})
|
||||
req.write(payload)
|
||||
req.end()
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* RouterOS REST для одиночных меню (`/system/resource`, `/system/identity`) отдаёт JSON-массив из одного объекта `[{...}]`,
|
||||
* а не сам объект. Без распаковки поля читаются с массива → всегда undefined и в БД уходят нули при «успехе».
|
||||
* @see https://help.mikrotik.com/docs/spaces/ROS/pages/47579162/REST+API
|
||||
*/
|
||||
function unwrapRestSingle<T extends object>(raw: unknown): T {
|
||||
if (Array.isArray(raw)) {
|
||||
const first = raw[0]
|
||||
if (first && typeof first === "object") return first as T
|
||||
throw new Error("RouterOS REST: пустой массив вместо записи")
|
||||
}
|
||||
if (raw && typeof raw === "object") return raw as T
|
||||
throw new Error("RouterOS REST: неожиданное тело ответа")
|
||||
}
|
||||
|
||||
// ── MikrotikClient ─────────────────────────────────────────────────────────────
|
||||
|
||||
export class MikrotikClient {
|
||||
@@ -228,14 +299,43 @@ export class MikrotikClient {
|
||||
return rosDelete(this.params, path, timeoutMs)
|
||||
}
|
||||
|
||||
async patch<T>(path: string, body: Record<string, string>, timeoutMs = 15_000): Promise<T> {
|
||||
return rosPatch(this.params, path, body, timeoutMs) as Promise<T>
|
||||
}
|
||||
|
||||
// ── typed helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
async getIdentity(): Promise<RosIdentity> {
|
||||
return this.get<RosIdentity>("/system/identity")
|
||||
const raw = await this.get<unknown>("/system/identity")
|
||||
return unwrapRestSingle<RosIdentity>(raw)
|
||||
}
|
||||
|
||||
async getResource(): Promise<RosResource> {
|
||||
return this.get<RosResource>("/system/resource")
|
||||
const raw = await this.get<unknown>("/system/resource")
|
||||
if (Array.isArray(raw)) {
|
||||
if (raw.length === 0) throw new Error("RouterOS REST: пустой массив вместо записи /system/resource")
|
||||
if (raw.length === 1) return unwrapRestSingle<RosResource>(raw[0])
|
||||
/** Несколько объектов: взять строку с total-memory (агрегат), а не дочернюю запись с «чужим» uptime. */
|
||||
const objs = raw.filter((x): x is Record<string, unknown> => x != null && typeof x === "object" && !Array.isArray(x))
|
||||
/** На части железа несколько строк; «первая с непустым total-memory» может парситься в 0 — берём запись с максимальным RAM. */
|
||||
let best: Record<string, unknown> | undefined
|
||||
let bestBytes = -1
|
||||
for (const o of objs) {
|
||||
const tm = parseRosDataSizeBytes(o["total-memory"])
|
||||
const fm = parseRosDataSizeBytes(o["free-memory"])
|
||||
const score = tm > 0 ? tm : fm
|
||||
if (score > bestBytes) {
|
||||
bestBytes = score
|
||||
best = o
|
||||
}
|
||||
}
|
||||
const main = objs.find((o) => {
|
||||
const tm = o["total-memory"]
|
||||
return tm != null && String(tm).trim() !== ""
|
||||
})
|
||||
return ((bestBytes > 0 ? best : undefined) ?? main ?? objs[0]) as unknown as RosResource
|
||||
}
|
||||
return unwrapRestSingle<RosResource>(raw)
|
||||
}
|
||||
|
||||
async getInterfaces(): Promise<RosInterface[]> {
|
||||
@@ -277,6 +377,11 @@ export class MikrotikClient {
|
||||
return this.get<RosBfdSession[]>("/routing/bfd/session")
|
||||
}
|
||||
|
||||
async setOspfInterfaceTemplateCost(templateId: string, cost: number): Promise<void> {
|
||||
const safeId = templateId.startsWith("*") ? templateId : `*${templateId.replace(/^\*/, "")}`
|
||||
await this.patch(`/routing/ospf/interface-template/${encodeURIComponent(safeId)}`, { cost: String(cost) })
|
||||
}
|
||||
|
||||
// ── extra endpoints for exec route ────────────────────────────────────────
|
||||
|
||||
async getIpRoutes(): Promise<RosIpRoute[]> {
|
||||
|
||||
@@ -4,6 +4,7 @@ import { servers, serverSnapshots } from "../db/schema.js"
|
||||
import type { SnapshotInsert } from "../db/schema.js"
|
||||
import type { SnapshotRead } from "../types/server.js"
|
||||
import { MikrotikClient } from "./mikrotik.js"
|
||||
import { parseRosCpuLoadPercent, parseRosDataSizeBytes } from "./ros-metric-parse.js"
|
||||
|
||||
// ── pollServer ─────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -45,10 +46,9 @@ export async function pollServer(serverId: number): Promise<SnapshotRead> {
|
||||
|
||||
const latencyMs = performance.now() - t0
|
||||
|
||||
// Parse RouterOS string values (RouterOS REST API returns everything as strings)
|
||||
const cpuLoad = parseInt(resource["cpu-load"], 10)
|
||||
const freeMem = parseInt(resource["free-memory"], 10)
|
||||
const totalMem = parseInt(resource["total-memory"], 10)
|
||||
const cpuLoad = parseRosCpuLoadPercent(resource["cpu-load"])
|
||||
const freeMem = parseRosDataSizeBytes(resource["free-memory"])
|
||||
const totalMem = parseRosDataSizeBytes(resource["total-memory"])
|
||||
|
||||
Object.assign(partialSnap, {
|
||||
status: "online",
|
||||
@@ -57,9 +57,9 @@ export async function pollServer(serverId: number): Promise<SnapshotRead> {
|
||||
rosVersion: resource["version"],
|
||||
boardName: resource["board-name"],
|
||||
uptime: resource["uptime"],
|
||||
cpuLoad: isNaN(cpuLoad) ? null : cpuLoad,
|
||||
freeMemory: isNaN(freeMem) ? null : freeMem,
|
||||
totalMemory: isNaN(totalMem) ? null : totalMem,
|
||||
cpuLoad,
|
||||
freeMemory: freeMem,
|
||||
totalMemory: totalMem,
|
||||
rawInterfaces: JSON.stringify(ifaces),
|
||||
rawIpAddresses: JSON.stringify(addresses),
|
||||
} satisfies Partial<SnapshotInsert>)
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
/**
|
||||
* Разбор полей /system/resource из RouterOS REST: не всегда «сырые» байты в виде десятичной строки.
|
||||
* На железе часто встречаются значения вида "224.0MiB", "94.2MiB"; parseInt от этого даёт мусор или 0.
|
||||
*/
|
||||
|
||||
export function parseRosDataSizeBytes(raw: unknown): number {
|
||||
if (raw == null) return 0
|
||||
if (typeof raw === "number" && Number.isFinite(raw)) return Math.max(0, Math.floor(raw))
|
||||
let s = String(raw).trim().replace(/\s+/g, "")
|
||||
if (!s) return 0
|
||||
if (/^\d+$/.test(s)) {
|
||||
const n = Number.parseInt(s, 10)
|
||||
return Number.isFinite(n) ? Math.max(0, n) : 0
|
||||
}
|
||||
const m = s.match(/^(\d+(?:[.,]\d+)?)(KiB|MiB|GiB|TiB|Ki|Mi|Gi|Ti|KB|MB|GB|TB)$/i)
|
||||
if (m) {
|
||||
const v = Number.parseFloat(m[1].replace(",", "."))
|
||||
if (!Number.isFinite(v) || v < 0) return 0
|
||||
const u = m[2].toLowerCase()
|
||||
const mult =
|
||||
u === "ki" || u === "kib" || u === "kb" ? 1024
|
||||
: u === "mi" || u === "mib" || u === "mb" ? 1024 ** 2
|
||||
: u === "gi" || u === "gib" || u === "gb" ? 1024 ** 3
|
||||
: u === "ti" || u === "tib" || u === "tb" ? 1024 ** 4
|
||||
: 1
|
||||
return Math.max(0, Math.floor(v * mult))
|
||||
}
|
||||
const fallback = Number.parseInt(s, 10)
|
||||
return Number.isFinite(fallback) ? Math.max(0, fallback) : 0
|
||||
}
|
||||
|
||||
/** cpu-load: "7%", "7", 7 */
|
||||
export function parseRosCpuLoadPercent(raw: unknown): number {
|
||||
if (raw == null) return 0
|
||||
if (typeof raw === "number" && Number.isFinite(raw)) {
|
||||
return Math.min(100, Math.max(0, Math.round(raw)))
|
||||
}
|
||||
const s = String(raw).trim().replace("%", "").replace(",", ".").trim()
|
||||
if (!s) return 0
|
||||
const n = Number.parseFloat(s)
|
||||
if (!Number.isFinite(n)) return 0
|
||||
return Math.min(100, Math.max(0, Math.round(n)))
|
||||
}
|
||||
|
||||
/**
|
||||
* `uptime` из /system/resource (REST): обычно "2d20h12m20s", "39m", "29s".
|
||||
* Встречаются варианты: только секунды строкой, `d.hh:mm:ss`, `hh:mm:ss` без суффиксов.
|
||||
* @see https://help.mikrotik.com/docs/display/ROS/Resource
|
||||
*/
|
||||
export function parseRosUptimeToSeconds(raw: unknown): number {
|
||||
if (raw == null) return 0
|
||||
if (typeof raw === "number" && Number.isFinite(raw)) {
|
||||
return Math.max(0, Math.floor(raw))
|
||||
}
|
||||
const s0 = String(raw).trim().replace(/\u00a0/g, " ").trim()
|
||||
if (!s0) return 0
|
||||
|
||||
// Только десятичные цифры — секунды (некоторые клиенты/прокси)
|
||||
if (/^\d+$/.test(s0)) {
|
||||
const n = Number.parseInt(s0, 10)
|
||||
return Number.isFinite(n) ? Math.max(0, n) : 0
|
||||
}
|
||||
|
||||
const s = s0.toLowerCase().replace(/\s+/g, "")
|
||||
|
||||
// 1d18:27:15
|
||||
const dClock = s.match(/^(\d+)d(\d{1,3}):(\d{2}):(\d{2})$/i)
|
||||
if (dClock) {
|
||||
const d = Number.parseInt(dClock[1], 10)
|
||||
const hh = Number.parseInt(dClock[2], 10)
|
||||
const mm = Number.parseInt(dClock[3], 10)
|
||||
const ss = Number.parseInt(dClock[4], 10)
|
||||
if ([d, hh, mm, ss].every((x) => Number.isFinite(x) && x >= 0)) {
|
||||
return d * 86400 + hh * 3600 + mm * 60 + ss
|
||||
}
|
||||
}
|
||||
|
||||
// 18:27:15 — только если нет суффиксов w/d/h/m/s (иначе это часть "2d20h12m20s")
|
||||
if (!/[wdhms]/i.test(s0) && /^\d{1,7}:\d{2}:\d{2}$/.test(s)) {
|
||||
const p = s.split(":")
|
||||
if (p.length === 3) {
|
||||
const hh = Number.parseInt(p[0], 10)
|
||||
const mm = Number.parseInt(p[1], 10)
|
||||
const sec = Number.parseInt(p[2], 10)
|
||||
if ([hh, mm, sec].every((x) => Number.isFinite(x) && x >= 0)) {
|
||||
return hh * 3600 + mm * 60 + sec
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total = 0
|
||||
for (const m of s.matchAll(/(\d+)(w|d|h|m(?!s)|s)/gi)) {
|
||||
const n = Number.parseInt(m[1], 10)
|
||||
if (!Number.isFinite(n) || n < 0) continue
|
||||
const u = m[2].toLowerCase()
|
||||
switch (u) {
|
||||
case "w": total += n * 604800; break
|
||||
case "d": total += n * 86400; break
|
||||
case "h": total += n * 3600; break
|
||||
case "m": total += n * 60; break
|
||||
case "s": total += n; break
|
||||
default: break
|
||||
}
|
||||
}
|
||||
return Math.max(0, total)
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
/**
|
||||
* Реестр активных джоб планировщика (без импорта `scheduler.ts` → нет цикла с `run-once`).
|
||||
* Слот берётся до `await` в теле джобы — так `alert_engine` может дождаться REST и др. до чтения БД.
|
||||
*/
|
||||
const runningSchedulerJobs = new Set<string>()
|
||||
|
||||
const ALERT_SNAPSHOT_SOURCE_JOB_KEYS = new Set<string>([
|
||||
"traffic",
|
||||
"servers_rest_ping",
|
||||
"uptime_resources",
|
||||
"uptime_ping",
|
||||
"uptime_speed",
|
||||
"gre_bgp",
|
||||
])
|
||||
|
||||
/** true, если слот занят (джоба уже выполняется). */
|
||||
export function tryBeginSchedulerJob(jobKey: string): boolean {
|
||||
if (runningSchedulerJobs.has(jobKey)) return false
|
||||
runningSchedulerJobs.add(jobKey)
|
||||
return true
|
||||
}
|
||||
|
||||
export function endSchedulerJob(jobKey: string): void {
|
||||
runningSchedulerJobs.delete(jobKey)
|
||||
}
|
||||
|
||||
export function isSchedulerJobRunning(jobKey: string): boolean {
|
||||
return runningSchedulerJobs.has(jobKey)
|
||||
}
|
||||
|
||||
/** Есть ли коллектор, который пишет таблицы снимка для `buildSignalSnapshot`. */
|
||||
export function isAnyAlertSnapshotSourceJobRunning(): boolean {
|
||||
for (const k of ALERT_SNAPSHOT_SOURCE_JOB_KEYS) {
|
||||
if (runningSchedulerJobs.has(k)) return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,362 @@
|
||||
/**
|
||||
* In-process планировщик: отдельный интервал на job (`refreshScheduler`).
|
||||
*
|
||||
* **Приоритеты при конкуренции** (для дальнейшего per-server mutex; сейчас зафиксировано в дизайне):
|
||||
* 1. `uptime_ping` — выше (короткий критичный сигнал).
|
||||
* 2. `traffic`, `uptime_resources`, `servers_rest_ping` — средний (REST `/system/identity` по каталогу).
|
||||
* 3. `uptime_speed` — ниже (BW-test, тяжёлый); локи узлов — `withBtestNodeLocks` в speed-сервисе.
|
||||
*
|
||||
* **Оповещения (`alert_engine`):** читают SQLite после джоб сбора (см. `buildSignalSnapshot`), в т.ч.
|
||||
* `gre_bgp` → `alert_gre_tunnel_samples` / `alert_bgp_peer_samples`. После успешного завершения джоб
|
||||
* `traffic`, `uptime_*`, `servers_rest_ping`, `gre_bgp` планируется **дополнительный** прогон движка
|
||||
* (debounce), см. [`alert-collector-hooks.ts`](./alert-collector-hooks.ts); любой другой писатель сэмплов
|
||||
* для снимка оповещений тоже должен вызывать `scheduleAlertEngineAfterDataCollectors()` после коммита.
|
||||
*
|
||||
* **Кастомные job без произвольного кода (MVP):** пер-тайминг ping в `uptime_probes.interval_sec`;
|
||||
* таблица `scheduler_job_instances` (kind + config JSON) — при необходимости следующий этап.
|
||||
*/
|
||||
import { desc, eq, lt } from "drizzle-orm"
|
||||
import { db } from "../db/index.js"
|
||||
import { schedulerRuns } from "../db/schema.js"
|
||||
import type { SchedulerRunSnapshot } from "../types/scheduler-run-snapshot.js"
|
||||
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
|
||||
import {
|
||||
collectServersRestPingOnce,
|
||||
getServersApiPingSettings,
|
||||
} from "./servers-rest-ping-collector.js"
|
||||
import { collectTrafficOnce, getTrafficSettings } from "./traffic-collector.js"
|
||||
import {
|
||||
collectPingProbesOnce,
|
||||
collectResourceSamplesOnce,
|
||||
getSettings as getUptimeSettings,
|
||||
} from "./uptime-collector.js"
|
||||
import { runScheduledSpeedProbesOnce } from "./uptime-speed-service.js"
|
||||
import { collectGreBgpSnapshotOnce } from "./gre-bgp-snapshot-collector.js"
|
||||
import { runAlertEngineOnce } from "./alert-engine/run-once.js"
|
||||
import {
|
||||
clearAlertCollectorHooksTimer,
|
||||
scheduleAlertEngineAfterDataCollectors,
|
||||
wireAlertEngineRunner,
|
||||
} from "./alert-collector-hooks.js"
|
||||
import {
|
||||
endSchedulerJob,
|
||||
isSchedulerJobRunning,
|
||||
tryBeginSchedulerJob,
|
||||
} from "./scheduler-running.js"
|
||||
|
||||
export const JOB_KEYS = [
|
||||
"traffic",
|
||||
"servers_rest_ping",
|
||||
"uptime_resources",
|
||||
"uptime_ping",
|
||||
"uptime_speed",
|
||||
"gre_bgp",
|
||||
"alert_engine",
|
||||
] as const
|
||||
export type SchedulerJobKey = (typeof JOB_KEYS)[number]
|
||||
|
||||
const timers = new Map<string, ReturnType<typeof setInterval>>()
|
||||
|
||||
const RUN_LOG_RETENTION_MS = 30 * 24 * 60 * 60 * 1000
|
||||
|
||||
function newRunId(): string {
|
||||
return `sch-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`
|
||||
}
|
||||
|
||||
function appendSchedulerRun(row: {
|
||||
jobKey: string
|
||||
startedAt: string
|
||||
finishedAt: string
|
||||
status: "ok" | "error"
|
||||
error: string | null
|
||||
durationMs: number
|
||||
result?: SchedulerRunSnapshot | null
|
||||
}) {
|
||||
db.insert(schedulerRuns).values({
|
||||
id: newRunId(),
|
||||
jobKey: row.jobKey,
|
||||
startedAt: row.startedAt,
|
||||
finishedAt: row.finishedAt,
|
||||
status: row.status,
|
||||
error: row.error,
|
||||
durationMs: row.durationMs,
|
||||
resultJson: row.result ? JSON.stringify(row.result) : null,
|
||||
}).run()
|
||||
const cutoff = new Date(Date.now() - RUN_LOG_RETENTION_MS).toISOString()
|
||||
db.delete(schedulerRuns).where(lt(schedulerRuns.finishedAt, cutoff)).run()
|
||||
}
|
||||
|
||||
async function runSchedulerJobBody(jobKey: SchedulerJobKey): Promise<void> {
|
||||
const startedAt = Date.now()
|
||||
const startedIso = new Date().toISOString()
|
||||
let snapshot: SchedulerRunSnapshot | undefined
|
||||
try {
|
||||
switch (jobKey) {
|
||||
case "traffic":
|
||||
snapshot = await collectTrafficOnce()
|
||||
break
|
||||
case "uptime_resources":
|
||||
snapshot = await collectResourceSamplesOnce()
|
||||
break
|
||||
case "uptime_ping":
|
||||
snapshot = await collectPingProbesOnce()
|
||||
break
|
||||
case "uptime_speed":
|
||||
snapshot = await runScheduledSpeedProbesOnce()
|
||||
break
|
||||
case "servers_rest_ping":
|
||||
snapshot = await collectServersRestPingOnce()
|
||||
break
|
||||
case "gre_bgp":
|
||||
snapshot = await collectGreBgpSnapshotOnce()
|
||||
break
|
||||
case "alert_engine": {
|
||||
const r = await runAlertEngineOnce()
|
||||
snapshot = {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "alert_engine",
|
||||
sampledAt: r.sampledAt,
|
||||
rulesChecked: r.rulesChecked,
|
||||
standaloneFires: r.standaloneFires,
|
||||
groupFires: r.groupFires,
|
||||
skippedNoTelegram: r.skippedNoTelegram,
|
||||
...(r.ruleDiag.length ? { ruleDiag: r.ruleDiag } : {}),
|
||||
...(r.errors.length ? { errors: r.errors } : {}),
|
||||
}
|
||||
break
|
||||
}
|
||||
default:
|
||||
throw new Error(`Unknown job: ${jobKey}`)
|
||||
}
|
||||
const finishedIso = new Date().toISOString()
|
||||
appendSchedulerRun({
|
||||
jobKey,
|
||||
startedAt: startedIso,
|
||||
finishedAt: finishedIso,
|
||||
status: "ok",
|
||||
error: null,
|
||||
durationMs: Date.now() - startedAt,
|
||||
result: snapshot ?? null,
|
||||
})
|
||||
if (
|
||||
jobKey === "traffic" ||
|
||||
jobKey === "servers_rest_ping" ||
|
||||
jobKey === "uptime_resources" ||
|
||||
jobKey === "uptime_ping" ||
|
||||
jobKey === "uptime_speed" ||
|
||||
jobKey === "gre_bgp"
|
||||
) {
|
||||
scheduleAlertEngineAfterDataCollectors()
|
||||
}
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e)
|
||||
const finishedIso = new Date().toISOString()
|
||||
appendSchedulerRun({
|
||||
jobKey,
|
||||
startedAt: startedIso,
|
||||
finishedAt: finishedIso,
|
||||
status: "error",
|
||||
error: msg,
|
||||
durationMs: Date.now() - startedAt,
|
||||
result: snapshot ?? null,
|
||||
})
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
/** Фоновый тик: пропуск, если предыдущий прогон ещё идёт. */
|
||||
export async function executeSchedulerJob(jobKey: SchedulerJobKey): Promise<void> {
|
||||
if (!tryBeginSchedulerJob(jobKey)) return
|
||||
try {
|
||||
await runSchedulerJobBody(jobKey)
|
||||
} catch {
|
||||
/* залогировано в runSchedulerJobBody */
|
||||
} finally {
|
||||
endSchedulerJob(jobKey)
|
||||
}
|
||||
}
|
||||
|
||||
/** Ручной запуск: 409, если job уже выполняется. */
|
||||
export async function runSchedulerJobNow(jobKey: string): Promise<void> {
|
||||
if (!JOB_KEYS.includes(jobKey as SchedulerJobKey)) {
|
||||
throw new Error(`Unknown job key: ${jobKey}`)
|
||||
}
|
||||
if (!tryBeginSchedulerJob(jobKey)) {
|
||||
const err = new Error("Job already running")
|
||||
;(err as Error & { statusCode?: number }).statusCode = 409
|
||||
throw err
|
||||
}
|
||||
try {
|
||||
await runSchedulerJobBody(jobKey as SchedulerJobKey)
|
||||
} finally {
|
||||
endSchedulerJob(jobKey)
|
||||
}
|
||||
}
|
||||
|
||||
function clearAllTimers() {
|
||||
clearAlertCollectorHooksTimer()
|
||||
for (const t of timers.values()) clearInterval(t)
|
||||
timers.clear()
|
||||
}
|
||||
|
||||
/** Пересоздать интервалы после смены настроек. */
|
||||
export function refreshScheduler(): void {
|
||||
clearAllTimers()
|
||||
|
||||
const traffic = getTrafficSettings()
|
||||
if (traffic.enabled) {
|
||||
const ms = Math.max(5_000, traffic.intervalSec * 1000)
|
||||
void executeSchedulerJob("traffic").catch(() => {})
|
||||
timers.set(
|
||||
"traffic",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("traffic").catch(() => {})
|
||||
}, ms),
|
||||
)
|
||||
}
|
||||
|
||||
const apiPing = getServersApiPingSettings()
|
||||
if (apiPing.enabled) {
|
||||
const apiMs = Math.max(10_000, apiPing.intervalSec * 1000)
|
||||
void executeSchedulerJob("servers_rest_ping").catch(() => {})
|
||||
timers.set(
|
||||
"servers_rest_ping",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("servers_rest_ping").catch(() => {})
|
||||
}, apiMs),
|
||||
)
|
||||
}
|
||||
|
||||
const uptime = getUptimeSettings()
|
||||
const resOn = uptime.resourcesEnabled ?? uptime.enabled
|
||||
const pingOn = uptime.pingEnabled ?? uptime.enabled
|
||||
const spdOn = uptime.speedEnabled ?? uptime.enabled
|
||||
|
||||
if (resOn) {
|
||||
const resMs = Math.max(5_000, uptime.intervalSec * 1000)
|
||||
void executeSchedulerJob("uptime_resources").catch(() => {})
|
||||
timers.set(
|
||||
"uptime_resources",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("uptime_resources").catch(() => {})
|
||||
}, resMs),
|
||||
)
|
||||
}
|
||||
|
||||
if (pingOn) {
|
||||
const pingMs = Math.max(1_000, uptime.probeIntervalSec * 1000)
|
||||
void executeSchedulerJob("uptime_ping").catch(() => {})
|
||||
timers.set(
|
||||
"uptime_ping",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("uptime_ping").catch(() => {})
|
||||
}, pingMs),
|
||||
)
|
||||
}
|
||||
|
||||
if (spdOn) {
|
||||
const spdMs = Math.max(10_000, uptime.speedIntervalSec * 1000)
|
||||
void executeSchedulerJob("uptime_speed").catch(() => {})
|
||||
timers.set(
|
||||
"uptime_speed",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("uptime_speed").catch(() => {})
|
||||
}, spdMs),
|
||||
)
|
||||
}
|
||||
|
||||
const greBgpMs = 30_000
|
||||
void executeSchedulerJob("gre_bgp").catch(() => {})
|
||||
timers.set(
|
||||
"gre_bgp",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("gre_bgp").catch(() => {})
|
||||
}, greBgpMs),
|
||||
)
|
||||
|
||||
const alertMs = 20_000
|
||||
void executeSchedulerJob("alert_engine").catch(() => {})
|
||||
timers.set(
|
||||
"alert_engine",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("alert_engine").catch(() => {})
|
||||
}, alertMs),
|
||||
)
|
||||
}
|
||||
|
||||
export function stopScheduler(): void {
|
||||
clearAllTimers()
|
||||
}
|
||||
|
||||
export function isJobRunning(jobKey: string): boolean {
|
||||
return isSchedulerJobRunning(jobKey)
|
||||
}
|
||||
|
||||
function lastRunForJob(jobKey: string) {
|
||||
return db.select().from(schedulerRuns)
|
||||
.where(eq(schedulerRuns.jobKey, jobKey))
|
||||
.orderBy(desc(schedulerRuns.finishedAt))
|
||||
.limit(1)
|
||||
.all()[0]
|
||||
}
|
||||
|
||||
export function getSchedulerStatus() {
|
||||
const traffic = getTrafficSettings()
|
||||
const uptime = getUptimeSettings()
|
||||
const apiPing = getServersApiPingSettings()
|
||||
|
||||
const resOn = uptime.resourcesEnabled ?? uptime.enabled
|
||||
const pingOn = uptime.pingEnabled ?? uptime.enabled
|
||||
const spdOn = uptime.speedEnabled ?? uptime.enabled
|
||||
|
||||
const jobMeta: Record<SchedulerJobKey, { enabled: boolean; intervalSec: number }> = {
|
||||
traffic: { enabled: traffic.enabled, intervalSec: traffic.intervalSec },
|
||||
servers_rest_ping: { enabled: apiPing.enabled, intervalSec: apiPing.intervalSec },
|
||||
uptime_resources: { enabled: resOn, intervalSec: uptime.intervalSec },
|
||||
uptime_ping: { enabled: pingOn, intervalSec: uptime.probeIntervalSec },
|
||||
uptime_speed: { enabled: spdOn, intervalSec: uptime.speedIntervalSec },
|
||||
gre_bgp: { enabled: true, intervalSec: 30 },
|
||||
alert_engine: { enabled: true, intervalSec: 20 },
|
||||
}
|
||||
|
||||
return {
|
||||
jobs: JOB_KEYS.map((jobKey) => {
|
||||
const last = lastRunForJob(jobKey)
|
||||
const m = jobMeta[jobKey]
|
||||
return {
|
||||
jobKey,
|
||||
enabled: m.enabled,
|
||||
intervalSec: m.intervalSec,
|
||||
running: isSchedulerJobRunning(jobKey),
|
||||
lastFinishedAt: last?.finishedAt ?? null,
|
||||
lastStatus: last?.status ?? null,
|
||||
lastDurationMs: last?.durationMs ?? null,
|
||||
lastError: last?.error ?? null,
|
||||
}
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
export function listSchedulerRuns(opts: { jobKey?: string; limit: number; offset: number }) {
|
||||
const limit = Math.min(200, Math.max(1, opts.limit))
|
||||
const offset = Math.max(0, opts.offset)
|
||||
if (opts.jobKey) {
|
||||
return {
|
||||
runs: db.select().from(schedulerRuns)
|
||||
.where(eq(schedulerRuns.jobKey, opts.jobKey))
|
||||
.orderBy(desc(schedulerRuns.finishedAt))
|
||||
.limit(limit)
|
||||
.offset(offset)
|
||||
.all(),
|
||||
}
|
||||
}
|
||||
return {
|
||||
runs: db.select().from(schedulerRuns)
|
||||
.orderBy(desc(schedulerRuns.finishedAt))
|
||||
.limit(limit)
|
||||
.offset(offset)
|
||||
.all(),
|
||||
}
|
||||
}
|
||||
|
||||
wireAlertEngineRunner(() => executeSchedulerJob("alert_engine"))
|
||||
@@ -0,0 +1,202 @@
|
||||
import { eq, or } from "drizzle-orm"
|
||||
import { db, sqliteDatabase } from "../db/index.js"
|
||||
import { servers, serversApiPingSettings, serversRestPingSamples } from "../db/schema.js"
|
||||
import type { ServersRestPingRunSnapshot } from "../types/scheduler-run-snapshot.js"
|
||||
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
|
||||
import { MikrotikClient } from "./mikrotik.js"
|
||||
|
||||
let collecting = false
|
||||
|
||||
function getSettingsRow() {
|
||||
const row = db.select().from(serversApiPingSettings).where(eq(serversApiPingSettings.id, 1)).limit(1).all()[0]
|
||||
if (row) return row
|
||||
const now = new Date().toISOString()
|
||||
db.insert(serversApiPingSettings).values({
|
||||
id: 1,
|
||||
enabled: false,
|
||||
intervalSec: 120,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
}).run()
|
||||
return db.select().from(serversApiPingSettings).where(eq(serversApiPingSettings.id, 1)).limit(1).all()[0]
|
||||
}
|
||||
|
||||
/** Как uptime: включённые в каталоге или Home Router (часто «выкл.» при доступе только из LAN). */
|
||||
function serversForRestProbe() {
|
||||
return db
|
||||
.select()
|
||||
.from(servers)
|
||||
.where(or(eq(servers.enabled, true), eq(servers.type, "home-router")))
|
||||
.all()
|
||||
}
|
||||
|
||||
export function getServersApiPingSettings() {
|
||||
return getSettingsRow()
|
||||
}
|
||||
|
||||
export function updateServersApiPingSettings(patch: { enabled?: boolean; intervalSec?: number }) {
|
||||
const prev = getSettingsRow()
|
||||
const next = {
|
||||
enabled: patch.enabled ?? prev.enabled,
|
||||
intervalSec: patch.intervalSec ?? prev.intervalSec,
|
||||
updatedAt: new Date().toISOString(),
|
||||
}
|
||||
db.update(serversApiPingSettings).set(next).where(eq(serversApiPingSettings.id, 1)).run()
|
||||
return getSettingsRow()
|
||||
}
|
||||
|
||||
export function getServersRestPingCollectorState(): { running: boolean } {
|
||||
return { running: collecting }
|
||||
}
|
||||
|
||||
function loadLastRestSampleByServer(): Map<number, { ok: boolean }> {
|
||||
const rows = sqliteDatabase
|
||||
.prepare(
|
||||
`SELECT s.server_id AS serverId, s.ok
|
||||
FROM servers_rest_ping_samples s
|
||||
INNER JOIN (
|
||||
SELECT server_id, MAX(id) AS mid FROM servers_rest_ping_samples GROUP BY server_id
|
||||
) t ON t.mid = s.id`,
|
||||
)
|
||||
.all() as { serverId: number; ok: number }[]
|
||||
const m = new Map<number, { ok: boolean }>()
|
||||
for (const r of rows) m.set(r.serverId, { ok: Boolean(r.ok) })
|
||||
return m
|
||||
}
|
||||
|
||||
/** Последний сэмпл ресурсов — для согласования «восстановился» с REST без второго тика uptime. */
|
||||
function loadLatestResourceDownServerIds(): Set<number> {
|
||||
const rows = sqliteDatabase
|
||||
.prepare(
|
||||
`WITH r AS (
|
||||
SELECT urs.server_id AS serverId, urs.status,
|
||||
ROW_NUMBER() OVER (PARTITION BY urs.server_id ORDER BY urs.id DESC) AS rn
|
||||
FROM uptime_resource_samples urs
|
||||
)
|
||||
SELECT serverId FROM r WHERE rn = 1 AND status IN ('offline', 'degraded')`,
|
||||
)
|
||||
.all() as { serverId: number }[]
|
||||
return new Set(rows.map((x) => x.serverId))
|
||||
}
|
||||
|
||||
/**
|
||||
* Лёгкая проверка: GET `/system/identity` (REST) на узел, замер RTT.
|
||||
* Пишет строки в `servers_rest_ping_samples` для алерт-движка (см. `buildSignalSnapshot`).
|
||||
*/
|
||||
export async function collectServersRestPingOnce(): Promise<ServersRestPingRunSnapshot> {
|
||||
const sampledAt = new Date().toISOString()
|
||||
if (collecting) {
|
||||
return {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "servers_rest_ping",
|
||||
sampledAt,
|
||||
skipped: true,
|
||||
servers: [],
|
||||
}
|
||||
}
|
||||
collecting = true
|
||||
const startedAt = Date.now()
|
||||
const snapshot: ServersRestPingRunSnapshot = {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "servers_rest_ping",
|
||||
sampledAt,
|
||||
servers: [],
|
||||
}
|
||||
|
||||
try {
|
||||
const targets = serversForRestProbe()
|
||||
const rows = await Promise.all(
|
||||
targets.map(async (srv) => {
|
||||
const name = srv.name || srv.host
|
||||
const t0 = performance.now()
|
||||
try {
|
||||
const client = MikrotikClient.fromServer(srv)
|
||||
await client.getIdentity()
|
||||
const latencyMs = Math.round(performance.now() - t0)
|
||||
return {
|
||||
serverId: srv.id,
|
||||
name,
|
||||
host: srv.host,
|
||||
ok: true as const,
|
||||
latencyMs,
|
||||
}
|
||||
} catch (err) {
|
||||
return {
|
||||
serverId: srv.id,
|
||||
name,
|
||||
host: srv.host,
|
||||
ok: false as const,
|
||||
latencyMs: null as number | null,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
}
|
||||
}
|
||||
}),
|
||||
)
|
||||
snapshot.servers = rows.sort((a, b) => a.serverId - b.serverId)
|
||||
|
||||
const lastByServer = loadLastRestSampleByServer()
|
||||
const resourceDown = loadLatestResourceDownServerIds()
|
||||
|
||||
db.transaction((tx) => {
|
||||
for (const row of snapshot.servers) {
|
||||
const sid = row.serverId
|
||||
const last = lastByServer.get(sid)
|
||||
const errText = row.ok ? null : (row.error ?? null)
|
||||
|
||||
const insertOne = (ok: boolean, latencyMs: number | null, error: string | null) => {
|
||||
tx.insert(serversRestPingSamples).values({
|
||||
serverId: sid,
|
||||
sampledAt,
|
||||
ok,
|
||||
latencyMs,
|
||||
error,
|
||||
}).run()
|
||||
}
|
||||
|
||||
const lat = row.latencyMs ?? null
|
||||
|
||||
if (row.ok) {
|
||||
const downRes = resourceDown.has(sid)
|
||||
if (last === undefined && downRes) {
|
||||
insertOne(false, null, null)
|
||||
}
|
||||
insertOne(true, lat, null)
|
||||
if (last?.ok === false || (last === undefined && downRes)) {
|
||||
insertOne(true, lat, null)
|
||||
}
|
||||
} else {
|
||||
insertOne(false, null, errText)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
sqliteDatabase
|
||||
.prepare(`DELETE FROM servers_rest_ping_samples WHERE sampled_at < datetime('now', '-30 days')`)
|
||||
.run()
|
||||
|
||||
db.update(serversApiPingSettings)
|
||||
.set({
|
||||
lastCollectedAt: sampledAt,
|
||||
lastDurationMs: Date.now() - startedAt,
|
||||
lastError: "",
|
||||
updatedAt: sampledAt,
|
||||
})
|
||||
.where(eq(serversApiPingSettings.id, 1))
|
||||
.run()
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e)
|
||||
snapshot.fatalError = msg
|
||||
db.update(serversApiPingSettings)
|
||||
.set({
|
||||
lastCollectedAt: sampledAt,
|
||||
lastDurationMs: Date.now() - startedAt,
|
||||
lastError: msg,
|
||||
updatedAt: sampledAt,
|
||||
})
|
||||
.where(eq(serversApiPingSettings.id, 1))
|
||||
.run()
|
||||
} finally {
|
||||
collecting = false
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
import { and, asc, eq, gte, lt } from "drizzle-orm"
|
||||
import { db } from "../db/index.js"
|
||||
import { servers, trafficSamples, trafficSettings } from "../db/schema.js"
|
||||
import type { TrafficRunSnapshot } from "../types/scheduler-run-snapshot.js"
|
||||
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
|
||||
import { MikrotikClient } from "./mikrotik.js"
|
||||
|
||||
interface RosIfaceTraffic {
|
||||
@@ -19,13 +21,7 @@ export interface TrafficCollectorState {
|
||||
lastError: string | null
|
||||
}
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null
|
||||
let collecting = false
|
||||
const state: TrafficCollectorState = {
|
||||
running: false,
|
||||
lastRunAt: null,
|
||||
lastError: null,
|
||||
}
|
||||
|
||||
function toNum(raw: unknown): number {
|
||||
const n = Number.parseFloat(String(raw ?? "0"))
|
||||
@@ -54,36 +50,74 @@ function cleanupOldSamples(retentionDays: number) {
|
||||
.run()
|
||||
}
|
||||
|
||||
export async function collectTrafficOnce(): Promise<void> {
|
||||
if (collecting) return
|
||||
export async function collectTrafficOnce(): Promise<TrafficRunSnapshot> {
|
||||
const sampledAt = new Date().toISOString()
|
||||
if (collecting) {
|
||||
return {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "traffic",
|
||||
sampledAt,
|
||||
skipped: true,
|
||||
servers: [],
|
||||
}
|
||||
}
|
||||
collecting = true
|
||||
const startedAt = Date.now()
|
||||
const now = new Date().toISOString()
|
||||
const now = sampledAt
|
||||
const settings = getSettingsRow()
|
||||
const snapshot: TrafficRunSnapshot = {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "traffic",
|
||||
sampledAt: now,
|
||||
servers: [],
|
||||
}
|
||||
|
||||
try {
|
||||
const enabledServers = db.select().from(servers).where(eq(servers.enabled, true)).all()
|
||||
|
||||
for (const srv of enabledServers) {
|
||||
const name = srv.name || srv.host
|
||||
try {
|
||||
const client = MikrotikClient.fromServer(srv)
|
||||
const ifaces = await client.get<RosIfaceTraffic[]>("/interface")
|
||||
if (ifaces.length === 0) continue
|
||||
db.insert(trafficSamples).values(
|
||||
ifaces.map((i) => ({
|
||||
serverId: srv.id,
|
||||
interfaceName: i.name ?? "unknown",
|
||||
sampledAt: now,
|
||||
rxBytes: toNum(i["rx-byte"]),
|
||||
txBytes: toNum(i["tx-byte"]),
|
||||
rxBps: toNum(i["rx-bits-per-second"]),
|
||||
txBps: toNum(i["tx-bits-per-second"]),
|
||||
running: (i.running ?? "false") === "true",
|
||||
disabled: (i.disabled ?? "false") === "true",
|
||||
})),
|
||||
).run()
|
||||
} catch {
|
||||
// Continue collecting from remaining servers
|
||||
let sumRx = 0
|
||||
let sumTx = 0
|
||||
for (const i of ifaces) {
|
||||
sumRx += toNum(i["rx-bits-per-second"]) / 1_000_000
|
||||
sumTx += toNum(i["tx-bits-per-second"]) / 1_000_000
|
||||
}
|
||||
if (ifaces.length > 0) {
|
||||
db.insert(trafficSamples).values(
|
||||
ifaces.map((i) => ({
|
||||
serverId: srv.id,
|
||||
interfaceName: i.name ?? "unknown",
|
||||
sampledAt: now,
|
||||
rxBytes: toNum(i["rx-byte"]),
|
||||
txBytes: toNum(i["tx-byte"]),
|
||||
rxBps: toNum(i["rx-bits-per-second"]),
|
||||
txBps: toNum(i["tx-bits-per-second"]),
|
||||
running: (i.running ?? "false") === "true",
|
||||
disabled: (i.disabled ?? "false") === "true",
|
||||
})),
|
||||
).run()
|
||||
}
|
||||
snapshot.servers.push({
|
||||
serverId: srv.id,
|
||||
name,
|
||||
host: srv.host,
|
||||
ok: true,
|
||||
interfaces: ifaces.length,
|
||||
sumRxMbps: Math.round(sumRx),
|
||||
sumTxMbps: Math.round(sumTx),
|
||||
})
|
||||
} catch (err) {
|
||||
snapshot.servers.push({
|
||||
serverId: srv.id,
|
||||
name,
|
||||
host: srv.host,
|
||||
ok: false,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -94,45 +128,38 @@ export async function collectTrafficOnce(): Promise<void> {
|
||||
lastError: "",
|
||||
updatedAt: now,
|
||||
}).where(eq(trafficSettings.id, 1)).run()
|
||||
|
||||
state.lastRunAt = now
|
||||
state.lastError = null
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e)
|
||||
snapshot.fatalError = msg
|
||||
db.update(trafficSettings).set({
|
||||
lastCollectedAt: now,
|
||||
lastDurationMs: Date.now() - startedAt,
|
||||
lastError: msg,
|
||||
updatedAt: now,
|
||||
}).where(eq(trafficSettings.id, 1)).run()
|
||||
state.lastRunAt = now
|
||||
state.lastError = msg
|
||||
} finally {
|
||||
collecting = false
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
export function stopTrafficCollector() {
|
||||
if (timer) {
|
||||
clearInterval(timer)
|
||||
timer = null
|
||||
}
|
||||
state.running = false
|
||||
}
|
||||
/** Таймеры трафика ведёт планировщик. */
|
||||
export function stopTrafficCollector() {}
|
||||
|
||||
export function restartTrafficCollector() {
|
||||
stopTrafficCollector()
|
||||
const settings = getSettingsRow()
|
||||
if (!settings.enabled) return
|
||||
const intervalMs = Math.max(5, settings.intervalSec) * 1000
|
||||
timer = setInterval(() => {
|
||||
void collectTrafficOnce()
|
||||
}, intervalMs)
|
||||
state.running = true
|
||||
}
|
||||
export function restartTrafficCollector() {}
|
||||
|
||||
export function getTrafficCollectorState(): TrafficCollectorState {
|
||||
return { ...state }
|
||||
const s = getSettingsRow()
|
||||
return {
|
||||
running: Boolean(s.enabled),
|
||||
lastRunAt: s.lastCollectedAt ?? null,
|
||||
lastError: s.lastError || null,
|
||||
}
|
||||
}
|
||||
|
||||
/** Идёт активный прогон `collectTrafficOnce` (запись в SQLite ещё не завершена). */
|
||||
export function isTrafficCollecting(): boolean {
|
||||
return collecting
|
||||
}
|
||||
|
||||
export function getTrafficSettings() {
|
||||
@@ -152,7 +179,6 @@ export function updateTrafficSettings(patch: {
|
||||
updatedAt: new Date().toISOString(),
|
||||
}
|
||||
db.update(trafficSettings).set(next).where(eq(trafficSettings.id, 1)).run()
|
||||
restartTrafficCollector()
|
||||
return getSettingsRow()
|
||||
}
|
||||
|
||||
@@ -166,4 +192,3 @@ export function readServerSamplesInRange(serverId: number, sinceIso: string) {
|
||||
.orderBy(asc(trafficSamples.sampledAt))
|
||||
.all()
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { and, asc, eq, gte, lt } from "drizzle-orm"
|
||||
import { and, asc, eq, gte, inArray, lt, max, or } from "drizzle-orm"
|
||||
import { db } from "../db/index.js"
|
||||
import {
|
||||
servers,
|
||||
@@ -7,31 +7,18 @@ import {
|
||||
uptimeResourceSamples,
|
||||
uptimeSettings,
|
||||
} from "../db/schema.js"
|
||||
import type { PingRunSnapshot, ResourcesRunSnapshot } from "../types/scheduler-run-snapshot.js"
|
||||
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
|
||||
import { PING_PROBE_WARN_RTT_MS } from "../constants/ping-probe.js"
|
||||
import { MikrotikClient } from "./mikrotik.js"
|
||||
import { parseRosCpuLoadPercent, parseRosDataSizeBytes, parseRosUptimeToSeconds } from "./ros-metric-parse.js"
|
||||
|
||||
let timer: ReturnType<typeof setInterval> | null = null
|
||||
let collecting = false
|
||||
let collectingResources = false
|
||||
let collectingPing = false
|
||||
|
||||
function parseDurationToSeconds(raw: string | undefined): number {
|
||||
if (!raw) return 0
|
||||
let total = 0
|
||||
for (const m of raw.matchAll(/(\d+)(w|d|h|m(?!s)|s)/g)) {
|
||||
const n = Number.parseInt(m[1], 10)
|
||||
if (!Number.isFinite(n)) continue
|
||||
switch (m[2]) {
|
||||
case "w": total += n * 604800; break
|
||||
case "d": total += n * 86400; break
|
||||
case "h": total += n * 3600; break
|
||||
case "m": total += n * 60; break
|
||||
case "s": total += n; break
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
function toNum(raw: unknown): number {
|
||||
const n = Number.parseInt(String(raw ?? "0"), 10)
|
||||
return Number.isFinite(n) ? Math.max(0, n) : 0
|
||||
/** Параллельный прогон сборщиков uptime — чтобы `alert_engine` не читал БД до коммита сэмплов. */
|
||||
export function getUptimeCollectorsState(): { resources: boolean; ping: boolean } {
|
||||
return { resources: collectingResources, ping: collectingPing }
|
||||
}
|
||||
|
||||
function parsePingTimeMs(raw: string | undefined): number | null {
|
||||
@@ -44,7 +31,6 @@ function parsePingTimeMs(raw: string | undefined): number | null {
|
||||
const sec = s.match(/^(\d+(?:\.\d+)?)\s*s$/)
|
||||
if (sec) return Number.parseFloat(sec[1]) * 1000
|
||||
|
||||
// HH:MM:SS(.sss) from some RouterOS outputs
|
||||
const clock = s.match(/^(\d+):(\d+):(\d+(?:\.\d+)?)$/)
|
||||
if (clock) {
|
||||
const h = Number.parseFloat(clock[1])
|
||||
@@ -57,16 +43,19 @@ function parsePingTimeMs(raw: string | undefined): number | null {
|
||||
return Number.isFinite(n) ? n : null
|
||||
}
|
||||
|
||||
function getSettings() {
|
||||
export function getSettings() {
|
||||
const row = db.select().from(uptimeSettings).where(eq(uptimeSettings.id, 1)).limit(1).all()[0]
|
||||
if (row) return row
|
||||
const now = new Date().toISOString()
|
||||
db.insert(uptimeSettings).values({
|
||||
id: 1,
|
||||
enabled: true,
|
||||
intervalSec: 15,
|
||||
resourcesEnabled: true,
|
||||
pingEnabled: true,
|
||||
speedEnabled: true,
|
||||
intervalSec: 300,
|
||||
probeIntervalSec: 15,
|
||||
speedIntervalSec: 60,
|
||||
speedIntervalSec: 3600,
|
||||
retentionDays: 14,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
@@ -105,38 +94,101 @@ export function parsePing(results: Array<{ time?: string; status?: string; sent?
|
||||
typeof lossSource === "number" && Number.isFinite(lossSource)
|
||||
? Math.max(0, Math.min(100, lossSource))
|
||||
: 100
|
||||
const status: "up" | "warn" | "down" = loss >= 100 ? "down" : (loss > 1 || (avgRtt ?? 0) > 60 ? "warn" : "up")
|
||||
const status: "up" | "warn" | "down" =
|
||||
loss >= 100 ? "down" : (loss > 1 || (avgRtt ?? 0) > PING_PROBE_WARN_RTT_MS ? "warn" : "up")
|
||||
return { avgRtt, loss, status }
|
||||
}
|
||||
|
||||
export async function collectUptimeOnce(): Promise<void> {
|
||||
if (collecting) return
|
||||
collecting = true
|
||||
/** Узлы с REST-опросом для uptime: включённые в каталоге или любой Home Router (см. /servers — HR часто «выкл.» при доступе только из LAN). */
|
||||
function serversForUptimeRosPoll() {
|
||||
return db
|
||||
.select()
|
||||
.from(servers)
|
||||
.where(or(eq(servers.enabled, true), eq(servers.type, "home-router")))
|
||||
.all()
|
||||
}
|
||||
|
||||
function lastProbeSampleTimes(): Map<string, string> {
|
||||
const rows = db.select({
|
||||
probeId: uptimeProbeSamples.probeId,
|
||||
lastAt: max(uptimeProbeSamples.sampledAt),
|
||||
}).from(uptimeProbeSamples).groupBy(uptimeProbeSamples.probeId).all()
|
||||
const m = new Map<string, string>()
|
||||
for (const r of rows) {
|
||||
if (r.lastAt) m.set(r.probeId, String(r.lastAt))
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
/** Сбор метрик ресурсов (CPU/RAM/…) по всем включённым серверам. */
|
||||
export async function collectResourceSamplesOnce(): Promise<ResourcesRunSnapshot> {
|
||||
const sampledAt = new Date().toISOString()
|
||||
if (collectingResources) {
|
||||
return {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "uptime_resources",
|
||||
sampledAt,
|
||||
skipped: true,
|
||||
servers: [],
|
||||
}
|
||||
}
|
||||
collectingResources = true
|
||||
const started = Date.now()
|
||||
const now = new Date().toISOString()
|
||||
const now = sampledAt
|
||||
const settings = getSettings()
|
||||
const snapshot: ResourcesRunSnapshot = {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "uptime_resources",
|
||||
sampledAt: now,
|
||||
servers: [],
|
||||
}
|
||||
|
||||
try {
|
||||
const enabledServers = db.select().from(servers).where(eq(servers.enabled, true)).all()
|
||||
const resourceTargets = serversForUptimeRosPoll()
|
||||
|
||||
for (const s of enabledServers) {
|
||||
for (const s of resourceTargets) {
|
||||
const name = s.name || s.host
|
||||
try {
|
||||
const client = MikrotikClient.fromServer(s)
|
||||
const resource = await client.getResource()
|
||||
const cpuLoad = parseRosCpuLoadPercent(resource["cpu-load"])
|
||||
const freeMem = parseRosDataSizeBytes(resource["free-memory"])
|
||||
const totalMem = parseRosDataSizeBytes(resource["total-memory"])
|
||||
const freeHdd = parseRosDataSizeBytes(resource["free-hdd-space"])
|
||||
const totalHdd = parseRosDataSizeBytes(resource["total-hdd-space"])
|
||||
const uptimeSeconds = parseRosUptimeToSeconds(resource["uptime"])
|
||||
db.insert(uptimeResourceSamples).values({
|
||||
serverId: s.id,
|
||||
sampledAt: now,
|
||||
status: "online",
|
||||
cpuLoad: toNum(resource["cpu-load"]),
|
||||
freeMemory: toNum(resource["free-memory"]),
|
||||
totalMemory: toNum(resource["total-memory"]),
|
||||
freeHddSpace: toNum(resource["free-hdd-space"]),
|
||||
totalHddSpace: toNum(resource["total-hdd-space"]),
|
||||
uptimeSeconds: parseDurationToSeconds(resource["uptime"]),
|
||||
cpuLoad,
|
||||
freeMemory: freeMem,
|
||||
totalMemory: totalMem,
|
||||
freeHddSpace: freeHdd,
|
||||
totalHddSpace: totalHdd,
|
||||
uptimeSeconds,
|
||||
boardName: String(resource["board-name"] ?? ""),
|
||||
rosVersion: String(resource["version"] ?? ""),
|
||||
}).run()
|
||||
} catch {
|
||||
const memUsedMb = totalMem > 0 ? Math.round((totalMem - freeMem) / (1024 * 1024)) : 0
|
||||
const memTotalMb = totalMem > 0 ? Math.round(totalMem / (1024 * 1024)) : 0
|
||||
snapshot.servers.push({
|
||||
serverId: s.id,
|
||||
name,
|
||||
host: s.host,
|
||||
status: "online",
|
||||
cpuLoadPct: cpuLoad,
|
||||
memUsedMb,
|
||||
memTotalMb,
|
||||
memUsedPct: totalMem > 0 ? Math.round(((totalMem - freeMem) / totalMem) * 100) : undefined,
|
||||
diskFreeMb: freeHdd > 0 ? Math.round(freeHdd / (1024 * 1024)) : 0,
|
||||
diskTotalMb: totalHdd > 0 ? Math.round(totalHdd / (1024 * 1024)) : 0,
|
||||
uptimeSeconds,
|
||||
boardName: String(resource["board-name"] ?? "") || undefined,
|
||||
rosVersion: String(resource["version"] ?? "") || undefined,
|
||||
})
|
||||
} catch (err) {
|
||||
console.warn(`[uptime-collector] resource serverId=${s.id} (${s.host}):`, err instanceof Error ? err.message : err)
|
||||
db.insert(uptimeResourceSamples).values({
|
||||
serverId: s.id,
|
||||
sampledAt: now,
|
||||
@@ -150,12 +202,180 @@ export async function collectUptimeOnce(): Promise<void> {
|
||||
boardName: "",
|
||||
rosVersion: "",
|
||||
}).run()
|
||||
snapshot.servers.push({
|
||||
serverId: s.id,
|
||||
name,
|
||||
host: s.host,
|
||||
status: "offline",
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
cleanup(Math.max(1, settings.retentionDays))
|
||||
db.update(uptimeSettings).set({
|
||||
lastCollectedAt: now,
|
||||
lastDurationMs: Date.now() - started,
|
||||
lastError: "",
|
||||
updatedAt: now,
|
||||
}).where(eq(uptimeSettings.id, 1)).run()
|
||||
} catch (e) {
|
||||
snapshot.fatalError = e instanceof Error ? e.message : String(e)
|
||||
db.update(uptimeSettings).set({
|
||||
lastCollectedAt: now,
|
||||
lastDurationMs: Date.now() - started,
|
||||
lastError: e instanceof Error ? e.message : String(e),
|
||||
updatedAt: now,
|
||||
}).where(eq(uptimeSettings.id, 1)).run()
|
||||
} finally {
|
||||
collectingResources = false
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
/** Ping-пробы: учитывается interval_sec у пробы или глобальный probe_interval_sec. */
|
||||
export async function collectPingProbesOnce(): Promise<PingRunSnapshot> {
|
||||
const sampledAt = new Date().toISOString()
|
||||
if (collectingPing) {
|
||||
return {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "uptime_ping",
|
||||
sampledAt,
|
||||
skipped: true,
|
||||
probes: [],
|
||||
}
|
||||
}
|
||||
collectingPing = true
|
||||
const started = Date.now()
|
||||
const now = sampledAt
|
||||
const settings = getSettings()
|
||||
const defaultProbeSec = Math.max(5, settings.probeIntervalSec ?? settings.intervalSec)
|
||||
const lastByProbe = lastProbeSampleTimes()
|
||||
const nowMs = Date.now()
|
||||
const snapshot: PingRunSnapshot = {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "uptime_ping",
|
||||
sampledAt: now,
|
||||
probes: [],
|
||||
skippedByInterval: 0,
|
||||
}
|
||||
|
||||
try {
|
||||
const rosPollServers = serversForUptimeRosPoll()
|
||||
const probes = db.select().from(uptimeProbes).where(eq(uptimeProbes.enabled, true)).orderBy(asc(uptimeProbes.sortOrder)).all()
|
||||
|
||||
for (const p of probes) {
|
||||
const src = enabledServers.find((s) => s.id === p.srcServerId)
|
||||
const effectiveSec = (p.intervalSec != null && p.intervalSec > 0) ? p.intervalSec : defaultProbeSec
|
||||
const lastIso = lastByProbe.get(p.id)
|
||||
if (lastIso) {
|
||||
const elapsed = nowMs - Date.parse(lastIso)
|
||||
if (elapsed < effectiveSec * 1000) {
|
||||
snapshot.skippedByInterval = (snapshot.skippedByInterval ?? 0) + 1
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
const src = rosPollServers.find((s) => s.id === p.srcServerId)
|
||||
if (!src) continue
|
||||
const srcServerName = src.name || src.host
|
||||
try {
|
||||
const client = MikrotikClient.fromServer(src)
|
||||
const results = await client.ping(p.target, 4, p.srcInterface || undefined)
|
||||
const parsed = parsePing(results)
|
||||
db.insert(uptimeProbeSamples).values({
|
||||
probeId: p.id,
|
||||
sampledAt: now,
|
||||
rttMs: parsed.avgRtt,
|
||||
lossPct: parsed.loss,
|
||||
status: parsed.status,
|
||||
}).run()
|
||||
snapshot.probes.push({
|
||||
probeId: p.id,
|
||||
name: p.name,
|
||||
target: p.target,
|
||||
srcServerId: p.srcServerId,
|
||||
srcServerName,
|
||||
srcInterface: p.srcInterface || "",
|
||||
ok: true,
|
||||
rttMs: parsed.avgRtt,
|
||||
lossPct: parsed.loss,
|
||||
status: parsed.status,
|
||||
})
|
||||
} catch (err) {
|
||||
db.insert(uptimeProbeSamples).values({
|
||||
probeId: p.id,
|
||||
sampledAt: now,
|
||||
rttMs: null,
|
||||
lossPct: 100,
|
||||
status: "down",
|
||||
}).run()
|
||||
snapshot.probes.push({
|
||||
probeId: p.id,
|
||||
name: p.name,
|
||||
target: p.target,
|
||||
srcServerId: p.srcServerId,
|
||||
srcServerName,
|
||||
srcInterface: p.srcInterface || "",
|
||||
ok: false,
|
||||
rttMs: null,
|
||||
lossPct: 100,
|
||||
status: "down",
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
cleanup(Math.max(1, settings.retentionDays))
|
||||
db.update(uptimeSettings).set({
|
||||
lastCollectedAt: now,
|
||||
lastDurationMs: Date.now() - started,
|
||||
lastError: "",
|
||||
updatedAt: now,
|
||||
}).where(eq(uptimeSettings.id, 1)).run()
|
||||
} catch (e) {
|
||||
snapshot.fatalError = e instanceof Error ? e.message : String(e)
|
||||
db.update(uptimeSettings).set({
|
||||
lastCollectedAt: now,
|
||||
lastDurationMs: Date.now() - started,
|
||||
lastError: e instanceof Error ? e.message : String(e),
|
||||
updatedAt: now,
|
||||
}).where(eq(uptimeSettings.id, 1)).run()
|
||||
} finally {
|
||||
collectingPing = false
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
const MAX_PROBE_IDS_PER_GROUP_COLLECT = 64
|
||||
|
||||
/**
|
||||
* Немедленный ping по указанным пробам (без учёта interval_sec) — запись в uptime_probe_samples.
|
||||
* Используется кнопкой «Обновить» у группы «сервер + назначение» на вкладке проб.
|
||||
*/
|
||||
export async function collectPingForProbeIds(probeIds: string[]): Promise<{ polled: number }> {
|
||||
const uniqueIds = [...new Set(probeIds.filter((id) => typeof id === "string" && id.trim() !== ""))].slice(
|
||||
0,
|
||||
MAX_PROBE_IDS_PER_GROUP_COLLECT,
|
||||
)
|
||||
if (uniqueIds.length === 0) return { polled: 0 }
|
||||
|
||||
if (collectingPing) {
|
||||
const e = new Error("Сбор ping уже выполняется, подождите")
|
||||
;(e as Error & { statusCode?: number }).statusCode = 409
|
||||
throw e
|
||||
}
|
||||
|
||||
collectingPing = true
|
||||
const now = new Date().toISOString()
|
||||
const settings = getSettings()
|
||||
let polled = 0
|
||||
|
||||
try {
|
||||
const rosPollServers = serversForUptimeRosPoll()
|
||||
const rows = db.select().from(uptimeProbes).where(inArray(uptimeProbes.id, uniqueIds)).all()
|
||||
|
||||
for (const p of rows) {
|
||||
const src = rosPollServers.find((s) => s.id === p.srcServerId)
|
||||
if (!src) continue
|
||||
try {
|
||||
const client = MikrotikClient.fromServer(src)
|
||||
@@ -177,64 +397,74 @@ export async function collectUptimeOnce(): Promise<void> {
|
||||
status: "down",
|
||||
}).run()
|
||||
}
|
||||
polled += 1
|
||||
}
|
||||
|
||||
cleanup(Math.max(1, settings.retentionDays))
|
||||
db.update(uptimeSettings).set({
|
||||
lastCollectedAt: now,
|
||||
lastDurationMs: Date.now() - started,
|
||||
lastError: "",
|
||||
updatedAt: now,
|
||||
}).where(eq(uptimeSettings.id, 1)).run()
|
||||
} catch (e) {
|
||||
db.update(uptimeSettings).set({
|
||||
lastCollectedAt: now,
|
||||
lastDurationMs: Date.now() - started,
|
||||
lastError: e instanceof Error ? e.message : String(e),
|
||||
updatedAt: now,
|
||||
}).where(eq(uptimeSettings.id, 1)).run()
|
||||
return { polled }
|
||||
} finally {
|
||||
collecting = false
|
||||
collectingPing = false
|
||||
}
|
||||
}
|
||||
|
||||
export function restartUptimeCollector() {
|
||||
const s = getSettings()
|
||||
if (timer) clearInterval(timer)
|
||||
timer = null
|
||||
if (!s.enabled) return
|
||||
const intervalMs = Math.max(5, s.probeIntervalSec || s.intervalSec) * 1000
|
||||
timer = setInterval(() => { void collectUptimeOnce() }, intervalMs)
|
||||
/** Совместимость: один проход ресурсы + ping (для ручного вызова без планировщика). */
|
||||
export async function collectUptimeOnce(): Promise<void> {
|
||||
await collectResourceSamplesOnce()
|
||||
await collectPingProbesOnce()
|
||||
}
|
||||
|
||||
export function stopUptimeCollector() {
|
||||
if (timer) clearInterval(timer)
|
||||
timer = null
|
||||
}
|
||||
/** no-op: таймеры управляются планировщиком */
|
||||
export function restartUptimeCollector() {}
|
||||
|
||||
export function stopUptimeCollector() {}
|
||||
|
||||
export function readUptimeSettings() {
|
||||
return getSettings()
|
||||
}
|
||||
|
||||
export function updateUptimeSettings(patch: {
|
||||
/** Устаревший: если передан без per-job флагов — выставляет все три job-флага. */
|
||||
enabled?: boolean
|
||||
resourcesEnabled?: boolean
|
||||
pingEnabled?: boolean
|
||||
speedEnabled?: boolean
|
||||
/** Интервал сбора метрик ресурсов (сек). */
|
||||
intervalSec?: number
|
||||
/** Интервал тика ping и значение по умолчанию для проб без своего interval_sec. */
|
||||
probeIntervalSec?: number
|
||||
speedIntervalSec?: number
|
||||
retentionDays?: number
|
||||
}) {
|
||||
const prev = getSettings()
|
||||
const nextProbeInterval = patch.probeIntervalSec ?? patch.intervalSec ?? prev.probeIntervalSec ?? prev.intervalSec
|
||||
const anyJobFlag =
|
||||
patch.resourcesEnabled !== undefined
|
||||
|| patch.pingEnabled !== undefined
|
||||
|| patch.speedEnabled !== undefined
|
||||
|
||||
let resourcesEnabled = patch.resourcesEnabled ?? prev.resourcesEnabled ?? prev.enabled
|
||||
let pingEnabled = patch.pingEnabled ?? prev.pingEnabled ?? prev.enabled
|
||||
let speedEnabled = patch.speedEnabled ?? prev.speedEnabled ?? prev.enabled
|
||||
|
||||
if (patch.enabled !== undefined && !anyJobFlag) {
|
||||
resourcesEnabled = patch.enabled
|
||||
pingEnabled = patch.enabled
|
||||
speedEnabled = patch.enabled
|
||||
}
|
||||
|
||||
const masterEnabled = Boolean(resourcesEnabled || pingEnabled || speedEnabled)
|
||||
|
||||
const next = {
|
||||
enabled: patch.enabled ?? prev.enabled,
|
||||
intervalSec: nextProbeInterval,
|
||||
probeIntervalSec: nextProbeInterval,
|
||||
enabled: masterEnabled,
|
||||
resourcesEnabled,
|
||||
pingEnabled,
|
||||
speedEnabled,
|
||||
intervalSec: patch.intervalSec ?? prev.intervalSec,
|
||||
probeIntervalSec: patch.probeIntervalSec ?? prev.probeIntervalSec,
|
||||
speedIntervalSec: patch.speedIntervalSec ?? prev.speedIntervalSec ?? 60,
|
||||
retentionDays: patch.retentionDays ?? prev.retentionDays,
|
||||
updatedAt: new Date().toISOString(),
|
||||
}
|
||||
db.update(uptimeSettings).set(next).where(eq(uptimeSettings.id, 1)).run()
|
||||
restartUptimeCollector()
|
||||
return readUptimeSettings()
|
||||
}
|
||||
|
||||
@@ -260,20 +490,20 @@ export function replaceProbes(rows: Array<{
|
||||
probeFilter: string
|
||||
enabled: boolean
|
||||
showOnDashboard?: boolean
|
||||
intervalSec?: number
|
||||
}>) {
|
||||
const now = new Date().toISOString()
|
||||
const existing = db.select({ id: uptimeProbes.id }).from(uptimeProbes).all()
|
||||
const nextIds = new Set(rows.map((r) => r.id))
|
||||
|
||||
// Remove only probes that user actually deleted (their samples are removed by FK cascade).
|
||||
for (const p of existing) {
|
||||
if (nextIds.has(p.id)) continue
|
||||
db.delete(uptimeProbes).where(eq(uptimeProbes.id, p.id)).run()
|
||||
}
|
||||
|
||||
// Upsert rows in-place to preserve sample history for unchanged probe ids.
|
||||
for (let i = 0; i < rows.length; i += 1) {
|
||||
const r = rows[i]
|
||||
const intervalSec = r.intervalSec != null && r.intervalSec > 0 ? Math.max(5, Math.floor(r.intervalSec)) : 0
|
||||
const patch = {
|
||||
srcServerId: r.srcServerId,
|
||||
srcInterface: r.srcInterface || "",
|
||||
@@ -282,6 +512,7 @@ export function replaceProbes(rows: Array<{
|
||||
probeFilter: r.probeFilter || "—",
|
||||
enabled: r.enabled,
|
||||
showOnDashboard: r.showOnDashboard ?? false,
|
||||
intervalSec,
|
||||
sortOrder: i,
|
||||
updatedAt: now,
|
||||
}
|
||||
@@ -311,4 +542,3 @@ export function readResourceSamplesSince(sinceIso: string, serverId: number) {
|
||||
.orderBy(asc(uptimeResourceSamples.sampledAt))
|
||||
.all()
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
import { asc, eq } from "drizzle-orm"
|
||||
import { db } from "../db/index.js"
|
||||
import { servers, uptimeSpeedProbes, uptimeSpeedTestRuns } from "../db/schema.js"
|
||||
import type { SpeedRunSnapshot, SpeedScheduledRunSnapshot } from "../types/scheduler-run-snapshot.js"
|
||||
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
|
||||
import type { RosIpAddress } from "../types/server.js"
|
||||
import { withBtestNodeLocks } from "./btest-node-locks.js"
|
||||
import { MikrotikClient } from "./mikrotik.js"
|
||||
import { parsePing } from "./uptime-collector.js"
|
||||
|
||||
function firstUsableIpOnInterface(rows: RosIpAddress[], iface: string): string {
|
||||
const name = iface.trim()
|
||||
const candidates = rows.filter((a) => {
|
||||
if (String(a.interface ?? "").trim() !== name) return false
|
||||
if (String(a.disabled ?? "false").toLowerCase() === "true") return false
|
||||
if (String(a.invalid ?? "false").toLowerCase() === "true") return false
|
||||
const addr = String(a.address ?? "").trim()
|
||||
return addr.includes("/")
|
||||
})
|
||||
candidates.sort((a, b) => {
|
||||
const ad = String(a.dynamic ?? "false").toLowerCase() === "true" ? 1 : 0
|
||||
const bd = String(b.dynamic ?? "false").toLowerCase() === "true" ? 1 : 0
|
||||
return ad - bd
|
||||
})
|
||||
const raw = candidates[0]?.address
|
||||
return raw ? String(raw).split("/")[0]?.trim() ?? "" : ""
|
||||
}
|
||||
|
||||
function parseRateToMbps(raw: unknown): number {
|
||||
const txt = String(raw ?? "").trim().toLowerCase()
|
||||
if (!txt) return 0
|
||||
const m = txt.match(/^(\d+(?:\.\d+)?)\s*(g|m|k)?(?:bps|bit\/s|bits\/s)?$/)
|
||||
if (m) {
|
||||
const n = Number.parseFloat(m[1])
|
||||
const u = m[2] ?? ""
|
||||
if (!Number.isFinite(n)) return 0
|
||||
if (u === "g") return n * 1000
|
||||
if (u === "m") return n
|
||||
if (u === "k") return n / 1000
|
||||
return n / 1_000_000
|
||||
}
|
||||
const n = Number.parseFloat(txt.replace(/[^\d.]/g, ""))
|
||||
if (!Number.isFinite(n)) return 0
|
||||
return n > 10000 ? n / 1_000_000 : n
|
||||
}
|
||||
|
||||
export interface SpeedTestRunParams {
|
||||
runId?: string
|
||||
probeId?: string
|
||||
srcServerId: number
|
||||
dstServerId: number
|
||||
srcInterface?: string
|
||||
dstInterface?: string
|
||||
protocol?: "tcp" | "udp"
|
||||
direction?: "transmit" | "receive" | "both"
|
||||
durationSec?: number
|
||||
}
|
||||
|
||||
export interface SpeedTestRunResult {
|
||||
runId: string
|
||||
srcInterface: string | null
|
||||
dstInterface: string | null
|
||||
address: string
|
||||
srcAddress: string
|
||||
dstAddress: string
|
||||
srcInterfaceAddress: string | null
|
||||
dstInterfaceAddress: string | null
|
||||
protocol: "tcp" | "udp"
|
||||
direction: "transmit" | "receive" | "both"
|
||||
durationSec: number
|
||||
txAvgMbps: number
|
||||
rxAvgMbps: number
|
||||
txSeries: number[]
|
||||
rxSeries: number[]
|
||||
raw: Array<Record<string, string>>
|
||||
afterBtPing: {
|
||||
rttMs: number | null
|
||||
lossPct: number | null
|
||||
error: string | null
|
||||
}
|
||||
}
|
||||
|
||||
function humanizeSpeedError(raw: string): string {
|
||||
return /timed out|socket hang up|ECONNRESET/i.test(raw)
|
||||
? "BTTest timeout/socket hang up: проверь доступность destination, /tool bandwidth-server, firewall и корректность интерфейсов"
|
||||
: raw
|
||||
}
|
||||
|
||||
/** Полный цикл bandwidth-test + after-ping + запись в БД. Вызывать внутри withBtestNodeLocks при необходимости — здесь lock уже снаружи. */
|
||||
export async function executeBandwidthSpeedTestInner(params: SpeedTestRunParams): Promise<SpeedTestRunResult> {
|
||||
const srcId = params.srcServerId
|
||||
const dstId = params.dstServerId
|
||||
const src = db.select().from(servers).where(eq(servers.id, srcId)).limit(1).all()[0]
|
||||
const dst = db.select().from(servers).where(eq(servers.id, dstId)).limit(1).all()[0]
|
||||
if (!src || !dst) throw new Error("Server not found")
|
||||
|
||||
const protocol = params.protocol === "udp" ? "udp" : "tcp"
|
||||
const direction = params.direction === "transmit" || params.direction === "receive" ? params.direction : "both"
|
||||
const durationSec = Math.max(3, Number.parseInt(String(params.durationSec ?? ""), 10) || 10)
|
||||
const srcInterface = String(params.srcInterface ?? "").trim()
|
||||
const dstInterface = String(params.dstInterface ?? "").trim()
|
||||
const srcClient = MikrotikClient.fromServer(src)
|
||||
const dstAddressRows = dstInterface
|
||||
? await MikrotikClient.fromServer(dst).getIpAddresses().catch(() => [])
|
||||
: []
|
||||
const srcAddressRows = srcInterface
|
||||
? await srcClient.getIpAddresses().catch(() => [])
|
||||
: []
|
||||
const dstAddress = dstInterface ? firstUsableIpOnInterface(dstAddressRows, dstInterface) : dst.host
|
||||
if (dstInterface && !dstAddress) {
|
||||
throw new Error(`На интерфейсе назначения '${dstInterface}' нет активного IP (проверьте /ip/address — не disabled).`)
|
||||
}
|
||||
const initialAddress = dstAddress || dst.host
|
||||
const srcAddress = srcInterface ? firstUsableIpOnInterface(srcAddressRows, srcInterface) : src.host
|
||||
const srcInterfaceAddress = srcInterface ? (srcAddress || null) : null
|
||||
const dstInterfaceAddress = dstInterface ? (dstAddress || null) : null
|
||||
|
||||
const rows = await srcClient.bandwidthTest({
|
||||
address: initialAddress,
|
||||
user: dst.username,
|
||||
password: dst.password,
|
||||
protocol,
|
||||
direction,
|
||||
durationSec,
|
||||
})
|
||||
const txSeries = rows
|
||||
.map((r) => parseRateToMbps(r["tx-current"] ?? r["tx-10-second-average"] ?? r["tx-total-average"]))
|
||||
.filter((v) => Number.isFinite(v) && v >= 0)
|
||||
const rxSeries = rows
|
||||
.map((r) => parseRateToMbps(r["rx-current"] ?? r["rx-10-second-average"] ?? r["rx-total-average"]))
|
||||
.filter((v) => Number.isFinite(v) && v >= 0)
|
||||
const last = rows[rows.length - 1] ?? {}
|
||||
const txAvg = parseRateToMbps(last["tx-total-average"] ?? last["tx-10-second-average"])
|
||||
const rxAvg = parseRateToMbps(last["rx-total-average"] ?? last["rx-10-second-average"])
|
||||
const txFinal = txAvg || (txSeries.length ? txSeries.reduce((a, b) => a + b, 0) / txSeries.length : 0)
|
||||
const rxFinal = rxAvg || (rxSeries.length ? rxSeries.reduce((a, b) => a + b, 0) / rxSeries.length : 0)
|
||||
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 3000))
|
||||
|
||||
const pingNow = new Date().toISOString()
|
||||
let afterBtPing: SpeedTestRunResult["afterBtPing"] = { rttMs: null, lossPct: null, error: null }
|
||||
try {
|
||||
const pingRows = await srcClient.ping(initialAddress, 4, srcInterface || undefined, { interval: "1s" })
|
||||
const pr = parsePing(pingRows as Parameters<typeof parsePing>[0])
|
||||
afterBtPing = { rttMs: pr.avgRtt, lossPct: pr.loss, error: null }
|
||||
} catch (pe) {
|
||||
afterBtPing = {
|
||||
rttMs: null,
|
||||
lossPct: null,
|
||||
error: pe instanceof Error ? pe.message : String(pe),
|
||||
}
|
||||
}
|
||||
|
||||
const probeId = String(params.probeId ?? "").trim()
|
||||
const runId = String(params.runId ?? "").trim() || `sr-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
|
||||
if (probeId) {
|
||||
db.update(uptimeSpeedProbes).set({
|
||||
lastRunAt: pingNow,
|
||||
lastTxAvgMbps: txFinal,
|
||||
lastRxAvgMbps: rxFinal,
|
||||
lastStatus: "done",
|
||||
lastError: "",
|
||||
lastPingRttMs: afterBtPing.rttMs,
|
||||
lastPingLossPct: afterBtPing.lossPct,
|
||||
lastPingAt: pingNow,
|
||||
lastPingError: afterBtPing.error ?? "",
|
||||
updatedAt: pingNow,
|
||||
}).where(eq(uptimeSpeedProbes.id, probeId)).run()
|
||||
}
|
||||
db.insert(uptimeSpeedTestRuns).values({
|
||||
id: runId,
|
||||
probeId: probeId || null,
|
||||
srcServerId: srcId,
|
||||
dstServerId: dstId,
|
||||
srcInterface: srcInterface || "",
|
||||
dstInterface: dstInterface || "",
|
||||
srcAddress: srcAddress || src.host,
|
||||
dstAddress: initialAddress,
|
||||
srcInterfaceAddress,
|
||||
dstInterfaceAddress,
|
||||
protocol,
|
||||
direction,
|
||||
durationSec,
|
||||
txAvgMbps: txFinal,
|
||||
rxAvgMbps: rxFinal,
|
||||
pingRttMs: afterBtPing.rttMs,
|
||||
pingLossPct: afterBtPing.lossPct,
|
||||
pingError: afterBtPing.error ?? null,
|
||||
status: "done",
|
||||
error: null,
|
||||
createdAt: pingNow,
|
||||
}).run()
|
||||
|
||||
return {
|
||||
runId,
|
||||
srcInterface: srcInterface || null,
|
||||
dstInterface: dstInterface || null,
|
||||
address: initialAddress,
|
||||
srcAddress: srcAddress || src.host,
|
||||
dstAddress: initialAddress,
|
||||
srcInterfaceAddress,
|
||||
dstInterfaceAddress,
|
||||
protocol,
|
||||
direction,
|
||||
durationSec,
|
||||
txAvgMbps: txFinal,
|
||||
rxAvgMbps: rxFinal,
|
||||
txSeries,
|
||||
rxSeries,
|
||||
raw: rows,
|
||||
afterBtPing,
|
||||
}
|
||||
}
|
||||
|
||||
export async function runBandwidthSpeedTest(params: SpeedTestRunParams): Promise<SpeedTestRunResult> {
|
||||
const srcId = params.srcServerId
|
||||
const dstId = params.dstServerId
|
||||
if (srcId === dstId) throw new Error("Source and destination must be different")
|
||||
return withBtestNodeLocks([srcId, dstId], () => executeBandwidthSpeedTestInner(params))
|
||||
}
|
||||
|
||||
export function recordSpeedTestFailure(params: {
|
||||
probeId?: string
|
||||
runId?: string
|
||||
srcServerId: number
|
||||
dstServerId: number
|
||||
srcInterface?: string
|
||||
dstInterface?: string
|
||||
protocol?: "tcp" | "udp"
|
||||
direction?: "transmit" | "receive" | "both"
|
||||
durationSec?: number
|
||||
message: string
|
||||
}) {
|
||||
const probeId = String(params.probeId ?? "").trim()
|
||||
const message = humanizeSpeedError(params.message)
|
||||
if (probeId) {
|
||||
db.update(uptimeSpeedProbes).set({
|
||||
lastRunAt: new Date().toISOString(),
|
||||
lastStatus: "error",
|
||||
lastError: message,
|
||||
updatedAt: new Date().toISOString(),
|
||||
}).where(eq(uptimeSpeedProbes.id, probeId)).run()
|
||||
}
|
||||
db.insert(uptimeSpeedTestRuns).values({
|
||||
id: String(params.runId ?? "").trim() || `sr-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
||||
probeId: probeId || null,
|
||||
srcServerId: params.srcServerId,
|
||||
dstServerId: params.dstServerId,
|
||||
srcInterface: String(params.srcInterface ?? "").trim(),
|
||||
dstInterface: String(params.dstInterface ?? "").trim(),
|
||||
srcAddress: null,
|
||||
dstAddress: null,
|
||||
srcInterfaceAddress: null,
|
||||
dstInterfaceAddress: null,
|
||||
protocol: params.protocol === "udp" ? "udp" : "tcp",
|
||||
direction: params.direction === "transmit" || params.direction === "receive" ? params.direction : "both",
|
||||
durationSec: Math.max(3, Number.parseInt(String(params.durationSec ?? ""), 10) || 10),
|
||||
txAvgMbps: null,
|
||||
rxAvgMbps: null,
|
||||
pingRttMs: null,
|
||||
pingLossPct: null,
|
||||
pingError: null,
|
||||
status: "error",
|
||||
error: message,
|
||||
createdAt: new Date().toISOString(),
|
||||
}).run()
|
||||
}
|
||||
|
||||
function serverDisplayName(id: number): string {
|
||||
const s = db.select().from(servers).where(eq(servers.id, id)).limit(1).all()[0]
|
||||
return s ? (s.name || s.host) : `#${id}`
|
||||
}
|
||||
|
||||
/** Фоновый прогон всех включённых speed-проб (по очереди). */
|
||||
export async function runScheduledSpeedProbesOnce(): Promise<SpeedScheduledRunSnapshot> {
|
||||
const sampledAt = new Date().toISOString()
|
||||
const runs: SpeedRunSnapshot[] = []
|
||||
const rows = db.select().from(uptimeSpeedProbes).orderBy(asc(uptimeSpeedProbes.sortOrder)).all()
|
||||
const enabled = rows.filter((r) => r.enabled !== false && r.srcServerId !== r.dstServerId)
|
||||
const protocol = (p: (typeof enabled)[number]) => (p.protocol === "udp" ? "udp" : "tcp")
|
||||
const direction = (p: (typeof enabled)[number]) =>
|
||||
p.direction === "transmit" || p.direction === "receive" ? p.direction : "both"
|
||||
const durationSec = (p: (typeof enabled)[number]) => Math.max(3, p.durationSec || 10)
|
||||
|
||||
for (const p of enabled) {
|
||||
try {
|
||||
const r = await runBandwidthSpeedTest({
|
||||
probeId: p.id,
|
||||
srcServerId: p.srcServerId,
|
||||
dstServerId: p.dstServerId,
|
||||
srcInterface: p.srcInterface || undefined,
|
||||
dstInterface: p.dstInterface || undefined,
|
||||
protocol: protocol(p) as "tcp" | "udp",
|
||||
direction: direction(p) as "transmit" | "receive" | "both",
|
||||
durationSec: durationSec(p),
|
||||
})
|
||||
runs.push({
|
||||
probeId: p.id,
|
||||
srcServerId: p.srcServerId,
|
||||
srcServerName: serverDisplayName(p.srcServerId),
|
||||
dstServerId: p.dstServerId,
|
||||
dstServerName: serverDisplayName(p.dstServerId),
|
||||
srcInterface: r.srcInterface ?? "",
|
||||
dstInterface: r.dstInterface ?? "",
|
||||
protocol: r.protocol,
|
||||
direction: r.direction,
|
||||
durationSec: r.durationSec,
|
||||
ok: true,
|
||||
txAvgMbps: r.txAvgMbps,
|
||||
rxAvgMbps: r.rxAvgMbps,
|
||||
pingRttMs: r.afterBtPing.rttMs,
|
||||
pingLossPct: r.afterBtPing.lossPct,
|
||||
pingError: r.afterBtPing.error,
|
||||
})
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e)
|
||||
recordSpeedTestFailure({
|
||||
probeId: p.id,
|
||||
srcServerId: p.srcServerId,
|
||||
dstServerId: p.dstServerId,
|
||||
srcInterface: p.srcInterface,
|
||||
dstInterface: p.dstInterface,
|
||||
protocol: protocol(p),
|
||||
direction: direction(p),
|
||||
durationSec: p.durationSec,
|
||||
message: msg,
|
||||
})
|
||||
runs.push({
|
||||
probeId: p.id,
|
||||
srcServerId: p.srcServerId,
|
||||
srcServerName: serverDisplayName(p.srcServerId),
|
||||
dstServerId: p.dstServerId,
|
||||
dstServerName: serverDisplayName(p.dstServerId),
|
||||
srcInterface: p.srcInterface || "",
|
||||
dstInterface: p.dstInterface || "",
|
||||
protocol: protocol(p),
|
||||
direction: direction(p),
|
||||
durationSec: durationSec(p),
|
||||
ok: false,
|
||||
error: msg,
|
||||
})
|
||||
}
|
||||
}
|
||||
return {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "uptime_speed",
|
||||
sampledAt,
|
||||
runs,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user