464 lines
18 KiB
TypeScript
464 lines
18 KiB
TypeScript
import { executeGuardedApplicationWrite } from "./config-mutation-policy.js"
|
|
import {deliverTelegramAlert,type TelegramAlertInput} from "./alerts-telegram-delivery.js"
|
|
import {readCollectorSourceSnapshot} from "./alert-engine/signals-from-collectors.js"
|
|
import {createCollectorContextReader} from "./alert-engine/collector-context-reader.js"
|
|
import {monitoringHistory} from "./alert-engine/monitoring-history.js"
|
|
import {alertsConfigRevision} from "./alerts-revision.js"
|
|
import {readAlertsOverview,buildAlertsMeta} from "./alerts-overview.js"
|
|
import {executeAlertsConfigReplacement} from "./alerts-config-execution.js"
|
|
import {sendTelegramRequest} from "./alerts-telegram-send.js"
|
|
import {executeTelegramSettingsUpdate} from "./alerts-telegram-settings.js"
|
|
import { asc, desc, eq, sql } from "drizzle-orm"
|
|
import { db } from "../db/index.js"
|
|
import {
|
|
alertGroups,
|
|
alertHistory,
|
|
alertRuleConditions,
|
|
alertRuleTargets,
|
|
alertRules,
|
|
alertTelegramSettings,
|
|
servers,
|
|
uptimeProbes,
|
|
uptimeSettings,
|
|
serversApiPingSettings,
|
|
trafficSettings,
|
|
} from "../db/schema.js"
|
|
|
|
export type ApiAlertGroup = {
|
|
id: string
|
|
name: string
|
|
combineMode: "any" | "all"
|
|
enabled: boolean
|
|
cooldownOverride: (typeof alertRules.$inferSelect)["cooldown"] | null
|
|
}
|
|
|
|
export type ApiAlertRule = {
|
|
id: string
|
|
name: string
|
|
type: (typeof alertRules.$inferSelect)["type"]
|
|
/** Сводная строка для списка / поиска */
|
|
target: string
|
|
/** Все объекты правила (OR) */
|
|
targets: string[]
|
|
groupId: string | null
|
|
/** Сводная строка условий для списка */
|
|
condition: string
|
|
/** Все условия (OR); если пусто в БД — используется `condition` */
|
|
conditions: string[]
|
|
severity: (typeof alertRules.$inferSelect)["severity"]
|
|
enabled: boolean
|
|
cooldown: (typeof alertRules.$inferSelect)["cooldown"]
|
|
/** Секунды стабильности перед отправкой; null — выключено */
|
|
confirmStabilitySec: number | null
|
|
/** Политика отправки recovery для этого правила. */
|
|
recoveryMode: "always" | "never" | "conditional"
|
|
/** Доп. задержка подтверждения recovery при `conditional`. */
|
|
recoveryStabilitySec: number | null
|
|
chatId: string
|
|
lastFiredAt: string | null
|
|
}
|
|
|
|
export type ApiAlertHistoryEntry = {
|
|
id: string
|
|
ruleName: string
|
|
severity: (typeof alertHistory.$inferSelect)["severity"]
|
|
message: string
|
|
sent: boolean
|
|
firedAt: string
|
|
source: "db" | "live"
|
|
}
|
|
|
|
async function ensureTelegramRow() {
|
|
let row = (await db.select().from(alertTelegramSettings).where(eq(alertTelegramSettings.id, 1)).limit(1))[0]
|
|
if (!row) {
|
|
await db.insert(alertTelegramSettings).values({ id: 1 }).onConflictDoNothing({target:alertTelegramSettings.id})
|
|
row = (await db.select().from(alertTelegramSettings).where(eq(alertTelegramSettings.id, 1)).limit(1))[0]
|
|
}
|
|
return row!
|
|
}
|
|
|
|
type AlertsReadContext=Pick<typeof db,"select"|"execute">
|
|
|
|
async function lastFiredByRuleId(context:AlertsReadContext=db): Promise<Map<string, string>> {
|
|
const result = await context.execute(sql`SELECT rule_id AS "ruleId", MAX(fired_at) AS mx FROM alert_history WHERE rule_id IS NOT NULL AND rule_id != '' GROUP BY rule_id`)
|
|
const rows = result.rows as {ruleId:string;mx:string|Date}[]
|
|
const m = new Map<string, string>()
|
|
for (const r of rows) {
|
|
if (r.ruleId && r.mx) m.set(r.ruleId, r.mx instanceof Date?r.mx.toISOString():r.mx)
|
|
}
|
|
return m
|
|
}
|
|
|
|
async function targetsByRuleId(context:AlertsReadContext=db): Promise<Map<string, string[]>> {
|
|
const rows = await context
|
|
.select()
|
|
.from(alertRuleTargets)
|
|
.orderBy(asc(alertRuleTargets.sortIndex), asc(alertRuleTargets.id))
|
|
const m = new Map<string, string[]>()
|
|
for (const t of rows) {
|
|
const rid = t.ruleId
|
|
const arr = m.get(rid) ?? []
|
|
arr.push(t.target)
|
|
m.set(rid, arr)
|
|
}
|
|
return m
|
|
}
|
|
|
|
async function conditionsByRuleId(context:AlertsReadContext=db): Promise<Map<string, string[]>> {
|
|
const rows = await context
|
|
.select()
|
|
.from(alertRuleConditions)
|
|
.orderBy(asc(alertRuleConditions.sortIndex), asc(alertRuleConditions.id))
|
|
const m = new Map<string, string[]>()
|
|
for (const t of rows) {
|
|
const rid = t.ruleId
|
|
const arr = m.get(rid) ?? []
|
|
arr.push(t.conditionLine)
|
|
m.set(rid, arr)
|
|
}
|
|
return m
|
|
}
|
|
|
|
export async function listAlertGroups(context:AlertsReadContext=db): Promise<ApiAlertGroup[]> {
|
|
const rows = await context
|
|
.select()
|
|
.from(alertGroups)
|
|
.orderBy(asc(alertGroups.name))
|
|
return rows.map((g) => ({
|
|
id: g.id,
|
|
name: g.name,
|
|
combineMode: g.combineMode,
|
|
enabled: g.enabled,
|
|
cooldownOverride: g.cooldownOverride ?? null,
|
|
}))
|
|
}
|
|
|
|
export async function listAlertRules(context:AlertsReadContext=db): Promise<ApiAlertRule[]> {
|
|
const lf = await lastFiredByRuleId(context)
|
|
const tgtMap = await targetsByRuleId(context)
|
|
const condMap = await conditionsByRuleId(context)
|
|
const rows = await context.select().from(alertRules)
|
|
return rows.map((r) => {
|
|
const targets = tgtMap.get(r.id)
|
|
const ts = targets?.length ? targets : [r.target]
|
|
const condLines = condMap.get(r.id)
|
|
const cs = condLines?.length ? condLines : [r.condition]
|
|
return {
|
|
id: r.id,
|
|
name: r.name,
|
|
type: r.type,
|
|
target: r.target,
|
|
targets: ts,
|
|
groupId: r.groupId ?? null,
|
|
condition: r.condition,
|
|
conditions: cs,
|
|
severity: r.severity,
|
|
enabled: r.enabled,
|
|
cooldown: r.cooldown,
|
|
confirmStabilitySec:
|
|
r.confirmStabilitySec != null && r.confirmStabilitySec > 0 ? r.confirmStabilitySec : null,
|
|
recoveryMode:
|
|
r.recoveryMode === "never" || r.recoveryMode === "conditional" ? r.recoveryMode : "always",
|
|
recoveryStabilitySec:
|
|
r.recoveryStabilitySec != null && r.recoveryStabilitySec > 0 ? r.recoveryStabilitySec : null,
|
|
chatId: r.chatId ?? "",
|
|
lastFiredAt: lf.get(r.id) ?? null,
|
|
}
|
|
})
|
|
}
|
|
|
|
function summarizeTargets(targets: string[]): string {
|
|
const u = targets.map((t) => t.trim()).filter(Boolean)
|
|
if (u.length === 0) return ""
|
|
if (u.length <= 2) return u.join(", ")
|
|
return `${u[0]}, ${u[1]} +${u.length - 2}`
|
|
}
|
|
|
|
function summarizeConditionLines(lines: string[]): string {
|
|
const u = lines.map((t) => t.trim()).filter(Boolean)
|
|
if (u.length === 0) return "—"
|
|
if (u.length === 1) return u[0]!
|
|
if (u.length <= 3) return u.join(" · ")
|
|
return `${u[0]} · ${u[1]} · +${u.length - 2}`
|
|
}
|
|
|
|
export type ReplaceAlertRuleInput = {
|
|
id: string
|
|
name: string
|
|
type: ApiAlertRule["type"]
|
|
/** @deprecated используйте targets; если нет targets — берётся target */
|
|
target?: string
|
|
targets?: string[]
|
|
/** Сводная строка; если не передана — из conditions */
|
|
condition?: string
|
|
/** Несколько условий (OR); если пусто — используется `condition` */
|
|
conditions?: string[]
|
|
severity: ApiAlertRule["severity"]
|
|
enabled: boolean
|
|
cooldown: ApiAlertRule["cooldown"]
|
|
/** Секунды; 0 / undefined / null — без задержки подтверждения */
|
|
confirmStabilitySec?: number | null
|
|
recoveryMode?: "always" | "never" | "conditional"
|
|
recoveryStabilitySec?: number | null
|
|
chatId: string
|
|
groupId?: string | null
|
|
}
|
|
|
|
export type ReplaceAlertGroupInput = {
|
|
id: string
|
|
name: string
|
|
combineMode: "any" | "all"
|
|
enabled: boolean
|
|
cooldownOverride: ApiAlertRule["cooldown"] | null
|
|
}
|
|
|
|
/** Атомарно заменяет группы, правила и строки targets. */
|
|
type AlertsWriteContext=Parameters<Parameters<typeof db.transaction>[0]>[0]
|
|
export function replaceAlertsConfig(payload: Parameters<typeof replaceAlertsConfigImpl>[0]) { return executeGuardedApplicationWrite({ kind: "alerts-configuration", id: "1" }, "all", () => replaceAlertsConfigImpl(payload)) }
|
|
async function replaceAlertsConfigImpl(payload: { groups?: ReplaceAlertGroupInput[]; rules: ReplaceAlertRuleInput[];expectedRevision?:string }) {
|
|
const now=sql`now()`
|
|
return executeAlertsConfigReplacement<AlertsWriteContext>(payload, {
|
|
transaction:work=>db.transaction(work),
|
|
lock:async tx=>{const result=await tx.execute(sql`SELECT pg_try_advisory_xact_lock(8723103) AS acquired`);return typeof result.rows[0]?.acquired==="boolean"?result.rows[0].acquired:undefined},
|
|
revision:async tx=>alertsConfigRevision({groups:await listAlertGroups(tx),rules:await listAlertRules(tx)}),
|
|
groups:tx=>listAlertGroups(tx),
|
|
write:async(tx,{groups,rules})=>{
|
|
await tx.delete(alertRuleConditions)
|
|
await tx.delete(alertRuleTargets)
|
|
await tx.delete(alertRules)
|
|
await tx.delete(alertGroups)
|
|
for (const g of groups) {
|
|
await tx.insert(alertGroups).values({
|
|
id: g.id,
|
|
name: g.name.trim() || g.id,
|
|
combineMode: g.combineMode,
|
|
enabled: g.enabled,
|
|
cooldownOverride: g.cooldownOverride ?? null,
|
|
updatedAt: now,
|
|
})
|
|
}
|
|
for (const r of rules) {
|
|
const rawTargets =
|
|
r.targets && r.targets.length > 0
|
|
? r.targets.map((t) => t.trim()).filter(Boolean)
|
|
: [String(r.target ?? "").trim()].filter(Boolean)
|
|
const summary = summarizeTargets(rawTargets.length ? rawTargets : ["—"])
|
|
const rawConds =
|
|
r.conditions && r.conditions.length > 0
|
|
? r.conditions.map((c) => c.trim()).filter(Boolean)
|
|
: [String(r.condition ?? "").trim()].filter(Boolean)
|
|
const condSummary = summarizeConditionLines(rawConds.length ? rawConds : ["—"])
|
|
const stab =
|
|
r.confirmStabilitySec != null &&
|
|
Number.isFinite(r.confirmStabilitySec) &&
|
|
r.confirmStabilitySec > 0
|
|
? Math.min(86400, Math.max(1, Math.floor(r.confirmStabilitySec)))
|
|
: null
|
|
const recoveryMode =
|
|
r.recoveryMode === "never" || r.recoveryMode === "conditional" ? r.recoveryMode : "always"
|
|
const recoveryStabilitySec =
|
|
r.recoveryStabilitySec != null &&
|
|
Number.isFinite(r.recoveryStabilitySec) &&
|
|
r.recoveryStabilitySec > 0
|
|
? Math.min(86400, Math.max(1, Math.floor(r.recoveryStabilitySec)))
|
|
: null
|
|
await tx.insert(alertRules).values({
|
|
id: r.id,
|
|
name: r.name,
|
|
type: r.type,
|
|
target: summary || "—",
|
|
condition: condSummary,
|
|
severity: r.severity,
|
|
enabled: r.enabled,
|
|
cooldown: r.cooldown,
|
|
confirmStabilitySec: stab,
|
|
recoveryMode,
|
|
recoveryStabilitySec,
|
|
chatId: r.chatId ?? "",
|
|
groupId: r.groupId && r.groupId.trim() ? r.groupId.trim() : null,
|
|
updatedAt: now,
|
|
})
|
|
for (const [i, t] of rawTargets.entries()) {
|
|
await tx.insert(alertRuleTargets).values({
|
|
id: `rt-${r.id}-${i}`,
|
|
ruleId: r.id,
|
|
target: t,
|
|
sortIndex: i,
|
|
})
|
|
}
|
|
for (const [i, c] of rawConds.entries()) {
|
|
await tx.insert(alertRuleConditions).values({
|
|
id: `rc-${r.id}-${i}`,
|
|
ruleId: r.id,
|
|
conditionLine: c,
|
|
sortIndex: i,
|
|
})
|
|
}
|
|
}
|
|
},
|
|
read:async tx=>{const groups=await listAlertGroups(tx),rules=await listAlertRules(tx);return {ok:true,revision:alertsConfigRevision({groups,rules}),groups,rules}},
|
|
})
|
|
}
|
|
|
|
/** Совместимость: только правила, группы не трогаем. */
|
|
export async function replaceAlertRules(rules: ReplaceAlertRuleInput[]) {
|
|
await replaceAlertsConfig({ rules })
|
|
}
|
|
|
|
export async function getTelegramSettingsRow() {
|
|
return await ensureTelegramRow()
|
|
}
|
|
|
|
export async function getTelegramPublic(context:AlertsReadContext=db) {
|
|
const row = (await context.select().from(alertTelegramSettings).where(eq(alertTelegramSettings.id,1)).limit(1))[0]
|
|
const messageThreadId = row?.messageThreadId ?? null
|
|
return {
|
|
chatId: row?.chatId ?? "",
|
|
tokenConfigured: Boolean(row?.botToken?.trim()),
|
|
messageThreadId,
|
|
}
|
|
}
|
|
|
|
export async function updateTelegramSettings(patch: {
|
|
token?: string | null
|
|
chatId?: string | undefined
|
|
/** undefined — не менять; null — сбросить тему */
|
|
messageThreadId?: number | null
|
|
}, beforeWrite: () => Promise<unknown> = async()=>{}) {
|
|
return executeGuardedApplicationWrite({kind:"alerts-telegram",id:"1"},"all",()=>executeTelegramSettingsUpdate(patch, {
|
|
ensure: async()=>{await beforeWrite();return ensureTelegramRow()},
|
|
update: async values => {
|
|
await beforeWrite()
|
|
const [row] = await db.update(alertTelegramSettings).set({...values, updatedAt:sql`now()`}).where(eq(alertTelegramSettings.id,1)).returning()
|
|
return row
|
|
},
|
|
}))
|
|
}
|
|
|
|
/** Токен только для внутреннего вызова Telegram API (не отдаётся клиенту). */
|
|
export async function getTelegramBotToken(): Promise<string> {
|
|
return (await ensureTelegramRow()).botToken?.trim() ?? ""
|
|
}
|
|
|
|
/** Для `sendMessage`: только если в БД задана валидная тема. */
|
|
export async function getTelegramMessageThreadIdForApi(): Promise<number | undefined> {
|
|
const { messageThreadId } = await getTelegramPublic()
|
|
return messageThreadId ?? undefined
|
|
}
|
|
|
|
/** Текст тестового сообщения по полям правила (как в UI предпросмотра, без записи в историю). */
|
|
export function formatAlertRuleTestTelegramText(opts: {
|
|
name: string
|
|
targets: string[]
|
|
conditionLine: string
|
|
severity: "critical" | "warning" | "info"
|
|
cooldown: string
|
|
}): string {
|
|
const emoji =
|
|
opts.severity === "critical" ? "🔴" : opts.severity === "warning" ? "🟡" : "🔵"
|
|
const obj = opts.targets.length ? opts.targets.join(", ") : "—"
|
|
return [
|
|
"🧪 MikroTik Manager — тестовая отправка по полям формы правила (в историю срабатываний не записывается).",
|
|
"",
|
|
`${emoji} ${opts.name}`,
|
|
`Объект: ${obj}`,
|
|
`Событие: ${opts.conditionLine}`,
|
|
`Cooldown: ${opts.cooldown}`,
|
|
].join("\n")
|
|
}
|
|
|
|
/** Отправка текста в Telegram (движок алертов и тест). */
|
|
export async function sendTelegramAlertMessage(opts:TelegramAlertInput){
|
|
return deliverTelegramAlert(opts,{readStored:getTelegramSettingsRow,send:sendTelegramRequest})
|
|
}
|
|
|
|
export async function appendAlertHistory(entry: {
|
|
id: string
|
|
ruleId?: string | null
|
|
groupId?: string | null
|
|
ruleName: string
|
|
severity: (typeof alertHistory.$inferSelect)["severity"]
|
|
message: string
|
|
sentOk: boolean
|
|
firedAt?: string
|
|
}) {
|
|
const firedAt = entry.firedAt ?? new Date().toISOString()
|
|
await db.insert(alertHistory)
|
|
.values({
|
|
id: entry.id,
|
|
ruleId: entry.ruleId ?? null,
|
|
groupId: entry.groupId ?? null,
|
|
ruleName: entry.ruleName,
|
|
severity: entry.severity,
|
|
message: entry.message,
|
|
sentOk: entry.sentOk,
|
|
firedAt,
|
|
})
|
|
}
|
|
|
|
async function toLiveHistoryEntries(context:AlertsReadContext): Promise<ApiAlertHistoryEntry[]> {
|
|
const snapshot = await readCollectorSourceSnapshot(Date.now, createCollectorContextReader(context))
|
|
return monitoringHistory(snapshot)
|
|
}
|
|
|
|
export async function listPersistedHistory(limit: number,context:AlertsReadContext=db): Promise<ApiAlertHistoryEntry[]> {
|
|
const rows = await context
|
|
.select()
|
|
.from(alertHistory)
|
|
.orderBy(desc(alertHistory.firedAt))
|
|
.limit(limit)
|
|
return rows.map((h) => ({
|
|
id: h.id,
|
|
ruleName: h.ruleName,
|
|
severity: h.severity,
|
|
message: h.message,
|
|
sent: h.sentOk,
|
|
firedAt: h.firedAt,
|
|
source: "db" as const,
|
|
}))
|
|
}
|
|
|
|
export async function listMergedHistory(persistLimit = 40, liveCap = 25,context:AlertsReadContext=db): Promise<ApiAlertHistoryEntry[]> {
|
|
const persisted = await listPersistedHistory(persistLimit,context)
|
|
const live = (await toLiveHistoryEntries(context)).slice(0, liveCap)
|
|
const merged = [...persisted, ...live].sort((a, b) => Date.parse(b.firedAt) - Date.parse(a.firedAt))
|
|
const seen = new Set<string>()
|
|
const dedup: ApiAlertHistoryEntry[] = []
|
|
for (const e of merged) {
|
|
if (seen.has(e.id)) continue
|
|
seen.add(e.id)
|
|
dedup.push(e)
|
|
if (dedup.length >= 50) break
|
|
}
|
|
return dedup
|
|
}
|
|
|
|
export async function getAlertsMeta(context:AlertsReadContext=db) {
|
|
const srvRows = await context
|
|
.select({
|
|
id: servers.id,
|
|
enabled: servers.enabled,
|
|
name: servers.name,
|
|
host: servers.host,
|
|
site: servers.site,
|
|
country: servers.country,
|
|
type: servers.type,
|
|
})
|
|
.from(servers)
|
|
const probes=await context.select({id:uptimeProbes.id,name:uptimeProbes.name,target:uptimeProbes.target,enabled:uptimeProbes.enabled,
|
|
srcServerId:uptimeProbes.srcServerId,srcInterface:uptimeProbes.srcInterface,intervalSec:uptimeProbes.intervalSec}).from(uptimeProbes)
|
|
const uptime=(await context.select().from(uptimeSettings).where(eq(uptimeSettings.id,1)).limit(1))[0]
|
|
const rest=(await context.select().from(serversApiPingSettings).where(eq(serversApiPingSettings.id,1)).limit(1))[0]
|
|
const traffic=(await context.select().from(trafficSettings).where(eq(trafficSettings.id,1)).limit(1))[0]
|
|
return buildAlertsMeta(srvRows,probes,{servers:srvRows,probes,uptime,rest,traffic})
|
|
}
|
|
|
|
export async function getAlertsOverview(){
|
|
return readAlertsOverview<AlertsWriteContext>({
|
|
transaction:(work,options)=>db.transaction(work,options),
|
|
telegram:async context=>{const value=await getTelegramPublic(context);return {...value,connected:value.tokenConfigured&&Boolean(value.chatId.trim())}},
|
|
groups:context=>listAlertGroups(context),rules:context=>listAlertRules(context),
|
|
history:context=>listMergedHistory(40,25,context),meta:context=>getAlertsMeta(context),
|
|
})
|
|
}
|