feat(alerts): implement customizable alert settings for server availability and resource usage, enhancing user control over notifications
Publish Fast Tabler Docker image / build-and-push-fast (push) Successful in 1m54s

This commit is contained in:
2026-02-24 00:48:34 +07:00
parent e4394ef35c
commit b35f787626
5 changed files with 331 additions and 54 deletions
+90 -38
View File
@@ -1,49 +1,97 @@
/**
* Система оповещений: агрегирует проблемы из availability и resources/stats.
* Критерии проблем: сервер недоступен, MikroTik недоступен, высокое CPU/RAM/HDD.
* Критерии и пороги задаются в ui-settings.alertSettings.
*/
const { sendError } = require('../middleware/errorHandler');
const { readServersFromS3 } = require('./serversRoutes');
const { checkOneServerFast } = require('./miscRoutes');
const { checkOneServerFast, loadUiSettingsSync } = require('./miscRoutes');
const { getResourceStatsData } = require('./resourceStatsRoutes');
const { getUptimeCacheData } = require('./mikrotikConfigRoutes');
/** Пороги для ресурсов (проценты). */
const THRESHOLD_CPU = 85;
const THRESHOLD_RAM = 85;
const THRESHOLD_HDD = 90;
/** Значения по умолчанию для настроек оповещений. */
const DEFAULT_ALERT_SETTINGS = {
serverOffline: { enabled: true },
mikrotikUnreachable: { enabled: true },
highCpu: { enabled: true, thresholdPercent: 85 },
highRam: { enabled: true, thresholdPercent: 85 },
highHdd: { enabled: true, thresholdPercent: 90 },
};
function getAlertSettings(uiSettings) {
const raw = uiSettings?.alertSettings && typeof uiSettings.alertSettings === 'object' ? uiSettings.alertSettings : {};
return {
serverOffline: { ...DEFAULT_ALERT_SETTINGS.serverOffline, ...raw.serverOffline },
mikrotikUnreachable: { ...DEFAULT_ALERT_SETTINGS.mikrotikUnreachable, ...raw.mikrotikUnreachable },
highCpu: { ...DEFAULT_ALERT_SETTINGS.highCpu, ...raw.highCpu },
highRam: { ...DEFAULT_ALERT_SETTINGS.highRam, ...raw.highRam },
highHdd: { ...DEFAULT_ALERT_SETTINGS.highHdd, ...raw.highHdd },
};
}
/**
* GET /api/alerts
* Возвращает список активных оповещений.
* Возвращает список активных оповещений (учёт настроек из ui-settings.alertSettings).
*/
async function getAlerts(req, res) {
try {
const uiSettings = await loadUiSettingsSync();
const settings = getAlertSettings(uiSettings);
const servers = await readServersFromS3();
const alerts = [];
const now = new Date().toISOString();
// 1) Доступность серверов (TCP 80/443)
const availabilityChecks = await Promise.allSettled(
servers.map((s) => checkOneServerFast(s))
);
servers.forEach((server, i) => {
const online = availabilityChecks[i].status === 'fulfilled' && availabilityChecks[i].value;
if (!online) {
const name = server.name || server.dns || server.ip || server.id || `Сервер #${i + 1}`;
alerts.push({
id: `server-offline-${server.id || server.dns || server.ip || i}`,
type: 'server_offline',
severity: 'critical',
title: 'Сервер недоступен',
description: `${name} не отвечает на TCP (80/443).`,
entity: name,
entityId: server.id || server.dns || server.ip,
link: '/servers',
at: now,
});
}
});
// 1) Доступность серверов: jumphost/home — по правилам Uptime Monitor (кеш), остальные — TCP 80/443
if (settings.serverOffline.enabled) {
const routerTypes = ['jumphost', 'home'];
const { results: uptimeResults } = await getUptimeCacheData();
const nonRouterServers = servers.filter(
(s) => !routerTypes.includes(String(s.type || '').toLowerCase())
);
const availabilityChecks = await Promise.allSettled(
nonRouterServers.map((s) => checkOneServerFast(s))
);
servers.forEach((server) => {
const serverId = server.id || server.dns || server.ip;
const name = server.name || server.dns || server.ip || server.id || 'Сервер';
const isRouter = routerTypes.includes(String(server.type || '').toLowerCase());
let offline = false;
let description = '';
if (isRouter) {
const entry = uptimeResults[serverId];
offline = entry ? entry.ok === false : false;
description = offline
? `${name} недоступен (по проверке Uptime Monitor).`
: '';
} else {
const idx = nonRouterServers.findIndex(
(s) => (s.id || s.dns || s.ip) === serverId
);
const online = idx >= 0 && availabilityChecks[idx].status === 'fulfilled' && availabilityChecks[idx].value;
offline = !online;
description = offline ? `${name} не отвечает на TCP (80/443).` : '';
}
if (offline && description) {
alerts.push({
id: `server-offline-${serverId}`,
type: 'server_offline',
severity: 'critical',
title: 'Сервер недоступен',
description,
entity: name,
entityId: serverId,
link: isRouter ? '/uptime-monitor' : '/servers',
at: now,
});
}
});
}
// 2) Ресурсы роутеров (MikroTik): ошибка доступа, высокое CPU/RAM/HDD
let resourceData;
@@ -54,12 +102,16 @@ async function getAlerts(req, res) {
resourceData = { routers: [] };
}
const thCpu = Math.max(1, Math.min(100, Number(settings.highCpu.thresholdPercent) || 85));
const thRam = Math.max(1, Math.min(100, Number(settings.highRam.thresholdPercent) || 85));
const thHdd = Math.max(1, Math.min(100, Number(settings.highHdd.thresholdPercent) || 90));
const routers = resourceData.routers || [];
routers.forEach((router) => {
const { serverId, name, error, resource } = router;
const entityName = name || serverId;
if (error) {
if (error && settings.mikrotikUnreachable.enabled) {
alerts.push({
id: `mikrotik-error-${serverId}`,
type: 'mikrotik_unreachable',
@@ -76,47 +128,47 @@ async function getAlerts(req, res) {
if (!resource) return;
if (resource.cpuLoad != null && resource.cpuLoad >= THRESHOLD_CPU) {
if (settings.highCpu.enabled && resource.cpuLoad != null && resource.cpuLoad >= thCpu) {
alerts.push({
id: `cpu-high-${serverId}`,
type: 'high_cpu',
severity: 'warning',
title: 'Высокое использование CPU',
description: `${entityName}: ${resource.cpuLoad}% (порог ${THRESHOLD_CPU}%).`,
description: `${entityName}: ${resource.cpuLoad}% (порог ${thCpu}%).`,
entity: entityName,
entityId: serverId,
value: resource.cpuLoad,
threshold: THRESHOLD_CPU,
threshold: thCpu,
link: '/resource-stats',
at: now,
});
}
if (resource.memoryUsagePercent != null && resource.memoryUsagePercent >= THRESHOLD_RAM) {
if (settings.highRam.enabled && resource.memoryUsagePercent != null && resource.memoryUsagePercent >= thRam) {
alerts.push({
id: `ram-high-${serverId}`,
type: 'high_ram',
severity: 'warning',
title: 'Высокое использование RAM',
description: `${entityName}: ${resource.memoryUsagePercent}% (порог ${THRESHOLD_RAM}%).`,
description: `${entityName}: ${resource.memoryUsagePercent}% (порог ${thRam}%).`,
entity: entityName,
entityId: serverId,
value: resource.memoryUsagePercent,
threshold: THRESHOLD_RAM,
threshold: thRam,
link: '/resource-stats',
at: now,
});
}
if (resource.hddUsagePercent != null && resource.hddUsagePercent >= THRESHOLD_HDD) {
if (settings.highHdd.enabled && resource.hddUsagePercent != null && resource.hddUsagePercent >= thHdd) {
alerts.push({
id: `hdd-high-${serverId}`,
type: 'high_hdd',
severity: 'warning',
title: 'Высокое использование диска',
description: `${entityName}: ${resource.hddUsagePercent}% (порог ${THRESHOLD_HDD}%).`,
description: `${entityName}: ${resource.hddUsagePercent}% (порог ${thHdd}%).`,
entity: entityName,
entityId: serverId,
value: resource.hddUsagePercent,
threshold: THRESHOLD_HDD,
threshold: thHdd,
link: '/resource-stats',
at: now,
});
+26 -13
View File
@@ -264,32 +264,44 @@ function updateUptimeCache(serverId, entry) {
}
/**
* GET /api/uptime/cache
* Возвращает закешированные результаты проверок, если кеш младше TTL из настроек.
* Получить данные кеша Uptime Monitor (для оповещений и др.).
* Учитывает TTL из ui-settings.uptimeMonitorCacheSeconds.
* @returns {Promise<{ results: Record<string, { ok, lastCheckTs, ms }>, updatedAt: number|null }>}
*/
async function getUptimeCache(req, res) {
async function getUptimeCacheData() {
try {
const uiSettings = await loadUiSettings();
const ttlSec = Math.max(0, parseInt(uiSettings.uptimeMonitorCacheSeconds, 10) || 120);
if (ttlSec === 0) {
return res.json({ results: {}, updatedAt: null });
}
if (ttlSec === 0) return { results: {}, updatedAt: null };
const data = await readS3TextObject(UPTIME_CACHE_KEY).catch(() => null);
if (!data?.body) {
return res.json({ results: {}, updatedAt: null });
}
if (!data?.body) return { results: {}, updatedAt: null };
let cache = { results: {}, updatedAt: null };
try {
const parsed = JSON.parse(data.body);
if (parsed && typeof parsed === 'object') cache = parsed;
} catch (_) {}
const now = Date.now();
if (!cache.updatedAt || now - cache.updatedAt > ttlSec * 1000) {
return res.json({ results: {}, updatedAt: null });
}
return res.json({
if (!cache.updatedAt || now - cache.updatedAt > ttlSec * 1000) return { results: {}, updatedAt: null };
return {
results: cache.results && typeof cache.results === 'object' ? cache.results : {},
updatedAt: cache.updatedAt,
};
} catch (e) {
console.warn('[uptime] getUptimeCacheData', e?.message);
return { results: {}, updatedAt: null };
}
}
/**
* GET /api/uptime/cache
* Возвращает закешированные результаты проверок, если кеш младше TTL из настроек.
*/
async function getUptimeCache(req, res) {
try {
const { results, updatedAt } = await getUptimeCacheData();
return res.json({
results: Object.keys(results).length ? results : {},
updatedAt,
});
} catch (e) {
console.error('[uptime] getUptimeCache', e);
@@ -1547,4 +1559,5 @@ module.exports = {
applyAddressListSummary,
uptimeCheck,
getUptimeCache,
getUptimeCacheData,
};
+1
View File
@@ -777,5 +777,6 @@ module.exports = {
getPingServices,
refreshPingServicesCache,
checkOneServerFast,
loadUiSettingsSync,
};