feat(alerts): implement customizable alert settings for server availability and resource usage, enhancing user control over notifications
Publish Fast Tabler Docker image / build-and-push-fast (push) Successful in 1m54s
Publish Fast Tabler Docker image / build-and-push-fast (push) Successful in 1m54s
This commit is contained in:
@@ -1,49 +1,97 @@
|
||||
/**
|
||||
* Система оповещений: агрегирует проблемы из availability и resources/stats.
|
||||
* Критерии проблем: сервер недоступен, MikroTik недоступен, высокое CPU/RAM/HDD.
|
||||
* Критерии и пороги задаются в ui-settings.alertSettings.
|
||||
*/
|
||||
|
||||
const { sendError } = require('../middleware/errorHandler');
|
||||
const { readServersFromS3 } = require('./serversRoutes');
|
||||
const { checkOneServerFast } = require('./miscRoutes');
|
||||
const { checkOneServerFast, loadUiSettingsSync } = require('./miscRoutes');
|
||||
const { getResourceStatsData } = require('./resourceStatsRoutes');
|
||||
const { getUptimeCacheData } = require('./mikrotikConfigRoutes');
|
||||
|
||||
/** Пороги для ресурсов (проценты). */
|
||||
const THRESHOLD_CPU = 85;
|
||||
const THRESHOLD_RAM = 85;
|
||||
const THRESHOLD_HDD = 90;
|
||||
/** Значения по умолчанию для настроек оповещений. */
|
||||
const DEFAULT_ALERT_SETTINGS = {
|
||||
serverOffline: { enabled: true },
|
||||
mikrotikUnreachable: { enabled: true },
|
||||
highCpu: { enabled: true, thresholdPercent: 85 },
|
||||
highRam: { enabled: true, thresholdPercent: 85 },
|
||||
highHdd: { enabled: true, thresholdPercent: 90 },
|
||||
};
|
||||
|
||||
function getAlertSettings(uiSettings) {
|
||||
const raw = uiSettings?.alertSettings && typeof uiSettings.alertSettings === 'object' ? uiSettings.alertSettings : {};
|
||||
return {
|
||||
serverOffline: { ...DEFAULT_ALERT_SETTINGS.serverOffline, ...raw.serverOffline },
|
||||
mikrotikUnreachable: { ...DEFAULT_ALERT_SETTINGS.mikrotikUnreachable, ...raw.mikrotikUnreachable },
|
||||
highCpu: { ...DEFAULT_ALERT_SETTINGS.highCpu, ...raw.highCpu },
|
||||
highRam: { ...DEFAULT_ALERT_SETTINGS.highRam, ...raw.highRam },
|
||||
highHdd: { ...DEFAULT_ALERT_SETTINGS.highHdd, ...raw.highHdd },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/alerts
|
||||
* Возвращает список активных оповещений.
|
||||
* Возвращает список активных оповещений (учёт настроек из ui-settings.alertSettings).
|
||||
*/
|
||||
async function getAlerts(req, res) {
|
||||
try {
|
||||
const uiSettings = await loadUiSettingsSync();
|
||||
const settings = getAlertSettings(uiSettings);
|
||||
|
||||
const servers = await readServersFromS3();
|
||||
const alerts = [];
|
||||
const now = new Date().toISOString();
|
||||
|
||||
// 1) Доступность серверов (TCP 80/443)
|
||||
const availabilityChecks = await Promise.allSettled(
|
||||
servers.map((s) => checkOneServerFast(s))
|
||||
);
|
||||
servers.forEach((server, i) => {
|
||||
const online = availabilityChecks[i].status === 'fulfilled' && availabilityChecks[i].value;
|
||||
if (!online) {
|
||||
const name = server.name || server.dns || server.ip || server.id || `Сервер #${i + 1}`;
|
||||
alerts.push({
|
||||
id: `server-offline-${server.id || server.dns || server.ip || i}`,
|
||||
type: 'server_offline',
|
||||
severity: 'critical',
|
||||
title: 'Сервер недоступен',
|
||||
description: `${name} не отвечает на TCP (80/443).`,
|
||||
entity: name,
|
||||
entityId: server.id || server.dns || server.ip,
|
||||
link: '/servers',
|
||||
at: now,
|
||||
});
|
||||
}
|
||||
});
|
||||
// 1) Доступность серверов: jumphost/home — по правилам Uptime Monitor (кеш), остальные — TCP 80/443
|
||||
if (settings.serverOffline.enabled) {
|
||||
const routerTypes = ['jumphost', 'home'];
|
||||
const { results: uptimeResults } = await getUptimeCacheData();
|
||||
|
||||
const nonRouterServers = servers.filter(
|
||||
(s) => !routerTypes.includes(String(s.type || '').toLowerCase())
|
||||
);
|
||||
const availabilityChecks = await Promise.allSettled(
|
||||
nonRouterServers.map((s) => checkOneServerFast(s))
|
||||
);
|
||||
|
||||
servers.forEach((server) => {
|
||||
const serverId = server.id || server.dns || server.ip;
|
||||
const name = server.name || server.dns || server.ip || server.id || 'Сервер';
|
||||
const isRouter = routerTypes.includes(String(server.type || '').toLowerCase());
|
||||
|
||||
let offline = false;
|
||||
let description = '';
|
||||
|
||||
if (isRouter) {
|
||||
const entry = uptimeResults[serverId];
|
||||
offline = entry ? entry.ok === false : false;
|
||||
description = offline
|
||||
? `${name} недоступен (по проверке Uptime Monitor).`
|
||||
: '';
|
||||
} else {
|
||||
const idx = nonRouterServers.findIndex(
|
||||
(s) => (s.id || s.dns || s.ip) === serverId
|
||||
);
|
||||
const online = idx >= 0 && availabilityChecks[idx].status === 'fulfilled' && availabilityChecks[idx].value;
|
||||
offline = !online;
|
||||
description = offline ? `${name} не отвечает на TCP (80/443).` : '';
|
||||
}
|
||||
|
||||
if (offline && description) {
|
||||
alerts.push({
|
||||
id: `server-offline-${serverId}`,
|
||||
type: 'server_offline',
|
||||
severity: 'critical',
|
||||
title: 'Сервер недоступен',
|
||||
description,
|
||||
entity: name,
|
||||
entityId: serverId,
|
||||
link: isRouter ? '/uptime-monitor' : '/servers',
|
||||
at: now,
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// 2) Ресурсы роутеров (MikroTik): ошибка доступа, высокое CPU/RAM/HDD
|
||||
let resourceData;
|
||||
@@ -54,12 +102,16 @@ async function getAlerts(req, res) {
|
||||
resourceData = { routers: [] };
|
||||
}
|
||||
|
||||
const thCpu = Math.max(1, Math.min(100, Number(settings.highCpu.thresholdPercent) || 85));
|
||||
const thRam = Math.max(1, Math.min(100, Number(settings.highRam.thresholdPercent) || 85));
|
||||
const thHdd = Math.max(1, Math.min(100, Number(settings.highHdd.thresholdPercent) || 90));
|
||||
|
||||
const routers = resourceData.routers || [];
|
||||
routers.forEach((router) => {
|
||||
const { serverId, name, error, resource } = router;
|
||||
const entityName = name || serverId;
|
||||
|
||||
if (error) {
|
||||
if (error && settings.mikrotikUnreachable.enabled) {
|
||||
alerts.push({
|
||||
id: `mikrotik-error-${serverId}`,
|
||||
type: 'mikrotik_unreachable',
|
||||
@@ -76,47 +128,47 @@ async function getAlerts(req, res) {
|
||||
|
||||
if (!resource) return;
|
||||
|
||||
if (resource.cpuLoad != null && resource.cpuLoad >= THRESHOLD_CPU) {
|
||||
if (settings.highCpu.enabled && resource.cpuLoad != null && resource.cpuLoad >= thCpu) {
|
||||
alerts.push({
|
||||
id: `cpu-high-${serverId}`,
|
||||
type: 'high_cpu',
|
||||
severity: 'warning',
|
||||
title: 'Высокое использование CPU',
|
||||
description: `${entityName}: ${resource.cpuLoad}% (порог ${THRESHOLD_CPU}%).`,
|
||||
description: `${entityName}: ${resource.cpuLoad}% (порог ${thCpu}%).`,
|
||||
entity: entityName,
|
||||
entityId: serverId,
|
||||
value: resource.cpuLoad,
|
||||
threshold: THRESHOLD_CPU,
|
||||
threshold: thCpu,
|
||||
link: '/resource-stats',
|
||||
at: now,
|
||||
});
|
||||
}
|
||||
if (resource.memoryUsagePercent != null && resource.memoryUsagePercent >= THRESHOLD_RAM) {
|
||||
if (settings.highRam.enabled && resource.memoryUsagePercent != null && resource.memoryUsagePercent >= thRam) {
|
||||
alerts.push({
|
||||
id: `ram-high-${serverId}`,
|
||||
type: 'high_ram',
|
||||
severity: 'warning',
|
||||
title: 'Высокое использование RAM',
|
||||
description: `${entityName}: ${resource.memoryUsagePercent}% (порог ${THRESHOLD_RAM}%).`,
|
||||
description: `${entityName}: ${resource.memoryUsagePercent}% (порог ${thRam}%).`,
|
||||
entity: entityName,
|
||||
entityId: serverId,
|
||||
value: resource.memoryUsagePercent,
|
||||
threshold: THRESHOLD_RAM,
|
||||
threshold: thRam,
|
||||
link: '/resource-stats',
|
||||
at: now,
|
||||
});
|
||||
}
|
||||
if (resource.hddUsagePercent != null && resource.hddUsagePercent >= THRESHOLD_HDD) {
|
||||
if (settings.highHdd.enabled && resource.hddUsagePercent != null && resource.hddUsagePercent >= thHdd) {
|
||||
alerts.push({
|
||||
id: `hdd-high-${serverId}`,
|
||||
type: 'high_hdd',
|
||||
severity: 'warning',
|
||||
title: 'Высокое использование диска',
|
||||
description: `${entityName}: ${resource.hddUsagePercent}% (порог ${THRESHOLD_HDD}%).`,
|
||||
description: `${entityName}: ${resource.hddUsagePercent}% (порог ${thHdd}%).`,
|
||||
entity: entityName,
|
||||
entityId: serverId,
|
||||
value: resource.hddUsagePercent,
|
||||
threshold: THRESHOLD_HDD,
|
||||
threshold: thHdd,
|
||||
link: '/resource-stats',
|
||||
at: now,
|
||||
});
|
||||
|
||||
@@ -264,32 +264,44 @@ function updateUptimeCache(serverId, entry) {
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/uptime/cache
|
||||
* Возвращает закешированные результаты проверок, если кеш младше TTL из настроек.
|
||||
* Получить данные кеша Uptime Monitor (для оповещений и др.).
|
||||
* Учитывает TTL из ui-settings.uptimeMonitorCacheSeconds.
|
||||
* @returns {Promise<{ results: Record<string, { ok, lastCheckTs, ms }>, updatedAt: number|null }>}
|
||||
*/
|
||||
async function getUptimeCache(req, res) {
|
||||
async function getUptimeCacheData() {
|
||||
try {
|
||||
const uiSettings = await loadUiSettings();
|
||||
const ttlSec = Math.max(0, parseInt(uiSettings.uptimeMonitorCacheSeconds, 10) || 120);
|
||||
if (ttlSec === 0) {
|
||||
return res.json({ results: {}, updatedAt: null });
|
||||
}
|
||||
if (ttlSec === 0) return { results: {}, updatedAt: null };
|
||||
const data = await readS3TextObject(UPTIME_CACHE_KEY).catch(() => null);
|
||||
if (!data?.body) {
|
||||
return res.json({ results: {}, updatedAt: null });
|
||||
}
|
||||
if (!data?.body) return { results: {}, updatedAt: null };
|
||||
let cache = { results: {}, updatedAt: null };
|
||||
try {
|
||||
const parsed = JSON.parse(data.body);
|
||||
if (parsed && typeof parsed === 'object') cache = parsed;
|
||||
} catch (_) {}
|
||||
const now = Date.now();
|
||||
if (!cache.updatedAt || now - cache.updatedAt > ttlSec * 1000) {
|
||||
return res.json({ results: {}, updatedAt: null });
|
||||
}
|
||||
return res.json({
|
||||
if (!cache.updatedAt || now - cache.updatedAt > ttlSec * 1000) return { results: {}, updatedAt: null };
|
||||
return {
|
||||
results: cache.results && typeof cache.results === 'object' ? cache.results : {},
|
||||
updatedAt: cache.updatedAt,
|
||||
};
|
||||
} catch (e) {
|
||||
console.warn('[uptime] getUptimeCacheData', e?.message);
|
||||
return { results: {}, updatedAt: null };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/uptime/cache
|
||||
* Возвращает закешированные результаты проверок, если кеш младше TTL из настроек.
|
||||
*/
|
||||
async function getUptimeCache(req, res) {
|
||||
try {
|
||||
const { results, updatedAt } = await getUptimeCacheData();
|
||||
return res.json({
|
||||
results: Object.keys(results).length ? results : {},
|
||||
updatedAt,
|
||||
});
|
||||
} catch (e) {
|
||||
console.error('[uptime] getUptimeCache', e);
|
||||
@@ -1547,4 +1559,5 @@ module.exports = {
|
||||
applyAddressListSummary,
|
||||
uptimeCheck,
|
||||
getUptimeCache,
|
||||
getUptimeCacheData,
|
||||
};
|
||||
|
||||
@@ -777,5 +777,6 @@ module.exports = {
|
||||
getPingServices,
|
||||
refreshPingServicesCache,
|
||||
checkOneServerFast,
|
||||
loadUiSettingsSync,
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user