denozord
d68b3aa104
Publish Fast Tabler Docker image / build-and-push-fast (push) Successful in 2m32s
378 lines
15 KiB
JavaScript
378 lines
15 KiB
JavaScript
/**
|
||
* Система оповещений: агрегирует проблемы из availability и resources/stats.
|
||
* Критерии и пороги задаются в ui-settings.alertSettings.
|
||
*/
|
||
|
||
const { sendError } = require('../middleware/errorHandler');
|
||
const { readServersFromS3 } = require('./serversRoutes');
|
||
const { checkOneServerFast, loadUiSettingsSync } = require('./miscRoutes');
|
||
const { getResourceStatsData } = require('./resourceStatsRoutes');
|
||
const { getUptimeCacheData } = require('./mikrotikConfigRoutes');
|
||
const { buildConnections } = require('../services/networkMapScheduler');
|
||
const { readS3TextObject } = require('../services/s3Service');
|
||
|
||
/** Значения по умолчанию для настроек оповещений. */
|
||
const DEFAULT_ALERT_SETTINGS = {
|
||
serverOffline: { enabled: true, offlineMinutes: 5 },
|
||
mikrotikUnreachable: { enabled: true, durationMinutes: 5 },
|
||
highCpu: { enabled: true, thresholdPercent: 85, durationMinutes: 10 },
|
||
highRam: { enabled: true, thresholdPercent: 85, durationMinutes: 10 },
|
||
highHdd: { enabled: true, thresholdPercent: 90, durationMinutes: 10 },
|
||
/** Глобальные пороги карты сети для UI-аналитики */
|
||
networkMapNodeThresholds: {
|
||
lowSpeedMbps: 40,
|
||
belowNormSpeedMbps: 80,
|
||
normalSpeedMbps: 120,
|
||
},
|
||
/** Индивидуальные пороги по туннелям (карта сети) */
|
||
tunnelThresholds: [],
|
||
};
|
||
|
||
function getAlertSettings(uiSettings) {
|
||
const raw = uiSettings?.alertSettings && typeof uiSettings.alertSettings === 'object' ? uiSettings.alertSettings : {};
|
||
return {
|
||
serverOffline: { ...DEFAULT_ALERT_SETTINGS.serverOffline, ...raw.serverOffline },
|
||
mikrotikUnreachable: { ...DEFAULT_ALERT_SETTINGS.mikrotikUnreachable, ...raw.mikrotikUnreachable },
|
||
highCpu: { ...DEFAULT_ALERT_SETTINGS.highCpu, ...raw.highCpu },
|
||
highRam: { ...DEFAULT_ALERT_SETTINGS.highRam, ...raw.highRam },
|
||
highHdd: { ...DEFAULT_ALERT_SETTINGS.highHdd, ...raw.highHdd },
|
||
networkMapNodeThresholds: {
|
||
...DEFAULT_ALERT_SETTINGS.networkMapNodeThresholds,
|
||
...(raw.networkMapNodeThresholds && typeof raw.networkMapNodeThresholds === 'object'
|
||
? raw.networkMapNodeThresholds
|
||
: {}),
|
||
},
|
||
tunnelThresholds: Array.isArray(raw.tunnelThresholds) ? raw.tunnelThresholds : [],
|
||
};
|
||
}
|
||
|
||
function edgePingKey(a, b) {
|
||
return [String(a), String(b)].sort().join(':');
|
||
}
|
||
|
||
function speedKey(key1, key2) {
|
||
return [String(key1), String(key2)].sort().join(':');
|
||
}
|
||
|
||
/**
|
||
* GET /api/alerts
|
||
* Возвращает список активных оповещений (учёт настроек из ui-settings.alertSettings).
|
||
*/
|
||
async function getAlerts(req, res) {
|
||
try {
|
||
const uiSettings = await loadUiSettingsSync();
|
||
const settings = getAlertSettings(uiSettings);
|
||
|
||
const servers = await readServersFromS3();
|
||
const alerts = [];
|
||
const now = new Date().toISOString();
|
||
|
||
// 1) Доступность серверов: jumphost/home — по правилам Uptime Monitor (кеш), остальные — TCP 80/443
|
||
if (settings.serverOffline.enabled) {
|
||
const routerTypes = ['jumphost', 'home'];
|
||
const { results: uptimeResults } = await getUptimeCacheData();
|
||
|
||
const nonRouterServers = servers.filter(
|
||
(s) => !routerTypes.includes(String(s.type || '').toLowerCase())
|
||
);
|
||
const availabilityChecks = await Promise.allSettled(
|
||
nonRouterServers.map((s) => checkOneServerFast(s))
|
||
);
|
||
|
||
servers.forEach((server) => {
|
||
const serverId = server.id || server.dns || server.ip;
|
||
const name = server.name || server.dns || server.ip || server.id || 'Сервер';
|
||
const isRouter = routerTypes.includes(String(server.type || '').toLowerCase());
|
||
|
||
let offline = false;
|
||
let description = '';
|
||
|
||
if (isRouter) {
|
||
const entry = uptimeResults[serverId];
|
||
const offlineMinutes = Math.max(1, Math.min(1440, Number(settings.serverOffline.offlineMinutes) || 5));
|
||
const offlineSinceMs = entry && entry.lastCheckTs ? (Date.now() - entry.lastCheckTs) : 0;
|
||
offline = entry && entry.ok === false && offlineSinceMs >= offlineMinutes * 60 * 1000;
|
||
description = offline
|
||
? `${name} недоступен более ${offlineMinutes} мин (по проверке Uptime Monitor).`
|
||
: '';
|
||
} else {
|
||
const idx = nonRouterServers.findIndex(
|
||
(s) => (s.id || s.dns || s.ip) === serverId
|
||
);
|
||
const online = idx >= 0 && availabilityChecks[idx].status === 'fulfilled' && availabilityChecks[idx].value;
|
||
offline = !online;
|
||
description = offline ? `${name} не отвечает на TCP (80/443).` : '';
|
||
}
|
||
|
||
if (offline && description) {
|
||
alerts.push({
|
||
id: `server-offline-${serverId}`,
|
||
type: 'server_offline',
|
||
severity: 'critical',
|
||
title: 'Сервер недоступен',
|
||
description,
|
||
entity: name,
|
||
entityId: serverId,
|
||
link: isRouter ? '/uptime-monitor' : '/servers',
|
||
at: now,
|
||
});
|
||
}
|
||
});
|
||
}
|
||
|
||
// 2) Ресурсы роутеров (MikroTik): ошибка доступа, высокое CPU/RAM/HDD
|
||
let resourceData;
|
||
try {
|
||
resourceData = await getResourceStatsData();
|
||
} catch (err) {
|
||
console.warn('alerts: getResourceStatsData failed', err?.message);
|
||
resourceData = { routers: [] };
|
||
}
|
||
|
||
const thCpu = Math.max(1, Math.min(100, Number(settings.highCpu.thresholdPercent) || 85));
|
||
const thRam = Math.max(1, Math.min(100, Number(settings.highRam.thresholdPercent) || 85));
|
||
const thHdd = Math.max(1, Math.min(100, Number(settings.highHdd.thresholdPercent) || 90));
|
||
|
||
const routers = resourceData.routers || [];
|
||
routers.forEach((router) => {
|
||
const { serverId, name, error, resource } = router;
|
||
const entityName = name || serverId;
|
||
|
||
if (error && settings.mikrotikUnreachable.enabled) {
|
||
alerts.push({
|
||
id: `mikrotik-error-${serverId}`,
|
||
type: 'mikrotik_unreachable',
|
||
severity: 'critical',
|
||
title: 'MikroTik недоступен',
|
||
description: `${entityName}: ${error}`,
|
||
entity: entityName,
|
||
entityId: serverId,
|
||
link: '/resource-stats',
|
||
at: now,
|
||
});
|
||
return;
|
||
}
|
||
|
||
if (!resource) return;
|
||
|
||
if (settings.highCpu.enabled && resource.cpuLoad != null && resource.cpuLoad >= thCpu) {
|
||
alerts.push({
|
||
id: `cpu-high-${serverId}`,
|
||
type: 'high_cpu',
|
||
severity: 'warning',
|
||
title: 'Высокое использование CPU',
|
||
description: `${entityName}: ${resource.cpuLoad}% (порог ${thCpu}%).`,
|
||
entity: entityName,
|
||
entityId: serverId,
|
||
value: resource.cpuLoad,
|
||
threshold: thCpu,
|
||
link: '/resource-stats',
|
||
at: now,
|
||
});
|
||
}
|
||
if (settings.highRam.enabled && resource.memoryUsagePercent != null && resource.memoryUsagePercent >= thRam) {
|
||
alerts.push({
|
||
id: `ram-high-${serverId}`,
|
||
type: 'high_ram',
|
||
severity: 'warning',
|
||
title: 'Высокое использование RAM',
|
||
description: `${entityName}: ${resource.memoryUsagePercent}% (порог ${thRam}%).`,
|
||
entity: entityName,
|
||
entityId: serverId,
|
||
value: resource.memoryUsagePercent,
|
||
threshold: thRam,
|
||
link: '/resource-stats',
|
||
at: now,
|
||
});
|
||
}
|
||
if (settings.highHdd.enabled && resource.hddUsagePercent != null && resource.hddUsagePercent >= thHdd) {
|
||
alerts.push({
|
||
id: `hdd-high-${serverId}`,
|
||
type: 'high_hdd',
|
||
severity: 'warning',
|
||
title: 'Высокое использование диска',
|
||
description: `${entityName}: ${resource.hddUsagePercent}% (порог ${thHdd}%).`,
|
||
entity: entityName,
|
||
entityId: serverId,
|
||
value: resource.hddUsagePercent,
|
||
threshold: thHdd,
|
||
link: '/resource-stats',
|
||
at: now,
|
||
});
|
||
}
|
||
});
|
||
|
||
// 3) Туннели (карта сети): индивидуальные пороги по пингу и скорости
|
||
const rawTunnelThresholds = Array.isArray(settings.tunnelThresholds)
|
||
? settings.tunnelThresholds
|
||
: [];
|
||
const tunnelThresholds = rawTunnelThresholds.filter(
|
||
(t) => t && (t.fromKey || t.toKey) && t.enabled !== false
|
||
);
|
||
|
||
if (tunnelThresholds.length > 0) {
|
||
let cache = null;
|
||
let connectionsData = null;
|
||
try {
|
||
const [rawCache, built] = await Promise.all([
|
||
readS3TextObject('network-map-cache/latest.json').catch(() => null),
|
||
buildConnections().catch(() => null),
|
||
]);
|
||
if (rawCache?.body) {
|
||
const parsed = JSON.parse(rawCache.body);
|
||
if (parsed && typeof parsed === 'object') cache = parsed;
|
||
}
|
||
if (built && typeof built === 'object') connectionsData = built;
|
||
} catch (e) {
|
||
// Ошибки при чтении кеша/конфигурации не должны ломать остальные оповещения
|
||
console.warn('alerts: network-map-cache read failed', e?.message || e);
|
||
}
|
||
|
||
if (cache && connectionsData && Array.isArray(connectionsData.connections)) {
|
||
const pingMap = cache.pingMap && typeof cache.pingMap === 'object' ? cache.pingMap : {};
|
||
const speedMap = cache.speedMap && typeof cache.speedMap === 'object' ? cache.speedMap : {};
|
||
|
||
const serversByIp = new Map();
|
||
servers.forEach((s) => {
|
||
if (s && s.ip) {
|
||
serversByIp.set(String(s.ip), s);
|
||
}
|
||
});
|
||
|
||
const thresholdsByKey = new Map();
|
||
const makeKey = (fromKey, toKey, iface) => {
|
||
const a = String(fromKey || '');
|
||
const b = String(toKey || '');
|
||
const pair = [a, b].sort().join('__');
|
||
return `${pair}::${iface || ''}`;
|
||
};
|
||
|
||
tunnelThresholds.forEach((t) => {
|
||
const key = makeKey(t.fromKey, t.toKey, t.interfaceName || '');
|
||
const maxPingMs =
|
||
typeof t.maxPingMs === 'number'
|
||
? t.maxPingMs
|
||
: t.thresholdMs != null
|
||
? Number(t.thresholdMs) || 0
|
||
: 0;
|
||
const minDownloadMbps =
|
||
typeof t.minDownloadMbps === 'number'
|
||
? t.minDownloadMbps
|
||
: t.minDownMbps != null
|
||
? Number(t.minDownMbps) || 0
|
||
: 0;
|
||
const minUploadMbps =
|
||
typeof t.minUploadMbps === 'number'
|
||
? t.minUploadMbps
|
||
: t.minUpMbps != null
|
||
? Number(t.minUpMbps) || 0
|
||
: 0;
|
||
thresholdsByKey.set(key, {
|
||
maxPingMs,
|
||
minDownloadMbps,
|
||
minUploadMbps,
|
||
});
|
||
});
|
||
|
||
connectionsData.connections.forEach((c) => {
|
||
const fromIp = String(c.from);
|
||
const toIp = String(c.to);
|
||
if (!fromIp || !toIp || fromIp === toIp) return;
|
||
|
||
const fromServer = serversByIp.get(fromIp) || null;
|
||
const toServer = serversByIp.get(toIp) || null;
|
||
const fromName = fromServer?.name || fromServer?.dns || fromIp;
|
||
const toName = toServer?.name || toServer?.dns || toIp;
|
||
|
||
const fromKey = c.fromKey || fromIp;
|
||
const toKey = c.toKey || toIp;
|
||
const key = makeKey(fromKey, toKey, c.interfaceName || '');
|
||
const th = thresholdsByKey.get(key);
|
||
if (!th) return;
|
||
|
||
const maxPingMs = Math.max(0, Number(th.maxPingMs) || 0);
|
||
const minDownMbps = Math.max(0, Number(th.minDownloadMbps) || 0);
|
||
const minUpMbps = Math.max(0, Number(th.minUploadMbps) || 0);
|
||
|
||
const pingMapKey = edgePingKey(fromIp, toIp);
|
||
const pingMs = typeof pingMap[pingMapKey] === 'number' ? pingMap[pingMapKey] : null;
|
||
|
||
if (maxPingMs > 0 && pingMs != null && pingMs >= maxPingMs) {
|
||
alerts.push({
|
||
id: `tunnel-ping-${pingMapKey}-${maxPingMs}`,
|
||
type: 'tunnel_high_ping',
|
||
severity: 'warning',
|
||
title: 'Высокий пинг по туннелю',
|
||
description: `Пинг между ${fromName} и ${toName}: ${pingMs} ms (порог ${maxPingMs} ms).`,
|
||
entity: `${fromName} ⇄ ${toName}`,
|
||
entityId: `${c.fromKey || fromIp}__${c.toKey || toIp}__${c.interfaceName || ''}`,
|
||
link: '/network-map',
|
||
at: now,
|
||
});
|
||
}
|
||
|
||
if ((minDownMbps > 0 || minUpMbps > 0) && c.fromKey && c.toKey) {
|
||
const sk = speedKey(c.fromKey, c.toKey);
|
||
const speed = speedMap[sk];
|
||
if (speed && typeof speed === 'object') {
|
||
const downMbps =
|
||
typeof speed.tcpDownloadBps === 'number' ? speed.tcpDownloadBps / 1e6 : null;
|
||
const upMbps =
|
||
typeof speed.tcpUploadBps === 'number' ? speed.tcpUploadBps / 1e6 : null;
|
||
|
||
const downTooLow = minDownMbps > 0 && (downMbps == null || downMbps < minDownMbps);
|
||
const upTooLow = minUpMbps > 0 && (upMbps == null || upMbps < minUpMbps);
|
||
|
||
if (downTooLow || upTooLow) {
|
||
const parts = [];
|
||
if (downTooLow) {
|
||
parts.push(
|
||
`↓ ${downMbps != null ? downMbps.toFixed(1) : '—'} Мбит/с (порог ${minDownMbps} Мбит/с)`
|
||
);
|
||
}
|
||
if (upTooLow) {
|
||
parts.push(
|
||
`↑ ${upMbps != null ? upMbps.toFixed(1) : '—'} Мбит/с (порог ${minUpMbps} Мбит/с)`
|
||
);
|
||
}
|
||
const ifaceLabel = c.interfaceName ? `, интерфейс ${c.interfaceName}` : '';
|
||
alerts.push({
|
||
id: `tunnel-speed-${sk}-${minDownMbps}-${minUpMbps}`,
|
||
type: 'tunnel_low_speed',
|
||
severity: 'warning',
|
||
title: 'Низкая скорость по туннелю',
|
||
description: `Туннель между ${fromName} и ${toName}${ifaceLabel}: ${parts.join(
|
||
'; '
|
||
)}.`,
|
||
entity: `${fromName} ⇄ ${toName}`,
|
||
entityId: `${c.fromKey || fromIp}__${c.toKey || toIp}__${c.interfaceName || ''}`,
|
||
link: '/network-map',
|
||
at: now,
|
||
});
|
||
}
|
||
}
|
||
}
|
||
});
|
||
}
|
||
}
|
||
|
||
return res.json({
|
||
alerts,
|
||
total: alerts.length,
|
||
at: now,
|
||
});
|
||
} catch (error) {
|
||
console.error('getAlerts:', error);
|
||
return sendError(
|
||
res,
|
||
500,
|
||
error.message || 'Ошибка загрузки оповещений',
|
||
'E_ALERTS'
|
||
);
|
||
}
|
||
}
|
||
|
||
module.exports = {
|
||
getAlerts,
|
||
};
|