/** * Система оповещений: агрегирует проблемы из availability и resources/stats. * Критерии и пороги задаются в ui-settings.alertSettings. */ const { sendError } = require('../middleware/errorHandler'); const { readServersFromS3 } = require('./serversRoutes'); const { checkOneServerFast, loadUiSettingsSync } = require('./miscRoutes'); const { getResourceStatsData } = require('./resourceStatsRoutes'); const { getUptimeCacheData } = require('./mikrotikConfigRoutes'); const { buildConnections } = require('../services/networkMapScheduler'); const { readS3TextObject } = require('../services/s3Service'); /** Значения по умолчанию для настроек оповещений. */ const DEFAULT_ALERT_SETTINGS = { serverOffline: { enabled: true, offlineMinutes: 5 }, mikrotikUnreachable: { enabled: true, durationMinutes: 5 }, highCpu: { enabled: true, thresholdPercent: 85, durationMinutes: 10 }, highRam: { enabled: true, thresholdPercent: 85, durationMinutes: 10 }, highHdd: { enabled: true, thresholdPercent: 90, durationMinutes: 10 }, /** Глобальные пороги карты сети для UI-аналитики */ networkMapNodeThresholds: { lowSpeedMbps: 40, belowNormSpeedMbps: 80, normalSpeedMbps: 120, }, /** Индивидуальные пороги по туннелям (карта сети) */ tunnelThresholds: [], }; function getAlertSettings(uiSettings) { const raw = uiSettings?.alertSettings && typeof uiSettings.alertSettings === 'object' ? uiSettings.alertSettings : {}; return { serverOffline: { ...DEFAULT_ALERT_SETTINGS.serverOffline, ...raw.serverOffline }, mikrotikUnreachable: { ...DEFAULT_ALERT_SETTINGS.mikrotikUnreachable, ...raw.mikrotikUnreachable }, highCpu: { ...DEFAULT_ALERT_SETTINGS.highCpu, ...raw.highCpu }, highRam: { ...DEFAULT_ALERT_SETTINGS.highRam, ...raw.highRam }, highHdd: { ...DEFAULT_ALERT_SETTINGS.highHdd, ...raw.highHdd }, networkMapNodeThresholds: { ...DEFAULT_ALERT_SETTINGS.networkMapNodeThresholds, ...(raw.networkMapNodeThresholds && typeof raw.networkMapNodeThresholds === 'object' ? raw.networkMapNodeThresholds : {}), }, tunnelThresholds: Array.isArray(raw.tunnelThresholds) ? raw.tunnelThresholds : [], }; } function edgePingKey(a, b) { return [String(a), String(b)].sort().join(':'); } function speedKey(key1, key2) { return [String(key1), String(key2)].sort().join(':'); } /** * GET /api/alerts * Возвращает список активных оповещений (учёт настроек из ui-settings.alertSettings). */ async function getAlerts(req, res) { try { const uiSettings = await loadUiSettingsSync(); const settings = getAlertSettings(uiSettings); const servers = await readServersFromS3(); const alerts = []; const now = new Date().toISOString(); // 1) Доступность серверов: jumphost/home — по правилам Uptime Monitor (кеш), остальные — TCP 80/443 if (settings.serverOffline.enabled) { const routerTypes = ['jumphost', 'home']; const { results: uptimeResults } = await getUptimeCacheData(); const nonRouterServers = servers.filter( (s) => !routerTypes.includes(String(s.type || '').toLowerCase()) ); const availabilityChecks = await Promise.allSettled( nonRouterServers.map((s) => checkOneServerFast(s)) ); servers.forEach((server) => { const serverId = server.id || server.dns || server.ip; const name = server.name || server.dns || server.ip || server.id || 'Сервер'; const isRouter = routerTypes.includes(String(server.type || '').toLowerCase()); let offline = false; let description = ''; if (isRouter) { const entry = uptimeResults[serverId]; const offlineMinutes = Math.max(1, Math.min(1440, Number(settings.serverOffline.offlineMinutes) || 5)); const offlineSinceMs = entry && entry.lastCheckTs ? (Date.now() - entry.lastCheckTs) : 0; offline = entry && entry.ok === false && offlineSinceMs >= offlineMinutes * 60 * 1000; description = offline ? `${name} недоступен более ${offlineMinutes} мин (по проверке Uptime Monitor).` : ''; } else { const idx = nonRouterServers.findIndex( (s) => (s.id || s.dns || s.ip) === serverId ); const online = idx >= 0 && availabilityChecks[idx].status === 'fulfilled' && availabilityChecks[idx].value; offline = !online; description = offline ? `${name} не отвечает на TCP (80/443).` : ''; } if (offline && description) { alerts.push({ id: `server-offline-${serverId}`, type: 'server_offline', severity: 'critical', title: 'Сервер недоступен', description, entity: name, entityId: serverId, link: isRouter ? '/uptime-monitor' : '/servers', at: now, }); } }); } // 2) Ресурсы роутеров (MikroTik): ошибка доступа, высокое CPU/RAM/HDD let resourceData; try { resourceData = await getResourceStatsData(); } catch (err) { console.warn('alerts: getResourceStatsData failed', err?.message); resourceData = { routers: [] }; } const thCpu = Math.max(1, Math.min(100, Number(settings.highCpu.thresholdPercent) || 85)); const thRam = Math.max(1, Math.min(100, Number(settings.highRam.thresholdPercent) || 85)); const thHdd = Math.max(1, Math.min(100, Number(settings.highHdd.thresholdPercent) || 90)); const routers = resourceData.routers || []; routers.forEach((router) => { const { serverId, name, error, resource } = router; const entityName = name || serverId; if (error && settings.mikrotikUnreachable.enabled) { alerts.push({ id: `mikrotik-error-${serverId}`, type: 'mikrotik_unreachable', severity: 'critical', title: 'MikroTik недоступен', description: `${entityName}: ${error}`, entity: entityName, entityId: serverId, link: '/resource-stats', at: now, }); return; } if (!resource) return; if (settings.highCpu.enabled && resource.cpuLoad != null && resource.cpuLoad >= thCpu) { alerts.push({ id: `cpu-high-${serverId}`, type: 'high_cpu', severity: 'warning', title: 'Высокое использование CPU', description: `${entityName}: ${resource.cpuLoad}% (порог ${thCpu}%).`, entity: entityName, entityId: serverId, value: resource.cpuLoad, threshold: thCpu, link: '/resource-stats', at: now, }); } if (settings.highRam.enabled && resource.memoryUsagePercent != null && resource.memoryUsagePercent >= thRam) { alerts.push({ id: `ram-high-${serverId}`, type: 'high_ram', severity: 'warning', title: 'Высокое использование RAM', description: `${entityName}: ${resource.memoryUsagePercent}% (порог ${thRam}%).`, entity: entityName, entityId: serverId, value: resource.memoryUsagePercent, threshold: thRam, link: '/resource-stats', at: now, }); } if (settings.highHdd.enabled && resource.hddUsagePercent != null && resource.hddUsagePercent >= thHdd) { alerts.push({ id: `hdd-high-${serverId}`, type: 'high_hdd', severity: 'warning', title: 'Высокое использование диска', description: `${entityName}: ${resource.hddUsagePercent}% (порог ${thHdd}%).`, entity: entityName, entityId: serverId, value: resource.hddUsagePercent, threshold: thHdd, link: '/resource-stats', at: now, }); } }); // 3) Туннели (карта сети): индивидуальные пороги по пингу и скорости const rawTunnelThresholds = Array.isArray(settings.tunnelThresholds) ? settings.tunnelThresholds : []; const tunnelThresholds = rawTunnelThresholds.filter( (t) => t && (t.fromKey || t.toKey) && t.enabled !== false ); if (tunnelThresholds.length > 0) { let cache = null; let connectionsData = null; try { const [rawCache, built] = await Promise.all([ readS3TextObject('network-map-cache/latest.json').catch(() => null), buildConnections().catch(() => null), ]); if (rawCache?.body) { const parsed = JSON.parse(rawCache.body); if (parsed && typeof parsed === 'object') cache = parsed; } if (built && typeof built === 'object') connectionsData = built; } catch (e) { // Ошибки при чтении кеша/конфигурации не должны ломать остальные оповещения console.warn('alerts: network-map-cache read failed', e?.message || e); } if (cache && connectionsData && Array.isArray(connectionsData.connections)) { const pingMap = cache.pingMap && typeof cache.pingMap === 'object' ? cache.pingMap : {}; const speedMap = cache.speedMap && typeof cache.speedMap === 'object' ? cache.speedMap : {}; const serversByIp = new Map(); servers.forEach((s) => { if (s && s.ip) { serversByIp.set(String(s.ip), s); } }); const thresholdsByKey = new Map(); const makeKey = (fromKey, toKey, iface) => { const a = String(fromKey || ''); const b = String(toKey || ''); const pair = [a, b].sort().join('__'); return `${pair}::${iface || ''}`; }; tunnelThresholds.forEach((t) => { const key = makeKey(t.fromKey, t.toKey, t.interfaceName || ''); const maxPingMs = typeof t.maxPingMs === 'number' ? t.maxPingMs : t.thresholdMs != null ? Number(t.thresholdMs) || 0 : 0; const minDownloadMbps = typeof t.minDownloadMbps === 'number' ? t.minDownloadMbps : t.minDownMbps != null ? Number(t.minDownMbps) || 0 : 0; const minUploadMbps = typeof t.minUploadMbps === 'number' ? t.minUploadMbps : t.minUpMbps != null ? Number(t.minUpMbps) || 0 : 0; thresholdsByKey.set(key, { maxPingMs, minDownloadMbps, minUploadMbps, }); }); connectionsData.connections.forEach((c) => { const fromIp = String(c.from); const toIp = String(c.to); if (!fromIp || !toIp || fromIp === toIp) return; const fromServer = serversByIp.get(fromIp) || null; const toServer = serversByIp.get(toIp) || null; const fromName = fromServer?.name || fromServer?.dns || fromIp; const toName = toServer?.name || toServer?.dns || toIp; const fromKey = c.fromKey || fromIp; const toKey = c.toKey || toIp; const key = makeKey(fromKey, toKey, c.interfaceName || ''); const th = thresholdsByKey.get(key); if (!th) return; const maxPingMs = Math.max(0, Number(th.maxPingMs) || 0); const minDownMbps = Math.max(0, Number(th.minDownloadMbps) || 0); const minUpMbps = Math.max(0, Number(th.minUploadMbps) || 0); const pingMapKey = edgePingKey(fromIp, toIp); const pingMs = typeof pingMap[pingMapKey] === 'number' ? pingMap[pingMapKey] : null; if (maxPingMs > 0 && pingMs != null && pingMs >= maxPingMs) { alerts.push({ id: `tunnel-ping-${pingMapKey}-${maxPingMs}`, type: 'tunnel_high_ping', severity: 'warning', title: 'Высокий пинг по туннелю', description: `Пинг между ${fromName} и ${toName}: ${pingMs} ms (порог ${maxPingMs} ms).`, entity: `${fromName} ⇄ ${toName}`, entityId: `${c.fromKey || fromIp}__${c.toKey || toIp}__${c.interfaceName || ''}`, link: '/network-map', at: now, }); } if ((minDownMbps > 0 || minUpMbps > 0) && c.fromKey && c.toKey) { const sk = speedKey(c.fromKey, c.toKey); const speed = speedMap[sk]; if (speed && typeof speed === 'object') { const downMbps = typeof speed.tcpDownloadBps === 'number' ? speed.tcpDownloadBps / 1e6 : null; const upMbps = typeof speed.tcpUploadBps === 'number' ? speed.tcpUploadBps / 1e6 : null; const downTooLow = minDownMbps > 0 && (downMbps == null || downMbps < minDownMbps); const upTooLow = minUpMbps > 0 && (upMbps == null || upMbps < minUpMbps); if (downTooLow || upTooLow) { const parts = []; if (downTooLow) { parts.push( `↓ ${downMbps != null ? downMbps.toFixed(1) : '—'} Мбит/с (порог ${minDownMbps} Мбит/с)` ); } if (upTooLow) { parts.push( `↑ ${upMbps != null ? upMbps.toFixed(1) : '—'} Мбит/с (порог ${minUpMbps} Мбит/с)` ); } const ifaceLabel = c.interfaceName ? `, интерфейс ${c.interfaceName}` : ''; alerts.push({ id: `tunnel-speed-${sk}-${minDownMbps}-${minUpMbps}`, type: 'tunnel_low_speed', severity: 'warning', title: 'Низкая скорость по туннелю', description: `Туннель между ${fromName} и ${toName}${ifaceLabel}: ${parts.join( '; ' )}.`, entity: `${fromName} ⇄ ${toName}`, entityId: `${c.fromKey || fromIp}__${c.toKey || toIp}__${c.interfaceName || ''}`, link: '/network-map', at: now, }); } } } }); } } return res.json({ alerts, total: alerts.length, at: now, }); } catch (error) { console.error('getAlerts:', error); return sendError( res, 500, error.message || 'Ошибка загрузки оповещений', 'E_ALERTS' ); } } module.exports = { getAlerts, };