/** * Разные утилитарные роуты (auto-urls, history, s3 meta, availability, etc.) */ const { s3, BUCKET_NAME, writeS3TextObject, headMeta, streamToString } = require('../services/s3Service'); const { sendError, sendOk } = require('../middleware/errorHandler'); const { resourceToKey, toIso } = require('../utils/helpers'); const { GetObjectCommand, PutObjectCommand, ListObjectVersionsCommand, CopyObjectCommand } = require('@aws-sdk/client-s3'); const validators = require('../lib/validators'); const net = require('net'); const http = require('http'); const https = require('https'); const { URL } = require('url'); // GET /api/s3/last-modified async function getS3LastModified(req, res) { try { const keys = [ { name: 'domainsNew', key: 'bgp_data/domains_community.txt' }, { name: 'asns', key: 'bgp_data/asns.txt' }, { name: 'servers', key: 'servers.json' }, { name: 'filters', key: 'filters.json' }, { name: 'ipRanges', key: 'bgp_data/ips.txt' }, { name: 'uiSettings', key: 'bgp_data/rt_ui_settings.json' } ]; const results = await Promise.allSettled( keys.map(k => headMeta(k.key)) ); const out = {}; results.forEach((r, idx) => { const name = keys[idx].name; if (r.status === 'fulfilled' && r.value) { out[name] = r.value; } else { out[name] = null; } }); res.json(out); } catch (error) { console.error('Error fetching last modified dates from S3:', error); return sendError(res, 500, 'Error fetching last modified dates from S3', 'E_S3'); } } // GET /api/history/:resource async function getHistory(req, res) { const { resource } = req.params; const { countOnly, format } = req.query || {}; const key = resourceToKey(resource); if (!key) return sendError(res, 400, 'Unknown resource', 'E_RESOURCE'); try { const out = await s3.send(new ListObjectVersionsCommand({ Bucket: BUCKET_NAME, Prefix: key, MaxKeys: 50 })); const versionsAll = (out.Versions || []).filter(v => v.Key === key); if (countOnly === 'true') { const total = versionsAll.length; return res.json(format === 'std' ? { items: [], total, meta: {} } : { total }); } const versions = versionsAll.slice(0, 10).map(v => ({ versionId: v.VersionId, isLatest: v.IsLatest, lastModified: toIso(v.LastModified), size: v.Size, etag: v.ETag })); return res.json(format === 'std' ? { items: versions, total: versionsAll.length, meta: {} } : { items: versions }); } catch (e) { console.error('history error', e); return sendError(res, 500, 'Error reading history', 'E_S3', { error: String(e?.message || e) }); } } // POST /api/history/:resource/rollback async function postRollback(req, res) { const { resource } = req.params; const { versionId } = req.body || {}; const key = resourceToKey(resource); if (!key || !versionId) { return sendError(res, 400, 'Bad request', 'E_BAD_REQUEST'); } try { await s3.send(new CopyObjectCommand({ Bucket: BUCKET_NAME, CopySource: `/${BUCKET_NAME}/${encodeURIComponent(key)}?versionId=${encodeURIComponent(versionId)}`, Key: key })); const meta = await headMeta(key); return sendOk(res, meta); } catch (e) { console.error('rollback error', e); return sendError(res, 500, 'Error rollback', 'E_S3', { error: String(e?.message || e) }); } } // GET /api/auto-urls async function getAutoUrls(req, res) { const { checkIfNoneMatch } = require('../middleware/errorHandler'); try { const { HeadObjectCommand } = require('@aws-sdk/client-s3'); const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/auto_url/urls.txt' })).catch(() => null); const etag = head?.ETag || null; if (etag) res.set('ETag', String(etag)); if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString()); if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength)); if (checkIfNoneMatch(req, res, etag)) return; const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/auto_url/urls.txt' })); const fileContent = await streamToString(data.Body); const urls = fileContent.split('\n').filter(line => line).map(line => { const parts = line.trim().split(/\s+/); return { url: parts[0] || '', community: parts[1] || '' }; }); res.json(urls); } catch (error) { if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) { res.json([]); } else { console.error(error); return sendError(res, 500, 'Error reading auto URLs from S3', 'E_S3'); } } } // POST /api/auto-urls async function postAutoUrls(req, res) { const { urls } = req.body; const fileContent = urls.map(u => `${u.url} ${u.community}`).join('\n'); try { const meta = await writeS3TextObject('bgp_data/auto_url/urls.txt', fileContent); return sendOk(res, meta); } catch (error) { console.error(error); return sendError(res, 500, 'Error writing auto URLs to S3', 'E_S3'); } } // POST /api/auto-urls/process async function processAutoUrls(req, res) { try { // Загрузка URL списков let urls = []; try { const urlsData = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/auto_url/urls.txt' })); const urlsContent = await streamToString(urlsData.Body); urls = urlsContent.split('\n').filter(line => line).map(line => { const parts = line.trim().split(/\s+/); return { url: parts[0] || '', community: parts[1] || '' }; }); } catch (error) { if (error.code !== 'NoSuchKey') throw error; } if (urls.length === 0) { return sendError(res, 400, 'No URLs to process', 'E_BAD_REQUEST'); } // Загрузка текущих IP и доменов let currentIps = []; try { const ipsData = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt' })); const ipsContent = await streamToString(ipsData.Body); currentIps = ipsContent.split('\n').filter(line => line).map(line => { const parts = line.trim().split(/\s+/); return { ipRange: parts[0] || '', community: parts[1] || '' }; }); } catch (error) { if (error.code !== 'NoSuchKey') throw error; } let currentDomains = []; try { const dData = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt' })); const dContent = await streamToString(dData.Body); currentDomains = dContent.split('\n').filter(line => line).map(line => { const parts = line.trim().split(/\s+/); return { domain: (parts[0] || '').toLowerCase(), community: parts[1] || '' }; }); } catch (error) { if (error.code !== 'NoSuchKey') throw error; } // Обработка URL const newIps = []; const newDomains = []; for (const urlData of urls) { try { const url = String(urlData.url || '').trim(); const community = String(urlData.community || '').trim(); if (!url || !community) continue; const content = await new Promise((resolve, reject) => { const protocol = url.startsWith('https:') ? https : http; const req = protocol.get(url, (r) => { let data = ''; r.on('data', (chunk) => { data += chunk; }); r.on('end', () => resolve(data)); }); req.on('error', reject); req.setTimeout(15000, () => req.destroy()); }); const lines = content.split('\n'); for (const raw of lines) { const line = String(raw || '').trim(); if (!line || line.startsWith('#') || line.startsWith('//')) continue; const token = line.split(/\s+/)[0]?.trim(); if (!token) continue; if (validators.isValidCIDRv4(token) || validators.isValidCIDRv6(token)) { newIps.push({ ipRange: token, community }); } else if (validators.isValidIPv4(token)) { newIps.push({ ipRange: `${token}/32`, community }); } else if (validators.isValidDomain(token)) { newDomains.push({ domain: token.toLowerCase(), community }); } } } catch (error) { console.error(`Error processing URL ${urlData.url}:`, error); } } // Объединение и дедупликация const existingIpRanges = new Set(currentIps.map(i => i.ipRange)); const uniqueNewIps = newIps.filter(i => !existingIpRanges.has(i.ipRange)); const allIps = [...currentIps, ...uniqueNewIps]; const existingDomains = new Set(currentDomains.map(d => d.domain)); const uniqueNewDomains = newDomains.filter(d => !existingDomains.has(d.domain)); const allDomains = [...currentDomains, ...uniqueNewDomains]; // Сохранение const updatedIpsContent = allIps.map(i => `${i.ipRange} ${i.community}`).join('\n'); await s3.send(new PutObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt', Body: updatedIpsContent, ContentType: 'text/plain' })); const updatedDomainsContent = allDomains.map(d => `${d.domain} ${d.community}`).join('\n'); await s3.send(new PutObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt', Body: updatedDomainsContent, ContentType: 'text/plain' })); const msg = `Обработано ${urls.length} URL, добавлено ${uniqueNewIps.length} IP и ${uniqueNewDomains.length} доменов`; return res.json({ success: true, message: msg, processedUrls: urls.length, newIpsCount: uniqueNewIps.length, newDomainsCount: uniqueNewDomains.length, totalIpsCount: allIps.length, totalDomainsCount: allDomains.length }); } catch (error) { console.error('Error processing auto URLs:', error); return sendError(res, 500, 'Error processing auto URLs', 'E_S3'); } } // GET /api/servers/availability const availabilityCache = { at: 0, data: null }; function tcpCheck(host, port, timeoutMs) { return new Promise((resolve) => { const socket = new net.Socket(); let settled = false; const settle = (ok) => { if (!settled) { settled = true; try { socket.destroy(); } catch {} resolve(ok); } }; socket.setTimeout(timeoutMs, () => settle(false)); socket.once('error', () => settle(false)); socket.connect(port, host, () => settle(true)); }); } function anyTrue(promises) { return new Promise((resolve) => { if (!Array.isArray(promises) || promises.length === 0) return resolve(false); let remaining = promises.length; let resolved = false; for (const p of promises) { Promise.resolve(p).then((v) => { if (v && !resolved) { resolved = true; resolve(true); } }).finally(() => { remaining -= 1; if (remaining === 0 && !resolved) resolve(false); }); } }); } async function checkOneServerFast(srv, perSocketTimeoutMs = 800, perServerBudgetMs = 1000) { const hosts = []; if (srv.ip) hosts.push(String(srv.ip)); if (srv.dns) hosts.push(String(srv.dns)); const tryOneHost = (host) => anyTrue([ tcpCheck(host, 443, perSocketTimeoutMs), tcpCheck(host, 80, perSocketTimeoutMs), ]); const run = anyTrue(hosts.map((h) => tryOneHost(h))); const timeout = new Promise((resolve) => setTimeout(() => resolve(false), perServerBudgetMs)); return Promise.race([run, timeout]); } async function getServersAvailability(req, res) { try { const ttlSeconds = Math.max(0, Math.min(300, Number(req.query.ttlSeconds) || 30)); const now = Date.now(); if (availabilityCache.data && (now - availabilityCache.at) < ttlSeconds * 1000) { return res.json({ ...availabilityCache.data, cached: true }); } const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'servers.json' })); let servers = []; try { servers = JSON.parse(await streamToString(data.Body)); if (!Array.isArray(servers)) servers = []; } catch { servers = []; } const checks = await Promise.allSettled(servers.map((s) => checkOneServerFast(s))); const statuses = servers.map((s, i) => ({ ip: s.ip, dns: s.dns, online: checks[i].status === 'fulfilled' ? Boolean(checks[i].value) : false })); const online = statuses.filter((x) => x.online).length; const payload = { online, total: servers.length, statuses }; availabilityCache.at = Date.now(); availabilityCache.data = payload; res.json(payload); } catch (e) { console.error('availability error', e); res.status(500).json({ online: 0, total: 0, statuses: [] }); } } // POST /api/update-bgp/background async function updateBgpBackground(req, res) { try { const targetUrl = process.env.BGP_BACKGROUND_URL; if (!targetUrl) { return sendError(res, 500, 'BGP_BACKGROUND_URL is not configured', 'E_CONFIG'); } const u = new URL(targetUrl); const client = u.protocol === 'https:' ? https : http; const options = { method: 'POST', hostname: u.hostname, port: u.port || (u.protocol === 'https:' ? 443 : 80), path: `${u.pathname}${u.search || ''}`, headers: { 'Content-Type': 'application/json', 'Accept': 'application/json', }, timeout: 15000, }; const body = req.body && Object.keys(req.body).length ? JSON.stringify(req.body) : ''; const upstream = client.request(options, (r) => { let data = ''; r.setEncoding('utf8'); r.on('data', (chunk) => { data += chunk; }); r.on('end', () => { const status = r.statusCode || 502; try { const json = data ? JSON.parse(data) : {}; return res.status(status).json(json); } catch (_) { return res.status(status).json({ ok: status >= 200 && status < 300, data }); } }); }); upstream.on('timeout', () => { try { upstream.destroy(); } catch {} return sendError(res, 504, 'Upstream timeout', 'E_UPSTREAM_TIMEOUT'); }); upstream.on('error', (e) => { return sendError(res, 502, 'Upstream error', 'E_UPSTREAM', { error: String(e?.message || e) }); }); if (body) upstream.write(body); upstream.end(); } catch (e) { return sendError(res, 500, 'Proxy error', 'E_PROXY', { error: String(e?.message || e) }); } } // GET /api/ws/url async function getWsUrl(req, res) { try { const settings = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/rt_ui_settings.json' })).then(async (d) => { try { return JSON.parse(await streamToString(d.Body)); } catch { return {}; } }).catch(() => ({})); const url = settings?.wsUpdateUrl || process.env.WS_UPDATE_URL || ''; return res.json({ url }); } catch (e) { return res.json({ url: '' }); } } // GET/POST /api/ui-settings async function getUiSettings(req, res) { const { checkIfNoneMatch } = require('../middleware/errorHandler'); try { const { HeadObjectCommand } = require('@aws-sdk/client-s3'); const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/rt_ui_settings.json' })).catch(() => null); const etag = head?.ETag || null; if (etag) res.set('ETag', String(etag)); if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString()); if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength)); if (checkIfNoneMatch(req, res, etag)) return; const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/rt_ui_settings.json' })); const jsonText = await streamToString(data.Body); let settings = {}; try { const parsed = JSON.parse(jsonText); if (parsed && typeof parsed === 'object') settings = parsed; } catch (parseError) { settings = {}; } return res.json(settings); } catch (error) { if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) { return res.json({}); } console.error('Error reading ui settings from S3:', error); return sendError(res, 500, 'Error reading UI settings from S3', 'E_S3'); } } async function postUiSettings(req, res) { const { settings, etag } = req.body || {}; const payload = (settings && typeof settings === 'object') ? settings : {}; try { const { headS3ObjectEtag } = require('../services/s3Service'); let current = null; const ifMatch = req.headers['if-match'] ? String(req.headers['if-match']).replace(/\"/g,'"') : null; try { current = await headS3ObjectEtag('bgp_data/rt_ui_settings.json'); } catch {} if (current && (ifMatch || etag) && current !== String(ifMatch || etag)) { const meta = await headMeta('bgp_data/rt_ui_settings.json'); return sendError(res, 412, 'Precondition Failed: ETag mismatch', 'E_ETAG_MISMATCH', { currentEtag: current, meta }); } } catch {} try { const { writeS3JsonObject } = require('../services/s3Service'); const meta = await writeS3JsonObject('bgp_data/rt_ui_settings.json', payload); return sendOk(res, meta); } catch (error) { console.error('Error writing UI settings to S3:', error); return sendError(res, 500, 'Error writing UI settings to S3', 'E_S3', { error: String(error?.message || error) }); } } module.exports = { getS3LastModified, getHistory, postRollback, getAutoUrls, postAutoUrls, processAutoUrls, getServersAvailability, updateBgpBackground, getWsUrl, getUiSettings, postUiSettings, };