diff --git a/backend/server.js b/backend/server.js index a9d7e8b..aeb41c4 100644 --- a/backend/server.js +++ b/backend/server.js @@ -1724,118 +1724,165 @@ app.post('/api/auto-urls', async (req, res) => { // Process auto URLs and update IPs app.post('/api/auto-urls/process', async (req, res) => { - const https = require('https'); - const http = require('http'); - + const https = require('https'); + const http = require('http'); + + // Helpers + const isValidIPv4 = (ip) => { + const octets = String(ip || '').trim().split('.'); + if (octets.length !== 4) return false; + return octets.every(o => /^\d{1,3}$/.test(o) && Number(o) >= 0 && Number(o) <= 255); + }; + const isValidCidrV4 = (value) => { + const v = String(value || '').trim(); + const parts = v.split('/'); + if (parts.length !== 2) return false; + const [ip, mask] = parts; + if (!isValidIPv4(ip)) return false; + if (!/^\d{1,2}$/.test(mask)) return false; + const m = Number(mask); + return m >= 0 && m <= 32; + }; + const isValidDomain = (value) => { + const v = String(value || '').trim().toLowerCase(); + if (v.startsWith('#')) return false; // comment line + return /^([a-z0-9-]+\.)+[a-z]{2,}$/i.test(v); + }; + + try { + // Load configured auto URLs + const urlsParams = { + Bucket: BUCKET_NAME, + Key: 'bgp_data/auto_url/urls.txt', + }; + + let urls = []; try { - // Get current auto URLs - const urlsParams = { - Bucket: BUCKET_NAME, - Key: 'bgp_data/auto_url/urls.txt', - }; - - let urls = []; - try { - const urlsData = await s3.getObject(urlsParams).promise(); - const urlsContent = urlsData.Body.toString('utf-8'); - urls = urlsContent.split('\n').filter(line => line).map(line => { - const parts = line.trim().split(/\s+/); - return { url: parts[0] || '', community: parts[1] || '' }; - }); - } catch (error) { - if (error.code !== 'NoSuchKey') { - throw error; - } - } - - if (urls.length === 0) { - return sendError(res, 400, 'No URLs to process', 'E_BAD_REQUEST'); - } - - // Get current IPs - const ipsParams = { - Bucket: BUCKET_NAME, - Key: 'bgp_data/ips.txt', - }; - - let currentIps = []; - try { - const ipsData = await s3.getObject(ipsParams).promise(); - const ipsContent = ipsData.Body.toString('utf-8'); - currentIps = ipsContent.split('\n').filter(line => line).map(line => { - const parts = line.trim().split(/\s+/); - return { ipRange: parts[0] || '', community: parts[1] || '' }; - }); - } catch (error) { - if (error.code !== 'NoSuchKey') { - throw error; - } - } - - // Process each URL - const newIps = []; - for (const urlData of urls) { - try { - const url = urlData.url.trim(); - const community = urlData.community.trim(); - - if (!url || !community) continue; - - // Download content from URL - const content = await new Promise((resolve, reject) => { - const protocol = url.startsWith('https:') ? https : http; - const req = protocol.get(url, (res) => { - let data = ''; - res.on('data', (chunk) => data += chunk); - res.on('end', () => resolve(data)); - }); - req.on('error', reject); - req.setTimeout(10000, () => req.destroy()); - }); - - // Parse IPs from content - const lines = content.split('\n'); - for (const line of lines) { - const ip = line.trim(); - if (ip && (ip.includes('.') || ip.includes(':'))) { - newIps.push({ ipRange: ip, community }); - } - } - - } catch (error) { - console.error(`Error processing URL ${urlData.url}:`, error); - } - } - - // Merge with existing IPs (avoid duplicates) - const existingIpRanges = new Set(currentIps.map(ip => ip.ipRange)); - const uniqueNewIps = newIps.filter(ip => !existingIpRanges.has(ip.ipRange)); - - const allIps = [...currentIps, ...uniqueNewIps]; - - // Save updated IPs - const updatedIpsContent = allIps.map(ip => `${ip.ipRange} ${ip.community}`).join('\n'); - const updateParams = { - Bucket: BUCKET_NAME, - Key: 'bgp_data/ips.txt', - Body: updatedIpsContent, - ContentType: 'text/plain', - }; - - await s3.putObject(updateParams).promise(); - - res.json({ - success: true, - message: `Обработано ${urls.length} URL, добавлено ${uniqueNewIps.length} новых IP-адресов`, - processedUrls: urls.length, - newIpsCount: uniqueNewIps.length, - totalIpsCount: allIps.length - }); - + const urlsData = await s3.getObject(urlsParams).promise(); + const urlsContent = urlsData.Body.toString('utf-8'); + urls = urlsContent.split('\n').filter(line => line).map(line => { + const parts = line.trim().split(/\s+/); + return { url: parts[0] || '', community: parts[1] || '' }; + }); } catch (error) { - console.error('Error processing auto URLs:', error); - return sendError(res, 500, 'Error processing auto URLs', 'E_S3'); + if (error.code !== 'NoSuchKey') { + throw error; + } } + + if (urls.length === 0) { + return sendError(res, 400, 'No URLs to process', 'E_BAD_REQUEST'); + } + + // Load current IP ranges + const ipsParams = { Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt' }; + let currentIps = []; + try { + const ipsData = await s3.getObject(ipsParams).promise(); + const ipsContent = ipsData.Body.toString('utf-8'); + currentIps = ipsContent.split('\n').filter(line => line).map(line => { + const parts = line.trim().split(/\s+/); + return { ipRange: parts[0] || '', community: parts[1] || '' }; + }); + } catch (error) { + if (error.code !== 'NoSuchKey') throw error; + } + + // Load current domains + const domainsParams = { Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt' }; + let currentDomains = []; + try { + const dData = await s3.getObject(domainsParams).promise(); + const dContent = dData.Body.toString('utf-8'); + currentDomains = dContent.split('\n').filter(line => line).map(line => { + const parts = line.trim().split(/\s+/); + return { domain: (parts[0] || '').toLowerCase(), community: parts[1] || '' }; + }); + } catch (error) { + if (error.code !== 'NoSuchKey') throw error; + } + + // Process each URL + const newIps = []; + const newDomains = []; + for (const urlData of urls) { + try { + const url = String(urlData.url || '').trim(); + const community = String(urlData.community || '').trim(); + if (!url || !community) continue; + + // Download content + const content = await new Promise((resolve, reject) => { + const protocol = url.startsWith('https:') ? https : http; + const req = protocol.get(url, (r) => { + let data = ''; + r.on('data', (chunk) => { data += chunk; }); + r.on('end', () => resolve(data)); + }); + req.on('error', reject); + req.setTimeout(15000, () => req.destroy()); + }); + + const lines = content.split('\n'); + for (const raw of lines) { + const line = String(raw || '').trim(); + if (!line) continue; + if (line.startsWith('#') || line.startsWith('//')) continue; + const token = line.split(/\s+/)[0]?.trim(); + if (!token) continue; + + // Decide destination + if (isValidCidrV4(token)) { + newIps.push({ ipRange: token, community }); + continue; + } + if (isValidIPv4(token)) { + // single IPv4 → normalize to /32 + newIps.push({ ipRange: `${token}/32`, community }); + continue; + } + if (isValidDomain(token)) { + newDomains.push({ domain: token.toLowerCase(), community }); + continue; + } + // ignore everything else + } + } catch (error) { + console.error(`Error processing URL ${urlData.url}:`, error); + } + } + + // Merge & deduplicate + const existingIpRanges = new Set(currentIps.map(i => i.ipRange)); + const uniqueNewIps = newIps.filter(i => !existingIpRanges.has(i.ipRange)); + const allIps = [...currentIps, ...uniqueNewIps]; + + const existingDomains = new Set(currentDomains.map(d => d.domain)); + const uniqueNewDomains = newDomains.filter(d => !existingDomains.has(d.domain)); + const allDomains = [...currentDomains, ...uniqueNewDomains]; + + // Save updated IPs + const updatedIpsContent = allIps.map(i => `${i.ipRange} ${i.community}`).join('\n'); + await s3.putObject({ Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt', Body: updatedIpsContent, ContentType: 'text/plain' }).promise(); + + // Save updated Domains + const updatedDomainsContent = allDomains.map(d => `${d.domain} ${d.community}`).join('\n'); + await s3.putObject({ Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt', Body: updatedDomainsContent, ContentType: 'text/plain' }).promise(); + + const msg = `Обработано ${urls.length} URL, добавлено ${uniqueNewIps.length} IP и ${uniqueNewDomains.length} доменов`; + return res.json({ + success: true, + message: msg, + processedUrls: urls.length, + newIpsCount: uniqueNewIps.length, + newDomainsCount: uniqueNewDomains.length, + totalIpsCount: allIps.length, + totalDomainsCount: allDomains.length + }); + } catch (error) { + console.error('Error processing auto URLs:', error); + return sendError(res, 500, 'Error processing auto URLs', 'E_S3'); + } }); // --- Proxy: Background BGP Update (avoids CORS from browser) --- diff --git a/frontend/src/ASNsNewManager.jsx b/frontend/src/ASNsNewManager.jsx index 1f914a2..f48f29d 100644 --- a/frontend/src/ASNsNewManager.jsx +++ b/frontend/src/ASNsNewManager.jsx @@ -407,8 +407,8 @@ function ASNsNewManager() { disableExport={items.length === 0} onClear={clearInvalid} disableClear={items.length === 0} - onClearCommunities={() => setClearCommunitiesOpen(true)} - disableClearCommunities={items.length === 0} + onClearCommunities={() => { if (!filterCommunity) { setError('Выберите community в фильтре для очистки'); return; } setClearCommunitiesOpen(true); }} + disableClearCommunities={items.length === 0 || !filterCommunity} onSave={handleSaveChanges} disableSave={loading} onHistory={() => setHistoryOpen(true)} @@ -694,11 +694,11 @@ function ASNsNewManager() { /> { setItems(prev => prev.map(i => ({ ...i, community: '' }))); setClearCommunitiesOpen(false); }} + onConfirm={() => { setItems(prev => prev.filter(i => i.community !== filterCommunity)); setClearCommunitiesOpen(false); }} onCancel={() => setClearCommunitiesOpen(false)} /> {/* datalist больше не нужен, т.к. используем кастомный автокомплит */} diff --git a/frontend/src/AutoUrlManager.jsx b/frontend/src/AutoUrlManager.jsx index a3f496d..716cb69 100644 --- a/frontend/src/AutoUrlManager.jsx +++ b/frontend/src/AutoUrlManager.jsx @@ -158,7 +158,7 @@ function AutoUrlManager() {

Автоматические URL

-
Управление автоматическими URL-адресами для загрузки IP-списков
+
Управление автоматическими URL-адресами для загрузки списков IP и доменов
@@ -338,7 +338,7 @@ function AutoUrlManager() { ) : ( <> - Загрузить IP-списки + Загрузить списки )} @@ -358,7 +358,7 @@ function AutoUrlManager() { diff --git a/frontend/src/DomainsNewManager.jsx b/frontend/src/DomainsNewManager.jsx index a42bf45..7a948f3 100644 --- a/frontend/src/DomainsNewManager.jsx +++ b/frontend/src/DomainsNewManager.jsx @@ -414,8 +414,8 @@ function DomainsNewManager() { disableExport={items.length === 0} onClear={clearInvalid} disableClear={items.length === 0} - onClearCommunities={() => setClearCommunitiesOpen(true)} - disableClearCommunities={items.length === 0} + onClearCommunities={() => { if (!filterCommunity) { setError('Выберите community в фильтре для очистки'); return; } setClearCommunitiesOpen(true); }} + disableClearCommunities={items.length === 0 || !filterCommunity} onSave={handleSaveChanges} disableSave={loading} onHistory={() => setHistoryOpen(true)} @@ -703,11 +703,11 @@ function DomainsNewManager() { /> { setItems(prev => prev.map(i => ({ ...i, community: '' }))); setClearCommunitiesOpen(false); }} + onConfirm={() => { setItems(prev => prev.filter(i => i.community !== filterCommunity)); setClearCommunitiesOpen(false); }} onCancel={() => setClearCommunitiesOpen(false)} /> {/* Diff Modal */} diff --git a/frontend/src/IPRangesManager.jsx b/frontend/src/IPRangesManager.jsx index 0326e79..4b24696 100644 --- a/frontend/src/IPRangesManager.jsx +++ b/frontend/src/IPRangesManager.jsx @@ -423,8 +423,8 @@ function IPRangesManager() { disableExport={items.length === 0} onClear={clearInvalid} disableClear={items.length === 0} - onClearCommunities={() => setClearCommunitiesOpen(true)} - disableClearCommunities={items.length === 0} + onClearCommunities={() => { if (!filterCommunity) { setError('Выберите community в фильтре для очистки'); return; } setClearCommunitiesOpen(true); }} + disableClearCommunities={items.length === 0 || !filterCommunity} onSave={handleSaveChanges} disableSave={loading} onHistory={() => setHistoryOpen(true)} @@ -715,11 +715,11 @@ function IPRangesManager() { /> { setItems(prev => prev.map(i => ({ ...i, community: '' }))); setClearCommunitiesOpen(false); }} + onConfirm={() => { setItems(prev => prev.filter(i => i.community !== filterCommunity)); setClearCommunitiesOpen(false); }} onCancel={() => setClearCommunitiesOpen(false)} /> {/* Diff Modal */} diff --git a/frontend/src/components/PageHeaderActions.jsx b/frontend/src/components/PageHeaderActions.jsx index 17e8f8d..5615429 100644 --- a/frontend/src/components/PageHeaderActions.jsx +++ b/frontend/src/components/PageHeaderActions.jsx @@ -94,7 +94,7 @@ function PageHeaderActions({ )} {onClearCommunities && ( - )}