From 7642d86a82b4b5a58ee1e6bb0181192b7012fe34 Mon Sep 17 00:00:00 2001 From: Denis Shatskiy Date: Mon, 21 Jul 2025 14:11:41 +0700 Subject: [PATCH] feat: Implement auto URL management in server.js and integrate AutoUrlManager in App.jsx for enhanced URL processing and IP updates --- backend/server.js | 166 +++++++++++++++++++ frontend/src/App.jsx | 6 +- frontend/src/AutoUrlManager.jsx | 275 ++++++++++++++++++++++++++++++++ 3 files changed, 446 insertions(+), 1 deletion(-) create mode 100644 frontend/src/AutoUrlManager.jsx diff --git a/backend/server.js b/backend/server.js index 1d046f0..de209f9 100644 --- a/backend/server.js +++ b/backend/server.js @@ -942,6 +942,172 @@ app.post('/api/simple-filters', async (req, res) => { } }); +// --- Auto URL Routes --- + +// Get auto URLs from S3 +app.get('/api/auto-urls', async (req, res) => { + const params = { + Bucket: BUCKET_NAME, + Key: 'bgp_data/auto_url/urls.txt', + }; + + try { + const data = await s3.getObject(params).promise(); + const fileContent = data.Body.toString('utf-8'); + const urls = fileContent.split('\n').filter(line => line).map(line => { + const parts = line.trim().split(/\s*\|\s*/); + const url = parts[0] || ''; + const community = parts[1] || ''; + return { url, community }; + }); + res.json(urls); + } catch (error) { + if (error.code === 'NoSuchKey') { + res.json([]); // Return empty array if file does not exist + } else { + console.error(error); + res.status(500).send('Error reading auto URLs from S3'); + } + } +}); + +// Update auto URLs in S3 +app.post('/api/auto-urls', async (req, res) => { + const { urls } = req.body; + const fileContent = urls.map(u => `${u.url} | ${u.community}`).join('\n'); + + const params = { + Bucket: BUCKET_NAME, + Key: 'bgp_data/auto_url/urls.txt', + Body: fileContent, + ContentType: 'text/plain', + }; + + try { + await s3.putObject(params).promise(); + res.send('Auto URLs updated successfully'); + } catch (error) { + console.error(error); + res.status(500).send('Error writing auto URLs to S3'); + } +}); + +// Process auto URLs and update IPs +app.post('/api/auto-urls/process', async (req, res) => { + const https = require('https'); + const http = require('http'); + + try { + // Get current auto URLs + const urlsParams = { + Bucket: BUCKET_NAME, + Key: 'bgp_data/auto_url/urls.txt', + }; + + let urls = []; + try { + const urlsData = await s3.getObject(urlsParams).promise(); + const urlsContent = urlsData.Body.toString('utf-8'); + urls = urlsContent.split('\n').filter(line => line).map(line => { + const parts = line.trim().split(/\s*\|\s*/); + return { url: parts[0] || '', community: parts[1] || '' }; + }); + } catch (error) { + if (error.code !== 'NoSuchKey') { + throw error; + } + } + + if (urls.length === 0) { + return res.status(400).send('No URLs to process'); + } + + // Get current IPs + const ipsParams = { + Bucket: BUCKET_NAME, + Key: 'bgp_data/ips.txt', + }; + + let currentIps = []; + try { + const ipsData = await s3.getObject(ipsParams).promise(); + const ipsContent = ipsData.Body.toString('utf-8'); + currentIps = ipsContent.split('\n').filter(line => line).map(line => { + const parts = line.trim().split(/\s+/); + return { ipRange: parts[0] || '', community: parts[1] || '' }; + }); + } catch (error) { + if (error.code !== 'NoSuchKey') { + throw error; + } + } + + // Process each URL + const newIps = []; + for (const urlData of urls) { + try { + const url = urlData.url.trim(); + const community = urlData.community.trim(); + + if (!url || !community) continue; + + // Download content from URL + const content = await new Promise((resolve, reject) => { + const protocol = url.startsWith('https:') ? https : http; + const req = protocol.get(url, (res) => { + let data = ''; + res.on('data', (chunk) => data += chunk); + res.on('end', () => resolve(data)); + }); + req.on('error', reject); + req.setTimeout(10000, () => req.destroy()); + }); + + // Parse IPs from content + const lines = content.split('\n'); + for (const line of lines) { + const ip = line.trim(); + if (ip && (ip.includes('.') || ip.includes(':'))) { + newIps.push({ ipRange: ip, community }); + } + } + + } catch (error) { + console.error(`Error processing URL ${urlData.url}:`, error); + } + } + + // Merge with existing IPs (avoid duplicates) + const existingIpRanges = new Set(currentIps.map(ip => ip.ipRange)); + const uniqueNewIps = newIps.filter(ip => !existingIpRanges.has(ip.ipRange)); + + const allIps = [...currentIps, ...uniqueNewIps]; + + // Save updated IPs + const updatedIpsContent = allIps.map(ip => `${ip.ipRange} ${ip.community}`).join('\n'); + const updateParams = { + Bucket: BUCKET_NAME, + Key: 'bgp_data/ips.txt', + Body: updatedIpsContent, + ContentType: 'text/plain', + }; + + await s3.putObject(updateParams).promise(); + + res.json({ + success: true, + message: `Обработано ${urls.length} URL, добавлено ${uniqueNewIps.length} новых IP-адресов`, + processedUrls: urls.length, + newIpsCount: uniqueNewIps.length, + totalIpsCount: allIps.length + }); + + } catch (error) { + console.error('Error processing auto URLs:', error); + res.status(500).send('Error processing auto URLs'); + } +}); + // The "catchall" handler: for any request that doesn't // match one above, send back React's index.html file. app.get('*', (req, res) => { diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 84ff028..d523323 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -17,7 +17,8 @@ import { IconAlertTriangle, IconAlertCircle, IconServer, - IconFilter + IconFilter, + IconDownload } from '@tabler/icons-react'; import DataManager from './DataManager'; import ServerManager from './ServerManager'; @@ -25,6 +26,7 @@ import FilterManager from './FilterManager'; import DomainsNewManager from './DomainsNewManager'; import IPRangesManager from './IPRangesManager'; import ASNsNewManager from './ASNsNewManager'; +import AutoUrlManager from './AutoUrlManager'; import './App.css'; import axios from 'axios'; import { @@ -56,6 +58,7 @@ function MainLayout() { { id: 'domains-new', title: 'Домены New', icon: IconWorld, path: '/domains-new' }, { id: 'ip-ranges', title: 'IP-диапазоны', icon: IconNetwork, path: '/ip-ranges' }, { id: 'asns', title: 'AS', icon: IconNetwork, path: '/asns' }, + { id: 'auto-urls', title: 'Авто URL', icon: IconDownload, path: '/auto-urls' }, { id: 'servers', title: 'Серверы', icon: IconServer, path: '/servers' }, { id: 'filters', title: 'Фильтры', icon: IconFilter, path: '/filters' }, { id: 'files', title: 'Файлы', icon: IconFileText, path: '/files' }, @@ -103,6 +106,7 @@ function MainLayout() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/frontend/src/AutoUrlManager.jsx b/frontend/src/AutoUrlManager.jsx new file mode 100644 index 0000000..9190a43 --- /dev/null +++ b/frontend/src/AutoUrlManager.jsx @@ -0,0 +1,275 @@ +import { useState, useEffect } from 'react'; +import axios from 'axios'; +import { + IconPlus, + IconTrash, + IconDownload, + IconAlertCircle, + IconCheck, + IconLoader +} from '@tabler/icons-react'; + +function AutoUrlManager() { + const [urls, setUrls] = useState([]); + const [loading, setLoading] = useState(true); + const [saving, setSaving] = useState(false); + const [processing, setProcessing] = useState(false); + const [message, setMessage] = useState(''); + const [messageType, setMessageType] = useState(''); + + useEffect(() => { + fetchUrls(); + }, []); + + const fetchUrls = async () => { + try { + setLoading(true); + const response = await axios.get('/api/auto-urls'); + setUrls(response.data); + } catch (error) { + console.error('Error fetching URLs:', error); + setMessage('Ошибка при загрузке URL-адресов'); + setMessageType('error'); + } finally { + setLoading(false); + } + }; + + const addUrl = () => { + setUrls([...urls, { url: '', community: '' }]); + }; + + const removeUrl = (index) => { + setUrls(urls.filter((_, i) => i !== index)); + }; + + const updateUrl = (index, field, value) => { + const newUrls = [...urls]; + newUrls[index][field] = value; + setUrls(newUrls); + }; + + const saveUrls = async () => { + try { + setSaving(true); + setMessage(''); + + // Validate URLs + const validUrls = urls.filter(u => u.url.trim() && u.community.trim()); + if (validUrls.length === 0) { + setMessage('Добавьте хотя бы один URL с community'); + setMessageType('error'); + return; + } + + await axios.post('/api/auto-urls', { urls: validUrls }); + setUrls(validUrls); + setMessage('URL-адреса сохранены успешно'); + setMessageType('success'); + } catch (error) { + console.error('Error saving URLs:', error); + setMessage('Ошибка при сохранении URL-адресов'); + setMessageType('error'); + } finally { + setSaving(false); + } + }; + + const processUrls = async () => { + try { + setProcessing(true); + setMessage(''); + + const response = await axios.post('/api/auto-urls/process'); + setMessage(response.data.message); + setMessageType('success'); + } catch (error) { + console.error('Error processing URLs:', error); + setMessage(error.response?.data || 'Ошибка при обработке URL-адресов'); + setMessageType('error'); + } finally { + setProcessing(false); + } + }; + + if (loading) { + return ( +
+ +
+ ); + } + + return ( +
+
+
+
+

Автоматические URL

+
Управление автоматическими URL-адресами для загрузки IP-списков
+
+
+
+ + {message && ( +
+
+ {messageType === 'error' ? : } + {message} +
+ +
+ )} + +
+
+

URL-адреса для автоматической загрузки

+
+ +
+
+
+ {urls.length === 0 ? ( +
+ +

Нет добавленных URL-адресов

+ +
+ ) : ( +
+ + + + + + + + + + {urls.map((url, index) => ( + + + + + + ))} + +
URLCommunityДействия
+ updateUrl(index, 'url', e.target.value)} + disabled={saving || processing} + /> + + updateUrl(index, 'community', e.target.value)} + disabled={saving || processing} + /> + + +
+
+ )} +
+
+
+
+ {urls.length > 0 && ( + Всего URL-адресов: {urls.length} + )} +
+
+ + +
+
+
+
+ +
+
+

Информация

+
+
+
+
+

Как это работает:

+
    +
  • Добавьте URL-адреса, которые содержат списки IP-адресов
    • +
  • Укажите community для каждого URL
    • +
  • Нажмите "Загрузить IP-списки" для обработки всех URL
    • +
  • IP-адреса будут добавлены в файл bgp_data/ips.txt
    • +
+
+
+

Формат файла:

+

Файл сохраняется в формате:

+ 
+{`https://test.com/ips.txt | 555
+https://example.com/blacklist.txt | 666`}
+
+
+
+
+
+
+ ); +} + +export default AutoUrlManager; \ No newline at end of file