feat: Implement auto URL management in server.js and integrate AutoUrlManager in App.jsx for enhanced URL processing and IP updates

backend/server.js

@@ -942,6 +942,172 @@ app.post('/api/simple-filters', async (req, res) => {
     }
 });
 
+// --- Auto URL Routes ---
+
+// Get auto URLs from S3
+app.get('/api/auto-urls', async (req, res) => {
+    const params = {
+        Bucket: BUCKET_NAME,
+        Key: 'bgp_data/auto_url/urls.txt',
+    };
+
+    try {
+        const data = await s3.getObject(params).promise();
+        const fileContent = data.Body.toString('utf-8');
+        const urls = fileContent.split('\n').filter(line => line).map(line => {
+            const parts = line.trim().split(/\s*\|\s*/);
+            const url = parts[0] || '';
+            const community = parts[1] || '';
+            return { url, community };
+        });
+        res.json(urls);
+    } catch (error) {
+        if (error.code === 'NoSuchKey') {
+            res.json([]); // Return empty array if file does not exist
+        } else {
+            console.error(error);
+            res.status(500).send('Error reading auto URLs from S3');
+        }
+    }
+});
+
+// Update auto URLs in S3
+app.post('/api/auto-urls', async (req, res) => {
+    const { urls } = req.body;
+    const fileContent = urls.map(u => `${u.url} | ${u.community}`).join('\n');
+
+    const params = {
+        Bucket: BUCKET_NAME,
+        Key: 'bgp_data/auto_url/urls.txt',
+        Body: fileContent,
+        ContentType: 'text/plain',
+    };
+
+    try {
+        await s3.putObject(params).promise();
+        res.send('Auto URLs updated successfully');
+    } catch (error) {
+        console.error(error);
+        res.status(500).send('Error writing auto URLs to S3');
+    }
+});
+
+// Process auto URLs and update IPs
+app.post('/api/auto-urls/process', async (req, res) => {
+    const https = require('https');
+    const http = require('http');
+    
+    try {
+        // Get current auto URLs
+        const urlsParams = {
+            Bucket: BUCKET_NAME,
+            Key: 'bgp_data/auto_url/urls.txt',
+        };
+        
+        let urls = [];
+        try {
+            const urlsData = await s3.getObject(urlsParams).promise();
+            const urlsContent = urlsData.Body.toString('utf-8');
+            urls = urlsContent.split('\n').filter(line => line).map(line => {
+                const parts = line.trim().split(/\s*\|\s*/);
+                return { url: parts[0] || '', community: parts[1] || '' };
+            });
+        } catch (error) {
+            if (error.code !== 'NoSuchKey') {
+                throw error;
+            }
+        }
+        
+        if (urls.length === 0) {
+            return res.status(400).send('No URLs to process');
+        }
+        
+        // Get current IPs
+        const ipsParams = {
+            Bucket: BUCKET_NAME,
+            Key: 'bgp_data/ips.txt',
+        };
+        
+        let currentIps = [];
+        try {
+            const ipsData = await s3.getObject(ipsParams).promise();
+            const ipsContent = ipsData.Body.toString('utf-8');
+            currentIps = ipsContent.split('\n').filter(line => line).map(line => {
+                const parts = line.trim().split(/\s+/);
+                return { ipRange: parts[0] || '', community: parts[1] || '' };
+            });
+        } catch (error) {
+            if (error.code !== 'NoSuchKey') {
+                throw error;
+            }
+        }
+        
+        // Process each URL
+        const newIps = [];
+        for (const urlData of urls) {
+            try {
+                const url = urlData.url.trim();
+                const community = urlData.community.trim();
+                
+                if (!url || !community) continue;
+                
+                // Download content from URL
+                const content = await new Promise((resolve, reject) => {
+                    const protocol = url.startsWith('https:') ? https : http;
+                    const req = protocol.get(url, (res) => {
+                        let data = '';
+                        res.on('data', (chunk) => data += chunk);
+                        res.on('end', () => resolve(data));
+                    });
+                    req.on('error', reject);
+                    req.setTimeout(10000, () => req.destroy());
+                });
+                
+                // Parse IPs from content
+                const lines = content.split('\n');
+                for (const line of lines) {
+                    const ip = line.trim();
+                    if (ip && (ip.includes('.') || ip.includes(':'))) {
+                        newIps.push({ ipRange: ip, community });
+                    }
+                }
+                
+            } catch (error) {
+                console.error(`Error processing URL ${urlData.url}:`, error);
+            }
+        }
+        
+        // Merge with existing IPs (avoid duplicates)
+        const existingIpRanges = new Set(currentIps.map(ip => ip.ipRange));
+        const uniqueNewIps = newIps.filter(ip => !existingIpRanges.has(ip.ipRange));
+        
+        const allIps = [...currentIps, ...uniqueNewIps];
+        
+        // Save updated IPs
+        const updatedIpsContent = allIps.map(ip => `${ip.ipRange} ${ip.community}`).join('\n');
+        const updateParams = {
+            Bucket: BUCKET_NAME,
+            Key: 'bgp_data/ips.txt',
+            Body: updatedIpsContent,
+            ContentType: 'text/plain',
+        };
+        
+        await s3.putObject(updateParams).promise();
+        
+        res.json({
+            success: true,
+            message: `Обработано ${urls.length} URL, добавлено ${uniqueNewIps.length} новых IP-адресов`,
+            processedUrls: urls.length,
+            newIpsCount: uniqueNewIps.length,
+            totalIpsCount: allIps.length
+        });
+        
+    } catch (error) {
+        console.error('Error processing auto URLs:', error);
+        res.status(500).send('Error processing auto URLs');
+    }
+});
+
 // The "catchall" handler: for any request that doesn't
 // match one above, send back React's index.html file.
 app.get('*', (req, res) => {

frontend/src/App.jsx

@@ -17,7 +17,8 @@ import {
   IconAlertTriangle,
   IconAlertCircle,
   IconServer,
-  IconFilter
+  IconFilter,
+  IconDownload
 } from '@tabler/icons-react';
 import DataManager from './DataManager';
 import ServerManager from './ServerManager';
@@ -25,6 +26,7 @@ import FilterManager from './FilterManager';
 import DomainsNewManager from './DomainsNewManager';
 import IPRangesManager from './IPRangesManager';
 import ASNsNewManager from './ASNsNewManager';
+import AutoUrlManager from './AutoUrlManager';
 import './App.css';
 import axios from 'axios';
 import {
@@ -56,6 +58,7 @@ function MainLayout() {
     { id: 'domains-new', title: 'Домены New', icon: IconWorld, path: '/domains-new' },
     { id: 'ip-ranges', title: 'IP-диапазоны', icon: IconNetwork, path: '/ip-ranges' },
     { id: 'asns', title: 'AS', icon: IconNetwork, path: '/asns' },
+    { id: 'auto-urls', title: 'Авто URL', icon: IconDownload, path: '/auto-urls' },
     { id: 'servers', title: 'Серверы', icon: IconServer, path: '/servers' },
     { id: 'filters', title: 'Фильтры', icon: IconFilter, path: '/filters' },
     { id: 'files', title: 'Файлы', icon: IconFileText, path: '/files' },
@@ -103,6 +106,7 @@ function MainLayout() {
           <Route path="/domains-new" element={<DomainsNewManager />} />
           <Route path="/ip-ranges" element={<IPRangesManager />} />
           <Route path="/asns" element={<ASNsNewManager />} />
+          <Route path="/auto-urls" element={<AutoUrlManager />} />
           <Route path="/servers" element={<ServerManager />} />
           <Route path="/filters" element={<FilterManager />} />
           <Route path="/" element={<Navigate to="/domains" replace />} />

frontend/src/AutoUrlManager.jsx

@@ -0,0 +1,275 @@
+import { useState, useEffect } from 'react';
+import axios from 'axios';
+import {
+  IconPlus,
+  IconTrash,
+  IconDownload,
+  IconAlertCircle,
+  IconCheck,
+  IconLoader
+} from '@tabler/icons-react';
+
+function AutoUrlManager() {
+  const [urls, setUrls] = useState([]);
+  const [loading, setLoading] = useState(true);
+  const [saving, setSaving] = useState(false);
+  const [processing, setProcessing] = useState(false);
+  const [message, setMessage] = useState('');
+  const [messageType, setMessageType] = useState('');
+
+  useEffect(() => {
+    fetchUrls();
+  }, []);
+
+  const fetchUrls = async () => {
+    try {
+      setLoading(true);
+      const response = await axios.get('/api/auto-urls');
+      setUrls(response.data);
+    } catch (error) {
+      console.error('Error fetching URLs:', error);
+      setMessage('Ошибка при загрузке URL-адресов');
+      setMessageType('error');
+    } finally {
+      setLoading(false);
+    }
+  };
+
+  const addUrl = () => {
+    setUrls([...urls, { url: '', community: '' }]);
+  };
+
+  const removeUrl = (index) => {
+    setUrls(urls.filter((_, i) => i !== index));
+  };
+
+  const updateUrl = (index, field, value) => {
+    const newUrls = [...urls];
+    newUrls[index][field] = value;
+    setUrls(newUrls);
+  };
+
+  const saveUrls = async () => {
+    try {
+      setSaving(true);
+      setMessage('');
+      
+      // Validate URLs
+      const validUrls = urls.filter(u => u.url.trim() && u.community.trim());
+      if (validUrls.length === 0) {
+        setMessage('Добавьте хотя бы один URL с community');
+        setMessageType('error');
+        return;
+      }
+
+      await axios.post('/api/auto-urls', { urls: validUrls });
+      setUrls(validUrls);
+      setMessage('URL-адреса сохранены успешно');
+      setMessageType('success');
+    } catch (error) {
+      console.error('Error saving URLs:', error);
+      setMessage('Ошибка при сохранении URL-адресов');
+      setMessageType('error');
+    } finally {
+      setSaving(false);
+    }
+  };
+
+  const processUrls = async () => {
+    try {
+      setProcessing(true);
+      setMessage('');
+      
+      const response = await axios.post('/api/auto-urls/process');
+      setMessage(response.data.message);
+      setMessageType('success');
+    } catch (error) {
+      console.error('Error processing URLs:', error);
+      setMessage(error.response?.data || 'Ошибка при обработке URL-адресов');
+      setMessageType('error');
+    } finally {
+      setProcessing(false);
+    }
+  };
+
+  if (loading) {
+    return (
+      <div className="d-flex justify-content-center align-items-center" style={{ minHeight: '200px' }}>
+        <IconLoader className="animate-spin" size={32} />
+      </div>
+    );
+  }
+
+  return (
+    <div>
+      <div className="page-header d-print-none mb-4">
+        <div className="row align-items-center">
+          <div className="col">
+            <h2 className="page-title">Автоматические URL</h2>
+            <div className="page-pretitle">Управление автоматическими URL-адресами для загрузки IP-списков</div>
+          </div>
+        </div>
+      </div>
+
+      {message && (
+        <div className={`alert alert-${messageType === 'error' ? 'danger' : 'success'} alert-dismissible`} role="alert">
+          <div className="d-flex">
+            {messageType === 'error' ? <IconAlertCircle className="me-2" /> : <IconCheck className="me-2" />}
+            {message}
+          </div>
+          <button type="button" className="btn-close" onClick={() => setMessage('')}></button>
+        </div>
+      )}
+
+      <div className="card">
+        <div className="card-header">
+          <h3 className="card-title">URL-адреса для автоматической загрузки</h3>
+          <div className="card-actions">
+            <button 
+              className="btn btn-primary btn-sm" 
+              onClick={addUrl}
+              disabled={saving || processing}
+            >
+              <IconPlus className="me-1" />
+              Добавить URL
+            </button>
+          </div>
+        </div>
+        <div className="card-body">
+          {urls.length === 0 ? (
+            <div className="text-center text-muted py-4">
+              <IconAlertCircle size={48} className="mb-3" />
+              <p>Нет добавленных URL-адресов</p>
+              <button 
+                className="btn btn-primary" 
+                onClick={addUrl}
+                disabled={saving || processing}
+              >
+                <IconPlus className="me-1" />
+                Добавить первый URL
+              </button>
+            </div>
+          ) : (
+            <div className="table-responsive">
+              <table className="table table-vcenter">
+                <thead>
+                  <tr>
+                    <th>URL</th>
+                    <th>Community</th>
+                    <th width="100">Действия</th>
+                  </tr>
+                </thead>
+                <tbody>
+                  {urls.map((url, index) => (
+                    <tr key={index}>
+                      <td>
+                        <input
+                          type="text"
+                          className="form-control"
+                          placeholder="https://example.com/ips.txt"
+                          value={url.url}
+                          onChange={(e) => updateUrl(index, 'url', e.target.value)}
+                          disabled={saving || processing}
+                        />
+                      </td>
+                      <td>
+                        <input
+                          type="text"
+                          className="form-control"
+                          placeholder="555"
+                          value={url.community}
+                          onChange={(e) => updateUrl(index, 'community', e.target.value)}
+                          disabled={saving || processing}
+                        />
+                      </td>
+                      <td>
+                        <button
+                          className="btn btn-outline-danger btn-sm"
+                          onClick={() => removeUrl(index)}
+                          disabled={saving || processing}
+                        >
+                          <IconTrash size={16} />
+                        </button>
+                      </td>
+                    </tr>
+                  ))}
+                </tbody>
+              </table>
+            </div>
+          )}
+        </div>
+        <div className="card-footer">
+          <div className="d-flex justify-content-between align-items-center">
+            <div className="text-muted">
+              {urls.length > 0 && (
+                <span>Всего URL-адресов: {urls.length}</span>
+              )}
+            </div>
+            <div>
+              <button
+                className="btn btn-outline-primary me-2"
+                onClick={saveUrls}
+                disabled={saving || processing || urls.length === 0}
+              >
+                {saving ? (
+                  <>
+                    <IconLoader className="animate-spin me-1" />
+                    Сохранение...
+                  </>
+                ) : (
+                  'Сохранить'
+                )}
+              </button>
+              <button
+                className="btn btn-primary"
+                onClick={processUrls}
+                disabled={saving || processing || urls.length === 0}
+              >
+                {processing ? (
+                  <>
+                    <IconLoader className="animate-spin me-1" />
+                    Обработка...
+                  </>
+                ) : (
+                  <>
+                    <IconDownload className="me-1" />
+                    Загрузить IP-списки
+                  </>
+                )}
+              </button>
+            </div>
+          </div>
+        </div>
+      </div>
+
+      <div className="card mt-4">
+        <div className="card-header">
+          <h3 className="card-title">Информация</h3>
+        </div>
+        <div className="card-body">
+          <div className="row">
+            <div className="col-md-6">
+              <h4>Как это работает:</h4>
+              <ul>
+                <li>Добавьте URL-адреса, которые содержат списки IP-адресов</li>
+                <li>Укажите community для каждого URL</li>
+                <li>Нажмите "Загрузить IP-списки" для обработки всех URL</li>
+                <li>IP-адреса будут добавлены в файл bgp_data/ips.txt</li>
+              </ul>
+            </div>
+            <div className="col-md-6">
+              <h4>Формат файла:</h4>
+              <p>Файл сохраняется в формате:</p>
+              <pre className="bg-light p-2 rounded">
+{`https://test.com/ips.txt | 555
+https://example.com/blacklist.txt | 666`}
+              </pre>
+            </div>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+}
+
+export default AutoUrlManager;