feat: Enhance auto URL processing to include domain management and improve error handling; update UI text for clarity in ASNs, Domains, and IPRanges managers

backend/server.js

@@ -1724,118 +1724,165 @@ app.post('/api/auto-urls', async (req, res) => {
 
 // Process auto URLs and update IPs
 app.post('/api/auto-urls/process', async (req, res) => {
-    const https = require('https');
+  const https = require('https');
-    const http = require('http');
+  const http = require('http');
 
+  // Helpers
+  const isValidIPv4 = (ip) => {
+    const octets = String(ip || '').trim().split('.');
+    if (octets.length !== 4) return false;
+    return octets.every(o => /^\d{1,3}$/.test(o) && Number(o) >= 0 && Number(o) <= 255);
+  };
+  const isValidCidrV4 = (value) => {
+    const v = String(value || '').trim();
+    const parts = v.split('/');
+    if (parts.length !== 2) return false;
+    const [ip, mask] = parts;
+    if (!isValidIPv4(ip)) return false;
+    if (!/^\d{1,2}$/.test(mask)) return false;
+    const m = Number(mask);
+    return m >= 0 && m <= 32;
+  };
+  const isValidDomain = (value) => {
+    const v = String(value || '').trim().toLowerCase();
+    if (v.startsWith('#')) return false; // comment line
+    return /^([a-z0-9-]+\.)+[a-z]{2,}$/i.test(v);
+  };
+
+  try {
+    // Load configured auto URLs
+    const urlsParams = {
+      Bucket: BUCKET_NAME,
+      Key: 'bgp_data/auto_url/urls.txt',
+    };
+
+    let urls = [];
     try {
-        // Get current auto URLs
+      const urlsData = await s3.getObject(urlsParams).promise();
-        const urlsParams = {
+      const urlsContent = urlsData.Body.toString('utf-8');
-            Bucket: BUCKET_NAME,
+      urls = urlsContent.split('\n').filter(line => line).map(line => {
-            Key: 'bgp_data/auto_url/urls.txt',
+        const parts = line.trim().split(/\s+/);
-        };
+        return { url: parts[0] || '', community: parts[1] || '' };
+      });
+    } catch (error) {
+      if (error.code !== 'NoSuchKey') {
+        throw error;
+      }
+    }
 
-        let urls = [];
+    if (urls.length === 0) {
-        try {
+      return sendError(res, 400, 'No URLs to process', 'E_BAD_REQUEST');
-            const urlsData = await s3.getObject(urlsParams).promise();
+    }
-            const urlsContent = urlsData.Body.toString('utf-8');
-            urls = urlsContent.split('\n').filter(line => line).map(line => {
-                const parts = line.trim().split(/\s+/);
-                return { url: parts[0] || '', community: parts[1] || '' };
-            });
-        } catch (error) {
-            if (error.code !== 'NoSuchKey') {
-                throw error;
-            }
-        }
 
-        if (urls.length === 0) {
+    // Load current IP ranges
-            return sendError(res, 400, 'No URLs to process', 'E_BAD_REQUEST');
+    const ipsParams = { Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt' };
-        }
+    let currentIps = [];
+    try {
+      const ipsData = await s3.getObject(ipsParams).promise();
+      const ipsContent = ipsData.Body.toString('utf-8');
+      currentIps = ipsContent.split('\n').filter(line => line).map(line => {
+        const parts = line.trim().split(/\s+/);
+        return { ipRange: parts[0] || '', community: parts[1] || '' };
+      });
+    } catch (error) {
+      if (error.code !== 'NoSuchKey') throw error;
+    }
 
-        // Get current IPs
+    // Load current domains
-        const ipsParams = {
+    const domainsParams = { Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt' };
-            Bucket: BUCKET_NAME,
+    let currentDomains = [];
-            Key: 'bgp_data/ips.txt',
+    try {
-        };
+      const dData = await s3.getObject(domainsParams).promise();
+      const dContent = dData.Body.toString('utf-8');
+      currentDomains = dContent.split('\n').filter(line => line).map(line => {
+        const parts = line.trim().split(/\s+/);
+        return { domain: (parts[0] || '').toLowerCase(), community: parts[1] || '' };
+      });
+    } catch (error) {
+      if (error.code !== 'NoSuchKey') throw error;
+    }
 
-        let currentIps = [];
+    // Process each URL
-        try {
+    const newIps = [];
-            const ipsData = await s3.getObject(ipsParams).promise();
+    const newDomains = [];
-            const ipsContent = ipsData.Body.toString('utf-8');
+    for (const urlData of urls) {
-            currentIps = ipsContent.split('\n').filter(line => line).map(line => {
+      try {
-                const parts = line.trim().split(/\s+/);
+        const url = String(urlData.url || '').trim();
-                return { ipRange: parts[0] || '', community: parts[1] || '' };
+        const community = String(urlData.community || '').trim();
-            });
+        if (!url || !community) continue;
-        } catch (error) {
-            if (error.code !== 'NoSuchKey') {
-                throw error;
-            }
-        }
 
-        // Process each URL
+        // Download content
-        const newIps = [];
+        const content = await new Promise((resolve, reject) => {
-        for (const urlData of urls) {
+          const protocol = url.startsWith('https:') ? https : http;
-            try {
+          const req = protocol.get(url, (r) => {
-                const url = urlData.url.trim();
+            let data = '';
-                const community = urlData.community.trim();
+            r.on('data', (chunk) => { data += chunk; });
-                
+            r.on('end', () => resolve(data));
-                if (!url || !community) continue;
+          });
-                
+          req.on('error', reject);
-                // Download content from URL
+          req.setTimeout(15000, () => req.destroy());
-                const content = await new Promise((resolve, reject) => {
-                    const protocol = url.startsWith('https:') ? https : http;
-                    const req = protocol.get(url, (res) => {
-                        let data = '';
-                        res.on('data', (chunk) => data += chunk);
-                        res.on('end', () => resolve(data));
-                    });
-                    req.on('error', reject);
-                    req.setTimeout(10000, () => req.destroy());
-                });
-                
-                // Parse IPs from content
-                const lines = content.split('\n');
-                for (const line of lines) {
-                    const ip = line.trim();
-                    if (ip && (ip.includes('.') || ip.includes(':'))) {
-                        newIps.push({ ipRange: ip, community });
-                    }
-                }
-                
-            } catch (error) {
-                console.error(`Error processing URL ${urlData.url}:`, error);
-            }
-        }
-        
-        // Merge with existing IPs (avoid duplicates)
-        const existingIpRanges = new Set(currentIps.map(ip => ip.ipRange));
-        const uniqueNewIps = newIps.filter(ip => !existingIpRanges.has(ip.ipRange));
-        
-        const allIps = [...currentIps, ...uniqueNewIps];
-        
-        // Save updated IPs
-        const updatedIpsContent = allIps.map(ip => `${ip.ipRange} ${ip.community}`).join('\n');
-        const updateParams = {
-            Bucket: BUCKET_NAME,
-            Key: 'bgp_data/ips.txt',
-            Body: updatedIpsContent,
-            ContentType: 'text/plain',
-        };
-        
-        await s3.putObject(updateParams).promise();
-        
-        res.json({
-            success: true,
-            message: `Обработано ${urls.length} URL, добавлено ${uniqueNewIps.length} новых IP-адресов`,
-            processedUrls: urls.length,
-            newIpsCount: uniqueNewIps.length,
-            totalIpsCount: allIps.length
         });
 
-    } catch (error) {
+        const lines = content.split('\n');
-        console.error('Error processing auto URLs:', error);
+        for (const raw of lines) {
-        return sendError(res, 500, 'Error processing auto URLs', 'E_S3');
+          const line = String(raw || '').trim();
+          if (!line) continue;
+          if (line.startsWith('#') || line.startsWith('//')) continue;
+          const token = line.split(/\s+/)[0]?.trim();
+          if (!token) continue;
+
+          // Decide destination
+          if (isValidCidrV4(token)) {
+            newIps.push({ ipRange: token, community });
+            continue;
+          }
+          if (isValidIPv4(token)) {
+            // single IPv4 → normalize to /32
+            newIps.push({ ipRange: `${token}/32`, community });
+            continue;
+          }
+          if (isValidDomain(token)) {
+            newDomains.push({ domain: token.toLowerCase(), community });
+            continue;
+          }
+          // ignore everything else
+        }
+      } catch (error) {
+        console.error(`Error processing URL ${urlData.url}:`, error);
+      }
     }
+
+    // Merge & deduplicate
+    const existingIpRanges = new Set(currentIps.map(i => i.ipRange));
+    const uniqueNewIps = newIps.filter(i => !existingIpRanges.has(i.ipRange));
+    const allIps = [...currentIps, ...uniqueNewIps];
+
+    const existingDomains = new Set(currentDomains.map(d => d.domain));
+    const uniqueNewDomains = newDomains.filter(d => !existingDomains.has(d.domain));
+    const allDomains = [...currentDomains, ...uniqueNewDomains];
+
+    // Save updated IPs
+    const updatedIpsContent = allIps.map(i => `${i.ipRange} ${i.community}`).join('\n');
+    await s3.putObject({ Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt', Body: updatedIpsContent, ContentType: 'text/plain' }).promise();
+
+    // Save updated Domains
+    const updatedDomainsContent = allDomains.map(d => `${d.domain} ${d.community}`).join('\n');
+    await s3.putObject({ Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt', Body: updatedDomainsContent, ContentType: 'text/plain' }).promise();
+
+    const msg = `Обработано ${urls.length} URL, добавлено ${uniqueNewIps.length} IP и ${uniqueNewDomains.length} доменов`;
+    return res.json({
+      success: true,
+      message: msg,
+      processedUrls: urls.length,
+      newIpsCount: uniqueNewIps.length,
+      newDomainsCount: uniqueNewDomains.length,
+      totalIpsCount: allIps.length,
+      totalDomainsCount: allDomains.length
+    });
+  } catch (error) {
+    console.error('Error processing auto URLs:', error);
+    return sendError(res, 500, 'Error processing auto URLs', 'E_S3');
+  }
 });
 
 // --- Proxy: Background BGP Update (avoids CORS from browser) ---

frontend/src/ASNsNewManager.jsx

@@ -407,8 +407,8 @@ function ASNsNewManager() {
               disableExport={items.length === 0}
               onClear={clearInvalid}
               disableClear={items.length === 0}
-              onClearCommunities={() => setClearCommunitiesOpen(true)}
+              onClearCommunities={() => { if (!filterCommunity) { setError('Выберите community в фильтре для очистки'); return; } setClearCommunitiesOpen(true); }}
-              disableClearCommunities={items.length === 0}
+              disableClearCommunities={items.length === 0 || !filterCommunity}
               onSave={handleSaveChanges}
               disableSave={loading}
               onHistory={() => setHistoryOpen(true)}
@@ -694,11 +694,11 @@ function ASNsNewManager() {
       />
       <ConfirmDialog
         open={clearCommunitiesOpen}
-        title={'Очистить все community?'}
+        title={'Удалить записи community'}
-        message={'Поле community будет очищено у всех записей в таблице. Это действие не сохраняет изменения автоматически.'}
+        message={`Будут удалены все записи с community "${filterCommunity}". Действие НЕ сохраняет изменения автоматически.`}
-        confirmText={'Очистить'}
+        confirmText={'Удалить'}
         cancelText={'Отмена'}
-        onConfirm={() => { setItems(prev => prev.map(i => ({ ...i, community: '' }))); setClearCommunitiesOpen(false); }}
+        onConfirm={() => { setItems(prev => prev.filter(i => i.community !== filterCommunity)); setClearCommunitiesOpen(false); }}
         onCancel={() => setClearCommunitiesOpen(false)}
       />
       {/* datalist больше не нужен, т.к. используем кастомный автокомплит */}

frontend/src/AutoUrlManager.jsx

@@ -158,7 +158,7 @@ function AutoUrlManager() {
         <div className="row align-items-center">
           <div className="col">
             <h2 className="page-title">Автоматические URL</h2>
-            <div className="page-pretitle">Управление автоматическими URL-адресами для загрузки IP-списков</div>
+            <div className="page-pretitle">Управление автоматическими URL-адресами для загрузки списков IP и доменов</div>
           </div>
         </div>
       </div>
@@ -338,7 +338,7 @@ function AutoUrlManager() {
                 ) : (
                   <>
                     <IconDownload className="me-1" />
-                    Загрузить IP-списки
+                    Загрузить списки
                   </>
                 )}
               </button>
@@ -358,7 +358,7 @@ function AutoUrlManager() {
               <ul className="list-unstyled m-0">
                 <li className="d-flex align-items-start mb-2">
                   <span className="text-blue me-2"><IconCircleCheck size={18} /></span>
-                  <span>Добавьте URL-адреса, которые содержат списки IP-адресов</span>
+                  <span>Добавьте URL-адреса, которые содержат списки IP-адресов или доменных имён</span>
                 </li>
                 <li className="d-flex align-items-start mb-2">
                   <span className="text-blue me-2"><IconCircleCheck size={18} /></span>
@@ -366,11 +366,11 @@ function AutoUrlManager() {
                 </li>
                 <li className="d-flex align-items-start mb-2">
                   <span className="text-blue me-2"><IconCircleCheck size={18} /></span>
-                  <span>Нажмите «Загрузить IP-списки» для обработки всех URL</span>
+                  <span>Нажмите «Загрузить списки» для обработки всех URL</span>
                 </li>
                 <li className="d-flex align-items-start">
                   <span className="text-blue me-2"><IconCircleCheck size={18} /></span>
-                  <span>IP-адреса будут добавлены в файл <code>bgp_data/ips.txt</code></span>
+                  <span>IP-диапазоны попадут в <code>bgp_data/ips.txt</code>, домены — в <code>bgp_data/domains_community.txt</code></span>
                 </li>
               </ul>
             </div>

frontend/src/DomainsNewManager.jsx

@@ -414,8 +414,8 @@ function DomainsNewManager() {
               disableExport={items.length === 0}
               onClear={clearInvalid}
               disableClear={items.length === 0}
-              onClearCommunities={() => setClearCommunitiesOpen(true)}
+              onClearCommunities={() => { if (!filterCommunity) { setError('Выберите community в фильтре для очистки'); return; } setClearCommunitiesOpen(true); }}
-              disableClearCommunities={items.length === 0}
+              disableClearCommunities={items.length === 0 || !filterCommunity}
               onSave={handleSaveChanges}
               disableSave={loading}
               onHistory={() => setHistoryOpen(true)}
@@ -703,11 +703,11 @@ function DomainsNewManager() {
       />
       <ConfirmDialog
         open={clearCommunitiesOpen}
-        title={'Очистить все community?'}
+        title={'Удалить записи community'}
-        message={'Поле community будет очищено у всех доменов. Это действие не сохраняет изменения автоматически.'}
+        message={`Будут удалены все домены с community "${filterCommunity}". Действие НЕ сохраняет изменения автоматически.`}
-        confirmText={'Очистить'}
+        confirmText={'Удалить'}
         cancelText={'Отмена'}
-        onConfirm={() => { setItems(prev => prev.map(i => ({ ...i, community: '' }))); setClearCommunitiesOpen(false); }}
+        onConfirm={() => { setItems(prev => prev.filter(i => i.community !== filterCommunity)); setClearCommunitiesOpen(false); }}
         onCancel={() => setClearCommunitiesOpen(false)}
       />
       {/* Diff Modal */}

frontend/src/IPRangesManager.jsx

@@ -423,8 +423,8 @@ function IPRangesManager() {
               disableExport={items.length === 0}
               onClear={clearInvalid}
               disableClear={items.length === 0}
-              onClearCommunities={() => setClearCommunitiesOpen(true)}
+              onClearCommunities={() => { if (!filterCommunity) { setError('Выберите community в фильтре для очистки'); return; } setClearCommunitiesOpen(true); }}
-              disableClearCommunities={items.length === 0}
+              disableClearCommunities={items.length === 0 || !filterCommunity}
               onSave={handleSaveChanges}
               disableSave={loading}
               onHistory={() => setHistoryOpen(true)}
@@ -715,11 +715,11 @@ function IPRangesManager() {
       />
       <ConfirmDialog
         open={clearCommunitiesOpen}
-        title={'Очистить все community?'}
+        title={'Удалить записи community'}
-        message={'Поле community будет очищено у всех IP-диапазонов. Это действие не сохраняет изменения автоматически.'}
+        message={`Будут удалены все IP-диапазоны с community "${filterCommunity}". Действие НЕ сохраняет изменения автоматически.`}
-        confirmText={'Очистить'}
+        confirmText={'Удалить'}
         cancelText={'Отмена'}
-        onConfirm={() => { setItems(prev => prev.map(i => ({ ...i, community: '' }))); setClearCommunitiesOpen(false); }}
+        onConfirm={() => { setItems(prev => prev.filter(i => i.community !== filterCommunity)); setClearCommunitiesOpen(false); }}
         onCancel={() => setClearCommunitiesOpen(false)}
       />
       {/* Diff Modal */}

frontend/src/components/PageHeaderActions.jsx

@@ -94,7 +94,7 @@ function PageHeaderActions({
           </button>
         )}
         {onClearCommunities && (
-          <button className="btn btn-outline-secondary" type="button" onClick={onClearCommunities} disabled={disableClearCommunities} title="Сбросить community у всех записей">
+          <button className="btn btn-outline-secondary" type="button" onClick={onClearCommunities} disabled={disableClearCommunities} title="Удалить все записи выбранного community">
             <IconEraser className="me-1" /> Очистить community
           </button>
         )}