feat: Enhance auto URL processing to include domain management and improve error handling; update UI text for clarity in ASNs, Domains, and IPRanges managers
Publish Fast Tabler Docker image / build-and-push-fast (push) Successful in 18m27s
Publish Fast Tabler Docker image / build-and-push-fast (push) Successful in 18m27s
This commit is contained in:
+156
-109
@@ -1724,118 +1724,165 @@ app.post('/api/auto-urls', async (req, res) => {
|
||||
|
||||
// Process auto URLs and update IPs
|
||||
app.post('/api/auto-urls/process', async (req, res) => {
|
||||
const https = require('https');
|
||||
const http = require('http');
|
||||
|
||||
const https = require('https');
|
||||
const http = require('http');
|
||||
|
||||
// Helpers
|
||||
const isValidIPv4 = (ip) => {
|
||||
const octets = String(ip || '').trim().split('.');
|
||||
if (octets.length !== 4) return false;
|
||||
return octets.every(o => /^\d{1,3}$/.test(o) && Number(o) >= 0 && Number(o) <= 255);
|
||||
};
|
||||
const isValidCidrV4 = (value) => {
|
||||
const v = String(value || '').trim();
|
||||
const parts = v.split('/');
|
||||
if (parts.length !== 2) return false;
|
||||
const [ip, mask] = parts;
|
||||
if (!isValidIPv4(ip)) return false;
|
||||
if (!/^\d{1,2}$/.test(mask)) return false;
|
||||
const m = Number(mask);
|
||||
return m >= 0 && m <= 32;
|
||||
};
|
||||
const isValidDomain = (value) => {
|
||||
const v = String(value || '').trim().toLowerCase();
|
||||
if (v.startsWith('#')) return false; // comment line
|
||||
return /^([a-z0-9-]+\.)+[a-z]{2,}$/i.test(v);
|
||||
};
|
||||
|
||||
try {
|
||||
// Load configured auto URLs
|
||||
const urlsParams = {
|
||||
Bucket: BUCKET_NAME,
|
||||
Key: 'bgp_data/auto_url/urls.txt',
|
||||
};
|
||||
|
||||
let urls = [];
|
||||
try {
|
||||
// Get current auto URLs
|
||||
const urlsParams = {
|
||||
Bucket: BUCKET_NAME,
|
||||
Key: 'bgp_data/auto_url/urls.txt',
|
||||
};
|
||||
|
||||
let urls = [];
|
||||
try {
|
||||
const urlsData = await s3.getObject(urlsParams).promise();
|
||||
const urlsContent = urlsData.Body.toString('utf-8');
|
||||
urls = urlsContent.split('\n').filter(line => line).map(line => {
|
||||
const parts = line.trim().split(/\s+/);
|
||||
return { url: parts[0] || '', community: parts[1] || '' };
|
||||
});
|
||||
} catch (error) {
|
||||
if (error.code !== 'NoSuchKey') {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
if (urls.length === 0) {
|
||||
return sendError(res, 400, 'No URLs to process', 'E_BAD_REQUEST');
|
||||
}
|
||||
|
||||
// Get current IPs
|
||||
const ipsParams = {
|
||||
Bucket: BUCKET_NAME,
|
||||
Key: 'bgp_data/ips.txt',
|
||||
};
|
||||
|
||||
let currentIps = [];
|
||||
try {
|
||||
const ipsData = await s3.getObject(ipsParams).promise();
|
||||
const ipsContent = ipsData.Body.toString('utf-8');
|
||||
currentIps = ipsContent.split('\n').filter(line => line).map(line => {
|
||||
const parts = line.trim().split(/\s+/);
|
||||
return { ipRange: parts[0] || '', community: parts[1] || '' };
|
||||
});
|
||||
} catch (error) {
|
||||
if (error.code !== 'NoSuchKey') {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Process each URL
|
||||
const newIps = [];
|
||||
for (const urlData of urls) {
|
||||
try {
|
||||
const url = urlData.url.trim();
|
||||
const community = urlData.community.trim();
|
||||
|
||||
if (!url || !community) continue;
|
||||
|
||||
// Download content from URL
|
||||
const content = await new Promise((resolve, reject) => {
|
||||
const protocol = url.startsWith('https:') ? https : http;
|
||||
const req = protocol.get(url, (res) => {
|
||||
let data = '';
|
||||
res.on('data', (chunk) => data += chunk);
|
||||
res.on('end', () => resolve(data));
|
||||
});
|
||||
req.on('error', reject);
|
||||
req.setTimeout(10000, () => req.destroy());
|
||||
});
|
||||
|
||||
// Parse IPs from content
|
||||
const lines = content.split('\n');
|
||||
for (const line of lines) {
|
||||
const ip = line.trim();
|
||||
if (ip && (ip.includes('.') || ip.includes(':'))) {
|
||||
newIps.push({ ipRange: ip, community });
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error(`Error processing URL ${urlData.url}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
// Merge with existing IPs (avoid duplicates)
|
||||
const existingIpRanges = new Set(currentIps.map(ip => ip.ipRange));
|
||||
const uniqueNewIps = newIps.filter(ip => !existingIpRanges.has(ip.ipRange));
|
||||
|
||||
const allIps = [...currentIps, ...uniqueNewIps];
|
||||
|
||||
// Save updated IPs
|
||||
const updatedIpsContent = allIps.map(ip => `${ip.ipRange} ${ip.community}`).join('\n');
|
||||
const updateParams = {
|
||||
Bucket: BUCKET_NAME,
|
||||
Key: 'bgp_data/ips.txt',
|
||||
Body: updatedIpsContent,
|
||||
ContentType: 'text/plain',
|
||||
};
|
||||
|
||||
await s3.putObject(updateParams).promise();
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
message: `Обработано ${urls.length} URL, добавлено ${uniqueNewIps.length} новых IP-адресов`,
|
||||
processedUrls: urls.length,
|
||||
newIpsCount: uniqueNewIps.length,
|
||||
totalIpsCount: allIps.length
|
||||
});
|
||||
|
||||
const urlsData = await s3.getObject(urlsParams).promise();
|
||||
const urlsContent = urlsData.Body.toString('utf-8');
|
||||
urls = urlsContent.split('\n').filter(line => line).map(line => {
|
||||
const parts = line.trim().split(/\s+/);
|
||||
return { url: parts[0] || '', community: parts[1] || '' };
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error processing auto URLs:', error);
|
||||
return sendError(res, 500, 'Error processing auto URLs', 'E_S3');
|
||||
if (error.code !== 'NoSuchKey') {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
if (urls.length === 0) {
|
||||
return sendError(res, 400, 'No URLs to process', 'E_BAD_REQUEST');
|
||||
}
|
||||
|
||||
// Load current IP ranges
|
||||
const ipsParams = { Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt' };
|
||||
let currentIps = [];
|
||||
try {
|
||||
const ipsData = await s3.getObject(ipsParams).promise();
|
||||
const ipsContent = ipsData.Body.toString('utf-8');
|
||||
currentIps = ipsContent.split('\n').filter(line => line).map(line => {
|
||||
const parts = line.trim().split(/\s+/);
|
||||
return { ipRange: parts[0] || '', community: parts[1] || '' };
|
||||
});
|
||||
} catch (error) {
|
||||
if (error.code !== 'NoSuchKey') throw error;
|
||||
}
|
||||
|
||||
// Load current domains
|
||||
const domainsParams = { Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt' };
|
||||
let currentDomains = [];
|
||||
try {
|
||||
const dData = await s3.getObject(domainsParams).promise();
|
||||
const dContent = dData.Body.toString('utf-8');
|
||||
currentDomains = dContent.split('\n').filter(line => line).map(line => {
|
||||
const parts = line.trim().split(/\s+/);
|
||||
return { domain: (parts[0] || '').toLowerCase(), community: parts[1] || '' };
|
||||
});
|
||||
} catch (error) {
|
||||
if (error.code !== 'NoSuchKey') throw error;
|
||||
}
|
||||
|
||||
// Process each URL
|
||||
const newIps = [];
|
||||
const newDomains = [];
|
||||
for (const urlData of urls) {
|
||||
try {
|
||||
const url = String(urlData.url || '').trim();
|
||||
const community = String(urlData.community || '').trim();
|
||||
if (!url || !community) continue;
|
||||
|
||||
// Download content
|
||||
const content = await new Promise((resolve, reject) => {
|
||||
const protocol = url.startsWith('https:') ? https : http;
|
||||
const req = protocol.get(url, (r) => {
|
||||
let data = '';
|
||||
r.on('data', (chunk) => { data += chunk; });
|
||||
r.on('end', () => resolve(data));
|
||||
});
|
||||
req.on('error', reject);
|
||||
req.setTimeout(15000, () => req.destroy());
|
||||
});
|
||||
|
||||
const lines = content.split('\n');
|
||||
for (const raw of lines) {
|
||||
const line = String(raw || '').trim();
|
||||
if (!line) continue;
|
||||
if (line.startsWith('#') || line.startsWith('//')) continue;
|
||||
const token = line.split(/\s+/)[0]?.trim();
|
||||
if (!token) continue;
|
||||
|
||||
// Decide destination
|
||||
if (isValidCidrV4(token)) {
|
||||
newIps.push({ ipRange: token, community });
|
||||
continue;
|
||||
}
|
||||
if (isValidIPv4(token)) {
|
||||
// single IPv4 → normalize to /32
|
||||
newIps.push({ ipRange: `${token}/32`, community });
|
||||
continue;
|
||||
}
|
||||
if (isValidDomain(token)) {
|
||||
newDomains.push({ domain: token.toLowerCase(), community });
|
||||
continue;
|
||||
}
|
||||
// ignore everything else
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error processing URL ${urlData.url}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
// Merge & deduplicate
|
||||
const existingIpRanges = new Set(currentIps.map(i => i.ipRange));
|
||||
const uniqueNewIps = newIps.filter(i => !existingIpRanges.has(i.ipRange));
|
||||
const allIps = [...currentIps, ...uniqueNewIps];
|
||||
|
||||
const existingDomains = new Set(currentDomains.map(d => d.domain));
|
||||
const uniqueNewDomains = newDomains.filter(d => !existingDomains.has(d.domain));
|
||||
const allDomains = [...currentDomains, ...uniqueNewDomains];
|
||||
|
||||
// Save updated IPs
|
||||
const updatedIpsContent = allIps.map(i => `${i.ipRange} ${i.community}`).join('\n');
|
||||
await s3.putObject({ Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt', Body: updatedIpsContent, ContentType: 'text/plain' }).promise();
|
||||
|
||||
// Save updated Domains
|
||||
const updatedDomainsContent = allDomains.map(d => `${d.domain} ${d.community}`).join('\n');
|
||||
await s3.putObject({ Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt', Body: updatedDomainsContent, ContentType: 'text/plain' }).promise();
|
||||
|
||||
const msg = `Обработано ${urls.length} URL, добавлено ${uniqueNewIps.length} IP и ${uniqueNewDomains.length} доменов`;
|
||||
return res.json({
|
||||
success: true,
|
||||
message: msg,
|
||||
processedUrls: urls.length,
|
||||
newIpsCount: uniqueNewIps.length,
|
||||
newDomainsCount: uniqueNewDomains.length,
|
||||
totalIpsCount: allIps.length,
|
||||
totalDomainsCount: allDomains.length
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error processing auto URLs:', error);
|
||||
return sendError(res, 500, 'Error processing auto URLs', 'E_S3');
|
||||
}
|
||||
});
|
||||
|
||||
// --- Proxy: Background BGP Update (avoids CORS from browser) ---
|
||||
|
||||
Reference in New Issue
Block a user