feat: Обновить интеграцию с AWS S3, заменив методы SDK v2 на команды SDK v3 для улучшения производительности и обработки объектов; добавить обработку ошибок для отсутствующих ключей
Publish Fast Tabler Docker image / build-and-push-fast (push) Successful in 1m37s
Publish Fast Tabler Docker image / build-and-push-fast (push) Successful in 1m37s
This commit is contained in:
+33
-25
@@ -1179,7 +1179,7 @@ app.get('/api/s3/last-modified', async (req, res) => {
|
||||
{ name: 'uiSettings', key: 'bgp_data/rt_ui_settings.json' }
|
||||
];
|
||||
const results = await Promise.allSettled(
|
||||
keys.map(k => s3.headObject({ Bucket: BUCKET_NAME, Key: k.key }).promise())
|
||||
keys.map(k => s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: k.key })))
|
||||
);
|
||||
const out = {};
|
||||
results.forEach((r, idx) => {
|
||||
@@ -1239,10 +1239,7 @@ app.get('/api/history/:resource', async (req, res) => {
|
||||
const key = resourceToKey(resource);
|
||||
if (!key) return sendError(res, 400, 'Unknown resource', 'E_RESOURCE');
|
||||
try {
|
||||
if (!s3.listObjectVersions) {
|
||||
return sendError(res, 501, 'S3 listObjectVersions not available', 'E_NOT_SUPPORTED');
|
||||
}
|
||||
const out = await s3.listObjectVersions({ Bucket: BUCKET_NAME, Prefix: key, MaxKeys: 20 }).promise();
|
||||
const out = await s3.send(new ListObjectVersionsCommand({ Bucket: BUCKET_NAME, Prefix: key, MaxKeys: 20 }));
|
||||
const versions = (out.Versions || [])
|
||||
.filter(v => v.Key === key)
|
||||
.slice(0, 10)
|
||||
@@ -1261,7 +1258,7 @@ app.post('/api/history/:resource/rollback', async (req, res) => {
|
||||
if (!key || !versionId) return sendError(res, 400, 'Bad request', 'E_BAD_REQUEST');
|
||||
try {
|
||||
// Copy specific version over same key to rollback
|
||||
await s3.copyObject({ Bucket: BUCKET_NAME, CopySource: `/${BUCKET_NAME}/${encodeURIComponent(key)}?versionId=${encodeURIComponent(versionId)}`, Key: key }).promise();
|
||||
await s3.send(new CopyObjectCommand({ Bucket: BUCKET_NAME, CopySource: `/${BUCKET_NAME}/${encodeURIComponent(key)}?versionId=${encodeURIComponent(versionId)}`, Key: key }));
|
||||
const meta = await headMeta(key);
|
||||
return sendOk(res, meta);
|
||||
} catch (e) {
|
||||
@@ -1393,7 +1390,7 @@ app.post('/api/filters/export-config', async (req, res) => {
|
||||
ContentType: 'text/plain',
|
||||
};
|
||||
|
||||
await s3.putObject(exportParams).promise();
|
||||
await s3.send(new PutObjectCommand(exportParams));
|
||||
res.json({ success: true, message: 'Конфигурация экспортирована в S3' });
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
@@ -1709,8 +1706,15 @@ app.get('/api/simple-filters', async (req, res) => {
|
||||
};
|
||||
|
||||
try {
|
||||
const data = await s3.getObject(params).promise();
|
||||
const fileContent = data.Body.toString('utf-8');
|
||||
const head = await s3.send(new HeadObjectCommand(params)).catch(() => null);
|
||||
const etag = head?.ETag || null;
|
||||
if (etag) res.set('ETag', String(etag));
|
||||
if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
|
||||
if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
|
||||
if (checkIfNoneMatch(req, res, etag)) return;
|
||||
|
||||
const data = await s3.send(new GetObjectCommand(params));
|
||||
const fileContent = await streamToString(data.Body);
|
||||
let filters = [];
|
||||
|
||||
try {
|
||||
@@ -1725,7 +1729,7 @@ app.get('/api/simple-filters', async (req, res) => {
|
||||
|
||||
res.json(filters);
|
||||
} catch (error) {
|
||||
if (error.code === 'NoSuchKey') {
|
||||
if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) {
|
||||
res.json([]); // Return empty array if file does not exist
|
||||
} else {
|
||||
console.error(error);
|
||||
@@ -1814,8 +1818,15 @@ app.get('/api/ui-settings', async (req, res) => {
|
||||
};
|
||||
|
||||
try {
|
||||
const data = await s3.getObject(params).promise();
|
||||
const jsonText = data.Body.toString('utf-8');
|
||||
const head = await s3.send(new HeadObjectCommand(params)).catch(() => null);
|
||||
const etag = head?.ETag || null;
|
||||
if (etag) res.set('ETag', String(etag));
|
||||
if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
|
||||
if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
|
||||
if (checkIfNoneMatch(req, res, etag)) return;
|
||||
|
||||
const data = await s3.send(new GetObjectCommand(params));
|
||||
const jsonText = await streamToString(data.Body);
|
||||
let settings = {};
|
||||
try {
|
||||
const parsed = JSON.parse(jsonText);
|
||||
@@ -1823,12 +1834,9 @@ app.get('/api/ui-settings', async (req, res) => {
|
||||
} catch (parseError) {
|
||||
settings = {};
|
||||
}
|
||||
if (data.ETag) res.set('ETag', String(data.ETag));
|
||||
if (data.LastModified) res.set('Last-Modified', new Date(data.LastModified).toUTCString());
|
||||
if (typeof data.ContentLength === 'number') res.set('Content-Length-Source', String(data.ContentLength));
|
||||
return res.json(settings);
|
||||
} catch (error) {
|
||||
if (error.code === 'NoSuchKey') {
|
||||
if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) {
|
||||
return res.json({});
|
||||
}
|
||||
console.error('Error reading ui settings from S3:', error);
|
||||
@@ -1860,7 +1868,7 @@ app.post('/api/ui-settings', async (req, res) => {
|
||||
};
|
||||
|
||||
try {
|
||||
await s3.putObject(params).promise();
|
||||
await s3.send(new PutObjectCommand(params));
|
||||
const meta = await headMeta('bgp_data/rt_ui_settings.json');
|
||||
return sendOk(res, meta);
|
||||
} catch (error) {
|
||||
@@ -2004,8 +2012,8 @@ app.post('/api/auto-urls/process', async (req, res) => {
|
||||
|
||||
let urls = [];
|
||||
try {
|
||||
const urlsData = await s3.getObject(urlsParams).promise();
|
||||
const urlsContent = urlsData.Body.toString('utf-8');
|
||||
const urlsData = await s3.send(new GetObjectCommand(urlsParams));
|
||||
const urlsContent = await streamToString(urlsData.Body);
|
||||
urls = urlsContent.split('\n').filter(line => line).map(line => {
|
||||
const parts = line.trim().split(/\s+/);
|
||||
return { url: parts[0] || '', community: parts[1] || '' };
|
||||
@@ -2024,8 +2032,8 @@ app.post('/api/auto-urls/process', async (req, res) => {
|
||||
const ipsParams = { Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt' };
|
||||
let currentIps = [];
|
||||
try {
|
||||
const ipsData = await s3.getObject(ipsParams).promise();
|
||||
const ipsContent = ipsData.Body.toString('utf-8');
|
||||
const ipsData = await s3.send(new GetObjectCommand(ipsParams));
|
||||
const ipsContent = await streamToString(ipsData.Body);
|
||||
currentIps = ipsContent.split('\n').filter(line => line).map(line => {
|
||||
const parts = line.trim().split(/\s+/);
|
||||
return { ipRange: parts[0] || '', community: parts[1] || '' };
|
||||
@@ -2038,8 +2046,8 @@ app.post('/api/auto-urls/process', async (req, res) => {
|
||||
const domainsParams = { Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt' };
|
||||
let currentDomains = [];
|
||||
try {
|
||||
const dData = await s3.getObject(domainsParams).promise();
|
||||
const dContent = dData.Body.toString('utf-8');
|
||||
const dData = await s3.send(new GetObjectCommand(domainsParams));
|
||||
const dContent = await streamToString(dData.Body);
|
||||
currentDomains = dContent.split('\n').filter(line => line).map(line => {
|
||||
const parts = line.trim().split(/\s+/);
|
||||
return { domain: (parts[0] || '').toLowerCase(), community: parts[1] || '' };
|
||||
@@ -2109,11 +2117,11 @@ app.post('/api/auto-urls/process', async (req, res) => {
|
||||
|
||||
// Save updated IPs
|
||||
const updatedIpsContent = allIps.map(i => `${i.ipRange} ${i.community}`).join('\n');
|
||||
await s3.putObject({ Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt', Body: updatedIpsContent, ContentType: 'text/plain' }).promise();
|
||||
await s3.send(new PutObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt', Body: updatedIpsContent, ContentType: 'text/plain' }));
|
||||
|
||||
// Save updated Domains
|
||||
const updatedDomainsContent = allDomains.map(d => `${d.domain} ${d.community}`).join('\n');
|
||||
await s3.putObject({ Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt', Body: updatedDomainsContent, ContentType: 'text/plain' }).promise();
|
||||
await s3.send(new PutObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt', Body: updatedDomainsContent, ContentType: 'text/plain' }));
|
||||
|
||||
const msg = `Обработано ${urls.length} URL, добавлено ${uniqueNewIps.length} IP и ${uniqueNewDomains.length} доменов`;
|
||||
return res.json({
|
||||
|
||||
Reference in New Issue
Block a user