From 20ee4704a4fb942f57eab460b99d202137c3fcaf Mon Sep 17 00:00:00 2001 From: shats Date: Wed, 27 Aug 2025 12:19:13 +0700 Subject: [PATCH] =?UTF-8?q?feat:=20=D0=9E=D0=B1=D0=BD=D0=BE=D0=B2=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20=D0=B8=D0=BD=D1=82=D0=B5=D0=B3=D1=80=D0=B0=D1=86?= =?UTF-8?q?=D0=B8=D1=8E=20=D1=81=20AWS=20S3,=20=D0=B7=D0=B0=D0=BC=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B2=20=D1=81=D1=82=D0=B0=D1=80=D1=8B=D0=B9=20SDK?= =?UTF-8?q?=20=D0=BD=D0=B0=20=D0=BD=D0=BE=D0=B2=D1=8B=D0=B9;=20=D1=83?= =?UTF-8?q?=D0=BB=D1=83=D1=87=D1=88=D0=B8=D1=82=D1=8C=20=D0=BE=D0=B1=D1=80?= =?UTF-8?q?=D0=B0=D0=B1=D0=BE=D1=82=D0=BA=D1=83=20=D0=BE=D0=B1=D1=8A=D0=B5?= =?UTF-8?q?=D0=BA=D1=82=D0=BE=D0=B2=20S3=20=D1=81=20=D0=B8=D1=81=D0=BF?= =?UTF-8?q?=D0=BE=D0=BB=D1=8C=D0=B7=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8=D0=B5?= =?UTF-8?q?=D0=BC=20=D0=BA=D0=BE=D0=BC=D0=B0=D0=BD=D0=B4=20SDK=20v3=20?= =?UTF-8?q?=D0=B8=20=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D1=82=D1=8C=20?= =?UTF-8?q?=D0=BA=D1=8D=D1=88=D0=B8=D1=80=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8?= =?UTF-8?q?=D0=B5=20=D0=B4=D0=BB=D1=8F=20=D0=BF=D0=BE=D0=B2=D1=8B=D1=88?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=BF=D1=80=D0=BE=D0=B8=D0=B7=D0=B2?= =?UTF-8?q?=D0=BE=D0=B4=D0=B8=D1=82=D0=B5=D0=BB=D1=8C=D0=BD=D0=BE=D1=81?= =?UTF-8?q?=D1=82=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/package.json | 1 - backend/server.js | 471 ++++++++++++++++++++++++++++++------------- 2 files changed, 326 insertions(+), 146 deletions(-) diff --git a/backend/package.json b/backend/package.json index fc9cacc..ac7cbf4 100644 --- a/backend/package.json +++ b/backend/package.json @@ -16,7 +16,6 @@ "license": "ISC", "description": "", "dependencies": { - "aws-sdk": "^2.1692.0", "cors": "^2.8.5", "dotenv": "^17.0.1", "express": "^4.19.2", diff --git a/backend/server.js b/backend/server.js index 2ef2fb5..2ce1eff 100644 --- a/backend/server.js +++ b/backend/server.js @@ -1,7 +1,7 @@ // History endpoints are registered below after app is initialized require('dotenv').config(); const express = require('express'); -const AWS = require('aws-sdk'); +const { S3Client, GetObjectCommand, HeadObjectCommand, PutObjectCommand, DeleteObjectCommand, CopyObjectCommand, ListObjectVersionsCommand } = require('@aws-sdk/client-s3'); const cors = require('cors'); const path = require('path'); const compression = require('compression'); @@ -109,7 +109,7 @@ function toIso(x) { async function headMeta(key) { try { - const h = await s3.headObject({ Bucket: BUCKET_NAME, Key: key }).promise(); + const h = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: key })); return { etag: h.ETag || null, lastModified: h.LastModified ? toIso(h.LastModified) : null, @@ -157,17 +157,20 @@ function mapAjvErrors(errors) { })); } +// Helpers +function splitWhitespace(line) { + return String(line || '').trim().split(/\s+/); +} + // Serve static files from the React app app.use(express.static(path.join(__dirname, 'public'))); -// Configure AWS S3 -const s3 = new AWS.S3({ +// Configure AWS S3 (SDK v3) +const s3 = new S3Client({ endpoint: 'https://storage.yandexcloud.net', region: process.env.AWS_REGION, - s3ForcePathStyle: true, - signatureVersion: 'v4', - httpOptions: { timeout: 15000 }, - maxRetries: 3, + forcePathStyle: true, + maxAttempts: 3, credentials: { accessKeyId: process.env.AWS_ACCESS_KEY_ID || process.env.S3_ACCESS_KEY_ID, secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || process.env.S3_SECRET_ACCESS_KEY, @@ -297,6 +300,7 @@ function buildNestedGatewayBlocks(gatewayGroups, baseIndentSpaces = 4) { // In-memory LRU-ish cache for small texts and head meta const s3Cache = { text: new Map(), head: new Map(), max: 100, ttlMs: 30_000 }; +const countOnlyCache = { map: new Map(), ttlMs: 10_000 }; function getCache(map, key) { const v = map.get(key); if (!v) return null; @@ -308,15 +312,51 @@ function setCache(map, key, value) { map.set(key, { value, at: Date.now() }); } +function invalidateCacheForKey(key) { + try { s3Cache.text.delete(key); } catch {} + try { s3Cache.head.delete(key); } catch {} +} + +function getCountOnlyCache(cacheKey) { + const v = countOnlyCache.map.get(cacheKey); + if (!v) return null; + if (Date.now() > v.at + countOnlyCache.ttlMs) { countOnlyCache.map.delete(cacheKey); return null; } + return v.value; +} +function setCountOnlyCache(cacheKey, value) { + countOnlyCache.map.set(cacheKey, { value, at: Date.now() }); +} + +function checkIfNoneMatch(req, res, etag) { + const inm = req.headers && (req.headers['if-none-match'] || req.headers['If-None-Match']); + if (inm && etag && String(inm) === String(etag)) { + res.status(304).end(); + return true; + } + return false; +} + // Helper: read text file from S3 and return { body, etag, lastModified, contentLength } +async function streamToString(stream) { + if (!stream) return ''; + if (typeof stream.transformToString === 'function') { + return await stream.transformToString(); + } + return await new Promise((resolve, reject) => { + let chunks = []; + stream.on('data', (c) => chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(String(c)))); + stream.once('error', reject); + stream.once('end', () => resolve(Buffer.concat(chunks).toString('utf-8'))); + }); +} + async function readS3TextObject(key) { const cached = getCache(s3Cache.text, key); if (cached) return cached; - const params = { Bucket: BUCKET_NAME, Key: key }; - const data = await s3.getObject(params).promise(); + const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: key })); const out = { - body: data.Body.toString('utf-8'), - etag: data.ETag ? String(data.ETag).replace(/\"/g, '"') : undefined, + body: await streamToString(data.Body), + etag: data.ETag || undefined, lastModified: data.LastModified ? data.LastModified.toISOString() : undefined, contentLength: typeof data.ContentLength === 'number' ? data.ContentLength : undefined }; @@ -328,9 +368,9 @@ async function readS3TextObject(key) { async function headS3ObjectEtag(key) { const cached = getCache(s3Cache.head, key); if (cached && cached.etag) return cached.etag; - const head = await s3.headObject({ Bucket: BUCKET_NAME, Key: key }).promise(); - setCache(s3Cache.head, key, { etag: head.ETag ? String(head.ETag).replace(/\"/g, '"') : undefined }); - return head.ETag ? String(head.ETag).replace(/\"/g, '"') : undefined; + const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: key })); + setCache(s3Cache.head, key, { etag: head.ETag || undefined }); + return head.ETag || undefined; } // Helper: stream and paginate big text files (line-based) @@ -344,43 +384,62 @@ async function streamPaginatedText({ key, mapLine, q, offset = 0, limit = 0 }) { if (!q) return true; return line.toLowerCase().includes(String(q).toLowerCase()); }; - const stream = s3.getObject({ Bucket: BUCKET_NAME, Key: key }).createReadStream(); - stream.on('data', (chunk) => { - buffered += chunk.toString('utf-8'); - let lines = buffered.split('\n'); - buffered = lines.pop(); - for (const lnRaw of lines) { - const line = lnRaw.trim(); - if (!line) continue; - if (!matchesQuery(line)) continue; - total++; - const pos = total - 1; // position among matches - if (limit > 0) { - if (pos >= offset && sent < limit) { - items.push(mapLine(line)); - sent++; - } - } else { - items.push(mapLine(line)); - } - } - }); - stream.on('end', () => { - const last = (buffered || '').trim(); - if (last) { - if (!q || last.toLowerCase().includes(String(q).toLowerCase())) { + try { + const resp = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: key })); + const stream = resp.Body; + if (!stream || typeof stream.on !== 'function') { + const text = await streamToString(resp.Body); + const lines = text.split('\n'); + for (let i = 0; i < lines.length; i++) { + const line = String(lines[i] || '').trim(); + if (!line) continue; + if (!matchesQuery(line)) continue; total++; + const pos = total - 1; if (limit > 0) { - const pos = total - 1; - if (pos >= offset && items.length < limit) items.push(mapLine(last)); + if (pos >= offset && sent < limit) { items.push(mapLine(line)); sent++; } } else { - items.push(mapLine(last)); + items.push(mapLine(line)); } } + return resolve({ items, total }); } - resolve({ items, total }); - }); - stream.on('error', reject); + stream.on('data', (chunk) => { + buffered += chunk.toString('utf-8'); + let lines = buffered.split('\n'); + buffered = lines.pop(); + for (const lnRaw of lines) { + const line = lnRaw.trim(); + if (!line) continue; + if (!matchesQuery(line)) continue; + total++; + const pos = total - 1; + if (limit > 0) { + if (pos >= offset && sent < limit) { items.push(mapLine(line)); sent++; } + } else { + items.push(mapLine(line)); + } + } + }); + stream.on('end', () => { + const last = (buffered || '').trim(); + if (last) { + if (!q || last.toLowerCase().includes(String(q).toLowerCase())) { + total++; + if (limit > 0) { + const pos = total - 1; + if (pos >= offset && items.length < limit) items.push(mapLine(last)); + } else { + items.push(mapLine(last)); + } + } + } + resolve({ items, total }); + }); + stream.on('error', reject); + } catch (e) { + reject(e); + } }); } @@ -396,37 +455,52 @@ function cleanupExpiredLocks() { } setInterval(cleanupExpiredLocks, 30_000); -// Get domains from S3 +// Get domains from S3 (supports countOnly/std and If-None-Match) app.get('/api/domains', async (req, res) => { - const params = { - Bucket: BUCKET_NAME, - Key: FILE_KEY, - }; - + const { q = '', offset, limit, countOnly, format } = req.query || {}; try { - const data = await s3.getObject(params).promise(); - const fileContent = data.Body.toString('utf-8'); - const domains = fileContent.split('\n').filter(line => line).map(line => { - // Trim the line to remove any leading/trailing whitespace, including \r - // And split by any whitespace to be more robust - const parts = line.trim().split(/\s+/); - const domain = parts[0] || ''; - const type = parts[1] || ''; - return { domain, type }; - }); - if (data.ETag) { - res.set('ETag', String(data.ETag)); + if (countOnly === 'true') { + const cacheKey = `domains:count:${q}`; + const cached = getCountOnlyCache(cacheKey); + if (cached != null) return res.json(format === 'std' ? { items: [], total: cached, meta: {} } : { total: cached }); + const { total } = await streamPaginatedText({ key: FILE_KEY, q, offset: 0, limit: 0, mapLine: () => ({}) }); + setCountOnlyCache(cacheKey, total); + return res.json(format === 'std' ? { items: [], total, meta: {} } : { total }); + } else if (typeof limit !== 'undefined') { + const { items, total } = await streamPaginatedText({ + key: FILE_KEY, + q, + offset: Number(offset) || 0, + limit: Number(limit) || 0, + mapLine: (line) => { + const parts = splitWhitespace(line); + return { domain: parts[0] || '', type: parts[1] || '' }; + } + }); + if (!validateAsns(items)) return sendError(res, 500, 'Invalid data format', 'E_SCHEMA'); + return res.json(format === 'std' ? { items, total, meta: {} } : { items, total }); + } else { + const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: FILE_KEY })).catch(() => null); + const etag = head?.ETag || null; + if (etag) res.set('ETag', String(etag)); + if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString()); + if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength)); + if (checkIfNoneMatch(req, res, etag)) return; // 304 + + const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: FILE_KEY })); + const fileContent = await streamToString(data.Body); + const domains = fileContent.split('\n').filter(line => line).map(line => { + const parts = splitWhitespace(line); + const domain = parts[0] || ''; + const type = parts[1] || ''; + return { domain, type }; + }); + if (format === 'std') return res.json({ items: domains, total: domains.length, meta: {} }); + res.json(domains); } - if (data.LastModified) { - res.set('Last-Modified', new Date(data.LastModified).toUTCString()); - } - if (typeof data.ContentLength === 'number') { - res.set('Content-Length-Source', String(data.ContentLength)); - } - res.json(domains); } catch (error) { - if (error.code === 'NoSuchKey') { - res.json([]); // Return empty array if file does not exist + if (error?.name === 'NotFound' || error?.$metadata?.httpStatusCode === 404) { + res.json([]); } else { console.error(error); return sendError(res, 500, 'Error reading from S3', 'E_S3', { error: String(error?.message || error) }); @@ -459,9 +533,10 @@ app.post('/api/domains', async (req, res) => { }; try { - const put = await s3.putObject(params).promise(); - res.set('ETag', put.ETag || ''); - res.send('File updated successfully'); + await s3.send(new PutObjectCommand(params)); + invalidateCacheForKey(FILE_KEY); + const meta = await headMeta(FILE_KEY); + return sendOk(res, meta); } catch (error) { console.error(error); return sendError(res, 500, 'Error writing to S3', 'E_S3'); @@ -500,8 +575,15 @@ app.get('/api/asns', async (req, res) => { if (!validateAsns(items)) return sendError(res, 500, 'Invalid data format', 'E_SCHEMA'); return res.json(format === 'std' ? { items, total, meta: {} } : { items, total }); } else { - const data = await s3.getObject(params).promise(); - const fileContent = data.Body.toString('utf-8'); + const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/asns.txt' })).catch(() => null); + const etag = head?.ETag || null; + if (etag) res.set('ETag', String(etag)); + if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString()); + if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength)); + if (checkIfNoneMatch(req, res, etag)) return; + + const data = await s3.send(new GetObjectCommand(params)); + const fileContent = await streamToString(data.Body); const asns = fileContent.split('\n').filter(line => line).map(line => { const parts = line.trim().split(/\s+/); const domain = parts[0] || ''; @@ -509,14 +591,11 @@ app.get('/api/asns', async (req, res) => { return { domain, type }; }); if (!validateAsns(asns)) return sendError(res, 500, 'Invalid data format', 'E_SCHEMA'); - if (data.ETag) res.set('ETag', String(data.ETag)); - if (data.LastModified) res.set('Last-Modified', new Date(data.LastModified).toUTCString()); - if (typeof data.ContentLength === 'number') res.set('Content-Length-Source', String(data.ContentLength)); if (format === 'std') return res.json({ items: asns, total: asns.length, meta: {} }); res.json(asns); } } catch (error) { - if (error.code === 'NoSuchKey') { + if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) { res.json([]); } else { console.error(error); @@ -551,7 +630,8 @@ app.post('/api/asns', async (req, res) => { }; try { - const put = await s3.putObject(params).promise(); + await s3.send(new PutObjectCommand(params)); + invalidateCacheForKey('bgp_data/asns.txt'); const meta = await headMeta('bgp_data/asns.txt'); return sendOk(res, meta); } catch (error) { @@ -591,8 +671,15 @@ app.get('/api/domains-new', async (req, res) => { if (!validateDomainsNew(items)) return sendError(res, 500, 'Invalid data format', 'E_SCHEMA'); return res.json(format === 'std' ? { items, total, meta: {} } : { items, total }); } else { - const data = await s3.getObject(params).promise(); - const fileContent = data.Body.toString('utf-8'); + const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt' })).catch(() => null); + const etag = head?.ETag || null; + if (etag) res.set('ETag', String(etag)); + if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString()); + if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength)); + if (checkIfNoneMatch(req, res, etag)) return; + + const data = await s3.send(new GetObjectCommand(params)); + const fileContent = await streamToString(data.Body); const domains = fileContent.split('\n').filter(line => line).map(line => { const parts = line.trim().split(/\s+/); const domain = parts[0] || ''; @@ -600,14 +687,11 @@ app.get('/api/domains-new', async (req, res) => { return { domain, community }; }); if (!validateDomainsNew(domains)) return sendError(res, 500, 'Invalid data format', 'E_SCHEMA'); - if (data.ETag) res.set('ETag', String(data.ETag)); - if (data.LastModified) res.set('Last-Modified', new Date(data.LastModified).toUTCString()); - if (typeof data.ContentLength === 'number') res.set('Content-Length-Source', String(data.ContentLength)); if (format === 'std') return res.json({ items: domains, total: domains.length, meta: {} }); res.json(domains); } } catch (error) { - if (error.code === 'NoSuchKey') { + if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) { res.json([]); // Return empty array if file does not exist } else { console.error(error); @@ -642,7 +726,8 @@ app.post('/api/domains-new', async (req, res) => { }; try { - const put = await s3.putObject(params).promise(); + await s3.send(new PutObjectCommand(params)); + invalidateCacheForKey('bgp_data/domains_community.txt'); const meta = await headMeta('bgp_data/domains_community.txt'); return sendOk(res, meta); } catch (error) { @@ -682,8 +767,15 @@ app.get('/api/ip-ranges', async (req, res) => { if (!validateIpRanges(items)) return sendError(res, 500, 'Invalid data format', 'E_SCHEMA'); return res.json(format === 'std' ? { items, total, meta: {} } : { items, total }); } else { - const data = await s3.getObject(params).promise(); - const fileContent = data.Body.toString('utf-8'); + const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt' })).catch(() => null); + const etag = head?.ETag || null; + if (etag) res.set('ETag', String(etag)); + if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString()); + if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength)); + if (checkIfNoneMatch(req, res, etag)) return; + + const data = await s3.send(new GetObjectCommand(params)); + const fileContent = await streamToString(data.Body); const ipRanges = fileContent.split('\n').filter(line => line).map(line => { const parts = line.trim().split(/\s+/); const ipRange = parts[0] || ''; @@ -691,14 +783,11 @@ app.get('/api/ip-ranges', async (req, res) => { return { ipRange, community }; }); if (!validateIpRanges(ipRanges)) return sendError(res, 500, 'Invalid data format', 'E_SCHEMA'); - if (data.ETag) res.set('ETag', String(data.ETag)); - if (data.LastModified) res.set('Last-Modified', new Date(data.LastModified).toUTCString()); - if (typeof data.ContentLength === 'number') res.set('Content-Length-Source', String(data.ContentLength)); if (format === 'std') return res.json({ items: ipRanges, total: ipRanges.length, meta: {} }); res.json(ipRanges); } } catch (error) { - if (error.code === 'NoSuchKey') { + if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) { res.json([]); // Return empty array if file does not exist } else { console.error(error); @@ -733,7 +822,8 @@ app.post('/api/ip-ranges', async (req, res) => { }; try { - const put = await s3.putObject(params).promise(); + await s3.send(new PutObjectCommand(params)); + invalidateCacheForKey('bgp_data/ips.txt'); const meta = await headMeta('bgp_data/ips.txt'); return sendOk(res, meta); } catch (error) { @@ -752,8 +842,15 @@ app.get('/api/communities', async (req, res) => { }; try { - const data = await s3.getObject(params).promise(); - const fileContent = data.Body.toString('utf-8'); + const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/communities.json' })).catch(() => null); + const etag = head?.ETag || null; + if (etag) res.set('ETag', String(etag)); + if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString()); + if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength)); + if (checkIfNoneMatch(req, res, etag)) return; + + const data = await s3.send(new GetObjectCommand(params)); + const fileContent = await streamToString(data.Body); let communities = []; try { @@ -826,8 +923,10 @@ app.post('/api/communities', async (req, res) => { }; try { - await s3.putObject(params).promise(); - res.send('Communities updated successfully'); + await s3.send(new PutObjectCommand(params)); + invalidateCacheForKey('bgp_data/communities.json'); + const meta = await headMeta('bgp_data/communities.json'); + return sendOk(res, meta); } catch (error) { console.error('Error writing communities to S3:', error); return sendError(res, 500, 'Error writing communities to S3', 'E_S3'); @@ -844,8 +943,15 @@ app.get('/api/servers', async (req, res) => { }; try { - const data = await s3.getObject(params).promise(); - const fileContent = data.Body.toString('utf-8'); + const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'servers.json' })).catch(() => null); + const etag = head?.ETag || null; + if (etag) res.set('ETag', String(etag)); + if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString()); + if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength)); + if (checkIfNoneMatch(req, res, etag)) return; + + const data = await s3.send(new GetObjectCommand(params)); + const fileContent = await streamToString(data.Body); let servers = []; try { @@ -895,8 +1001,10 @@ app.post('/api/servers', async (req, res) => { }; try { - await s3.putObject(params).promise(); - res.send('File updated successfully'); + await s3.send(new PutObjectCommand(params)); + invalidateCacheForKey('servers.json'); + const meta = await headMeta('servers.json'); + return sendOk(res, meta); } catch (error) { console.error(error); return sendError(res, 500, 'Error writing to S3', 'E_S3'); @@ -913,8 +1021,15 @@ app.get('/api/billing', async (req, res) => { }; try { - const data = await s3.getObject(params).promise(); - const fileContent = data.Body.toString('utf-8'); + const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'servers-billing.json' })).catch(() => null); + const etag = head?.ETag || null; + if (etag) res.set('ETag', String(etag)); + if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString()); + if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength)); + if (checkIfNoneMatch(req, res, etag)) return; + + const data = await s3.send(new GetObjectCommand(params)); + const fileContent = await streamToString(data.Body); let billingData = []; try { @@ -964,8 +1079,10 @@ app.post('/api/billing', async (req, res) => { }; try { - await s3.putObject(params).promise(); - res.send('File updated successfully'); + await s3.send(new PutObjectCommand(params)); + invalidateCacheForKey('servers-billing.json'); + const meta = await headMeta('servers-billing.json'); + return sendOk(res, meta); } catch (error) { console.error(error); return sendError(res, 500, 'Error writing to S3', 'E_S3'); @@ -982,8 +1099,15 @@ app.get('/api/filters', async (req, res) => { }; try { - const data = await s3.getObject(params).promise(); - const fileContent = data.Body.toString('utf-8'); + const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'filters.json' })).catch(() => null); + const etag = head?.ETag || null; + if (etag) res.set('ETag', String(etag)); + if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString()); + if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength)); + if (checkIfNoneMatch(req, res, etag)) return; + + const data = await s3.send(new GetObjectCommand(params)); + const fileContent = await streamToString(data.Body); let filters = []; try { @@ -1033,8 +1157,10 @@ app.post('/api/filters', async (req, res) => { }; try { - await s3.putObject(params).promise(); - res.send('File updated successfully'); + await s3.send(new PutObjectCommand(params)); + invalidateCacheForKey('filters.json'); + const meta = await headMeta('filters.json'); + return sendOk(res, meta); } catch (error) { console.error(error); return sendError(res, 500, 'Error writing to S3', 'E_S3'); @@ -1152,8 +1278,15 @@ app.get('/api/filters/generate-config', async (req, res) => { }; try { - const data = await s3.getObject(params).promise(); - const fileContent = data.Body.toString('utf-8'); + const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'server-configs.json' })).catch(() => null); + const etag = head?.ETag || null; + if (etag) res.set('ETag', String(etag)); + if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString()); + if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength)); + if (checkIfNoneMatch(req, res, etag)) return; + + const data = await s3.send(new GetObjectCommand(params)); + const fileContent = await streamToString(data.Body); let filters = []; try { @@ -1208,8 +1341,15 @@ app.post('/api/filters/export-config', async (req, res) => { }; try { - const data = await s3.getObject(params).promise(); - const fileContent = data.Body.toString('utf-8'); + const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: `filter-manager/server-filters-${serverId}.json` })).catch(() => null); + const etag = head?.ETag || null; + if (etag) res.set('ETag', String(etag)); + if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString()); + if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength)); + if (checkIfNoneMatch(req, res, etag)) return; + + const data = await s3.send(new GetObjectCommand(params)); + const fileContent = await streamToString(data.Body); let filters = []; try { @@ -1271,8 +1411,15 @@ app.get('/api/server-configs', async (req, res) => { }; try { - const data = await s3.getObject(params).promise(); - const fileContent = data.Body.toString('utf-8'); + const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'filter-manager/simple-filters.json' })).catch(() => null); + const etag = head?.ETag || null; + if (etag) res.set('ETag', String(etag)); + if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString()); + if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength)); + if (checkIfNoneMatch(req, res, etag)) return; + + const data = await s3.send(new GetObjectCommand(params)); + const fileContent = await streamToString(data.Body); let servers = []; try { @@ -1321,8 +1468,10 @@ app.post('/api/server-configs', async (req, res) => { }; try { - await s3.putObject(params).promise(); - res.send('Server configs updated successfully'); + await s3.send(new PutObjectCommand(params)); + invalidateCacheForKey('server-configs.json'); + const meta = await headMeta('server-configs.json'); + return sendOk(res, meta); } catch (error) { console.error(error); return sendError(res, 500, 'Error writing server configs to S3', 'E_S3'); @@ -1338,11 +1487,18 @@ app.get('/api/server-configs/:serverId', async (req, res) => { }; try { - const data = await s3.getObject(params).promise(); - const config = data.Body.toString('utf-8'); + const head = await s3.send(new HeadObjectCommand(params)).catch(() => null); + const etag = head?.ETag || null; + if (etag) res.set('ETag', String(etag)); + if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString()); + if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength)); + if (checkIfNoneMatch(req, res, etag)) return; + + const data = await s3.send(new GetObjectCommand(params)); + const config = await streamToString(data.Body); res.json({ config }); } catch (error) { - if (error.code === 'NoSuchKey') { + if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) { res.json({ config: '// Конфигурация не найдена' }); } else { console.error(error); @@ -1368,8 +1524,10 @@ app.post('/api/server-configs/:serverId', async (req, res) => { }; try { - await s3.putObject(params).promise(); - res.send('Server config saved successfully'); + await s3.send(new PutObjectCommand(params)); + invalidateCacheForKey(`filter-manager/config-${serverId}.txt`); + const meta = await headMeta(`filter-manager/config-${serverId}.txt`); + return sendOk(res, meta); } catch (error) { console.error(error); return sendError(res, 500, 'Error writing server config to S3', 'E_S3'); @@ -1385,8 +1543,10 @@ app.delete('/api/server-configs/:serverId', async (req, res) => { }; try { - await s3.deleteObject(params).promise(); - res.send('Server config deleted successfully'); + await s3.send(new DeleteObjectCommand(params)); + invalidateCacheForKey(`filter-manager/config-${serverId}.txt`); + const meta = await headMeta(`filter-manager/config-${serverId}.txt`); + return sendOk(res, meta); } catch (error) { console.error(error); return sendError(res, 500, 'Error deleting server config from S3', 'E_S3'); @@ -1412,11 +1572,12 @@ app.delete('/api/server-configs/:serverId/complete', async (req, res) => { // Удаляем оба файла параллельно await Promise.allSettled([ - s3.deleteObject(configParams).promise(), - s3.deleteObject(filtersParams).promise() + s3.send(new DeleteObjectCommand(configParams)), + s3.send(new DeleteObjectCommand(filtersParams)) ]); - - res.send('Server and all associated files deleted successfully'); + invalidateCacheForKey(configParams.Key); + invalidateCacheForKey(filtersParams.Key); + return res.json({ ok: true, etag: null, lastModified: null, contentLength: null }); } catch (error) { console.error(error); return sendError(res, 500, 'Error deleting server files from S3', 'E_S3'); @@ -1434,8 +1595,15 @@ app.get('/api/server-filters/:serverId', async (req, res) => { }; try { - const data = await s3.getObject(params).promise(); - const fileContent = data.Body.toString('utf-8'); + const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/auto_url/urls.txt' })).catch(() => null); + const etag = head?.ETag || null; + if (etag) res.set('ETag', String(etag)); + if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString()); + if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength)); + if (checkIfNoneMatch(req, res, etag)) return; + + const data = await s3.send(new GetObjectCommand(params)); + const fileContent = await streamToString(data.Body); let filters = []; try { @@ -1521,8 +1689,10 @@ app.post('/api/server-filters/:serverId', async (req, res) => { }; try { - await s3.putObject(params).promise(); - res.send('Server filters updated successfully'); + await s3.send(new PutObjectCommand(params)); + invalidateCacheForKey(`filter-manager/server-filters-${serverId}.json`); + const meta = await headMeta(`filter-manager/server-filters-${serverId}.json`); + return sendOk(res, meta); } catch (error) { console.error(error); return sendError(res, 500, 'Error writing server filters to S3', 'E_S3'); @@ -1589,8 +1759,10 @@ app.post('/api/simple-filters', async (req, res) => { }; try { - await s3.putObject(params).promise(); - res.send('Simple filters updated successfully'); + await s3.send(new PutObjectCommand(params)); + invalidateCacheForKey('filter-manager/simple-filters.json'); + const meta = await headMeta('filter-manager/simple-filters.json'); + return sendOk(res, meta); } catch (error) { console.error(error); return sendError(res, 500, 'Error writing simple filters to S3', 'E_S3'); @@ -1607,8 +1779,15 @@ app.get('/api/auto-urls', async (req, res) => { }; try { - const data = await s3.getObject(params).promise(); - const fileContent = data.Body.toString('utf-8'); + const head = await s3.send(new HeadObjectCommand(params)).catch(() => null); + const etag = head?.ETag || null; + if (etag) res.set('ETag', String(etag)); + if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString()); + if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength)); + if (checkIfNoneMatch(req, res, etag)) return; + + const data = await s3.send(new GetObjectCommand(params)); + const fileContent = await streamToString(data.Body); const urls = fileContent.split('\n').filter(line => line).map(line => { const parts = line.trim().split(/\s+/); const url = parts[0] || ''; @@ -1617,7 +1796,7 @@ app.get('/api/auto-urls', async (req, res) => { }); res.json(urls); } catch (error) { - if (error.code === 'NoSuchKey') { + if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) { res.json([]); // Return empty array if file does not exist } else { console.error(error); @@ -1744,10 +1923,10 @@ app.get('/api/servers/availability', async (req, res) => { return res.json({ ...availabilityCache.data, cached: true }); } - const data = await s3.getObject({ Bucket: BUCKET_NAME, Key: 'servers.json' }).promise(); + const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'servers.json' })); let servers = []; try { - servers = JSON.parse(data.Body.toString('utf-8')); + servers = JSON.parse(await streamToString(data.Body)); if (!Array.isArray(servers)) servers = []; } catch { servers = []; @@ -1779,8 +1958,10 @@ app.post('/api/auto-urls', async (req, res) => { }; try { - await s3.putObject(params).promise(); - res.send('Auto URLs updated successfully'); + await s3.send(new PutObjectCommand(params)); + invalidateCacheForKey('bgp_data/auto_url/urls.txt'); + const meta = await headMeta('bgp_data/auto_url/urls.txt'); + return sendOk(res, meta); } catch (error) { console.error(error); return sendError(res, 500, 'Error writing auto URLs to S3', 'E_S3');