feat: Integrate logging and metrics with pino and prom-client; enhance S3 data handling with caching and improved response formats for ASNs, Domains, and IP Ranges APIs

backend/package.json

@@ -23,7 +23,11 @@
     "compression": "^1.7.4",
     "ajv": "^8.17.1",
     "helmet": "^7.1.0",
-    "express-rate-limit": "^6.11.2"
+    "express-rate-limit": "^6.11.2",
+    "pino": "^9.4.0",
+    "pino-http": "^10.3.0",
+    "prom-client": "^15.1.3",
+    "@aws-sdk/client-s3": "^3.687.0"
   },
   "devDependencies": {
     "nodemon": "^3.1.10"

backend/server.js

@@ -10,10 +10,24 @@ const net = require('net');
 const crypto = require('crypto');
 const helmet = require('helmet');
 const rateLimit = require('express-rate-limit');
+const pino = require('pino');
+const pinoHttp = require('pino-http');
+const promClient = require('prom-client');
 
 const app = express();
 const port = Number(process.env.PORT) || 3001;
 
+// Logger with requestId
+const logger = pino({ level: process.env.LOG_LEVEL || 'info' });
+app.use(pinoHttp({
+  logger,
+  genReqId: (req) => req.headers['x-request-id'] || crypto.randomBytes(8).toString('hex'),
+  serializers: {
+    req(req) { return { id: req.id, method: req.method, url: req.url }; },
+    res(res) { return { statusCode: res.statusCode }; },
+  },
+}));
+
 // CORS: по умолчанию максимально разрешаем, можно сузить через CORS_ORIGINS
 const allowed = (process.env.CORS_ORIGINS || '').split(',').map(s => s.trim()).filter(Boolean);
 if (allowed.length > 0) {
@@ -56,6 +70,33 @@ app.use((req, res, next) => {
   next();
 });
 
+// --- Metrics ---
+promClient.collectDefaultMetrics();
+const httpDuration = new promClient.Histogram({ name: 'http_request_duration_seconds', help: 'HTTP request duration', labelNames: ['method', 'route', 'code'], buckets: [0.05,0.1,0.2,0.5,1,2,5] });
+const httpErrors = new promClient.Counter({ name: 'http_errors_total', help: 'HTTP error count', labelNames: ['route','code'] });
+app.use((req, res, next) => {
+  const start = process.hrtime.bigint();
+  res.on('finish', () => {
+    try {
+      const dur = Number(process.hrtime.bigint() - start) / 1e9;
+      httpDuration.labels(req.method, req.route?.path || req.path, String(res.statusCode)).observe(dur);
+      if (res.statusCode >= 400) httpErrors.labels(req.route?.path || req.path, String(res.statusCode)).inc();
+    } catch {}
+  });
+  next();
+});
+
+app.get('/health', (req, res) => res.json({ ok: true }));
+app.get('/ready', (req, res) => res.json({ ok: true }));
+app.get('/metrics', async (req, res) => {
+  try {
+    res.set('Content-Type', promClient.register.contentType);
+    res.end(await promClient.register.metrics());
+  } catch (e) {
+    res.status(500).end(String(e?.message || e));
+  }
+});
+
 // Централизованный обработчик ошибок (должен быть подключён ПОСЛЕ роутов — см. ниже второе use)
 
 // Helpers: meta and responses
@@ -250,21 +291,41 @@ function buildNestedGatewayBlocks(gatewayGroups, baseIndentSpaces = 4) {
     return buildAt(0, baseIndentSpaces);
 }
 
+// In-memory LRU-ish cache for small texts and head meta
+const s3Cache = { text: new Map(), head: new Map(), max: 100, ttlMs: 30_000 };
+function getCache(map, key) {
+  const v = map.get(key);
+  if (!v) return null;
+  if (Date.now() > v.at + s3Cache.ttlMs) { map.delete(key); return null; }
+  return v.value;
+}
+function setCache(map, key, value) {
+  if (map.size >= s3Cache.max) { const firstKey = map.keys().next().value; if (firstKey) map.delete(firstKey); }
+  map.set(key, { value, at: Date.now() });
+}
+
 // Helper: read text file from S3 and return { body, etag, lastModified, contentLength }
 async function readS3TextObject(key) {
+    const cached = getCache(s3Cache.text, key);
+    if (cached) return cached;
     const params = { Bucket: BUCKET_NAME, Key: key };
     const data = await s3.getObject(params).promise();
-    return {
+    const out = {
         body: data.Body.toString('utf-8'),
         etag: data.ETag ? String(data.ETag).replace(/\"/g, '"') : undefined,
         lastModified: data.LastModified ? data.LastModified.toISOString() : undefined,
         contentLength: typeof data.ContentLength === 'number' ? data.ContentLength : undefined
     };
+    setCache(s3Cache.text, key, out);
+    return out;
 }
 
 // Helper: head object and return current ETag
 async function headS3ObjectEtag(key) {
+    const cached = getCache(s3Cache.head, key);
+    if (cached && cached.etag) return cached.etag;
     const head = await s3.headObject({ Bucket: BUCKET_NAME, Key: key }).promise();
+    setCache(s3Cache.head, key, { etag: head.ETag ? String(head.ETag).replace(/\"/g, '"') : undefined });
     return head.ETag ? String(head.ETag).replace(/\"/g, '"') : undefined;
 }
 
@@ -407,7 +468,7 @@ app.post('/api/domains', async (req, res) => {
 
 // Get ASNs from S3
 app.get('/api/asns', async (req, res) => {
-    const { q = '', offset, limit, countOnly } = req.query || {};
+    const { q = '', offset, limit, countOnly, format } = req.query || {};
     const params = {
         Bucket: BUCKET_NAME,
         Key: 'bgp_data/asns.txt',
@@ -420,7 +481,7 @@ app.get('/api/asns', async (req, res) => {
                 key: 'bgp_data/asns.txt', q, offset: 0, limit: 0,
                 mapLine: (line) => ({})
             });
-            return res.json({ total });
+            return res.json(format === 'std' ? { items: [], total, meta: {} } : { total });
         } else if (limit !== undefined) {
             const { items, total } = await streamPaginatedText({
                 key: 'bgp_data/asns.txt',
@@ -433,7 +494,7 @@ app.get('/api/asns', async (req, res) => {
                 }
             });
             if (!validateAsns(items)) return sendError(res, 500, 'Invalid data format', 'E_SCHEMA');
-            return res.json({ items, total });
+            return res.json(format === 'std' ? { items, total, meta: {} } : { items, total });
         } else {
             const data = await s3.getObject(params).promise();
             const fileContent = data.Body.toString('utf-8');
@@ -447,6 +508,7 @@ app.get('/api/asns', async (req, res) => {
             if (data.ETag) res.set('ETag', String(data.ETag));
             if (data.LastModified) res.set('Last-Modified', new Date(data.LastModified).toUTCString());
             if (typeof data.ContentLength === 'number') res.set('Content-Length-Source', String(data.ContentLength));
+            if (format === 'std') return res.json({ items: asns, total: asns.length, meta: {} });
             res.json(asns);
         }
     } catch (error) {
@@ -469,8 +531,9 @@ app.post('/api/asns', async (req, res) => {
             return sendError(res, 400, 'Invalid payload format for asns', 'E_SCHEMA');
         }
         let current = null;
+        const ifMatch = req.headers['if-match'] ? String(req.headers['if-match']).replace(/\"/g,'"') : null;
         try { current = await headS3ObjectEtag('bgp_data/asns.txt'); } catch {}
-        if (current && etag && current !== String(etag)) {
+        if (current && (ifMatch || etag) && current !== String(ifMatch || etag)) {
             const meta = await headMeta('bgp_data/asns.txt');
             return sendError(res, 412, 'Precondition Failed: ETag mismatch', 'E_ETAG_MISMATCH', { currentEtag: current, meta });
         }
@@ -497,7 +560,7 @@ app.post('/api/asns', async (req, res) => {
 
 // Get domains-new from S3
 app.get('/api/domains-new', async (req, res) => {
-    const { q = '', offset, limit, countOnly } = req.query || {};
+    const { q = '', offset, limit, countOnly, format } = req.query || {};
     const params = {
         Bucket: BUCKET_NAME,
         Key: 'bgp_data/domains_community.txt',
@@ -509,7 +572,7 @@ app.get('/api/domains-new', async (req, res) => {
                 key: 'bgp_data/domains_community.txt', q, offset: 0, limit: 0,
                 mapLine: (line) => ({})
             });
-            return res.json({ total });
+            return res.json(format === 'std' ? { items: [], total, meta: {} } : { total });
         } else if (limit !== undefined) {
             const { items, total } = await streamPaginatedText({
                 key: 'bgp_data/domains_community.txt',
@@ -522,7 +585,7 @@ app.get('/api/domains-new', async (req, res) => {
                 }
             });
             if (!validateDomainsNew(items)) return sendError(res, 500, 'Invalid data format', 'E_SCHEMA');
-            return res.json({ items, total });
+            return res.json(format === 'std' ? { items, total, meta: {} } : { items, total });
         } else {
             const data = await s3.getObject(params).promise();
             const fileContent = data.Body.toString('utf-8');
@@ -536,6 +599,7 @@ app.get('/api/domains-new', async (req, res) => {
             if (data.ETag) res.set('ETag', String(data.ETag));
             if (data.LastModified) res.set('Last-Modified', new Date(data.LastModified).toUTCString());
             if (typeof data.ContentLength === 'number') res.set('Content-Length-Source', String(data.ContentLength));
+            if (format === 'std') return res.json({ items: domains, total: domains.length, meta: {} });
             res.json(domains);
         }
     } catch (error) {
@@ -558,8 +622,9 @@ app.post('/api/domains-new', async (req, res) => {
             return sendError(res, 400, 'Invalid payload format for domains-new', 'E_SCHEMA');
         }
         let current = null;
+        const ifMatch = req.headers['if-match'] ? String(req.headers['if-match']).replace(/\"/g,'"') : null;
         try { current = await headS3ObjectEtag('bgp_data/domains_community.txt'); } catch {}
-        if (current && etag && current !== String(etag)) {
+        if (current && (ifMatch || etag) && current !== String(ifMatch || etag)) {
             const meta = await headMeta('bgp_data/domains_community.txt');
             return sendError(res, 412, 'Precondition Failed: ETag mismatch', 'E_ETAG_MISMATCH', { currentEtag: current, meta });
         }
@@ -586,7 +651,7 @@ app.post('/api/domains-new', async (req, res) => {
 
 // Get IP ranges from S3
 app.get('/api/ip-ranges', async (req, res) => {
-    const { q = '', offset, limit, countOnly } = req.query || {};
+    const { q = '', offset, limit, countOnly, format } = req.query || {};
     const params = {
         Bucket: BUCKET_NAME,
         Key: 'bgp_data/ips.txt',
@@ -598,7 +663,7 @@ app.get('/api/ip-ranges', async (req, res) => {
                 key: 'bgp_data/ips.txt', q, offset: 0, limit: 0,
                 mapLine: (line) => ({})
             });
-            return res.json({ total });
+            return res.json(format === 'std' ? { items: [], total, meta: {} } : { total });
         } else if (limit !== undefined) {
             const { items, total } = await streamPaginatedText({
                 key: 'bgp_data/ips.txt',
@@ -611,7 +676,7 @@ app.get('/api/ip-ranges', async (req, res) => {
                 }
             });
             if (!validateIpRanges(items)) return sendError(res, 500, 'Invalid data format', 'E_SCHEMA');
-            return res.json({ items, total });
+            return res.json(format === 'std' ? { items, total, meta: {} } : { items, total });
         } else {
             const data = await s3.getObject(params).promise();
             const fileContent = data.Body.toString('utf-8');
@@ -625,6 +690,7 @@ app.get('/api/ip-ranges', async (req, res) => {
             if (data.ETag) res.set('ETag', String(data.ETag));
             if (data.LastModified) res.set('Last-Modified', new Date(data.LastModified).toUTCString());
             if (typeof data.ContentLength === 'number') res.set('Content-Length-Source', String(data.ContentLength));
+            if (format === 'std') return res.json({ items: ipRanges, total: ipRanges.length, meta: {} });
             res.json(ipRanges);
         }
     } catch (error) {
@@ -647,8 +713,9 @@ app.post('/api/ip-ranges', async (req, res) => {
             return sendError(res, 400, 'Invalid payload format for ip-ranges', 'E_SCHEMA');
         }
         let current = null;
+        const ifMatch = req.headers['if-match'] ? String(req.headers['if-match']).replace(/\"/g,'"') : null;
         try { current = await headS3ObjectEtag('bgp_data/ips.txt'); } catch {}
-        if (current && etag && current !== String(etag)) {
+        if (current && (ifMatch || etag) && current !== String(ifMatch || etag)) {
             const meta = await headMeta('bgp_data/ips.txt');
             return sendError(res, 412, 'Precondition Failed: ETag mismatch', 'E_ETAG_MISMATCH', { currentEtag: current, meta });
         }