Backend: SQLite storage, EvoBGP integration, filters in SQL

Made-with: Cursor

backend/routes/miscRoutes.js

@@ -2,10 +2,18 @@
  * Разные утилитарные роуты (auto-urls, history, s3 meta, availability, etc.)
  */
 
-const { s3, BUCKET_NAME, writeS3TextObject, writeS3JsonObject, readS3TextObject, headMeta, streamToString } = require('../services/s3Service');
+const {
+  writeS3TextObject,
+  writeS3JsonObject,
+  readS3TextObject,
+  headMeta,
+  listBlobVersionsForKey,
+  rollbackBlobVersion,
+} = require('../services/s3Service');
+const evobgpClient = require('../services/evobgpClient');
+const filterRulesStorage = require('../services/filterRulesStorage');
 const { sendError, sendOk } = require('../middleware/errorHandler');
 const { resourceToKey, toIso } = require('../utils/helpers');
-const { GetObjectCommand, PutObjectCommand, ListObjectVersionsCommand, CopyObjectCommand } = require('@aws-sdk/client-s3');
 const validators = require('../lib/validators');
 const net = require('net');
 const http = require('http');
@@ -19,66 +27,97 @@ async function getS3LastModified(req, res) {
       { name: 'domainsNew', key: 'bgp_data/domains_community.txt' },
       { name: 'asns', key: 'bgp_data/asns.txt' },
       { name: 'servers', key: 'servers.json' },
-      { name: 'filters', key: 'filters.json' },
+      { name: 'filters', key: null },
       { name: 'ipRanges', key: 'bgp_data/ips.txt' },
       { name: 'uiSettings', key: 'bgp_data/rt_ui_settings.json' }
     ];
     
-    const results = await Promise.allSettled(
-      keys.map(k => headMeta(k.key))
-    );
-    
     const out = {};
-    results.forEach((r, idx) => {
-      const name = keys[idx].name;
-      if (r.status === 'fulfilled' && r.value) {
-        out[name] = r.value;
-      } else {
-        out[name] = null;
+    for (const k of keys) {
+      if (k.name === 'filters') {
+        try {
+          const head = filterRulesStorage.headMetaForScope('global');
+          out.filters = {
+            etag: head.etag,
+            lastModified: head.lastModified,
+            contentLength: head.contentLength,
+          };
+        } catch {
+          out.filters = null;
+        }
+        continue;
       }
-    });
-    
+      if (['domainsNew', 'asns', 'ipRanges'].includes(k.name) && evobgpClient.isConfigured()) {
+        try {
+          await evobgpClient.fetchRouterListsCatalog();
+          out[k.name] = { etag: null, lastModified: new Date().toISOString(), contentLength: null };
+        } catch {
+          out[k.name] = null;
+        }
+        continue;
+      }
+      out[k.name] = await headMeta(k.key).catch(() => null);
+    }
+
     res.json(out);
   } catch (error) {
-    console.error('Error fetching last modified dates from S3:', error);
-    return sendError(res, 500, 'Error fetching last modified dates from S3', 'E_S3');
+    console.error('Error fetching last modified dates:', error);
+    return sendError(res, 500, 'Error fetching last modified dates', 'E_STORAGE');
   }
 }
 
+const EVOBGP_ONLY_HISTORY_RESOURCES = new Set(['domains-new', 'asns', 'ip-ranges', 'filters']);
+
 // GET /api/history/:resource
 async function getHistory(req, res) {
   const { resource } = req.params;
   const { countOnly, format } = req.query || {};
+  if (EVOBGP_ONLY_HISTORY_RESOURCES.has(resource)) {
+    const note =
+      resource === 'filters'
+        ? 'Фильтры в SQL-таблице app_filter_rules; локальной истории отката через этот API нет'
+        : 'Версии хранятся в EvoBGP, локальной истории нет';
+    if (countOnly === 'true') {
+      return res.json(format === 'std' ? { items: [], total: 0, meta: { noBlobHistory: true } } : { total: 0 });
+    }
+    return res.json(
+      format === 'std'
+        ? { items: [], total: 0, meta: { noBlobHistory: true, note } }
+        : { items: [], total: 0 },
+    );
+  }
+
   const key = resourceToKey(resource);
-  
+
   if (!key) return sendError(res, 400, 'Unknown resource', 'E_RESOURCE');
   
   try {
-    const out = await s3.send(new ListObjectVersionsCommand({ 
-      Bucket: BUCKET_NAME, 
-      Prefix: key, 
-      MaxKeys: 50 
+    const rows = listBlobVersionsForKey(key, 50);
+    const versionsAll = rows.map((r) => ({
+      id: r.id,
+      object_key: r.object_key,
+      byte_size: r.byte_size,
+      etag: r.etag,
+      created_at: r.created_at,
     }));
-    
-    const versionsAll = (out.Versions || []).filter(v => v.Key === key);
-    
+
     if (countOnly === 'true') {
       const total = versionsAll.length;
       return res.json(format === 'std' ? { items: [], total, meta: {} } : { total });
     }
-    
-    const versions = versionsAll.slice(0, 10).map(v => ({
-      versionId: v.VersionId,
-      isLatest: v.IsLatest,
-      lastModified: toIso(v.LastModified),
-      size: v.Size,
-      etag: v.ETag
+
+    const versions = versionsAll.slice(0, 10).map((v, idx) => ({
+      versionId: String(v.id),
+      isLatest: idx === 0,
+      lastModified: toIso(v.created_at),
+      size: v.byte_size,
+      etag: v.etag,
     }));
-    
+
     return res.json(format === 'std' ? { items: versions, total: versionsAll.length, meta: {} } : { items: versions });
   } catch (e) {
     console.error('history error', e);
-    return sendError(res, 500, 'Error reading history', 'E_S3', { error: String(e?.message || e) });
+    return sendError(res, 500, 'Error reading history', 'E_STORAGE', { error: String(e?.message || e) });
   }
 }
 
@@ -86,125 +125,76 @@ async function getHistory(req, res) {
 async function postRollback(req, res) {
   const { resource } = req.params;
   const { versionId } = req.body || {};
+  if (EVOBGP_ONLY_HISTORY_RESOURCES.has(resource)) {
+    const msg =
+      resource === 'filters'
+        ? 'Откат версий для фильтров не поддерживается (данные в SQL, не в blob_versions)'
+        : 'Откат версий для этого справочника не поддерживается (данные в EvoBGP)';
+    return sendError(res, 400, msg, resource === 'filters' ? 'E_HISTORY_FILTERS_SQL' : 'E_HISTORY_EV');
+  }
+
   const key = resourceToKey(resource);
-  
+
   if (!key || !versionId) {
     return sendError(res, 400, 'Bad request', 'E_BAD_REQUEST');
   }
   
   try {
-    await s3.send(new CopyObjectCommand({
-      Bucket: BUCKET_NAME,
-      CopySource: `/${BUCKET_NAME}/${encodeURIComponent(key)}?versionId=${encodeURIComponent(versionId)}`,
-      Key: key
-    }));
-    
+    rollbackBlobVersion(key, versionId);
     const meta = await headMeta(key);
     return sendOk(res, meta);
   } catch (e) {
     console.error('rollback error', e);
-    return sendError(res, 500, 'Error rollback', 'E_S3', { error: String(e?.message || e) });
+    return sendError(res, 500, 'Error rollback', 'E_STORAGE', { error: String(e?.message || e) });
   }
 }
 
 // GET /api/auto-urls
 async function getAutoUrls(req, res) {
-  const { checkIfNoneMatch } = require('../middleware/errorHandler');
-  
   try {
-    const { HeadObjectCommand } = require('@aws-sdk/client-s3');
-    const head = await s3.send(new HeadObjectCommand({ 
-      Bucket: BUCKET_NAME, 
-      Key: 'bgp_data/auto_url/urls.txt' 
-    })).catch(() => null);
-    
-    const etag = head?.ETag || null;
-    if (etag) res.set('ETag', String(etag));
-    if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
-    if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
-    if (checkIfNoneMatch(req, res, etag)) return;
-
-    const data = await s3.send(new GetObjectCommand({ 
-      Bucket: BUCKET_NAME, 
-      Key: 'bgp_data/auto_url/urls.txt' 
-    }));
-    const fileContent = await streamToString(data.Body);
-    const urls = fileContent.split('\n').filter(line => line).map(line => {
-      const parts = line.trim().split(/\s+/);
-      return { url: parts[0] || '', community: parts[1] || '' };
-    });
-    res.json(urls);
-  } catch (error) {
-    if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) {
-      res.json([]);
-    } else {
-      console.error(error);
-      return sendError(res, 500, 'Error reading auto URLs from S3', 'E_S3');
+    if (!evobgpClient.isConfigured()) {
+      return sendError(res, 503, 'EvoBGP не настроен', 'E_EVOBGP_NOT_CONFIGURED');
     }
+    const rows = await evobgpClient.fetchAutoUrlsData();
+    res.json(rows);
+  } catch (error) {
+    console.error(error);
+    return sendError(res, 500, 'Error reading auto URLs', 'E_EVOBGP');
   }
 }
 
 // POST /api/auto-urls
 async function postAutoUrls(req, res) {
   const { urls } = req.body;
-  const fileContent = urls.map(u => `${u.url} ${u.community}`).join('\n');
-
+  if (!Array.isArray(urls)) {
+    return sendError(res, 400, 'urls must be an array', 'E_BAD_REQUEST');
+  }
   try {
-    const meta = await writeS3TextObject('bgp_data/auto_url/urls.txt', fileContent);
-    return sendOk(res, meta);
+    if (!evobgpClient.isConfigured()) {
+      return sendError(res, 503, 'EvoBGP не настроен', 'E_EVOBGP_NOT_CONFIGURED');
+    }
+    await evobgpClient.saveAutoUrlsData(urls);
+    return sendOk(res, { etag: null, lastModified: new Date().toISOString(), contentLength: null });
   } catch (error) {
     console.error(error);
-    return sendError(res, 500, 'Error writing auto URLs to S3', 'E_S3');
+    return sendError(res, 500, 'Error writing auto URLs', 'E_EVOBGP');
   }
 }
 
 // POST /api/auto-urls/process
 async function processAutoUrls(req, res) {
   try {
-    // Загрузка URL списков
-    let urls = [];
-    try {
-      const urlsData = await s3.send(new GetObjectCommand({ 
-        Bucket: BUCKET_NAME, 
-        Key: 'bgp_data/auto_url/urls.txt' 
-      }));
-      const urlsContent = await streamToString(urlsData.Body);
-      urls = urlsContent.split('\n').filter(line => line).map(line => {
-        const parts = line.trim().split(/\s+/);
-        return { url: parts[0] || '', community: parts[1] || '' };
-      });
-    } catch (error) {
-      if (error.code !== 'NoSuchKey') throw error;
+    if (!evobgpClient.isConfigured()) {
+      return sendError(res, 503, 'EvoBGP не настроен', 'E_EVOBGP_NOT_CONFIGURED');
     }
 
+    const urls = await evobgpClient.fetchAutoUrlsData();
     if (urls.length === 0) {
       return sendError(res, 400, 'No URLs to process', 'E_BAD_REQUEST');
     }
 
-    // Загрузка текущих IP и доменов
-    let currentIps = [];
-    try {
-      const ipsData = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt' }));
-      const ipsContent = await streamToString(ipsData.Body);
-      currentIps = ipsContent.split('\n').filter(line => line).map(line => {
-        const parts = line.trim().split(/\s+/);
-        return { ipRange: parts[0] || '', community: parts[1] || '' };
-      });
-    } catch (error) {
-      if (error.code !== 'NoSuchKey') throw error;
-    }
-
-    let currentDomains = [];
-    try {
-      const dData = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt' }));
-      const dContent = await streamToString(dData.Body);
-      currentDomains = dContent.split('\n').filter(line => line).map(line => {
-        const parts = line.trim().split(/\s+/);
-        return { domain: (parts[0] || '').toLowerCase(), community: parts[1] || '' };
-      });
-    } catch (error) {
-      if (error.code !== 'NoSuchKey') throw error;
-    }
+    const currentIps = await evobgpClient.fetchIpRangesData();
+    const currentDomains = await evobgpClient.fetchDomainsData();
 
     // Обработка URL
     const newIps = [];
@@ -256,22 +246,8 @@ async function processAutoUrls(req, res) {
     const uniqueNewDomains = newDomains.filter(d => !existingDomains.has(d.domain));
     const allDomains = [...currentDomains, ...uniqueNewDomains];
 
-    // Сохранение
-    const updatedIpsContent = allIps.map(i => `${i.ipRange} ${i.community}`).join('\n');
-    await s3.send(new PutObjectCommand({ 
-      Bucket: BUCKET_NAME, 
-      Key: 'bgp_data/ips.txt', 
-      Body: updatedIpsContent, 
-      ContentType: 'text/plain' 
-    }));
-
-    const updatedDomainsContent = allDomains.map(d => `${d.domain} ${d.community}`).join('\n');
-    await s3.send(new PutObjectCommand({ 
-      Bucket: BUCKET_NAME, 
-      Key: 'bgp_data/domains_community.txt', 
-      Body: updatedDomainsContent, 
-      ContentType: 'text/plain' 
-    }));
+    await evobgpClient.saveIpRangesData(allIps, currentIps);
+    await evobgpClient.saveDomainsData(allDomains, currentDomains);
 
     const msg = `Обработано ${urls.length} URL, добавлено ${uniqueNewIps.length} IP и ${uniqueNewDomains.length} доменов`;
     return res.json({
@@ -285,7 +261,7 @@ async function processAutoUrls(req, res) {
     });
   } catch (error) {
     console.error('Error processing auto URLs:', error);
-    return sendError(res, 500, 'Error processing auto URLs', 'E_S3');
+    return sendError(res, 500, 'Error processing auto URLs', 'E_EVOBGP');
   }
 }
 
@@ -357,9 +333,8 @@ function getPingServicesListFromSettings(uiSettings) {
 /** Загрузить UI-настройки из S3 (для pingServicesSource и др.) */
 async function loadUiSettingsSync() {
   try {
-    const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/rt_ui_settings.json' }));
-    const jsonText = await streamToString(data.Body);
-    const parsed = JSON.parse(jsonText || '{}');
+    const data = await readS3TextObject('bgp_data/rt_ui_settings.json');
+    const parsed = JSON.parse(data.body || '{}');
     return parsed && typeof parsed === 'object' ? parsed : {};
   } catch {
     return {};
@@ -598,10 +573,10 @@ async function getServersAvailability(req, res) {
       return res.json({ ...availabilityCache.data, cached: true });
     }
 
-    const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'servers.json' }));
     let servers = [];
     try {
-      servers = JSON.parse(await streamToString(data.Body));
+      const data = await readS3TextObject('servers.json');
+      servers = JSON.parse(data.body);
       if (!Array.isArray(servers)) servers = [];
     } catch {
       servers = [];
@@ -683,12 +658,13 @@ async function updateBgpBackground(req, res) {
 // GET /api/ws/url
 async function getWsUrl(req, res) {
   try {
-    const settings = await s3.send(new GetObjectCommand({ 
-      Bucket: BUCKET_NAME, 
-      Key: 'bgp_data/rt_ui_settings.json' 
-    })).then(async (d) => {
-      try { return JSON.parse(await streamToString(d.Body)); } catch { return {}; }
-    }).catch(() => ({}));
+    let settings = {};
+    try {
+      const d = await readS3TextObject('bgp_data/rt_ui_settings.json');
+      settings = JSON.parse(d.body || '{}');
+    } catch (_) {
+      settings = {};
+    }
     
     const url = settings?.wsUpdateUrl || process.env.WS_UPDATE_URL || '';
     return res.json({ url });
@@ -700,25 +676,22 @@ async function getWsUrl(req, res) {
 // GET/POST /api/ui-settings
 async function getUiSettings(req, res) {
   const { checkIfNoneMatch } = require('../middleware/errorHandler');
-  
+
   try {
-    const { HeadObjectCommand } = require('@aws-sdk/client-s3');
-    const head = await s3.send(new HeadObjectCommand({ 
-      Bucket: BUCKET_NAME, 
-      Key: 'bgp_data/rt_ui_settings.json' 
-    })).catch(() => null);
-    
-    const etag = head?.ETag || null;
+    const head = await headMeta('bgp_data/rt_ui_settings.json');
+    const etag = head?.etag || null;
     if (etag) res.set('ETag', String(etag));
-    if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
-    if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
+    if (head?.lastModified) res.set('Last-Modified', new Date(head.lastModified).toUTCString());
+    if (typeof head?.contentLength === 'number') res.set('Content-Length-Source', String(head.contentLength));
     if (checkIfNoneMatch(req, res, etag)) return;
 
-    const data = await s3.send(new GetObjectCommand({ 
-      Bucket: BUCKET_NAME, 
-      Key: 'bgp_data/rt_ui_settings.json' 
-    }));
-    const jsonText = await streamToString(data.Body);
+    let jsonText = '{}';
+    try {
+      const data = await readS3TextObject('bgp_data/rt_ui_settings.json');
+      jsonText = data.body;
+    } catch (e) {
+      if (e?.code !== 'NoSuchKey') throw e;
+    }
     let settings = {};
     try {
       const parsed = JSON.parse(jsonText);
@@ -728,11 +701,11 @@ async function getUiSettings(req, res) {
     }
     return res.json(settings);
   } catch (error) {
-    if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) {
+    if (error?.code === 'NoSuchKey') {
       return res.json({});
     }
-    console.error('Error reading ui settings from S3:', error);
-    return sendError(res, 500, 'Error reading UI settings from S3', 'E_S3');
+    console.error('Error reading ui settings:', error);
+    return sendError(res, 500, 'Error reading UI settings', 'E_STORAGE');
   }
 }
 
@@ -756,8 +729,8 @@ async function postUiSettings(req, res) {
     const meta = await writeS3JsonObject('bgp_data/rt_ui_settings.json', payload);
     return sendOk(res, meta);
   } catch (error) {
-    console.error('Error writing UI settings to S3:', error);
-    return sendError(res, 500, 'Error writing UI settings to S3', 'E_S3', { error: String(error?.message || error) });
+    console.error('Error writing UI settings:', error);
+    return sendError(res, 500, 'Error writing UI settings', 'E_STORAGE', { error: String(error?.message || error) });
   }
 }