Backend: SQLite storage, EvoBGP integration, filters in SQL

Made-with: Cursor

backend/services/s3Service.js

@@ -1,58 +1,35 @@
 /**
- * Сервис для работы с S3 (Yandex Object Storage)
- * Централизованные операции чтения/записи/кэширования
+ * Локальное хранилище объектов (SQLite). Сохранены имена экспортов readS3TextObject / writeS3JsonObject и т.д. для совместимости с роутами.
+ * Справочники EvoBGP не хранятся здесь — см. evobgpClient.js
  */
 
-const { S3Client, GetObjectCommand, HeadObjectCommand, PutObjectCommand, DeleteObjectCommand, ListObjectsV2Command } = require('@aws-sdk/client-s3');
-const { NodeHttpHandler } = require('@smithy/node-http-handler');
-const http = require('http');
-const https = require('https');
+const { openDatabase } = require('../db/sqliteDb');
+const blobStorage = require('./blobStorage');
 
-// S3 Client setup
-const s3 = new S3Client({
-    endpoint: 'https://storage.yandexcloud.net',
-    region: process.env.AWS_REGION,
-    forcePathStyle: true,
-    maxAttempts: 3,
-    requestHandler: new NodeHttpHandler({
-      httpAgent: new http.Agent({ keepAlive: true }),
-      httpsAgent: new https.Agent({ keepAlive: true })
-    }),
-    credentials: {
-        accessKeyId: process.env.AWS_ACCESS_KEY_ID || process.env.S3_ACCESS_KEY_ID,
-        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || process.env.S3_SECRET_ACCESS_KEY,
-    }
-});
+openDatabase();
 
-const BUCKET_NAME = process.env.S3_BUCKET_NAME;
+const BUCKET_NAME = process.env.SQLITE_BUCKET_LABEL || 'sqlite';
 
-// In-memory cache
 const cache = {
   text: new Map(),
   head: new Map(),
   max: 100,
-  ttlMs: 30_000
+  ttlMs: 30_000,
 };
 
-/**
- * Конвертирует stream в строку
- */
 async function streamToString(stream) {
-    if (!stream) return '';
-    if (typeof stream.transformToString === 'function') {
-        return await stream.transformToString();
-    }
-    return await new Promise((resolve, reject) => {
-        let chunks = [];
-        stream.on('data', (c) => chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(String(c))));
-        stream.once('error', reject);
-        stream.once('end', () => resolve(Buffer.concat(chunks).toString('utf-8')));
-    });
+  if (!stream) return '';
+  if (typeof stream.transformToString === 'function') {
+    return await stream.transformToString();
+  }
+  return await new Promise((resolve, reject) => {
+    const chunks = [];
+    stream.on('data', (c) => chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(String(c))));
+    stream.once('error', reject);
+    stream.once('end', () => resolve(Buffer.concat(chunks).toString('utf-8')));
+  });
 }
 
-/**
- * Получить значение из кэша
- */
 function getCache(map, key) {
   const v = map.get(key);
   if (!v) return null;
@@ -63,9 +40,6 @@ function getCache(map, key) {
   return v.value;
 }
 
-/**
- * Установить значение в кэш
- */
 function setCache(map, key, value) {
   if (map.size >= cache.max) {
     const firstKey = map.keys().next().value;
@@ -74,213 +48,115 @@ function setCache(map, key, value) {
   map.set(key, { value, at: Date.now() });
 }
 
-/**
- * Инвалидировать кэш для ключа
- */
 function invalidateCacheForKey(key) {
-  try { cache.text.delete(key); } catch {}
-  try { cache.head.delete(key); } catch {}
-}
-
-/**
- * Прочитать текстовый объект из S3
- */
-async function readS3TextObject(key, s3Duration = null) {
-    const cached = getCache(cache.text, key);
-    if (cached) return cached;
-    
-    const s3Start = Date.now();
-    const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: key }));
-    if (s3Duration) {
-      try { s3Duration.labels('getObject').observe((Date.now() - s3Start)/1000); } catch {}
-    }
-    
-    const out = {
-        body: await streamToString(data.Body),
-        etag: data.ETag || undefined,
-        lastModified: data.LastModified ? data.LastModified.toISOString() : undefined,
-        contentLength: typeof data.ContentLength === 'number' ? data.ContentLength : undefined
-    };
-    setCache(cache.text, key, out);
-    return out;
-}
-
-/**
- * Получить ETag объекта из S3
- */
-async function headS3ObjectEtag(key, s3Duration = null) {
-    const cached = getCache(cache.head, key);
-    if (cached && cached.etag) return cached.etag;
-    
-    const s3Start = Date.now();
-    const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: key }));
-    if (s3Duration) {
-      try { s3Duration.labels('headObject').observe((Date.now() - s3Start)/1000); } catch {}
-    }
-    
-    setCache(cache.head, key, { etag: head.ETag || undefined });
-    return head.ETag || undefined;
-}
-
-/**
- * Получить метаданные объекта (etag, lastModified, contentLength)
- */
-async function headMeta(key) {
   try {
-    const h = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: key }));
-    return {
-      etag: h.ETag || null,
-      lastModified: h.LastModified ? new Date(h.LastModified).toISOString() : null,
-      contentLength: typeof h.ContentLength === 'number' ? h.ContentLength : null,
-    };
-  } catch (e) {
-    return { etag: null, lastModified: null, contentLength: null };
+    cache.text.delete(key);
+  } catch (_) {}
+  try {
+    cache.head.delete(key);
+  } catch (_) {}
+}
+
+async function readS3TextObject(key) {
+  const cached = getCache(cache.text, key);
+  if (cached) return cached;
+  const row = blobStorage.readBlob(key);
+  if (!row) {
+    const err = new Error('NoSuchKey');
+    err.code = 'NoSuchKey';
+    throw err;
   }
-}
-
-/**
- * Записать текстовый объект в S3
- */
-async function writeS3TextObject(key, content, contentType = 'text/plain') {
-  await s3.send(new PutObjectCommand({
-    Bucket: BUCKET_NAME,
-    Key: key,
-    Body: content,
-    ContentType: contentType,
-  }));
-  invalidateCacheForKey(key);
-  return await headMeta(key);
-}
-
-/**
- * Записать JSON объект в S3
- */
-async function writeS3JsonObject(key, data) {
-  return await writeS3TextObject(key, JSON.stringify(data, null, 2), 'application/json');
-}
-
-/**
- * Удалить объект из S3
- */
-async function deleteS3Object(key) {
-  await s3.send(new DeleteObjectCommand({ Bucket: BUCKET_NAME, Key: key }));
-  invalidateCacheForKey(key);
-}
-
-/**
- * Список объектов в S3 по префиксу
- * Используется для истории/бэкапов (количество ограничено для безопасности)
- */
-async function listS3Objects(prefix, { maxKeys = 100 } = {}) {
-  const out = [];
-  let continuationToken = undefined;
-
-  while (out.length < maxKeys) {
-    const resp = await s3.send(new ListObjectsV2Command({
-      Bucket: BUCKET_NAME,
-      Prefix: prefix,
-      ContinuationToken: continuationToken,
-      MaxKeys: Math.min(1000, maxKeys - out.length),
-    }));
-
-    const contents = resp.Contents || [];
-    for (const obj of contents) {
-      out.push({
-        key: obj.Key,
-        size: typeof obj.Size === 'number' ? obj.Size : null,
-        lastModified: obj.LastModified ? new Date(obj.LastModified).toISOString() : null,
-        etag: obj.ETag || null,
-      });
-      if (out.length >= maxKeys) break;
-    }
-
-    if (!resp.IsTruncated || !resp.NextContinuationToken || out.length >= maxKeys) {
-      break;
-    }
-    continuationToken = resp.NextContinuationToken;
-  }
-
+  const out = {
+    body: row.body,
+    etag: row.etag,
+    lastModified: row.lastModified,
+    contentLength: row.contentLength,
+  };
+  setCache(cache.text, key, out);
   return out;
 }
 
-/**
- * Потоковое чтение с пагинацией больших текстовых файлов
- */
-async function streamPaginatedText({ key, mapLine, q, offset = 0, limit = 0 }) {
-  return new Promise(async (resolve, reject) => {
-    let total = 0;
-    const items = [];
-    let sent = 0;
-    let buffered = '';
-    
-    const matchesQuery = (line) => {
-      if (!q) return true;
-      return line.toLowerCase().includes(String(q).toLowerCase());
-    };
-    
-    try {
-      const resp = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: key }));
-      const stream = resp.Body;
-      
-      if (!stream || typeof stream.on !== 'function') {
-        const text = await streamToString(resp.Body);
-        const lines = text.split('\n');
-        for (let i = 0; i < lines.length; i++) {
-          const line = String(lines[i] || '').trim();
-          if (!line) continue;
-          if (!matchesQuery(line)) continue;
-          total++;
-          const pos = total - 1;
-          if (limit > 0) {
-            if (pos >= offset && sent < limit) { items.push(mapLine(line)); sent++; }
-          } else {
-            items.push(mapLine(line));
-          }
-        }
-        return resolve({ items, total });
-      }
-      
-      stream.on('data', (chunk) => {
-        buffered += chunk.toString('utf-8');
-        let lines = buffered.split('\n');
-        buffered = lines.pop();
-        for (const lnRaw of lines) {
-          const line = lnRaw.trim();
-          if (!line) continue;
-          if (!matchesQuery(line)) continue;
-          total++;
-          const pos = total - 1;
-          if (limit > 0) {
-            if (pos >= offset && sent < limit) { items.push(mapLine(line)); sent++; }
-          } else {
-            items.push(mapLine(line));
-          }
-        }
-      });
-      
-      stream.on('end', () => {
-        const last = (buffered || '').trim();
-        if (last) {
-          if (!q || last.toLowerCase().includes(String(q).toLowerCase())) {
-            total++;
-            if (limit > 0) {
-              const pos = total - 1;
-              if (pos >= offset && items.length < limit) items.push(mapLine(last));
-            } else {
-              items.push(mapLine(last));
-            }
-          }
-        }
-        resolve({ items, total });
-      });
-      
-      stream.on('error', reject);
-    } catch (e) {
-      reject(e);
-    }
-  });
+async function headS3ObjectEtag(key) {
+  const cached = getCache(cache.head, key);
+  if (cached && cached.etag) return cached.etag;
+  const etag = blobStorage.headEtag(key);
+  if (etag) setCache(cache.head, key, { etag });
+  return etag;
 }
 
+async function headMeta(key) {
+  return blobStorage.headMetaRow(key);
+}
+
+async function writeS3TextObject(key, content, contentType = 'text/plain') {
+  const db = openDatabase();
+  blobStorage.writeBlobTx(db, key, content, contentType);
+  invalidateCacheForKey(key);
+  return headMeta(key);
+}
+
+async function writeS3JsonObject(key, data) {
+  return writeS3TextObject(key, JSON.stringify(data, null, 2), 'application/json');
+}
+
+async function deleteS3Object(key) {
+  blobStorage.deleteBlob(key);
+  invalidateCacheForKey(key);
+}
+
+async function listS3Objects(prefix, { maxKeys = 100 } = {}) {
+  return blobStorage.listByPrefix(prefix, maxKeys);
+}
+
+async function streamPaginatedText({ key, mapLine, q, offset = 0, limit = 0 }) {
+  let text = '';
+  try {
+    const row = await readS3TextObject(key);
+    text = row.body || '';
+  } catch (e) {
+    if (e?.code === 'NoSuchKey') {
+      return { items: [], total: 0 };
+    }
+    throw e;
+  }
+
+  const lines = text.split('\n');
+  let total = 0;
+  const items = [];
+  const qstr = q ? String(q).toLowerCase() : '';
+  const matchesQuery = (line) => !qstr || line.toLowerCase().includes(qstr);
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = String(lines[i] || '').trim();
+    if (!line) continue;
+    if (!matchesQuery(line)) continue;
+    total++;
+    const pos = total - 1;
+    if (limit > 0) {
+      if (pos >= offset && items.length < limit) items.push(mapLine(line));
+    } else {
+      items.push(mapLine(line));
+    }
+  }
+  return { items, total };
+}
+
+function listBlobVersionsForKey(key, maxKeys = 50) {
+  return blobStorage.listVersions(key, maxKeys);
+}
+
+function rollbackBlobVersion(key, versionId) {
+  openDatabase();
+  return blobStorage.rollbackToVersionId(key, versionId);
+}
+
+/** Заглушка: прямой вызов S3 SDK больше не используется. */
+const s3 = {
+  send() {
+    throw new Error('S3 SDK removed: use readS3TextObject / writeS3TextObject or evobgpClient');
+  },
+};
+
 module.exports = {
   s3,
   BUCKET_NAME,
@@ -294,5 +170,6 @@ module.exports = {
   streamPaginatedText,
   invalidateCacheForKey,
   listS3Objects,
+  listBlobVersionsForKey,
+  rollbackBlobVersion,
 };
-