Backend: SQLite storage, EvoBGP integration, filters in SQL

Made-with: Cursor

backend/services/backupFsService.js

@@ -0,0 +1,103 @@
+/**
+ * Бэкапы MikroTik на локальной ФС (вместо S3).
+ * Логические ключи сохраняют вид backups/mikrotik/{serverId}/{ts}-{rand}.rsc
+ */
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+
+const BACKUP_PREFIX = 'backups/mikrotik';
+
+function backupRootDir() {
+  const raw = process.env.MIKROTIK_BACKUP_DIR || path.join(__dirname, '..', 'data', 'backups', 'mikrotik');
+  if (!fs.existsSync(raw)) {
+    fs.mkdirSync(raw, { recursive: true });
+  }
+  return raw;
+}
+
+function parseBackupKey(key) {
+  const m = String(key || '').match(/^backups\/mikrotik\/([^/]+)\/(.+)$/);
+  if (!m) return null;
+  return { safeId: m[1], fileName: m[2] };
+}
+
+function fsPathForKey(key) {
+  const p = parseBackupKey(key);
+  if (!p) return null;
+  return path.join(backupRootDir(), p.safeId, p.fileName);
+}
+
+function buildBackupKey(serverId, createdAt, suffix) {
+  const safeId = String(serverId || 'unknown').replace(/[^a-zA-Z0-9._-]/g, '_');
+  const ts = createdAt.toISOString().replace(/[:-]/g, '').replace(/\.\d+Z$/, 'Z');
+  const rand = suffix || crypto.randomBytes(3).toString('hex');
+  return `${BACKUP_PREFIX}/${safeId}/${ts}-${rand}.rsc`;
+}
+
+function etagForContent(content) {
+  const h = crypto.createHash('sha256').update(Buffer.from(String(content), 'utf8')).digest('hex');
+  return `"${h}"`;
+}
+
+async function writeBackupFile(key, content, contentType = 'text/plain') {
+  const p = parseBackupKey(key);
+  if (!p) throw new Error('Invalid backup key');
+  const dir = path.join(backupRootDir(), p.safeId);
+  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+  const fp = path.join(dir, p.fileName);
+  await fs.promises.writeFile(fp, String(content), 'utf8');
+  const st = await fs.promises.stat(fp);
+  const etag = etagForContent(String(content));
+  return {
+    etag,
+    lastModified: st.mtime.toISOString(),
+    contentLength: st.size,
+  };
+}
+
+async function readBackupFile(key) {
+  const fp = fsPathForKey(key);
+  if (!fp) throw new Error('Invalid backup key');
+  const body = await fs.promises.readFile(fp, 'utf8');
+  const st = await fs.promises.stat(fp);
+  const etag = etagForContent(body);
+  return {
+    body,
+    etag,
+    lastModified: st.mtime.toISOString(),
+    contentLength: st.size,
+  };
+}
+
+async function listBackupObjects(serverId, maxKeys = 200) {
+  const safeId = String(serverId || '').replace(/[^a-zA-Z0-9._-]/g, '_');
+  const dir = path.join(backupRootDir(), safeId);
+  if (!fs.existsSync(dir)) return [];
+  const names = await fs.promises.readdir(dir);
+  const out = [];
+  for (const name of names) {
+    if (out.length >= maxKeys) break;
+    const fp = path.join(dir, name);
+    const st = await fs.promises.stat(fp);
+    if (!st.isFile()) continue;
+    const key = `${BACKUP_PREFIX}/${safeId}/${name}`;
+    out.push({
+      key,
+      size: st.size,
+      lastModified: st.mtime.toISOString(),
+      etag: etagForContent(await fs.promises.readFile(fp, 'utf8')),
+    });
+  }
+  return out;
+}
+
+module.exports = {
+  BACKUP_PREFIX,
+  buildBackupKey,
+  writeBackupFile,
+  readBackupFile,
+  listBackupObjects,
+  backupRootDir,
+};

backend/services/blobStorage.js

@@ -0,0 +1,167 @@
+/**
+ * SQLite blob store (logical S3 keys → rows in `blobs`).
+ */
+
+const crypto = require('crypto');
+const { openDatabase } = require('../db/sqliteDb');
+
+function sha256Etag(buf) {
+  const h = crypto.createHash('sha256').update(buf).digest('hex');
+  return `"${h}"`;
+}
+
+function namespaceFromKey(objectKey) {
+  const k = String(objectKey || '');
+  if (k.startsWith('ping-cache/') || k.startsWith('speed-test-cache/') || k.startsWith('network-map-cache/') ||
+      k.startsWith('uptime-monitor-cache/') || k.startsWith('ping-services/')) {
+    return 'cache';
+  }
+  if (k.startsWith('filter-manager/config-') || k.startsWith('filter-manager/server-filters-')) {
+    return 'per_server';
+  }
+  if (k === 'mikrotik-frouting-config.txt') {
+    return 'derived';
+  }
+  return 'config';
+}
+
+function isVersionedKey(db, objectKey) {
+  const row = db.prepare('SELECT 1 FROM versioned_keys WHERE object_key = ?').get(objectKey);
+  return Boolean(row);
+}
+
+function trimVersions(db, objectKey, keep = 50) {
+  db.prepare(
+    `DELETE FROM blob_versions WHERE object_key = ? AND id NOT IN (
+       SELECT id FROM blob_versions WHERE object_key = ? ORDER BY created_at DESC, id DESC LIMIT ?
+     )`
+  ).run(objectKey, objectKey, keep);
+}
+
+/**
+ * @param {Buffer|string} body
+ */
+function writeBlobTx(db, objectKey, body, contentType) {
+  const buf = Buffer.isBuffer(body) ? body : Buffer.from(String(body), 'utf8');
+  const ns = namespaceFromKey(objectKey);
+  const etag = sha256Etag(buf);
+  const now = Date.now();
+  const size = buf.length;
+
+  const existing = db.prepare('SELECT body, content_type, etag, updated_at, created_at FROM blobs WHERE object_key = ?').get(objectKey);
+
+  const run = () => {
+    if (existing && isVersionedKey(db, objectKey)) {
+      const oldBuf = existing.body;
+      db.prepare(
+        `INSERT INTO blob_versions (object_key, body, byte_size, etag, created_at)
+         VALUES (?, ?, ?, ?, ?)`
+      ).run(objectKey, oldBuf, oldBuf.length, existing.etag, existing.updated_at);
+      trimVersions(db, objectKey);
+    }
+
+    if (existing) {
+      db.prepare(
+        `UPDATE blobs SET namespace = ?, content_type = ?, body = ?, byte_size = ?, etag = ?, updated_at = ?
+         WHERE object_key = ?`
+      ).run(ns, contentType, buf, size, etag, now, objectKey);
+    } else {
+      db.prepare(
+        `INSERT INTO blobs (object_key, namespace, content_type, body, byte_size, etag, updated_at, created_at)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
+      ).run(objectKey, ns, contentType, buf, size, etag, now, now);
+    }
+  };
+
+  db.transaction(run)();
+  return {
+    etag,
+    lastModified: new Date(now).toISOString(),
+    contentLength: size,
+  };
+}
+
+function readBlob(objectKey) {
+  const db = openDatabase();
+  const row = db.prepare(
+    'SELECT body, etag, updated_at, byte_size FROM blobs WHERE object_key = ?'
+  ).get(objectKey);
+  if (!row) return null;
+  const bodyStr = row.body.toString('utf8');
+  return {
+    body: bodyStr,
+    etag: row.etag,
+    lastModified: new Date(row.updated_at).toISOString(),
+    contentLength: row.byte_size,
+  };
+}
+
+function headMetaRow(objectKey) {
+  const db = openDatabase();
+  const row = db.prepare('SELECT etag, updated_at, byte_size FROM blobs WHERE object_key = ?').get(objectKey);
+  if (!row) {
+    return { etag: null, lastModified: null, contentLength: null };
+  }
+  return {
+    etag: row.etag,
+    lastModified: new Date(row.updated_at).toISOString(),
+    contentLength: row.byte_size,
+  };
+}
+
+function headEtag(objectKey) {
+  const m = headMetaRow(objectKey);
+  return m.etag || undefined;
+}
+
+function deleteBlob(objectKey) {
+  const db = openDatabase();
+  db.prepare('DELETE FROM blobs WHERE object_key = ?').run(objectKey);
+}
+
+function listByPrefix(prefix, maxKeys = 100) {
+  const db = openDatabase();
+  const rows = db.prepare(
+    `SELECT object_key AS key, byte_size AS size, updated_at, etag FROM blobs
+     WHERE object_key >= ? AND object_key < ? ORDER BY object_key LIMIT ?`
+  ).all(prefix, prefix + '\uffff', maxKeys);
+  return rows.map((r) => ({
+    key: r.key,
+    size: r.size,
+    lastModified: new Date(r.updated_at).toISOString(),
+    etag: r.etag,
+  }));
+}
+
+function listVersions(objectKey, maxKeys = 50) {
+  const db = openDatabase();
+  return db.prepare(
+    `SELECT id, object_key, byte_size, etag, created_at FROM blob_versions
+     WHERE object_key = ? ORDER BY created_at DESC, id DESC LIMIT ?`
+  ).all(objectKey, maxKeys);
+}
+
+function rollbackToVersionId(objectKey, versionId) {
+  const db = openDatabase();
+  const vid = Number(versionId);
+  if (!Number.isFinite(vid)) throw new Error('Invalid versionId');
+  const ver = db.prepare('SELECT body, etag, created_at FROM blob_versions WHERE id = ? AND object_key = ?').get(vid, objectKey);
+  if (!ver) throw new Error('Version not found');
+  const buf = ver.body;
+  const contentType = objectKey.endsWith('.json') ? 'application/json' : 'text/plain';
+  writeBlobTx(db, objectKey, buf, contentType);
+  return headMetaRow(objectKey);
+}
+
+module.exports = {
+  namespaceFromKey,
+  writeBlobTx,
+  readBlob,
+  headMetaRow,
+  headEtag,
+  deleteBlob,
+  listByPrefix,
+  listVersions,
+  rollbackToVersionId,
+  sha256Etag,
+};

backend/services/evobgpClient.js

@@ -0,0 +1,426 @@
+/**
+ * Серверный клиент EvoBGP API (Bearer, base /v1). Логика выборки совместима с frontend/src/lib/evobgpData.js
+ */
+
+const { normalizeEvobgpBaseUrl } = require('../routes/evobgpProxyRoutes');
+
+const PAGE_LIMIT = 500;
+const DEFAULT_TIMEOUT_MS = Number(process.env.EVOBGP_CLIENT_TIMEOUT_MS) || 30000;
+
+function getBaseAndToken() {
+  const base = normalizeEvobgpBaseUrl(process.env.EVOBGP_API_URL || '');
+  const token = String(process.env.EVOBGP_API_TOKEN || '').trim();
+  return { base, token };
+}
+
+function isConfigured() {
+  const { base, token } = getBaseAndToken();
+  return Boolean(base && token);
+}
+
+async function evobgpRequest(method, pathname, { query, body } = {}) {
+  const { base, token } = getBaseAndToken();
+  if (!base || !token) {
+    const err = new Error('EvoBGP not configured');
+    err.code = 'E_EVOBGP_NOT_CONFIGURED';
+    throw err;
+  }
+  const u = new URL(pathname.startsWith('/') ? pathname.slice(1) : pathname, base.endsWith('/') ? base : `${base}/`);
+  if (query && typeof query === 'object') {
+    for (const [k, v] of Object.entries(query)) {
+      if (v !== undefined && v !== null) u.searchParams.set(k, String(v));
+    }
+  }
+  const ctrl = new AbortController();
+  const t = setTimeout(() => ctrl.abort(), DEFAULT_TIMEOUT_MS);
+  try {
+    const res = await fetch(u.toString(), {
+      method,
+      headers: {
+        Authorization: `Bearer ${token}`,
+        Accept: 'application/json',
+        ...(body !== undefined ? { 'Content-Type': 'application/json' } : {}),
+      },
+      body: body !== undefined ? JSON.stringify(body) : undefined,
+      signal: ctrl.signal,
+    });
+    const text = await res.text();
+    let data = null;
+    try {
+      data = text ? JSON.parse(text) : null;
+    } catch (_) {
+      data = text;
+    }
+    if (!res.ok) {
+      const err = new Error(typeof data === 'object' && data?.message ? data.message : res.statusText);
+      err.code = 'E_EVOBGP_HTTP';
+      err.status = res.status;
+      err.body = data;
+      throw err;
+    }
+    return data;
+  } finally {
+    clearTimeout(t);
+  }
+}
+
+async function paginate(path, params = {}) {
+  const items = [];
+  let cursor = null;
+  while (true) {
+    const res = await evobgpRequest('GET', path, {
+      query: { ...params, limit: PAGE_LIMIT, ...(cursor ? { cursor } : {}) },
+    });
+    const chunk = Array.isArray(res?.items) ? res.items : [];
+    items.push(...chunk);
+    if (!res?.has_more || !res?.next_cursor) break;
+    cursor = res.next_cursor;
+  }
+  return items;
+}
+
+async function fetchRouterListsCatalog() {
+  return evobgpRequest('GET', '/router-lists/catalog');
+}
+
+function getCatalogItems(section) {
+  if (!section || typeof section !== 'object') return [];
+  return Array.isArray(section.items) ? section.items : [];
+}
+
+async function listCommunities() {
+  let communities = [];
+  try {
+    const catalog = await fetchRouterListsCatalog();
+    communities = getCatalogItems(catalog.communities);
+  } catch {
+    communities = await paginate('/communities');
+  }
+  return communities.map((c) => ({
+    id: c.id,
+    value: String(c.community || ''),
+    name: String(c.title || ''),
+    description: '',
+    tags: [],
+    color: '',
+    icon: '',
+  }));
+}
+
+async function getOrCreateModule(moduleType, defaultName) {
+  const modules = await paginate('/modules', { type: moduleType });
+  if (modules.length > 0) return modules[0];
+  const created = await evobgpRequest('POST', '/modules', {
+    body: { type: moduleType, name: defaultName, enabled: true },
+  });
+  return created?.data || created;
+}
+
+function normalizeList(items, keyName) {
+  const out = [];
+  const seen = new Set();
+  for (const item of items || []) {
+    const key = String(item?.[keyName] || '').trim().toLowerCase();
+    if (!key || seen.has(key)) continue;
+    seen.add(key);
+    out.push({ ...item, [keyName]: key });
+  }
+  return out;
+}
+
+async function buildCommunityMaps() {
+  const communities = await listCommunities();
+  const byId = new Map(communities.map((c) => [c.id, c.value]));
+  const byValue = new Map(communities.map((c) => [c.value, c.id]));
+  return { communities, byId, byValue };
+}
+
+function mapCommunityToValue(communityId, byId) {
+  if (!communityId) return '';
+  return byId.get(communityId) || String(communityId);
+}
+
+async function syncEntries({
+  moduleType,
+  moduleName,
+  listPath,
+  keyName,
+  createBody,
+  patchBody,
+  currentItems,
+  originalItems,
+  requireCommunity = false,
+}) {
+  const module = await getOrCreateModule(moduleType, moduleName);
+  const moduleId = module.id;
+  const { byId, byValue } = await buildCommunityMaps();
+  const existing = await paginate(`/modules/${moduleId}/${listPath}`);
+
+  const existingByKey = new Map();
+  for (const row of existing) {
+    const entryKey = String(row?.[keyName] || '').trim().toLowerCase();
+    if (!entryKey) continue;
+    existingByKey.set(entryKey, row);
+  }
+
+  const normalizedCurrent = normalizeList(currentItems, keyName);
+  const normalizedOriginal = normalizeList(originalItems, keyName);
+  const currentMap = new Map(normalizedCurrent.map((i) => [String(i[keyName]).toLowerCase(), i]));
+  const originalKeys = new Set(normalizedOriginal.map((i) => String(i[keyName]).toLowerCase()));
+
+  for (const key of originalKeys) {
+    if (!currentMap.has(key) && existingByKey.has(key)) {
+      const row = existingByKey.get(key);
+      await evobgpRequest('DELETE', `/modules/${moduleId}/${listPath}/${row.id}`);
+      existingByKey.delete(key);
+    }
+  }
+
+  for (const [key, item] of currentMap.entries()) {
+    const rawCommunity = String(item.community || '').trim();
+    const communityId = byValue.get(rawCommunity) || null;
+    if (requireCommunity && !communityId) {
+      throw new Error(`Community "${rawCommunity}" не найдена в справочнике`);
+    }
+
+    const existingRow = existingByKey.get(key);
+    if (!existingRow) {
+      await evobgpRequest('POST', `/modules/${moduleId}/${listPath}`, { body: createBody(item, communityId) });
+      continue;
+    }
+
+    const currentCommunity = mapCommunityToValue(existingRow.community_id, byId);
+    if (currentCommunity !== rawCommunity) {
+      await evobgpRequest('PATCH', `/modules/${moduleId}/${listPath}/${existingRow.id}`, {
+        body: patchBody(item, communityId),
+      });
+    }
+  }
+}
+
+async function fetchDomainsData() {
+  try {
+    const catalog = await fetchRouterListsCatalog();
+    const communities = getCatalogItems(catalog.communities);
+    const byId = new Map(communities.map((c) => [c.id, String(c.community || '')]));
+    const rows = getCatalogItems(catalog.domains);
+    return rows
+      .map((row) => ({
+        id: row?.entry?.id || row?.id || '',
+        domain: String(row?.entry?.fqdn || row?.fqdn || '')
+          .trim()
+          .toLowerCase(),
+        community: mapCommunityToValue(row?.entry?.community_id || row?.community_id || null, byId),
+      }))
+      .filter((row) => row.domain);
+  } catch {
+    const module = await getOrCreateModule('DOMAINS', 'Domains');
+    const { byId } = await buildCommunityMaps();
+    const rows = await paginate(`/modules/${module.id}/domain-entries`);
+    return rows.map((row) => ({
+      id: row.id,
+      domain: String(row.fqdn || '')
+        .trim()
+        .toLowerCase(),
+      community: mapCommunityToValue(row.community_id, byId),
+    }));
+  }
+}
+
+async function fetchIpRangesData() {
+  try {
+    const catalog = await fetchRouterListsCatalog();
+    const communities = getCatalogItems(catalog.communities);
+    const byId = new Map(communities.map((c) => [c.id, String(c.community || '')]));
+    const rows = getCatalogItems(catalog.ip_ranges);
+    return rows
+      .map((row) => ({
+        id: row?.entry?.id || row?.id || '',
+        ipRange: String(row?.entry?.prefix || row?.prefix || '').trim(),
+        community: mapCommunityToValue(row?.entry?.community_id || row?.community_id || null, byId),
+      }))
+      .filter((row) => row.ipRange);
+  } catch {
+    const module = await getOrCreateModule('IP_RANGES', 'IP ranges');
+    const { byId } = await buildCommunityMaps();
+    const rows = await paginate(`/modules/${module.id}/ip-range-entries`);
+    return rows.map((row) => ({
+      id: row.id,
+      ipRange: String(row.prefix || '').trim(),
+      community: mapCommunityToValue(row.community_id, byId),
+    }));
+  }
+}
+
+async function fetchAsnsData() {
+  try {
+    const catalog = await fetchRouterListsCatalog();
+    const communities = getCatalogItems(catalog.communities);
+    const byId = new Map(communities.map((c) => [c.id, String(c.community || '')]));
+    const rows = getCatalogItems(catalog.asns);
+    return rows
+      .map((row) => ({
+        id: row?.entry?.id || row?.id || '',
+        asn: String(row?.entry?.asn || row?.asn || '').trim(),
+        community: mapCommunityToValue(row?.entry?.community_id || row?.community_id || null, byId),
+      }))
+      .filter((row) => row.asn);
+  } catch {
+    const module = await getOrCreateModule('AS_PREFIXES', 'ASNs');
+    const { byId } = await buildCommunityMaps();
+    const rows = await paginate(`/modules/${module.id}/as-entries`);
+    return rows.map((row) => ({
+      id: row.id,
+      asn: String(row.asn || '').trim(),
+      community: mapCommunityToValue(row.community_id, byId),
+    }));
+  }
+}
+
+/** Auto-URL: секция каталога или модуль AUTO_URLS / URL_LIST (настраивается env). */
+async function fetchAutoUrlsData() {
+  try {
+    const catalog = await fetchRouterListsCatalog();
+    for (const name of ['auto_urls', 'auto_url', 'urls', 'url_list']) {
+      const sec = catalog[name];
+      const items = getCatalogItems(sec);
+      if (items.length) {
+        const communities = getCatalogItems(catalog.communities);
+        const byId = new Map(communities.map((c) => [c.id, String(c.community || '')]));
+        return items.map((row) => {
+          const url = String(row?.entry?.url || row?.entry?.fqdn || row?.url || row?.fqdn || '').trim();
+          const community = mapCommunityToValue(
+            row?.entry?.community_id || row?.community_id || null,
+            byId,
+          );
+          return { url, community };
+        }).filter((r) => r.url);
+      }
+    }
+  } catch (_) {}
+  const modType = process.env.EVOBGP_AUTO_URL_MODULE_TYPE || 'AUTO_URLS';
+  try {
+    const module = await getOrCreateModule(modType, 'Auto URLs');
+    const listPath = process.env.EVOBGP_AUTO_URL_LIST_PATH || 'url-entries';
+    const rows = await paginate(`/modules/${module.id}/${listPath}`);
+    const { byId } = await buildCommunityMaps();
+    return rows.map((row) => ({
+      url: String(row.url || row.fqdn || '').trim(),
+      community: mapCommunityToValue(row.community_id, byId),
+    })).filter((r) => r.url);
+  } catch {
+    return [];
+  }
+}
+
+async function saveDomainsData(currentItems, originalItems) {
+  await syncEntries({
+    moduleType: 'DOMAINS',
+    moduleName: 'Domains',
+    listPath: 'domain-entries',
+    keyName: 'domain',
+    currentItems,
+    originalItems,
+    createBody: (item, communityId) => ({ fqdn: item.domain, community_id: communityId }),
+    patchBody: (item, communityId) => ({ fqdn: item.domain, community_id: communityId }),
+  });
+}
+
+async function saveIpRangesData(currentItems, originalItems) {
+  await syncEntries({
+    moduleType: 'IP_RANGES',
+    moduleName: 'IP ranges',
+    listPath: 'ip-range-entries',
+    keyName: 'ipRange',
+    currentItems,
+    originalItems,
+    requireCommunity: true,
+    createBody: (item, communityId) => ({ prefix: item.ipRange, community_id: communityId }),
+    patchBody: (item, communityId) => ({ prefix: item.ipRange, community_id: communityId }),
+  });
+}
+
+async function saveAsnsData(currentItems, originalItems) {
+  await syncEntries({
+    moduleType: 'AS_PREFIXES',
+    moduleName: 'ASNs',
+    listPath: 'as-entries',
+    keyName: 'asn',
+    currentItems,
+    originalItems,
+    createBody: (item, communityId) => ({ asn: Number(item.asn), community_id: communityId }),
+    patchBody: (item, communityId) => ({ asn: Number(item.asn), community_id: communityId }),
+  });
+}
+
+async function saveCommunitiesData(currentItems) {
+  const existing = await paginate('/communities');
+  const existingByValue = new Map(existing.map((c) => [String(c.community || ''), c]));
+  const nextByValue = new Map();
+  for (const item of currentItems || []) {
+    const value = String(item?.value || '').trim();
+    if (!value) continue;
+    nextByValue.set(value, item);
+  }
+
+  for (const [value, row] of existingByValue.entries()) {
+    if (!nextByValue.has(value)) {
+      await evobgpRequest('DELETE', `/communities/${row.id}`);
+    }
+  }
+
+  for (const [value, item] of nextByValue.entries()) {
+    const title = String(item?.name || '').trim();
+    const row = existingByValue.get(value);
+    if (!row) {
+      await evobgpRequest('POST', '/communities', { body: { community: value, title } });
+      continue;
+    }
+    if (String(row.title || '') !== title) {
+      await evobgpRequest('PATCH', `/communities/${row.id}`, { body: { community: value, title } });
+    }
+  }
+}
+
+async function saveAutoUrlsData(urls) {
+  const originalItems = await fetchAutoUrlsData();
+  const list = (urls || []).map((u) => ({
+    url: String(u?.url || '').trim(),
+    community: String(u?.community || '').trim(),
+  })).filter((u) => u.url && u.community);
+  const modType = process.env.EVOBGP_AUTO_URL_MODULE_TYPE || 'AUTO_URLS';
+  const listPath = process.env.EVOBGP_AUTO_URL_LIST_PATH || 'url-entries';
+  const keyName = process.env.EVOBGP_AUTO_URL_KEY_FIELD || 'url';
+  const currentItems = list.map((u) => ({ [keyName]: u.url, community: u.community }));
+  const origMapped = originalItems.map((u) => ({ [keyName]: u.url, community: u.community }));
+  await syncEntries({
+    moduleType: modType,
+    moduleName: 'Auto URLs',
+    listPath,
+    keyName,
+    currentItems,
+    originalItems: origMapped,
+    requireCommunity: true,
+    createBody: (item, communityId) => ({ url: item[keyName], community_id: communityId }),
+    patchBody: (item, communityId) => ({ url: item[keyName], community_id: communityId }),
+  });
+}
+
+module.exports = {
+  isConfigured,
+  evobgpRequest,
+  paginate,
+  fetchRouterListsCatalog,
+  listCommunities,
+  fetchDomainsData,
+  fetchIpRangesData,
+  fetchAsnsData,
+  fetchAutoUrlsData,
+  saveCommunitiesData,
+  saveDomainsData,
+  saveIpRangesData,
+  saveAsnsData,
+  saveAutoUrlsData,
+  buildCommunityMaps,
+  mapCommunityToValue,
+};

backend/services/filterRulesStorage.js

@@ -0,0 +1,124 @@
+/**
+ * Глобальные и simple-фильтры в таблице app_filter_rules (не в blobs JSON).
+ */
+
+const crypto = require('crypto');
+const { openDatabase } = require('../db/sqliteDb');
+
+/** @typedef {'global'|'simple'} FilterScope */
+
+function assertScope(scope) {
+  if (scope !== 'global' && scope !== 'simple') throw new Error('Invalid filter scope');
+}
+
+function mtimeMetaKey(scope) {
+  return `filter_rules_mtime_${scope}`;
+}
+
+function sha256EtagFromString(s) {
+  const h = crypto.createHash('sha256').update(Buffer.from(s, 'utf8')).digest('hex');
+  return `"${h}"`;
+}
+
+function rowToItem(row) {
+  let extra = {};
+  try {
+    extra = JSON.parse(row.extra_json || '{}');
+    if (!extra || typeof extra !== 'object' || Array.isArray(extra)) extra = {};
+  } catch {
+    extra = {};
+  }
+  const out = {
+    community: row.community,
+    gateway: row.gateway,
+    description: row.description || '',
+  };
+  for (const k of Object.keys(extra).sort()) {
+    if (k === 'community' || k === 'gateway' || k === 'description') continue;
+    out[k] = extra[k];
+  }
+  return out;
+}
+
+/**
+ * @param {FilterScope} scope
+ */
+function listRules(scope) {
+  assertScope(scope);
+  const db = openDatabase();
+  const rows = db
+    .prepare(
+      `SELECT community, gateway, description, extra_json FROM app_filter_rules WHERE scope = ? ORDER BY position ASC`,
+    )
+    .all(scope);
+  return rows.map(rowToItem);
+}
+
+/**
+ * @param {FilterScope} scope
+ * @returns {{ etag: string, lastModified: string|null, contentLength: number }}
+ */
+function headMetaForScope(scope) {
+  assertScope(scope);
+  const items = listRules(scope);
+  const body = JSON.stringify(items);
+  const etag = sha256EtagFromString(body);
+  const contentLength = Buffer.byteLength(body, 'utf8');
+  const db = openDatabase();
+  const row = db.prepare('SELECT value FROM app_meta WHERE key = ?').get(mtimeMetaKey(scope));
+  const lastModified = row?.value ? new Date(Number(row.value)).toISOString() : null;
+  return { etag, lastModified, contentLength };
+}
+
+/**
+ * @param {FilterScope} scope
+ * @param {any[]} items
+ * @param {{ validateItem?: (item: any, i: number) => string|null }} [opts]
+ */
+function replaceRules(scope, items, opts = {}) {
+  assertScope(scope);
+  const { validateItem } = opts;
+  if (!Array.isArray(items)) throw new Error('items must be an array');
+  if (validateItem) {
+    for (let i = 0; i < items.length; i++) {
+      const err = validateItem(items[i], i);
+      if (err) {
+        const e = new Error(err);
+        e.code = 'E_SCHEMA';
+        throw e;
+      }
+    }
+  }
+  const db = openDatabase();
+  const now = Date.now();
+  const del = db.prepare('DELETE FROM app_filter_rules WHERE scope = ?');
+  const ins = db.prepare(
+    `INSERT INTO app_filter_rules (scope, position, community, gateway, description, extra_json, updated_at)
+     VALUES (?, ?, ?, ?, ?, ?, ?)`,
+  );
+  const metaUpsert = db.prepare('INSERT OR REPLACE INTO app_meta (key, value) VALUES (?, ?)');
+
+  const run = () => {
+    del.run(scope);
+    for (let i = 0; i < items.length; i++) {
+      const it = items[i] || {};
+      const community = String(it.community || '').trim();
+      const gateway = String(it.gateway || '').trim();
+      const description = String(it.description || '').trim();
+      const extra = { ...it };
+      delete extra.community;
+      delete extra.gateway;
+      delete extra.description;
+      ins.run(scope, i, community, gateway, description, JSON.stringify(extra), now);
+    }
+    metaUpsert.run(mtimeMetaKey(scope), String(now));
+  };
+  db.transaction(run)();
+  return headMetaForScope(scope);
+}
+
+module.exports = {
+  listRules,
+  headMetaForScope,
+  replaceRules,
+};

backend/services/mikrotikBackupScheduler.js

@@ -4,7 +4,7 @@
  * Работает внутри Node-процесса backend:
  * - периодически обходит jumphost-сервера из servers.json
  * - вызывает RouterOS REST API /rest/export (compact)
- * - сохраняет конфиг в S3 через saveBackupForServer
+ * - сохраняет конфиг на диск через saveBackupForServer
  *
  * Управляется через переменные окружения:
  * - MIKROTIK_BACKUP_ENABLED=true|false (по умолчанию true)
@@ -131,7 +131,7 @@ async function runBackupOnce(logger) {
 
         log.info(
           { component: 'mikrotik-backup', serverId: id, key: result.key },
-          'Automatic backup saved to S3',
+          'Automatic backup saved to disk',
         );
       } catch (err) {
         log.error(

backend/services/s3Service.js

@@ -1,58 +1,35 @@
 /**
- * Сервис для работы с S3 (Yandex Object Storage)
- * Централизованные операции чтения/записи/кэширования
+ * Локальное хранилище объектов (SQLite). Сохранены имена экспортов readS3TextObject / writeS3JsonObject и т.д. для совместимости с роутами.
+ * Справочники EvoBGP не хранятся здесь — см. evobgpClient.js
  */
 
-const { S3Client, GetObjectCommand, HeadObjectCommand, PutObjectCommand, DeleteObjectCommand, ListObjectsV2Command } = require('@aws-sdk/client-s3');
-const { NodeHttpHandler } = require('@smithy/node-http-handler');
-const http = require('http');
-const https = require('https');
+const { openDatabase } = require('../db/sqliteDb');
+const blobStorage = require('./blobStorage');
 
-// S3 Client setup
-const s3 = new S3Client({
-    endpoint: 'https://storage.yandexcloud.net',
-    region: process.env.AWS_REGION,
-    forcePathStyle: true,
-    maxAttempts: 3,
-    requestHandler: new NodeHttpHandler({
-      httpAgent: new http.Agent({ keepAlive: true }),
-      httpsAgent: new https.Agent({ keepAlive: true })
-    }),
-    credentials: {
-        accessKeyId: process.env.AWS_ACCESS_KEY_ID || process.env.S3_ACCESS_KEY_ID,
-        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || process.env.S3_SECRET_ACCESS_KEY,
-    }
-});
+openDatabase();
 
-const BUCKET_NAME = process.env.S3_BUCKET_NAME;
+const BUCKET_NAME = process.env.SQLITE_BUCKET_LABEL || 'sqlite';
 
-// In-memory cache
 const cache = {
   text: new Map(),
   head: new Map(),
   max: 100,
-  ttlMs: 30_000
+  ttlMs: 30_000,
 };
 
-/**
- * Конвертирует stream в строку
- */
 async function streamToString(stream) {
-    if (!stream) return '';
-    if (typeof stream.transformToString === 'function') {
-        return await stream.transformToString();
-    }
-    return await new Promise((resolve, reject) => {
-        let chunks = [];
-        stream.on('data', (c) => chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(String(c))));
-        stream.once('error', reject);
-        stream.once('end', () => resolve(Buffer.concat(chunks).toString('utf-8')));
-    });
+  if (!stream) return '';
+  if (typeof stream.transformToString === 'function') {
+    return await stream.transformToString();
+  }
+  return await new Promise((resolve, reject) => {
+    const chunks = [];
+    stream.on('data', (c) => chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(String(c))));
+    stream.once('error', reject);
+    stream.once('end', () => resolve(Buffer.concat(chunks).toString('utf-8')));
+  });
 }
 
-/**
- * Получить значение из кэша
- */
 function getCache(map, key) {
   const v = map.get(key);
   if (!v) return null;
@@ -63,9 +40,6 @@ function getCache(map, key) {
   return v.value;
 }
 
-/**
- * Установить значение в кэш
- */
 function setCache(map, key, value) {
   if (map.size >= cache.max) {
     const firstKey = map.keys().next().value;
@@ -74,213 +48,115 @@ function setCache(map, key, value) {
   map.set(key, { value, at: Date.now() });
 }
 
-/**
- * Инвалидировать кэш для ключа
- */
 function invalidateCacheForKey(key) {
-  try { cache.text.delete(key); } catch {}
-  try { cache.head.delete(key); } catch {}
-}
-
-/**
- * Прочитать текстовый объект из S3
- */
-async function readS3TextObject(key, s3Duration = null) {
-    const cached = getCache(cache.text, key);
-    if (cached) return cached;
-    
-    const s3Start = Date.now();
-    const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: key }));
-    if (s3Duration) {
-      try { s3Duration.labels('getObject').observe((Date.now() - s3Start)/1000); } catch {}
-    }
-    
-    const out = {
-        body: await streamToString(data.Body),
-        etag: data.ETag || undefined,
-        lastModified: data.LastModified ? data.LastModified.toISOString() : undefined,
-        contentLength: typeof data.ContentLength === 'number' ? data.ContentLength : undefined
-    };
-    setCache(cache.text, key, out);
-    return out;
-}
-
-/**
- * Получить ETag объекта из S3
- */
-async function headS3ObjectEtag(key, s3Duration = null) {
-    const cached = getCache(cache.head, key);
-    if (cached && cached.etag) return cached.etag;
-    
-    const s3Start = Date.now();
-    const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: key }));
-    if (s3Duration) {
-      try { s3Duration.labels('headObject').observe((Date.now() - s3Start)/1000); } catch {}
-    }
-    
-    setCache(cache.head, key, { etag: head.ETag || undefined });
-    return head.ETag || undefined;
-}
-
-/**
- * Получить метаданные объекта (etag, lastModified, contentLength)
- */
-async function headMeta(key) {
   try {
-    const h = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: key }));
-    return {
-      etag: h.ETag || null,
-      lastModified: h.LastModified ? new Date(h.LastModified).toISOString() : null,
-      contentLength: typeof h.ContentLength === 'number' ? h.ContentLength : null,
-    };
-  } catch (e) {
-    return { etag: null, lastModified: null, contentLength: null };
+    cache.text.delete(key);
+  } catch (_) {}
+  try {
+    cache.head.delete(key);
+  } catch (_) {}
+}
+
+async function readS3TextObject(key) {
+  const cached = getCache(cache.text, key);
+  if (cached) return cached;
+  const row = blobStorage.readBlob(key);
+  if (!row) {
+    const err = new Error('NoSuchKey');
+    err.code = 'NoSuchKey';
+    throw err;
   }
-}
-
-/**
- * Записать текстовый объект в S3
- */
-async function writeS3TextObject(key, content, contentType = 'text/plain') {
-  await s3.send(new PutObjectCommand({
-    Bucket: BUCKET_NAME,
-    Key: key,
-    Body: content,
-    ContentType: contentType,
-  }));
-  invalidateCacheForKey(key);
-  return await headMeta(key);
-}
-
-/**
- * Записать JSON объект в S3
- */
-async function writeS3JsonObject(key, data) {
-  return await writeS3TextObject(key, JSON.stringify(data, null, 2), 'application/json');
-}
-
-/**
- * Удалить объект из S3
- */
-async function deleteS3Object(key) {
-  await s3.send(new DeleteObjectCommand({ Bucket: BUCKET_NAME, Key: key }));
-  invalidateCacheForKey(key);
-}
-
-/**
- * Список объектов в S3 по префиксу
- * Используется для истории/бэкапов (количество ограничено для безопасности)
- */
-async function listS3Objects(prefix, { maxKeys = 100 } = {}) {
-  const out = [];
-  let continuationToken = undefined;
-
-  while (out.length < maxKeys) {
-    const resp = await s3.send(new ListObjectsV2Command({
-      Bucket: BUCKET_NAME,
-      Prefix: prefix,
-      ContinuationToken: continuationToken,
-      MaxKeys: Math.min(1000, maxKeys - out.length),
-    }));
-
-    const contents = resp.Contents || [];
-    for (const obj of contents) {
-      out.push({
-        key: obj.Key,
-        size: typeof obj.Size === 'number' ? obj.Size : null,
-        lastModified: obj.LastModified ? new Date(obj.LastModified).toISOString() : null,
-        etag: obj.ETag || null,
-      });
-      if (out.length >= maxKeys) break;
-    }
-
-    if (!resp.IsTruncated || !resp.NextContinuationToken || out.length >= maxKeys) {
-      break;
-    }
-    continuationToken = resp.NextContinuationToken;
-  }
-
+  const out = {
+    body: row.body,
+    etag: row.etag,
+    lastModified: row.lastModified,
+    contentLength: row.contentLength,
+  };
+  setCache(cache.text, key, out);
   return out;
 }
 
-/**
- * Потоковое чтение с пагинацией больших текстовых файлов
- */
-async function streamPaginatedText({ key, mapLine, q, offset = 0, limit = 0 }) {
-  return new Promise(async (resolve, reject) => {
-    let total = 0;
-    const items = [];
-    let sent = 0;
-    let buffered = '';
-    
-    const matchesQuery = (line) => {
-      if (!q) return true;
-      return line.toLowerCase().includes(String(q).toLowerCase());
-    };
-    
-    try {
-      const resp = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: key }));
-      const stream = resp.Body;
-      
-      if (!stream || typeof stream.on !== 'function') {
-        const text = await streamToString(resp.Body);
-        const lines = text.split('\n');
-        for (let i = 0; i < lines.length; i++) {
-          const line = String(lines[i] || '').trim();
-          if (!line) continue;
-          if (!matchesQuery(line)) continue;
-          total++;
-          const pos = total - 1;
-          if (limit > 0) {
-            if (pos >= offset && sent < limit) { items.push(mapLine(line)); sent++; }
-          } else {
-            items.push(mapLine(line));
-          }
-        }
-        return resolve({ items, total });
-      }
-      
-      stream.on('data', (chunk) => {
-        buffered += chunk.toString('utf-8');
-        let lines = buffered.split('\n');
-        buffered = lines.pop();
-        for (const lnRaw of lines) {
-          const line = lnRaw.trim();
-          if (!line) continue;
-          if (!matchesQuery(line)) continue;
-          total++;
-          const pos = total - 1;
-          if (limit > 0) {
-            if (pos >= offset && sent < limit) { items.push(mapLine(line)); sent++; }
-          } else {
-            items.push(mapLine(line));
-          }
-        }
-      });
-      
-      stream.on('end', () => {
-        const last = (buffered || '').trim();
-        if (last) {
-          if (!q || last.toLowerCase().includes(String(q).toLowerCase())) {
-            total++;
-            if (limit > 0) {
-              const pos = total - 1;
-              if (pos >= offset && items.length < limit) items.push(mapLine(last));
-            } else {
-              items.push(mapLine(last));
-            }
-          }
-        }
-        resolve({ items, total });
-      });
-      
-      stream.on('error', reject);
-    } catch (e) {
-      reject(e);
-    }
-  });
+async function headS3ObjectEtag(key) {
+  const cached = getCache(cache.head, key);
+  if (cached && cached.etag) return cached.etag;
+  const etag = blobStorage.headEtag(key);
+  if (etag) setCache(cache.head, key, { etag });
+  return etag;
 }
 
+async function headMeta(key) {
+  return blobStorage.headMetaRow(key);
+}
+
+async function writeS3TextObject(key, content, contentType = 'text/plain') {
+  const db = openDatabase();
+  blobStorage.writeBlobTx(db, key, content, contentType);
+  invalidateCacheForKey(key);
+  return headMeta(key);
+}
+
+async function writeS3JsonObject(key, data) {
+  return writeS3TextObject(key, JSON.stringify(data, null, 2), 'application/json');
+}
+
+async function deleteS3Object(key) {
+  blobStorage.deleteBlob(key);
+  invalidateCacheForKey(key);
+}
+
+async function listS3Objects(prefix, { maxKeys = 100 } = {}) {
+  return blobStorage.listByPrefix(prefix, maxKeys);
+}
+
+async function streamPaginatedText({ key, mapLine, q, offset = 0, limit = 0 }) {
+  let text = '';
+  try {
+    const row = await readS3TextObject(key);
+    text = row.body || '';
+  } catch (e) {
+    if (e?.code === 'NoSuchKey') {
+      return { items: [], total: 0 };
+    }
+    throw e;
+  }
+
+  const lines = text.split('\n');
+  let total = 0;
+  const items = [];
+  const qstr = q ? String(q).toLowerCase() : '';
+  const matchesQuery = (line) => !qstr || line.toLowerCase().includes(qstr);
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = String(lines[i] || '').trim();
+    if (!line) continue;
+    if (!matchesQuery(line)) continue;
+    total++;
+    const pos = total - 1;
+    if (limit > 0) {
+      if (pos >= offset && items.length < limit) items.push(mapLine(line));
+    } else {
+      items.push(mapLine(line));
+    }
+  }
+  return { items, total };
+}
+
+function listBlobVersionsForKey(key, maxKeys = 50) {
+  return blobStorage.listVersions(key, maxKeys);
+}
+
+function rollbackBlobVersion(key, versionId) {
+  openDatabase();
+  return blobStorage.rollbackToVersionId(key, versionId);
+}
+
+/** Заглушка: прямой вызов S3 SDK больше не используется. */
+const s3 = {
+  send() {
+    throw new Error('S3 SDK removed: use readS3TextObject / writeS3TextObject or evobgpClient');
+  },
+};
+
 module.exports = {
   s3,
   BUCKET_NAME,
@@ -294,5 +170,6 @@ module.exports = {
   streamPaginatedText,
   invalidateCacheForKey,
   listS3Objects,
+  listBlobVersionsForKey,
+  rollbackBlobVersion,
 };
-