denozord/router-lists-ui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Backend: SQLite storage, EvoBGP integration, filters in SQL

Browse Source 
Made-with: Cursor
This commit is contained in:
denozord
2026-04-20 01:17:00 +07:00
parent 7f3eaea40a
commit 1c6e6ab24a
27 changed files with 2406 additions and 2542 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/routes/communitiesRoutes.js
+92 -98
View File
@@ -1,38 +1,52 @@
/**
* Роуты для работы с communities (справочник BGP Community)
* Роуты для работы с communities (справочник BGP Community) — данные из EvoBGP API
*/
const { s3, BUCKET_NAME, writeS3JsonObject, invalidateCacheForKey } = require('../services/s3Service');
const evobgpClient = require('../services/evobgpClient');
const filterRulesStorage = require('../services/filterRulesStorage');
const { sendError, sendOk } = require('../middleware/errorHandler');
const { GetObjectCommand, PutObjectCommand } = require('@aws-sdk/client-s3');
const { streamToString } = require('../services/s3Service');
const validators = require('../lib/validators');
const { getS3JsonWithHeaders } = require('../utils/s3Helpers');
const S3_KEY = 'bgp_data/communities.json';
function requireEvobgp(res) {
if (!evobgpClient.isConfigured()) {
return sendError(
res,
503,
'EvoBGP не настроен: задайте EVOBGP_API_URL и EVOBGP_API_TOKEN',
'E_EVOBGP_NOT_CONFIGURED',
);
}
return null;
}
// GET /api/communities
async function getCommunities(req, res) {
await getS3JsonWithHeaders(S3_KEY, req, res, {
transform: (communities) => {
// Нормализация
return communities
.filter((c) => c && typeof c.value === 'string' && c.value.trim().length > 0)
.map((c) => ({
value: String(c.value).trim(),
name: c.name ? String(c.name) : '',
description: c.description ? String(c.description) : '',
tags: Array.isArray(c.tags) ? c.tags.map(String) : [],
color: c.color ? String(c.color) : '',
icon: c.icon ? String(c.icon) : ''
}));
},
defaultValue: []
});
const err = requireEvobgp(res);
if (err) return err;
try {
const communities = await evobgpClient.listCommunities();
const normalized = communities
.filter((c) => c && typeof c.value === 'string' && c.value.trim().length > 0)
.map((c) => ({
value: String(c.value).trim(),
name: c.name ? String(c.name) : '',
description: c.description ? String(c.description) : '',
tags: Array.isArray(c.tags) ? c.tags.map(String) : [],
color: c.color ? String(c.color) : '',
icon: c.icon ? String(c.icon) : '',
}));
res.json(normalized);
} catch (e) {
console.error('getCommunities', e);
return sendError(res, 500, e.message || 'EvoBGP error', 'E_EVOBGP');
}
}
// POST /api/communities
async function postCommunities(req, res) {
const err = requireEvobgp(res);
if (err) return err;
const { communities } = req.body;
if (!Array.isArray(communities)) {
@@ -42,26 +56,26 @@ async function postCommunities(req, res) {
const seen = new Set();
const normalized = [];
const validationErrors = [];
for (let i = 0; i < communities.length; i++) {
const entry = communities[i] || {};
const value = typeof entry.value === 'string' ? entry.value.trim() : '';
if (!value) {
validationErrors.push(`Community at index ${i} is missing required field: value`);
continue;
}
if (!validators.isValidCommunity(value)) {
validationErrors.push(`Community at index ${i} has invalid value: ${value}`);
continue;
}
if (seen.has(value)) {
validationErrors.push(`Duplicate community value at index ${i}: ${value}`);
continue;
}
seen.add(value);
normalized.push({
value,
@@ -75,97 +89,78 @@ async function postCommunities(req, res) {
enabled: typeof entry.enabled === 'boolean' ? entry.enabled : true,
});
}
if (validationErrors.length > 0) {
return sendError(res, 400, 'Validation errors', 'E_VALIDATION', { errors: validationErrors });
}
try {
await s3.send(new PutObjectCommand({
Bucket: BUCKET_NAME,
Key: S3_KEY,
Body: JSON.stringify(normalized, null, 2),
ContentType: 'application/json',
}));
invalidateCacheForKey(S3_KEY);
const { headMeta } = require('../services/s3Service');
const meta = await headMeta(S3_KEY);
return sendOk(res, meta);
await evobgpClient.saveCommunitiesData(normalized);
return sendOk(res, {
etag: null,
lastModified: new Date().toISOString(),
contentLength: null,
});
} catch (error) {
console.error('Error writing communities to S3:', error);
return sendError(res, 500, 'Error writing communities to S3', 'E_S3');
console.error('Error writing communities to EvoBGP:', error);
return sendError(res, 500, error.message || 'EvoBGP write failed', 'E_EVOBGP');
}
}
function countCommunityInLines(rows, getCommunity) {
const usage = new Map();
for (const row of rows || []) {
const c = getCommunity(row);
if (c) usage.set(c, (usage.get(c) || 0) + 1);
}
return usage;
}
// GET /api/communities/stats
async function getCommunityStats(req, res) {
const err = requireEvobgp(res);
if (err) return err;
try {
const [domainsRes, ipRangesRes, asnsRes, filtersRes] = await Promise.allSettled([
s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt' })),
s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt' })),
s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/asns.txt' })),
s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'filter-manager/simple-filters.json' })),
const [domains, ipRanges, asns] = await Promise.allSettled([
evobgpClient.fetchDomainsData(),
evobgpClient.fetchIpRangesData(),
evobgpClient.fetchAsnsData(),
]);
const communityUsage = new Map();
// Подсчет использования в доменах
if (domainsRes.status === 'fulfilled') {
const text = await streamToString(domainsRes.value.Body);
text.split('\n').filter(Boolean).forEach(line => {
const parts = line.trim().split(/\s+/);
if (parts[1]) {
communityUsage.set(parts[1], (communityUsage.get(parts[1]) || 0) + 1);
}
});
if (domains.status === 'fulfilled') {
const m = countCommunityInLines(domains.value, (r) => r.community);
for (const [k, v] of m) communityUsage.set(k, (communityUsage.get(k) || 0) + v);
}
// Подсчет в IP ranges
if (ipRangesRes.status === 'fulfilled') {
const text = await streamToString(ipRangesRes.value.Body);
text.split('\n').filter(Boolean).forEach(line => {
const parts = line.trim().split(/\s+/);
if (parts[1]) {
communityUsage.set(parts[1], (communityUsage.get(parts[1]) || 0) + 1);
}
});
if (ipRanges.status === 'fulfilled') {
const m = countCommunityInLines(ipRanges.value, (r) => r.community);
for (const [k, v] of m) communityUsage.set(k, (communityUsage.get(k) || 0) + v);
}
// Подсчет в ASNs
if (asnsRes.status === 'fulfilled') {
const text = await streamToString(asnsRes.value.Body);
text.split('\n').filter(Boolean).forEach(line => {
const parts = line.trim().split(/\s+/);
if (parts[1]) {
communityUsage.set(parts[1], (communityUsage.get(parts[1]) || 0) + 1);
}
});
if (asns.status === 'fulfilled') {
const m = countCommunityInLines(asns.value, (r) => r.community);
for (const [k, v] of m) communityUsage.set(k, (communityUsage.get(k) || 0) + v);
}
// Подсчет в фильтрах
if (filtersRes.status === 'fulfilled') {
try {
const text = await streamToString(filtersRes.value.Body);
const filters = JSON.parse(text);
if (Array.isArray(filters)) {
filters.forEach(f => {
if (f.community) {
communityUsage.set(f.community, (communityUsage.get(f.community) || 0) + 1);
}
});
try {
const simpleFilters = filterRulesStorage.listRules('simple');
for (const f of simpleFilters) {
if (f.community) {
communityUsage.set(f.community, (communityUsage.get(f.community) || 0) + 1);
}
} catch {}
}
const stats = Array.from(communityUsage.entries()).map(([community, count]) => ({
community,
count,
})).sort((a, b) => b.count - a.count);
}
} catch (_) {}
const stats = Array.from(communityUsage.entries())
.map(([community, count]) => ({ community, count }))
.sort((a, b) => b.count - a.count);
res.json({ stats, total: stats.reduce((sum, s) => sum + s.count, 0) });
} catch (error) {
console.error('Error getting community stats:', error);
return sendError(res, 500, 'Error getting community stats', 'E_S3');
return sendError(res, 500, 'Error getting community stats', 'E_EVOBGP');
}
}
@@ -174,4 +169,3 @@ module.exports = {
postCommunities,
getCommunityStats,
};
backend/routes/evobgpReferenceRoutes.js
+257
View File
@@ -0,0 +1,257 @@
/**
* GET/POST для справочников из EvoBGP (ранее textDataRoutes + S3).
*/
const crypto = require('crypto');
const evobgpClient = require('../services/evobgpClient');
const { sendError, sendOk, checkIfNoneMatch } = require('../middleware/errorHandler');
const countOnlyCache = { map: new Map(), ttlMs: 10_000 };
function getCountOnlyCache(cacheKey) {
const v = countOnlyCache.map.get(cacheKey);
if (!v) return null;
if (Date.now() > v.at + countOnlyCache.ttlMs) {
countOnlyCache.map.delete(cacheKey);
return null;
}
return v.value;
}
function setCountOnlyCache(cacheKey, value) {
countOnlyCache.map.set(cacheKey, { value, at: Date.now() });
}
function etagForJson(obj) {
const s = JSON.stringify(obj);
const h = crypto.createHash('sha256').update(Buffer.from(s, 'utf8')).digest('hex');
return `"${h}"`;
}
function filterPage(items, q, offset, limit) {
const qstr = q ? String(q).toLowerCase() : '';
const filtered = !qstr
? items
: items.filter((it) => JSON.stringify(it).toLowerCase().includes(qstr));
const total = filtered.length;
if (limit > 0) {
return { items: filtered.slice(offset, offset + limit), total };
}
return { items: filtered, total };
}
function createEvobgpListGET({ fetchItems, cachePrefix, validate }) {
return async (req, res) => {
const { q = '', offset, limit, countOnly, format } = req.query || {};
if (!evobgpClient.isConfigured()) {
return sendError(res, 503, 'EvoBGP не настроен', 'E_EVOBGP_NOT_CONFIGURED');
}
try {
if (countOnly === 'true') {
const cacheKey = `${cachePrefix}:count:${q}`;
const cached = getCountOnlyCache(cacheKey);
if (cached != null) {
return res.json(format === 'std' ? { items: [], total: cached, meta: {} } : { total: cached });
}
const all = await fetchItems();
const { total } = filterPage(all, q, 0, 0);
setCountOnlyCache(cacheKey, total);
return res.json(format === 'std' ? { items: [], total, meta: {} } : { total });
}
if (Number(limit) > 0) {
const all = await fetchItems();
if (!validate(all)) {
return sendError(res, 500, 'Invalid data format', 'E_SCHEMA');
}
const { items, total } = filterPage(all, q, Number(offset) || 0, Number(limit) || 0);
return res.json(format === 'std' ? { items, total, meta: {} } : { items, total });
}
const all = await fetchItems();
if (!validate(all)) {
return sendError(res, 500, 'Invalid data format', 'E_SCHEMA');
}
const etag = etagForJson(all);
res.set('ETag', etag);
res.set('Last-Modified', new Date().toUTCString());
res.set('Content-Length-Source', String(Buffer.byteLength(JSON.stringify(all), 'utf8')));
if (checkIfNoneMatch(req, res, etag)) return;
if (format === 'std') {
return res.json({ items: all, total: all.length, meta: {} });
}
return res.json(all);
} catch (e) {
console.error(cachePrefix, e);
return sendError(res, 500, e.message || 'EvoBGP error', 'E_EVOBGP');
}
};
}
function readIfMatch(req) {
const raw = req.headers['if-match'] || req.headers['If-Match'];
if (!raw) return null;
return String(raw).trim();
}
function createEvobgpDomainsNewPOST({ validate, validateItem }) {
return async (req, res) => {
const { domains: items, etag: bodyEtag } = req.body || {};
if (!Array.isArray(items)) {
return sendError(res, 400, 'Data must be an array', 'E_BAD_REQUEST');
}
if (!validate(items)) {
return sendError(res, 400, 'Invalid payload format', 'E_SCHEMA');
}
if (validateItem) {
const validationErrors = [];
for (let i = 0; i < items.length; i++) {
const errors = validateItem(items[i], i);
if (errors.length > 0) validationErrors.push(...errors);
}
if (validationErrors.length > 0) {
return sendError(res, 400, 'Validation errors', 'E_VALIDATION', {
errors: validationErrors.slice(0, 10),
});
}
}
if (!evobgpClient.isConfigured()) {
return sendError(res, 503, 'EvoBGP не настроен', 'E_EVOBGP_NOT_CONFIGURED');
}
try {
const current = await evobgpClient.fetchDomainsData();
const currentEtag = etagForJson(current);
const ifMatch = readIfMatch(req);
const expected = ifMatch || bodyEtag;
if (expected && String(expected).trim() !== currentEtag) {
return sendError(res, 412, 'ETag mismatch', 'E_ETAG_MISMATCH');
}
const normalized = items.map((d) => ({
domain: String(d.domain || '').trim().toLowerCase(),
community: String(d.community || '').trim(),
}));
await evobgpClient.saveDomainsData(normalized, current);
return sendOk(res, { etag: null, lastModified: new Date().toISOString(), contentLength: null });
} catch (e) {
console.error(e);
return sendError(res, 500, e.message || 'EvoBGP write failed', 'E_EVOBGP');
}
};
}
function createEvobgpAsnsPOST({ validate, validateItem }) {
return async (req, res) => {
const { domains: items, etag: bodyEtag } = req.body || {};
if (!Array.isArray(items)) {
return sendError(res, 400, 'Data must be an array', 'E_BAD_REQUEST');
}
if (!validate(items)) {
return sendError(res, 400, 'Invalid payload format', 'E_SCHEMA');
}
if (validateItem) {
const validationErrors = [];
for (let i = 0; i < items.length; i++) {
const errors = validateItem(items[i], i);
if (errors.length > 0) validationErrors.push(...errors);
}
if (validationErrors.length > 0) {
return sendError(res, 400, 'Validation errors', 'E_VALIDATION', {
errors: validationErrors.slice(0, 10),
});
}
}
if (!evobgpClient.isConfigured()) {
return sendError(res, 503, 'EvoBGP не настроен', 'E_EVOBGP_NOT_CONFIGURED');
}
try {
const current = await evobgpClient.fetchAsnsData();
const currentEtag = etagForJson(
current.map((a) => ({ domain: a.asn, type: a.community })),
);
const ifMatch = readIfMatch(req);
const expected = ifMatch || bodyEtag;
if (expected && String(expected).trim() !== currentEtag) {
return sendError(res, 412, 'ETag mismatch', 'E_ETAG_MISMATCH');
}
const normalized = items.map((a) => ({
asn: String(a.domain || a.asn || '').trim(),
community: String(a.type || a.community || '').trim(),
}));
await evobgpClient.saveAsnsData(normalized, current);
return sendOk(res, { etag: null, lastModified: new Date().toISOString(), contentLength: null });
} catch (e) {
console.error(e);
return sendError(res, 500, e.message || 'EvoBGP write failed', 'E_EVOBGP');
}
};
}
function createEvobgpIpRangesPOST({ validate, validateItem }) {
return async (req, res) => {
const payload = Array.isArray(req.body?.domains)
? req.body.domains
: Array.isArray(req.body?.ipRanges)
? req.body.ipRanges
: null;
const { etag: bodyEtag } = req.body || {};
if (!Array.isArray(payload)) {
return sendError(res, 400, 'Data must be an array', 'E_BAD_REQUEST');
}
if (!validate(payload)) {
return sendError(res, 400, 'Invalid payload format', 'E_SCHEMA');
}
if (validateItem) {
const validationErrors = [];
for (let i = 0; i < payload.length; i++) {
const errors = validateItem(payload[i], i);
if (errors.length > 0) validationErrors.push(...errors);
}
if (validationErrors.length > 0) {
return sendError(res, 400, 'Validation errors', 'E_VALIDATION', {
errors: validationErrors.slice(0, 10),
});
}
}
if (!evobgpClient.isConfigured()) {
return sendError(res, 503, 'EvoBGP не настроен', 'E_EVOBGP_NOT_CONFIGURED');
}
try {
const current = await evobgpClient.fetchIpRangesData();
const currentEtag = etagForJson(current);
const ifMatch = readIfMatch(req);
const expected = ifMatch || bodyEtag;
if (expected && String(expected).trim() !== currentEtag) {
return sendError(res, 412, 'ETag mismatch', 'E_ETAG_MISMATCH');
}
const normalized = payload.map((ip) => ({
ipRange: String(ip.ipRange || '').trim(),
community: String(ip.community || '').trim(),
}));
await evobgpClient.saveIpRangesData(normalized, current);
return sendOk(res, { etag: null, lastModified: new Date().toISOString(), contentLength: null });
} catch (e) {
console.error(e);
return sendError(res, 500, e.message || 'EvoBGP write failed', 'E_EVOBGP');
}
};
}
/** Legacy /api/domains: тот же модуль ASN, ответ в полях domain + type. */
function createEvobgpLegacyDomainsGET({ validate }) {
return createEvobgpListGET({
cachePrefix: 'domains',
validate,
fetchItems: async () => {
const rows = await evobgpClient.fetchAsnsData();
return rows.map((r) => ({ domain: r.asn, type: r.community }));
},
});
}
module.exports = {
createEvobgpListGET,
createEvobgpDomainsNewPOST,
createEvobgpAsnsPOST,
createEvobgpIpRangesPOST,
createEvobgpLegacyDomainsGET,
};
backend/routes/filterRulesRoutes.js
+53
View File
@@ -0,0 +1,53 @@
/**
* GET/POST /api/filters и /api/simple-filters — данные в app_filter_rules, не в blobs.
*/
const filterRulesStorage = require('../services/filterRulesStorage');
const { sendError, sendOk, checkIfNoneMatch } = require('../middleware/errorHandler');
/**
* @param {'global'|'simple'} scope
* @param {(item: any, i: number) => string|null} validateItem
*/
function createFilterRulesRoutes(scope, validateItem) {
return {
get: async (req, res) => {
try {
const head = filterRulesStorage.headMetaForScope(scope);
if (head.etag) res.set('ETag', String(head.etag));
if (head.lastModified) {
res.set('Last-Modified', new Date(head.lastModified).toUTCString());
}
if (typeof head.contentLength === 'number') {
res.set('Content-Length-Source', String(head.contentLength));
}
if (checkIfNoneMatch(req, res, head.etag)) return;
res.json(filterRulesStorage.listRules(scope));
} catch (e) {
console.error('filterRules get', e);
return sendError(res, 500, 'Error reading storage', 'E_STORAGE');
}
},
post: async (req, res) => {
const { domains: items } = req.body;
if (!Array.isArray(items)) {
return sendError(res, 400, 'Data must be an array', 'E_BAD_REQUEST');
}
try {
const meta = filterRulesStorage.replaceRules(scope, items, { validateItem });
return sendOk(res, meta);
} catch (e) {
if (e.code === 'E_SCHEMA') {
return sendError(res, 400, e.message || 'Validation failed', 'E_SCHEMA');
}
console.error('filterRules post', e);
return sendError(res, 500, 'Error writing storage', 'E_STORAGE');
}
},
};
}
module.exports = {
createFilterRulesRoutes,
};
backend/routes/filtersRoutes.js
+16 -25
View File
@@ -2,12 +2,10 @@
* Роуты для фильтров и генерации конфигураций MikroTik
*/
const { s3, BUCKET_NAME, writeS3TextObject, headMeta } = require('../services/s3Service');
const { sendError, sendOk } = require('../middleware/errorHandler');
const { GetObjectCommand } = require('@aws-sdk/client-s3');
const { streamToString } = require('../services/s3Service');
const { writeS3TextObject } = require('../services/s3Service');
const filterRulesStorage = require('../services/filterRulesStorage');
const { sendError, checkIfNoneMatch } = require('../middleware/errorHandler');
const { generateMikrotikConfig } = require('../utils/mikrotikConfig');
const { getS3JsonWithHeaders } = require('../utils/s3Helpers');
// GET /api/filters/generate-config?format=text|json
async function generateConfig(req, res) {
@@ -16,13 +14,16 @@ async function generateConfig(req, res) {
if (!['text', 'json'].includes(format)) {
return sendError(res, 400, 'format must be "text" or "json"', 'E_BAD_REQUEST');
}
await getS3JsonWithHeaders('filters.json', req, res, {
transform: (filters) => {
const result = generateMikrotikConfig(filters, { format });
return format === 'json' ? result : { config: result };
},
defaultValue: [],
});
const head = filterRulesStorage.headMetaForScope('global');
if (head.etag) res.set('ETag', String(head.etag));
if (head.lastModified) res.set('Last-Modified', new Date(head.lastModified).toUTCString());
if (typeof head.contentLength === 'number') {
res.set('Content-Length-Source', String(head.contentLength));
}
if (checkIfNoneMatch(req, res, head.etag)) return;
const filters = filterRulesStorage.listRules('global');
const result = generateMikrotikConfig(filters, { format });
res.json(format === 'json' ? result : { config: result });
} catch (error) {
console.error(error);
return sendError(res, 500, 'Error generating configuration', 'E_S3');
@@ -32,17 +33,7 @@ async function generateConfig(req, res) {
// POST /api/filters/export-config
async function exportConfig(req, res) {
try {
const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'filters.json' }));
const fileContent = await streamToString(data.Body);
let filters = [];
try {
filters = JSON.parse(fileContent);
if (!Array.isArray(filters)) filters = [];
} catch (parseError) {
console.error('Error parsing filters.json:', parseError);
filters = [];
}
const filters = filterRulesStorage.listRules('global');
if (filters.length === 0) {
return res.json({ success: false, message: 'Нет фильтров для экспорта' });
@@ -50,10 +41,10 @@ async function exportConfig(req, res) {
const config = generateMikrotikConfig(filters);
await writeS3TextObject('mikrotik-frouting-config.txt', config);
res.json({ success: true, message: 'Конфигурация экспортирована в S3' });
res.json({ success: true, message: 'Конфигурация экспортирована в локальное хранилище' });
} catch (error) {
console.error(error);
return sendError(res, 500, 'Error exporting configuration', 'E_S3');
return sendError(res, 500, 'Error exporting configuration', 'E_STORAGE');
}
}
backend/routes/jsonDataRoutes.js
+30 -46
View File
@@ -2,14 +2,12 @@
* Общие роуты для JSON данных (servers, filters, billing и т.д.)
*/
const { s3, BUCKET_NAME, writeS3JsonObject } = require('../services/s3Service');
const { writeS3JsonObject, headMeta, readS3TextObject } = require('../services/s3Service');
const { sendError, sendOk, checkIfNoneMatch } = require('../middleware/errorHandler');
const { GetObjectCommand, HeadObjectCommand } = require('@aws-sdk/client-s3');
const { streamToString } = require('../services/s3Service');
/**
* Создать GET эндпоинт для JSON данных
* @param {string} s3Key - Ключ в S3
* @param {string} s3Key - Ключ объекта (логический, как раньше в S3)
* @param {Object} options - Опции
* @param {boolean} options.singleObject - Если true, возвращает объект вместо массива
* @param {any} options.defaultValue - Значение по умолчанию ([] для массива, {} для объекта)
@@ -17,66 +15,63 @@ const { streamToString } = require('../services/s3Service');
function createJsonDataGET(s3Key, options = {}) {
const { singleObject = false, defaultValue } = options;
const fallback = defaultValue !== undefined ? defaultValue : (singleObject ? {} : []);
return async (req, res) => {
try {
const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: s3Key })).catch(() => null);
const etag = head?.ETag || null;
const head = await headMeta(s3Key);
const etag = head?.etag || null;
if (etag) res.set('ETag', String(etag));
if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
if (head?.lastModified) res.set('Last-Modified', new Date(head.lastModified).toUTCString());
if (typeof head?.contentLength === 'number') res.set('Content-Length-Source', String(head.contentLength));
if (checkIfNoneMatch(req, res, etag)) return;
const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: s3Key }));
const fileContent = await streamToString(data.Body);
let fileContent;
try {
const data = await readS3TextObject(s3Key);
fileContent = data.body;
} catch (e) {
if (e?.code === 'NoSuchKey') {
return res.json(fallback);
}
throw e;
}
let items = fallback;
try {
const parsed = JSON.parse(fileContent);
if (singleObject) {
// Для одиночного объекта
items = (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) ? parsed : fallback;
items = typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed) ? parsed : fallback;
} else {
// Для массива
items = Array.isArray(parsed) ? parsed : fallback;
}
} catch (parseError) {
console.error(`Error parsing ${s3Key}:`, parseError);
items = fallback;
}
res.json(items);
} catch (error) {
if (error?.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) {
res.json(fallback);
} else {
console.error(error);
return sendError(res, 500, 'Error reading from S3', 'E_S3');
}
console.error(error);
return sendError(res, 500, 'Error reading storage', 'E_STORAGE');
}
};
}
/**
* Создать POST эндпоинт для JSON данных
* @param {string} s3Key - Ключ в S3
* @param {Function} validateItem - Функция валидации (для массива - валидирует каждый элемент, для объекта - весь объект)
* @param {Object} options - Опции
* @param {boolean} options.singleObject - Если true, ожидает объект вместо массива
*/
function createJsonDataPOST(s3Key, validateItem = null, options = {}) {
const { singleObject = false } = options;
return async (req, res) => {
const { domains: items } = req.body; // Используем 'domains' для обратной совместимости
const { domains: items } = req.body;
if (singleObject) {
// Для одиночного объекта
if (typeof items !== 'object' || items === null || Array.isArray(items)) {
return sendError(res, 400, 'Data must be an object', 'E_BAD_REQUEST');
}
// Валидация объекта целиком
if (validateItem) {
const error = validateItem(items);
if (error) {
@@ -84,12 +79,10 @@ function createJsonDataPOST(s3Key, validateItem = null, options = {}) {
}
}
} else {
// Для массива
if (!Array.isArray(items)) {
return sendError(res, 400, 'Data must be an array', 'E_BAD_REQUEST');
}
// Валидация каждого элемента массива
if (validateItem) {
for (let i = 0; i < items.length; i++) {
const error = validateItem(items[i], i);
@@ -105,23 +98,15 @@ function createJsonDataPOST(s3Key, validateItem = null, options = {}) {
return sendOk(res, meta);
} catch (error) {
console.error(error);
return sendError(res, 500, 'Error writing to S3', 'E_S3');
return sendError(res, 500, 'Error writing storage', 'E_STORAGE');
}
};
}
/**
* Создать роуты для JSON данных (GET + POST)
* @param {string} s3Key - Ключ в S3
* @param {Function} validateItem - Функция валидации
* @param {Object} options - Опции
* @param {boolean} options.singleObject - Если true, работает с объектом вместо массива
* @param {any} options.defaultValue - Значение по умолчанию
*/
function createJsonDataRoutes(s3Key, validateItem = null, options = {}) {
return {
get: createJsonDataGET(s3Key, options),
post: createJsonDataPOST(s3Key, validateItem, options)
post: createJsonDataPOST(s3Key, validateItem, options),
};
}
@@ -130,4 +115,3 @@ module.exports = {
createJsonDataGET,
createJsonDataPOST,
};
backend/routes/mikrotikBackupRoutes.js
+18 -19
View File
@@ -1,8 +1,8 @@
/**
* Роуты для бэкапов конфигурации MikroTik
* Хранят сырый текст конфигурации (например, вывод /export) в S3.
* Хранят сырый текст конфигурации (например, вывод /export) на локальной ФС.
*
* Ключи в S3: backups/mikrotik/{serverId}/{timestamp}-{rand}.rsc
* Логические ключи: backups/mikrotik/{serverId}/{timestamp}-{rand}.rsc
*
* Задачи:
* - создать бэкап (ручной или из планировщика)
@@ -13,7 +13,7 @@
const crypto = require('crypto');
const { sendError, sendOk } = require('../middleware/errorHandler');
const { writeS3TextObject, readS3TextObject, listS3Objects } = require('../services/s3Service');
const { writeBackupFile, readBackupFile, listBackupObjects } = require('../services/backupFsService');
const { readServersFromS3 } = require('./serversRoutes');
const { createRosClient, fetchExportViaFile } = require('../services/mikrotikApplyService');
const { encrypt, decrypt } = require('../utils/encryption');
@@ -61,7 +61,7 @@ async function saveBackupForServer(serverId, config, { source, comment } = {}) {
let content = `${headerLines.join('\n')}\n${String(config || '').replace(/^\uFEFF/, '')}`;
// Шифруем содержимое на S3, если задан ENCRYPTION_KEY
// Шифруем содержимое файла, если задан ENCRYPTION_KEY
if (process.env.ENCRYPTION_KEY) {
try {
content = ENCRYPTED_PREFIX + encrypt(content);
@@ -71,15 +71,15 @@ async function saveBackupForServer(serverId, config, { source, comment } = {}) {
}
}
const meta = await writeS3TextObject(key, content, 'text/plain');
return { key, createdAt: now.toISOString(), s3: meta };
const meta = await writeBackupFile(key, content, 'text/plain');
return { key, createdAt: now.toISOString(), storage: meta };
}
/**
* POST /api/mikrotik/backups
* Body: { serverId: string, config: string, source?: string, comment?: string }
*
* Сохраняет текст конфигурации (обычно вывод /export) в S3.
* Сохраняет текст конфигурации (обычно вывод /export) в каталог бэкапов на диске.
* Автоматический бэкап можно организовать внешним планировщиком, дергающим этот эндпоинт.
*/
async function createBackup(req, res) {
@@ -93,14 +93,14 @@ async function createBackup(req, res) {
return sendError(res, 400, 'config (MikroTik export text) is required', 'E_BAD_REQUEST');
}
const { key, createdAt, s3 } = await saveBackupForServer(serverId, config, { source, comment });
const { key, createdAt, storage } = await saveBackupForServer(serverId, config, { source, comment });
return res.json({
ok: true,
key,
serverId,
createdAt,
s3,
storage,
});
} catch (error) {
console.error('createBackup error:', error);
@@ -120,12 +120,11 @@ async function listBackups(req, res) {
return sendError(res, 400, 'serverId is required', 'E_BAD_REQUEST');
}
const prefix = `${BACKUP_PREFIX}/${String(serverId).replace(/[^a-zA-Z0-9._-]/g, '_')}/`;
const objects = await listS3Objects(prefix, { maxKeys: 200 });
const safeId = String(serverId).replace(/[^a-zA-Z0-9._-]/g, '_');
const objects = await listBackupObjects(safeId, 200);
const items = objects
.filter(o => o.key && o.key.startsWith(prefix))
.map(o => ({
.map((o) => ({
key: o.key,
serverId,
createdAt: o.lastModified,
@@ -156,7 +155,7 @@ async function getBackup(req, res) {
return sendError(res, 400, 'Invalid backup key', 'E_BAD_REQUEST');
}
const data = await readS3TextObject(key);
const data = await readBackupFile(key);
const config = decryptBackupContent(data.body || '');
return res.json({
@@ -191,8 +190,8 @@ async function diffBackups(req, res) {
}
const [a, b] = await Promise.all([
readS3TextObject(keyA),
readS3TextObject(keyB),
readBackupFile(keyA),
readBackupFile(keyB),
]);
const bodyA = decryptBackupContent(a.body || '');
@@ -265,7 +264,7 @@ async function getMikrotikClientForServerId(serverId) {
* POST /api/mikrotik/backups/run
* Body: { serverId: string }
*
* Немедленно делает /rest/export compact на указанном MikroTik и сохраняет результат в S3.
* Немедленно делает /rest/export compact на указанном MikroTik и сохраняет результат на диск.
*/
async function runBackupNow(req, res) {
try {
@@ -293,7 +292,7 @@ async function runBackupNow(req, res) {
return sendError(res, 502, 'Empty export result from MikroTik', 'E_EMPTY_EXPORT');
}
const { key, createdAt, s3 } = await saveBackupForServer(serverId, config, {
const { key, createdAt, storage } = await saveBackupForServer(serverId, config, {
source: 'manual',
comment: 'Manual backup from UI (/api/mikrotik/backups/run)',
});
@@ -303,7 +302,7 @@ async function runBackupNow(req, res) {
key,
serverId,
createdAt,
s3,
storage,
});
} catch (error) {
console.error('runBackupNow error:', error?.message, error?.stack, error?.response?.data);
backend/routes/mikrotikConfigRoutes.js
+4 -6
View File
@@ -4,8 +4,7 @@
*/
const crypto = require('crypto');
const { GetObjectCommand } = require('@aws-sdk/client-s3');
const { s3, BUCKET_NAME, streamToString, readS3TextObject, writeS3JsonObject } = require('../services/s3Service');
const { readS3TextObject, writeS3JsonObject } = require('../services/s3Service');
const { sendError, sendOk } = require('../middleware/errorHandler');
const { decrypt } = require('../utils/encryption');
const { readServersFromS3 } = require('./serversRoutes');
@@ -54,12 +53,11 @@ function speedTestCacheKey(serverId, interfaceName) {
/** Загрузить network-config из S3 */
async function loadNetworkConfig() {
try {
const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: NETWORK_CONFIG_KEY }));
const body = await streamToString(data.Body);
const parsed = JSON.parse(body || '{}');
const data = await readS3TextObject(NETWORK_CONFIG_KEY);
const parsed = JSON.parse(data.body || '{}');
return parsed && typeof parsed === 'object' ? parsed : {};
} catch (e) {
if (e?.name === 'NoSuchKey' || e?.$metadata?.httpStatusCode === 404) return {};
if (e?.code === 'NoSuchKey') return {};
throw e;
}
}
backend/routes/miscRoutes.js
+136 -163
View File
@@ -2,10 +2,18 @@
* Разные утилитарные роуты (auto-urls, history, s3 meta, availability, etc.)
*/
const { s3, BUCKET_NAME, writeS3TextObject, writeS3JsonObject, readS3TextObject, headMeta, streamToString } = require('../services/s3Service');
const {
writeS3TextObject,
writeS3JsonObject,
readS3TextObject,
headMeta,
listBlobVersionsForKey,
rollbackBlobVersion,
} = require('../services/s3Service');
const evobgpClient = require('../services/evobgpClient');
const filterRulesStorage = require('../services/filterRulesStorage');
const { sendError, sendOk } = require('../middleware/errorHandler');
const { resourceToKey, toIso } = require('../utils/helpers');
const { GetObjectCommand, PutObjectCommand, ListObjectVersionsCommand, CopyObjectCommand } = require('@aws-sdk/client-s3');
const validators = require('../lib/validators');
const net = require('net');
const http = require('http');
@@ -19,66 +27,97 @@ async function getS3LastModified(req, res) {
{ name: 'domainsNew', key: 'bgp_data/domains_community.txt' },
{ name: 'asns', key: 'bgp_data/asns.txt' },
{ name: 'servers', key: 'servers.json' },
{ name: 'filters', key: 'filters.json' },
{ name: 'filters', key: null },
{ name: 'ipRanges', key: 'bgp_data/ips.txt' },
{ name: 'uiSettings', key: 'bgp_data/rt_ui_settings.json' }
];
const results = await Promise.allSettled(
keys.map(k => headMeta(k.key))
);
const out = {};
results.forEach((r, idx) => {
const name = keys[idx].name;
if (r.status === 'fulfilled' && r.value) {
out[name] = r.value;
} else {
out[name] = null;
for (const k of keys) {
if (k.name === 'filters') {
try {
const head = filterRulesStorage.headMetaForScope('global');
out.filters = {
etag: head.etag,
lastModified: head.lastModified,
contentLength: head.contentLength,
};
} catch {
out.filters = null;
}
continue;
}
});
if (['domainsNew', 'asns', 'ipRanges'].includes(k.name) && evobgpClient.isConfigured()) {
try {
await evobgpClient.fetchRouterListsCatalog();
out[k.name] = { etag: null, lastModified: new Date().toISOString(), contentLength: null };
} catch {
out[k.name] = null;
}
continue;
}
out[k.name] = await headMeta(k.key).catch(() => null);
}
res.json(out);
} catch (error) {
console.error('Error fetching last modified dates from S3:', error);
return sendError(res, 500, 'Error fetching last modified dates from S3', 'E_S3');
console.error('Error fetching last modified dates:', error);
return sendError(res, 500, 'Error fetching last modified dates', 'E_STORAGE');
}
}
const EVOBGP_ONLY_HISTORY_RESOURCES = new Set(['domains-new', 'asns', 'ip-ranges', 'filters']);
// GET /api/history/:resource
async function getHistory(req, res) {
const { resource } = req.params;
const { countOnly, format } = req.query || {};
if (EVOBGP_ONLY_HISTORY_RESOURCES.has(resource)) {
const note =
resource === 'filters'
? 'Фильтры в SQL-таблице app_filter_rules; локальной истории отката через этот API нет'
: 'Версии хранятся в EvoBGP, локальной истории нет';
if (countOnly === 'true') {
return res.json(format === 'std' ? { items: [], total: 0, meta: { noBlobHistory: true } } : { total: 0 });
}
return res.json(
format === 'std'
? { items: [], total: 0, meta: { noBlobHistory: true, note } }
: { items: [], total: 0 },
);
}
const key = resourceToKey(resource);
if (!key) return sendError(res, 400, 'Unknown resource', 'E_RESOURCE');
try {
const out = await s3.send(new ListObjectVersionsCommand({
Bucket: BUCKET_NAME,
Prefix: key,
MaxKeys: 50
const rows = listBlobVersionsForKey(key, 50);
const versionsAll = rows.map((r) => ({
id: r.id,
object_key: r.object_key,
byte_size: r.byte_size,
etag: r.etag,
created_at: r.created_at,
}));
const versionsAll = (out.Versions || []).filter(v => v.Key === key);
if (countOnly === 'true') {
const total = versionsAll.length;
return res.json(format === 'std' ? { items: [], total, meta: {} } : { total });
}
const versions = versionsAll.slice(0, 10).map(v => ({
versionId: v.VersionId,
isLatest: v.IsLatest,
lastModified: toIso(v.LastModified),
size: v.Size,
etag: v.ETag
const versions = versionsAll.slice(0, 10).map((v, idx) => ({
versionId: String(v.id),
isLatest: idx === 0,
lastModified: toIso(v.created_at),
size: v.byte_size,
etag: v.etag,
}));
return res.json(format === 'std' ? { items: versions, total: versionsAll.length, meta: {} } : { items: versions });
} catch (e) {
console.error('history error', e);
return sendError(res, 500, 'Error reading history', 'E_S3', { error: String(e?.message || e) });
return sendError(res, 500, 'Error reading history', 'E_STORAGE', { error: String(e?.message || e) });
}
}
@@ -86,125 +125,76 @@ async function getHistory(req, res) {
async function postRollback(req, res) {
const { resource } = req.params;
const { versionId } = req.body || {};
if (EVOBGP_ONLY_HISTORY_RESOURCES.has(resource)) {
const msg =
resource === 'filters'
? 'Откат версий для фильтров не поддерживается (данные в SQL, не в blob_versions)'
: 'Откат версий для этого справочника не поддерживается (данные в EvoBGP)';
return sendError(res, 400, msg, resource === 'filters' ? 'E_HISTORY_FILTERS_SQL' : 'E_HISTORY_EV');
}
const key = resourceToKey(resource);
if (!key || !versionId) {
return sendError(res, 400, 'Bad request', 'E_BAD_REQUEST');
}
try {
await s3.send(new CopyObjectCommand({
Bucket: BUCKET_NAME,
CopySource: `/${BUCKET_NAME}/${encodeURIComponent(key)}?versionId=${encodeURIComponent(versionId)}`,
Key: key
}));
rollbackBlobVersion(key, versionId);
const meta = await headMeta(key);
return sendOk(res, meta);
} catch (e) {
console.error('rollback error', e);
return sendError(res, 500, 'Error rollback', 'E_S3', { error: String(e?.message || e) });
return sendError(res, 500, 'Error rollback', 'E_STORAGE', { error: String(e?.message || e) });
}
}
// GET /api/auto-urls
async function getAutoUrls(req, res) {
const { checkIfNoneMatch } = require('../middleware/errorHandler');
try {
const { HeadObjectCommand } = require('@aws-sdk/client-s3');
const head = await s3.send(new HeadObjectCommand({
Bucket: BUCKET_NAME,
Key: 'bgp_data/auto_url/urls.txt'
})).catch(() => null);
const etag = head?.ETag || null;
if (etag) res.set('ETag', String(etag));
if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
if (checkIfNoneMatch(req, res, etag)) return;
const data = await s3.send(new GetObjectCommand({
Bucket: BUCKET_NAME,
Key: 'bgp_data/auto_url/urls.txt'
}));
const fileContent = await streamToString(data.Body);
const urls = fileContent.split('\n').filter(line => line).map(line => {
const parts = line.trim().split(/\s+/);
return { url: parts[0] || '', community: parts[1] || '' };
});
res.json(urls);
} catch (error) {
if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) {
res.json([]);
} else {
console.error(error);
return sendError(res, 500, 'Error reading auto URLs from S3', 'E_S3');
if (!evobgpClient.isConfigured()) {
return sendError(res, 503, 'EvoBGP не настроен', 'E_EVOBGP_NOT_CONFIGURED');
}
const rows = await evobgpClient.fetchAutoUrlsData();
res.json(rows);
} catch (error) {
console.error(error);
return sendError(res, 500, 'Error reading auto URLs', 'E_EVOBGP');
}
}
// POST /api/auto-urls
async function postAutoUrls(req, res) {
const { urls } = req.body;
const fileContent = urls.map(u => `${u.url} ${u.community}`).join('\n');
if (!Array.isArray(urls)) {
return sendError(res, 400, 'urls must be an array', 'E_BAD_REQUEST');
}
try {
const meta = await writeS3TextObject('bgp_data/auto_url/urls.txt', fileContent);
return sendOk(res, meta);
if (!evobgpClient.isConfigured()) {
return sendError(res, 503, 'EvoBGP не настроен', 'E_EVOBGP_NOT_CONFIGURED');
}
await evobgpClient.saveAutoUrlsData(urls);
return sendOk(res, { etag: null, lastModified: new Date().toISOString(), contentLength: null });
} catch (error) {
console.error(error);
return sendError(res, 500, 'Error writing auto URLs to S3', 'E_S3');
return sendError(res, 500, 'Error writing auto URLs', 'E_EVOBGP');
}
}
// POST /api/auto-urls/process
async function processAutoUrls(req, res) {
try {
// Загрузка URL списков
let urls = [];
try {
const urlsData = await s3.send(new GetObjectCommand({
Bucket: BUCKET_NAME,
Key: 'bgp_data/auto_url/urls.txt'
}));
const urlsContent = await streamToString(urlsData.Body);
urls = urlsContent.split('\n').filter(line => line).map(line => {
const parts = line.trim().split(/\s+/);
return { url: parts[0] || '', community: parts[1] || '' };
});
} catch (error) {
if (error.code !== 'NoSuchKey') throw error;
if (!evobgpClient.isConfigured()) {
return sendError(res, 503, 'EvoBGP не настроен', 'E_EVOBGP_NOT_CONFIGURED');
}
const urls = await evobgpClient.fetchAutoUrlsData();
if (urls.length === 0) {
return sendError(res, 400, 'No URLs to process', 'E_BAD_REQUEST');
}
// Загрузка текущих IP и доменов
let currentIps = [];
try {
const ipsData = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt' }));
const ipsContent = await streamToString(ipsData.Body);
currentIps = ipsContent.split('\n').filter(line => line).map(line => {
const parts = line.trim().split(/\s+/);
return { ipRange: parts[0] || '', community: parts[1] || '' };
});
} catch (error) {
if (error.code !== 'NoSuchKey') throw error;
}
let currentDomains = [];
try {
const dData = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt' }));
const dContent = await streamToString(dData.Body);
currentDomains = dContent.split('\n').filter(line => line).map(line => {
const parts = line.trim().split(/\s+/);
return { domain: (parts[0] || '').toLowerCase(), community: parts[1] || '' };
});
} catch (error) {
if (error.code !== 'NoSuchKey') throw error;
}
const currentIps = await evobgpClient.fetchIpRangesData();
const currentDomains = await evobgpClient.fetchDomainsData();
// Обработка URL
const newIps = [];
@@ -256,22 +246,8 @@ async function processAutoUrls(req, res) {
const uniqueNewDomains = newDomains.filter(d => !existingDomains.has(d.domain));
const allDomains = [...currentDomains, ...uniqueNewDomains];
// Сохранение
const updatedIpsContent = allIps.map(i => `${i.ipRange} ${i.community}`).join('\n');
await s3.send(new PutObjectCommand({
Bucket: BUCKET_NAME,
Key: 'bgp_data/ips.txt',
Body: updatedIpsContent,
ContentType: 'text/plain'
}));
const updatedDomainsContent = allDomains.map(d => `${d.domain} ${d.community}`).join('\n');
await s3.send(new PutObjectCommand({
Bucket: BUCKET_NAME,
Key: 'bgp_data/domains_community.txt',
Body: updatedDomainsContent,
ContentType: 'text/plain'
}));
await evobgpClient.saveIpRangesData(allIps, currentIps);
await evobgpClient.saveDomainsData(allDomains, currentDomains);
const msg = `Обработано ${urls.length} URL, добавлено ${uniqueNewIps.length} IP и ${uniqueNewDomains.length} доменов`;
return res.json({
@@ -285,7 +261,7 @@ async function processAutoUrls(req, res) {
});
} catch (error) {
console.error('Error processing auto URLs:', error);
return sendError(res, 500, 'Error processing auto URLs', 'E_S3');
return sendError(res, 500, 'Error processing auto URLs', 'E_EVOBGP');
}
}
@@ -357,9 +333,8 @@ function getPingServicesListFromSettings(uiSettings) {
/** Загрузить UI-настройки из S3 (для pingServicesSource и др.) */
async function loadUiSettingsSync() {
try {
const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/rt_ui_settings.json' }));
const jsonText = await streamToString(data.Body);
const parsed = JSON.parse(jsonText || '{}');
const data = await readS3TextObject('bgp_data/rt_ui_settings.json');
const parsed = JSON.parse(data.body || '{}');
return parsed && typeof parsed === 'object' ? parsed : {};
} catch {
return {};
@@ -598,10 +573,10 @@ async function getServersAvailability(req, res) {
return res.json({ ...availabilityCache.data, cached: true });
}
const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'servers.json' }));
let servers = [];
try {
servers = JSON.parse(await streamToString(data.Body));
const data = await readS3TextObject('servers.json');
servers = JSON.parse(data.body);
if (!Array.isArray(servers)) servers = [];
} catch {
servers = [];
@@ -683,12 +658,13 @@ async function updateBgpBackground(req, res) {
// GET /api/ws/url
async function getWsUrl(req, res) {
try {
const settings = await s3.send(new GetObjectCommand({
Bucket: BUCKET_NAME,
Key: 'bgp_data/rt_ui_settings.json'
})).then(async (d) => {
try { return JSON.parse(await streamToString(d.Body)); } catch { return {}; }
}).catch(() => ({}));
let settings = {};
try {
const d = await readS3TextObject('bgp_data/rt_ui_settings.json');
settings = JSON.parse(d.body || '{}');
} catch (_) {
settings = {};
}
const url = settings?.wsUpdateUrl || process.env.WS_UPDATE_URL || '';
return res.json({ url });
@@ -700,25 +676,22 @@ async function getWsUrl(req, res) {
// GET/POST /api/ui-settings
async function getUiSettings(req, res) {
const { checkIfNoneMatch } = require('../middleware/errorHandler');
try {
const { HeadObjectCommand } = require('@aws-sdk/client-s3');
const head = await s3.send(new HeadObjectCommand({
Bucket: BUCKET_NAME,
Key: 'bgp_data/rt_ui_settings.json'
})).catch(() => null);
const etag = head?.ETag || null;
const head = await headMeta('bgp_data/rt_ui_settings.json');
const etag = head?.etag || null;
if (etag) res.set('ETag', String(etag));
if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
if (head?.lastModified) res.set('Last-Modified', new Date(head.lastModified).toUTCString());
if (typeof head?.contentLength === 'number') res.set('Content-Length-Source', String(head.contentLength));
if (checkIfNoneMatch(req, res, etag)) return;
const data = await s3.send(new GetObjectCommand({
Bucket: BUCKET_NAME,
Key: 'bgp_data/rt_ui_settings.json'
}));
const jsonText = await streamToString(data.Body);
let jsonText = '{}';
try {
const data = await readS3TextObject('bgp_data/rt_ui_settings.json');
jsonText = data.body;
} catch (e) {
if (e?.code !== 'NoSuchKey') throw e;
}
let settings = {};
try {
const parsed = JSON.parse(jsonText);
@@ -728,11 +701,11 @@ async function getUiSettings(req, res) {
}
return res.json(settings);
} catch (error) {
if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) {
if (error?.code === 'NoSuchKey') {
return res.json({});
}
console.error('Error reading ui settings from S3:', error);
return sendError(res, 500, 'Error reading UI settings from S3', 'E_S3');
console.error('Error reading ui settings:', error);
return sendError(res, 500, 'Error reading UI settings', 'E_STORAGE');
}
}
@@ -756,8 +729,8 @@ async function postUiSettings(req, res) {
const meta = await writeS3JsonObject('bgp_data/rt_ui_settings.json', payload);
return sendOk(res, meta);
} catch (error) {
console.error('Error writing UI settings to S3:', error);
return sendError(res, 500, 'Error writing UI settings to S3', 'E_S3', { error: String(error?.message || error) });
console.error('Error writing UI settings:', error);
return sendError(res, 500, 'Error writing UI settings', 'E_STORAGE', { error: String(error?.message || error) });
}
}
backend/routes/routeOptimizerRoutes.js
+3 -3
View File
@@ -5,6 +5,7 @@
const { sendError } = require('../middleware/errorHandler');
const { readS3TextObject, listS3Objects } = require('../services/s3Service');
const evobgpClient = require('../services/evobgpClient');
const { readServersFromS3 } = require('./serversRoutes');
const NETWORK_MAP_CACHE_KEY = 'network-map-cache/latest.json';
@@ -173,9 +174,8 @@ function findPinnedCommunityGateway(pinnedCommunityGateways, jumphost, community
async function loadCommunitiesIndex() {
try {
const raw = await readS3TextObject('bgp_data/communities.json').catch(() => null);
if (!raw?.body) return new Map();
const arr = JSON.parse(raw.body || '[]');
if (!evobgpClient.isConfigured()) return new Map();
const arr = await evobgpClient.listCommunities();
const m = new Map();
if (!Array.isArray(arr)) return m;
for (const c of arr) {
backend/routes/serverConfigsRoutes.js
+14 -13
View File
@@ -5,10 +5,8 @@
* Это нужно для EasySwitch, который работает только с jumphost серверами
*/
const { s3, BUCKET_NAME, writeS3TextObject, deleteS3Object, headMeta, writeS3JsonObject } = require('../services/s3Service');
const { writeS3TextObject, deleteS3Object, headMeta, writeS3JsonObject, readS3TextObject } = require('../services/s3Service');
const { sendError, sendOk, checkIfNoneMatch } = require('../middleware/errorHandler');
const { GetObjectCommand, HeadObjectCommand } = require('@aws-sdk/client-s3');
const { streamToString } = require('../services/s3Service');
const { getS3TextWithHeaders, getS3JsonWithHeaders } = require('../utils/s3Helpers');
const { isValidServerId } = require('../lib/validators');
@@ -41,15 +39,20 @@ function validateJumphostServer(server, i) {
// GET /api/server-configs (список только jumphost серверов)
async function getServerConfigsList(req, res) {
try {
const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: S3_KEY })).catch(() => null);
const etag = head?.ETag || null;
const head = await headMeta(S3_KEY);
const etag = head?.etag || null;
if (etag) res.set('ETag', String(etag));
if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
if (head?.lastModified) res.set('Last-Modified', new Date(head.lastModified).toUTCString());
if (typeof head?.contentLength === 'number') res.set('Content-Length-Source', String(head.contentLength));
if (checkIfNoneMatch(req, res, etag)) return;
const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: S3_KEY }));
const fileContent = await streamToString(data.Body);
let fileContent = '[]';
try {
const data = await readS3TextObject(S3_KEY);
fileContent = data.body;
} catch (e) {
if (e?.code !== 'NoSuchKey') throw e;
}
let allServers = [];
try {
@@ -113,12 +116,10 @@ async function postServerConfigsList(req, res) {
// Читаем текущие данные
let allServers = [];
try {
const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: S3_KEY }));
const fileContent = await streamToString(data.Body);
allServers = JSON.parse(fileContent);
const data = await readS3TextObject(S3_KEY);
allServers = JSON.parse(data.body);
if (!Array.isArray(allServers)) allServers = [];
} catch (e) {
// Файл не существует или пустой - это нормально
allServers = [];
}
backend/routes/serversRoutes.js
+22 -29
View File
@@ -3,8 +3,7 @@
* Для jumphost и home (входной роутер): mikrotikHost, mikrotikPort, mikrotikUser, encryptedMikrotikPassword
*/
const { GetObjectCommand, PutObjectCommand, HeadObjectCommand } = require('@aws-sdk/client-s3');
const { s3, BUCKET_NAME, streamToString } = require('../services/s3Service');
const { readS3TextObject, writeS3JsonObject, headMeta } = require('../services/s3Service');
const { sendError, sendOk, checkIfNoneMatch } = require('../middleware/errorHandler');
const { encrypt, decrypt } = require('../utils/encryption');
@@ -28,7 +27,7 @@ function validateServer(server, i) {
if (!Array.isArray(server.gateways) || server.gateways.length === 0) {
return `Server at index ${i} must have gateways for type ${normalizedType}`;
}
const primaries = server.gateways.filter(g => g && g.primary);
const primaries = server.gateways.filter((g) => g && g.primary);
if (primaries.length !== 1) {
return `Server at index ${i} must have exactly one primary gateway`;
}
@@ -45,24 +44,17 @@ function validateServer(server, i) {
async function readServersFromS3() {
try {
const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: S3_KEY }));
const body = await streamToString(data.Body);
const parsed = JSON.parse(body || '[]');
const data = await readS3TextObject(S3_KEY);
const parsed = JSON.parse(data.body || '[]');
return Array.isArray(parsed) ? parsed : [];
} catch (e) {
if (e?.name === 'NoSuchKey' || e?.$metadata?.httpStatusCode === 404) return [];
if (e?.code === 'NoSuchKey') return [];
throw e;
}
}
async function writeServersToS3(servers) {
const body = JSON.stringify(servers, null, 2);
await s3.send(new PutObjectCommand({
Bucket: BUCKET_NAME,
Key: S3_KEY,
Body: body,
ContentType: 'application/json',
}));
await writeS3JsonObject(S3_KEY, servers);
}
/**
@@ -70,15 +62,15 @@ async function writeServersToS3(servers) {
*/
async function getServers(req, res) {
try {
const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: S3_KEY })).catch(() => null);
const etag = head?.ETag || null;
const head = await headMeta(S3_KEY);
const etag = head?.etag || null;
if (etag) res.set('ETag', String(etag));
if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
if (head?.lastModified) res.set('Last-Modified', new Date(head.lastModified).toUTCString());
if (typeof head?.contentLength === 'number') res.set('Content-Length-Source', String(head.contentLength));
if (checkIfNoneMatch(req, res, etag)) return;
const servers = await readServersFromS3();
const sanitized = servers.map(s => {
const sanitized = servers.map((s) => {
const out = { ...s };
const hasMikrotikApi = s.type === 'jumphost' || s.type === 'home';
if (hasMikrotikApi && out.encryptedMikrotikPassword) {
@@ -91,11 +83,11 @@ async function getServers(req, res) {
});
res.json(sanitized);
} catch (error) {
if (error?.name === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) {
if (error?.code === 'NoSuchKey') {
return res.json([]);
}
console.error(error);
return sendError(res, 500, 'Error reading servers', 'E_S3');
return sendError(res, 500, 'Error reading servers', 'E_STORAGE');
}
}
@@ -115,9 +107,10 @@ async function postServers(req, res) {
try {
const currentServers = await readServersFromS3();
const findCurrent = (srv) => currentServers.find(c =>
(srv.id && c.id === srv.id) || (srv.dns && c.dns === srv.dns) || (srv.ip && c.ip === srv.ip)
);
const findCurrent = (srv) =>
currentServers.find(
(c) => (srv.id && c.id === srv.id) || (srv.dns && c.dns === srv.dns) || (srv.ip && c.ip === srv.ip),
);
const toSave = items.map((srv) => {
const current = findCurrent(srv);
@@ -140,15 +133,15 @@ async function postServers(req, res) {
});
await writeServersToS3(toSave);
const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: S3_KEY })).catch(() => null);
const meta = await headMeta(S3_KEY);
return sendOk(res, {
etag: head?.ETag || null,
lastModified: head?.LastModified ? head.LastModified.toISOString() : null,
contentLength: typeof head?.ContentLength === 'number' ? head.ContentLength : null,
etag: meta?.etag || null,
lastModified: meta?.lastModified || null,
contentLength: typeof meta?.contentLength === 'number' ? meta.contentLength : null,
});
} catch (error) {
console.error(error);
return sendError(res, 500, error.message || 'Error saving servers', 'E_S3');
return sendError(res, 500, error.message || 'Error saving servers', 'E_STORAGE');
}
}
backend/routes/textDataRoutes.js
+34 -69
View File
@@ -1,14 +1,10 @@
/**
* Общие роуты для текстовых данных (domains, asns, ip-ranges)
* Уменьшают дублирование кода для похожих эндпоинтов
*/
const { streamPaginatedText, headS3ObjectEtag, writeS3TextObject } = require('../services/s3Service');
const { streamPaginatedText, headS3ObjectEtag, writeS3TextObject, headMeta, readS3TextObject } = require('../services/s3Service');
const { sendError, sendOk, checkIfNoneMatch } = require('../middleware/errorHandler');
const { splitWhitespace } = require('../utils/helpers');
const { s3, BUCKET_NAME } = require('../services/s3Service');
const { GetObjectCommand, HeadObjectCommand } = require('@aws-sdk/client-s3');
const { streamToString } = require('../services/s3Service');
// Кэш для countOnly запросов
const countOnlyCache = { map: new Map(), ttlMs: 10_000 };
@@ -27,17 +23,10 @@ function setCountOnlyCache(cacheKey, value) {
countOnlyCache.map.set(cacheKey, { value, at: Date.now() });
}
/**
* Создать GET эндпоинт для текстовых данных
* @param {string} s3Key - Ключ S3 файла
* @param {function} mapLine - Функция маппинга строки в объект
* @param {function} validate - Функция валидации данных
* @param {string} cachePrefix - Префикс для кэша
*/
function createTextDataGET(s3Key, mapLine, validate, cachePrefix) {
return async (req, res) => {
const { q = '', offset, limit, countOnly, format } = req.query || {};
try {
if (countOnly === 'true') {
const cacheKey = `${cachePrefix}:count:${q}`;
@@ -54,24 +43,32 @@ function createTextDataGET(s3Key, mapLine, validate, cachePrefix) {
q,
offset: Number(offset) || 0,
limit: Number(limit) || 0,
mapLine
mapLine,
});
if (!validate(items)) {
return sendError(res, 500, 'Invalid data format', 'E_SCHEMA');
}
return res.json(format === 'std' ? { items, total, meta: {} } : { items, total });
} else {
const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: s3Key })).catch(() => null);
const etag = head?.ETag || null;
const head = await headMeta(s3Key);
const etag = head?.etag || null;
if (etag) res.set('ETag', String(etag));
if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
if (head?.lastModified) res.set('Last-Modified', new Date(head.lastModified).toUTCString());
if (typeof head?.contentLength === 'number') res.set('Content-Length-Source', String(head.contentLength));
if (checkIfNoneMatch(req, res, etag)) return;
const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: s3Key }));
const fileContent = await streamToString(data.Body);
const items = fileContent.split('\n').filter(line => line).map(line => mapLine(line.trim()));
let fileContent;
try {
const data = await readS3TextObject(s3Key);
fileContent = data.body;
} catch (e) {
if (e?.code === 'NoSuchKey') {
return res.json([]);
}
throw e;
}
const items = fileContent.split('\n').filter((line) => line).map((line) => mapLine(line.trim()));
if (!validate(items)) {
return sendError(res, 500, 'Invalid data format', 'E_SCHEMA');
}
@@ -85,24 +82,16 @@ function createTextDataGET(s3Key, mapLine, validate, cachePrefix) {
res.json([]);
} else {
console.error(error);
return sendError(res, 500, 'Error reading from S3', 'E_S3', { error: String(error?.message || error) });
return sendError(res, 500, 'Error reading storage', 'E_STORAGE', { error: String(error?.message || error) });
}
}
};
}
/**
* Создать POST эндпоинт для текстовых данных
* @param {string} s3Key - Ключ S3 файла
* @param {function} formatLine - Функция форматирования объекта в строку
* @param {function} validate - Функция валидации данных
* @param {function} validateItem - Функция валидации отдельного элемента (опционально)
*/
function createTextDataPOST(s3Key, formatLine, validate, validateItem = null) {
return async (req, res) => {
const { domains, ipRanges, items, etag } = req.body; // 'domains' основной ключ, но поддерживаем и альтернативные
// Поддержка обратной совместимости: принимаем domains | ipRanges | items
const { domains, ipRanges, items, etag } = req.body;
const payload = Array.isArray(domains)
? domains
: Array.isArray(ipRanges)
@@ -114,13 +103,11 @@ function createTextDataPOST(s3Key, formatLine, validate, validateItem = null) {
if (!Array.isArray(payload)) {
return sendError(res, 400, 'Data must be an array', 'E_BAD_REQUEST');
}
// Валидация с помощью AJV
if (!validate(payload || [])) {
return sendError(res, 400, 'Invalid payload format', 'E_SCHEMA');
}
// Дополнительная валидация элементов, если предоставлена
if (validateItem) {
const validationErrors = [];
for (let i = 0; i < payload.length; i++) {
@@ -129,62 +116,41 @@ function createTextDataPOST(s3Key, formatLine, validate, validateItem = null) {
validationErrors.push(...errors);
}
}
if (validationErrors.length > 0) {
return sendError(res, 400, 'Validation errors', 'E_VALIDATION', {
errors: validationErrors.slice(0, 10)
errors: validationErrors.slice(0, 10),
});
}
}
const fileContent = (payload || []).map(formatLine).filter(Boolean).join('\n');
try {
// Проверка ETag для optimistic concurrency
let current = null;
const ifMatch = req.headers['if-match'] ? String(req.headers['if-match']).replace(/\"/g,'"') : null;
const ifMatch = req.headers['if-match'] ? String(req.headers['if-match']).replace(/\"/g, '"') : null;
try {
current = await headS3ObjectEtag(s3Key);
} catch {}
} catch (_) {}
if (current && (ifMatch || etag) && current !== String(ifMatch || etag)) {
const { headMeta } = require('../services/s3Service');
const meta = await headMeta(s3Key);
return sendError(res, 412, 'Precondition Failed: ETag mismatch', 'E_ETAG_MISMATCH', {
currentEtag: current,
meta
});
return sendError(res, 412, 'ETag mismatch', 'E_ETAG_MISMATCH');
}
} catch {}
try {
const meta = await writeS3TextObject(s3Key, fileContent);
return sendOk(res, meta);
} catch (error) {
console.error(error);
return sendError(res, 500, 'Error writing to S3', 'E_S3', {
error: String(error?.message || error)
});
return sendError(res, 500, 'Error writing storage', 'E_STORAGE');
}
};
}
/**
* Создать роуты для текстовых данных (GET + POST)
*/
function createTextDataRoutes(config) {
const {
s3Key,
mapLine,
formatLine,
validate,
validateItem,
cachePrefix
} = config;
const { s3Key, mapLine, formatLine, validate, validateItem, cachePrefix } = config;
return {
get: createTextDataGET(s3Key, mapLine, validate, cachePrefix),
post: createTextDataPOST(s3Key, formatLine, validate, validateItem)
post: createTextDataPOST(s3Key, formatLine, validate, validateItem),
};
}
@@ -193,4 +159,3 @@ module.exports = {
createTextDataGET,
createTextDataPOST,
};