From 9de7fd104c26ac8ad7fb17535dcc6be106ddcc72 Mon Sep 17 00:00:00 2001 From: shats Date: Tue, 24 Feb 2026 14:59:12 +0700 Subject: [PATCH] feat(mikrotik): add concurrent processing for address list entries to improve performance and efficiency --- .cursor/debug-378b5f.log | 43 ++++++++++++++++++ backend/routes/mikrotikConfigRoutes.js | 62 +++++++++++++++++++------- 2 files changed, 89 insertions(+), 16 deletions(-) diff --git a/.cursor/debug-378b5f.log b/.cursor/debug-378b5f.log index 1593617..c0b99d4 100644 --- a/.cursor/debug-378b5f.log +++ b/.cursor/debug-378b5f.log @@ -65,3 +65,46 @@ {"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:response","message":"API response received","data":{"method":"GET","url":"/alerts","status":200},"timestamp":1771918635314} {"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"GET","url":"/alerts"},"timestamp":1771918694983} {"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:response","message":"API response received","data":{"method":"GET","url":"/alerts","status":200},"timestamp":1771918695315} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"GET","url":"/alerts"},"timestamp":1771919865915} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"GET","url":"/communities"},"timestamp":1771919865915} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H5","location":"frontend/src/App.jsx:route-change","message":"Route changed in MainLayout","data":{"pathname":"/easy-switch","layout":"fluid"},"timestamp":1771919865914} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"GET","url":"/servers"},"timestamp":1771919865941} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:response","message":"API response received","data":{"method":"GET","url":"/communities","status":200},"timestamp":1771919865941} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:response","message":"API response received","data":{"method":"GET","url":"/alerts","status":200},"timestamp":1771919865942} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:response","message":"API response received","data":{"method":"GET","url":"/servers","status":200},"timestamp":1771919865956} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"GET","url":"/network-config"},"timestamp":1771919865956} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:response","message":"API response received","data":{"method":"GET","url":"/network-config","status":200},"timestamp":1771919866039} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"GET","url":"/server-configs"},"timestamp":1771919866040} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"GET","url":"/server-filters/msk-ihor"},"timestamp":1771919866114} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"GET","url":"/server-filters/msk-vpsville"},"timestamp":1771919866114} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:response","message":"API response received","data":{"method":"GET","url":"/server-configs","status":200},"timestamp":1771919866113} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"GET","url":"/server-filters/srv-5"},"timestamp":1771919866114} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"GET","url":"/server-filters/msk-dc"},"timestamp":1771919866114} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"GET","url":"/server-filters/nsk-servhost"},"timestamp":1771919866114} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:response","message":"API response received","data":{"method":"GET","url":"/server-filters/msk-ihor","status":200},"timestamp":1771919866194} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:response","message":"API response received","data":{"method":"GET","url":"/server-filters/msk-vpsville","status":200},"timestamp":1771919866308} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:response","message":"API response received","data":{"method":"GET","url":"/server-filters/srv-5","status":200},"timestamp":1771919866314} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:response","message":"API response received","data":{"method":"GET","url":"/server-filters/msk-dc","status":200},"timestamp":1771919866322} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:response","message":"API response received","data":{"method":"GET","url":"/server-filters/nsk-servhost","status":200},"timestamp":1771919866323} +{"sessionId":"378b5f","runId":"post-fix","hypothesisId":"H6","location":"frontend/src/contexts/PingContext.jsx:requestPings","message":"Ping batch scheduled","data":{"scope":"easy-switch","count":20,"force":false},"timestamp":1771919866323} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866325} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866326} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866326} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866326} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866326} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866326} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866326} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866326} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866327} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866326} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866327} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866327} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866327} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866327} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866327} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866327} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866327} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866327} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866327} +{"sessionId":"378b5f","runId":"baseline","hypothesisId":"H1_H2","location":"frontend/src/lib/api.js:request","message":"API request started","data":{"method":"POST","url":"/mikrotik/ping"},"timestamp":1771919866327} +{"sessionId":"378b5f","runId":"post-fix","hypothesisId":"H6","location":"frontend/src/contexts/PingContext.jsx:requestPings","message":"Ping batch scheduled","data":{"scope":"easy-switch","count":0,"force":false},"timestamp":1771919866389} diff --git a/backend/routes/mikrotikConfigRoutes.js b/backend/routes/mikrotikConfigRoutes.js index d7b2c55..1aff825 100644 --- a/backend/routes/mikrotikConfigRoutes.js +++ b/backend/routes/mikrotikConfigRoutes.js @@ -1476,6 +1476,7 @@ async function getAddressLists(req, res) { const ADDRESS_LIST_PATH = 'ip/firewall/address-list'; const ADDRESS_LIST_REMOVE_CHUNK_SIZE = 100; +const ADDRESS_LIST_ADD_CONCURRENCY = 8; function chunkArray(items, size) { if (!Array.isArray(items) || items.length === 0) return []; @@ -1528,6 +1529,49 @@ async function removeAddressListIds(client, ids) { return removed; } +async function addAddressListEntries(client, entries) { + const normalized = [...new Map( + (Array.isArray(entries) ? entries : []) + .map((entry) => { + const address = String(entry?.address || '').trim(); + const list = String(entry?.list || 'ban').trim(); + const comment = entry?.comment != null ? String(entry.comment).trim() : ''; + if (!address || !list) return null; + const params = { address, list }; + if (comment) params.comment = comment; + const key = `${list}|${address}|${comment}`; + return [key, params]; + }) + .filter(Boolean) + ).values()]; + + if (normalized.length === 0) return { added: 0, failed: 0 }; + + let index = 0; + let added = 0; + let failed = 0; + const workersCount = Math.min(ADDRESS_LIST_ADD_CONCURRENCY, normalized.length); + + const workers = Array.from({ length: workersCount }, async () => { + while (true) { + const current = index; + index += 1; + if (current >= normalized.length) break; + const params = normalized[current]; + try { + await rosAdd(client, ADDRESS_LIST_PATH, params); + added++; + } catch (err) { + failed++; + console.warn('[address-lists] add:', err?.message); + } + } + }); + + await Promise.all(workers); + return { added, failed }; +} + /** * POST /api/mikrotik/address-lists/apply-summary * Body: { serverId, removeIds: string[], addEntries: { address, list, comment? }[] } @@ -1563,22 +1607,8 @@ async function applyAddressListSummary(req, res) { removed = await removeAddressListIds(client, ids); - for (const entry of entries) { - try { - const params = { - address: String(entry.address || '').trim(), - list: String(entry.list || 'ban').trim(), - }; - if (entry.comment != null && String(entry.comment).trim()) { - params.comment = String(entry.comment).trim(); - } - if (!params.address || !params.list) continue; - await rosAdd(client, ADDRESS_LIST_PATH, params); - added++; - } catch (err) { - console.warn('[address-lists] add:', err?.message); - } - } + const addResult = await addAddressListEntries(client, entries); + added = addResult.added; return res.json({ ok: true, removed, added }); } catch (error) {