// History endpoints are registered below after app is initialized
require('dotenv').config();
const express = require('express');
const { S3Client, GetObjectCommand, HeadObjectCommand, PutObjectCommand, DeleteObjectCommand, CopyObjectCommand, ListObjectVersionsCommand } = require('@aws-sdk/client-s3');
const { NodeHttpHandler } = require('@smithy/node-http-handler');
const cors = require('cors');
const path = require('path');
const compression = require('compression');
const Ajv = require('ajv');
const net = require('net');
const crypto = require('crypto');
const helmet = require('helmet');
const rateLimit = require('express-rate-limit');
const pino = require('pino');
const pinoHttp = require('pino-http');
const promClient = require('prom-client');
const http = require('http');
const https = require('https');
const { URL } = require('url');

const app = express();
const port = Number(process.env.PORT) || 3001;

// Logger with requestId
const logger = pino({ level: process.env.LOG_LEVEL || 'info' });
app.use(pinoHttp({
  logger,
  genReqId: (req) => req.headers['x-request-id'] || crypto.randomBytes(8).toString('hex'),
  serializers: {
    req(req) { return { id: req.id, method: req.method, url: req.url }; },
    res(res) { return { statusCode: res.statusCode }; },
  },
}));

// CORS: по умолчанию максимально разрешаем, можно сузить через CORS_ORIGINS
const allowed = (process.env.CORS_ORIGINS || '').split(',').map(s => s.trim()).filter(Boolean);
if (allowed.length > 0) {
  app.use(cors({
    origin: (origin, cb) => {
      if (!origin || allowed.includes(origin)) return cb(null, true);
      return cb(new Error('CORS blocked'));
    },
    credentials: true,
  }));
} else {
  app.use(cors({ origin: true, credentials: true }));
}
// Разрешаем preflight для всех путей
app.options('*', cors());
app.use(helmet({
  contentSecurityPolicy: false,
  crossOriginEmbedderPolicy: false,
  crossOriginOpenerPolicy: { policy: 'same-origin-allow-popups' },
  crossOriginResourcePolicy: { policy: 'cross-origin' },
}));
// Если приложение работает за прокси/ингрессом (Docker/NGINX), доверяем первому прокси для корректной работы rate-limit
app.set('trust proxy', 1);
app.disable('x-powered-by');
const limiter = rateLimit({
  windowMs: 15 * 60 * 1000,
  max: 1000,
  standardHeaders: true,
  legacyHeaders: false,
});
app.use(limiter);
app.use(express.json({ limit: process.env.JSON_LIMIT || '1mb' }));
app.use(compression());
// Disable Express auto-ETag to avoid weak ETags on JSON bodies
app.set('etag', false);

// Expose important headers to browser JS (for CORS)
app.use((req, res, next) => {
  res.setHeader('Access-Control-Expose-Headers', 'ETag, Last-Modified, Content-Length-Source');
  next();
});

// --- Metrics ---
promClient.collectDefaultMetrics();
const httpDuration = new promClient.Histogram({ name: 'http_request_duration_seconds', help: 'HTTP request duration', labelNames: ['method', 'route', 'code'], buckets: [0.05,0.1,0.2,0.5,1,2,5] });
const httpErrors = new promClient.Counter({ name: 'http_errors_total', help: 'HTTP error count', labelNames: ['route','code'] });
const s3Duration = new promClient.Histogram({ name: 's3_request_duration_seconds', help: 'S3 request duration', labelNames: ['op'], buckets: [0.01,0.05,0.1,0.2,0.5,1,2] });
const http304 = new promClient.Counter({ name: 'http_304_total', help: 'HTTP 304 responses' });
const http412 = new promClient.Counter({ name: 'http_412_total', help: 'HTTP 412 responses' });
const http423 = new promClient.Counter({ name: 'http_423_total', help: 'HTTP 423 responses' });
app.use((req, res, next) => {
  const start = process.hrtime.bigint();
  res.on('finish', () => {
    try {
      const dur = Number(process.hrtime.bigint() - start) / 1e9;
      httpDuration.labels(req.method, req.route?.path || req.path, String(res.statusCode)).observe(dur);
      if (res.statusCode >= 400) httpErrors.labels(req.route?.path || req.path, String(res.statusCode)).inc();
    } catch {}
  });
  next();
});

app.get('/health', (req, res) => res.json({ ok: true }));
app.get('/ready', (req, res) => res.json({ ok: true }));
app.get('/metrics', async (req, res) => {
  try {
    res.set('Content-Type', promClient.register.contentType);
    res.end(await promClient.register.metrics());
  } catch (e) {
    res.status(500).end(String(e?.message || e));
  }
});

// Version endpoint (static env-based)
app.get('/api/version', (req, res) => {
  res.json({
    version: process.env.APP_VERSION || null,
    gitSha: process.env.GIT_SHA || null,
    buildAt: process.env.BUILD_AT || null
  });
});

// Централизованный обработчик ошибок (должен быть подключён ПОСЛЕ роутов — см. ниже второе use)

// Helpers: meta and responses
function toIso(x) {
  try { return new Date(x).toISOString(); } catch { return null; }
}

async function headMeta(key) {
  try {
    const h = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: key }));
    return {
      etag: h.ETag || null,
      lastModified: h.LastModified ? toIso(h.LastModified) : null,
      contentLength: typeof h.ContentLength === 'number' ? h.ContentLength : null,
    };
  } catch (e) {
    return { etag: null, lastModified: null, contentLength: null };
  }
}

function sendOk(res, meta) {
  if (meta?.etag) res.set('ETag', String(meta.etag));
  if (meta?.lastModified) res.set('Last-Modified', new Date(meta.lastModified).toUTCString());
  if (typeof meta?.contentLength === 'number') res.set('Content-Length-Source', String(meta.contentLength));
  return res.json({ ok: true, etag: meta?.etag || null, lastModified: meta?.lastModified || null, contentLength: meta?.contentLength ?? null });
}

function sendError(res, status, message, code, details) {
  const requestId = res.req?.id;
  try { if (status === 304) http304.inc(); if (status === 412) http412.inc(); if (status === 423) http423.inc(); } catch {}
  return res.status(status).json({ code, message, details, requestId });
}

// Map UI resource -> S3 key (for history endpoints)
function resourceToKey(resource) {
  switch (resource) {
    case 'domains-new': return 'bgp_data/domains_community.txt';
    case 'ip-ranges': return 'bgp_data/ips.txt';
    case 'asns': return 'bgp_data/asns.txt';
    default: return null;
  }
}

// Compute sha256 of string
function sha256OfString(s) {
  return crypto.createHash('sha256').update(Buffer.from(String(s), 'utf-8')).digest('hex');
}

function mapAjvErrors(errors) {
  if (!Array.isArray(errors)) return [];
  return errors.map((e) => ({
    message: e.message,
    instancePath: e.instancePath,
    keyword: e.keyword,
    params: e.params,
  }));
}

// Helpers
function splitWhitespace(line) {
  return String(line || '').trim().split(/\s+/);
}

// Serve static files from the React app
app.use(express.static(path.join(__dirname, 'public')));

// Configure AWS S3 (SDK v3)
const s3 = new S3Client({
    endpoint: 'https://storage.yandexcloud.net',
    region: process.env.AWS_REGION,
    forcePathStyle: true,
    maxAttempts: 3,
    requestHandler: new NodeHttpHandler({
      httpAgent: new http.Agent({ keepAlive: true }),
      httpsAgent: new https.Agent({ keepAlive: true })
    }),
    credentials: {
        accessKeyId: process.env.AWS_ACCESS_KEY_ID || process.env.S3_ACCESS_KEY_ID,
        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || process.env.S3_SECRET_ACCESS_KEY,
    }
});
// Default Cache-Control for GETs
const DEFAULT_CACHE_TTL = Math.max(0, Math.min(300, Number(process.env.CACHE_TTL_SECONDS) || 30));
app.use((req, res, next) => {
  if (req.method === 'GET') {
    res.set('Cache-Control', `private, max-age=${DEFAULT_CACHE_TTL}`);
  }
  next();
});


const BUCKET_NAME = process.env.S3_BUCKET_NAME;
const FILE_KEY = 'bgp_data/domains.txt';

// AJV setup and schemas
const ajv = new Ajv({ allErrors: true, removeAdditional: 'failing' });

const schemaDomainsNew = {
  type: 'array',
  items: {
    type: 'object',
    required: ['domain', 'community'],
    additionalProperties: false,
    properties: {
      domain: { type: 'string' },
      community: { type: 'string' }
    }
  }
};
const schemaAsns = {
  type: 'array',
  items: {
    type: 'object',
    required: ['domain', 'type'],
    additionalProperties: false,
    properties: {
      domain: { type: 'string' },
      type: { type: 'string' }
    }
  }
};
const schemaIpRanges = {
  type: 'array',
  items: {
    type: 'object',
    required: ['ipRange', 'community'],
    additionalProperties: false,
    properties: {
      ipRange: { type: 'string' },
      community: { type: 'string' }
    }
  }
};
const schemaFilters = {
  type: 'array',
  items: {
    type: 'object',
    required: ['community', 'gateway'],
    additionalProperties: true,
    properties: {
      community: { type: 'string' },
      gateway: { type: 'string' },
      description: { type: 'string' }
    }
  }
};
const schemaServers = {
  type: 'array',
  items: {
    type: 'object',
    required: ['ip', 'dns', 'country', 'provider', 'tunnel'],
    additionalProperties: true,
    properties: {
      ip: { type: 'string' },
      dns: { type: 'string' },
      country: { type: 'string' },
      provider: { type: 'string' },
      tunnel: { type: 'string' },
      gateway: { type: 'string' }
    }
  }
};
const schemaBilling = {
  type: 'array',
  items: {
    type: 'object',
    required: ['hostName', 'country', 'provider'],
    additionalProperties: true,
    properties: {
      hostName: { type: 'string' },
      country: { type: 'string' },
      provider: { type: 'string' }
    }
  }
};

const validateDomainsNew = ajv.compile(schemaDomainsNew);
const validateAsns = ajv.compile(schemaAsns);
const validateIpRanges = ajv.compile(schemaIpRanges);
const validateFilters = ajv.compile(schemaFilters);
const validateServers = ajv.compile(schemaServers);
const validateBilling = ajv.compile(schemaBilling);

// Helper: build MikroTik nested if/else blocks (RouterOS v7 filter language does not support 'else if')
function buildNestedGatewayBlocks(gatewayGroups, baseIndentSpaces = 4) {
    const indent = (n) => ' '.repeat(n);
    const entries = Object.entries(gatewayGroups);
    if (entries.length === 0) return '';

    function buildAt(index, pad) {
        const [gateway, communities] = entries[index];
        let s = '';
        s += `${indent(pad)}if (\n`;
        communities.forEach((community, i) => {
            s += `${indent(pad + 4)}(bgp-communities includes ${community})`;
            if (i < communities.length - 1) s += ' or \n';
        });
        s += `\n${indent(pad)})\n`;
        s += `${indent(pad)}{\n${indent(pad + 8 - 4)}set gw ${gateway}; accept;\n${indent(pad)}}\n`;
        if (index < entries.length - 1) {
            s += `${indent(pad)}else\n${indent(pad)}{\n`;
            s += buildAt(index + 1, pad + 4);
            s += `\n${indent(pad)}}`;
        } else {
            s += `${indent(pad)}else\n${indent(pad)}{\n${indent(pad + 4)}reject;\n${indent(pad)}}`;
        }
        return s;
    }

    return buildAt(0, baseIndentSpaces);
}

// In-memory LRU-ish cache for small texts and head meta
const s3Cache = { text: new Map(), head: new Map(), max: 100, ttlMs: 30_000 };
const countOnlyCache = { map: new Map(), ttlMs: 10_000 };
function getCache(map, key) {
  const v = map.get(key);
  if (!v) return null;
  if (Date.now() > v.at + s3Cache.ttlMs) { map.delete(key); return null; }
  return v.value;
}
function setCache(map, key, value) {
  if (map.size >= s3Cache.max) { const firstKey = map.keys().next().value; if (firstKey) map.delete(firstKey); }
  map.set(key, { value, at: Date.now() });
}

function invalidateCacheForKey(key) {
  try { s3Cache.text.delete(key); } catch {}
  try { s3Cache.head.delete(key); } catch {}
}

function getCountOnlyCache(cacheKey) {
  const v = countOnlyCache.map.get(cacheKey);
  if (!v) return null;
  if (Date.now() > v.at + countOnlyCache.ttlMs) { countOnlyCache.map.delete(cacheKey); return null; }
  return v.value;
}
function setCountOnlyCache(cacheKey, value) {
  countOnlyCache.map.set(cacheKey, { value, at: Date.now() });
}

function checkIfNoneMatch(req, res, etag) {
  const inm = req.headers && (req.headers['if-none-match'] || req.headers['If-None-Match']);
  if (inm && etag && String(inm) === String(etag)) {
    try { http304.inc(); } catch {}
    res.status(304).end();
    return true;
  }
  return false;
}

// Helper: read text file from S3 and return { body, etag, lastModified, contentLength }
async function streamToString(stream) {
    if (!stream) return '';
    if (typeof stream.transformToString === 'function') {
        return await stream.transformToString();
    }
    return await new Promise((resolve, reject) => {
        let chunks = [];
        stream.on('data', (c) => chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(String(c))));
        stream.once('error', reject);
        stream.once('end', () => resolve(Buffer.concat(chunks).toString('utf-8')));
    });
}

async function readS3TextObject(key) {
    const cached = getCache(s3Cache.text, key);
    if (cached) return cached;
    const s3Start = Date.now();
    const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: key }));
    try { s3Duration.labels('getObject').observe((Date.now() - s3Start)/1000); } catch {}
    const out = {
        body: await streamToString(data.Body),
        etag: data.ETag || undefined,
        lastModified: data.LastModified ? data.LastModified.toISOString() : undefined,
        contentLength: typeof data.ContentLength === 'number' ? data.ContentLength : undefined
    };
    setCache(s3Cache.text, key, out);
    return out;
}

// Helper: head object and return current ETag
async function headS3ObjectEtag(key) {
    const cached = getCache(s3Cache.head, key);
    if (cached && cached.etag) return cached.etag;
    const s3Start = Date.now();
    const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: key }));
    try { s3Duration.labels('headObject').observe((Date.now() - s3Start)/1000); } catch {}
    setCache(s3Cache.head, key, { etag: head.ETag || undefined });
    return head.ETag || undefined;
}

// Helper: stream and paginate big text files (line-based)
async function streamPaginatedText({ key, mapLine, q, offset = 0, limit = 0 }) {
  return new Promise(async (resolve, reject) => {
    let total = 0;
    const items = [];
    let sent = 0;
    let buffered = '';
    const matchesQuery = (line) => {
      if (!q) return true;
      return line.toLowerCase().includes(String(q).toLowerCase());
    };
    try {
      const resp = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: key }));
      const stream = resp.Body;
      if (!stream || typeof stream.on !== 'function') {
        const text = await streamToString(resp.Body);
        const lines = text.split('\n');
        for (let i = 0; i < lines.length; i++) {
          const line = String(lines[i] || '').trim();
          if (!line) continue;
          if (!matchesQuery(line)) continue;
          total++;
          const pos = total - 1;
          if (limit > 0) {
            if (pos >= offset && sent < limit) { items.push(mapLine(line)); sent++; }
          } else {
            items.push(mapLine(line));
          }
        }
        return resolve({ items, total });
      }
      stream.on('data', (chunk) => {
        buffered += chunk.toString('utf-8');
        let lines = buffered.split('\n');
        buffered = lines.pop();
        for (const lnRaw of lines) {
          const line = lnRaw.trim();
          if (!line) continue;
          if (!matchesQuery(line)) continue;
          total++;
          const pos = total - 1;
          if (limit > 0) {
            if (pos >= offset && sent < limit) { items.push(mapLine(line)); sent++; }
          } else {
            items.push(mapLine(line));
          }
          // Не прерываем поток раньше конца, чтобы корректно посчитать total
        }
      });
      stream.on('end', () => {
        const last = (buffered || '').trim();
        if (last) {
          if (!q || last.toLowerCase().includes(String(q).toLowerCase())) {
            total++;
            if (limit > 0) {
              const pos = total - 1;
              if (pos >= offset && items.length < limit) items.push(mapLine(last));
            } else {
              items.push(mapLine(last));
            }
          }
        }
        resolve({ items, total });
      });
      stream.on('error', reject);
    } catch (e) {
      reject(e);
    }
  });
}

// Simple in-memory soft locks with TTL
const locks = new Map(); // key -> { owner, expiresAt }
function cleanupExpiredLocks() {
    const now = Date.now();
    for (const [k, v] of locks.entries()) {
        if (!v || typeof v.expiresAt !== 'number' || v.expiresAt <= now) {
            locks.delete(k);
        }
    }
}
setInterval(cleanupExpiredLocks, 30_000);

// Get domains from S3 (supports countOnly/std and If-None-Match)
app.get('/api/domains', async (req, res) => {
    const { q = '', offset, limit, countOnly, format } = req.query || {};
    try {
        if (countOnly === 'true') {
            const cacheKey = `domains:count:${q}`;
            const cached = getCountOnlyCache(cacheKey);
            if (cached != null) return res.json(format === 'std' ? { items: [], total: cached, meta: {} } : { total: cached });
            const { total } = await streamPaginatedText({ key: FILE_KEY, q, offset: 0, limit: 0, mapLine: () => ({}) });
            setCountOnlyCache(cacheKey, total);
            return res.json(format === 'std' ? { items: [], total, meta: {} } : { total });
        } else if (Number(limit) > 0) {
            const { items, total } = await streamPaginatedText({
                key: FILE_KEY,
                q,
                offset: Number(offset) || 0,
                limit: Number(limit) || 0,
                mapLine: (line) => {
                    const parts = splitWhitespace(line);
                    return { domain: parts[0] || '', type: parts[1] || '' };
                }
            });
            if (!validateAsns(items)) return sendError(res, 500, 'Invalid data format', 'E_SCHEMA');
            return res.json(format === 'std' ? { items, total, meta: {} } : { items, total });
        } else {
            const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: FILE_KEY })).catch(() => null);
            const etag = head?.ETag || null;
            if (etag) res.set('ETag', String(etag));
            if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
            if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
            if (checkIfNoneMatch(req, res, etag)) return; // 304

            const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: FILE_KEY }));
            const fileContent = await streamToString(data.Body);
            const domains = fileContent.split('\n').filter(line => line).map(line => {
                const parts = splitWhitespace(line);
                const domain = parts[0] || '';
                const type = parts[1] || '';
                return { domain, type };
            });
            if (format === 'std') return res.json({ items: domains, total: domains.length, meta: {} });
            res.json(domains);
        }
    } catch (error) {
        if (error?.name === 'NotFound' || error?.$metadata?.httpStatusCode === 404) {
            res.json([]);
        } else {
            console.error(error);
            return sendError(res, 500, 'Error reading from S3', 'E_S3', { error: String(error?.message || error) });
        }
    }
});

// Update domains in S3 with optimistic concurrency via ETag check
app.post('/api/domains', async (req, res) => {
    const { domains, etag } = req.body;
    const fileContent = (domains || []).map(d => `${String(d.domain || '').trim()} ${String(d.type || '').trim()}`.trim()).filter(Boolean).join('\n');

    // Concurrency guard: if client sent etag, ensure current ETag matches
    try {
        if (etag) {
            const current = await headS3ObjectEtag(FILE_KEY).catch(() => undefined);
            if (current && current.replace(/\"/g, '"') !== String(etag)) {
                const meta = await headMeta(FILE_KEY);
                return sendError(res, 412, 'Precondition Failed: ETag mismatch', 'E_ETAG_MISMATCH', { currentEtag: current, meta });
            }
        }
    } catch (e) {
        // ignore if head fails due to NoSuchKey; proceed to create
    }

    const params = {
        Bucket: BUCKET_NAME,
        Key: FILE_KEY,
        Body: fileContent,
        ContentType: 'text/plain',
    };

    try {
        await s3.send(new PutObjectCommand(params));
        invalidateCacheForKey(FILE_KEY);
        const meta = await headMeta(FILE_KEY);
        return sendOk(res, meta);
    } catch (error) {
        console.error(error);
        return sendError(res, 500, 'Error writing to S3', 'E_S3');
    }
});

// --- ASNs Routes ---

// Get ASNs from S3
app.get('/api/asns', async (req, res) => {
    const { q = '', offset, limit, countOnly, format } = req.query || {};
    const params = {
        Bucket: BUCKET_NAME,
        Key: 'bgp_data/asns.txt',
    };

    try {
        if (countOnly === 'true') {
            const cacheKey = `asns:count:${q}`;
            const cached = getCountOnlyCache(cacheKey);
            if (cached != null) return res.json(format === 'std' ? { items: [], total: cached, meta: {} } : { total: cached });
            const { total } = await streamPaginatedText({ key: 'bgp_data/asns.txt', q, offset: 0, limit: 0, mapLine: () => ({}) });
            setCountOnlyCache(cacheKey, total);
            return res.json(format === 'std' ? { items: [], total, meta: {} } : { total });
        } else if (Number(limit) > 0) {
            const { items, total } = await streamPaginatedText({
                key: 'bgp_data/asns.txt',
                q,
                offset: Number(offset) || 0,
                limit: Number(limit) || 0,
                mapLine: (line) => {
                    const parts = line.trim().split(/\s+/);
                    return { domain: parts[0] || '', type: parts[1] || '' };
                }
            });
            if (!validateAsns(items)) return sendError(res, 500, 'Invalid data format', 'E_SCHEMA');
            return res.json(format === 'std' ? { items, total, meta: {} } : { items, total });
        } else {
            const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/asns.txt' })).catch(() => null);
            const etag = head?.ETag || null;
            if (etag) res.set('ETag', String(etag));
            if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
            if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
            if (checkIfNoneMatch(req, res, etag)) return;

            const data = await s3.send(new GetObjectCommand(params));
            const fileContent = await streamToString(data.Body);
            const asns = fileContent.split('\n').filter(line => line).map(line => {
                const parts = line.trim().split(/\s+/);
                const domain = parts[0] || '';
                const type = parts[1] || '';
                return { domain, type };
            });
            if (!validateAsns(asns)) return sendError(res, 500, 'Invalid data format', 'E_SCHEMA');
            if (format === 'std') return res.json({ items: asns, total: asns.length, meta: {} });
            res.json(asns);
        }
    } catch (error) {
        if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) {
            res.json([]);
        } else {
            console.error(error);
            return sendError(res, 500, 'Error reading from S3', 'E_S3');
        }
    }
});

// Update ASNs in S3
app.post('/api/asns', async (req, res) => {
    const { domains: asns, etag } = req.body; // Keep name 'domains' for consistency
    const fileContent = (asns || []).map(a => `${String(a.domain || '').trim()} ${String(a.type || '').trim()}`.trim()).filter(Boolean).join('\n');

    try {
        if (!validateAsns(asns || [])) {
            return sendError(res, 400, 'Invalid payload format for asns', 'E_SCHEMA');
        }
        let current = null;
        const ifMatch = req.headers['if-match'] ? String(req.headers['if-match']).replace(/\"/g,'"') : null;
        try { current = await headS3ObjectEtag('bgp_data/asns.txt'); } catch {}
        if (current && (ifMatch || etag) && current !== String(ifMatch || etag)) {
            const meta = await headMeta('bgp_data/asns.txt');
            return sendError(res, 412, 'Precondition Failed: ETag mismatch', 'E_ETAG_MISMATCH', { currentEtag: current, meta });
        }
    } catch {}

    const params = {
        Bucket: BUCKET_NAME,
        Key: 'bgp_data/asns.txt',
        Body: fileContent,
        ContentType: 'text/plain',
    };

    try {
        await s3.send(new PutObjectCommand(params));
        invalidateCacheForKey('bgp_data/asns.txt');
        const meta = await headMeta('bgp_data/asns.txt');
        return sendOk(res, meta);
    } catch (error) {
        console.error(error);
        return sendError(res, 500, 'Error writing to S3', 'E_S3', { error: String(error?.message || error) });
    }
});

// --- Domains New Routes ---

// Get domains-new from S3
app.get('/api/domains-new', async (req, res) => {
    const { q = '', offset, limit, countOnly, format } = req.query || {};
    const params = {
        Bucket: BUCKET_NAME,
        Key: 'bgp_data/domains_community.txt',
    };

    try {
        if (countOnly === 'true') {
            const cacheKey = `domains-new:count:${q}`;
            const cached = getCountOnlyCache(cacheKey);
            if (cached != null) return res.json(format === 'std' ? { items: [], total: cached, meta: {} } : { total: cached });
            const { total } = await streamPaginatedText({ key: 'bgp_data/domains_community.txt', q, offset: 0, limit: 0, mapLine: () => ({}) });
            setCountOnlyCache(cacheKey, total);
            return res.json(format === 'std' ? { items: [], total, meta: {} } : { total });
        } else if (Number(limit) > 0) {
            const { items, total } = await streamPaginatedText({
                key: 'bgp_data/domains_community.txt',
                q,
                offset: Number(offset) || 0,
                limit: Number(limit) || 0,
                mapLine: (line) => {
                    const parts = line.trim().split(/\s+/);
                    return { domain: parts[0] || '', community: parts[1] || '' };
                }
            });
            if (!validateDomainsNew(items)) return sendError(res, 500, 'Invalid data format', 'E_SCHEMA');
            return res.json(format === 'std' ? { items, total, meta: {} } : { items, total });
        } else {
            const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt' })).catch(() => null);
            const etag = head?.ETag || null;
            if (etag) res.set('ETag', String(etag));
            if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
            if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
            if (checkIfNoneMatch(req, res, etag)) return;

            const data = await s3.send(new GetObjectCommand(params));
            const fileContent = await streamToString(data.Body);
            const domains = fileContent.split('\n').filter(line => line).map(line => {
                const parts = line.trim().split(/\s+/);
                const domain = parts[0] || '';
                const community = parts[1] || '';
                return { domain, community };
            });
            if (!validateDomainsNew(domains)) return sendError(res, 500, 'Invalid data format', 'E_SCHEMA');
            if (format === 'std') return res.json({ items: domains, total: domains.length, meta: {} });
            res.json(domains);
        }
    } catch (error) {
        if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) {
            res.json([]); // Return empty array if file does not exist
        } else {
            console.error(error);
            return sendError(res, 500, 'Error reading from S3', 'E_S3');
        }
    }
});

// Update domains-new in S3
app.post('/api/domains-new', async (req, res) => {
    const { domains, etag } = req.body;
    const fileContent = (domains || []).map(d => `${String(d.domain || '').trim()} ${String(d.community || '').trim()}`.trim()).filter(Boolean).join('\n');

    try {
        if (!validateDomainsNew(domains || [])) {
            return sendError(res, 400, 'Invalid payload format for domains-new', 'E_SCHEMA');
        }
        let current = null;
        const ifMatch = req.headers['if-match'] ? String(req.headers['if-match']).replace(/\"/g,'"') : null;
        try { current = await headS3ObjectEtag('bgp_data/domains_community.txt'); } catch {}
        if (current && (ifMatch || etag) && current !== String(ifMatch || etag)) {
            const meta = await headMeta('bgp_data/domains_community.txt');
            return sendError(res, 412, 'Precondition Failed: ETag mismatch', 'E_ETAG_MISMATCH', { currentEtag: current, meta });
        }
    } catch {}

    const params = {
        Bucket: BUCKET_NAME,
        Key: 'bgp_data/domains_community.txt',
        Body: fileContent,
        ContentType: 'text/plain',
    };

    try {
        await s3.send(new PutObjectCommand(params));
        invalidateCacheForKey('bgp_data/domains_community.txt');
        const meta = await headMeta('bgp_data/domains_community.txt');
        return sendOk(res, meta);
    } catch (error) {
        console.error(error);
        return sendError(res, 500, 'Error writing to S3', 'E_S3', { error: String(error?.message || error) });
    }
});

// --- IP Ranges Routes ---

// Get IP ranges from S3
app.get('/api/ip-ranges', async (req, res) => {
    const { q = '', offset, limit, countOnly, format } = req.query || {};
    const params = {
        Bucket: BUCKET_NAME,
        Key: 'bgp_data/ips.txt',
    };

    try {
        if (countOnly === 'true') {
            const cacheKey = `ip-ranges:count:${q}`;
            const cached = getCountOnlyCache(cacheKey);
            if (cached != null) return res.json(format === 'std' ? { items: [], total: cached, meta: {} } : { total: cached });
            const { total } = await streamPaginatedText({ key: 'bgp_data/ips.txt', q, offset: 0, limit: 0, mapLine: () => ({}) });
            setCountOnlyCache(cacheKey, total);
            return res.json(format === 'std' ? { items: [], total, meta: {} } : { total });
        } else if (Number(limit) > 0) {
            const { items, total } = await streamPaginatedText({
                key: 'bgp_data/ips.txt',
                q,
                offset: Number(offset) || 0,
                limit: Number(limit) || 0,
                mapLine: (line) => {
                    const parts = line.trim().split(/\s+/);
                    return { ipRange: parts[0] || '', community: parts[1] || '' };
                }
            });
            if (!validateIpRanges(items)) return sendError(res, 500, 'Invalid data format', 'E_SCHEMA');
            return res.json(format === 'std' ? { items, total, meta: {} } : { items, total });
        } else {
            const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt' })).catch(() => null);
            const etag = head?.ETag || null;
            if (etag) res.set('ETag', String(etag));
            if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
            if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
            if (checkIfNoneMatch(req, res, etag)) return;

            const data = await s3.send(new GetObjectCommand(params));
            const fileContent = await streamToString(data.Body);
            const ipRanges = fileContent.split('\n').filter(line => line).map(line => {
                const parts = line.trim().split(/\s+/);
                const ipRange = parts[0] || '';
                const community = parts[1] || '';
                return { ipRange, community };
            });
            if (!validateIpRanges(ipRanges)) return sendError(res, 500, 'Invalid data format', 'E_SCHEMA');
            if (format === 'std') return res.json({ items: ipRanges, total: ipRanges.length, meta: {} });
            res.json(ipRanges);
        }
    } catch (error) {
        if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) {
            res.json([]); // Return empty array if file does not exist
        } else {
            console.error(error);
            return sendError(res, 500, 'Error reading from S3', 'E_S3');
        }
    }
});

// Update IP ranges in S3
app.post('/api/ip-ranges', async (req, res) => {
    const { ipRanges, etag } = req.body;
    const fileContent = (ipRanges || []).map(ip => `${String(ip.ipRange || '').trim()} ${String(ip.community || '').trim()}`.trim()).filter(Boolean).join('\n');

    try {
        if (!validateIpRanges(ipRanges || [])) {
            return sendError(res, 400, 'Invalid payload format for ip-ranges', 'E_SCHEMA');
        }
        let current = null;
        const ifMatch = req.headers['if-match'] ? String(req.headers['if-match']).replace(/\"/g,'"') : null;
        try { current = await headS3ObjectEtag('bgp_data/ips.txt'); } catch {}
        if (current && (ifMatch || etag) && current !== String(ifMatch || etag)) {
            const meta = await headMeta('bgp_data/ips.txt');
            return sendError(res, 412, 'Precondition Failed: ETag mismatch', 'E_ETAG_MISMATCH', { currentEtag: current, meta });
        }
    } catch {}

    const params = {
        Bucket: BUCKET_NAME,
        Key: 'bgp_data/ips.txt',
        Body: fileContent,
        ContentType: 'text/plain',
    };

    try {
        await s3.send(new PutObjectCommand(params));
        invalidateCacheForKey('bgp_data/ips.txt');
        const meta = await headMeta('bgp_data/ips.txt');
        return sendOk(res, meta);
    } catch (error) {
        console.error(error);
        return sendError(res, 500, 'Error writing to S3', 'E_S3', { error: String(error?.message || error) });
    }
});

// --- Communities Directory Routes ---

// Get communities from S3
app.get('/api/communities', async (req, res) => {
    const params = {
        Bucket: BUCKET_NAME,
        Key: 'bgp_data/communities.json',
    };

    try {
        const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/communities.json' })).catch(() => null);
        const etag = head?.ETag || null;
        if (etag) res.set('ETag', String(etag));
        if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
        if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
        if (checkIfNoneMatch(req, res, etag)) return;

        const data = await s3.send(new GetObjectCommand(params));
        const fileContent = await streamToString(data.Body);
        let communities = [];

        try {
            const parsed = JSON.parse(fileContent);
            communities = Array.isArray(parsed) ? parsed : [];
        } catch (parseError) {
            console.error('Error parsing communities.json:', parseError);
            communities = [];
        }

        // Basic normalization
        communities = communities
            .filter((c) => c && typeof c.value === 'string' && c.value.trim().length > 0)
            .map((c) => ({
                value: String(c.value).trim(),
                name: c.name ? String(c.name) : '',
                description: c.description ? String(c.description) : '',
                tags: Array.isArray(c.tags) ? c.tags.map(String) : [],
                gatewayDefault: c.gatewayDefault ? String(c.gatewayDefault) : '',
                color: c.color ? String(c.color) : ''
            }));

        res.json(communities);
    } catch (error) {
        if (error.code === 'NoSuchKey') {
            // If file missing, start with empty list
            return res.json([]);
        }
        console.error('Error reading communities from S3:', error);
        return sendError(res, 500, 'Error reading communities from S3', 'E_S3');
    }
});

// Update communities in S3
app.post('/api/communities', async (req, res) => {
    const { communities } = req.body;

    if (!Array.isArray(communities)) {
        return sendError(res, 400, 'communities must be an array', 'E_BAD_REQUEST');
    }

    // Validate entries and ensure unique values
    const seen = new Set();
    const normalized = [];
    for (let i = 0; i < communities.length; i++) {
        const entry = communities[i] || {};
        const value = typeof entry.value === 'string' ? entry.value.trim() : '';
        if (!value) {
            return sendError(res, 400, `Community at index ${i} is missing required field: value`, 'E_SCHEMA');
        }
        if (seen.has(value)) {
            return sendError(res, 400, `Duplicate community value at index ${i}: ${value}`, 'E_SCHEMA');
        }
        seen.add(value);
        normalized.push({
            value,
            name: entry.name ? String(entry.name) : '',
            description: entry.description ? String(entry.description) : '',
            tags: Array.isArray(entry.tags) ? entry.tags.map(String) : [],
            gatewayDefault: entry.gatewayDefault ? String(entry.gatewayDefault) : '',
            color: entry.color ? String(entry.color) : ''
        });
    }

    const params = {
        Bucket: BUCKET_NAME,
        Key: 'bgp_data/communities.json',
        Body: JSON.stringify(normalized, null, 2),
        ContentType: 'application/json',
    };

    try {
        await s3.send(new PutObjectCommand(params));
        invalidateCacheForKey('bgp_data/communities.json');
        const meta = await headMeta('bgp_data/communities.json');
        return sendOk(res, meta);
    } catch (error) {
        console.error('Error writing communities to S3:', error);
        return sendError(res, 500, 'Error writing communities to S3', 'E_S3');
    }
});

// --- Servers Routes (JSON format) ---

// Get servers from S3
app.get('/api/servers', async (req, res) => {
    const params = {
        Bucket: BUCKET_NAME,
        Key: 'servers.json',
    };

    try {
        const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'servers.json' })).catch(() => null);
        const etag = head?.ETag || null;
        if (etag) res.set('ETag', String(etag));
        if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
        if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
        if (checkIfNoneMatch(req, res, etag)) return;

        const data = await s3.send(new GetObjectCommand(params));
        const fileContent = await streamToString(data.Body);
        let servers = [];
        
        try {
            servers = JSON.parse(fileContent);
            // Ensure it's an array
            if (!Array.isArray(servers)) {
                servers = [];
            }
        } catch (parseError) {
            console.error('Error parsing servers.json:', parseError);
            servers = [];
        }
        
        res.json(servers);
    } catch (error) {
        if (error.code === 'NoSuchKey') {
            res.json([]); // Return empty array if file does not exist
        } else {
            console.error(error);
            return sendError(res, 500, 'Error reading from S3', 'E_S3');
        }
    }
});

// Update servers in S3
app.post('/api/servers', async (req, res) => {
    const { domains: servers } = req.body; // Keep name 'domains' for consistency
    
    // Validate servers structure
    if (!Array.isArray(servers)) {
        return sendError(res, 400, 'Servers must be an array', 'E_BAD_REQUEST');
    }
    
    // Validate each server has required fields
    for (let i = 0; i < servers.length; i++) {
        const server = servers[i];
        if (!server.ip || !server.dns || !server.country || !server.provider || !server.tunnel) {
            return sendError(res, 400, `Server at index ${i} is missing required fields`, 'E_SCHEMA');
        }
    }

    const params = {
        Bucket: BUCKET_NAME,
        Key: 'servers.json',
        Body: JSON.stringify(servers, null, 2), // Pretty print JSON
        ContentType: 'application/json',
    };

    try {
        await s3.send(new PutObjectCommand(params));
        invalidateCacheForKey('servers.json');
        const meta = await headMeta('servers.json');
        return sendOk(res, meta);
    } catch (error) {
        console.error(error);
        return sendError(res, 500, 'Error writing to S3', 'E_S3');
    }
});

// --- Billing Routes ---

// Get billing data from S3
app.get('/api/billing', async (req, res) => {
    const params = {
        Bucket: BUCKET_NAME,
        Key: 'servers-billing.json',
    };

    try {
        const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'servers-billing.json' })).catch(() => null);
        const etag = head?.ETag || null;
        if (etag) res.set('ETag', String(etag));
        if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
        if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
        if (checkIfNoneMatch(req, res, etag)) return;

        const data = await s3.send(new GetObjectCommand(params));
        const fileContent = await streamToString(data.Body);
        let billingData = [];
        
        try {
            billingData = JSON.parse(fileContent);
            // Ensure it's an array
            if (!Array.isArray(billingData)) {
                billingData = [];
            }
        } catch (parseError) {
            console.error('Error parsing servers-billing.json:', parseError);
            billingData = [];
        }
        
        res.json(billingData);
    } catch (error) {
        if (error.code === 'NoSuchKey') {
            res.json([]); // Return empty array if file does not exist
        } else {
            console.error(error);
            return sendError(res, 500, 'Error reading from S3', 'E_S3');
        }
    }
});

// Update billing data in S3
app.post('/api/billing', async (req, res) => {
    const { domains: billingData } = req.body; // Keep name 'domains' for consistency
    
    // Validate billing data structure
    if (!Array.isArray(billingData)) {
        return sendError(res, 400, 'Billing data must be an array', 'E_BAD_REQUEST');
    }
    
    // Validate each billing item has required fields
    for (let i = 0; i < billingData.length; i++) {
        const item = billingData[i];
        if (!item.hostName || !item.country || !item.provider) {
            return sendError(res, 400, `Billing item at index ${i} is missing required fields: hostName, country, provider`, 'E_SCHEMA');
        }
    }

    const params = {
        Bucket: BUCKET_NAME,
        Key: 'servers-billing.json',
        Body: JSON.stringify(billingData, null, 2), // Pretty print JSON
        ContentType: 'application/json',
    };

    try {
        await s3.send(new PutObjectCommand(params));
        invalidateCacheForKey('servers-billing.json');
        const meta = await headMeta('servers-billing.json');
        return sendOk(res, meta);
    } catch (error) {
        console.error(error);
        return sendError(res, 500, 'Error writing to S3', 'E_S3');
    }
});

// --- Filters Routes (JSON format) ---

// Get filters from S3
app.get('/api/filters', async (req, res) => {
    const params = {
        Bucket: BUCKET_NAME,
        Key: 'filters.json',
    };

    try {
        const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'filters.json' })).catch(() => null);
        const etag = head?.ETag || null;
        if (etag) res.set('ETag', String(etag));
        if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
        if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
        if (checkIfNoneMatch(req, res, etag)) return;

        const data = await s3.send(new GetObjectCommand(params));
        const fileContent = await streamToString(data.Body);
        let filters = [];
        
        try {
            filters = JSON.parse(fileContent);
            // Ensure it's an array
            if (!Array.isArray(filters)) {
                filters = [];
            }
        } catch (parseError) {
            console.error('Error parsing filters.json:', parseError);
            filters = [];
        }
        
        res.json(filters);
    } catch (error) {
        if (error.code === 'NoSuchKey') {
            res.json([]); // Return empty array if file does not exist
        } else {
            console.error(error);
            return sendError(res, 500, 'Error reading from S3', 'E_S3');
        }
    }
});

// Update filters in S3
app.post('/api/filters', async (req, res) => {
    const { domains: filters } = req.body; // Keep name 'domains' for consistency
    
    // Validate filters structure
    if (!Array.isArray(filters)) {
        return sendError(res, 400, 'Filters must be an array', 'E_BAD_REQUEST');
    }
    
    // Validate each filter has required fields
    for (let i = 0; i < filters.length; i++) {
        const filter = filters[i];
        if (!filter.community || !filter.gateway) {
            return sendError(res, 400, `Filter at index ${i} is missing required fields`, 'E_SCHEMA');
        }
    }

    const params = {
        Bucket: BUCKET_NAME,
        Key: 'filters.json',
        Body: JSON.stringify(filters, null, 2), // Pretty print JSON
        ContentType: 'application/json',
    };

    try {
        await s3.send(new PutObjectCommand(params));
        invalidateCacheForKey('filters.json');
        const meta = await headMeta('filters.json');
        return sendOk(res, meta);
    } catch (error) {
        console.error(error);
        return sendError(res, 500, 'Error writing to S3', 'E_S3');
    }
});

// Эндпоинт метаданных S3 по ключевым файлам (Last-Modified, ETag, Content-Length)
app.get('/api/s3/last-modified', async (req, res) => {
    try {
        const keys = [
            { name: 'domainsNew', key: 'bgp_data/domains_community.txt' },
            { name: 'asns', key: 'bgp_data/asns.txt' },
            { name: 'servers', key: 'servers.json' },
            { name: 'filters', key: 'filters.json' },
            { name: 'ipRanges', key: 'bgp_data/ips.txt' },
            { name: 'uiSettings', key: 'bgp_data/rt_ui_settings.json' }
        ];
        const results = await Promise.allSettled(
            keys.map(k => s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: k.key })))
        );
        const out = {};
        results.forEach((r, idx) => {
            const name = keys[idx].name;
            if (r.status === 'fulfilled') {
                out[name] = {
                    lastModified: r.value.LastModified ? r.value.LastModified.toISOString() : null,
                    etag: r.value.ETag || null,
                    contentLength: typeof r.value.ContentLength === 'number' ? r.value.ContentLength : null
                };
            } else {
                out[name] = null;
            }
        });
        res.json(out);
    } catch (error) {
        console.error('Error fetching last modified dates from S3:', error);
        return sendError(res, 500, 'Error fetching last modified dates from S3', 'E_S3');
    }
});

// Soft-lock endpoints
// GET lock status
app.get('/api/locks/:resource', (req, res) => {
    cleanupExpiredLocks();
    const { resource } = req.params;
    const info = locks.get(resource);
    if (!info) return res.json({ locked: false });
    res.json({ locked: true, owner: info.owner, expiresAt: info.expiresAt });
});

// POST acquire/refresh lock
app.post('/api/locks/:resource', (req, res) => {
    cleanupExpiredLocks();
    const { resource } = req.params;
    const { owner = 'anonymous', ttlSeconds = 120 } = req.body || {};
    const now = Date.now();
    const existing = locks.get(resource);
    if (existing && existing.expiresAt > now && existing.owner !== owner) {
        return sendError(res, 423, 'Resource is locked by another user', 'E_RESOURCE_LOCKED', { owner: existing.owner, expiresAt: existing.expiresAt });
    }
    const expiresAt = now + Math.max(30, Math.min(600, Number(ttlSeconds) || 120)) * 1000;
    locks.set(resource, { owner, expiresAt });
    res.json({ locked: true, owner, expiresAt });
});

// DELETE release lock
app.delete('/api/locks/:resource', (req, res) => {
    const { resource } = req.params;
    locks.delete(resource);
    res.json({ released: true });
});

// History endpoints (require bucket versioning to be enabled). If versioning disabled, best effort.
app.get('/api/history/:resource', async (req, res) => {
  const { resource } = req.params;
  const { countOnly, format } = req.query || {};
  const key = resourceToKey(resource);
  if (!key) return sendError(res, 400, 'Unknown resource', 'E_RESOURCE');
  try {
    const out = await s3.send(new ListObjectVersionsCommand({ Bucket: BUCKET_NAME, Prefix: key, MaxKeys: 50 }));
    const versionsAll = (out.Versions || []).filter(v => v.Key === key);
    if (countOnly === 'true') {
      const total = versionsAll.length;
      return res.json(format === 'std' ? { items: [], total, meta: {} } : { total });
    }
    const versions = versionsAll.slice(0, 10).map(v => ({ versionId: v.VersionId, isLatest: v.IsLatest, lastModified: toIso(v.LastModified), size: v.Size, etag: v.ETag }));
    return res.json(format === 'std' ? { items: versions, total: versionsAll.length, meta: {} } : { items: versions });
  } catch (e) {
    console.error('history error', e);
    return sendError(res, 500, 'Error reading history', 'E_S3', { error: String(e?.message || e) });
  }
});

app.post('/api/history/:resource/rollback', async (req, res) => {
  const { resource } = req.params;
  const { versionId } = req.body || {};
  const key = resourceToKey(resource);
  if (!key || !versionId) return sendError(res, 400, 'Bad request', 'E_BAD_REQUEST');
  try {
    // Copy specific version over same key to rollback
    await s3.send(new CopyObjectCommand({ Bucket: BUCKET_NAME, CopySource: `/${BUCKET_NAME}/${encodeURIComponent(key)}?versionId=${encodeURIComponent(versionId)}`, Key: key }));
    const meta = await headMeta(key);
    return sendOk(res, meta);
  } catch (e) {
    console.error('rollback error', e);
    return sendError(res, 500, 'Error rollback', 'E_S3', { error: String(e?.message || e) });
  }
});

// Generate MikroTik configuration from filters
app.get('/api/filters/generate-config', async (req, res) => {
    const params = {
        Bucket: BUCKET_NAME,
        Key: 'filters.json',
    };

    try {
        const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'server-configs.json' })).catch(() => null);
        const etag = head?.ETag || null;
        if (etag) res.set('ETag', String(etag));
        if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
        if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
        if (checkIfNoneMatch(req, res, etag)) return;

        const data = await s3.send(new GetObjectCommand(params));
        const fileContent = await streamToString(data.Body);
        let filters = [];
        
        try {
            filters = JSON.parse(fileContent);
            if (!Array.isArray(filters)) {
                filters = [];
            }
        } catch (parseError) {
            console.error('Error parsing filters.json:', parseError);
            filters = [];
        }

        if (filters.length === 0) {
            return res.json({ config: '// No filters to generate configuration' });
        }

        // Group filters by gateway
        const gatewayGroups = {};
        filters.forEach(filter => {
            if (!gatewayGroups[filter.gateway]) {
                gatewayGroups[filter.gateway] = [];
            }
            gatewayGroups[filter.gateway].push(filter.community);
        });

        let config = '// Frouting filter configuration for MikroTik 7.14+\n';
        config += '// Generated automatically\n';
        config += `// Date: ${new Date().toISOString()}\n\n`;
        config += '/routing filter bgp-in-tmp {\n';

        // Строим вложенные if/else (RouterOS не поддерживает else if в данном контексте)
        config += buildNestedGatewayBlocks(gatewayGroups, 4);
        
        config += '}\n';

        res.json({ config });
    } catch (error) {
        if (error.code === 'NoSuchKey') {
            res.json({ config: '// filters.json file not found' });
        } else {
            console.error(error);
            return sendError(res, 500, 'Error generating configuration', 'E_S3');
        }
    }
});

// Export MikroTik configuration to S3
app.post('/api/filters/export-config', async (req, res) => {
    const params = {
        Bucket: BUCKET_NAME,
        Key: 'filters.json',
    };

    try {
        const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: `filter-manager/server-filters-${serverId}.json` })).catch(() => null);
        const etag = head?.ETag || null;
        if (etag) res.set('ETag', String(etag));
        if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
        if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
        if (checkIfNoneMatch(req, res, etag)) return;

        const data = await s3.send(new GetObjectCommand(params));
        const fileContent = await streamToString(data.Body);
        let filters = [];
        
        try {
            filters = JSON.parse(fileContent);
            if (!Array.isArray(filters)) {
                filters = [];
            }
        } catch (parseError) {
            console.error('Error parsing filters.json:', parseError);
            filters = [];
        }

        if (filters.length === 0) {
            return res.json({ success: false, message: 'Нет фильтров для экспорта' });
        }

        // Group filters by gateway
        const gatewayGroups = {};
        filters.forEach(filter => {
            if (!gatewayGroups[filter.gateway]) {
                gatewayGroups[filter.gateway] = [];
            }
            gatewayGroups[filter.gateway].push(filter.community);
        });

        let config = '// Frouting filter configuration for MikroTik 7.14+\n';
        config += '// Generated automatically\n';
        config += `// Date: ${new Date().toISOString()}\n\n`;
        config += '/routing filter bgp-in-tmp {\n';

        // Строим вложенные if/else (RouterOS не поддерживает else if в данном контексте)
        config += buildNestedGatewayBlocks(gatewayGroups, 4);
        
        config += '}\n';

        // Save configuration to S3
        const exportParams = {
            Bucket: BUCKET_NAME,
            Key: 'mikrotik-frouting-config.txt',
            Body: config,
            ContentType: 'text/plain',
        };

        await s3.send(new PutObjectCommand(exportParams));
        res.json({ success: true, message: 'Конфигурация экспортирована в S3' });
    } catch (error) {
        console.error(error);
        return sendError(res, 500, 'Error exporting configuration', 'E_S3');
    }
});

// --- Server Configs Routes ---

// Get server configs list
app.get('/api/server-configs', async (req, res) => {
    const params = {
        Bucket: BUCKET_NAME,
        Key: 'server-configs.json',
    };

    try {
        const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'filter-manager/simple-filters.json' })).catch(() => null);
        const etag = head?.ETag || null;
        if (etag) res.set('ETag', String(etag));
        if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
        if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
        if (checkIfNoneMatch(req, res, etag)) return;

        const data = await s3.send(new GetObjectCommand(params));
        const fileContent = await streamToString(data.Body);
        let servers = [];
        
        try {
            servers = JSON.parse(fileContent);
            if (!Array.isArray(servers)) {
                servers = [];
            }
        } catch (parseError) {
            console.error('Error parsing server-configs.json:', parseError);
            servers = [];
        }
        
        res.json(servers);
    } catch (error) {
        if (error.code === 'NoSuchKey') {
            res.json([]); // Return empty array if file does not exist
        } else {
            console.error(error);
            return sendError(res, 500, 'Error reading server configs from S3', 'E_S3');
        }
    }
});

// Update server configs list
app.post('/api/server-configs', async (req, res) => {
    const { servers } = req.body;
    
    // Validate servers structure
    if (!Array.isArray(servers)) {
        return sendError(res, 400, 'Servers must be an array', 'E_BAD_REQUEST');
    }
    
    // Validate each server has required fields
    for (let i = 0; i < servers.length; i++) {
        const server = servers[i];
        if (!server.id || !server.name) {
            return sendError(res, 400, `Server at index ${i} is missing required fields`, 'E_SCHEMA');
        }
    }

    const params = {
        Bucket: BUCKET_NAME,
        Key: 'server-configs.json',
        Body: JSON.stringify(servers, null, 2), // Pretty print JSON
        ContentType: 'application/json',
    };

    try {
        await s3.send(new PutObjectCommand(params));
        invalidateCacheForKey('server-configs.json');
        const meta = await headMeta('server-configs.json');
        return sendOk(res, meta);
    } catch (error) {
        console.error(error);
        return sendError(res, 500, 'Error writing server configs to S3', 'E_S3');
    }
});

// Get specific server config
app.get('/api/server-configs/:serverId', async (req, res) => {
    const { serverId } = req.params;
    const params = {
        Bucket: BUCKET_NAME,
        Key: `filter-manager/config-${serverId}.txt`,
    };

    try {
        const head = await s3.send(new HeadObjectCommand(params)).catch(() => null);
        const etag = head?.ETag || null;
        if (etag) res.set('ETag', String(etag));
        if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
        if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
        if (checkIfNoneMatch(req, res, etag)) return;

        const data = await s3.send(new GetObjectCommand(params));
        const config = await streamToString(data.Body);
        res.json({ config });
    } catch (error) {
        if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) {
            res.json({ config: '// Конфигурация не найдена' });
        } else {
            console.error(error);
            return sendError(res, 500, 'Error reading server config from S3', 'E_S3');
        }
    }
});

// Save specific server config
app.post('/api/server-configs/:serverId', async (req, res) => {
    const { serverId } = req.params;
    const { config } = req.body;
    
    if (!config) {
        return sendError(res, 400, 'Config is required', 'E_BAD_REQUEST');
    }

    const params = {
        Bucket: BUCKET_NAME,
        Key: `filter-manager/config-${serverId}.txt`,
        Body: config,
        ContentType: 'text/plain',
    };

    try {
        await s3.send(new PutObjectCommand(params));
        invalidateCacheForKey(`filter-manager/config-${serverId}.txt`);
        const meta = await headMeta(`filter-manager/config-${serverId}.txt`);
        return sendOk(res, meta);
    } catch (error) {
        console.error(error);
        return sendError(res, 500, 'Error writing server config to S3', 'E_S3');
    }
});

// Delete specific server config
app.delete('/api/server-configs/:serverId', async (req, res) => {
    const { serverId } = req.params;
    const params = {
        Bucket: BUCKET_NAME,
        Key: `filter-manager/config-${serverId}.txt`,
    };

    try {
        await s3.send(new DeleteObjectCommand(params));
        invalidateCacheForKey(`filter-manager/config-${serverId}.txt`);
        const meta = await headMeta(`filter-manager/config-${serverId}.txt`);
        return sendOk(res, meta);
    } catch (error) {
        console.error(error);
        return sendError(res, 500, 'Error deleting server config from S3', 'E_S3');
    }
});

// Delete server completely (config + filters)
app.delete('/api/server-configs/:serverId/complete', async (req, res) => {
    const { serverId } = req.params;
    
    try {
        // Удаляем конфигурацию сервера
        const configParams = {
            Bucket: BUCKET_NAME,
            Key: `filter-manager/config-${serverId}.txt`,
        };
        
        // Удаляем фильтры сервера
        const filtersParams = {
            Bucket: BUCKET_NAME,
            Key: `filter-manager/server-filters-${serverId}.json`,
        };
        
        // Удаляем оба файла параллельно
        await Promise.allSettled([
            s3.send(new DeleteObjectCommand(configParams)),
            s3.send(new DeleteObjectCommand(filtersParams))
        ]);
        invalidateCacheForKey(configParams.Key);
        invalidateCacheForKey(filtersParams.Key);
        return res.json({ ok: true, etag: null, lastModified: null, contentLength: null });
    } catch (error) {
        console.error(error);
        return sendError(res, 500, 'Error deleting server files from S3', 'E_S3');
    }
});

// --- Server Filters Routes ---

// Get server filters
app.get('/api/server-filters/:serverId', async (req, res) => {
    const { serverId } = req.params;
    const params = {
        Bucket: BUCKET_NAME,
        Key: `filter-manager/server-filters-${serverId}.json`,
    };

    try {
        const head = await s3.send(new HeadObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/auto_url/urls.txt' })).catch(() => null);
        const etag = head?.ETag || null;
        if (etag) res.set('ETag', String(etag));
        if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
        if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
        if (checkIfNoneMatch(req, res, etag)) return;

        const data = await s3.send(new GetObjectCommand(params));
        const fileContent = await streamToString(data.Body);
        let filters = [];
        
        try {
            filters = JSON.parse(fileContent);
            if (!Array.isArray(filters)) {
                filters = [];
            }
        } catch (parseError) {
            console.error('Error parsing server filters:', parseError);
            filters = [];
        }
        
        res.json(filters);
    } catch (error) {
        if (error.code === 'NoSuchKey') {
            res.json([]); // Return empty array if file does not exist
        } else {
            console.error(error);
            return sendError(res, 500, 'Error reading server filters from S3', 'E_S3');
        }
    }
});

// Generate MikroTik configuration from server filters
app.post('/api/server-filters/generate-config', async (req, res) => {
    console.log('Received generate-config request with filters:', req.body);
    const { filters } = req.body;
    
    if (!Array.isArray(filters) || filters.length === 0) {
        console.log('No filters provided, returning empty config');
        return res.json({ config: '// No filters to generate configuration' });
    }

    // Group filters by gateway
    const gatewayGroups = {};
    filters.forEach(filter => {
        if (!gatewayGroups[filter.gateway]) {
            gatewayGroups[filter.gateway] = [];
        }
        gatewayGroups[filter.gateway].push(filter.community);
    });

    console.log('Grouped filters by gateway:', gatewayGroups);

    let config = '// Frouting filter configuration for MikroTik 7.14+\n';
    config += '// Generated automatically\n';
    config += `// Date: ${new Date().toISOString()}\n\n`;
    config += '/routing filter bgp-in-tmp {\n';

    // Строим вложенные if/else (RouterOS не поддерживает else if в данном контексте)
    config += buildNestedGatewayBlocks(gatewayGroups, 4);
    
    config += '}\n';

    console.log('Generated config:', config);
    console.log('Sending response:', { config });
    res.json({ config });
});

// Update server filters
app.post('/api/server-filters/:serverId', async (req, res) => {
    const { serverId } = req.params;
    const { filters } = req.body;
    
    // Validate filters structure
    if (!Array.isArray(filters)) {
        return sendError(res, 400, 'Filters must be an array', 'E_BAD_REQUEST');
    }
    
    // Validate each filter has required fields
    for (let i = 0; i < filters.length; i++) {
        const filter = filters[i];
        if (!filter.community || !filter.gateway) {
            return sendError(res, 400, `Filter at index ${i} is missing required fields`, 'E_SCHEMA');
        }
    }

    const params = {
        Bucket: BUCKET_NAME,
        Key: `filter-manager/server-filters-${serverId}.json`,
        Body: JSON.stringify(filters, null, 2), // Pretty print JSON
        ContentType: 'application/json',
    };

    try {
        await s3.send(new PutObjectCommand(params));
        invalidateCacheForKey(`filter-manager/server-filters-${serverId}.json`);
        const meta = await headMeta(`filter-manager/server-filters-${serverId}.json`);
        return sendOk(res, meta);
    } catch (error) {
        console.error(error);
        return sendError(res, 500, 'Error writing server filters to S3', 'E_S3');
    }
});

// --- Simple Filters Routes ---

// Get simple filters
app.get('/api/simple-filters', async (req, res) => {
    const params = {
        Bucket: BUCKET_NAME,
        Key: 'filter-manager/simple-filters.json',
    };

    try {
        const head = await s3.send(new HeadObjectCommand(params)).catch(() => null);
        const etag = head?.ETag || null;
        if (etag) res.set('ETag', String(etag));
        if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
        if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
        if (checkIfNoneMatch(req, res, etag)) return;

        const data = await s3.send(new GetObjectCommand(params));
        const fileContent = await streamToString(data.Body);
        let filters = [];
        
        try {
            filters = JSON.parse(fileContent);
            if (!Array.isArray(filters)) {
                filters = [];
            }
        } catch (parseError) {
            console.error('Error parsing simple filters:', parseError);
            filters = [];
        }
        
        res.json(filters);
    } catch (error) {
        if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) {
            res.json([]); // Return empty array if file does not exist
        } else {
            console.error(error);
            return sendError(res, 500, 'Error reading simple filters from S3', 'E_S3');
        }
    }
});

// Update simple filters
app.post('/api/simple-filters', async (req, res) => {
    const { filters } = req.body;
    
    // Validate filters structure
    if (!Array.isArray(filters)) {
        return sendError(res, 400, 'Filters must be an array', 'E_BAD_REQUEST');
    }
    
    // Validate each filter has required fields
    for (let i = 0; i < filters.length; i++) {
        const filter = filters[i];
        if (!filter.community || !filter.gateway) {
            return sendError(res, 400, `Filter at index ${i} is missing required fields`, 'E_SCHEMA');
        }
    }

    const params = {
        Bucket: BUCKET_NAME,
        Key: 'filter-manager/simple-filters.json',
        Body: JSON.stringify(filters, null, 2), // Pretty print JSON
        ContentType: 'application/json',
    };

    try {
        await s3.send(new PutObjectCommand(params));
        invalidateCacheForKey('filter-manager/simple-filters.json');
        const meta = await headMeta('filter-manager/simple-filters.json');
        return sendOk(res, meta);
    } catch (error) {
        console.error(error);
        return sendError(res, 500, 'Error writing simple filters to S3', 'E_S3');
    }
});

// --- Auto URL Routes ---

// Get auto URLs from S3
app.get('/api/auto-urls', async (req, res) => {
    const params = {
        Bucket: BUCKET_NAME,
        Key: 'bgp_data/auto_url/urls.txt',
    };

    try {
        const head = await s3.send(new HeadObjectCommand(params)).catch(() => null);
        const etag = head?.ETag || null;
        if (etag) res.set('ETag', String(etag));
        if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
        if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
        if (checkIfNoneMatch(req, res, etag)) return;

        const data = await s3.send(new GetObjectCommand(params));
        const fileContent = await streamToString(data.Body);
        const urls = fileContent.split('\n').filter(line => line).map(line => {
            const parts = line.trim().split(/\s+/);
            const url = parts[0] || '';
            const community = parts[1] || '';
            return { url, community };
        });
        res.json(urls);
    } catch (error) {
        if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) {
            res.json([]); // Return empty array if file does not exist
        } else {
            console.error(error);
            return sendError(res, 500, 'Error reading auto URLs from S3', 'E_S3');
        }
    }
});

// --- UI Settings (rt_ui_settings.json in bgp_data) ---
// Get UI settings
app.get('/api/ui-settings', async (req, res) => {
    const params = {
        Bucket: BUCKET_NAME,
        Key: 'bgp_data/rt_ui_settings.json',
    };

    try {
        const head = await s3.send(new HeadObjectCommand(params)).catch(() => null);
        const etag = head?.ETag || null;
        if (etag) res.set('ETag', String(etag));
        if (head?.LastModified) res.set('Last-Modified', new Date(head.LastModified).toUTCString());
        if (typeof head?.ContentLength === 'number') res.set('Content-Length-Source', String(head.ContentLength));
        if (checkIfNoneMatch(req, res, etag)) return;

        const data = await s3.send(new GetObjectCommand(params));
        const jsonText = await streamToString(data.Body);
        let settings = {};
        try {
            const parsed = JSON.parse(jsonText);
            if (parsed && typeof parsed === 'object') settings = parsed;
        } catch (parseError) {
            settings = {};
        }
        return res.json(settings);
    } catch (error) {
        if (error.code === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) {
            return res.json({});
        }
        console.error('Error reading ui settings from S3:', error);
        return sendError(res, 500, 'Error reading UI settings from S3', 'E_S3');
    }
});

// Update UI settings
app.post('/api/ui-settings', async (req, res) => {
    const { settings, etag } = req.body || {};
    const payload = (settings && typeof settings === 'object') ? settings : {};

    try {
        // optimistic concurrency if ETag provided (or If-Match header)
        let current = null;
        const ifMatch = req.headers['if-match'] ? String(req.headers['if-match']).replace(/\"/g,'"') : null;
        try { current = await headS3ObjectEtag('bgp_data/rt_ui_settings.json'); } catch {}
        if (current && (ifMatch || etag) && current !== String(ifMatch || etag)) {
            const meta = await headMeta('bgp_data/rt_ui_settings.json');
            return sendError(res, 412, 'Precondition Failed: ETag mismatch', 'E_ETAG_MISMATCH', { currentEtag: current, meta });
        }
    } catch {}

    const params = {
        Bucket: BUCKET_NAME,
        Key: 'bgp_data/rt_ui_settings.json',
        Body: JSON.stringify(payload, null, 2),
        ContentType: 'application/json',
    };

    try {
        await s3.send(new PutObjectCommand(params));
        const meta = await headMeta('bgp_data/rt_ui_settings.json');
        return sendOk(res, meta);
    } catch (error) {
        console.error('Error writing UI settings to S3:', error);
        return sendError(res, 500, 'Error writing UI settings to S3', 'E_S3', { error: String(error?.message || error) });
    }
});

// --- Servers availability check ---
// Optimized TCP check: parallelize ports and hosts, cap per-host time, add in-memory TTL cache
function tcpCheck(host, port, timeoutMs) {
  return new Promise((resolve) => {
    const socket = new net.Socket();
    let settled = false;
    const settle = (ok) => { if (!settled) { settled = true; try { socket.destroy(); } catch {} resolve(ok); } };
    socket.setTimeout(timeoutMs, () => settle(false));
    socket.once('error', () => settle(false));
    socket.connect(port, host, () => settle(true));
  });
}

function anyTrue(promises) {
  return new Promise((resolve) => {
    if (!Array.isArray(promises) || promises.length === 0) return resolve(false);
    let remaining = promises.length;
    let resolved = false;
    for (const p of promises) {
      Promise.resolve(p).then((v) => {
        if (v && !resolved) { resolved = true; resolve(true); }
      }).finally(() => {
        remaining -= 1;
        if (remaining === 0 && !resolved) resolve(false);
      });
    }
  });
}

async function checkOneServerFast(srv, perSocketTimeoutMs = 800, perServerBudgetMs = 1000) {
  const hosts = [];
  if (srv.ip) hosts.push(String(srv.ip));
  if (srv.dns) hosts.push(String(srv.dns));
  const tryOneHost = (host) => anyTrue([
    // try common ports simultaneously
    tcpCheck(host, 443, perSocketTimeoutMs),
    tcpCheck(host, 80, perSocketTimeoutMs),
  ]);
  const run = anyTrue(hosts.map((h) => tryOneHost(h)));
  // Per-server overall budget
  const timeout = new Promise((resolve) => setTimeout(() => resolve(false), perServerBudgetMs));
  return Promise.race([run, timeout]);
}

const availabilityCache = { at: 0, data: null };

app.get('/api/servers/availability', async (req, res) => {
  try {
    const ttlSeconds = Math.max(0, Math.min(300, Number(req.query.ttlSeconds) || 30));
    const now = Date.now();
    if (availabilityCache.data && (now - availabilityCache.at) < ttlSeconds * 1000) {
      return res.json({ ...availabilityCache.data, cached: true });
    }

    const data = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'servers.json' }));
    let servers = [];
    try {
      servers = JSON.parse(await streamToString(data.Body));
      if (!Array.isArray(servers)) servers = [];
    } catch {
      servers = [];
    }

    const checks = await Promise.allSettled(servers.map((s) => checkOneServerFast(s)));
    const statuses = servers.map((s, i) => ({ ip: s.ip, dns: s.dns, online: checks[i].status === 'fulfilled' ? Boolean(checks[i].value) : false }));
    const online = statuses.filter((x) => x.online).length;
    const payload = { online, total: servers.length, statuses };
    availabilityCache.at = Date.now();
    availabilityCache.data = payload;
    res.json(payload);
  } catch (e) {
    console.error('availability error', e);
    res.status(500).json({ online: 0, total: 0, statuses: [] });
  }
});

// Update auto URLs in S3
app.post('/api/auto-urls', async (req, res) => {
    const { urls } = req.body;
    const fileContent = urls.map(u => `${u.url} ${u.community}`).join('\n');

    const params = {
        Bucket: BUCKET_NAME,
        Key: 'bgp_data/auto_url/urls.txt',
        Body: fileContent,
        ContentType: 'text/plain',
    };

    try {
        await s3.send(new PutObjectCommand(params));
        invalidateCacheForKey('bgp_data/auto_url/urls.txt');
        const meta = await headMeta('bgp_data/auto_url/urls.txt');
        return sendOk(res, meta);
    } catch (error) {
        console.error(error);
        return sendError(res, 500, 'Error writing auto URLs to S3', 'E_S3');
    }
});

// Process auto URLs and update IPs
app.post('/api/auto-urls/process', async (req, res) => {
  const https = require('https');
  const http = require('http');

  // Helpers
  const isValidIPv4 = (ip) => {
    const octets = String(ip || '').trim().split('.');
    if (octets.length !== 4) return false;
    return octets.every(o => /^\d{1,3}$/.test(o) && Number(o) >= 0 && Number(o) <= 255);
  };
  const isValidCidrV4 = (value) => {
    const v = String(value || '').trim();
    const parts = v.split('/');
    if (parts.length !== 2) return false;
    const [ip, mask] = parts;
    if (!isValidIPv4(ip)) return false;
    if (!/^\d{1,2}$/.test(mask)) return false;
    const m = Number(mask);
    return m >= 0 && m <= 32;
  };
  const isValidDomain = (value) => {
    const v = String(value || '').trim().toLowerCase();
    if (v.startsWith('#')) return false; // comment line
    return /^([a-z0-9-]+\.)+[a-z]{2,}$/i.test(v);
  };

  try {
    // Load configured auto URLs
    const urlsParams = {
      Bucket: BUCKET_NAME,
      Key: 'bgp_data/auto_url/urls.txt',
    };

    let urls = [];
    try {
      const urlsData = await s3.send(new GetObjectCommand(urlsParams));
      const urlsContent = await streamToString(urlsData.Body);
      urls = urlsContent.split('\n').filter(line => line).map(line => {
        const parts = line.trim().split(/\s+/);
        return { url: parts[0] || '', community: parts[1] || '' };
      });
    } catch (error) {
      if (error.code !== 'NoSuchKey') {
        throw error;
      }
    }

    if (urls.length === 0) {
      return sendError(res, 400, 'No URLs to process', 'E_BAD_REQUEST');
    }

    // Load current IP ranges
    const ipsParams = { Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt' };
    let currentIps = [];
    try {
      const ipsData = await s3.send(new GetObjectCommand(ipsParams));
      const ipsContent = await streamToString(ipsData.Body);
      currentIps = ipsContent.split('\n').filter(line => line).map(line => {
        const parts = line.trim().split(/\s+/);
        return { ipRange: parts[0] || '', community: parts[1] || '' };
      });
    } catch (error) {
      if (error.code !== 'NoSuchKey') throw error;
    }

    // Load current domains
    const domainsParams = { Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt' };
    let currentDomains = [];
    try {
      const dData = await s3.send(new GetObjectCommand(domainsParams));
      const dContent = await streamToString(dData.Body);
      currentDomains = dContent.split('\n').filter(line => line).map(line => {
        const parts = line.trim().split(/\s+/);
        return { domain: (parts[0] || '').toLowerCase(), community: parts[1] || '' };
      });
    } catch (error) {
      if (error.code !== 'NoSuchKey') throw error;
    }

    // Process each URL
    const newIps = [];
    const newDomains = [];
    for (const urlData of urls) {
      try {
        const url = String(urlData.url || '').trim();
        const community = String(urlData.community || '').trim();
        if (!url || !community) continue;

        // Download content
        const content = await new Promise((resolve, reject) => {
          const protocol = url.startsWith('https:') ? https : http;
          const req = protocol.get(url, (r) => {
            let data = '';
            r.on('data', (chunk) => { data += chunk; });
            r.on('end', () => resolve(data));
          });
          req.on('error', reject);
          req.setTimeout(15000, () => req.destroy());
        });

        const lines = content.split('\n');
        for (const raw of lines) {
          const line = String(raw || '').trim();
          if (!line) continue;
          if (line.startsWith('#') || line.startsWith('//')) continue;
          const token = line.split(/\s+/)[0]?.trim();
          if (!token) continue;

          // Decide destination
          if (isValidCidrV4(token)) {
            newIps.push({ ipRange: token, community });
            continue;
          }
          if (isValidIPv4(token)) {
            // single IPv4 → normalize to /32
            newIps.push({ ipRange: `${token}/32`, community });
            continue;
          }
          if (isValidDomain(token)) {
            newDomains.push({ domain: token.toLowerCase(), community });
            continue;
          }
          // ignore everything else
        }
      } catch (error) {
        console.error(`Error processing URL ${urlData.url}:`, error);
      }
    }

    // Merge & deduplicate
    const existingIpRanges = new Set(currentIps.map(i => i.ipRange));
    const uniqueNewIps = newIps.filter(i => !existingIpRanges.has(i.ipRange));
    const allIps = [...currentIps, ...uniqueNewIps];

    const existingDomains = new Set(currentDomains.map(d => d.domain));
    const uniqueNewDomains = newDomains.filter(d => !existingDomains.has(d.domain));
    const allDomains = [...currentDomains, ...uniqueNewDomains];

    // Save updated IPs
    const updatedIpsContent = allIps.map(i => `${i.ipRange} ${i.community}`).join('\n');
    await s3.send(new PutObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/ips.txt', Body: updatedIpsContent, ContentType: 'text/plain' }));

    // Save updated Domains
    const updatedDomainsContent = allDomains.map(d => `${d.domain} ${d.community}`).join('\n');
    await s3.send(new PutObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/domains_community.txt', Body: updatedDomainsContent, ContentType: 'text/plain' }));

    const msg = `Обработано ${urls.length} URL, добавлено ${uniqueNewIps.length} IP и ${uniqueNewDomains.length} доменов`;
    return res.json({
      success: true,
      message: msg,
      processedUrls: urls.length,
      newIpsCount: uniqueNewIps.length,
      newDomainsCount: uniqueNewDomains.length,
      totalIpsCount: allIps.length,
      totalDomainsCount: allDomains.length
    });
  } catch (error) {
    console.error('Error processing auto URLs:', error);
    return sendError(res, 500, 'Error processing auto URLs', 'E_S3');
  }
});

// --- Proxy: Background BGP Update (avoids CORS from browser) ---
app.post('/api/update-bgp/background', async (req, res) => {
  try {
    const targetUrl = process.env.BGP_BACKGROUND_URL;
    if (!targetUrl) {
      return sendError(res, 500, 'BGP_BACKGROUND_URL is not configured', 'E_CONFIG');
    }
    const u = new URL(targetUrl);
    const client = u.protocol === 'https:' ? https : http;
    const options = {
      method: 'POST',
      hostname: u.hostname,
      port: u.port || (u.protocol === 'https:' ? 443 : 80),
      path: `${u.pathname}${u.search || ''}`,
      headers: {
        'Content-Type': 'application/json',
        'Accept': 'application/json',
      },
      timeout: 15000,
    };

    const body = req.body && Object.keys(req.body).length ? JSON.stringify(req.body) : '';

    const upstream = client.request(options, (r) => {
      let data = '';
      r.setEncoding('utf8');
      r.on('data', (chunk) => { data += chunk; });
      r.on('end', () => {
        const status = r.statusCode || 502;
        // Try to parse JSON; fallback to text
        try {
          const json = data ? JSON.parse(data) : {};
          return res.status(status).json(json);
        } catch (_) {
          return res.status(status).json({ ok: status >= 200 && status < 300, data });
        }
      });
    });
    upstream.on('timeout', () => {
      try { upstream.destroy(); } catch {}
      return sendError(res, 504, 'Upstream timeout', 'E_UPSTREAM_TIMEOUT');
    });
    upstream.on('error', (e) => {
      return sendError(res, 502, 'Upstream error', 'E_UPSTREAM', { error: String(e?.message || e) });
    });
    if (body) upstream.write(body);
    upstream.end();
  } catch (e) {
    return sendError(res, 500, 'Proxy error', 'E_PROXY', { error: String(e?.message || e) });
  }
});

// Provide ws url to UI from settings/env to avoid exposing keys in bundle
app.get('/api/ws/url', async (req, res) => {
  try {
    const settings = await s3.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: 'bgp_data/rt_ui_settings.json' })).then(async (d) => {
      try { return JSON.parse(await streamToString(d.Body)); } catch { return {}; }
    }).catch(() => ({}));
    const url = settings?.wsUpdateUrl || process.env.WS_UPDATE_URL || '';
    return res.json({ url });
  } catch (e) {
    return res.json({ url: '' });
  }
});

// The "catchall" handler: for any request that doesn't
// match one above, send back React's index.html file.
app.get('*', (req, res) => {
  res.sendFile(path.join(__dirname, 'public', 'index.html'));
});

// Централизованный error handler (последний middleware)
// eslint-disable-next-line no-unused-vars
app.use((err, req, res, next) => {
  const status = typeof err?.status === 'number' ? err.status : 500;
  const code = err?.code || 'E_INTERNAL';
  const message = status === 500 && process.env.NODE_ENV === 'production' ? 'Internal Server Error' : (err?.message || 'Error');
  const details = err?.details;
  const requestId = req?.id;
  try { req.log?.error({ err, code, requestId }, 'request error'); } catch {}
  res.status(status).json({ code, message, details, requestId });
});

app.listen(port, () => {
  console.log(`Server is running on http://localhost:${port}`);
});