Backend: SQLite storage, EvoBGP integration, filters in SQL
Made-with: Cursor
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
-- Router-lists local storage (SQLite). EvoBGP holds reference lists; this DB holds config + caches.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS app_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
version INTEGER NOT NULL UNIQUE,
|
||||
applied_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS blobs (
|
||||
object_key TEXT PRIMARY KEY,
|
||||
namespace TEXT NOT NULL CHECK (namespace IN (
|
||||
'config',
|
||||
'derived',
|
||||
'per_server',
|
||||
'cache'
|
||||
)),
|
||||
content_type TEXT NOT NULL DEFAULT 'application/octet-stream',
|
||||
body BLOB NOT NULL,
|
||||
byte_size INTEGER NOT NULL,
|
||||
etag TEXT NOT NULL,
|
||||
updated_at INTEGER NOT NULL,
|
||||
created_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_blobs_namespace_updated
|
||||
ON blobs (namespace, updated_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_blobs_prefix
|
||||
ON blobs (object_key);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS blob_versions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
object_key TEXT NOT NULL,
|
||||
body BLOB NOT NULL,
|
||||
byte_size INTEGER NOT NULL,
|
||||
etag TEXT,
|
||||
created_at INTEGER NOT NULL,
|
||||
FOREIGN KEY (object_key) REFERENCES blobs(object_key) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_blob_versions_key_time
|
||||
ON blob_versions (object_key, created_at DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS versioned_keys (
|
||||
object_key TEXT PRIMARY KEY
|
||||
);
|
||||
|
||||
INSERT OR IGNORE INTO versioned_keys (object_key) VALUES
|
||||
('filters.json'),
|
||||
('servers.json');
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cache_entries (
|
||||
cache_key TEXT PRIMARY KEY,
|
||||
namespace TEXT NOT NULL DEFAULT 'cache',
|
||||
body BLOB NOT NULL,
|
||||
byte_size INTEGER NOT NULL,
|
||||
etag TEXT,
|
||||
updated_at INTEGER NOT NULL,
|
||||
expires_at INTEGER
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cache_expires
|
||||
ON cache_entries (expires_at);
|
||||
@@ -0,0 +1,16 @@
|
||||
-- Нормализованные глобальные и simple-фильтры (вместо filters.json / simple-filters.json в blobs)
|
||||
|
||||
CREATE TABLE IF NOT EXISTS app_filter_rules (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
scope TEXT NOT NULL CHECK (scope IN ('global', 'simple')),
|
||||
position INTEGER NOT NULL,
|
||||
community TEXT NOT NULL,
|
||||
gateway TEXT NOT NULL,
|
||||
description TEXT NOT NULL DEFAULT '',
|
||||
extra_json TEXT NOT NULL DEFAULT '{}',
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_app_filter_rules_scope_pos ON app_filter_rules(scope, position);
|
||||
|
||||
DELETE FROM versioned_keys WHERE object_key = 'filters.json';
|
||||
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* Single-process SQLite connection, migrations, PRAGMAs.
|
||||
* Designed so the same API can later move to a dedicated storage service process.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const Database = require('better-sqlite3');
|
||||
|
||||
let _db = null;
|
||||
let _cachePurgeTimer = null;
|
||||
|
||||
function defaultDbPath() {
|
||||
return path.join(__dirname, '..', 'data', 'router-lists.db');
|
||||
}
|
||||
|
||||
function getDbPath() {
|
||||
const p = process.env.SQLITE_PATH || defaultDbPath();
|
||||
const dir = path.dirname(p);
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
function applyMigration001(db) {
|
||||
const sql = fs.readFileSync(path.join(__dirname, 'migrations', '001_init.sql'), 'utf8');
|
||||
db.exec(sql);
|
||||
}
|
||||
|
||||
function applyMigration002(db) {
|
||||
const sql = fs.readFileSync(path.join(__dirname, 'migrations', '002_filter_rules.sql'), 'utf8');
|
||||
db.exec(sql);
|
||||
|
||||
const migrateBlobToRules = (objectKey, scope) => {
|
||||
const row = db.prepare('SELECT body FROM blobs WHERE object_key = ?').get(objectKey);
|
||||
if (!row) return;
|
||||
let bodyStr = '';
|
||||
try {
|
||||
bodyStr = Buffer.isBuffer(row.body) ? row.body.toString('utf8') : String(row.body);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
let arr = [];
|
||||
try {
|
||||
const parsed = JSON.parse(bodyStr);
|
||||
arr = Array.isArray(parsed) ? parsed : [];
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const now = Date.now();
|
||||
const delRules = db.prepare('DELETE FROM app_filter_rules WHERE scope = ?');
|
||||
const ins = db.prepare(
|
||||
`INSERT INTO app_filter_rules (scope, position, community, gateway, description, extra_json, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
);
|
||||
const run = () => {
|
||||
delRules.run(scope);
|
||||
for (let i = 0; i < arr.length; i++) {
|
||||
const it = arr[i] || {};
|
||||
const community = String(it.community || '').trim();
|
||||
const gateway = String(it.gateway || '').trim();
|
||||
const description = String(it.description || '').trim();
|
||||
const extra = { ...it };
|
||||
delete extra.community;
|
||||
delete extra.gateway;
|
||||
delete extra.description;
|
||||
ins.run(scope, i, community, gateway, description, JSON.stringify(extra), now);
|
||||
}
|
||||
db.prepare('INSERT OR REPLACE INTO app_meta (key, value) VALUES (?, ?)').run(
|
||||
`filter_rules_mtime_${scope}`,
|
||||
String(now),
|
||||
);
|
||||
db.prepare('DELETE FROM blob_versions WHERE object_key = ?').run(objectKey);
|
||||
db.prepare('DELETE FROM blobs WHERE object_key = ?').run(objectKey);
|
||||
};
|
||||
db.transaction(run)();
|
||||
};
|
||||
|
||||
migrateBlobToRules('filters.json', 'global');
|
||||
migrateBlobToRules('filter-manager/simple-filters.json', 'simple');
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {import('better-sqlite3').Database}
|
||||
*/
|
||||
function openDatabase() {
|
||||
if (_db) return _db;
|
||||
|
||||
const filePath = getDbPath();
|
||||
const db = new Database(filePath);
|
||||
|
||||
db.pragma('journal_mode = WAL');
|
||||
db.pragma('synchronous = NORMAL');
|
||||
db.pragma('foreign_keys = ON');
|
||||
db.pragma(`busy_timeout = ${Number(process.env.SQLITE_BUSY_MS) || 8000}`);
|
||||
const cacheKb = Number(process.env.SQLITE_CACHE_KB);
|
||||
if (Number.isFinite(cacheKb) && cacheKb !== 0) {
|
||||
db.pragma(`cache_size = ${-Math.abs(Math.floor(cacheKb))}`);
|
||||
}
|
||||
|
||||
let userVersion = db.pragma('user_version', { simple: true });
|
||||
if (userVersion < 1) {
|
||||
applyMigration001(db);
|
||||
db.pragma('user_version = 1');
|
||||
const now = Date.now();
|
||||
db.prepare('INSERT OR IGNORE INTO schema_migrations (version, applied_at) VALUES (1, ?)').run(now);
|
||||
userVersion = 1;
|
||||
}
|
||||
if (userVersion < 2) {
|
||||
applyMigration002(db);
|
||||
db.pragma('user_version = 2');
|
||||
const now2 = Date.now();
|
||||
db.prepare('INSERT OR IGNORE INTO schema_migrations (version, applied_at) VALUES (2, ?)').run(now2);
|
||||
}
|
||||
|
||||
_db = db;
|
||||
|
||||
if (!_cachePurgeTimer) {
|
||||
const intervalMs = Math.max(60_000, Number(process.env.SQLITE_CACHE_PURGE_MS) || 300_000);
|
||||
_cachePurgeTimer = setInterval(purgeExpiredCacheEntries, intervalMs);
|
||||
_cachePurgeTimer.unref?.();
|
||||
}
|
||||
|
||||
return db;
|
||||
}
|
||||
|
||||
function purgeExpiredCacheEntries() {
|
||||
const db = openDatabase();
|
||||
const now = Date.now();
|
||||
try {
|
||||
db.prepare('DELETE FROM cache_entries WHERE expires_at IS NOT NULL AND expires_at < ?').run(now);
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
openDatabase,
|
||||
getDbPath,
|
||||
purgeExpiredCacheEntries,
|
||||
};
|
||||
Reference in New Issue
Block a user