feat(db): перевести хранилище с SQLite на PostgreSQL
Docker images / prepare-release (push) Successful in 12s
Docker images / backend-test (push) Successful in 4m9s
Docker images / frontend-image (push) Successful in 4m28s
Docker images / updater-image (push) Successful in 58s
Docker images / backend-image (push) Successful in 2m49s
Docker images / notify-webhook (push) Skipped
Docker images / publish-release (push) Successful in 11s

При старте backend накатывает схему PostgreSQL 18 и, если база пустая, один раз импортирует mikrotik.db с тома. Повторный старт не копирует данные. Бэкап в UI идёт через pg_dump.

Co-authored-by: Cursor <[email protected]>
This commit is contained in:
Denozordec
2026-09-08 01:36:48 +07:00
co-authored by Cursor
parent 0e5fb065e2
commit ec43591a99
128 changed files with 4304 additions and 3639 deletions
+33
View File
@@ -0,0 +1,33 @@
import { pool } from "./index.js"
import { applySqlMigrations } from "./migrate.js"
import { dropExpiredPartitions, ensurePartitionsAround } from "./partitions.js"
import { importSqliteToPostgres, shouldImportSqlite } from "./sqlite-import.js"
import { env } from "../config.js"
const ETL_LOCK = 8723101
export async function initDatabase(): Promise<void> {
await pool.query("SELECT 1")
await applySqlMigrations(pool)
await ensurePartitionsAround(pool)
const client = await pool.connect()
try {
await client.query("SELECT pg_advisory_lock($1)", [ETL_LOCK])
if (await shouldImportSqlite(pool, env.DATABASE_PATH)) {
console.log(`SQLite → PostgreSQL: импорт ${env.DATABASE_PATH}`)
const report = await importSqliteToPostgres(pool, env.DATABASE_PATH, { strict: true })
console.log(
`SQLite → PostgreSQL: готово за ${report.durationMs}ms, таблиц ${Object.keys(report.tables).length}`,
)
}
} finally {
try {
await client.query("SELECT pg_advisory_unlock($1)", [ETL_LOCK])
} catch {
/* ignore */
}
client.release()
}
await dropExpiredPartitions(pool)
await ensurePartitionsAround(pool)
}
+51 -1045
View File
File diff suppressed because it is too large Load Diff
+43
View File
@@ -0,0 +1,43 @@
/** JSONB from node-pg may already be an object; SQLite leftovers may be strings. */
export function parseJsonObject(value: unknown): Record<string, unknown> {
if (value && typeof value === "object" && !Array.isArray(value)) {
return value as Record<string, unknown>
}
if (typeof value === "string" && value.trim()) {
try {
const parsed = JSON.parse(value) as unknown
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
return parsed as Record<string, unknown>
}
} catch {
/* ignore */
}
}
return {}
}
export function parseJsonArray(value: unknown): unknown[] {
if (Array.isArray(value)) return value
if (typeof value === "string" && value.trim()) {
try {
const parsed = JSON.parse(value) as unknown
if (Array.isArray(parsed)) return parsed
} catch {
/* ignore */
}
}
return []
}
export function toJsonb(value: unknown): unknown {
if (value == null) return null
if (typeof value === "string") {
try {
return JSON.parse(value) as unknown
} catch {
return value
}
}
return value
}
+43
View File
@@ -0,0 +1,43 @@
import { readFileSync } from "node:fs"
import { dirname, join } from "node:path"
import { fileURLToPath } from "node:url"
import type { Pool } from "pg"
const MIGRATION_ID = "0000_postgresql"
function migrationsDir(): string {
const here = dirname(fileURLToPath(import.meta.url))
const candidates = [
join(here, "..", "..", "drizzle"),
join(process.cwd(), "drizzle"),
join(process.cwd(), "backend", "drizzle"),
]
for (const dir of candidates) {
try {
readFileSync(join(dir, `${MIGRATION_ID}.sql`), "utf8")
return dir
} catch {
/* try next */
}
}
throw new Error("Не найден backend/drizzle/0000_postgresql.sql")
}
export async function applySqlMigrations(pool: Pool): Promise<void> {
await pool.query(`
CREATE TABLE IF NOT EXISTS schema_migrations (
id TEXT PRIMARY KEY,
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
`)
const { rows } = await pool.query<{ id: string }>(
`SELECT id FROM schema_migrations WHERE id = $1`,
[MIGRATION_ID],
)
if (rows.length > 0) return
const sql = readFileSync(join(migrationsDir(), `${MIGRATION_ID}.sql`), "utf8")
await pool.query(sql)
await pool.query(`INSERT INTO schema_migrations (id) VALUES ($1) ON CONFLICT (id) DO NOTHING`, [
MIGRATION_ID,
])
}
+129
View File
@@ -0,0 +1,129 @@
import type { Pool } from "pg"
export type PartitionKind = "day" | "week" | "month"
export interface PartitionSpec {
parent: string
kind: PartitionKind
keepDays: number
}
export const PARTITION_SPECS: PartitionSpec[] = [
{ parent: "flow_buckets", kind: "day", keepDays: 4 },
{ parent: "flow_minute_stats", kind: "day", keepDays: 4 },
{ parent: "flow_minute_dims", kind: "day", keepDays: 4 },
{ parent: "flow_daily_dims", kind: "month", keepDays: 420 },
{ parent: "traffic_samples", kind: "week", keepDays: 21 },
{ parent: "servers_rest_ping_samples", kind: "week", keepDays: 35 },
{ parent: "uptime_probe_samples", kind: "week", keepDays: 21 },
{ parent: "uptime_resource_samples", kind: "week", keepDays: 21 },
{ parent: "server_snapshots", kind: "week", keepDays: 21 },
{ parent: "internet_path_snapshots", kind: "week", keepDays: 21 },
]
function utcDate(d: Date): Date {
return new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()))
}
function addUtcDays(d: Date, n: number): Date {
const x = utcDate(d)
x.setUTCDate(x.getUTCDate() + n)
return x
}
function startOfWeekUtc(d: Date): Date {
const x = utcDate(d)
const day = x.getUTCDay() || 7
x.setUTCDate(x.getUTCDate() - (day - 1))
return x
}
function startOfMonthUtc(d: Date): Date {
return new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), 1))
}
function fmtDay(d: Date): string {
return d.toISOString().slice(0, 10)
}
function partName(parent: string, kind: PartitionKind, start: Date): string {
if (kind === "day") return `${parent}_d_${fmtDay(start).replaceAll("-", "")}`
if (kind === "week") return `${parent}_w_${fmtDay(start).replaceAll("-", "")}`
return `${parent}_m_${start.getUTCFullYear()}${String(start.getUTCMonth() + 1).padStart(2, "0")}`
}
function rangeFor(kind: PartitionKind, ts: Date): { start: Date; end: Date } {
if (kind === "day") {
const start = utcDate(ts)
return { start, end: addUtcDays(start, 1) }
}
if (kind === "week") {
const start = startOfWeekUtc(ts)
return { start, end: addUtcDays(start, 7) }
}
const start = startOfMonthUtc(ts)
const end = new Date(Date.UTC(start.getUTCFullYear(), start.getUTCMonth() + 1, 1))
return { start, end }
}
export async function ensurePartitionFor(
pool: Pool,
parent: string,
kind: PartitionKind,
ts: Date,
): Promise<string> {
const { start, end } = rangeFor(kind, ts)
const name = partName(parent, kind, start)
const from = fmtDay(start)
const to = fmtDay(end)
await pool.query(
`CREATE TABLE IF NOT EXISTS ${name} PARTITION OF ${parent} FOR VALUES FROM ('${from}') TO ('${to}')`,
)
return name
}
export async function ensurePartitionsAround(pool: Pool, around = new Date()): Promise<void> {
for (const spec of PARTITION_SPECS) {
const daysAhead = spec.kind === "month" ? 40 : spec.kind === "week" ? 21 : 8
const start = addUtcDays(around, -spec.keepDays)
const end = addUtcDays(around, daysAhead)
for (let t = new Date(start.getTime()); t < end; ) {
await ensurePartitionFor(pool, spec.parent, spec.kind, t)
if (spec.kind === "day") t = addUtcDays(t, 1)
else if (spec.kind === "week") t = addUtcDays(t, 7)
else t = new Date(Date.UTC(t.getUTCFullYear(), t.getUTCMonth() + 1, 1))
}
}
}
export async function dropExpiredPartitions(pool: Pool, around = new Date()): Promise<void> {
for (const spec of PARTITION_SPECS) {
const cutoff = addUtcDays(around, -spec.keepDays)
const { rows } = await pool.query<{ relname: string }>(
`SELECT c.relname
FROM pg_inherits i
JOIN pg_class c ON c.oid = i.inhrelid
JOIN pg_class p ON p.oid = i.inhparent
WHERE p.relname = $1`,
[spec.parent],
)
for (const row of rows) {
const m = row.relname.match(/_([dwm])_(\d{8}|\d{6})$/)
if (!m) continue
const stamp = m[2]
let start: Date
if (stamp.length === 6) {
start = new Date(Date.UTC(Number(stamp.slice(0, 4)), Number(stamp.slice(4, 6)) - 1, 1))
} else {
start = new Date(`${stamp.slice(0, 4)}-${stamp.slice(4, 6)}-${stamp.slice(6, 8)}T00:00:00Z`)
}
if (start < cutoff) {
await pool.query(`DROP TABLE IF EXISTS ${row.relname}`)
}
}
}
}
export function specForParent(parent: string): PartitionSpec | undefined {
return PARTITION_SPECS.find((s) => s.parent === parent)
}
+56
View File
@@ -0,0 +1,56 @@
import assert from "node:assert/strict"
import { withPgOrSkip } from "../test/pg.js"
import { dbQuery, pool } from "./index.js"
import { ensurePartitionFor } from "./partitions.js"
if (!(await withPgOrSkip())) {
console.log("pg-schema.test.ts: skip")
process.exit(0)
}
{
const { rows } = await dbQuery<{ n: string }>(`SELECT COUNT(*)::text AS n FROM servers`)
assert.ok(rows[0])
}
{
await dbQuery(`
INSERT INTO servers (name, host) VALUES ('pg-schema-test', '127.0.0.1')
`)
const { rows } = await dbQuery<{ id: number }>(`SELECT id FROM servers WHERE name = 'pg-schema-test' LIMIT 1`)
const id = rows[0]?.id
assert.ok(id)
await ensurePartitionFor(pool, "flow_buckets", "day", new Date())
const bucketAt = new Date().toISOString()
await dbQuery(`
INSERT INTO flow_buckets (
server_id, bucket_at, src, dst, proto, src_port, dst_port, bytes, packets, in_iface
) VALUES ($1, $2, '10.0.0.1', '8.8.8.8', 6, 1, 443, 10, 1, 'wg-flow')
ON CONFLICT DO NOTHING
`, [id, bucketAt])
await dbQuery(`DELETE FROM flow_buckets WHERE server_id = $1`, [id])
await dbQuery(`DELETE FROM servers WHERE id = $1`, [id])
}
{
await dbQuery(`
INSERT INTO alert_outbox (id, dedupe_key, payload_json, next_attempt_at)
VALUES ('pg-test-1', 'pg-dedupe-key', '{}'::jsonb, now())
ON CONFLICT (dedupe_key) DO NOTHING
`)
let threw = false
try {
await dbQuery(`
INSERT INTO alert_outbox (id, dedupe_key, payload_json, next_attempt_at)
VALUES ('pg-test-2', 'pg-dedupe-key', '{}'::jsonb, now())
`)
} catch {
threw = true
}
assert.equal(threw, true, "alert_outbox.dedupe_key UNIQUE")
await dbQuery(`DELETE FROM alert_outbox WHERE dedupe_key = 'pg-dedupe-key'`)
}
console.log("pg-schema.test.ts: ok")
+348 -364
View File
File diff suppressed because it is too large Load Diff
+21
View File
@@ -0,0 +1,21 @@
import assert from "node:assert/strict"
import { bindSql } from "./sql-bind.js"
{
const q = bindSql("SELECT 1")
assert.deepEqual(q, { text: "SELECT 1", values: [] })
}
{
const q = bindSql("SELECT * FROM t WHERE a = ? AND b = ?", [1, "x"])
assert.equal(q.text, "SELECT * FROM t WHERE a = $1 AND b = $2")
assert.deepEqual(q.values, [1, "x"])
}
{
const q = bindSql("INSERT INTO t (a, b) VALUES (@a, @b)", { a: 3, b: "y" })
assert.equal(q.text, "INSERT INTO t (a, b) VALUES ($1, $2)")
assert.deepEqual(q.values, [3, "y"])
}
console.log("sql-bind.test.ts: ok")
+22
View File
@@ -0,0 +1,22 @@
/** Convert SQLite-style @name / ? placeholders to node-pg $n. */
export function bindSql(
sql: string,
params?: unknown[] | Record<string, unknown>,
): { text: string; values: unknown[] } {
if (params == null) return { text: sql, values: [] }
if (Array.isArray(params)) {
let i = 0
const text = sql.replace(/\?/g, () => {
i += 1
return `$${i}`
})
return { text, values: params }
}
const values: unknown[] = []
const text = sql.replace(/@([a-zA-Z_][a-zA-Z0-9_]*)/g, (_, name: string) => {
values.push(params[name])
return `$${values.length}`
})
return { text, values }
}
+479
View File
@@ -0,0 +1,479 @@
import { createHash } from "node:crypto"
import { existsSync, readFileSync } from "node:fs"
import Database from "better-sqlite3"
import type { Pool } from "pg"
import { env } from "../config.js"
import { ensurePartitionFor, specForParent } from "./partitions.js"
export interface ImportReport {
sqlitePath: string
sqliteSha256: string
tables: Record<string, { sqlite: number; copied: number; skipped: number }>
rejects: string[]
durationMs: number
}
const SNAPSHOT_RETENTION_DAYS = 14
type ColKind = "ts" | "date" | "bool" | "json" | "json-null" | "bigint-id" | "int" | "text" | "num"
interface TableCopy {
table: string
columns: Array<[string, ColKind]>
timeCol?: string
retentionDays?: number
identity?: boolean
upsert?: boolean
}
const TABLES: TableCopy[] = [
{ table: "servers", identity: true, columns: [
["id", "int"], ["name", "text"], ["host", "text"], ["port", "int"], ["username", "text"],
["password", "text"], ["use_ssl", "bool"], ["verify_ssl", "bool"], ["type", "text"],
["site", "text"], ["country", "text"], ["asn", "text"], ["comment", "text"], ["enabled", "bool"],
["lan_subnet", "text"], ["wan_uplinks", "json"], ["mgmt_tunnel_ip", "text"],
["created_at", "ts"], ["updated_at", "ts"],
]},
{ table: "traffic_settings", upsert: true, columns: [
["id", "int"], ["enabled", "bool"], ["interval_sec", "int"], ["retention_days", "int"],
["last_collected_at", "ts"], ["last_duration_ms", "int"], ["last_error", "text"],
["created_at", "ts"], ["updated_at", "ts"],
]},
{ table: "servers_api_ping_settings", upsert: true, columns: [
["id", "int"], ["enabled", "bool"], ["interval_sec", "int"],
["last_collected_at", "ts"], ["last_duration_ms", "int"], ["last_error", "text"],
["created_at", "ts"], ["updated_at", "ts"],
]},
{ table: "traffic_flow_settings", upsert: true, columns: [
["id", "int"], ["enabled", "bool"], ["collector_ip", "text"], ["flow_listen_port", "int"],
["wg_listen_port", "int"], ["prefix", "text"], ["public_endpoint", "text"],
["host_public_key", "text"], ["host_private_key", "text"], ["hub_server_id", "int"],
["retention_hours", "int"], ["top_n", "int"], ["map_service_min_share_pct", "num"],
["last_datagram_at", "ts"], ["last_exporter_ip", "text"], ["last_error", "text"],
["packets_received", "int"], ["peers_json", "json"], ["created_at", "ts"], ["updated_at", "ts"],
]},
{ table: "uptime_settings", upsert: true, columns: [
["id", "int"], ["enabled", "bool"], ["resources_enabled", "bool"], ["ping_enabled", "bool"],
["speed_enabled", "bool"], ["interval_sec", "int"], ["probe_interval_sec", "int"],
["speed_interval_sec", "int"], ["retention_days", "int"], ["last_collected_at", "ts"],
["last_duration_ms", "int"], ["last_error", "text"], ["created_at", "ts"], ["updated_at", "ts"],
]},
{ table: "evobgp_settings", upsert: true, columns: [
["id", "int"], ["base_url", "text"], ["api_key", "text"], ["enabled", "bool"], ["updated_at", "ts"],
]},
{ table: "alert_telegram_settings", upsert: true, columns: [
["id", "int"], ["bot_token", "text"], ["chat_id", "text"], ["message_thread_id", "int"], ["updated_at", "ts"],
]},
{ table: "acme_settings", upsert: true, columns: [
["id", "int"], ["directory_url", "text"], ["cloudflare_api_token", "text"],
["default_zone_id", "text"], ["account_private_key", "text"], ["updated_at", "ts"],
]},
{ table: "certificate_renew_settings", upsert: true, columns: [
["id", "int"], ["enabled", "bool"], ["interval_sec", "int"], ["renew_before_days", "int"],
["last_collected_at", "ts"], ["last_duration_ms", "int"], ["last_error", "text"], ["updated_at", "ts"],
]},
{ table: "backup_schedule_settings", upsert: true, columns: [
["id", "int"], ["enabled", "bool"], ["frequency", "text"], ["hour", "int"], ["minute", "int"],
["week_day", "int"], ["month_day", "int"], ["keep_count", "int"], ["format", "text"],
["server_ids_json", "json"], ["last_run_at", "ts"], ["last_duration_ms", "int"],
["last_error", "text"], ["updated_at", "ts"],
]},
{ table: "internet_path_settings", upsert: true, columns: [
["id", "int"], ["enabled", "bool"], ["interval_sec", "int"], ["retention_days", "int"],
["last_collected_at", "ts"], ["last_duration_ms", "int"], ["last_error", "text"],
["created_at", "ts"], ["updated_at", "ts"],
]},
{ table: "alert_engine_cursor", upsert: true, columns: [
["id", "int"], ["last_source_finished_at", "ts"], ["updated_at", "ts"],
]},
{ table: "filter_rules", identity: true, columns: [
["id", "int"], ["server_id", "int"], ["sort_order", "int"], ["community", "text"],
["community_name", "text"], ["action", "text"], ["gateway", "text"],
["gateway_tunnel_id", "text"], ["description", "text"], ["created_at", "ts"], ["updated_at", "ts"],
]},
{ table: "recursive_routes", identity: true, columns: [
["id", "int"], ["server_id", "int"], ["sort_order", "int"], ["dst_address", "text"],
["gateway", "text"], ["distance", "int"], ["scope", "int"], ["target_scope", "int"],
["routing_table", "text"], ["check_gateway", "text"], ["country", "text"], ["comment", "text"],
["disabled", "bool"], ["created_at", "ts"], ["updated_at", "ts"],
]},
{ table: "server_snapshots", identity: true, timeCol: "polled_at", retentionDays: SNAPSHOT_RETENTION_DAYS, columns: [
["id", "int"], ["server_id", "int"], ["polled_at", "ts"], ["status", "text"], ["latency_ms", "num"],
["ros_version", "text"], ["board_name", "text"], ["uptime", "text"], ["cpu_load", "int"],
["free_memory", "int"], ["total_memory", "int"], ["identity_name", "text"],
["raw_interfaces", "json-null"], ["raw_ip_addresses", "json-null"],
]},
{ table: "traffic_samples", identity: true, timeCol: "sampled_at", retentionDays: 14, columns: [
["id", "int"], ["server_id", "int"], ["interface_name", "text"], ["peer_public_key", "text"],
["sampled_at", "ts"], ["rx_bytes", "int"], ["tx_bytes", "int"], ["rx_bps", "int"], ["tx_bps", "int"],
["running", "bool"], ["disabled", "bool"],
]},
{ table: "servers_rest_ping_samples", identity: true, timeCol: "sampled_at", retentionDays: 30, columns: [
["id", "int"], ["server_id", "int"], ["sampled_at", "ts"], ["ok", "bool"], ["latency_ms", "int"], ["error", "text"],
]},
{ table: "flow_buckets", timeCol: "bucket_at", retentionDays: 2, columns: [
["server_id", "int"], ["bucket_at", "ts"], ["src", "text"], ["dst", "text"], ["proto", "int"],
["src_port", "int"], ["dst_port", "int"], ["bytes", "int"], ["packets", "int"],
["in_iface", "text"], ["out_iface", "text"], ["next_hop", "text"],
["flow_start_ms", "int"], ["flow_end_ms", "int"],
]},
{ table: "flow_minute_stats", timeCol: "bucket_at", retentionDays: 3, columns: [
["server_id", "int"], ["bucket_at", "ts"], ["bytes", "int"], ["packets", "int"],
["unique_src", "int"], ["unique_dst", "int"], ["conversations", "int"],
]},
{ table: "flow_minute_dims", timeCol: "bucket_at", retentionDays: 3, columns: [
["server_id", "int"], ["bucket_at", "ts"], ["dim", "text"], ["key", "text"], ["bytes", "int"], ["packets", "int"],
]},
{ table: "flow_daily_dims", timeCol: "day", retentionDays: 396, columns: [
["server_id", "int"], ["day", "date"], ["dim", "text"], ["key", "text"], ["bytes", "int"], ["packets", "int"],
]},
{ table: "flow_ip_meta", columns: [
["prefix", "text"], ["asn", "int"], ["country", "text"], ["lat", "num"], ["lng", "num"],
["holder", "text"], ["ok", "int"], ["fetched_at", "ts"],
]},
{ table: "flow_asn_meta", columns: [
["asn", "int"], ["holder", "text"], ["fetched_at", "ts"],
]},
{ table: "uptime_probes", columns: [
["id", "text"], ["src_server_id", "int"], ["src_interface", "text"], ["name", "text"],
["target", "text"], ["probe_filter", "text"], ["enabled", "bool"], ["interval_sec", "int"],
["show_on_dashboard", "bool"], ["sort_order", "int"], ["created_at", "ts"], ["updated_at", "ts"],
]},
{ table: "uptime_probe_samples", identity: true, timeCol: "sampled_at", retentionDays: 14, columns: [
["id", "int"], ["probe_id", "text"], ["sampled_at", "ts"], ["rtt_ms", "int"], ["loss_pct", "int"], ["status", "text"],
]},
{ table: "uptime_resource_samples", identity: true, timeCol: "sampled_at", retentionDays: 14, columns: [
["id", "int"], ["server_id", "int"], ["sampled_at", "ts"], ["status", "text"], ["cpu_load", "int"],
["free_memory", "int"], ["total_memory", "int"], ["free_hdd_space", "int"], ["total_hdd_space", "int"],
["uptime_seconds", "int"], ["board_name", "text"], ["ros_version", "text"],
]},
{ table: "uptime_speed_probes", columns: [
["id", "text"], ["src_server_id", "int"], ["dst_server_id", "int"], ["src_interface", "text"],
["dst_interface", "text"], ["protocol", "text"], ["direction", "text"], ["duration_sec", "int"],
["enabled", "bool"], ["last_run_at", "ts"], ["last_tx_avg_mbps", "num"], ["last_rx_avg_mbps", "num"],
["last_status", "text"], ["last_error", "text"], ["last_ping_rtt_ms", "int"],
["last_ping_loss_pct", "int"], ["last_ping_at", "ts"], ["last_ping_error", "text"],
["sort_order", "int"], ["created_at", "ts"], ["updated_at", "ts"],
]},
{ table: "uptime_speed_test_runs", columns: [
["id", "text"], ["probe_id", "text"], ["src_server_id", "int"], ["dst_server_id", "int"],
["src_interface", "text"], ["dst_interface", "text"], ["src_address", "text"], ["dst_address", "text"],
["src_interface_address", "text"], ["dst_interface_address", "text"], ["protocol", "text"],
["direction", "text"], ["duration_sec", "int"], ["tx_avg_mbps", "num"], ["rx_avg_mbps", "num"],
["ping_rtt_ms", "int"], ["ping_loss_pct", "int"], ["ping_error", "text"], ["status", "text"],
["error", "text"], ["created_at", "ts"],
]},
{ table: "app_users", columns: [
["id", "text"], ["name", "text"], ["login", "text"], ["email", "text"], ["role", "text"],
["active", "bool"], ["avatar", "text"], ["last_seen", "ts"], ["sections_json", "json"],
["servers_json", "json"], ["created_at", "ts"], ["updated_at", "ts"],
]},
{ table: "user_interface_bindings", columns: [
["id", "text"], ["user_id", "text"], ["server_id", "int"], ["interface_name", "text"],
["interface_type", "text"], ["peer_public_key", "text"], ["peer_name", "text"],
["comment", "text"], ["created_at", "ts"], ["updated_at", "ts"],
]},
{ table: "alert_groups", columns: [
["id", "text"], ["name", "text"], ["combine_mode", "text"], ["enabled", "bool"],
["cooldown_override", "text"], ["created_at", "ts"], ["updated_at", "ts"],
]},
{ table: "alert_rules", columns: [
["id", "text"], ["name", "text"], ["type", "text"], ["target", "text"], ["condition", "text"],
["severity", "text"], ["enabled", "bool"], ["cooldown", "text"], ["rule_chat_id", "text"],
["confirm_stability_sec", "int"], ["recovery_mode", "text"], ["recovery_stability_sec", "int"],
["group_id", "text"], ["created_at", "ts"], ["updated_at", "ts"],
]},
{ table: "alert_rule_targets", columns: [
["id", "text"], ["rule_id", "text"], ["target", "text"], ["sort_index", "int"],
]},
{ table: "alert_rule_conditions", columns: [
["id", "text"], ["rule_id", "text"], ["condition_line", "text"], ["sort_index", "int"],
]},
{ table: "alert_engine_state", columns: [
["scope_key", "text"], ["last_fired_at", "text"], ["last_payload_hash", "text"],
]},
{ table: "alert_engine_prev_live", columns: [
["kind", "text"], ["payload_json", "json"], ["updated_at", "ts"],
]},
{ table: "alert_engine_confirm_pending", columns: [
["rule_id", "text"], ["payload_hash", "text"], ["since_at", "ts"],
]},
{ table: "alert_history", columns: [
["id", "text"], ["rule_id", "text"], ["group_id", "text"], ["rule_name", "text"],
["severity", "text"], ["message", "text"], ["sent_ok", "bool"], ["fired_at", "ts"],
]},
{ table: "alert_outbox", columns: [
["id", "text"], ["dedupe_key", "text"], ["channel", "text"], ["status", "text"],
["retry_count", "int"], ["max_retries", "int"], ["next_attempt_at", "ts"],
["payload_json", "json"], ["created_at", "ts"], ["sent_at", "ts"], ["last_error", "text"],
]},
{ table: "certificate_issue_jobs", columns: [
["id", "text"], ["status", "text"], ["step", "text"], ["source", "text"],
["server_id", "bigint-id"], ["cert_name", "text"], ["domain_names", "json"],
["key_type", "text"], ["trust_store", "text"], ["requested_at", "ts"],
["started_at", "ts"], ["finished_at", "ts"], ["error", "text"],
]},
{ table: "backup_entries", columns: [
["id", "text"], ["server_id", "bigint-id"], ["server_name", "text"], ["filename", "text"],
["size_bytes", "int"], ["kind", "text"], ["notes", "text"], ["created_at", "ts"],
]},
{ table: "scheduler_runs", timeCol: "started_at", retentionDays: 30, columns: [
["id", "text"], ["job_key", "text"], ["started_at", "ts"], ["finished_at", "ts"],
["status", "text"], ["error", "text"], ["duration_ms", "int"], ["result_json", "json-null"],
]},
{ table: "events", timeCol: "created_at", retentionDays: 30, columns: [
["id", "text"], ["created_at", "ts"], ["level", "text"], ["event_type", "text"],
["source_module", "text"], ["title", "text"], ["message", "text"],
["entity_type", "text"], ["entity_id", "text"], ["payload_json", "json-null"],
]},
{ table: "internet_path_snapshots", identity: true, timeCol: "sampled_at", retentionDays: 14, columns: [
["id", "int"], ["sampled_at", "ts"], ["payload_json", "json"],
]},
]
function parseTs(value: unknown, strict: boolean, rejects: string[], ctx: string): string | null {
if (value == null || value === "") return null
const s = String(value)
try {
if (/^\d{4}-\d{2}-\d{2}T/.test(s)) return new Date(s).toISOString()
if (/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/.test(s)) {
return new Date(`${s.replace(" ", "T")}Z`).toISOString()
}
const d = new Date(s)
if (!Number.isNaN(d.getTime())) return d.toISOString()
} catch {
/* fallthrough */
}
const msg = `${ctx}: неразобранный timestamp ${s}`
if (strict) throw new Error(msg)
rejects.push(msg)
return null
}
function parseDate(value: unknown): string | null {
if (value == null || value === "") return null
return String(value).slice(0, 10)
}
function parseBool(value: unknown): boolean {
return value === true || value === 1 || value === "1" || value === "true"
}
function parseJson(value: unknown, fallback: unknown): unknown {
if (value == null || value === "") return fallback
if (typeof value !== "string") return value
try {
return JSON.parse(value) as unknown
} catch {
return fallback
}
}
function coerce(kind: ColKind, value: unknown, strict: boolean, rejects: string[], ctx: string): unknown {
switch (kind) {
case "ts":
return parseTs(value, strict, rejects, ctx)
case "date":
return parseDate(value)
case "bool":
return parseBool(value)
case "json":
return parseJson(value, [])
case "json-null":
return value == null || value === "" ? null : parseJson(value, null)
case "bigint-id": {
const t = String(value ?? "").trim()
if (!t) return null
const n = Number(t)
if (!Number.isFinite(n)) {
const msg = `${ctx}: server_id ${t}`
if (strict) throw new Error(msg)
rejects.push(msg)
return null
}
return n
}
case "int":
if (value == null || value === "") return null
return Number(value)
case "num":
if (value == null || value === "") return null
return Number(value)
case "text":
return value == null ? "" : String(value)
default:
return value == null ? null : String(value)
}
}
function sqliteTableExists(sqlite: Database.Database, name: string): boolean {
const row = sqlite.prepare(
`SELECT 1 AS ok FROM sqlite_master WHERE type = 'table' AND name = ?`,
).get(name) as { ok?: number } | undefined
return Boolean(row)
}
async function setval(pool: Pool, table: string): Promise<void> {
await pool.query(
`SELECT setval(
pg_get_serial_sequence($1, 'id'),
GREATEST(COALESCE((SELECT MAX(id) FROM ${table}), 1), 1),
true
)`,
[table],
)
}
async function copyTable(
sqlite: Database.Database,
pool: Pool,
spec: TableCopy,
opts: { strict: boolean; fullHistory: boolean; rejects: string[] },
): Promise<{ sqlite: number; copied: number; skipped: number }> {
if (!sqliteTableExists(sqlite, spec.table)) {
return { sqlite: 0, copied: 0, skipped: 0 }
}
let where = ""
if (spec.timeCol && spec.retentionDays && !opts.fullHistory) {
const cutoff = new Date(Date.now() - spec.retentionDays * 86400_000).toISOString()
where = ` WHERE ${spec.timeCol} >= '${cutoff.replace("T", " ").slice(0, 19)}' OR ${spec.timeCol} >= '${cutoff}'`
}
const total = (sqlite.prepare(`SELECT COUNT(*) AS c FROM ${spec.table}${where}`).get() as { c: number }).c
const part = specForParent(spec.table)
const cols = spec.columns.map(([c]) => c)
const placeholders = cols.map((_, i) => `$${i + 1}`).join(", ")
const conflictSql = spec.upsert
? `ON CONFLICT (id) DO UPDATE SET ${cols.filter((c) => c !== "id").map((c) => `${c} = EXCLUDED.${c}`).join(", ")}`
: `ON CONFLICT DO NOTHING`
const insertSql = `INSERT INTO ${spec.table} (${cols.join(", ")}) VALUES (${placeholders}) ${conflictSql}`
let copied = 0
let skipped = 0
const stmt = sqlite.prepare(`SELECT * FROM ${spec.table}${where}`)
const batch: unknown[][] = []
const flush = async () => {
if (batch.length === 0) return
const client = await pool.connect()
try {
await client.query("BEGIN")
for (const values of batch) {
await client.query(insertSql, values)
copied += 1
}
await client.query("COMMIT")
} catch (err) {
await client.query("ROLLBACK")
throw err
} finally {
client.release()
batch.length = 0
}
}
for (const row of stmt.iterate() as Iterable<Record<string, unknown>>) {
try {
if (part && spec.timeCol) {
const raw = row[spec.timeCol]
const ts = spec.columns.find((c) => c[0] === spec.timeCol)?.[1] === "date"
? `${String(raw).slice(0, 10)}T00:00:00Z`
: parseTs(raw, false, opts.rejects, spec.table)
if (ts) await ensurePartitionFor(pool, spec.table, part.kind, new Date(ts))
}
const values = spec.columns.map(([col, kind]) =>
coerce(kind, row[col], opts.strict, opts.rejects, `${spec.table}.${col}`),
)
if (spec.table === "certificate_issue_jobs" && values[4] == null) {
skipped += 1
continue
}
batch.push(values)
if (batch.length >= 200) await flush()
} catch (err) {
skipped += 1
const msg = `${spec.table}: ${err instanceof Error ? err.message : String(err)}`
opts.rejects.push(msg)
if (opts.strict) throw err
}
}
await flush()
if (spec.identity) {
try {
await setval(pool, spec.table)
} catch {
/* partitioned identity sequence name may differ */
}
}
return { sqlite: total, copied, skipped }
}
export function sqliteFileLooksPresent(path: string): boolean {
if (!existsSync(path) || path === ":memory:") return false
try {
const buf = readFileSync(path)
return buf.subarray(0, 16).toString("utf8").startsWith("SQLite format 3")
} catch {
return false
}
}
export async function shouldImportSqlite(pool: Pool, sqlitePath: string): Promise<boolean> {
if (!sqliteFileLooksPresent(sqlitePath)) return false
const marker = await pool.query<{ sqlite_imported_at: string | null }>(
`SELECT sqlite_imported_at FROM data_migration WHERE id = 1`,
)
if (marker.rows[0]?.sqlite_imported_at) return false
const servers = await pool.query<{ c: string }>(`SELECT COUNT(*)::text AS c FROM servers`)
if (Number(servers.rows[0]?.c ?? 0) > 0) return false
return true
}
export async function importSqliteToPostgres(
pool: Pool,
sqlitePath: string,
opts?: { dryRun?: boolean; strict?: boolean; fullHistory?: boolean },
): Promise<ImportReport> {
const started = Date.now()
const rejects: string[] = []
const sqlite = new Database(sqlitePath, { readonly: true, fileMustExist: true })
try {
sqlite.pragma("journal_mode = WAL")
} catch {
/* readonly */
}
const sha = createHash("sha256").update(readFileSync(sqlitePath)).digest("hex")
const report: ImportReport = {
sqlitePath,
sqliteSha256: sha,
tables: {},
rejects,
durationMs: 0,
}
const fullHistory = opts?.fullHistory ?? env.sqliteImportFullHistory
const strict = opts?.strict ?? true
if (opts?.dryRun) {
for (const spec of TABLES) {
if (!sqliteTableExists(sqlite, spec.table)) {
report.tables[spec.table] = { sqlite: 0, copied: 0, skipped: 0 }
continue
}
const c = (sqlite.prepare(`SELECT COUNT(*) AS c FROM ${spec.table}`).get() as { c: number }).c
report.tables[spec.table] = { sqlite: c, copied: 0, skipped: 0 }
}
sqlite.close()
report.durationMs = Date.now() - started
return report
}
for (const spec of TABLES) {
report.tables[spec.table] = await copyTable(sqlite, pool, spec, { strict, fullHistory, rejects })
}
sqlite.close()
await pool.query(
`UPDATE data_migration
SET sqlite_imported_at = now(), sqlite_path = $1, sqlite_sha256 = $2, report_json = $3::jsonb
WHERE id = 1`,
[sqlitePath, sha, JSON.stringify(report.tables)],
)
report.durationMs = Date.now() - started
if (strict && rejects.length > 0) {
throw new Error(`SQLite import strict: ${rejects.length} rejects\n${rejects.slice(0, 20).join("\n")}`)
}
return report
}