perf(db): сжать схему PostgreSQL 18 и повторно загрузить SQLite

При первом рестарте wipe всех таблиц и импорт из SQLite в компактную схему. Retention через DROP PARTITION, lz4 и AIO worker.

Co-authored-by: Cursor <[email protected]>

backend/src/db/sqlite-import.ts

@@ -3,7 +3,8 @@ import { existsSync, readFileSync } from "node:fs"
 import Database from "better-sqlite3"
 import type { Pool } from "pg"
 import { env } from "../config.js"
-import { ensurePartitionFor, specForParent } from "./partitions.js"
+import { ensurePartitionsBetween, specForParent } from "./partitions.js"
+import { encodeTrafficFlags } from "./traffic-flags.js"
 
 export interface ImportReport {
   sqlitePath: string
@@ -15,7 +16,9 @@ export interface ImportReport {
 
 const SNAPSHOT_RETENTION_DAYS = 14
 
-type ColKind = "ts" | "date" | "bool" | "json" | "json-null" | "bigint-id" | "int" | "text" | "num"
+type ColKind = "ts" | "date" | "bool" | "json" | "json-null" | "bigint-id" | "int" | "text" | "num" | "flags" | "inet"
+
+const INSERT_CHUNK = 1000
 
 interface TableCopy {
   table: string
@@ -103,18 +106,18 @@ const TABLES: TableCopy[] = [
     ["free_memory", "int"], ["total_memory", "int"], ["identity_name", "text"],
     ["raw_interfaces", "json-null"], ["raw_ip_addresses", "json-null"],
   ]},
-  { table: "traffic_samples", identity: true, timeCol: "sampled_at", retentionDays: 14, columns: [
-    ["id", "int"], ["server_id", "int"], ["interface_name", "text"], ["peer_public_key", "text"],
+  { table: "traffic_samples", timeCol: "sampled_at", retentionDays: 14, columns: [
+    ["server_id", "int"], ["interface_name", "text"], ["peer_public_key", "text"],
     ["sampled_at", "ts"], ["rx_bytes", "int"], ["tx_bytes", "int"], ["rx_bps", "int"], ["tx_bps", "int"],
-    ["running", "bool"], ["disabled", "bool"],
+    ["flags", "flags"],
   ]},
-  { table: "servers_rest_ping_samples", identity: true, timeCol: "sampled_at", retentionDays: 30, columns: [
-    ["id", "int"], ["server_id", "int"], ["sampled_at", "ts"], ["ok", "bool"], ["latency_ms", "int"], ["error", "text"],
+  { table: "servers_rest_ping_samples", timeCol: "sampled_at", retentionDays: 30, columns: [
+    ["server_id", "int"], ["sampled_at", "ts"], ["ok", "bool"], ["latency_ms", "int"], ["error", "text"],
   ]},
   { table: "flow_buckets", timeCol: "bucket_at", retentionDays: 2, columns: [
-    ["server_id", "int"], ["bucket_at", "ts"], ["src", "text"], ["dst", "text"], ["proto", "int"],
+    ["server_id", "int"], ["bucket_at", "ts"], ["src", "inet"], ["dst", "inet"], ["proto", "int"],
     ["src_port", "int"], ["dst_port", "int"], ["bytes", "int"], ["packets", "int"],
-    ["in_iface", "text"], ["out_iface", "text"], ["next_hop", "text"],
+    ["in_iface", "text"], ["out_iface", "text"], ["next_hop", "inet"],
     ["flow_start_ms", "int"], ["flow_end_ms", "int"],
   ]},
   { table: "flow_minute_stats", timeCol: "bucket_at", retentionDays: 3, columns: [
@@ -139,13 +142,13 @@ const TABLES: TableCopy[] = [
     ["target", "text"], ["probe_filter", "text"], ["enabled", "bool"], ["interval_sec", "int"],
     ["show_on_dashboard", "bool"], ["sort_order", "int"], ["created_at", "ts"], ["updated_at", "ts"],
   ]},
-  { table: "uptime_probe_samples", identity: true, timeCol: "sampled_at", retentionDays: 14, columns: [
-    ["id", "int"], ["probe_id", "text"], ["sampled_at", "ts"], ["rtt_ms", "int"], ["loss_pct", "int"], ["status", "text"],
+  { table: "uptime_probe_samples", timeCol: "sampled_at", retentionDays: 14, columns: [
+    ["probe_id", "text"], ["sampled_at", "ts"], ["rtt_ms", "int"], ["loss_pct", "int"], ["status", "text"],
   ]},
-  { table: "uptime_resource_samples", identity: true, timeCol: "sampled_at", retentionDays: 14, columns: [
-    ["id", "int"], ["server_id", "int"], ["sampled_at", "ts"], ["status", "text"], ["cpu_load", "int"],
+  { table: "uptime_resource_samples", timeCol: "sampled_at", retentionDays: 14, columns: [
+    ["server_id", "int"], ["sampled_at", "ts"], ["status", "text"], ["cpu_load", "int"],
     ["free_memory", "int"], ["total_memory", "int"], ["free_hdd_space", "int"], ["total_hdd_space", "int"],
-    ["uptime_seconds", "int"], ["board_name", "text"], ["ros_version", "text"],
+    ["uptime_seconds", "int"],
   ]},
   { table: "uptime_speed_probes", columns: [
     ["id", "text"], ["src_server_id", "int"], ["dst_server_id", "int"], ["src_interface", "text"],
@@ -269,7 +272,20 @@ function parseJson(value: unknown, fallback: unknown): unknown {
   }
 }
 
-function coerce(kind: ColKind, value: unknown, strict: boolean, rejects: string[], ctx: string): unknown {
+function parseInet(value: unknown): string | null {
+  const s = String(value ?? "").trim()
+  if (!s) return null
+  return s
+}
+
+function coerce(
+  kind: ColKind,
+  value: unknown,
+  strict: boolean,
+  rejects: string[],
+  ctx: string,
+  row?: Record<string, unknown>,
+): unknown {
   switch (kind) {
     case "ts":
       return parseTs(value, strict, rejects, ctx)
@@ -303,6 +319,10 @@ function coerce(kind: ColKind, value: unknown, strict: boolean, rejects: string[
       return Number(value)
     case "text":
       return value == null ? "" : String(value)
+    case "flags":
+      return encodeTrafficFlags(parseBool(row?.running), parseBool(row?.disabled))
+    case "inet":
+      return parseInet(value)
     default:
       return value == null ? null : String(value)
   }
@@ -326,6 +346,35 @@ async function setval(pool: Pool, table: string): Promise<void> {
   )
 }
 
+function placeholderFor(kind: ColKind, index: number): string {
+  if (kind === "json" || kind === "json-null") return `$${index}::jsonb`
+  if (kind === "inet") return `$${index}::inet`
+  return `$${index}`
+}
+
+async function precreatePartitions(
+  sqlite: Database.Database,
+  pool: Pool,
+  spec: TableCopy,
+  where: string,
+): Promise<void> {
+  const part = specForParent(spec.table)
+  if (!part || !spec.timeCol) return
+  const bounds = sqlite.prepare(
+    `SELECT MIN(${spec.timeCol}) AS a, MAX(${spec.timeCol}) AS b FROM ${spec.table}${where}`,
+  ).get() as { a?: unknown; b?: unknown }
+  if (bounds?.a == null || bounds?.b == null) return
+  const isDate = spec.columns.find((c) => c[0] === spec.timeCol)?.[1] === "date"
+  const fromIso = isDate
+    ? `${String(bounds.a).slice(0, 10)}T00:00:00Z`
+    : parseTs(bounds.a, false, [], spec.table)
+  const toIso = isDate
+    ? `${String(bounds.b).slice(0, 10)}T00:00:00Z`
+    : parseTs(bounds.b, false, [], spec.table)
+  if (!fromIso || !toIso) return
+  await ensurePartitionsBetween(pool, spec.table, part.kind, new Date(fromIso), new Date(toIso))
+}
+
 async function copyTable(
   sqlite: Database.Database,
   pool: Pool,
@@ -341,28 +390,27 @@ async function copyTable(
     where = ` WHERE ${spec.timeCol} >= '${cutoff.replace("T", " ").slice(0, 19)}' OR ${spec.timeCol} >= '${cutoff}'`
   }
   const total = (sqlite.prepare(`SELECT COUNT(*) AS c FROM ${spec.table}${where}`).get() as { c: number }).c
-  const part = specForParent(spec.table)
+  await precreatePartitions(sqlite, pool, spec, where)
   const cols = spec.columns.map(([c]) => c)
-  const placeholders = spec.columns
-    .map(([, kind], i) => (kind === "json" || kind === "json-null" ? `$${i + 1}::jsonb` : `$${i + 1}`))
-    .join(", ")
   const conflictSql = spec.upsert
     ? `ON CONFLICT (id) DO UPDATE SET ${cols.filter((c) => c !== "id").map((c) => `${c} = EXCLUDED.${c}`).join(", ")}`
     : `ON CONFLICT DO NOTHING`
-  const insertSql = `INSERT INTO ${spec.table} (${cols.join(", ")}) VALUES (${placeholders}) ${conflictSql}`
   let copied = 0
   let skipped = 0
   const stmt = sqlite.prepare(`SELECT * FROM ${spec.table}${where}`)
   const batch: unknown[][] = []
   const flush = async () => {
     if (batch.length === 0) return
+    const n = spec.columns.length
+    const valuesSql = batch.map((_, i) =>
+      `(${spec.columns.map(([, kind], j) => placeholderFor(kind, i * n + j + 1)).join(", ")})`,
+    ).join(", ")
+    const insertSql = `INSERT INTO ${spec.table} (${cols.join(", ")}) VALUES ${valuesSql} ${conflictSql}`
     const client = await pool.connect()
     try {
       await client.query("BEGIN")
-      for (const values of batch) {
-        await client.query(insertSql, values)
-        copied += 1
-      }
+      await client.query(insertSql, batch.flat())
+      copied += batch.length
       await client.query("COMMIT")
     } catch (err) {
       await client.query("ROLLBACK")
@@ -375,22 +423,15 @@ async function copyTable(
   }
   for (const row of stmt.iterate() as Iterable<Record<string, unknown>>) {
     try {
-      if (part && spec.timeCol) {
-        const raw = row[spec.timeCol]
-        const ts = spec.columns.find((c) => c[0] === spec.timeCol)?.[1] === "date"
-          ? `${String(raw).slice(0, 10)}T00:00:00Z`
-          : parseTs(raw, false, opts.rejects, spec.table)
-        if (ts) await ensurePartitionFor(pool, spec.table, part.kind, new Date(ts))
-      }
       const values = spec.columns.map(([col, kind]) =>
-        coerce(kind, row[col], opts.strict, opts.rejects, `${spec.table}.${col}`),
+        coerce(kind, row[col], opts.strict, opts.rejects, `${spec.table}.${col}`, row),
       )
       if (spec.table === "certificate_issue_jobs" && values[4] == null) {
         skipped += 1
         continue
       }
       batch.push(values)
-      if (batch.length >= 200) await flush()
+      if (batch.length >= INSERT_CHUNK) await flush()
     } catch (err) {
       skipped += 1
       const msg = `${spec.table}: ${err instanceof Error ? err.message : String(err)}`