perf(db): сжать схему PostgreSQL 18 и повторно загрузить SQLite

При первом рестарте wipe всех таблиц и импорт из SQLite в компактную схему. Retention через DROP PARTITION, lz4 и AIO worker.

Co-authored-by: Cursor <[email protected]>

backend/src/services/internet-path-collector.ts

@@ -1,4 +1,4 @@
-import { and, asc, desc, eq, lt } from "drizzle-orm"
+import { and, asc, desc, eq } from "drizzle-orm"
 import { db } from "../db/index.js"
 import {
   filterRules,
@@ -45,11 +45,6 @@ async function getSettingsRow() {
   return (await db.select().from(internetPathSettings).where(eq(internetPathSettings.id, 1)).limit(1))[0]
 }
 
-async function cleanupSnapshots(retentionDays: number) {
-  const cutoff = new Date(Date.now() - retentionDays * 24 * 60 * 60 * 1000).toISOString()
-  await db.delete(internetPathSnapshots).where(lt(internetPathSnapshots.sampledAt, cutoff))
-}
-
 async function buildRulesets() {
   const enabled = await db.select().from(servers).where(eq(servers.enabled, true))
   const rules = await db.select().from(filterRules).orderBy(asc(filterRules.serverId), asc(filterRules.sortOrder))
@@ -273,7 +268,6 @@ export async function collectInternetPathSnapshotOnce(): Promise<InternetPathRun
       sampledAt,
       payloadJson: payload,
     })
-    await cleanupSnapshots(Math.max(1, settings.retentionDays))
     await db.update(internetPathSettings).set({
       lastCollectedAt: sampledAt,
       lastDurationMs: Date.now() - started,

backend/src/services/poller.ts

@@ -1,5 +1,5 @@
-import { eq, lt } from "drizzle-orm"
-import { db, pool } from "../db/index.js"
+import { eq } from "drizzle-orm"
+import { db, dbQuery, pool } from "../db/index.js"
 import { dropExpiredPartitions, ensurePartitionsAround } from "../db/partitions.js"
 import { servers, serverSnapshots } from "../db/schema.js"
 import type { SnapshotInsert } from "../db/schema.js"
@@ -82,11 +82,18 @@ export async function pollServer(serverId: number): Promise<SnapshotRead> {
     .values(partialSnap as SnapshotInsert)
     .returning()
 
-  const cutoffIso = new Date(Date.now() - 14 * 24 * 3600_000).toISOString()
-  await db.delete(serverSnapshots).where(lt(serverSnapshots.polledAt, cutoffIso))
+  if (inserted) {
+    await dbQuery(
+      `UPDATE server_snapshots
+       SET raw_interfaces = NULL, raw_ip_addresses = NULL
+       WHERE server_id = $1 AND polled_at < $2
+         AND (raw_interfaces IS NOT NULL OR raw_ip_addresses IS NOT NULL)`,
+      [serverId, inserted.polledAt],
+    )
+  }
   void dropExpiredPartitions(pool).then(() => ensurePartitionsAround(pool))
 
-  return toSnapshotRead(inserted)
+  return toSnapshotRead(inserted!)
 }
 
 // ── helper ─────────────────────────────────────────────────────────────────────

backend/src/services/servers-rest-ping-collector.ts

@@ -162,8 +162,6 @@ export async function collectServersRestPingOnce(): Promise<ServersRestPingRunSn
       }
     })
 
-    await dbQuery(`DELETE FROM servers_rest_ping_samples WHERE sampled_at < now() - interval '30 days'`)
-
     await db.update(serversApiPingSettings)
       .set({
         lastCollectedAt: sampledAt,

backend/src/services/traffic-collector.ts

@@ -1,6 +1,7 @@
-import { and, asc, desc, eq, gte, lt } from "drizzle-orm"
+import { and, asc, desc, eq, gte } from "drizzle-orm"
 import { db } from "../db/index.js"
 import { servers, trafficSamples, trafficSettings } from "../db/schema.js"
+import { decodeTrafficSampleFlags, encodeTrafficFlags } from "../db/traffic-flags.js"
 import type { TrafficRunSnapshot } from "../types/scheduler-run-snapshot.js"
 import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
 import { MikrotikClient } from "./mikrotik.js"
@@ -79,12 +80,6 @@ async function getSettingsRow() {
   return (await db.select().from(trafficSettings).where(eq(trafficSettings.id, 1)).limit(1))[0]
 }
 
-async function cleanupOldSamples(retentionDays: number) {
-  const cutoff = new Date(Date.now() - retentionDays * 24 * 60 * 60 * 1000).toISOString()
-  await db.delete(trafficSamples)
-    .where(lt(trafficSamples.sampledAt, cutoff))
-}
-
 async function readPreviousWave(serverId: number): Promise<Map<string, { rxBytes: number; txBytes: number; sampledAt: string }>> {
   const last = (await db
     .select({ sampledAt: trafficSamples.sampledAt })
@@ -164,8 +159,7 @@ export async function collectTrafficOnce(): Promise<TrafficRunSnapshot> {
             txBytes,
             rxBps,
             txBps,
-            running,
-            disabled,
+            flags: encodeTrafficFlags(running, disabled),
           }
         })
         try {
@@ -194,8 +188,7 @@ export async function collectTrafficOnce(): Promise<TrafficRunSnapshot> {
               txBytes,
               rxBps,
               txBps,
-              running,
-              disabled,
+              flags: encodeTrafficFlags(running, disabled),
             })
           }
         } catch {
@@ -224,7 +217,6 @@ export async function collectTrafficOnce(): Promise<TrafficRunSnapshot> {
       }
     }
 
-    await cleanupOldSamples(Math.max(1, settings.retentionDays))
     await db.update(trafficSettings).set({
       lastCollectedAt: now,
       lastDurationMs: Date.now() - startedAt,
@@ -286,11 +278,12 @@ export async function updateTrafficSettings(patch: {
 }
 
 export async function readServerSamplesInRange(serverId: number, sinceIso: string) {
-  return await db.select()
+  const rows = await db.select()
     .from(trafficSamples)
     .where(and(
       eq(trafficSamples.serverId, serverId),
       gte(trafficSamples.sampledAt, sinceIso),
     ))
     .orderBy(asc(trafficSamples.sampledAt))
+  return rows.map((r) => ({ ...r, ...decodeTrafficSampleFlags(r.flags) }))
 }

backend/src/services/traffic-flow-engine.ts

@@ -48,6 +48,30 @@ export interface PendingFlowRow {
   flowEndMs: number
 }
 
+function inetOrNull(value: string | null | undefined): string | null {
+  const s = String(value ?? "").trim()
+  return s.length > 0 ? s : null
+}
+
+function flowUpsertParams(r: PendingFlowRow) {
+  return {
+    serverId: r.serverId,
+    bucketAt: r.bucketAt,
+    src: r.src,
+    dst: r.dst,
+    proto: r.proto,
+    srcPort: r.srcPort,
+    dstPort: r.dstPort,
+    bytes: r.bytes,
+    packets: r.packets,
+    inIface: r.inIface,
+    outIface: r.outIface,
+    nextHop: inetOrNull(r.nextHop),
+    flowStartMs: r.flowStartMs,
+    flowEndMs: r.flowEndMs,
+  }
+}
+
 export interface EngineStats {
   packetsReceived: number
   lastExporterIp: string | null
@@ -668,7 +692,7 @@ export async function flushPending(): Promise<void> {
       bytes = flow_buckets.bytes + excluded.bytes,
       packets = flow_buckets.packets + excluded.packets,
       out_iface = CASE WHEN excluded.out_iface != '' THEN excluded.out_iface ELSE flow_buckets.out_iface END,
-      next_hop = CASE WHEN excluded.next_hop != '' THEN excluded.next_hop ELSE flow_buckets.next_hop END,
+      next_hop = COALESCE(excluded.next_hop, flow_buckets.next_hop),
       flow_start_ms = CASE
         WHEN excluded.flow_start_ms > 0 AND (flow_buckets.flow_start_ms = 0 OR excluded.flow_start_ms < flow_buckets.flow_start_ms)
         THEN excluded.flow_start_ms ELSE flow_buckets.flow_start_ms END,
@@ -678,22 +702,7 @@ export async function flushPending(): Promise<void> {
   try {
     for (const r of rows) {
       await ensureParentPartition("flow_buckets", r.bucketAt)
-      await dbQuery(upsertSql, {
-        serverId: r.serverId,
-        bucketAt: r.bucketAt,
-        src: r.src,
-        dst: r.dst,
-        proto: r.proto,
-        srcPort: r.srcPort,
-        dstPort: r.dstPort,
-        bytes: r.bytes,
-        packets: r.packets,
-        inIface: r.inIface,
-        outIface: r.outIface,
-        nextHop: r.nextHop,
-        flowStartMs: r.flowStartMs,
-        flowEndMs: r.flowEndMs,
-      })
+      await dbQuery(upsertSql, flowUpsertParams(r))
     }
     lastFlushUsedTransaction = true
     rowsStored += rows.length
@@ -702,22 +711,7 @@ export async function flushPending(): Promise<void> {
     for (const r of rows) {
       try {
         await ensureParentPartition("flow_buckets", r.bucketAt)
-        await dbQuery(upsertSql, {
-          serverId: r.serverId,
-          bucketAt: r.bucketAt,
-          src: r.src,
-          dst: r.dst,
-          proto: r.proto,
-          srcPort: r.srcPort,
-          dstPort: r.dstPort,
-          bytes: r.bytes,
-          packets: r.packets,
-          inIface: r.inIface,
-          outIface: r.outIface,
-          nextHop: r.nextHop,
-          flowStartMs: r.flowStartMs,
-          flowEndMs: r.flowEndMs,
-        })
+        await dbQuery(upsertSql, flowUpsertParams(r))
         rowsStored += 1
       } catch {
         /* ignore single-row failures */

backend/src/services/uptime-collector.ts

@@ -1,4 +1,4 @@
-import { and, asc, eq, gte, inArray, lt, max, or } from "drizzle-orm"
+import { and, asc, eq, gte, inArray, max, or } from "drizzle-orm"
 import { db } from "../db/index.js"
 import {
   servers,
@@ -63,12 +63,6 @@ export async function getSettings() {
   return (await db.select().from(uptimeSettings).where(eq(uptimeSettings.id, 1)).limit(1))[0]
 }
 
-async function cleanup(retentionDays: number) {
-  const cutoff = new Date(Date.now() - retentionDays * 24 * 60 * 60 * 1000).toISOString()
-  await db.delete(uptimeProbeSamples).where(lt(uptimeProbeSamples.sampledAt, cutoff))
-  await db.delete(uptimeResourceSamples).where(lt(uptimeResourceSamples.sampledAt, cutoff))
-}
-
 export function parsePing(results: Array<{ time?: string; status?: string; sent?: string; received?: string; "packet-loss"?: string; "avg-rtt"?: string }>) {
   const sum = [...results].reverse().find((r) =>
     r.sent != null || r.received != null || r["packet-loss"] != null || r["avg-rtt"] != null,
@@ -166,8 +160,6 @@ export async function collectResourceSamplesOnce(): Promise<ResourcesRunSnapshot
           freeHddSpace: freeHdd,
           totalHddSpace: totalHdd,
           uptimeSeconds,
-          boardName: String(resource["board-name"] ?? ""),
-          rosVersion: String(resource["version"] ?? ""),
         })
         const memUsedMb = totalMem > 0 ? Math.round((totalMem - freeMem) / (1024 * 1024)) : 0
         const memTotalMb = totalMem > 0 ? Math.round(totalMem / (1024 * 1024)) : 0
@@ -198,8 +190,6 @@ export async function collectResourceSamplesOnce(): Promise<ResourcesRunSnapshot
           freeHddSpace: 0,
           totalHddSpace: 0,
           uptimeSeconds: 0,
-          boardName: "",
-          rosVersion: "",
         })
         snapshot.servers.push({
           serverId: s.id,
@@ -211,7 +201,6 @@ export async function collectResourceSamplesOnce(): Promise<ResourcesRunSnapshot
       }
     }
 
-    await cleanup(Math.max(1, settings.retentionDays))
     await db.update(uptimeSettings).set({
       lastCollectedAt: now,
       lastDurationMs: Date.now() - started,
@@ -324,7 +313,6 @@ export async function collectPingProbesOnce(): Promise<PingRunSnapshot> {
       }
     }
 
-    await cleanup(Math.max(1, settings.retentionDays))
     await db.update(uptimeSettings).set({
       lastCollectedAt: now,
       lastDurationMs: Date.now() - started,
@@ -399,7 +387,6 @@ export async function collectPingForProbeIds(probeIds: string[]): Promise<{ poll
       polled += 1
     }
 
-    await cleanup(Math.max(1, settings.retentionDays))
     return { polled }
   } finally {
     collectingPing = false