fix(db): исправить импорт jsonb из sqlite

JSON-колонки передаются строкой с приведением ::jsonb, иначе node-pg кодирует массивы как PostgreSQL array и ETL падает. Повтор импорта идёт, пока нет маркера. Том PG18 монтируется в /var/lib/postgresql.

Co-authored-by: Cursor <[email protected]>

README.md

@@ -128,7 +128,7 @@ sequenceDiagram
 | Зависимость | Реализация |
 |-------------|------------|
 | PostgreSQL | Контейнер `mmapp-postgres` (`postgres:18-alpine`). `DATABASE_URL=postgres://mmapp:…@postgres:5432/mmapp`. |
-| SQLite (ETL) | Файл `mikrotik.db` на томе `/app/data` (`DATABASE_PATH=/app/data/mikrotik.db`). При первом старте, если PG пустой, backend сам импортирует данные и ставит маркер. Повторный старт не копирует заново. |
+| SQLite (ETL) | Файл `mikrotik.db` на томе `/app/data` (`DATABASE_PATH=/app/data/mikrotik.db`). При старте, пока нет маркера `data_migration.sqlite_imported_at`, backend импортирует sqlite в PG (`ON CONFLICT` / upsert). После успешного импорта повтор не копирует заново. Том PG18: `mmapp-pgdata:/var/lib/postgresql` (не `.../data`). |
 | Docker socket | Только у контейнера updater: `/var/run/docker.sock` — доступ к Docker API хоста (управление контейнерами, pull). |
 
 ## Локальная разработка

backend/package.json

@@ -17,7 +17,7 @@
     "test:traffic-rate": "tsx src/services/traffic-rate.test.ts",
     "test:traffic-flow": "tsx src/services/traffic-flow-parse.test.ts && tsx src/services/traffic-flow-map-exporter.test.ts && tsx src/services/traffic-flow-ifaces.test.ts && tsx src/services/traffic-flow-dedup.test.ts && tsx src/services/traffic-flow-planes.test.ts && tsx src/services/traffic-flow-ip.test.ts && tsx src/services/traffic-flow-classify.test.ts && tsx src/services/traffic-flow-ripe.test.ts && tsx src/services/traffic-flow-brands.test.ts && tsx src/services/traffic-flow-ingest.test.ts && tsx src/services/traffic-flow-analytics.test.ts && tsx src/services/traffic-flow-map-hops.test.ts && tsx src/services/traffic-flow-purge.test.ts",
     "test:users": "tsx src/modules/users/iface-type.test.ts && tsx src/modules/users/bindings.test.ts",
-    "test:pg": "tsx src/db/sql-bind.test.ts && tsx src/db/pg-schema.test.ts",
+    "test:pg": "tsx src/db/sql-bind.test.ts && tsx src/db/sqlite-json.test.ts && tsx src/db/pg-schema.test.ts",
     "test": "npm run test:alert-engine && npm run test:auth && npm run test:wireguard && npm run test:traffic-rate && npm run test:traffic-flow && npm run test:users && npm run test:pg"
   },
   "dependencies": {

backend/src/db/pg-schema.test.ts

@@ -53,4 +53,31 @@ if (!(await withPgOrSkip())) {
   await dbQuery(`DELETE FROM alert_outbox WHERE dedupe_key = 'pg-dedupe-key'`)
 }
 
+{
+  const peers = [{ endpoint: "msk-gw02.rtnt.top:13232", publicKey: "x" }]
+  await dbQuery(
+    `INSERT INTO alert_outbox (id, dedupe_key, payload_json, next_attempt_at)
+     VALUES ('pg-json-arr', 'pg-json-arr', $1::jsonb, now())`,
+    [JSON.stringify(peers)],
+  )
+  const { rows } = await dbQuery<{ payload_json: unknown }>(
+    `SELECT payload_json FROM alert_outbox WHERE id = 'pg-json-arr'`,
+  )
+  assert.equal(Array.isArray(rows[0]?.payload_json), true)
+  await dbQuery(`DELETE FROM alert_outbox WHERE id = 'pg-json-arr'`)
+
+  let arrayAsPgArrayFailed = false
+  try {
+    await dbQuery(
+      `INSERT INTO alert_outbox (id, dedupe_key, payload_json, next_attempt_at)
+       VALUES ('pg-json-bad', 'pg-json-bad', $1, now())`,
+      [peers],
+    )
+  } catch (err) {
+    arrayAsPgArrayFailed = err instanceof Error && /json|22P02/i.test(err.message)
+  }
+  await dbQuery(`DELETE FROM alert_outbox WHERE id = 'pg-json-bad'`).catch(() => undefined)
+  assert.equal(arrayAsPgArrayFailed, true, "JS array must not be bound as jsonb without stringify")
+}
+
 console.log("pg-schema.test.ts: ok")

backend/src/db/sqlite-import.ts

@@ -278,9 +278,11 @@ function coerce(kind: ColKind, value: unknown, strict: boolean, rejects: string[
     case "bool":
       return parseBool(value)
     case "json":
-      return parseJson(value, [])
+      // Always JSON text. JS arrays must not go to node-pg as values — it encodes
+      // them as PG arrays (`{...}`), which jsonb rejects (22P02).
+      return JSON.stringify(parseJson(value, []))
     case "json-null":
-      return value == null || value === "" ? null : parseJson(value, null)
+      return value == null || value === "" ? null : JSON.stringify(parseJson(value, null))
     case "bigint-id": {
       const t = String(value ?? "").trim()
       if (!t) return null
@@ -341,7 +343,9 @@ async function copyTable(
   const total = (sqlite.prepare(`SELECT COUNT(*) AS c FROM ${spec.table}${where}`).get() as { c: number }).c
   const part = specForParent(spec.table)
   const cols = spec.columns.map(([c]) => c)
-  const placeholders = cols.map((_, i) => `$${i + 1}`).join(", ")
+  const placeholders = spec.columns
+    .map(([, kind], i) => (kind === "json" || kind === "json-null" ? `$${i + 1}::jsonb` : `$${i + 1}`))
+    .join(", ")
   const conflictSql = spec.upsert
     ? `ON CONFLICT (id) DO UPDATE SET ${cols.filter((c) => c !== "id").map((c) => `${c} = EXCLUDED.${c}`).join(", ")}`
     : `ON CONFLICT DO NOTHING`
@@ -362,7 +366,8 @@ async function copyTable(
       await client.query("COMMIT")
     } catch (err) {
       await client.query("ROLLBACK")
-      throw err
+      const detail = err instanceof Error ? err.message : String(err)
+      throw new Error(`${spec.table}: ${detail}`)
     } finally {
       client.release()
       batch.length = 0
@@ -421,7 +426,11 @@ export async function shouldImportSqlite(pool: Pool, sqlitePath: string): Promis
   )
   if (marker.rows[0]?.sqlite_imported_at) return false
   const servers = await pool.query<{ c: string }>(`SELECT COUNT(*)::text AS c FROM servers`)
-  if (Number(servers.rows[0]?.c ?? 0) > 0) return false
+  if (Number(servers.rows[0]?.c ?? 0) > 0) {
+    console.warn(
+      "SQLite → PostgreSQL: повтор недописанного импорта (маркера нет, servers уже не пустые)",
+    )
+  }
   return true
 }
 
@@ -462,6 +471,7 @@ export async function importSqliteToPostgres(
     return report
   }
   for (const spec of TABLES) {
+    console.log(`SQLite → PostgreSQL: таблица ${spec.table}`)
     report.tables[spec.table] = await copyTable(sqlite, pool, spec, { strict, fullHistory, rejects })
   }
   sqlite.close()

backend/src/db/sqlite-json.test.ts

@@ -0,0 +1,11 @@
+import assert from "node:assert/strict"
+
+/** node-pg encodes a JS array as a PostgreSQL array (`{...}`), not JSON (`[...]`). */
+{
+  const peers = [{ endpoint: "msk-gw02.rtnt.top:13232" }]
+  const asJson = JSON.stringify(peers)
+  assert.equal(asJson.startsWith("["), true)
+  assert.equal(asJson.includes("msk-gw02.rtnt.top:13232"), true)
+}
+
+console.log("sqlite-json.test.ts: ok")

deploy/docker-compose.cdn-mm.yml

@@ -146,7 +146,8 @@ services:
       - -c
       - wal_compression=on
     volumes:
-      - mmapp-pgdata:/var/lib/postgresql/data
+      # PostgreSQL 18+: VOLUME is /var/lib/postgresql (PGDATA = .../18/docker)
+      - mmapp-pgdata:/var/lib/postgresql
     networks:
       - mmapp
     healthcheck:

deploy/docker-compose.postgres.yml

@@ -32,7 +32,8 @@ services:
       - -c
       - wal_compression=on
     volumes:
-      - mmapp-pgdata:/var/lib/postgresql/data
+      # PostgreSQL 18+: VOLUME is /var/lib/postgresql (PGDATA = .../18/docker)
+      - mmapp-pgdata:/var/lib/postgresql
     ports:
       - "127.0.0.1:${POSTGRES_PORT:-5432}:5432"
     healthcheck:

deploy/docker-compose.traefik-cdn.yml

@@ -48,7 +48,8 @@ services:
       - -c
       - wal_compression=on
     volumes:
-      - mmapp-pgdata:/var/lib/postgresql/data
+      # PostgreSQL 18+: VOLUME is /var/lib/postgresql (PGDATA = .../18/docker)
+      - mmapp-pgdata:/var/lib/postgresql
     networks:
       - mmapp
     healthcheck:

deploy/docker-compose.traefik.yml

@@ -83,7 +83,8 @@ services:
       - -c
       - wal_compression=on
     volumes:
-      - mmapp-pgdata:/var/lib/postgresql/data
+      # PostgreSQL 18+: VOLUME is /var/lib/postgresql (PGDATA = .../18/docker)
+      - mmapp-pgdata:/var/lib/postgresql
     networks:
       - mmapp
     healthcheck:

deploy/docker-compose.yml

@@ -28,7 +28,8 @@ services:
       - -c
       - wal_compression=on
     volumes:
-      - mmapp-pgdata:/var/lib/postgresql/data
+      # PostgreSQL 18+: VOLUME is /var/lib/postgresql (PGDATA = .../18/docker)
+      - mmapp-pgdata:/var/lib/postgresql
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U mmapp -d mmapp"]
       interval: 5s

deploy/run-beside-cdn-traefik.sh

@@ -97,7 +97,7 @@ if ! docker inspect mmapp-postgres >/dev/null 2>&1; then
     -e POSTGRES_INITDB_ARGS="--encoding=UTF8 --locale=C.UTF-8 --data-checksums" \
     -e TZ=UTC \
     -e PGTZ=UTC \
-    -v mmapp-pgdata:/var/lib/postgresql/data \
+    -v mmapp-pgdata:/var/lib/postgresql \
     --health-cmd="pg_isready -U mmapp -d mmapp" \
     --health-interval=5s \
     --health-timeout=5s \