diff --git a/README.md b/README.md index 50f126c..3cf691c 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ sequenceDiagram | Зависимость | Реализация | |-------------|------------| | PostgreSQL | Контейнер `mmapp-postgres` (`postgres:18-alpine`). `DATABASE_URL=postgres://mmapp:…@postgres:5432/mmapp`. | -| SQLite (ETL) | Файл `mikrotik.db` на томе `/app/data` (`DATABASE_PATH=/app/data/mikrotik.db`). При первом старте, если PG пустой, backend сам импортирует данные и ставит маркер. Повторный старт не копирует заново. | +| SQLite (ETL) | Файл `mikrotik.db` на томе `/app/data` (`DATABASE_PATH=/app/data/mikrotik.db`). При старте, пока нет маркера `data_migration.sqlite_imported_at`, backend импортирует sqlite в PG (`ON CONFLICT` / upsert). После успешного импорта повтор не копирует заново. Том PG18: `mmapp-pgdata:/var/lib/postgresql` (не `.../data`). | | Docker socket | Только у контейнера updater: `/var/run/docker.sock` — доступ к Docker API хоста (управление контейнерами, pull). | ## Локальная разработка diff --git a/backend/package.json b/backend/package.json index 2ce9008..66e6340 100644 --- a/backend/package.json +++ b/backend/package.json @@ -17,7 +17,7 @@ "test:traffic-rate": "tsx src/services/traffic-rate.test.ts", "test:traffic-flow": "tsx src/services/traffic-flow-parse.test.ts && tsx src/services/traffic-flow-map-exporter.test.ts && tsx src/services/traffic-flow-ifaces.test.ts && tsx src/services/traffic-flow-dedup.test.ts && tsx src/services/traffic-flow-planes.test.ts && tsx src/services/traffic-flow-ip.test.ts && tsx src/services/traffic-flow-classify.test.ts && tsx src/services/traffic-flow-ripe.test.ts && tsx src/services/traffic-flow-brands.test.ts && tsx src/services/traffic-flow-ingest.test.ts && tsx src/services/traffic-flow-analytics.test.ts && tsx src/services/traffic-flow-map-hops.test.ts && tsx src/services/traffic-flow-purge.test.ts", "test:users": "tsx src/modules/users/iface-type.test.ts && tsx src/modules/users/bindings.test.ts", - "test:pg": "tsx src/db/sql-bind.test.ts && tsx src/db/pg-schema.test.ts", + "test:pg": "tsx src/db/sql-bind.test.ts && tsx src/db/sqlite-json.test.ts && tsx src/db/pg-schema.test.ts", "test": "npm run test:alert-engine && npm run test:auth && npm run test:wireguard && npm run test:traffic-rate && npm run test:traffic-flow && npm run test:users && npm run test:pg" }, "dependencies": { diff --git a/backend/src/db/pg-schema.test.ts b/backend/src/db/pg-schema.test.ts index 7cd0074..74a559f 100644 --- a/backend/src/db/pg-schema.test.ts +++ b/backend/src/db/pg-schema.test.ts @@ -53,4 +53,31 @@ if (!(await withPgOrSkip())) { await dbQuery(`DELETE FROM alert_outbox WHERE dedupe_key = 'pg-dedupe-key'`) } +{ + const peers = [{ endpoint: "msk-gw02.rtnt.top:13232", publicKey: "x" }] + await dbQuery( + `INSERT INTO alert_outbox (id, dedupe_key, payload_json, next_attempt_at) + VALUES ('pg-json-arr', 'pg-json-arr', $1::jsonb, now())`, + [JSON.stringify(peers)], + ) + const { rows } = await dbQuery<{ payload_json: unknown }>( + `SELECT payload_json FROM alert_outbox WHERE id = 'pg-json-arr'`, + ) + assert.equal(Array.isArray(rows[0]?.payload_json), true) + await dbQuery(`DELETE FROM alert_outbox WHERE id = 'pg-json-arr'`) + + let arrayAsPgArrayFailed = false + try { + await dbQuery( + `INSERT INTO alert_outbox (id, dedupe_key, payload_json, next_attempt_at) + VALUES ('pg-json-bad', 'pg-json-bad', $1, now())`, + [peers], + ) + } catch (err) { + arrayAsPgArrayFailed = err instanceof Error && /json|22P02/i.test(err.message) + } + await dbQuery(`DELETE FROM alert_outbox WHERE id = 'pg-json-bad'`).catch(() => undefined) + assert.equal(arrayAsPgArrayFailed, true, "JS array must not be bound as jsonb without stringify") +} + console.log("pg-schema.test.ts: ok") diff --git a/backend/src/db/sqlite-import.ts b/backend/src/db/sqlite-import.ts index 00dc0d6..18b174d 100644 --- a/backend/src/db/sqlite-import.ts +++ b/backend/src/db/sqlite-import.ts @@ -278,9 +278,11 @@ function coerce(kind: ColKind, value: unknown, strict: boolean, rejects: string[ case "bool": return parseBool(value) case "json": - return parseJson(value, []) + // Always JSON text. JS arrays must not go to node-pg as values — it encodes + // them as PG arrays (`{...}`), which jsonb rejects (22P02). + return JSON.stringify(parseJson(value, [])) case "json-null": - return value == null || value === "" ? null : parseJson(value, null) + return value == null || value === "" ? null : JSON.stringify(parseJson(value, null)) case "bigint-id": { const t = String(value ?? "").trim() if (!t) return null @@ -341,7 +343,9 @@ async function copyTable( const total = (sqlite.prepare(`SELECT COUNT(*) AS c FROM ${spec.table}${where}`).get() as { c: number }).c const part = specForParent(spec.table) const cols = spec.columns.map(([c]) => c) - const placeholders = cols.map((_, i) => `$${i + 1}`).join(", ") + const placeholders = spec.columns + .map(([, kind], i) => (kind === "json" || kind === "json-null" ? `$${i + 1}::jsonb` : `$${i + 1}`)) + .join(", ") const conflictSql = spec.upsert ? `ON CONFLICT (id) DO UPDATE SET ${cols.filter((c) => c !== "id").map((c) => `${c} = EXCLUDED.${c}`).join(", ")}` : `ON CONFLICT DO NOTHING` @@ -362,7 +366,8 @@ async function copyTable( await client.query("COMMIT") } catch (err) { await client.query("ROLLBACK") - throw err + const detail = err instanceof Error ? err.message : String(err) + throw new Error(`${spec.table}: ${detail}`) } finally { client.release() batch.length = 0 @@ -421,7 +426,11 @@ export async function shouldImportSqlite(pool: Pool, sqlitePath: string): Promis ) if (marker.rows[0]?.sqlite_imported_at) return false const servers = await pool.query<{ c: string }>(`SELECT COUNT(*)::text AS c FROM servers`) - if (Number(servers.rows[0]?.c ?? 0) > 0) return false + if (Number(servers.rows[0]?.c ?? 0) > 0) { + console.warn( + "SQLite → PostgreSQL: повтор недописанного импорта (маркера нет, servers уже не пустые)", + ) + } return true } @@ -462,6 +471,7 @@ export async function importSqliteToPostgres( return report } for (const spec of TABLES) { + console.log(`SQLite → PostgreSQL: таблица ${spec.table}`) report.tables[spec.table] = await copyTable(sqlite, pool, spec, { strict, fullHistory, rejects }) } sqlite.close() diff --git a/backend/src/db/sqlite-json.test.ts b/backend/src/db/sqlite-json.test.ts new file mode 100644 index 0000000..e434d5d --- /dev/null +++ b/backend/src/db/sqlite-json.test.ts @@ -0,0 +1,11 @@ +import assert from "node:assert/strict" + +/** node-pg encodes a JS array as a PostgreSQL array (`{...}`), not JSON (`[...]`). */ +{ + const peers = [{ endpoint: "msk-gw02.rtnt.top:13232" }] + const asJson = JSON.stringify(peers) + assert.equal(asJson.startsWith("["), true) + assert.equal(asJson.includes("msk-gw02.rtnt.top:13232"), true) +} + +console.log("sqlite-json.test.ts: ok") diff --git a/deploy/docker-compose.cdn-mm.yml b/deploy/docker-compose.cdn-mm.yml index affd110..b3a98d0 100644 --- a/deploy/docker-compose.cdn-mm.yml +++ b/deploy/docker-compose.cdn-mm.yml @@ -146,7 +146,8 @@ services: - -c - wal_compression=on volumes: - - mmapp-pgdata:/var/lib/postgresql/data + # PostgreSQL 18+: VOLUME is /var/lib/postgresql (PGDATA = .../18/docker) + - mmapp-pgdata:/var/lib/postgresql networks: - mmapp healthcheck: diff --git a/deploy/docker-compose.postgres.yml b/deploy/docker-compose.postgres.yml index 5007edd..671ae77 100644 --- a/deploy/docker-compose.postgres.yml +++ b/deploy/docker-compose.postgres.yml @@ -32,7 +32,8 @@ services: - -c - wal_compression=on volumes: - - mmapp-pgdata:/var/lib/postgresql/data + # PostgreSQL 18+: VOLUME is /var/lib/postgresql (PGDATA = .../18/docker) + - mmapp-pgdata:/var/lib/postgresql ports: - "127.0.0.1:${POSTGRES_PORT:-5432}:5432" healthcheck: diff --git a/deploy/docker-compose.traefik-cdn.yml b/deploy/docker-compose.traefik-cdn.yml index 3ae79fa..1485bb4 100644 --- a/deploy/docker-compose.traefik-cdn.yml +++ b/deploy/docker-compose.traefik-cdn.yml @@ -48,7 +48,8 @@ services: - -c - wal_compression=on volumes: - - mmapp-pgdata:/var/lib/postgresql/data + # PostgreSQL 18+: VOLUME is /var/lib/postgresql (PGDATA = .../18/docker) + - mmapp-pgdata:/var/lib/postgresql networks: - mmapp healthcheck: diff --git a/deploy/docker-compose.traefik.yml b/deploy/docker-compose.traefik.yml index e05720d..3cce3ed 100644 --- a/deploy/docker-compose.traefik.yml +++ b/deploy/docker-compose.traefik.yml @@ -83,7 +83,8 @@ services: - -c - wal_compression=on volumes: - - mmapp-pgdata:/var/lib/postgresql/data + # PostgreSQL 18+: VOLUME is /var/lib/postgresql (PGDATA = .../18/docker) + - mmapp-pgdata:/var/lib/postgresql networks: - mmapp healthcheck: diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index cb48a91..45ba7ae 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -28,7 +28,8 @@ services: - -c - wal_compression=on volumes: - - mmapp-pgdata:/var/lib/postgresql/data + # PostgreSQL 18+: VOLUME is /var/lib/postgresql (PGDATA = .../18/docker) + - mmapp-pgdata:/var/lib/postgresql healthcheck: test: ["CMD-SHELL", "pg_isready -U mmapp -d mmapp"] interval: 5s diff --git a/deploy/run-beside-cdn-traefik.sh b/deploy/run-beside-cdn-traefik.sh index 46e94c3..91a4996 100644 --- a/deploy/run-beside-cdn-traefik.sh +++ b/deploy/run-beside-cdn-traefik.sh @@ -97,7 +97,7 @@ if ! docker inspect mmapp-postgres >/dev/null 2>&1; then -e POSTGRES_INITDB_ARGS="--encoding=UTF8 --locale=C.UTF-8 --data-checksums" \ -e TZ=UTC \ -e PGTZ=UTC \ - -v mmapp-pgdata:/var/lib/postgresql/data \ + -v mmapp-pgdata:/var/lib/postgresql \ --health-cmd="pg_isready -U mmapp -d mmapp" \ --health-interval=5s \ --health-timeout=5s \