242 lines
18 KiB
TypeScript
242 lines
18 KiB
TypeScript
import assert from "node:assert/strict"
|
|
import { createHash } from "node:crypto"
|
|
import { readFileSync } from "node:fs"
|
|
import { captureMigrationData, compareMigrationData, MIGRATION_DATA_CONTEXT_QUERY, MigrationDataInventoryError, type CapturedMigrationData } from "./migration-data-inventory.js"
|
|
import { MIGRATION_SCHEMA_QUERIES, type SchemaSection } from "./migration-schema-inventory.js"
|
|
|
|
type Row = Record<string, unknown>
|
|
const schemaMetadata = { backend_pid: "71", transaction_started: "2026-10-25 10:00:00+00", read_only: "on", isolation: "repeatable read", search_path: "pg_catalog", timezone: "UTC", date_style: "ISO, YMD", standard_strings: "on", server_version: "180000", statement_timeout: "5s" }
|
|
const metadata = { ...schemaMetadata, row_security: "off", extra_float_digits: "3", bytea_output: "hex", interval_style: "postgres", client_encoding: "UTF8", lc_monetary: "C", lc_numeric: "C" }
|
|
const relation = (name: string, kind = "r", extra: Row = {}) => ({ namespace: "public", name, kind, persistence: "p", owner: "postgres", acl: null, options: null, comment: null, access_method: ["r", "p", "m"].includes(kind) ? "heap" : null, tablespace: null, toast_options: null, of_type: null, row_security: false, force_row_security: false, replica_identity: "d", is_partition: false, populated: true, partition_key: kind === "p" ? "RANGE (id)" : null, partition_bound: null, ...extra })
|
|
function schemaFixture() {
|
|
const data = Object.fromEntries(Object.keys(MIGRATION_SCHEMA_QUERIES).map(key => [key, []])) as unknown as Record<SchemaSection, Row[]>
|
|
data.database = [{ encoding: "UTF8", collate: "C", ctype: "C", locale_provider: "c", locale: null, locale_rules: null, collation_version: null, default_tablespace: "pg_default", owner: "postgres", acl: null, comment: null }]
|
|
data.namespaces = [{ namespace: "public", owner: "postgres", acl: null, comment: null }]
|
|
data.relations = [relation("servers"), relation("parent", "p"), relation("child", "r", { is_partition: true, partition_bound: "FOR VALUES FROM (0) TO (10)" }), relation("cached", "m"), relation("never_execute", "v"), relation("ids", "S")]
|
|
data.views = ["cached", "never_execute"].map(name => ({ namespace: "public", name, kind: name === "cached" ? "m" : "v", definition: "SELECT 1" }))
|
|
data.inheritance = [{ namespace: "public", child: "child", parent_namespace: "public", parent: "parent", position: 1, detach_pending: false }]
|
|
data.sequences = [{ namespace: "public", name: "ids", type: "bigint", start: "1", increment: "1", maximum: "9223372036854775807", minimum: "1", cache: "1", cycle: false, owner_namespace: null, owner_relation: null, owner_column: null, owner_dependency: null }]
|
|
return data
|
|
}
|
|
interface FakeOptions {
|
|
rows?: Record<string, string[]>
|
|
metadata?: Row
|
|
mutate?: (row: Row, kind: string, index: number) => void
|
|
sequence?: (read: number) => { last_value: string; is_called: boolean }
|
|
queryError?: string
|
|
preserveOrder?: boolean
|
|
includeControl?: boolean
|
|
}
|
|
function fake(options: FakeOptions = {}) {
|
|
const calls: string[] = [], data = schemaFixture()
|
|
if (options.includeControl) {
|
|
data.namespaces.push({ ...data.namespaces[0], namespace: "mmapp_control" })
|
|
data.relations.push(relation("control_log", "r", { namespace: "mmapp_control" }), relation("control_ids", "S", { namespace: "mmapp_control" }))
|
|
data.sequences.push({ ...data.sequences[0], namespace: "mmapp_control", name: "control_ids" })
|
|
}
|
|
let cursor: string | undefined, offset = 0, reads = 0, rowIndex = 0
|
|
const envelope = (kind: string, fields: Row = {}) => {
|
|
const row = { ...metadata, ...options.metadata, ...fields }
|
|
options.mutate?.(row, kind, rowIndex++)
|
|
return row
|
|
}
|
|
return { calls, data, query: async (sql: string, values: unknown[] = []) => {
|
|
calls.push(sql)
|
|
const section = (Object.keys(MIGRATION_SCHEMA_QUERIES) as SchemaSection[]).find(section => MIGRATION_SCHEMA_QUERIES[section] === sql)
|
|
if (section) {
|
|
assert.deepEqual(values, ["database", "namespaces"].includes(section) ? [] : [options.includeControl ? ["mmapp_control", "public"] : ["public"]])
|
|
return { rows: [{ ...schemaMetadata, objects: structuredClone(data[section]) }] }
|
|
}
|
|
assert.deepEqual(values, [])
|
|
if (sql === MIGRATION_DATA_CONTEXT_QUERY) return { rows: [envelope("context")] }
|
|
if (sql.startsWith("/* migration-data:sequence */")) {
|
|
assert(sql.endsWith('FROM "public"."ids"'))
|
|
return { rows: [envelope("sequence", options.sequence?.(++reads) ?? { last_value: "42", is_called: true })] }
|
|
}
|
|
if (sql.startsWith("/* migration-data:declare */")) {
|
|
assert.equal(cursor, undefined)
|
|
const match = /FROM ONLY "public"\."([^"]+)" AS t/.exec(sql)
|
|
assert(match); assert.notEqual(match[1], "never_execute")
|
|
assert(sql.includes('ORDER BY row_text COLLATE pg_catalog."C"'))
|
|
assert(sql.includes("octet_length(row_text)<=1048576"))
|
|
assert(sql.includes("pg_catalog.jsonb_build_array(pg_catalog.to_jsonb(t),pg_catalog.record_out(t)::text)::text AS row_text"))
|
|
cursor = match[1]; offset = 0
|
|
return { rows: [] }
|
|
}
|
|
if (sql === "/* migration-data:fetch */ FETCH FORWARD 16 FROM mm_migration_data_inventory") {
|
|
assert(cursor)
|
|
if (options.queryError) throw new Error(options.queryError)
|
|
const raw = [...(options.rows?.[cursor] ?? [])]
|
|
if (!options.preserveOrder) raw.sort((a, b) => Buffer.compare(Buffer.from(a), Buffer.from(b)))
|
|
const batch = raw.slice(offset, offset + 16); offset += batch.length
|
|
return { rows: batch.map(text => envelope("row", { row_text: Buffer.byteLength(text) <= 1048576 ? text : null, row_bytes: String(Buffer.byteLength(text)) })) }
|
|
}
|
|
assert.equal(sql, "/* migration-data:close */ CLOSE mm_migration_data_inventory")
|
|
assert(cursor); cursor = undefined
|
|
return { rows: [] }
|
|
} }
|
|
}
|
|
let cases = 0
|
|
async function test(name: string, work: () => unknown | Promise<unknown>) { await work(); cases++; console.log("ok", name) }
|
|
const captured = await captureMigrationData(fake({ rows: { servers: ['{"id": 1}', '{"id": 1}', '{"id": 2}'], child: ['{"id": 3}'], cached: ['{"id": 4}'] } }))
|
|
const clone = () => structuredClone(captured)
|
|
await test("actual catalog relations drive bounded physical-only cursor capture", () => {
|
|
assert.equal(captured.rowCount, 5)
|
|
assert.deepEqual(captured.tables.map(table => table.name), ["cached", "child", "parent", "servers"])
|
|
assert.equal(captured.tables.find(table => table.name === "parent")?.rowCount, 0)
|
|
assert.deepEqual(captured.sequences, [{ namespace: "public", name: "ids", lastValue: "42", isCalled: true }])
|
|
assert(!captured.canonical.includes('"id": 1'))
|
|
assert(Object.isFrozen(captured)); assert(Object.isFrozen(captured.tables)); assert(Object.isFrozen(captured.schema.sections.relations[0]))
|
|
assert.equal(captured.ready, false); assert.equal(captured.allReplicaQuiescence, false)
|
|
assert.deepEqual(compareMigrationData(captured, clone()), { equal: true, evidenceOnly: true, enrollmentAuthorized: false, allReplicaQuiescence: false, ready: false, sha256: captured.sha256 })
|
|
})
|
|
await test("ordering is stable and duplicate rows affect the multiset digest", async () => {
|
|
const original = await captureMigrationData(fake({ rows: { servers: ['{"id": 1}', '{"id": 1}', '{"id": 2}'] } }))
|
|
const reordered = await captureMigrationData(fake({ rows: { servers: ['{"id": 2}', '{"id": 1}', '{"id": 1}'] } }))
|
|
assert.equal(original.sha256, reordered.sha256)
|
|
for (const rows of [['{"id": 1}', '{"id": 2}'], ['{"id": 1}', '{"id": 2}', '{"id": 2}']]) {
|
|
const different = await captureMigrationData(fake({ rows: { servers: rows } }))
|
|
assert.throws(() => compareMigrationData(original, different))
|
|
}
|
|
})
|
|
await test("scoped data binds control schema definitions and refuses extra, incomplete or altered scope vectors", async () => {
|
|
const scope = { applicationNamespaces: ["public", "mmapp_control"] }
|
|
const capture = (client = fake({ includeControl: true })) => captureMigrationData(client, { scope, dataNamespaces: ["public"] })
|
|
const reference = await capture()
|
|
assert.equal(reference.format, "pg18-application-data-v3")
|
|
assert.deepEqual(JSON.parse(reference.canonical).dataNamespaces, ["public"])
|
|
assert.equal(reference.tables.length, 4)
|
|
assert.equal(reference.sequences.length, 1)
|
|
assert(reference.schema.sections.relations.some(row => row.name === "control_log"))
|
|
assert.equal(compareMigrationData(reference, await capture()).equal, true)
|
|
const changedDefinition = fake({ includeControl: true })
|
|
changedDefinition.data.relations.find(row => row.name === "control_log")!.comment = "changed"
|
|
const changedSchema = await capture(changedDefinition)
|
|
assert.throws(() => compareMigrationData(reference, changedSchema))
|
|
const changedData = await capture(fake({ includeControl: true, rows: { servers: ['{"id": 1}'] } }))
|
|
assert.throws(() => compareMigrationData(reference, changedData))
|
|
for (const dataNamespaces of [[], ["public", "public"], ["public", "absent"]]) {
|
|
await assert.rejects(captureMigrationData(fake({ includeControl: true }), { scope, dataNamespaces }), MigrationDataInventoryError)
|
|
}
|
|
const expanded = { ...reference, dataNamespaces: ["mmapp_control", "public"] }
|
|
assert.throws(() => compareMigrationData(expanded, expanded), /Incomplete data evidence vector/)
|
|
const incomplete = { ...reference, sequences: [] }
|
|
assert.throws(() => compareMigrationData(incomplete, incomplete), /Incomplete data evidence vector/)
|
|
const extra = { ...reference, sequences: [...reference.sequences, { ...reference.sequences[0], namespace: "mmapp_control", name: "control_ids" }] }
|
|
assert.throws(() => compareMigrationData(extra, extra), /Incomplete data evidence vector/)
|
|
})
|
|
await test("v2 native-record supplement distinguishes identical JSONB with different array bounds", async () => {
|
|
// PostgreSQL renders these native arrays differently while to_jsonb yields
|
|
// the same ["a","b"]. The SQL acceptance owner verifies real PG output.
|
|
const logical = { values: ["a", "b"] }
|
|
const zeroBased = JSON.stringify([logical, '("[0:1]={a,b}")'])
|
|
const oneBased = JSON.stringify([logical, '("{a,b}")'])
|
|
assert.deepEqual(JSON.parse(zeroBased)[0], JSON.parse(oneBased)[0])
|
|
const zero = await captureMigrationData(fake({ rows: { servers: [zeroBased] } }))
|
|
const one = await captureMigrationData(fake({ rows: { servers: [oneBased] } }))
|
|
assert.equal(zero.format, "pg18-application-data-v2")
|
|
assert.equal(Object.hasOwn(zero, "dataNamespaces"), false)
|
|
assert.equal(Object.hasOwn(JSON.parse(zero.canonical), "dataNamespaces"), false)
|
|
assert.notEqual(zero.tables.find(table => table.name === "servers")!.sha256, one.tables.find(table => table.name === "servers")!.sha256)
|
|
assert.throws(() => compareMigrationData(zero, one))
|
|
})
|
|
await test("batches fetch to exhaustion beyond sixteen rows without truncation", async () => {
|
|
const client = fake({ rows: { servers: Array.from({ length: 33 }, (_, id) => JSON.stringify({ id })) } })
|
|
const result = await captureMigrationData(client)
|
|
assert.equal(result.rowCount, 33)
|
|
assert.equal(client.calls.filter(sql => sql.includes("migration-data:fetch")).length, 7)
|
|
assert.equal(client.calls.filter(sql => sql.includes("migration-data:declare")).length, 4)
|
|
assert.equal(client.calls.filter(sql => sql.includes("migration-data:close")).length, 4)
|
|
})
|
|
await test("bounds are fail closed and cleanup closes only the owned cursor", async () => {
|
|
for (const options of [{ maxRows: 1 }, { maxBytes: 3 }]) {
|
|
const client = fake({ rows: { servers: ['{"id": 1}', '{"id": 2}'] } })
|
|
await assert.rejects(() => captureMigrationData(client, options), MigrationDataInventoryError)
|
|
assert.equal(client.calls.filter(sql => sql.includes("migration-data:close")).length, 4)
|
|
}
|
|
const client = fake({ rows: { servers: ['"' + "x".repeat(1048576) + '"'] } })
|
|
await assert.rejects(() => captureMigrationData(client), MigrationDataInventoryError)
|
|
assert(client.calls.at(-1)?.includes("migration-data:close"))
|
|
})
|
|
await test("invalid trusted bounds are rejected before any client query", async () => {
|
|
for (const options of [{ maxRows: 0 }, { maxRows: 10000001 }, { maxBytes: 1073741825 }, { maxBytes: 0 }, { deadlineAt: Date.now() - 1 }, { deadlineAt: Date.now() + 121000 }]) {
|
|
const client = fake(); await assert.rejects(() => captureMigrationData(client, options), MigrationDataInventoryError); assert.equal(client.calls.length, 0)
|
|
}
|
|
})
|
|
await test("unsafe GUCs, backend/transaction drift and per-row drift are refused", async () => {
|
|
for (const [key, value] of Object.entries({ row_security: "on", extra_float_digits: "0", bytea_output: "escape", interval_style: "iso_8601", client_encoding: "LATIN1", lc_monetary: "en_US.UTF-8", lc_numeric: "de_DE.UTF-8", read_only: "off", isolation: "read committed", timezone: "Europe/Moscow" })) {
|
|
await assert.rejects(() => captureMigrationData(fake({ metadata: { [key]: value } })), MigrationDataInventoryError)
|
|
}
|
|
for (const [key, value] of Object.entries({ backend_pid: "72", transaction_started: "2026-10-25 10:01:00+00", extra_float_digits: "2" })) {
|
|
await assert.rejects(() => captureMigrationData(fake({ rows: { servers: ['{"id": 1}'] }, mutate: (row, kind) => { if (kind === "row") row[key] = value } })), MigrationDataInventoryError)
|
|
}
|
|
})
|
|
await test("non-MVCC sequence instability and untrusted raw-error leakage are refused", async () => {
|
|
await assert.rejects(() => captureMigrationData(fake({ sequence: read => ({ last_value: String(read), is_called: true }) })), MigrationDataInventoryError)
|
|
for (const last_value of ["9223372036854775808", "-9223372036854775809", "-0", "00"]) await assert.rejects(() => captureMigrationData(fake({ sequence: () => ({ last_value, is_called: true }) })), MigrationDataInventoryError)
|
|
const privateValue = "PRIVATE CUSTOMER RECORD"
|
|
try { await captureMigrationData(fake({ queryError: privateValue })); assert.fail("must fail") }
|
|
catch (error) { assert(error instanceof MigrationDataInventoryError); assert(!String(error).includes(privateValue)); assert.equal(error.cause, undefined) }
|
|
})
|
|
await test("catalog identity ambiguity and sequence definitions without relations are refused", async () => {
|
|
const duplicate = fake(); duplicate.data.relations.push({ ...duplicate.data.relations[0], owner: "another" })
|
|
await assert.rejects(() => captureMigrationData(duplicate), MigrationDataInventoryError)
|
|
const extra = fake(); extra.data.sequences.push({ ...extra.data.sequences[0], name: "orphan" })
|
|
await assert.rejects(() => captureMigrationData(extra), MigrationDataInventoryError)
|
|
})
|
|
await test("cursor owner is not closed after connection ownership changes", async () => {
|
|
let declared = false
|
|
const base = fake({ rows: { servers: ['{"id": 1}'] }, mutate: (row, kind) => { if (declared && (kind === "row" || kind === "context")) row.backend_pid = "999" } })
|
|
const client = { query: async (sql: string, values?: unknown[]) => {
|
|
const result = await base.query(sql, values)
|
|
if (sql.includes("migration-data:declare")) declared = true
|
|
return result
|
|
} }
|
|
await assert.rejects(() => captureMigrationData(client), MigrationDataInventoryError)
|
|
assert.equal(base.calls.filter(sql => sql.includes("migration-data:close")).length, 0)
|
|
})
|
|
await test("concurrent capture cannot take over another capture cursor", async () => {
|
|
let release!: () => void, entered!: () => void
|
|
const waiting = new Promise<void>(resolve => { release = resolve }), arrived = new Promise<void>(resolve => { entered = resolve })
|
|
const base = fake(); let first = true
|
|
const client = { query: async (sql: string, values?: unknown[]) => {
|
|
if (first) { first = false; entered(); await waiting }
|
|
return base.query(sql, values)
|
|
} }
|
|
const capture = captureMigrationData(client); await arrived
|
|
await assert.rejects(() => captureMigrationData(client), MigrationDataInventoryError)
|
|
release(); await capture
|
|
await captureMigrationData(client)
|
|
})
|
|
await test("out-of-order rows, mismatched byte envelopes and extra fields fail closed", async () => {
|
|
await assert.rejects(() => captureMigrationData(fake({ rows: { servers: ['{"id": 2}', '{"id": 1}'] }, preserveOrder: true })), MigrationDataInventoryError)
|
|
for (const mutate of [(row: Row) => { row.row_bytes = "0" }, (row: Row) => { row.surprise = true }]) {
|
|
await assert.rejects(() => captureMigrationData(fake({ rows: { servers: ['{"id": 1}'] }, mutate: (row, kind) => { if (kind === "row") mutate(row) } })), MigrationDataInventoryError)
|
|
}
|
|
})
|
|
await test("comparison rejects missing/extra/duplicate tables, sequences and tampered hashes/counts/schema", () => {
|
|
const mutations: ((value: CapturedMigrationData) => void)[] = [
|
|
value => { value.tables.pop() }, value => { value.tables.push(value.tables[0]) }, value => { value.tables[1] = value.tables[0] },
|
|
value => { value.sequences = [] }, value => { value.sequences.push(value.sequences[0]) }, value => { value.sequences[0].lastValue = "43" },
|
|
value => { value.tables[0].sha256 = "a".repeat(64) }, value => { value.tables[0].rowCount++ }, value => { value.rowCount++ },
|
|
value => { value.tables[0].byteCount++ }, value => { value.sha256 = "a".repeat(64) }, value => { value.schemaSha256 = "a".repeat(64) },
|
|
value => { value.schema.sections.relations[0].owner = "another" }, value => { value.schema.observedPhysical.sha256 = "a".repeat(64) },
|
|
value => { Reflect.set(value, "authority", true) }, value => { Reflect.set(value, "ready", true) },
|
|
]
|
|
for (const mutate of mutations) { const value = clone(); mutate(value); assert.throws(() => compareMigrationData(captured, value)) }
|
|
})
|
|
await test("comparison recomputes an envelope rather than trusting a supplied matching hash", () => {
|
|
const value = clone(); value.tables[0].sha256 = "a".repeat(64); value.canonical = captured.canonical; value.sha256 = captured.sha256
|
|
assert.throws(() => compareMigrationData(value, value))
|
|
const empty = clone(); const table = empty.tables.find(table => table.rowCount === 0)!
|
|
table.sha256 = createHash("sha256").update("").digest("hex")
|
|
assert.throws(() => compareMigrationData(empty, empty))
|
|
})
|
|
await test("no configured pool, signing, transaction control, mutation or enrollment imports", () => {
|
|
const source = readFileSync(new URL("./migration-data-inventory.ts", import.meta.url), "utf8")
|
|
const imports = [...source.matchAll(/from "([^"]+)"/g)].map(match => match[1])
|
|
assert.deepEqual(imports, ["node:crypto", "./migration-schema-inventory.js"])
|
|
assert(!/\b(?:INSERT INTO|UPDATE public|DELETE FROM|nextval\(|setval\(|set_config\(|BEGIN TRANSACTION|COMMIT;|ROLLBACK;|createSign\()/.test(source))
|
|
})
|
|
console.log(`${cases} migration data inventory tests passed`)
|