Files
MikrotikManager/backend/src/services/database-maintenance-startup.test.ts
T

250 lines
19 KiB
TypeScript

import test from 'node:test'
import assert from 'node:assert/strict'
import {mkdtemp,readFile,writeFile,appendFile,mkdir,rm,readdir,link} from 'node:fs/promises'
import * as filesystem from 'node:fs/promises'
import {randomUUID} from 'node:crypto'
import path from 'node:path'
import {fileURLToPath,pathToFileURL} from 'node:url'
import {spawnSync} from 'node:child_process'
import {createRequire} from 'node:module'
import {runInNewContext} from 'node:vm'
import {inspectDatabaseMaintenanceStartup,assertDatabaseMaintenanceStartup,DatabaseMaintenanceStartupError} from './database-maintenance-startup.js'
import {executeMcpDatabaseMaintenance} from '../modules/mcp/database-maintenance.js'
import {SYSTEM_DATABASE_OPERATION_LOCK,SYSTEM_DATABASE_REQUEST_LOCK,withSystemDatabaseOperationLock} from '../db/operation-lock.js'
const testFile=fileURLToPath(import.meta.url),here=path.dirname(testFile),backend=path.resolve(here,'../..'),area=path.resolve(backend,'../tmp'),require=createRequire(import.meta.url),ts=require('typescript')
const stamp='2026-11-03T00:00:00.000Z'
// Same installed loader as the backend's owned test conventions, resolved to
// an absolute file URL rather than assuming a child's package resolution.
if(process.argv[2]==='--inspect-owned-maintenance'){
const value=await inspectDatabaseMaintenanceStartup(process.argv[3])
process.stdout.write(JSON.stringify(value)+'\n');process.exit(value.classification==='inspection-required'?1:0)
}
if(process.argv[2]==='--build-owned-app'){
const {buildApp}=await import('../index.js')
try{
const app=await buildApp({logger:false,startScheduler:process.argv[3]==='consumers'})
try{process.stdout.write(JSON.stringify({health:(await app.inject('/health')).statusCode})+'\n')}
finally{await app.close()}
process.exit(0)
}catch(error){
if(!(error instanceof DatabaseMaintenanceStartupError))throw error
process.stdout.write(JSON.stringify({reason:error.inspection.reason})+'\n');process.exit(1)
}
}
const record=(operationId:string,state:string)=>JSON.stringify({operationId,state,at:stamp})+'\n'
async function own(work:(root:string)=>Promise<void>){
await mkdir(area,{recursive:true});const root=await mkdtemp(path.join(area,'owned-maintenance-startup-'))
try{await work(root)}finally{assert(path.dirname(root)===area&&path.basename(root).startsWith('owned-maintenance-startup-'));await rm(root,{recursive:true,force:true})}
}
async function generation(root:string){await writeFile(path.join(root,'access-generation'),Buffer.alloc(32,7))}
async function completed(root:string){const id=randomUUID();await generation(root);await writeFile(path.join(root,'database-maintenance.jsonl'),record(id,'intent')+record(id,'applied'));return id}
async function outer(root:string,state:string,used=true,wrongClaim=false,warnings:string[]=[]){
const vault=path.join(root,'restore-vault');await mkdir(vault,{recursive:true})
const operationId=randomUUID(),archiveRef=randomUUID()
const value:Record<string,unknown>={operationId,archiveRef,state,startedAt:stamp,automaticRetry:false}
if(state==='applied')Object.assign(value,{finishedAt:stamp,accessRevoked:true,warnings})
else if(state!=='running')Object.assign(value,{errorCode:'RESTORE_UNCONFIRMED',finishedAt:stamp})
await writeFile(path.join(vault,operationId+'.receipt.json'),JSON.stringify(value))
if(used)await writeFile(path.join(vault,archiveRef+'.used'),JSON.stringify({operationId:wrongClaim?randomUUID():operationId,at:stamp}))
return {vault,operationId,archiveRef}
}
function child(root:string){
const value=spawnSync(process.execPath,['--import',pathToFileURL(require.resolve('tsx')).href,testFile,'--inspect-owned-maintenance',root],{cwd:backend,encoding:'utf8',timeout:10000})
assert.equal(value.error,undefined);assert.equal(value.signal,null)
return {code:value.status,inspection:JSON.parse(value.stdout)}
}
test('missing external state is read-only fresh legacy; empty existing state does not create authority',()=>own(async root=>{
const absent=path.join(root,'not-created'),before=await readdir(root)
assert.equal((await inspectDatabaseMaintenanceStartup(absent)).classification,'fresh-legacy')
assert.deepEqual(await readdir(root),before)
assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'fresh-legacy')
assert.deepEqual(await readdir(root),before)
}))
test('empty journal is only before-intent when an existing external access generation is readable',()=>own(async root=>{
await writeFile(path.join(root,'database-maintenance.jsonl'),'')
assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'inspection-required')
await generation(root)
const result=await inspectDatabaseMaintenanceStartup(root);assert.equal(result.classification,'empty-before-intent');assert.equal(result.accessGenerationObserved,true)
}))
test('actual fresh process refuses dangling intent and unconfirmed without replay or rewriting',()=>own(async root=>{
await generation(root);const file=path.join(root,'database-maintenance.jsonl'),id=randomUUID()
await writeFile(file,record(id,'intent'))
const before=await readFile(file)
const first=child(root);assert.equal(first.code,1);assert.equal(first.inspection.reason,'UNFINISHED_MAINTENANCE')
assert.deepEqual(await readFile(file),before)
await appendFile(file,record(id,'unconfirmed'))
const unknown=child(root);assert.equal(unknown.code,1);assert.equal(unknown.inspection.reason,'UNCONFIRMED_MAINTENANCE')
assert.equal(unknown.inspection.automaticRestore,false)
}))
test('application entrypoint checks persisted maintenance before HTTP or consumers start',()=>own(async root=>{
const run=(mode:'http'|'consumers')=>{
const result=spawnSync(process.execPath,['--import',pathToFileURL(require.resolve('tsx')).href,testFile,'--build-owned-app',mode],{
cwd:backend,encoding:'utf8',timeout:20000,
env:{...process.env,MCP_STATE_DIR:root,DATABASE_URL:'postgres://test:[email protected]:1/test'},
})
assert.equal(result.error,undefined);assert.equal(result.signal,null)
return {code:result.status,value:JSON.parse(result.stdout)}
}
const id=await completed(root)
assert.deepEqual(run('http'),{code:0,value:{health:200}})
await appendFile(path.join(root,'database-maintenance.jsonl'),record(randomUUID(),'intent'))
assert.deepEqual(run('http'),{code:1,value:{reason:'UNFINISHED_MAINTENANCE'}})
await writeFile(path.join(root,'database-maintenance.jsonl'),record(id,'intent')+record(id,'unconfirmed'))
assert.deepEqual(run('consumers'),{code:1,value:{reason:'UNCONFIRMED_MAINTENANCE'}})
}))
test('migration failure recorded by the restore writer refuses startup in a new process',()=>own(async root=>{
const file=path.join(root,'database-maintenance.jsonl'),id=randomUUID()
await executeMcpDatabaseMaintenance(async()=>({migrationError:'Synthetic failure'}),{
record:state=>appendFile(file,record(id,state)),invalidateAccess:()=>generation(root),close:async()=>{},
})
const result=child(root)
assert.equal(result.code,1)
assert.equal(result.inspection.reason,'UNCONFIRMED_MAINTENANCE')
assert.equal(await readFile(file,'utf8'),record(id,'intent')+record(id,'unconfirmed'))
}))
test('core applied allows ordinary bootstrap but never infers outer success, audit import or repaired schema',()=>own(async root=>{
await completed(root)
const result=child(root);assert.equal(result.code,0);assert.equal(result.inspection.classification,'completed-core-maintenance')
assert.equal(result.inspection.outerOperationInferred,false);assert.equal(result.inspection.auditImportStatus,'not-verified');assert.equal(result.inspection.postRepairSchemaStatus,'not-inspected')
}))
for(const state of ['running','unknown'])test('core applied cannot promote outer '+state+' receipt to completed',()=>own(async root=>{
await completed(root);await outer(root,state)
const result=await inspectDatabaseMaintenanceStartup(root);assert.equal(result.classification,'inspection-required');assert.equal(result.reason,'EXTERNAL_OPERATION_UNRESOLVED')
}))
test('outer applied receipt requires exact matching persisted used claim',()=>own(async root=>{
await completed(root);const f=await outer(root,'applied',true)
assert.equal((await inspectDatabaseMaintenanceStartup(root)).outerAppliedReceipts,1)
await writeFile(path.join(f.vault,f.archiveRef+'.used'),JSON.stringify({operationId:randomUUID(),at:stamp}))
assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'inspection-required')
}))
test('historical applied receipts with schema or journal warnings refuse ordinary startup',()=>own(async root=>{
await completed(root)
for(const warning of ['migrationError','maintenanceError']){
const fixture=await outer(root,'applied',true,false,[warning])
const result=await inspectDatabaseMaintenanceStartup(root)
assert.equal(result.classification,'inspection-required')
assert.equal(result.reason,'EXTERNAL_OPERATION_UNRESOLVED')
assert.equal(result.outerAppliedReceipts,1)
await rm(path.join(fixture.vault,fixture.operationId+'.receipt.json'))
await rm(path.join(fixture.vault,fixture.archiveRef+'.used'))
}
await outer(root,'applied',true,false,['cleanupError'])
assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'completed-core-maintenance')
}))
test('blocked predispatch operation may share an archive later successfully consumed by a different operation',()=>own(async root=>{
await completed(root);const a=await outer(root,'blocked',false),operationB=randomUUID()
await writeFile(path.join(a.vault,operationB+'.receipt.json'),JSON.stringify({operationId:operationB,archiveRef:a.archiveRef,state:'applied',startedAt:stamp,automaticRetry:false,finishedAt:stamp,accessRevoked:true,warnings:[]}))
await writeFile(path.join(a.vault,a.archiveRef+'.used'),JSON.stringify({operationId:operationB,at:stamp}))
const result=await inspectDatabaseMaintenanceStartup(root)
assert.equal(result.classification,'completed-core-maintenance');assert.equal(result.blockedReceipts,1);assert.equal(result.outerAppliedReceipts,1)
await writeFile(path.join(a.vault,a.archiveRef+'.used'),JSON.stringify({operationId:a.operationId,at:stamp}))
assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'inspection-required')
}))
test('missing journal with attempt markers is not a new clean install',()=>own(async root=>{
await outer(root,'applied',true)
const value=await inspectDatabaseMaintenanceStartup(root);assert.equal(value.classification,'inspection-required');assert.equal(value.reason,'JOURNAL_ABSENT_WITH_ATTEMPT_MARKERS')
}))
test('orphan used claim, unknown publication temporary file and blocked-with-used refuse',()=>own(async root=>{
await completed(root);const f=await outer(root,'blocked',true)
assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'inspection-required')
await rm(path.join(f.vault,f.operationId+'.receipt.json'))
assert.equal((await inspectDatabaseMaintenanceStartup(root)).reason,'EXTERNAL_OPERATION_UNRESOLVED')
await rm(path.join(f.vault,f.archiveRef+'.used'));await writeFile(path.join(f.vault,'receipt-publication.tmp'),'partial')
assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'inspection-required')
}))
test('malformed tails, duplicates, unknown fields/states and impossible transitions never admit startup',()=>own(async root=>{
await generation(root);const id=randomUUID(),good=record(id,'intent')
const bad=[good.slice(0,-1),good+'{',good+record(id,'intent'),good+record(id,'applied')+record(id,'applied'),record(id,'applied'),good+record(id,'future'),JSON.stringify({operationId:id,state:'intent',at:stamp,ready:true})+'\n',`{"operationId":"${id}","state":"intent","state":"applied","at":"${stamp}"}\n`]
for(const content of bad){await writeFile(path.join(root,'database-maintenance.jsonl'),content);assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'inspection-required')}
}))
test('journal byte bound refuses without accepting a truncated history',()=>own(async root=>{
await generation(root);const file=path.join(root,'database-maintenance.jsonl')
await writeFile(file,Buffer.alloc(8*1024*1024+1,32));assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'inspection-required')
}))
test('hardlinked external files are not accepted as complete history',async t=>own(async root=>{
await generation(root);const file=path.join(root,'database-maintenance.jsonl')
await writeFile(file,record(randomUUID(),'intent'))
try{await link(file,path.join(root,'other-link'))}catch(error){if((error as NodeJS.ErrnoException).code==='EPERM'){t.skip('Host filesystem policy refuses creating an owned hardlink');return}throw error}
assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'inspection-required')
}))
test('bootstrap checks recovery before connecting and holds both maintenance locks through startup writes',()=>own(async root=>{
const journal=path.join(root,'database-maintenance.jsonl'),intent=record(randomUUID(),'intent')
await generation(root);await writeFile(journal,intent)
const source=await readFile(path.join(backend,'src/db/bootstrap.ts'),'utf8'),events:string[]=[],exports:Record<string,unknown>={}
const held=new Set<number>();let appearedWhileWaiting=false,missingMigration=false
const write=(phase:string)=>{assert(held.has(SYSTEM_DATABASE_REQUEST_LOCK)&&held.has(SYSTEM_DATABASE_OPERATION_LOCK));events.push(phase)}
const client={query:async(sql:string,[key]:number[])=>{
if(sql.includes('pg_advisory_lock(')||sql.includes('pg_try_advisory_lock(')){
assert(!held.has(key));held.add(key)
if(key===SYSTEM_DATABASE_REQUEST_LOCK&&appearedWhileWaiting)await writeFile(journal,intent)
return {rows:[{acquired:true}]}
}
if(sql.includes('pg_advisory_unlock('))return {rows:[{released:held.delete(key)}]}
return {rows:[]}
},release:(discard:boolean)=>{assert.equal(discard,false);assert.equal(held.size,0);events.push('release')}}
const modules:Record<string,unknown>={
'./index.js':{pool:{connect:async()=>{events.push('connect');return client}}},
'./migrate.js':{applySqlMigrations:async()=>write('migrations'),assertSqlMigrationsApplied:async()=>{
write('verify-migrations');if(missingMigration)throw Error('Missing migration')
}},
'./operation-lock.js':{withSystemDatabaseOperationLock},
'./partitions.js':{dropExpiredPartitions:async()=>write('drop'),ensurePartitionsAround:async()=>write('partitions')},
'./sqlite-import.js':{shouldImportSqlite:async()=>true,importSqliteToPostgres:async()=>{write('import');return {durationMs:0,tables:{}}}},
'../config.js':{env:{DATABASE_PATH:'synthetic-only'}},'../modules/mcp/security.js':{mcpStateDirectory:()=>root},
'../services/database-maintenance-startup.js':{assertDatabaseMaintenanceStartup:async(directory:string)=>{events.push('guard');return assertDatabaseMaintenanceStartup(directory)}},
'../services/database-restore-audit.js':{reconcileDatabaseRestoreAudits:async()=>write('audit-reconcile')},
'../services/database-restore-checkpoint.js':{readDatabaseRestoreCheckpoint:async()=>null},
'./database-restore-quarantine.js':{databaseRestoreQuarantineEnabled:()=>false},
}
const code=ts.transpileModule(source,{compilerOptions:{module:ts.ModuleKind.CommonJS,target:ts.ScriptTarget.ES2022}}).outputText
runInNewContext(code,{exports,require:(name:string)=>{assert(name in modules);return modules[name]},console,process:{env:{}}})
const initialize=exports.initDatabase as (mode?:'operator-partitions')=>Promise<void>
await assert.rejects(initialize(),error=>error instanceof DatabaseMaintenanceStartupError)
assert.deepEqual(events,['guard'])
await writeFile(journal,'');events.length=0;appearedWhileWaiting=true
await assert.rejects(initialize(),error=>error instanceof DatabaseMaintenanceStartupError)
assert.deepEqual(events,['guard','connect','guard','release'])
await writeFile(journal,'');events.length=0;appearedWhileWaiting=false
await initialize()
assert.deepEqual(events,['guard','connect','guard','migrations','partitions','import','drop','partitions','audit-reconcile','release'])
events.length=0;missingMigration=true
await assert.rejects(initialize('operator-partitions'),/Missing migration/)
assert.deepEqual(events,['guard','connect','guard','verify-migrations','release'])
events.length=0;missingMigration=false
await initialize('operator-partitions')
assert.deepEqual(events,['guard','connect','guard','verify-migrations','drop','partitions','release'])
const main=await readFile(path.join(backend,'src/index.ts'),'utf8')
const start=main.indexOf('if (isMain)');assert(start>=0)
const body=main.slice(start),init=body.indexOf('await initDatabase()'),build=body.indexOf('await buildApp()')
assert(init>=0&&init<build&&build<body.indexOf('await app.listen'))
}))
test('actual journal append between metadata capture and open refuses an unstable snapshot',()=>own(async root=>{
await generation(root);const operation=randomUUID(),file=path.join(root,'database-maintenance.jsonl')
await writeFile(file,record(operation,'intent'))
const source=await readFile(path.join(backend,'src/services/database-maintenance-startup.ts'),'utf8'),exports:Record<string,unknown>={};let changed=false
const model={...filesystem,lstat:async(...args:Parameters<typeof filesystem.lstat>)=>{
const value=await Reflect.apply(filesystem.lstat,filesystem,args)
if(!changed&&args[0]===file){changed=true;await appendFile(file,record(operation,'applied'))}
return value
}}
const code=ts.transpileModule(source,{compilerOptions:{module:ts.ModuleKind.CommonJS,target:ts.ScriptTarget.ES2022,esModuleInterop:true}}).outputText
runInNewContext(code,{exports,require:(name:string)=>name==='node:fs/promises'?model:require(name),Buffer,TextDecoder,Date,Map,Set,Object,Error})
const inspect=exports.inspectDatabaseMaintenanceStartup as typeof inspectDatabaseMaintenanceStartup
const value=await inspect(root);assert.equal(changed,true);assert.equal(value.classification,'inspection-required')
assert.equal(value.reason,'EXTERNAL_STATE_UNAVAILABLE_OR_MALFORMED')
assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'completed-core-maintenance','a later distinct stable inspection may observe actual terminal bytes without rewriting them')
}))
test('existing non-directory or unverifiable ancestor never becomes a missing clean state',()=>own(async root=>{
await writeFile(path.join(root,'ordinary-file'),'not-directory')
assert.equal((await inspectDatabaseMaintenanceStartup(path.join(root,'ordinary-file','state'))).classification,'inspection-required')
const source=await readFile(path.join(backend,'src/services/database-maintenance-startup.ts'),'utf8'),exports:Record<string,unknown>={}
const model={...filesystem,realpath:async(file:Parameters<typeof filesystem.realpath>[0])=>{
if(file===root)throw Object.assign(Error('Synthetic ancestor became unverifiable'),{code:'ENOENT'})
return filesystem.realpath(file)
}}
const code=ts.transpileModule(source,{compilerOptions:{module:ts.ModuleKind.CommonJS,target:ts.ScriptTarget.ES2022,esModuleInterop:true}}).outputText
runInNewContext(code,{exports,require:(name:string)=>name==='node:fs/promises'?model:require(name),Buffer,TextDecoder,Date,Map,Set,Object,Error})
const inspect=exports.inspectDatabaseMaintenanceStartup as typeof inspectDatabaseMaintenanceStartup
assert.equal((await inspect(path.join(root,'not-created-state'))).classification,'inspection-required')
}))