250 lines
19 KiB
TypeScript
250 lines
19 KiB
TypeScript
import test from 'node:test'
|
|
import assert from 'node:assert/strict'
|
|
import {mkdtemp,readFile,writeFile,appendFile,mkdir,rm,readdir,link} from 'node:fs/promises'
|
|
import * as filesystem from 'node:fs/promises'
|
|
import {randomUUID} from 'node:crypto'
|
|
import path from 'node:path'
|
|
import {fileURLToPath,pathToFileURL} from 'node:url'
|
|
import {spawnSync} from 'node:child_process'
|
|
import {createRequire} from 'node:module'
|
|
import {runInNewContext} from 'node:vm'
|
|
import {inspectDatabaseMaintenanceStartup,assertDatabaseMaintenanceStartup,DatabaseMaintenanceStartupError} from './database-maintenance-startup.js'
|
|
import {executeMcpDatabaseMaintenance} from '../modules/mcp/database-maintenance.js'
|
|
import {SYSTEM_DATABASE_OPERATION_LOCK,SYSTEM_DATABASE_REQUEST_LOCK,withSystemDatabaseOperationLock} from '../db/operation-lock.js'
|
|
const testFile=fileURLToPath(import.meta.url),here=path.dirname(testFile),backend=path.resolve(here,'../..'),area=path.resolve(backend,'../tmp'),require=createRequire(import.meta.url),ts=require('typescript')
|
|
const stamp='2026-11-03T00:00:00.000Z'
|
|
// Same installed loader as the backend's owned test conventions, resolved to
|
|
// an absolute file URL rather than assuming a child's package resolution.
|
|
if(process.argv[2]==='--inspect-owned-maintenance'){
|
|
const value=await inspectDatabaseMaintenanceStartup(process.argv[3])
|
|
process.stdout.write(JSON.stringify(value)+'\n');process.exit(value.classification==='inspection-required'?1:0)
|
|
}
|
|
if(process.argv[2]==='--build-owned-app'){
|
|
const {buildApp}=await import('../index.js')
|
|
try{
|
|
const app=await buildApp({logger:false,startScheduler:process.argv[3]==='consumers'})
|
|
try{process.stdout.write(JSON.stringify({health:(await app.inject('/health')).statusCode})+'\n')}
|
|
finally{await app.close()}
|
|
process.exit(0)
|
|
}catch(error){
|
|
if(!(error instanceof DatabaseMaintenanceStartupError))throw error
|
|
process.stdout.write(JSON.stringify({reason:error.inspection.reason})+'\n');process.exit(1)
|
|
}
|
|
}
|
|
const record=(operationId:string,state:string)=>JSON.stringify({operationId,state,at:stamp})+'\n'
|
|
async function own(work:(root:string)=>Promise<void>){
|
|
await mkdir(area,{recursive:true});const root=await mkdtemp(path.join(area,'owned-maintenance-startup-'))
|
|
try{await work(root)}finally{assert(path.dirname(root)===area&&path.basename(root).startsWith('owned-maintenance-startup-'));await rm(root,{recursive:true,force:true})}
|
|
}
|
|
async function generation(root:string){await writeFile(path.join(root,'access-generation'),Buffer.alloc(32,7))}
|
|
async function completed(root:string){const id=randomUUID();await generation(root);await writeFile(path.join(root,'database-maintenance.jsonl'),record(id,'intent')+record(id,'applied'));return id}
|
|
async function outer(root:string,state:string,used=true,wrongClaim=false,warnings:string[]=[]){
|
|
const vault=path.join(root,'restore-vault');await mkdir(vault,{recursive:true})
|
|
const operationId=randomUUID(),archiveRef=randomUUID()
|
|
const value:Record<string,unknown>={operationId,archiveRef,state,startedAt:stamp,automaticRetry:false}
|
|
if(state==='applied')Object.assign(value,{finishedAt:stamp,accessRevoked:true,warnings})
|
|
else if(state!=='running')Object.assign(value,{errorCode:'RESTORE_UNCONFIRMED',finishedAt:stamp})
|
|
await writeFile(path.join(vault,operationId+'.receipt.json'),JSON.stringify(value))
|
|
if(used)await writeFile(path.join(vault,archiveRef+'.used'),JSON.stringify({operationId:wrongClaim?randomUUID():operationId,at:stamp}))
|
|
return {vault,operationId,archiveRef}
|
|
}
|
|
function child(root:string){
|
|
const value=spawnSync(process.execPath,['--import',pathToFileURL(require.resolve('tsx')).href,testFile,'--inspect-owned-maintenance',root],{cwd:backend,encoding:'utf8',timeout:10000})
|
|
assert.equal(value.error,undefined);assert.equal(value.signal,null)
|
|
return {code:value.status,inspection:JSON.parse(value.stdout)}
|
|
}
|
|
test('missing external state is read-only fresh legacy; empty existing state does not create authority',()=>own(async root=>{
|
|
const absent=path.join(root,'not-created'),before=await readdir(root)
|
|
assert.equal((await inspectDatabaseMaintenanceStartup(absent)).classification,'fresh-legacy')
|
|
assert.deepEqual(await readdir(root),before)
|
|
assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'fresh-legacy')
|
|
assert.deepEqual(await readdir(root),before)
|
|
}))
|
|
test('empty journal is only before-intent when an existing external access generation is readable',()=>own(async root=>{
|
|
await writeFile(path.join(root,'database-maintenance.jsonl'),'')
|
|
assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'inspection-required')
|
|
await generation(root)
|
|
const result=await inspectDatabaseMaintenanceStartup(root);assert.equal(result.classification,'empty-before-intent');assert.equal(result.accessGenerationObserved,true)
|
|
}))
|
|
test('actual fresh process refuses dangling intent and unconfirmed without replay or rewriting',()=>own(async root=>{
|
|
await generation(root);const file=path.join(root,'database-maintenance.jsonl'),id=randomUUID()
|
|
await writeFile(file,record(id,'intent'))
|
|
const before=await readFile(file)
|
|
const first=child(root);assert.equal(first.code,1);assert.equal(first.inspection.reason,'UNFINISHED_MAINTENANCE')
|
|
assert.deepEqual(await readFile(file),before)
|
|
await appendFile(file,record(id,'unconfirmed'))
|
|
const unknown=child(root);assert.equal(unknown.code,1);assert.equal(unknown.inspection.reason,'UNCONFIRMED_MAINTENANCE')
|
|
assert.equal(unknown.inspection.automaticRestore,false)
|
|
}))
|
|
test('application entrypoint checks persisted maintenance before HTTP or consumers start',()=>own(async root=>{
|
|
const run=(mode:'http'|'consumers')=>{
|
|
const result=spawnSync(process.execPath,['--import',pathToFileURL(require.resolve('tsx')).href,testFile,'--build-owned-app',mode],{
|
|
cwd:backend,encoding:'utf8',timeout:20000,
|
|
env:{...process.env,MCP_STATE_DIR:root,DATABASE_URL:'postgres://test:[email protected]:1/test'},
|
|
})
|
|
assert.equal(result.error,undefined);assert.equal(result.signal,null)
|
|
return {code:result.status,value:JSON.parse(result.stdout)}
|
|
}
|
|
const id=await completed(root)
|
|
assert.deepEqual(run('http'),{code:0,value:{health:200}})
|
|
await appendFile(path.join(root,'database-maintenance.jsonl'),record(randomUUID(),'intent'))
|
|
assert.deepEqual(run('http'),{code:1,value:{reason:'UNFINISHED_MAINTENANCE'}})
|
|
await writeFile(path.join(root,'database-maintenance.jsonl'),record(id,'intent')+record(id,'unconfirmed'))
|
|
assert.deepEqual(run('consumers'),{code:1,value:{reason:'UNCONFIRMED_MAINTENANCE'}})
|
|
}))
|
|
test('migration failure recorded by the restore writer refuses startup in a new process',()=>own(async root=>{
|
|
const file=path.join(root,'database-maintenance.jsonl'),id=randomUUID()
|
|
await executeMcpDatabaseMaintenance(async()=>({migrationError:'Synthetic failure'}),{
|
|
record:state=>appendFile(file,record(id,state)),invalidateAccess:()=>generation(root),close:async()=>{},
|
|
})
|
|
const result=child(root)
|
|
assert.equal(result.code,1)
|
|
assert.equal(result.inspection.reason,'UNCONFIRMED_MAINTENANCE')
|
|
assert.equal(await readFile(file,'utf8'),record(id,'intent')+record(id,'unconfirmed'))
|
|
}))
|
|
test('core applied allows ordinary bootstrap but never infers outer success, audit import or repaired schema',()=>own(async root=>{
|
|
await completed(root)
|
|
const result=child(root);assert.equal(result.code,0);assert.equal(result.inspection.classification,'completed-core-maintenance')
|
|
assert.equal(result.inspection.outerOperationInferred,false);assert.equal(result.inspection.auditImportStatus,'not-verified');assert.equal(result.inspection.postRepairSchemaStatus,'not-inspected')
|
|
}))
|
|
for(const state of ['running','unknown'])test('core applied cannot promote outer '+state+' receipt to completed',()=>own(async root=>{
|
|
await completed(root);await outer(root,state)
|
|
const result=await inspectDatabaseMaintenanceStartup(root);assert.equal(result.classification,'inspection-required');assert.equal(result.reason,'EXTERNAL_OPERATION_UNRESOLVED')
|
|
}))
|
|
test('outer applied receipt requires exact matching persisted used claim',()=>own(async root=>{
|
|
await completed(root);const f=await outer(root,'applied',true)
|
|
assert.equal((await inspectDatabaseMaintenanceStartup(root)).outerAppliedReceipts,1)
|
|
await writeFile(path.join(f.vault,f.archiveRef+'.used'),JSON.stringify({operationId:randomUUID(),at:stamp}))
|
|
assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'inspection-required')
|
|
}))
|
|
test('historical applied receipts with schema or journal warnings refuse ordinary startup',()=>own(async root=>{
|
|
await completed(root)
|
|
for(const warning of ['migrationError','maintenanceError']){
|
|
const fixture=await outer(root,'applied',true,false,[warning])
|
|
const result=await inspectDatabaseMaintenanceStartup(root)
|
|
assert.equal(result.classification,'inspection-required')
|
|
assert.equal(result.reason,'EXTERNAL_OPERATION_UNRESOLVED')
|
|
assert.equal(result.outerAppliedReceipts,1)
|
|
await rm(path.join(fixture.vault,fixture.operationId+'.receipt.json'))
|
|
await rm(path.join(fixture.vault,fixture.archiveRef+'.used'))
|
|
}
|
|
await outer(root,'applied',true,false,['cleanupError'])
|
|
assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'completed-core-maintenance')
|
|
}))
|
|
test('blocked predispatch operation may share an archive later successfully consumed by a different operation',()=>own(async root=>{
|
|
await completed(root);const a=await outer(root,'blocked',false),operationB=randomUUID()
|
|
await writeFile(path.join(a.vault,operationB+'.receipt.json'),JSON.stringify({operationId:operationB,archiveRef:a.archiveRef,state:'applied',startedAt:stamp,automaticRetry:false,finishedAt:stamp,accessRevoked:true,warnings:[]}))
|
|
await writeFile(path.join(a.vault,a.archiveRef+'.used'),JSON.stringify({operationId:operationB,at:stamp}))
|
|
const result=await inspectDatabaseMaintenanceStartup(root)
|
|
assert.equal(result.classification,'completed-core-maintenance');assert.equal(result.blockedReceipts,1);assert.equal(result.outerAppliedReceipts,1)
|
|
await writeFile(path.join(a.vault,a.archiveRef+'.used'),JSON.stringify({operationId:a.operationId,at:stamp}))
|
|
assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'inspection-required')
|
|
}))
|
|
test('missing journal with attempt markers is not a new clean install',()=>own(async root=>{
|
|
await outer(root,'applied',true)
|
|
const value=await inspectDatabaseMaintenanceStartup(root);assert.equal(value.classification,'inspection-required');assert.equal(value.reason,'JOURNAL_ABSENT_WITH_ATTEMPT_MARKERS')
|
|
}))
|
|
test('orphan used claim, unknown publication temporary file and blocked-with-used refuse',()=>own(async root=>{
|
|
await completed(root);const f=await outer(root,'blocked',true)
|
|
assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'inspection-required')
|
|
await rm(path.join(f.vault,f.operationId+'.receipt.json'))
|
|
assert.equal((await inspectDatabaseMaintenanceStartup(root)).reason,'EXTERNAL_OPERATION_UNRESOLVED')
|
|
await rm(path.join(f.vault,f.archiveRef+'.used'));await writeFile(path.join(f.vault,'receipt-publication.tmp'),'partial')
|
|
assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'inspection-required')
|
|
}))
|
|
test('malformed tails, duplicates, unknown fields/states and impossible transitions never admit startup',()=>own(async root=>{
|
|
await generation(root);const id=randomUUID(),good=record(id,'intent')
|
|
const bad=[good.slice(0,-1),good+'{',good+record(id,'intent'),good+record(id,'applied')+record(id,'applied'),record(id,'applied'),good+record(id,'future'),JSON.stringify({operationId:id,state:'intent',at:stamp,ready:true})+'\n',`{"operationId":"${id}","state":"intent","state":"applied","at":"${stamp}"}\n`]
|
|
for(const content of bad){await writeFile(path.join(root,'database-maintenance.jsonl'),content);assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'inspection-required')}
|
|
}))
|
|
test('journal byte bound refuses without accepting a truncated history',()=>own(async root=>{
|
|
await generation(root);const file=path.join(root,'database-maintenance.jsonl')
|
|
await writeFile(file,Buffer.alloc(8*1024*1024+1,32));assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'inspection-required')
|
|
}))
|
|
test('hardlinked external files are not accepted as complete history',async t=>own(async root=>{
|
|
await generation(root);const file=path.join(root,'database-maintenance.jsonl')
|
|
await writeFile(file,record(randomUUID(),'intent'))
|
|
try{await link(file,path.join(root,'other-link'))}catch(error){if((error as NodeJS.ErrnoException).code==='EPERM'){t.skip('Host filesystem policy refuses creating an owned hardlink');return}throw error}
|
|
assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'inspection-required')
|
|
}))
|
|
test('bootstrap checks recovery before connecting and holds both maintenance locks through startup writes',()=>own(async root=>{
|
|
const journal=path.join(root,'database-maintenance.jsonl'),intent=record(randomUUID(),'intent')
|
|
await generation(root);await writeFile(journal,intent)
|
|
const source=await readFile(path.join(backend,'src/db/bootstrap.ts'),'utf8'),events:string[]=[],exports:Record<string,unknown>={}
|
|
const held=new Set<number>();let appearedWhileWaiting=false,missingMigration=false
|
|
const write=(phase:string)=>{assert(held.has(SYSTEM_DATABASE_REQUEST_LOCK)&&held.has(SYSTEM_DATABASE_OPERATION_LOCK));events.push(phase)}
|
|
const client={query:async(sql:string,[key]:number[])=>{
|
|
if(sql.includes('pg_advisory_lock(')||sql.includes('pg_try_advisory_lock(')){
|
|
assert(!held.has(key));held.add(key)
|
|
if(key===SYSTEM_DATABASE_REQUEST_LOCK&&appearedWhileWaiting)await writeFile(journal,intent)
|
|
return {rows:[{acquired:true}]}
|
|
}
|
|
if(sql.includes('pg_advisory_unlock('))return {rows:[{released:held.delete(key)}]}
|
|
return {rows:[]}
|
|
},release:(discard:boolean)=>{assert.equal(discard,false);assert.equal(held.size,0);events.push('release')}}
|
|
const modules:Record<string,unknown>={
|
|
'./index.js':{pool:{connect:async()=>{events.push('connect');return client}}},
|
|
'./migrate.js':{applySqlMigrations:async()=>write('migrations'),assertSqlMigrationsApplied:async()=>{
|
|
write('verify-migrations');if(missingMigration)throw Error('Missing migration')
|
|
}},
|
|
'./operation-lock.js':{withSystemDatabaseOperationLock},
|
|
'./partitions.js':{dropExpiredPartitions:async()=>write('drop'),ensurePartitionsAround:async()=>write('partitions')},
|
|
'./sqlite-import.js':{shouldImportSqlite:async()=>true,importSqliteToPostgres:async()=>{write('import');return {durationMs:0,tables:{}}}},
|
|
'../config.js':{env:{DATABASE_PATH:'synthetic-only'}},'../modules/mcp/security.js':{mcpStateDirectory:()=>root},
|
|
'../services/database-maintenance-startup.js':{assertDatabaseMaintenanceStartup:async(directory:string)=>{events.push('guard');return assertDatabaseMaintenanceStartup(directory)}},
|
|
'../services/database-restore-audit.js':{reconcileDatabaseRestoreAudits:async()=>write('audit-reconcile')},
|
|
'../services/database-restore-checkpoint.js':{readDatabaseRestoreCheckpoint:async()=>null},
|
|
'./database-restore-quarantine.js':{databaseRestoreQuarantineEnabled:()=>false},
|
|
}
|
|
const code=ts.transpileModule(source,{compilerOptions:{module:ts.ModuleKind.CommonJS,target:ts.ScriptTarget.ES2022}}).outputText
|
|
runInNewContext(code,{exports,require:(name:string)=>{assert(name in modules);return modules[name]},console,process:{env:{}}})
|
|
const initialize=exports.initDatabase as (mode?:'operator-partitions')=>Promise<void>
|
|
await assert.rejects(initialize(),error=>error instanceof DatabaseMaintenanceStartupError)
|
|
assert.deepEqual(events,['guard'])
|
|
await writeFile(journal,'');events.length=0;appearedWhileWaiting=true
|
|
await assert.rejects(initialize(),error=>error instanceof DatabaseMaintenanceStartupError)
|
|
assert.deepEqual(events,['guard','connect','guard','release'])
|
|
await writeFile(journal,'');events.length=0;appearedWhileWaiting=false
|
|
await initialize()
|
|
assert.deepEqual(events,['guard','connect','guard','migrations','partitions','import','drop','partitions','audit-reconcile','release'])
|
|
events.length=0;missingMigration=true
|
|
await assert.rejects(initialize('operator-partitions'),/Missing migration/)
|
|
assert.deepEqual(events,['guard','connect','guard','verify-migrations','release'])
|
|
events.length=0;missingMigration=false
|
|
await initialize('operator-partitions')
|
|
assert.deepEqual(events,['guard','connect','guard','verify-migrations','drop','partitions','release'])
|
|
const main=await readFile(path.join(backend,'src/index.ts'),'utf8')
|
|
const start=main.indexOf('if (isMain)');assert(start>=0)
|
|
const body=main.slice(start),init=body.indexOf('await initDatabase()'),build=body.indexOf('await buildApp()')
|
|
assert(init>=0&&init<build&&build<body.indexOf('await app.listen'))
|
|
}))
|
|
test('actual journal append between metadata capture and open refuses an unstable snapshot',()=>own(async root=>{
|
|
await generation(root);const operation=randomUUID(),file=path.join(root,'database-maintenance.jsonl')
|
|
await writeFile(file,record(operation,'intent'))
|
|
const source=await readFile(path.join(backend,'src/services/database-maintenance-startup.ts'),'utf8'),exports:Record<string,unknown>={};let changed=false
|
|
const model={...filesystem,lstat:async(...args:Parameters<typeof filesystem.lstat>)=>{
|
|
const value=await Reflect.apply(filesystem.lstat,filesystem,args)
|
|
if(!changed&&args[0]===file){changed=true;await appendFile(file,record(operation,'applied'))}
|
|
return value
|
|
}}
|
|
const code=ts.transpileModule(source,{compilerOptions:{module:ts.ModuleKind.CommonJS,target:ts.ScriptTarget.ES2022,esModuleInterop:true}}).outputText
|
|
runInNewContext(code,{exports,require:(name:string)=>name==='node:fs/promises'?model:require(name),Buffer,TextDecoder,Date,Map,Set,Object,Error})
|
|
const inspect=exports.inspectDatabaseMaintenanceStartup as typeof inspectDatabaseMaintenanceStartup
|
|
const value=await inspect(root);assert.equal(changed,true);assert.equal(value.classification,'inspection-required')
|
|
assert.equal(value.reason,'EXTERNAL_STATE_UNAVAILABLE_OR_MALFORMED')
|
|
assert.equal((await inspectDatabaseMaintenanceStartup(root)).classification,'completed-core-maintenance','a later distinct stable inspection may observe actual terminal bytes without rewriting them')
|
|
}))
|
|
test('existing non-directory or unverifiable ancestor never becomes a missing clean state',()=>own(async root=>{
|
|
await writeFile(path.join(root,'ordinary-file'),'not-directory')
|
|
assert.equal((await inspectDatabaseMaintenanceStartup(path.join(root,'ordinary-file','state'))).classification,'inspection-required')
|
|
const source=await readFile(path.join(backend,'src/services/database-maintenance-startup.ts'),'utf8'),exports:Record<string,unknown>={}
|
|
const model={...filesystem,realpath:async(file:Parameters<typeof filesystem.realpath>[0])=>{
|
|
if(file===root)throw Object.assign(Error('Synthetic ancestor became unverifiable'),{code:'ENOENT'})
|
|
return filesystem.realpath(file)
|
|
}}
|
|
const code=ts.transpileModule(source,{compilerOptions:{module:ts.ModuleKind.CommonJS,target:ts.ScriptTarget.ES2022,esModuleInterop:true}}).outputText
|
|
runInNewContext(code,{exports,require:(name:string)=>name==='node:fs/promises'?model:require(name),Buffer,TextDecoder,Date,Map,Set,Object,Error})
|
|
const inspect=exports.inspectDatabaseMaintenanceStartup as typeof inspectDatabaseMaintenanceStartup
|
|
assert.equal((await inspect(path.join(root,'not-created-state'))).classification,'inspection-required')
|
|
}))
|