Files
MikrotikManager/backend/src/services/database-maintenance-startup.ts
T

236 lines
19 KiB
TypeScript

/** Read-only classification of the EXISTING external maintenance grammar.
* No restore, journal rewrite, generation rotation, SQL, role issuer or READY. */
import process from 'node:process'
import {constants,type BigIntStats} from 'node:fs'
import {lstat,open,readdir,realpath} from 'node:fs/promises'
import path from 'node:path'
import {createHash,createHmac,timingSafeEqual} from 'node:crypto'
import {readDatabaseRestoreJournalHead,type RestoreJournalHead} from '../db/database-restore-quarantine.js'
import {inspectMaintenanceJournal,parseMaintenanceJSON,evaluateMaintenanceAcceptances,runtimeDatabaseTargetSha256,RESTORE_DATA_JOURNAL_DOMAIN,type MaintenanceMarker,type RestoreDataJournalBinding,type RestoreDataJournalRecord} from './database-maintenance-resolution.js'
const MAX_JOURNAL_BYTES=8*1024*1024,MAX_VAULT_ENTRIES=4096,MAX_RECEIPT_BYTES=65536
const uuid=/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/
type Reason='NONE'|'UNFINISHED_MAINTENANCE'|'UNCONFIRMED_MAINTENANCE'|'EXTERNAL_OPERATION_UNRESOLVED'|'JOURNAL_ABSENT_WITH_ATTEMPT_MARKERS'|'EXTERNAL_STATE_UNAVAILABLE_OR_MALFORMED'
export interface MaintenanceStartupInspection {
readonly classification:'fresh-legacy'|'empty-before-intent'|'completed-core-maintenance'|'accepted-current-state'|'inspection-required'
readonly reason:Reason
readonly journalOperations:number
readonly coreApplied:number
readonly outerAppliedReceipts:number
readonly blockedReceipts:number
readonly accessGenerationObserved:boolean
readonly outerOperationInferred:false
readonly auditImportStatus:'not-verified'
readonly postRepairSchemaStatus:'not-inspected'
readonly acceptedHistoricResolutions:number
readonly historicOutcomesChanged:false
readonly automaticRestore:false
}
class Refusal extends Error {constructor(readonly reason:Reason='EXTERNAL_STATE_UNAVAILABLE_OR_MALFORMED'){super('External maintenance state requires inspection')}}
function refuse(reason?:Reason):never{throw new Refusal(reason)}
function plain(value:unknown):value is Record<string,unknown>{return value!==null&&typeof value==='object'&&!Array.isArray(value)}
function exact(value:unknown,keys:readonly string[]):asserts value is Record<string,unknown>{if(!plain(value)||Object.keys(value).sort().join('\0')!==[...keys].sort().join('\0'))refuse()}
function id(value:unknown):asserts value is string{if(typeof value!=='string'||!uuid.test(value))refuse()}
function instant(value:unknown):asserts value is string{if(typeof value!=='string'||!Number.isFinite(Date.parse(value))||new Date(value).toISOString()!==value)refuse()}
function json(raw:Buffer):unknown{
const text=new TextDecoder('utf8',{fatal:true}).decode(raw)
// All existing writers produce JSON.stringify output. Exact roundtrip also
// refuses duplicate keys and ambiguous/trailing non-JSON representations.
const value:unknown=JSON.parse(text)
if(JSON.stringify(value)!==text.trim())refuse()
return value
}
function identity(s:BigIntStats){return [s.dev,s.ino,s.size,s.mode,s.nlink,s.mtimeNs,s.ctimeNs].map(String).join(':')}
async function directory(file:string,optional=false){
let s
try{s=await lstat(file,{bigint:true})}
catch(error){if(optional&&(error as NodeJS.ErrnoException).code==='ENOENT')return null;throw error}
if(!s.isDirectory()||s.isSymbolicLink())refuse()
// Only initial lstat absence is a new/legacy absence. An existing ancestor
// whose realpath disappears or cannot be verified is ambiguous, not fresh.
try{if(await realpath(file)!==file)refuse()}catch{refuse()}
return identity(s)
}
async function existingAncestors(file:string){
for(let current=path.dirname(file);;current=path.dirname(current)){
await directory(current,true)
if(current===path.dirname(current))break
}
}
async function bounded(file:string,maximum:number,optional=false,sync=false){
let before
try{before=await lstat(file,{bigint:true})}catch(error){if(optional&&(error as NodeJS.ErrnoException).code==='ENOENT')return null;throw error}
if(!before.isFile()||before.isSymbolicLink()||before.nlink!==1n||before.size>BigInt(maximum)||await realpath(file)!==file)refuse()
const signature=(s:typeof before)=>[s.dev,s.ino,s.size,s.mode,s.nlink,s.mtimeNs,s.ctimeNs].map(String).join(':')
const handle=await open(file,(sync?constants.O_RDWR:constants.O_RDONLY)|(constants.O_NOFOLLOW??0))
try{
const current=await handle.stat({bigint:true});if(signature(current)!==signature(before))refuse()
const data=Buffer.alloc(Number(current.size)+1);let size=0
while(size<data.length){const got=await handle.read(data,size,data.length-size,size);if(!got.bytesRead)break;size+=got.bytesRead}
if(sync)await handle.sync()
if(BigInt(size)!==current.size||signature(await handle.stat({bigint:true}))!==signature(before)||signature(await lstat(file,{bigint:true}))!==signature(before))refuse()
return data.subarray(0,size)
}finally{await handle.close()}
}
export interface AppliedRestoreAuditSource {readonly operationId:string;readonly archiveRef:string;readonly finishedAt:string;readonly receiptSha256:string;readonly receipt:Readonly<Record<string,unknown>>}
interface VaultFacts {markers:number;applied:number;blocked:number;unresolved:boolean;auditSources:readonly AppliedRestoreAuditSource[];evidence:readonly MaintenanceMarker[];unresolvedFiles:readonly string[]}
async function vaultFacts(stateDirectory:string):Promise<VaultFacts>{
const root=path.join(stateDirectory,'restore-vault'),before=await directory(root,true)
if(before===null)return {markers:0,applied:0,blocked:0,unresolved:false,auditSources:[],evidence:[],unresolvedFiles:[]}
const auditSources:AppliedRestoreAuditSource[]=[],evidence:MaintenanceMarker[]=[],unresolvedFiles=new Set<string>()
const names=(await readdir(root)).sort();if(names.length>MAX_VAULT_ENTRIES)refuse()
const signatures=new Map<string,string>(),used=new Map<string,{operationId:string}>(),receipts=new Map<string,{state:string;archiveRef:string}>()
for(const name of names){
const match=/^([a-f0-9-]{36})\.(used|receipt\.json|archive\.json|archive\.bin)$/.exec(name)
if(!match)refuse();id(match[1])
const file=path.join(root,name),stat=await lstat(file,{bigint:true});if(!stat.isFile()||stat.isSymbolicLink()||stat.nlink!==1n||await realpath(file)!==file)refuse()
signatures.set(name,identity(stat))
if(match[2]==='used'){
const raw=(await bounded(file,4096))!;evidence.push({file:name,bytes:raw.length,sha256:createHash('sha256').update(raw).digest('hex')});const value=json(raw);exact(value,['operationId','at']);id(value.operationId);instant(value.at)
used.set(match[1],{operationId:value.operationId})
}else if(match[2]==='receipt.json'){
const raw=(await bounded(file,MAX_RECEIPT_BYTES))!,value=json(raw);evidence.push({file:name,bytes:raw.length,sha256:createHash('sha256').update(raw).digest('hex')})
if(!plain(value)||!['running','applied','unknown','blocked'].includes(String(value.state)))refuse()
const fields=['operationId','archiveRef','state','startedAt','automaticRetry']
if(value.state==='applied')fields.push('finishedAt','accessRevoked','warnings')
else if(value.state!=='running')fields.push('errorCode','finishedAt')
exact(value,fields);id(value.operationId);id(value.archiveRef);instant(value.startedAt)
if(value.operationId!==match[1]||value.automaticRetry!==false)refuse()
if(value.state!=='running')instant(value.finishedAt)
if(value.state==='applied'){
const allowed=new Set(['listenerError','schedulerError','cleanupError','migrationError','maintenanceError'])
if(value.accessRevoked!==true||!Array.isArray(value.warnings)||value.warnings.length>5||value.warnings.some(w=>typeof w!=='string'||!allowed.has(w))||new Set(value.warnings).size!==value.warnings.length)refuse()
// An older writer could record successful archive application despite an unconfirmed schema or journal.
if(value.warnings.includes('migrationError')||value.warnings.includes('maintenanceError')){
unresolvedFiles.add(name)
}
auditSources.push(Object.freeze({operationId:value.operationId,archiveRef:value.archiveRef,finishedAt:value.finishedAt as string,receiptSha256:createHash('sha256').update(raw).digest('hex'),receipt:Object.freeze({...value})}))
}else if(value.state!=='running'&&(typeof value.errorCode!=='string'||!/^[A-Z][A-Z0-9_]{0,79}$/.test(value.errorCode)))refuse()
receipts.set(match[1],{state:String(value.state),archiveRef:value.archiveRef})
}
}
let applied=0,blocked=0
for(const [operationId,receipt]of receipts){
const claim=used.get(receipt.archiveRef)
if(receipt.state==='applied'){if(!claim||claim.operationId!==operationId)refuse();applied++}
else if(receipt.state==='blocked'){if(claim?.operationId===operationId)refuse();blocked++}
else unresolvedFiles.add(operationId+'.receipt.json')
}
for(const [archiveRef,claim]of used){const receipt=receipts.get(claim.operationId);if(!receipt||receipt.archiveRef!==archiveRef||receipt.state!=='applied')unresolvedFiles.add(archiveRef+'.used')}
if(await directory(root)!==before||(await readdir(root)).sort().join('\0')!==[...names].sort().join('\0'))refuse()
for(const [name,signature]of signatures)if(identity(await lstat(path.join(root,name),{bigint:true}))!==signature)refuse()
return {markers:used.size+receipts.size,applied,blocked,unresolved:unresolvedFiles.size>0,auditSources:Object.freeze(auditSources),evidence:Object.freeze(evidence),unresolvedFiles:Object.freeze([...unresolvedFiles].sort())}
}
/** Strict inspection facts; no acceptance decision and no SQL. Operator commands
* use the same grammar, not a looser recovery parser. Existing single managed
* host/operator assumption applies; bounded scans are not a concurrent fence. */
export async function readMaintenanceRecoveryFacts(rawDirectory:string,syncJournal=false){
if(typeof rawDirectory!=='string'||!path.isAbsolute(rawDirectory)||path.resolve(rawDirectory)!==rawDirectory)refuse()
await existingAncestors(rawDirectory);const stateBefore=await directory(rawDirectory,true)
if(stateBefore===null)return {kind:'fresh-legacy' as const,journal:Buffer.alloc(0),operations:[],vault:{markers:0,applied:0,blocked:0,unresolved:false,auditSources:[],evidence:[],unresolvedFiles:[]} as VaultFacts,generation:null}
const journalFile=path.join(rawDirectory,'database-maintenance.jsonl'),raw=await bounded(journalFile,MAX_JOURNAL_BYTES,true,syncJournal),vault=await vaultFacts(rawDirectory)
if(await directory(rawDirectory)!==stateBefore)refuse()
if(raw===null||raw.length===0){if(vault.markers)refuse('JOURNAL_ABSENT_WITH_ATTEMPT_MARKERS');if(raw===null)return {kind:'fresh-legacy' as const,journal:Buffer.alloc(0),operations:[],vault,generation:null}
const generation=await bounded(path.join(rawDirectory,'access-generation'),32);if(generation?.length!==32)refuse();return {kind:'empty-before-intent' as const,journal:raw,operations:[],vault,generation}
}
const generation=await bounded(path.join(rawDirectory,'access-generation'),32);if(generation?.length!==32)refuse()
const {operations,authenticatedRecords}=inspectMaintenanceJournal(raw)
if(authenticatedRecords.length){
const material=await bounded(path.join(rawDirectory,'encryption-key'),32)
if(material?.length!==32)refuse()
const key=createHmac('sha256',material).update('MCP private fingerprint:'+RESTORE_DATA_JOURNAL_DOMAIN).digest()
material.fill(0)
try {
for(const {payload,mac} of authenticatedRecords){
const expected=createHmac('sha256',key).update(payload).digest()
if(!timingSafeEqual(expected,Buffer.from(mac,'hex')))refuse()
}
} finally {key.fill(0)}
}
if(!(await bounded(journalFile,MAX_JOURNAL_BYTES))!.equals(raw)||await directory(rawDirectory)!==stateBefore)refuse()
return {kind:'journal' as const,journal:raw,operations,vault,generation,authenticated:authenticatedRecords.length>0}
}
export function databaseRestoreJournalHead(journal:Buffer):RestoreJournalHead{
return {bytes:journal.length,sha256:createHash('sha256').update(journal).digest('hex')}
}
export async function readPendingRestoreCompletion(directory:string,operationId:string,intentHead:RestoreJournalHead,syncJournal=false):Promise<{
appliedHead:RestoreJournalHead;previousHead:RestoreJournalHead|null;binding:RestoreDataJournalBinding;appliedAt:number
}>{
const facts=await readMaintenanceRecoveryFacts(directory,syncJournal)
if(facts.kind!=='journal'||!facts.authenticated)refuse()
const {journal}=facts
const {bytes,sha256}=intentHead
if(!Number.isInteger(bytes)||bytes<2||bytes>=journal.length||journal[bytes-1]!==0x0a)refuse()
if(databaseRestoreJournalHead(journal.subarray(0,bytes)).sha256!==sha256)refuse()
if(journal.indexOf(0x0a,bytes)!==journal.length-1)refuse()
// The full scan validates each record, authenticates signed entries and matches the transition binding.
const intentStart=journal.lastIndexOf(0x0a,bytes-2)+1
const intent=parseMaintenanceJSON(journal.subarray(intentStart,bytes-1)) as RestoreDataJournalRecord
const applied=parseMaintenanceJSON(journal.subarray(bytes,journal.length-1)) as RestoreDataJournalRecord
if(intent.format!==RESTORE_DATA_JOURNAL_DOMAIN||intent.state!=='intent'||intent.operationId!==operationId||
applied.format!==RESTORE_DATA_JOURNAL_DOMAIN||applied.state!=='applied'||applied.operationId!==operationId)refuse()
const previous=journal.subarray(0,intentStart)
const previousHead=previous.length&&inspectMaintenanceJournal(previous).authenticatedRecords.length?databaseRestoreJournalHead(previous):null
return {appliedHead:databaseRestoreJournalHead(journal),previousHead,binding:applied.binding,appliedAt:Date.parse(applied.at)}
}
/** The caller holds the system database operation lock and has checked the active claim. */
export async function assertDatabaseRestoreJournalCurrent(client:Parameters<typeof readDatabaseRestoreJournalHead>[0],directory:string){
const head=await readDatabaseRestoreJournalHead(client)
const facts=await readMaintenanceRecoveryFacts(directory)
const authenticated=facts.kind==='journal'&&facts.authenticated
if(head){
const actual=databaseRestoreJournalHead(facts.journal)
if(!authenticated||actual.bytes!==head.bytes||actual.sha256!==head.sha256)refuse()
}else if(authenticated)refuse()
return head
}
/** Acceptance grants startup admission, so POSIX ownership/modes are verified.
* Native Windows protection has no implementation and refuses, never modeled. */
export async function readMaintenanceAcceptanceRecords(stateDirectory:string):Promise<readonly Buffer[]>{
const root=path.join(stateDirectory,'maintenance-acceptances'),before=await directory(root,true);if(before===null)return []
if(process.platform==='win32'||typeof process.getuid!=='function')refuse()
const uid=BigInt(process.getuid()),protectedStat=async(file:string,isDirectory:boolean)=>{const s=await lstat(file,{bigint:true});if(s.uid!==uid||(s.mode&0o077n)!==0n||(isDirectory?!s.isDirectory():!s.isFile()))refuse()}
await protectedStat(root,true)
for(let current=path.dirname(root);;current=path.dirname(current)){const stat=await lstat(current,{bigint:true});if(stat.isSymbolicLink()||!stat.isDirectory()||(stat.uid!==uid&&stat.uid!==0n)||((stat.mode&0o022n)!==0n&&!(stat.uid===0n&&(stat.mode&0o1000n)!==0n)))refuse();if(current===path.dirname(current))break}
const names=(await readdir(root)).sort();if(names.length>64)refuse();const records:Buffer[]=[]
for(const name of names){if(!/^[a-f0-9-]{36}\.json$/.test(name))refuse();id(name.slice(0,36));const file=path.join(root,name);await protectedStat(file,false);const raw=(await bounded(file,1048576))!,value=json(raw);if(!plain(value)||value.resolutionId!==name.slice(0,36))refuse();records.push(raw)}
if(await directory(root)!==before||(await readdir(root)).sort().join('\0')!==names.join('\0'))refuse();return records
}
const configuredTarget=()=>process.env.DATABASE_URL??'postgres://mmapp:[email protected]:5432/mmapp'
export async function inspectDatabaseMaintenanceStartup(rawDirectory:string,runtimeTarget=configuredTarget()):Promise<MaintenanceStartupInspection>{
const facts={journalOperations:0,coreApplied:0,outerAppliedReceipts:0,blockedReceipts:0,accessGenerationObserved:false,outerOperationInferred:false as const,auditImportStatus:'not-verified' as const,postRepairSchemaStatus:'not-inspected' as const,acceptedHistoricResolutions:0,historicOutcomesChanged:false as const,automaticRestore:false as const}
const result=(classification:MaintenanceStartupInspection['classification'],reason:Reason):MaintenanceStartupInspection=>Object.freeze({classification,reason,...facts})
try{
const state=await readMaintenanceRecoveryFacts(rawDirectory),records=await readMaintenanceAcceptanceRecords(rawDirectory)
facts.outerAppliedReceipts=state.vault.applied;facts.blockedReceipts=state.vault.blocked;facts.accessGenerationObserved=state.generation!==null
if(state.kind!=='journal'){if(records.length)refuse();return result(state.kind,'NONE')}
facts.journalOperations=state.operations.length;facts.coreApplied=state.operations.filter(op=>op.state==='applied').length
const accepted=evaluateMaintenanceAcceptances(records,state.journal,state.vault.evidence,records.length?runtimeDatabaseTargetSha256(runtimeTarget):'0'.repeat(64));facts.acceptedHistoricResolutions=accepted.count
const unresolved=state.operations.filter(op=>op.state!=='applied'&&!accepted.acceptedCoreIds.has(op.operationId))
if(unresolved.some(op=>op.state==='intent'))return result('inspection-required','UNFINISHED_MAINTENANCE')
if(unresolved.some(op=>op.state==='unconfirmed'))return result('inspection-required','UNCONFIRMED_MAINTENANCE')
if(state.vault.unresolvedFiles.some(file=>!accepted.acceptedVaultFiles.has(file)))return result('inspection-required','EXTERNAL_OPERATION_UNRESOLVED')
return result(accepted.count?'accepted-current-state':'completed-core-maintenance','NONE')
}catch(error){return result('inspection-required',error instanceof Refusal?error.reason:'EXTERNAL_STATE_UNAVAILABLE_OR_MALFORMED')}
}
export class DatabaseMaintenanceStartupError extends Error {
readonly code='DATABASE_MAINTENANCE_INSPECTION_REQUIRED'
constructor(readonly inspection:MaintenanceStartupInspection){super('External database maintenance requires read-only inspection before managed startup')}
}
export async function assertDatabaseMaintenanceStartup(directory:string,runtimeTarget=configuredTarget()):Promise<MaintenanceStartupInspection>{
const result=await inspectDatabaseMaintenanceStartup(directory,runtimeTarget)
if(result.classification==='inspection-required')throw new DatabaseMaintenanceStartupError(result)
return result
}
/** Audit recovery consumes the same strict grammar/claims as startup. No core
* applied record can manufacture an applied outer receipt or original actor. */
export async function readAppliedRestoreAuditSources(directory:string,runtimeTarget=configuredTarget()):Promise<readonly AppliedRestoreAuditSource[]>{
await assertDatabaseMaintenanceStartup(directory,runtimeTarget)
const before=await vaultFacts(directory)
await assertDatabaseMaintenanceStartup(directory,runtimeTarget)
const after=await vaultFacts(directory)
if(JSON.stringify(before)!==JSON.stringify(after))throw new DatabaseMaintenanceStartupError(await inspectDatabaseMaintenanceStartup(directory))
return after.auditSources
}