Files
MikrotikManager/backend/scripts/writer-profile/source58-foundation.mts
T

232 lines
19 KiB
TypeScript

import assert from 'node:assert/strict'
import {createPublicKey,verify} from 'node:crypto'
import type {PoolClient} from 'pg'
import {z} from 'zod'
import {deploymentRequest} from '../database-writer-bootstrap-operator.mjs'
import {assertJsDriverEnvironment,poolOptions,retainPublisherTlsPeer} from './operator-contract.mjs'
import {readProtected,sha} from './operator-files.mjs'
import {createProtectedDatabaseSourceBindingVerifier} from './protected-source-registry.mjs'
import {withActiveSource} from './source-activation.mjs'
import {BUILD57_PROVENANCE,FIXED57_ROLE_TEMPLATE} from './fixed57-build-definition.js'
import {BUILD58_PROVENANCE} from './fixed58-build-definition.js'
import {enrollWriterRegistryInTransaction,writerRegistryEnrollmentSql} from './registry-enrollment.mjs'
import {reviewedSource57Admission} from './source57-admission-plan.mjs'
import {collectSource57Catalogue} from './catalogue.mjs'
import {canonical} from './application-writer-profile-canonical.js'
import {readProtectedRegistryStructureData} from '../../src/db/application-writer-profile-registry-structure.js'
import {captureDatabaseSourceObservation} from '../../src/db/database-source-observation.js'
import {sourceBindingBytes,sourceBindingCanonical} from '../../src/db/database-source-binding.js'
import {captureMigrationSchema} from '../../src/db/migration-schema-inventory.js'
import {MIGRATION_MANIFEST,migrationManifestDigest} from '../../src/db/migration-manifest.js'
import {SYSTEM_DATABASE_OPERATION_LOCK,SYSTEM_DATABASE_REQUEST_LOCK} from '../../src/db/operation-lock.js'
export const SOURCE58_FOUNDATION_DOMAIN='mmapp:source58-foundation:ed25519:v1\n'
export const SOURCE57_ADMISSION_DOMAIN='mmapp:source57-admission:ed25519:v1\n'
const hash=z.string().regex(/^[a-f0-9]{64}$/)
const fields={
bootstrapSha256:hash,sourceHeadSha256:hash,sourceTlsSpkiSha256:hash,manifestSha256:hash,definitionSha256:hash,
preSchemaSha256:hash,expiresAt:z.number().int().positive(),
}
const approval=z.object({body:z.discriminatedUnion('format',[
z.object({...fields,format:z.literal('source58-foundation-approval-v1'),decision:z.literal('install-foundations'),foundationSha256:hash}).strict(),
z.object({...fields,format:z.literal('source57-admission-approval-v1'),decision:z.literal('install-admission'),admissionSha256:hash}).strict(),
]),
signature:z.string().regex(/^[A-Za-z0-9_-]{86}$/)}).strict()
const parse=(bytes:Buffer)=>JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(bytes))
type Request=ReturnType<typeof deploymentRequest>
type Sql={query:<T extends Record<string,unknown>>(text:string,values?:unknown[])=>Promise<{rows:T[];command?:string}>}
const keys=['candidate_require_preadmission','candidate_truncate_preadmitted','candidate_operator_ddl','server_partition_writer_guard']
const sorted=(rows:Record<string,unknown>[])=>rows.map(sourceBindingCanonical).sort()
const withSortedAcl=(row:Record<string,unknown>)=>({...row,acl:Array.isArray(row.acl)?[...row.acl].sort():row.acl})
const identifier=(name:string)=>{assert(/^[a-z_][a-z0-9_]{0,62}$/.test(name));return '"'+name+'"'}
const bind=(text:string,roles:Request['roles'])=>text.replace(/@mm-role:(operator|control|publisher|runtime|capture)@/g,(_,scope)=>roles[scope].name)
function reviewedFoundation(roles:Request['roles']){
const expected=FIXED57_ROLE_TEMPLATE.functions.filter(row=>keys.includes(String(row.proname))).map(row=>JSON.parse(bind(sourceBindingCanonical(row),roles)))
const publicGuard=expected.find(row=>row.namespace==='public')
const functions=keys.slice(0,3).map(name=>expected.find(row=>row.proname===name).definition.replace('CREATE OR REPLACE FUNCTION ','CREATE FUNCTION '))
const schemas=FIXED57_ROLE_TEMPLATE.schemas.map(row=>JSON.parse(bind(sourceBindingCanonical(row),roles)))
const events=FIXED57_ROLE_TEMPLATE.events.map(row=>JSON.parse(bind(sourceBindingCanonical(row),roles)))
const {operator,publisher,runtime,capture}=roles
const sql=[
'ALTER FUNCTION public.server_partition_writer_guard() SET search_path = pg_catalog',
'REVOKE ALL ON SCHEMA public, mmapp_control FROM PUBLIC',
`GRANT USAGE ON SCHEMA public TO ${[publisher,runtime,capture].map(role=>identifier(role.name)).join(', ')}`,
`GRANT USAGE ON SCHEMA mmapp_control TO ${[operator,publisher].map(role=>identifier(role.name)).join(', ')}`,
...functions,
`REVOKE ALL ON FUNCTION ${keys.slice(0,3).map(name=>'mmapp_control.'+name+'()').join(', ')} FROM PUBLIC`,
`GRANT EXECUTE ON FUNCTION mmapp_control.candidate_truncate_preadmitted() TO ${identifier(operator.name)}`,
'CREATE EVENT TRIGGER candidate_operator_ddl ON ddl_command_start EXECUTE FUNCTION mmapp_control.candidate_operator_ddl()',
]
return {expected,publicGuard,schemas,events,sql,sha256:sha(sourceBindingBytes([...sql,...writerRegistryEnrollmentSql(roles)]))}
}
export function source58FoundationSqlSha256(roles:Request['roles']){return reviewedFoundation(roles).sha256}
const functionSql=`SELECT n.nspname AS namespace,p.proname, pg_catalog.pg_get_function_identity_arguments(p.oid) AS arguments,
l.lanname AS language,pg_catalog.pg_get_userbyid(p.proowner) AS owner,p.proconfig,p.prokind,p.prosecdef,p.proacl::text[] AS acl,
pg_catalog.pg_get_functiondef(p.oid) AS definition
FROM pg_catalog.pg_proc p JOIN pg_catalog.pg_namespace n ON n.oid=p.pronamespace JOIN pg_catalog.pg_language l ON l.oid=p.prolang
WHERE (n.nspname='public' AND p.proname=$1) OR (n.nspname='mmapp_control' AND p.proname=ANY($2::text[])) LIMIT 5`
const eventSql=`SELECT e.evtname,e.evtevent,e.evtenabled,e.evttags, pg_catalog.pg_get_userbyid(e.evtowner) AS owner,
n.nspname AS function_namespace,p.proname AS function_name,n.nspname||'.'||p.proname||'()' AS function
FROM pg_catalog.pg_event_trigger e JOIN pg_catalog.pg_proc p ON p.oid=e.evtfoid
JOIN pg_catalog.pg_namespace n ON n.oid=p.pronamespace LIMIT 2`
async function checkTarget(sql:Sql,request:Request){
const subject=await sql.query(`SELECT (pg_catalog.pg_control_system()).system_identifier::text AS system,d.oid::text AS database,
d.datname AS name,pg_catalog.current_setting('server_version_num')::int AS version,r.oid::text AS role,r.rolsuper AS superuser,
n.nspowner::text AS control_owner, pub.nspowner::text AS public_owner
FROM pg_catalog.pg_database d CROSS JOIN pg_catalog.pg_roles r
JOIN pg_catalog.pg_namespace n ON n.nspname='mmapp_control'
JOIN pg_catalog.pg_namespace pub ON pub.nspname='public'
WHERE d.datname=pg_catalog.current_database() AND r.rolname=SESSION_USER AND SESSION_USER=CURRENT_USER
AND NOT pg_catalog.pg_is_in_recovery() AND pg_catalog.current_setting('session_replication_role')='origin'
AND pg_catalog.current_setting('event_triggers')='on'`)
const {source,roles}=request
assert.deepEqual(subject.rows,[{system:source.systemIdentifier,database:source.databaseOid,name:source.databaseName,version:source.serverVersionNum,
role:roles.control.signedOid,superuser:true,control_owner:roles.control.signedOid,public_owner:roles.operator.signedOid}],'Exact authorized control DBA and namespace owners required')
const {rows}=await sql.query(`SELECT rolname AS name,oid::text AS oid FROM pg_catalog.pg_roles WHERE rolname=ANY($1::text[])`,[Object.values(roles).map(role=>role.name)])
assert.deepEqual(sorted(rows),sorted(Object.values(roles).map(role=>({name:role.name,oid:role.signedOid}))),'Exact five role OIDs required')
const markers=await sql.query('SELECT id FROM ONLY public.schema_migrations ORDER BY id COLLATE "C" LIMIT $1',[MIGRATION_MANIFEST.length+1])
assert.deepEqual(markers.rows,MIGRATION_MANIFEST.map(pin=>({id:pin.file.slice(0,-4)})),'Exact current migration markers required')
}
async function checkFresh(sql:Sql,review:ReturnType<typeof reviewedFoundation>){
const fresh=await sql.query(`SELECT NOT EXISTS(SELECT FROM pg_catalog.pg_class WHERE relnamespace='mmapp_control'::pg_catalog.regnamespace)
AND NOT EXISTS(SELECT FROM pg_catalog.pg_proc WHERE pronamespace='mmapp_control'::pg_catalog.regnamespace)
AND NOT EXISTS(SELECT FROM pg_catalog.pg_type WHERE typnamespace='mmapp_control'::pg_catalog.regnamespace)
AND NOT EXISTS(SELECT FROM pg_catalog.pg_event_trigger)
AND NOT EXISTS(SELECT FROM pg_catalog.pg_trigger WHERE tgrelid='public.schema_migrations'::pg_catalog.regclass AND NOT tgisinternal) AS empty`)
assert.deepEqual(fresh.rows,[{empty:true}],'Unenrolled staging control namespace and marker required')
const funcs=await sql.query(functionSql,[keys[3],keys.slice(0,3)])
assert.deepEqual(funcs.rows,[{...review.publicGuard,proconfig:null,definition:review.publicGuard.definition.replace(" SET search_path TO 'pg_catalog'\n",'')}],
'Original numbered public guard required')
}
async function checkFoundation(sql:Sql,review:ReturnType<typeof reviewedFoundation>){
const functions=(await sql.query(functionSql,[keys[3],keys.slice(0,3)])).rows
assert.deepEqual(sorted(functions.map(withSortedAcl)),sorted(review.expected.map(withSortedAcl)),'Exact reviewed guard function readback required')
assert.deepEqual(sorted((await sql.query(eventSql)).rows),sorted(review.events),'Exact reviewed event trigger readback required')
const schemas=await sql.query(`SELECT nspname,pg_catalog.pg_get_userbyid(nspowner) AS owner,nspacl::text[] AS acl
FROM pg_catalog.pg_namespace WHERE nspname IN('public','mmapp_control')`)
assert.deepEqual(sorted(schemas.rows.map(withSortedAcl)),sorted(review.schemas.map(withSortedAcl)),'Exact reviewed schema ACL required')
}
async function checkUnenrolledRelations(sql:Sql,plan:ReturnType<typeof reviewedSource57Admission>,operatorOid:string){
const tables=await sql.query(`SELECT c.relname AS name,c.relkind::text AS kind,c.relowner::text AS owner FROM pg_catalog.pg_class c
WHERE c.relnamespace='public'::pg_catalog.regnamespace AND c.relkind IN ('r','p') LIMIT $1`,[plan.tables.length+1])
assert.deepEqual(sorted(tables.rows),sorted(plan.tables.map(row=>({...row,owner:operatorOid}))),'Exact unpartitioned staging relation inventory required')
const sequences=await sql.query(`SELECT c.relname AS name,c.relowner::text AS owner FROM pg_catalog.pg_class c
WHERE c.relnamespace='public'::pg_catalog.regnamespace AND c.relkind='S' LIMIT $1`,[plan.sequences.length+1])
assert.deepEqual(sorted(sequences.rows),sorted(plan.sequences.map(row=>({...row,owner:operatorOid}))),'Exact staging sequence inventory required')
const {rows}=await sql.query(`SELECT NOT EXISTS(SELECT FROM pg_catalog.pg_trigger t JOIN pg_catalog.pg_class c ON c.oid=t.tgrelid
WHERE c.relnamespace='public'::pg_catalog.regnamespace AND t.tgname IN ('candidate_application_statement_guard','candidate_truncate_preadmitted'))
AND NOT EXISTS(SELECT FROM mmapp_control.application_writer_profile_history) AND NOT EXISTS(SELECT FROM mmapp_control.application_writer_profile_current)
AND pg_catalog.to_regclass('mmapp_control.database_restore_quarantine') IS NULL AS fresh`)
assert.deepEqual(rows,[{fresh:true}],'Unenrolled public relations and empty Source57 writer registry required')
}
export async function source58FoundationMain(args:string[]){
const flags=['bootstrap','bootstrap-sha256','publisher-transport','publisher-transport-sha256','control-transport','control-transport-sha256','ca',
'source-registry','source-spki-sha256','activation-directory','activation-sha256','approval','approval-sha256','approval-public-key','approval-spki-sha256']
const admission=args[0]==='source57-admission-enroll'
assert((admission||args[0]==='source58-foundation-enroll')&&args.length===1+2*flags.length);assertJsDriverEnvironment(process.env)
const values=new Map<string,string>()
for(let index=1;index<args.length;index+=2){const name=args[index].slice(2);assert(args[index].startsWith('--')&&flags.includes(name)&&!values.has(name)&&args[index+1]);values.set(name,args[index+1])}
const get=(name:string)=>values.get(name)!
for(const flag of flags.filter(name=>name.endsWith('sha256')))hash.parse(get(flag))
const bootstrap=parse(await readProtected(get('bootstrap'),32768,get('bootstrap-sha256'))),request=deploymentRequest(bootstrap.deploymentRequest)
assert(bootstrap.format==='application-writer-bootstrap-v1'&&bootstrap.publicationAuthorized===false&&bootstrap.readyAuthorized===false)
const bytes=await readProtected(get('approval'),8192,get('approval-sha256')),raw=parse(bytes),{body,signature}=approval.parse(raw)
const pem=await readProtected(get('approval-public-key'),4096),key=createPublicKey(pem),sig=Buffer.from(signature,'base64url')
const sourcePin={registryDirectory:get('source-registry'),installationId:request.installationId,publicKeySpkiSha256:get('source-spki-sha256'),currentHeadSha256:request.sourceHeadSha256}
assert(!pem.includes('PRIVATE KEY')&&key.asymmetricKeyType==='ed25519'&&get('approval-spki-sha256')!==sourcePin.publicKeySpkiSha256&&
sha(key.export({type:'spki',format:'der'}))===get('approval-spki-sha256')&&sourceBindingBytes(raw).equals(bytes)&&sig.toString('base64url')===signature&&
verify(null,Buffer.from((admission?SOURCE57_ADMISSION_DOMAIN:SOURCE58_FOUNDATION_DOMAIN)+sourceBindingCanonical(body)),key,sig),'Independent signed operator approval required')
const review=reviewedFoundation(request.roles)
const relationPlan=admission?reviewedSource57Admission(request.roles):undefined
const planMatches=body.format==='source57-admission-approval-v1'?
admission&&body.definitionSha256===BUILD57_PROVENANCE.templateSha256&&body.admissionSha256===relationPlan?.sha256:
!admission&&body.definitionSha256===BUILD58_PROVENANCE.templateSha256&&body.foundationSha256===review.sha256
assert(body.bootstrapSha256===get('bootstrap-sha256')&&body.sourceHeadSha256===sourcePin.currentHeadSha256&&body.sourceTlsSpkiSha256===request.tls.spkiSha256&&
body.manifestSha256===migrationManifestDigest()&&planMatches&&body.expiresAt>Date.now()&&body.expiresAt-Date.now()<=900000,'Exact reviewed writer plan required')
const ca=await readProtected(get('ca'),65536,request.tls.caSha256)
const transport=async(role:'publisher'|'control')=>poolOptions(parse(await readProtected(get(role+'-transport'),16384,get(role+'-transport-sha256'))),request,ca,role)
const [publisherOptions,controlOptions]=await Promise.all([transport('publisher'),transport('control')])
const {default:pg}=await import('pg');assertJsDriverEnvironment(process.env)
const expectedSource={...request.source,databaseOid:Number(request.source.databaseOid),sessionUser:request.roles.publisher.name,currentUser:request.roles.publisher.name}
const deadlineAt=Math.min(body.expiresAt,Date.now()+120000)
return withActiveSource({directory:get('activation-directory'),expectedSha256:get('activation-sha256'),sourcePin,expectedSource,
tlsSpkiSha256:request.tls.spkiSha256,deadlineAt},createProtectedDatabaseSourceBindingVerifier(sourcePin),async verifier=>{
const publisher=new pg.Pool(publisherOptions),control=new pg.Pool(controlOptions)
let lost=false,commitSent=false,committed=false,cleanupUnconfirmed=false,client:PoolClient|undefined,registryStructureSha256:string|undefined,catalogueSha256:string|undefined
for(const pool of [publisher,control])pool.on('error',()=>{lost=true})
const retained=(client:PoolClient)=>{
const check=retainPublisherTlsPeer(client,request.tls)
return {check,sql:{query:async<T extends Record<string,unknown>>(text:string,values?:unknown[])=>{
assert(!lost&&Date.now()<deadlineAt);check()
const result=await client.query<T>(text,values)
check();assert(!lost&&Date.now()<deadlineAt);return result
}}}
}
async function checkPredecessor(sql:Sql){
await checkTarget(sql,request)
if(relationPlan){
await checkFoundation(sql,review)
await checkUnenrolledRelations(sql,relationPlan,request.roles.operator.signedOid)
}else await checkFresh(sql,review)
}
async function observe(){
const peer=await publisher.connect()
try{
const {sql}=retained(peer)
await sql.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY')
assert.equal((await sql.query('SELECT oid::text AS oid FROM pg_catalog.pg_roles WHERE rolname=CURRENT_USER AND rolname=SESSION_USER')).rows[0]?.oid,request.roles.publisher.signedOid)
await verifier.verify(sql,await captureDatabaseSourceObservation(sql,{deadlineAt}),{deadlineAt})
assert.equal((await sql.query('ROLLBACK')).command,'ROLLBACK')
}finally{peer.release(true)}
}
try{
client=await control.connect()
const {sql,check}=retained(client)
for(const lock of [SYSTEM_DATABASE_REQUEST_LOCK,SYSTEM_DATABASE_OPERATION_LOCK,8723110]){
const {rows:[row]}=await sql.query('SELECT pg_catalog.pg_try_advisory_lock($1::bigint) AS held',[lock]);assert.equal(row?.held,true,'Source58 foundation drain unavailable')
}
await observe()
await sql.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY')
await sql.query("SET LOCAL search_path=pg_catalog; SET LOCAL TimeZone='UTC'; SET LOCAL DateStyle='ISO, YMD'; SET LOCAL standard_conforming_strings=on; SET LOCAL statement_timeout='5s'")
await checkPredecessor(sql)
const before=await captureMigrationSchema(sql,{scope:{applicationNamespaces:['public','mmapp_control']},deadlineAt,includeControlEventTriggers:admission})
assert.equal(before.sha256,body.preSchemaSha256,admission?'Exact signed pre-admission catalogue required':'Exact signed pre-foundation catalogue required')
assert.equal((await sql.query('ROLLBACK')).command,'ROLLBACK')
await sql.query('BEGIN')
await sql.query("SET LOCAL search_path=pg_catalog; SET LOCAL synchronous_commit=on")
await checkPredecessor(sql)
if(relationPlan){
registryStructureSha256=(await readProtectedRegistryStructureData(sql,request.roles)).sha256
for(const statement of relationPlan.sql)await sql.query(statement)
const catalogue=await collectSource57Catalogue(sql,{roleBindings:request.roles})
catalogueSha256=sha(canonical({base:catalogue.catalogSha256,security:registryStructureSha256}))
}else{
for(const statement of review.sql)await sql.query(statement)
registryStructureSha256=(await enrollWriterRegistryInTransaction(sql,request.roles)).sha256
await checkFoundation(sql,review)
}
await observe()
check();assert(!lost&&Date.now()<deadlineAt)
commitSent=true
assert.equal((await client.query('COMMIT')).command,'COMMIT');committed=true
}catch(error){
if(commitSent)throw Object.assign(Error('Writer installation COMMIT must be inspected'),{code:admission?'SOURCE57_ADMISSION_UNKNOWN':'SOURCE58_FOUNDATION_UNKNOWN'})
throw error
}finally{
try{client?.release(true)}catch{cleanupUnconfirmed=true}
const closed=await Promise.allSettled([publisher.end(),control.end()]);cleanupUnconfirmed ||= closed.some(result=>result.status==='rejected')
}
assert(committed&&registryStructureSha256)
return {event:admission?'database-source57-admission':'database-source58-foundation',outcome:'committed',approvalSha256:sha(bytes),
...(relationPlan?{admissionSha256:relationPlan.sha256,catalogueSha256}:{foundationSha256:review.sha256}),
registryStructureSha256,cleanupUnconfirmed:cleanupUnconfirmed||lost,automaticReplay:false,readyAuthorized:false}
})
}