200 lines
19 KiB
TypeScript
200 lines
19 KiB
TypeScript
import assert from "node:assert/strict"
|
|
import { randomUUID } from "node:crypto"
|
|
import { createProbeJournalStorage,probeStorageFits,ProbeStorageRefused,ProbeStorageUnknown,PROBE_PAYLOAD_MAX_BYTES,PROBE_PERMANENT_RESERVATION_BYTES,type ProbeStorageDependencies,type ProbeStoragePolicy,type ProbeStorageScope } from "./server-probe-journal-maintenance.js"
|
|
import { probeMetadataGcEligible } from "./server-probe-journal-maintenance-model.js"
|
|
|
|
const digest="a".repeat(64),at=Date.parse("2026-10-25T00:00:00Z")
|
|
const policy:ProbeStoragePolicy={retentionDays:7,globalBytes:2000000,ownerBytes:1000000,globalFiles:4,ownerFiles:2,globalAdmissions:5,ownerAdmissions:3}
|
|
const request=():ProbeStorageScope=>({ownerId:"owner-a",receiptId:randomUUID(),identityKind:"rest",executionId:randomUUID(),bindingProof:digest,authorityExpiresAt:new Date(at+300000).toISOString()})
|
|
{
|
|
const original={authorityExpiresAt:new Date(at).toISOString(),lastActivityAt:new Date(at-1).toISOString()}
|
|
assert.equal(probeMetadataGcEligible(original,at+7*86400000+60000,7,60000),false)
|
|
assert.equal(probeMetadataGcEligible(original,at+7*86400000+60001,7,60000),true)
|
|
assert.equal(probeMetadataGcEligible({...original,lastActivityAt:new Date(at+86400000).toISOString()},at+7*86400000+60001,7,60000),false)
|
|
assert.equal(probeMetadataGcEligible({...original,authorityExpiresAt:"unavailable"},at+100*86400000,7,60000),false)
|
|
assert.equal(probeMetadataGcEligible(original,at+100*86400000,0,60000),false)
|
|
}
|
|
type Row=Record<string,unknown>
|
|
function fixture(){
|
|
const cancellations=new Map<string,Row>();let currentTime=at
|
|
const admissions=new Map<string,Row>(),payloads=new Map<string,Row>(),usage=new Map<string,Row>([["global",{scope:"global",admissions:0,reserved_bytes:0,reserved_files:0}]])
|
|
const effects:string[]=[],anchors=new Set<string>();let queries=0,allow=true,lostCommit=0,commits=0,closedUnsafe=false,phaseHook:((tag:string)=>void)|undefined
|
|
let tail=Promise.resolve()
|
|
const deps:ProbeStorageDependencies={policy,safetyWindowMs:60000,now:()=>currentTime,async authorize(){assert.equal(allow,true);return digest},async connect(){
|
|
let unlock:(()=>void)|undefined,saved:{admissions:Map<string,Row>;payloads:Map<string,Row>;usage:Map<string,Row>;anchors:Set<string>}|undefined
|
|
const restore=()=>{if(saved){for(const [target,source]of [[admissions,saved.admissions],[payloads,saved.payloads],[usage,saved.usage]]as const){target.clear();for(const [key,value]of source)target.set(key,value)}anchors.clear();for(const key of saved.anchors)anchors.add(key)}}
|
|
return {release(unsafe){closedUnsafe ||= !!unsafe;unlock?.();unlock=undefined},async query(text,values=[],options){
|
|
queries++;assert.ok(!options||options.timeoutMs<=10000);let rows:Row[]=[]
|
|
const tag=text.match(/probe-storage:([\w-]+)/)?.[1]??text
|
|
if(tag==="lock"){const before=tail;tail=new Promise<void>(resolve=>{unlock=resolve});await before;saved=structuredClone({admissions,payloads,usage,anchors})}
|
|
else if(tag==="gc-candidates")rows=[...admissions.values()].filter(row=>row.owner_id===values[0]&&probeMetadataGcEligible({authorityExpiresAt:String(row.authority_expires_at),lastActivityAt:String(row.authority_expires_at)},currentTime,Number(values[1]),Number(values[3]))).slice(0,Number(values[2])).map(row=>({...row,last_activity:row.authority_expires_at}))
|
|
else if(tag==="gc-payload"){for(const id of values[0]as string[])payloads.delete(id)}
|
|
else if(tag==="gc-stages"||tag==="gc-quarantines"){}
|
|
else if(tag==="gc-anchors"){for(const id of values[0]as string[])anchors.delete(id)}
|
|
else if(tag==="gc-admissions"){for(const id of values[1]as string[]){assert.equal(admissions.get(id)!.owner_id,values[0]);admissions.delete(id);rows.push({receipt_id:id})}}
|
|
else if(tag==="gc-release")rows=(values[0]as string[]).map(key=>{const row=usage.get(key)!;row.admissions=Number(row.admissions)-Number(values[1]);row.reserved_files=Number(row.reserved_files)-Number(values[2]);row.reserved_bytes=Number(row.reserved_bytes)-Number(values[3]);return {scope:key}})
|
|
else if(tag==="gc-owner-counter"){const key=String(values[0]),row=usage.get(key);if(row&&row.admissions===0&&row.reserved_files===0&&row.reserved_bytes===0)usage.delete(key)}
|
|
else if(tag==="gc-readback")rows=(values[0]as string[]).filter(id=>admissions.has(id)||payloads.has(id)||anchors.has(id)).map(receipt_id=>({receipt_id}))
|
|
else if(tag==="cancel-gc-candidates")rows=[...cancellations.values()].filter(row=>row.actor_id===values[0]&&row.owner_id===values[0]&&row.operation_name==="servers.probes.run"&&["applied","failed","unknown"].includes(String(row.state))&&row.finished_at!==null&&row.real_plan===true&&probeMetadataGcEligible({authorityExpiresAt:String(row.authority_expires_at),lastActivityAt:String(row.last_activity)},currentTime,Number(values[1]),Number(values[3]))).slice(0,Number(values[2]))
|
|
else if(tag==="cancel-gc-delete"){for(const id of values[0]as string[]){assert.equal(cancellations.get(id)!.actor_id,values[1]);cancellations.delete(id);rows.push({operation_id:id})}}
|
|
else if(tag==="cancel-gc-readback")rows=(values[0]as string[]).filter(id=>cancellations.has(id)).map(operation_id=>({operation_id}))
|
|
else if(tag==="owner-counter"){const key=String(values[0]);if(!usage.has(key))usage.set(key,{scope:key,admissions:0,reserved_bytes:0,reserved_files:0})}
|
|
else if(tag==="counters")rows=(values[0]as string[]).map(key=>usage.get(key)!)
|
|
else if(tag==="reserve"||tag==="cancel-reserve"){
|
|
assert.match(text,/authority_expires_at/);assert.match(text,/clock_timestamp\(\)/)
|
|
const [id,ownerId,kind,executionId,bindingProof,expires,token]=values
|
|
if(!admissions.has(String(id))&&![...admissions.values()].some(row=>row.identity_kind===kind&&row.execution_id===executionId)){
|
|
admissions.set(String(id),{receipt_id:id,owner_id:ownerId,identity_kind:kind,execution_id:executionId,binding_proof:bindingProof,authority_expires_at:expires,claim_token:tag==="cancel-reserve"?null:token,purged_at:null,claim_attempted_at:tag==="cancel-reserve"?new Date(at).toISOString():null,cancelled_at:tag==="cancel-reserve"?new Date(at).toISOString():null});rows=[{receipt_id:id}]
|
|
}
|
|
}else if(tag==="charge"||tag==="release"){
|
|
rows=(values[0]as string[]).map(key=>{const row=usage.get(key)!;if(tag==="charge"){row.admissions=Number(row.admissions)+1;row.reserved_files=Number(row.reserved_files)+1;row.reserved_bytes=Number(row.reserved_bytes)+Number(values[1])}else{row.reserved_files=Number(row.reserved_files)-Number(values[1]);row.reserved_bytes=Number(row.reserved_bytes)-Number(values[2])}return {scope:key}})
|
|
}else if(tag==="admission")rows=[...admissions.values()].filter(row=>row.receipt_id===values[0]||row.identity_kind===values[1]&&row.execution_id===values[2])
|
|
else if(tag==="cancel-existing"){const row=admissions.get(String(values[0]));if(row&&row.cancelled_at===null){row.cancelled_at=new Date(at).toISOString();row.claim_token=null;row.claim_attempted_at??=new Date(at).toISOString();rows=[{receipt_id:values[0]}]}}
|
|
else if(tag==="consume"){const row=admissions.get(String(values[0]));assert.match(text,/authority_expires_at>clock_timestamp/);assert.match(text,/cancelled_at IS NULL/);if(row?.claim_token===values[1]&&row.purged_at===null&&row.cancelled_at===null){row.claim_token=null;row.claim_attempted_at=new Date(at).toISOString();rows=[{receipt_id:values[0]}]}}
|
|
else if(tag==="publish"){const id=String(values[0]);assert.ok(anchors.has(id));assert.ok(!payloads.has(id));payloads.set(id,{receipt_ciphertext:values[1],receipt_proof:values[2]});rows=[{receipt_id:id}]}
|
|
else if(tag==="read"){const row=admissions.get(String(values[0]));if(row)rows=[{...row,...(payloads.get(String(values[0]))??{receipt_ciphertext:null,receipt_proof:null})}]}
|
|
else if(tag==="purge-candidates"){assert.match(text,/a.owner_id=\$1/);assert.match(text,/authority_expires_at/);rows=[...admissions.values()].filter(row=>row.owner_id===values[0]&&row.purged_at===null&&row.expired===true).slice(0,Number(values[2])).map(row=>({receipt_id:row.receipt_id}))}
|
|
else if(tag==="mark-purged"){for(const id of values[1]as string[]){const row=admissions.get(id)!;assert.equal(row.owner_id,values[0]);assert.equal(row.purged_at,null);row.purged_at=new Date(at).toISOString();row.claim_token=null;rows.push({receipt_id:id})}}
|
|
else if(tag==="delete-payload"){for(const id of values[0]as string[]){assert.notEqual(admissions.get(id)!.purged_at,null);if(payloads.delete(id))rows.push({receipt_id:id})}}
|
|
else if(tag==="purge-readback")rows=(values[0]as string[]).filter(id=>payloads.has(id)).map(id=>({receipt_id:id}))
|
|
else if(text==="test:anchor"){anchors.add(String(values[0]));effects.push(text)}
|
|
else if(text==="test:terminal")effects.push(text)
|
|
else if(text==="COMMIT"){commits++;unlock?.();unlock=undefined;if(commits===lostCommit)throw Error("private-driver-detail")}
|
|
else if(text==="ROLLBACK"){restore();unlock?.();unlock=undefined}
|
|
phaseHook?.(tag);return {rows:structuredClone(rows)}
|
|
}}
|
|
}}
|
|
return {cancellations,setNow(value:number){currentTime=value},deps,storage:createProbeJournalStorage(deps),admissions,payloads,usage,effects,anchors,options:{deadlineAt:at+120000},setAllow(value:boolean){allow=value},setHook(value:(tag:string)=>void){phaseHook=value},loseCommit(value:number){lostCommit=value},get queries(){return queries},get unsafe(){return closedUnsafe}}
|
|
}
|
|
async function claimed(f:ReturnType<typeof fixture>,scope=request()){await f.storage.claim(scope,f.options,sql=>sql.query("test:anchor",[scope.receiptId]));return scope}
|
|
|
|
{
|
|
const f=fixture(),scope=await claimed(f)
|
|
assert.equal(f.usage.get("global")!.reserved_bytes,PROBE_PAYLOAD_MAX_BYTES+PROBE_PERMANENT_RESERVATION_BYTES)
|
|
await f.storage.publish(scope,"sealed-payload",digest,f.options,sql=>sql.query("test:terminal"))
|
|
assert.equal((await f.storage.read(scope,f.options)).kind,"payload")
|
|
const foreign=await claimed(f,{...request(),ownerId:"owner-b"});f.admissions.get(scope.receiptId)!.expired=true;f.admissions.get(foreign.receiptId)!.expired=true
|
|
assert.deepEqual(await f.storage.purge(scope.ownerId,100,f.options),{purged:1,automaticReplay:false})
|
|
assert.ok(f.anchors.has(scope.receiptId));assert.ok(f.admissions.has(scope.receiptId));assert.equal(f.payloads.has(scope.receiptId),false)
|
|
assert.equal(f.admissions.get(foreign.receiptId)!.purged_at,null)
|
|
assert.equal((await f.storage.read(scope,f.options)).kind,"expired")
|
|
assert.equal(f.usage.get("owner:owner-a")!.admissions,1);assert.equal(f.usage.get("owner:owner-a")!.reserved_bytes,PROBE_PERMANENT_RESERVATION_BYTES)
|
|
assert.deepEqual(await f.storage.purge(scope.ownerId,100,f.options),{purged:0,automaticReplay:false})
|
|
await assert.rejects(f.storage.claim(scope,f.options,async()=>{throw Error("replayed")}),ProbeStorageRefused)
|
|
await assert.rejects(f.storage.publish(scope,"again",digest,f.options,async()=>{throw Error("republished")}),ProbeStorageRefused)
|
|
assert.equal(f.effects.filter(value=>value==="test:anchor").length,2)
|
|
}
|
|
{
|
|
const f=fixture();f.loseCommit(1)
|
|
await assert.rejects(claimed(f),ProbeStorageUnknown);assert.equal(f.effects.length,0);assert.equal(f.admissions.size,1);assert.equal(f.unsafe,true)
|
|
}
|
|
{
|
|
const f=fixture(),scope=request();f.loseCommit(2)
|
|
await assert.rejects(claimed(f,scope),ProbeStorageUnknown);assert.equal(f.effects.length,1)
|
|
await assert.rejects(claimed(f,scope),ProbeStorageRefused);assert.equal(f.effects.length,1)
|
|
}
|
|
{
|
|
const f=fixture(),scope=request();f.setHook(tag=>{if(tag==="consume")f.setAllow(false)})
|
|
await assert.rejects(claimed(f,scope),ProbeStorageRefused);assert.equal(f.effects.length,0);assert.equal(f.admissions.size,1)
|
|
}
|
|
{
|
|
const f=fixture(),scope=await claimed(f);f.setHook(tag=>{if(tag==="publish")f.setAllow(false)})
|
|
await assert.rejects(f.storage.publish(scope,"payload",digest,f.options,sql=>sql.query("test:terminal")),ProbeStorageRefused)
|
|
assert.equal(f.payloads.size,0)
|
|
}
|
|
{
|
|
const f=fixture(),scopes=[request(),request(),request()]
|
|
const outcomes=await Promise.allSettled(scopes.map(scope=>claimed(f,scope)))
|
|
assert.equal(outcomes.filter(value=>value.status==="fulfilled").length,2);assert.equal(f.admissions.size,2);assert.equal(f.usage.get("owner:owner-a")!.reserved_files,2)
|
|
assert.equal(probeStorageFits(policy,{admissions:0,reserved_bytes:Number.MAX_SAFE_INTEGER,reserved_files:0},{admissions:0,reserved_bytes:0,reserved_files:0}),false)
|
|
const before=f.queries;await assert.rejects(claimed(f,{...request(),authorityExpiresAt:new Date(at).toISOString()}),ProbeStorageRefused);assert.equal(f.queries,before)
|
|
await assert.rejects(f.storage.purge("owner-a",101,f.options),ProbeStorageRefused)
|
|
}
|
|
{
|
|
const f=fixture(),old=await claimed(f)
|
|
await f.storage.publish(old,"old-payload",digest,f.options,sql=>sql.query("test:terminal"))
|
|
const foreign=await claimed(f,{...request(),ownerId:"owner-b"})
|
|
const after=at+9*86400000,gcOptions={deadlineAt:after+120000};f.setNow(after)
|
|
assert.deepEqual(await f.storage.collectExpired("owner-a",100,gcOptions),{collected:1,automaticReplay:false})
|
|
assert.equal(f.admissions.has(old.receiptId),false);assert.equal(f.anchors.has(old.receiptId),false);assert.equal(f.payloads.has(old.receiptId),false)
|
|
assert.ok(f.admissions.has(foreign.receiptId));assert.equal(f.usage.has("owner:owner-a"),false)
|
|
assert.deepEqual(await f.storage.collectExpired("owner-a",100,gcOptions),{collected:0,automaticReplay:false})
|
|
const calls=f.queries;await assert.rejects(f.storage.claim(old,gcOptions,sql=>sql.query("test:anchor",[old.receiptId])),ProbeStorageRefused);assert.equal(f.queries,calls)
|
|
const fresh={...request(),authorityExpiresAt:new Date(after+300000).toISOString()}
|
|
await f.storage.claim(fresh,gcOptions,sql=>sql.query("test:anchor",[fresh.receiptId]));assert.ok(f.admissions.has(fresh.receiptId))
|
|
}
|
|
{
|
|
const f=fixture(),old=await claimed(f);f.admissions.get(old.receiptId)!.expired=true
|
|
await f.storage.purge("owner-a",100,f.options)
|
|
const after=at+9*86400000;f.setNow(after)
|
|
await f.storage.collectExpired("owner-a",100,{deadlineAt:after+120000})
|
|
assert.deepEqual(f.usage.get("global"),{scope:"global",admissions:0,reserved_bytes:0,reserved_files:0})
|
|
}
|
|
{
|
|
const f=fixture(),old=await claimed(f),after=at+9*86400000;f.setNow(after);f.setHook(tag=>{if(tag==="gc-admissions")f.setAllow(false)})
|
|
await assert.rejects(f.storage.collectExpired("owner-a",100,{deadlineAt:after+120000}),ProbeStorageRefused)
|
|
assert.ok(f.admissions.has(old.receiptId));assert.ok(f.anchors.has(old.receiptId))
|
|
}
|
|
{
|
|
const f=fixture(),old=await claimed(f),after=at+9*86400000;f.setNow(after);f.loseCommit(3)
|
|
await assert.rejects(f.storage.collectExpired("owner-a",100,{deadlineAt:after+120000}),ProbeStorageUnknown)
|
|
assert.equal(f.admissions.has(old.receiptId),false);assert.equal(f.unsafe,true)
|
|
}
|
|
{
|
|
const f=fixture(),after=at+9*86400000;f.setNow(after)
|
|
const base={connection_id:randomUUID(),actor_id:"owner-a",owner_id:"owner-a",operation_name:"servers.probes.run",real_plan:true,state:"applied",finished_at:new Date(at).toISOString(),authority_expires_at:new Date(at).toISOString(),last_activity:new Date(at).toISOString()}
|
|
for(const row of [base,{...base,state:"pending",finished_at:null},{...base,state:"running",finished_at:null},{...base,actor_id:"owner-b",owner_id:"owner-b"},{...base,real_plan:false}]){const id=randomUUID();f.cancellations.set(id,{...row,operation_id:id})}
|
|
f.usage.set("owner:owner-a",{scope:"owner:owner-a",admissions:4,reserved_files:4,reserved_bytes:4*294912});f.usage.set("global",{scope:"global",admissions:5,reserved_files:5,reserved_bytes:5*294912})
|
|
assert.deepEqual(await f.storage.collectExpiredCancellations("owner-a",100,{deadlineAt:after+120000}),{collected:1,automaticReplay:false})
|
|
assert.equal(f.cancellations.size,4);assert.equal(f.usage.get("owner:owner-a")!.reserved_files,4);assert.equal(f.usage.get("owner:owner-a")!.reserved_bytes,4*294912)
|
|
assert.deepEqual(await f.storage.collectExpiredCancellations("owner-a",100,{deadlineAt:after+120000}),{collected:0,automaticReplay:false})
|
|
}
|
|
|
|
console.log("Probe journal storage inert tests passed: preclaim budgets, concurrent admission, lost COMMIT, owner/late guards, payload retention and approved expiry GC; no SQL/network")
|
|
|
|
async function cancelled(f:ReturnType<typeof fixture>,bound=request()){
|
|
return f.storage.cancel(bound.ownerId,f.options,async sql=>{await sql.query("test:source-lock");return bound},async sql=>{await sql.query("test:cancel");return bound.receiptId})
|
|
}
|
|
{
|
|
const f=fixture(),bound={...request(),authorityExpiresAt:new Date(at-1).toISOString()},order:string[]=[]
|
|
f.setHook(tag=>order.push(tag));assert.equal(await cancelled(f,bound),bound.receiptId)
|
|
assert.ok(order.indexOf("lock")<order.indexOf("test:source-lock"));assert.equal(f.admissions.size,1)
|
|
const row=f.admissions.get(bound.receiptId)!;assert.equal(row.claim_token,null);assert.notEqual(row.claim_attempted_at,null);assert.notEqual(row.cancelled_at,null)
|
|
assert.equal(f.usage.get("global")!.reserved_bytes,294912)
|
|
await claimed(f);const before=structuredClone([...f.usage]);const timestamp=row.cancelled_at
|
|
await cancelled(f,bound);assert.deepEqual([...f.usage],before);assert.equal(row.cancelled_at,timestamp)
|
|
await assert.rejects(cancelled(f),ProbeStorageRefused)
|
|
await assert.rejects(claimed(f,bound),ProbeStorageRefused)
|
|
await assert.rejects(f.storage.publish(bound,"payload",digest,f.options,sql=>sql.query("test:terminal")),ProbeStorageRefused)
|
|
}
|
|
{
|
|
const f=fixture(),bound=await claimed(f),row=f.admissions.get(bound.receiptId)!,attempted=row.claim_attempted_at
|
|
await cancelled(f,bound);assert.equal(row.claim_attempted_at,attempted);assert.equal(f.usage.get("global")!.admissions,1)
|
|
await assert.rejects(cancelled(f,{...bound,ownerId:"other-owner"}),ProbeStorageRefused)
|
|
await assert.rejects(cancelled(f,{...bound,receiptId:randomUUID()}),ProbeStorageRefused)
|
|
await assert.rejects(cancelled(f,{...bound,executionId:randomUUID()}),ProbeStorageRefused)
|
|
}
|
|
{
|
|
const f=fixture(),bounds=[request(),request(),request()]
|
|
const result=await Promise.allSettled([cancelled(f,bounds[0]),claimed(f,bounds[1]),cancelled(f,bounds[2])])
|
|
assert.equal(result.filter(value=>value.status==="fulfilled").length,2);assert.equal(f.usage.get("global")!.admissions,2)
|
|
assert.equal(f.usage.get("global")!.reserved_bytes,2*294912)
|
|
}
|
|
{
|
|
const f=fixture(),bound=request();f.loseCommit(1)
|
|
await assert.rejects(cancelled(f,bound),ProbeStorageUnknown);assert.equal(f.unsafe,true)
|
|
assert.notEqual(f.admissions.get(bound.receiptId)!.cancelled_at,null)
|
|
await cancelled(f,bound);assert.equal(f.usage.get("global")!.admissions,1)
|
|
}
|
|
{
|
|
const f=fixture();const result=await f.storage.cancel("owner-a",f.options,async()=>null,async()=>{throw Error("terminal source must not write")})
|
|
assert.equal(result,undefined);assert.equal(f.admissions.size,0);assert.equal(f.usage.size,1)
|
|
}
|
|
{
|
|
const f=fixture(),bound=request();let boundChecks=0,wrote=false
|
|
f.deps.authorize=async(ownerId,operation,scope)=>{assert.equal(ownerId,bound.ownerId);assert.equal(operation,"cancel");if(scope){boundChecks++;throw Error("revoked source")};return digest}
|
|
await assert.rejects(f.storage.cancel(bound.ownerId,f.options,async()=>bound,async()=>{wrote=true}),ProbeStorageRefused)
|
|
assert.equal(boundChecks,1);assert.equal(wrote,false);assert.equal(f.admissions.size,0)
|
|
}
|
|
console.log("Probe cancellation shared quota tests passed: original expired authority, lock order, full charge, duplicate under exhaustion, collisions, execution race, bound fence, terminal no-op and unknown COMMIT")
|