fix(mcp): уточнить переключение маршрутов и хранение операций

This commit is contained in:
Denozordec committed 2026-11-04 19:04:51 +07:00
1 parent 5ce56b5168
commit bf4b9b5443
10 files changed
+293 -30

No files matched your search

@@ -0,0 +1,18 @@
BEGIN;
CREATE OR REPLACE FUNCTION public.mcp_ordinary_retirement_claim_immutable() RETURNS trigger
LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$
BEGIN
IF TG_OP='INSERT' AND NEW.claim_xid=pg_current_xact_id() AND EXISTS(
SELECT 1 FROM public.mcp_operations o JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
WHERE o.id=NEW.operation_id AND o.plan_id=NEW.plan_id AND o.connection_id=NEW.connection_id
AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()-interval '180 days'
AND (p.operation_id,p.operation_version) IN (
('logs.settings.update',1),('traffic.settings.update',2),('uptime.settings.update',2),
('servers.api-ping.settings.update',2),('internet-path.settings.update',1),('geoip.settings.update',1)
) AND p.state='completed'
) THEN RETURN NEW; END IF;
RAISE EXCEPTION 'ordinary operation retirement claim cannot be replaced or reopened';
END $$;
INSERT INTO public.schema_migrations(id) VALUES ('0054_mcp_geoip_retirement') ON CONFLICT(id) DO NOTHING;
COMMIT;
@@ -0,0 +1,18 @@
BEGIN;
CREATE OR REPLACE FUNCTION public.mcp_ordinary_retirement_claim_immutable() RETURNS trigger
LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$
BEGIN
IF TG_OP='INSERT' AND NEW.claim_xid=pg_current_xact_id() AND EXISTS(
SELECT 1 FROM public.mcp_operations o JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
WHERE o.id=NEW.operation_id AND o.plan_id=NEW.plan_id AND o.connection_id=NEW.connection_id
AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()-interval '180 days'
AND (p.operation_id,p.operation_version) IN (
('logs.settings.update',1),('traffic.settings.update',2),('uptime.settings.update',2),
('servers.api-ping.settings.update',2),('internet-path.settings.update',1),('geoip.settings.update',1),('backups.schedule.update',1)
) AND p.state='completed'
) THEN RETURN NEW; END IF;
RAISE EXCEPTION 'ordinary operation retirement claim cannot be replaced or reopened';
END $$;
INSERT INTO public.schema_migrations(id) VALUES ('0055_mcp_backup_schedule_retirement') ON CONFLICT(id) DO NOTHING;
COMMIT;
@@ -1,7 +1,7 @@
import EmbeddedPostgres from 'embedded-postgres'
import { createRequire } from 'node:module'
import { mkdir, mkdtemp, readdir, readFile, writeFile, cp, rm } from 'node:fs/promises'
import { resolve, join, relative, dirname } from 'node:path'
import { resolve, join, relative, dirname, basename } from 'node:path'
import { randomBytes, createHash } from 'node:crypto'
import { createServer } from 'node:net'
import { spawn } from 'node:child_process'
@@ -16,7 +16,8 @@ await readFile(join(root, 'backend/src/modules/mcp/routing-main-transition.ts'))
await readFile(join(root, 'backend/src/modules/mcp/routing-controller-baseline-acceptance.ts'))
if (process.argv.includes('--check-only')) { console.log(JSON.stringify({ mode, sourcePresent: true, acceptanceExecuted: false, productionApplied: false })); process.exit(0) }
await mkdir(join(root, 'tmp'), { recursive: true })
const directory = await mkdtemp(join(root, 'tmp/routing-controller-baseline-owned-')), output = join(directory, 'dist'), sourcePins = []
const ownedPrefix = 'mm-mcp-test-routing-controller-baseline-'
const directory = await mkdtemp(join(root, 'tmp', ownedPrefix)), output = join(directory, 'dist'), sourcePins = []
const hash = buffer => createHash('sha256').update(buffer).digest('hex')
async function compile(sourceDirectory) {
for (const entry of await readdir(sourceDirectory, { withFileTypes: true })) {
@@ -59,7 +60,7 @@ try {
// embedded-postgres waits for a new exit event; avoid waiting again if its
// owned child has already exited. This never targets another database.
if (postgres.process?.exitCode === null) await postgres.stop()
if (dirname(resolve(directory)) !== resolve(root, 'tmp') || !directory.includes('routing-controller-baseline-owned-')) throw new Error('Owned routing cleanup boundary changed')
if (dirname(resolve(directory)) !== resolve(root, 'tmp') || !basename(directory).startsWith(ownedPrefix)) throw new Error('Owned routing cleanup boundary changed')
await rm(join(directory, 'pg'), { recursive: true, force: true, maxRetries: 20, retryDelay: 100 })
console.log('Owned routing PostgreSQL stopped and its data removed; snapshot/log evidence retained')
}
+5 -5
View File
@@ -12,10 +12,10 @@ const reviewed = (source: string, extra: Partial<MigrationReview> = {}) => {
const bytes = Buffer.from(source), file = "9000_fixture.sql"
return () => analyzeMigration(file, bytes, { file, sha256: createHash("sha256").update(bytes).digest("hex"), transaction: "owner-transaction", selfMarker: false, opaqueBodies: 0, bodyReview: [], effects: ["Explicit inert fixture review"], ...extra })
}
await test("all 53 exact file pins and every top-level schema operation are inventoried", () => {
assert.equal(inventory.length, 53); assert.equal(MIGRATION_MANIFEST.length, 53)
await test("all 55 exact file pins and every top-level schema operation are inventoried", () => {
assert.equal(inventory.length, 55); assert.equal(MIGRATION_MANIFEST.length, 55)
assert.equal(inventory.filter(file => file.transaction === "owner-transaction").length, 17)
assert.equal(inventory.filter(file => file.transaction === "self-managed-single-transaction").length, 36)
assert.equal(inventory.filter(file => file.transaction === "self-managed-single-transaction").length, 38)
assert(inventory.every(file => file.operations.length === file.statementCount && file.effects.length > 0))
let tables = new Set<string>()
for (const file of inventory) {
@@ -32,8 +32,8 @@ await test("DO BEGIN and stored function BEGIN are never transaction boundaries"
assert.equal(endpoint.topLevelTransactionControls.length, 2); assert.equal(endpoint.opaqueBodies.length, 2)
assert.deepEqual(endpoint.opaqueBodies.map(body => body.kind), ["stored-function-body", "executed-do-body"])
})
await test("0035 through 0053 have exact atomic file-owned terminal markers", () => {
assert.deepEqual(inventory.filter(file => file.selfMarker).map(file => file.id), ["0035_server_probe_journal", "0036_diagnostic_cancellations", "0037_server_probe_storage_v2", "0038_server_poll_journal_and_writer_guards", "0039_server_poll_storage", "0040_mcp_routing_reservations", "0041_server_poll_terminal_archives", "0042_server_poll_settings_epoch", "0043_mcp_routing_forward_charges", "0044_mcp_routing_manual_inverse_charges", "0045_mcp_operation_replay_tombstones", "0046_server_poll_operation_retirement", "0047_server_poll_retirement_claims", "0048_mcp_ordinary_retirement_claims", "0049_mcp_ordinary_telemetry_retirement", "0050_server_probe_retired_sources", "0051_server_probe_retirement_claims", "0052_server_probe_retired_gc", "0053_server_probe_terminal_proofs"])
await test("0035 through 0055 have exact atomic file-owned terminal markers", () => {
assert.deepEqual(inventory.filter(file => file.selfMarker).map(file => file.id), ["0035_server_probe_journal", "0036_diagnostic_cancellations", "0037_server_probe_storage_v2", "0038_server_poll_journal_and_writer_guards", "0039_server_poll_storage", "0040_mcp_routing_reservations", "0041_server_poll_terminal_archives", "0042_server_poll_settings_epoch", "0043_mcp_routing_forward_charges", "0044_mcp_routing_manual_inverse_charges", "0045_mcp_operation_replay_tombstones", "0046_server_poll_operation_retirement", "0047_server_poll_retirement_claims", "0048_mcp_ordinary_retirement_claims", "0049_mcp_ordinary_telemetry_retirement", "0050_server_probe_retired_sources", "0051_server_probe_retirement_claims", "0052_server_probe_retired_gc", "0053_server_probe_terminal_proofs", "0054_mcp_geoip_retirement", "0055_mcp_backup_schedule_retirement"])
for (const file of inventory.filter(file => file.selfMarker)) { assert.equal(file.selfMarker!.atomicWithAllFileEffects, true); assert.equal(file.selfMarker!.statement, file.statementCount - 2); assert.equal(file.currentOuterMarkerGap, false) }
assert.equal(inventory.filter(file => file.currentOuterMarkerGap).length, 34)
})
+26
View File
@@ -828,6 +828,32 @@ const classifiedMigrationPins: readonly MigrationReview[] = [
"effects": [
"Creates empty bounded private terminal proof storage. Existing payload retention, scheduler, and archive GC remain unchanged."
]
},
{
"file": "0054_mcp_geoip_retirement.sql",
"sha256": "680390f98076e3047dbb5a34eab417b253853703932d16c17e4fa3a11060accc",
"transaction": "self-managed-single-transaction",
"selfMarker": true,
"opaqueBodies": 1,
"bodyReview": [
"Adds only exact settled GeoIP settings version 1 to the existing immutable ordinary retirement claim guard."
],
"effects": [
"Replaces the ordinary claim guard function without changing settings or retiring existing rows during migration."
]
},
{
"file": "0055_mcp_backup_schedule_retirement.sql",
"sha256": "07e63c1245ea547176d613933a81ad354e734fa64218dadced67ee8f965c01b6",
"transaction": "self-managed-single-transaction",
"selfMarker": true,
"opaqueBodies": 1,
"bodyReview": [
"Adds only exact settled backup schedule settings version 1 to the existing immutable ordinary retirement claim guard."
],
"effects": [
"Replaces the ordinary claim guard function without changing backup settings, metadata or entries during migration."
]
}
]
@@ -16,6 +16,8 @@ const {tickMcpWorkers}=await import("./workers.js")
const {pauseSchedulerJobsAndWait,resumeSchedulerJobs}=await import("../../services/scheduler-running.js")
const {Client}=await import("pg")
const {McpError}=await import("./security.js")
const {touchGeoipRunMeta}=await import("../../services/geoip-settings.js")
const {touchBackupScheduleRunMeta}=await import("../../services/backup-service.js")
const errorIs=(code:string,status:number)=>(error:unknown)=>error instanceof McpError&&error.code===code&&error.statusCode===status
try {
@@ -25,19 +27,20 @@ try {
const serverId=Number((await pool.query("INSERT INTO servers(name,host,enabled) VALUES('owned logs retention','127.0.0.1',false) RETURNING id")).rows[0].id)
async function access(){
const actor={id:randomUUID(),email:"[email protected]",name:"Owned settings retention",apps:["mm"],
permissions:["mm:settings:admin","mm:mcp:approve","mm:traffic:write","mm:uptime:write","mm:servers:write","mm:network:write"],
permissions:["mm:settings:admin","mm:mcp:approve","mm:traffic:write","mm:uptime:write","mm:servers:write","mm:network:write","mm:backups:write"],
issuedAt:Date.now()-1000,expiresAt:Date.now()+300000}
const issued=await createMcpConnection(mcpConnectionCreateSchema.parse({name:actor.name,mode:"approval",
permissions:actor.permissions,expiresAt:new Date(actor.expiresAt).toISOString(),
scope:{serverIds:[serverId],operationIds:["logs.settings.update","traffic.settings.update","uptime.settings.update",
"servers.api-ping.settings.update","internet-path.settings.update","operations.read","changes.apply"],allowGlobal:true},
limits:{callsPerMinute:100,mutationsPerHour:10}}),actor)
"servers.api-ping.settings.update","internet-path.settings.update","geoip.settings.update","backups.schedule.update","operations.read","changes.apply"],allowGlobal:true},
limits:{callsPerMinute:100,mutationsPerHour:20}}),actor)
return {actor,principal:await authenticateMcpToken(issued.token)}
}
const {actor,principal}=await access(),foreign=(await access()).principal
async function operation(retentionDays:number,ageMs:number,tool="mm_logs_settings_update_plan",
settings:Record<string,unknown>={retentionDays}){
const plan=await invokeMcpTool(principal,tool,{settings}) as {planId:string;state:string}
if(tool==="mm_geoip_settings_update_plan"||tool==="mm_backup_schedule_update_plan")assert.equal(plan.state,"awaiting-approval")
if(plan.state==="awaiting-approval")await approvePlan(plan.planId,actor,true)
const key=randomUUID(),applied=await applyPlan(principal,plan.planId,key)
let terminal
@@ -141,8 +144,83 @@ try {
await assert.rejects(applyPlan(principal,record.planId,record.key),errorIs("OPERATION_HISTORY_EXPIRED",410))
await assert.rejects(invokeMcpTool(foreign,"mm_operation_get",{operationId:record.id}),errorIs("NOT_FOUND",404))
}
// Keep the owned fixture offline even if a product scheduler refresh occurs.
await pool.query("UPDATE geoip_settings SET enabled=false WHERE id=1")
await touchGeoipRunMeta({lastCheckAt:new Date().toISOString(),lastError:"owned GeoIP metadata",etags:{"owned-test":"unchanged"}})
const geoip=await operation(0,age+60000,"mm_geoip_settings_update_plan",{enabled:false,updateIntervalSec:86400})
const geoipState=async()=>(await pool.query("SELECT to_jsonb(g) AS row FROM geoip_settings g WHERE id=1")).rows[0].row
const geoipBefore=await geoipState()
assert.equal(geoipBefore.enabled,false);assert.equal(geoipBefore.update_interval_sec,86400)
assert.equal(geoipBefore.last_error,"owned GeoIP metadata");assert.deepEqual(geoipBefore.etags_json,{"owned-test":"unchanged"})
const {rows:[geoipStep]}=await pool.query("SELECT result FROM mcp_operation_steps WHERE operation_id=$1",[geoip.id])
const assertGeoipUnclaimed=async()=>{
assert.equal((await pool.query("SELECT 1 FROM mcp_ordinary_retirement_claims WHERE operation_id=$1",[geoip.id])).rows.length,0)
assert.equal((await pool.query("SELECT 1 FROM mcp_operations WHERE id=$1",[geoip.id])).rows.length,1)
assert.deepEqual(await geoipState(),geoipBefore)
}
await pool.query("UPDATE mcp_operation_steps SET result=jsonb_set(result,'{settings,updateIntervalSec}','172800') WHERE operation_id=$1",[geoip.id])
await tickMcpWorkers();await tickMcpWorkers();await assertGeoipUnclaimed()
await pool.query("UPDATE mcp_operation_steps SET result=$2 WHERE operation_id=$1",[geoip.id,geoipStep.result])
const geoipHold=randomUUID()
await pool.query("INSERT INTO mcp_drift(id,operation_id,expected,observed) VALUES($1,$2,'{}','{}')",[geoipHold,geoip.id])
await tickMcpWorkers();await assertGeoipUnclaimed()
await pool.query("DELETE FROM mcp_drift WHERE id=$1",[geoipHold])
await tickMcpWorkers();await tickMcpWorkers()
const {rows:[geoipTombstone]}=await pool.query(`SELECT t.action_id,t.target_global,t.target_server_ids::text[] AS server_ids,
NOT EXISTS(SELECT 1 FROM mcp_operations WHERE id=$1) AS operation_gone,
NOT EXISTS(SELECT 1 FROM mcp_plans WHERE id=$2) AS plan_gone,
NOT EXISTS(SELECT 1 FROM mcp_operation_steps WHERE operation_id=$1) AS step_gone,
NOT EXISTS(SELECT 1 FROM mcp_operation_checkpoints WHERE operation_id=$1) AS checkpoint_gone
FROM mcp_operation_replay_tombstones t WHERE t.operation_id=$1 AND t.plan_id=$2`,[geoip.id,geoip.planId])
assert.deepEqual(geoipTombstone,{action_id:"geoip.settings.update",target_global:true,server_ids:[String(serverId)],
operation_gone:true,plan_gone:true,step_gone:true,checkpoint_gone:true})
await assert.rejects(applyPlan(principal,geoip.planId,geoip.key),errorIs("OPERATION_HISTORY_EXPIRED",410))
await assert.rejects(applyPlan(principal,randomUUID(),geoip.key),errorIs("STATE_CONFLICT",409))
await assert.rejects(invokeMcpTool(foreign,"mm_operation_get",{operationId:geoip.id}),errorIs("NOT_FOUND",404))
assert.deepEqual(await geoipState(),geoipBefore)
await pool.query("UPDATE backup_schedule_settings SET enabled=false,server_ids_json='[]' WHERE id=1")
await touchBackupScheduleRunMeta({lastRunAt:new Date().toISOString(),lastDurationMs:123,lastError:"owned backup scheduler metadata"})
await pool.query(`INSERT INTO backup_entries(id,server_id,server_name,filename,size_bytes,kind,notes,created_at)
VALUES($1,$2,'owned fixture','owned-retention.rsc',123,'auto','keep',clock_timestamp())`,[randomUUID(),serverId])
const backupState=async()=>(await pool.query(`SELECT to_jsonb(s) AS schedule,
(SELECT jsonb_agg(to_jsonb(b) ORDER BY id) FROM backup_entries b) AS entries
FROM backup_schedule_settings s WHERE id=1`)).rows[0]
const backupOutside=await operation(0,age+120000,"mm_backup_schedule_update_plan",{enabled:false,serverIds:[String(serverId+1000)]})
const outsideState=await backupState()
await tickMcpWorkers();await tickMcpWorkers()
assert.equal((await pool.query("SELECT 1 FROM mcp_ordinary_retirement_claims WHERE operation_id=$1",[backupOutside.id])).rows.length,0)
assert.equal((await pool.query("SELECT 1 FROM mcp_operations WHERE id=$1",[backupOutside.id])).rows.length,1)
assert.deepEqual(await backupState(),outsideState)
await pool.query("UPDATE backup_schedule_settings SET server_ids_json='[]' WHERE id=1")
const backup=await operation(0,age+60000,"mm_backup_schedule_update_plan",{enabled:false,keepCount:9,serverIds:[String(serverId)]})
const backupBefore=await backupState()
assert.equal(backupBefore.schedule.enabled,false);assert.equal(backupBefore.schedule.keep_count,9)
assert.deepEqual(backupBefore.schedule.server_ids_json,[String(serverId)])
assert.equal(backupBefore.schedule.last_duration_ms,123);assert.equal(backupBefore.schedule.last_error,"owned backup scheduler metadata")
assert.deepEqual(backupBefore.entries,outsideState.entries)
const {rows:[backupStep]}=await pool.query("SELECT result FROM mcp_operation_steps WHERE operation_id=$1",[backup.id])
await pool.query("UPDATE mcp_operation_steps SET result=jsonb_set(result,'{keepCount}','10') WHERE operation_id=$1",[backup.id])
await tickMcpWorkers();await tickMcpWorkers()
assert.equal((await pool.query("SELECT 1 FROM mcp_ordinary_retirement_claims WHERE operation_id=$1",[backup.id])).rows.length,0)
assert.deepEqual(await backupState(),backupBefore)
await pool.query("UPDATE mcp_operation_steps SET result=$2 WHERE operation_id=$1",[backup.id,backupStep.result])
await tickMcpWorkers();await tickMcpWorkers()
const {rows:[backupTombstone]}=await pool.query(`SELECT t.action_id,t.target_global,t.target_server_ids::text[] AS server_ids,
NOT EXISTS(SELECT 1 FROM mcp_operations WHERE id=$1) AS operation_gone,
NOT EXISTS(SELECT 1 FROM mcp_plans WHERE id=$2) AS plan_gone,
NOT EXISTS(SELECT 1 FROM mcp_operation_steps WHERE operation_id=$1) AS step_gone,
NOT EXISTS(SELECT 1 FROM mcp_operation_checkpoints WHERE operation_id=$1) AS checkpoint_gone
FROM mcp_operation_replay_tombstones t WHERE t.operation_id=$1 AND t.plan_id=$2`,[backup.id,backup.planId])
assert.deepEqual(backupTombstone,{action_id:"backups.schedule.update",target_global:true,server_ids:[String(serverId)],
operation_gone:true,plan_gone:true,step_gone:true,checkpoint_gone:true})
await assert.rejects(applyPlan(principal,backup.planId,backup.key),errorIs("OPERATION_HISTORY_EXPIRED",410))
await assert.rejects(applyPlan(principal,randomUUID(),backup.key),errorIs("STATE_CONFLICT",409))
await assert.rejects(invokeMcpTool(foreign,"mm_operation_get",{operationId:backup.id}),errorIs("NOT_FOUND",404))
assert.deepEqual(await backupState(),backupBefore)
await pool.query("UPDATE mcp_connections SET policy=jsonb_set(policy,'{scope,serverIds}','[]') WHERE id=$1",[principal.connectionId])
await assert.rejects(applyPlan(principal,due.planId,due.key),errorIs("NOT_FOUND",404))
for(const {record} of telemetry)await assert.rejects(applyPlan(principal,record.planId,record.key),errorIs("NOT_FOUND",404))
console.log("Owned settings operations: five exact families, authenticated apply, physical 180-day purge, preserved settings and scoped replay")
await assert.rejects(applyPlan(principal,geoip.planId,geoip.key),errorIs("NOT_FOUND",404))
await assert.rejects(applyPlan(principal,backup.planId,backup.key),errorIs("NOT_FOUND",404))
console.log("Owned settings operations: seven exact families, authenticated apply, physical 180-day purge, preserved settings and scoped replay")
}finally{resumeSchedulerJobs();await closePool()}
@@ -1,5 +1,7 @@
import { randomUUID } from "node:crypto"
import { logSettingsPatchSchema, logSettingsSchema } from "@mmapp/contracts"
import { geoipSettingsDtoSchema, geoipSettingsPatchSchema } from "@mmapp/contracts/geoip"
import { backupScheduleSettingsDtoSchema, putBackupScheduleSettingsSchema } from "@mmapp/contracts/backups"
import { z } from "zod"
import { connectProbeStorage } from "../../services/server-probe-storage-connection.js"
import type { ProbeStorageSql } from "../../services/server-probe-journal-maintenance.js"
@@ -15,6 +17,10 @@ const ids = z.array(positive).max(1000)
const retention = z.number().int().min(1).max(365)
const bool = z.boolean()
const interval = (min:number,max:number)=>z.number().int().min(min).max(max)
const backupServerIds=z.array(z.string().regex(/^[1-9]\d{0,15}$/).refine(value=>Number.isSafeInteger(Number(value))))
.max(1000).refine(values=>new Set(values).size===values.length)
const backupSchedule=backupScheduleSettingsDtoSchema.extend({serverIds:backupServerIds}).strict()
const backupBefore=z.object({value:backupSchedule,serverIds:ids}).strict()
// Drizzle's string-mode timestamptz keeps PostgreSQL's space and hour-only
// offset. Accept that exact wire form as well as an ISO timestamptz.
const timestamp = z.string().regex(/^\d{4}-\d\d-\d\d[ T]\d\d:\d\d:\d\d(?:\.\d{1,6})?(?:Z|[+-]\d\d(?::\d\d)?)$/)
@@ -37,6 +43,18 @@ const families={
speedEnabled:bool,intervalSec:interval(5,86400),probeIntervalSec:interval(5,86400),speedIntervalSec:interval(10,86400),retentionDays:retention}),true,true),
"servers.api-ping.settings.update":telemetryFamily(2,"servers-api-ping-settings",z.object({enabled:bool,intervalSec:interval(10,86400)}),false,true),
"internet-path.settings.update":telemetryFamily(1,"internet-path-settings",z.object({enabled:bool,intervalSec:interval(30,86400),retentionDays:retention}),true,false),
"geoip.settings.update":{
version:1,domain:"geoip-settings",
patch:geoipSettingsPatchSchema.strict().refine(value=>Object.keys(value).length>0),
before:z.object({value:z.object({enabled:bool,updateIntervalSec:positive}).strict(),serverIds:ids}).strict(),
effect:z.object({settings:geoipSettingsDtoSchema.strict(),verificationScope:z.literal("configuration-persistence")})
.strict().transform(result=>result.settings),
},
"backups.schedule.update":{
version:1,domain:"backup-schedule",
patch:putBackupScheduleSettingsSchema.extend({serverIds:backupServerIds.optional()}).strict().refine(value=>Object.keys(value).length>0),
before:backupBefore,effect:backupSchedule,
},
} as const
const planEvidence = z.object({
target:z.object({scope:z.literal("global")}).strict(),
@@ -64,7 +82,7 @@ const ELIGIBLE = `FROM public.mcp_operations o
JOIN public.mcp_operation_checkpoints k ON k.operation_id=o.id AND k.step_index=0
WHERE (p.operation_id,p.operation_version) IN (
('logs.settings.update',1),('traffic.settings.update',2),('uptime.settings.update',2),
('servers.api-ping.settings.update',2),('internet-path.settings.update',1)
('servers.api-ping.settings.update',2),('internet-path.settings.update',1),('geoip.settings.update',1),('backups.schedule.update',1)
) AND p.state='completed'
AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()-interval '180 days'
@@ -128,7 +146,7 @@ async function prepare(sql:ProbeStorageSql, operationId:string, connectionId:str
requireThat(fingerprint(step.intended_state)===fingerprint(redact(service)))
const patch=object.parse(family.patch.parse(service.input.settings))
const effect=object.parse(family.effect.parse(step.result))
requireThat(Object.entries(patch).every(([key,value])=>effect[key]===value))
requireThat(Object.entries(patch).every(([key,value])=>fingerprint(effect[key])===fingerprint(value)))
requireThat(fingerprint(operation.result)===fingerprint({confirmedWrites:1,verificationScope:"configuration"}))
for(const value of [checkpoint.before_sealed,checkpoint.after_sealed]){
requireThat(typeof value==="string"&&value.length<=1024&&fingerprint(await open({sealed:value}))===fingerprint([]))
@@ -145,6 +163,10 @@ async function prepare(sql:ProbeStorageSql, operationId:string, connectionId:str
&&domainObjects.length===1&&domainObjects[0].kind===family.domain&&domainObjects[0].id==="1"&&domainObjects[0].serverId===undefined)
requireThat("serverIds" in before&&fingerprint(before.serverIds)===fingerprint(serverIds))
}
if(action==="backups.schedule.update"){
const scope=new Set(serverIds.map(String))
requireThat([...backupBefore.parse(before).value.serverIds,...backupSchedule.parse(effect).serverIds].every(id=>scope.has(id)))
}
const signer=await privateFingerprinterExisting("mcp-ordinary-retirement-source-v1")
return {planId:uuid.parse(plan.id),action,serverIds,key:z.string().min(1).max(200).parse(operation.idempotency_key),proof:signer(fingerprint(raw))}
}
+95 -6
View File
@@ -43,14 +43,43 @@ test("new unknown attempts block older successful data independently for each me
test("failover requires three distinct failures and a freshly verified eligible standby", () => {
const ranking = rankRoutingCandidates(profile, [...a, sample("b1", 1, 4, "wan-b"), sample("b2", 4, 3, "exit-b")], now)
let state: ControllerState = {}
const health = (id: string, status: PathHealth["status"] = "failed"): PathHealth[] => [{ id, candidateId: "a", status, observedAt: at }, { id: "reserve", candidateId: "b", status: "passed", observedAt: at }]
const health = (id: string, time = now, status: PathHealth["status"] = "failed"): PathHealth[] => {
const observedAt = new Date(time).toISOString()
return [{ id, candidateId: "a", status, observedAt }, { id: "reserve", candidateId: "b", status: "passed", observedAt }]
}
for (let i = 0; i < 5; i++) { const next = decideRoutingSwitch(profile, state, ranking, "a", now, health("one")); state = next.state; assert.equal(next.switchTo, null) }
assert.equal(state.failureCount, 1)
const second = decideRoutingSwitch(profile, state, ranking, "a", now, health("two")); assert.equal(second.switchTo, null)
assert.equal(decideRoutingSwitch(profile, second.state, ranking, "a", now, health("three")).switchTo, "b")
assert.equal(decideRoutingSwitch(profile, second.state, ranking, "a", now, health("three", "unknown")).switchTo, null)
assert.equal(decideRoutingSwitch(profile, { ...second.state, switches: [at, at, at] }, ranking, "a", now, health("three")).switchTo, null)
assert.equal(decideRoutingSwitch(profile, second.state, ranking, "a", now + 100000, health("three")).switchTo, null)
const secondAt = now + 60000, thirdAt = now + 120000
const second = decideRoutingSwitch(profile, state, ranking, "a", secondAt, health("two", secondAt)); assert.equal(second.switchTo, null)
assert.equal(decideRoutingSwitch(profile, second.state, ranking, "a", thirdAt, health("three", thirdAt)).switchTo, "b")
const unknown = decideRoutingSwitch(profile, second.state, ranking, "a", thirdAt, health("three", thirdAt, "unknown"))
assert.equal(unknown.switchTo, null)
assert.equal(unknown.state.failureObservedAt, undefined)
assert.equal(decideRoutingSwitch(profile, { ...second.state, switches: [at, at, at] }, ranking, "a", thirdAt, health("three", thirdAt)).switchTo, null)
assert.equal(decideRoutingSwitch(profile, second.state, ranking, "a", thirdAt + 100000, health("three", thirdAt)).switchTo, null)
})
test("failover starts a fresh series after downtime or legacy state and ignores reordered failures", () => {
const ranking = rankRoutingCandidates(profile, [...a, sample("b1", 1, 4, "wan-b"), sample("b2", 4, 3, "exit-b")], now)
const last = now + 60000
const persisted: ControllerState = JSON.parse(JSON.stringify({ failureCount: 2, failureKey: "two",
failureObservedAt: new Date(last).toISOString(), failingCandidateId: "a" }))
const check = (state: ControllerState, id: string, time: number, clock = time) => {
const observedAt = new Date(time).toISOString()
return decideRoutingSwitch(profile, state, ranking, "a", clock,
[{ id, candidateId: "a", status: "failed", observedAt }, { id: "reserve", candidateId: "b", status: "passed", observedAt }])
}
for (const result of [check(persisted, "two", last + 60000), check(persisted, "older", last - 1000, last),
check(persisted, "clock-rollback", last - 1000), check(persisted, "equal-time", last)]) {
assert.equal(result.switchTo, null)
assert.equal(result.state.failureCount, 2)
assert.equal(result.state.failureObservedAt, persisted.failureObservedAt)
}
for (const [state, time] of [[persisted, now + 4 * 3600000], [{ ...persisted, failureObservedAt: undefined }, last + 60000]] as const) {
const result = check(state, "next", time)
assert.equal(result.switchTo, null)
assert.equal(result.state.failureCount, 1)
assert.equal(result.state.failureObservedAt, new Date(time).toISOString())
}
})
test("routing combines RTT, independent losses and bottleneck speed, never claims an observed path", () => {
const ranked = rankRoutingCandidates(profile, a, now).find(row => row.candidate.id === "a")!
@@ -92,3 +121,63 @@ test("controller does not count polling the same sample as consecutive confirmat
assert.equal(decideRoutingSwitch(profile, { ...state, switches: [at, at, at] }, ranks, "a", now).switchTo, null)
assert.equal(decideRoutingSwitch(profile, state, ranks, null, now).switchTo, null)
})
test("hysteresis retains pending confirmation without counting sub-threshold observations or releasing cooldown", () => {
const ranked = rankRoutingCandidates(profile, [...a, sample("b1", 1, 4, "wan-b"), sample("b2", 4, 3, "exit-b")], now)
const compare = (state: ControllerState, gain: number, observation: string, time = now) => {
const ranks = ["b", "a"].map(id => ({ ...ranked.find(row => row.candidate.id === id)!,
score: id === "a" ? 50 : 50 + gain, observationIds: [id + observation] }))
return decideRoutingSwitch(profile, state, ranks, "a", time)
}
const initial = compare({}, 15, "1").state
assert.equal(initial.consecutive, 1)
for (const [gain, observation] of [[6, "2"], [5, "3"]] as const) {
const result = compare(initial, gain, observation)
assert.equal(result.switchTo, null)
assert.deepEqual(result.state, initial)
}
const reset = compare(initial, 4, "4")
assert.equal(reset.state.pendingCandidateId, undefined)
assert.equal(reset.state.consecutive, 0)
const second = compare(initial, 15, "5")
assert.equal(second.switchTo, null)
const third = compare({ ...second.state, lastSwitchAt: at }, 15, "6")
assert.equal(third.state.consecutive, 3)
assert.equal(third.switchTo, null)
const afterCooldown = now + profile.cooldownSec * 1000
assert.equal(compare(third.state, 6, "7", afterCooldown).switchTo, null)
assert.equal(compare(third.state, 15, "8", afterCooldown).switchTo, "b")
})
test("a different pending candidate cannot borrow confirmations from a shared observation key", () => {
const ranks = rankRoutingCandidates(profile, [...a, sample("b1", 1, 4, "wan-b"), sample("b2", 4, 3, "exit-b")], now)
.map(row => ({ ...row, score: row.candidate.id === "a" ? 50 : 75 }))
.sort((left, right) => right.score - left.score)
const observationKey = ranks.flatMap(row => row.observationIds).sort().join(":")
const persisted: ControllerState = { pendingCandidateId: "previous-candidate", consecutive: 3, observationKey }
const selected = decideRoutingSwitch(profile, persisted, ranks, "a", now)
assert.equal(selected.switchTo, null)
assert.equal(selected.state.pendingCandidateId, "b")
assert.equal(selected.state.consecutive, 0)
assert.equal(selected.state.observationKey, observationKey)
const replay = decideRoutingSwitch(profile, selected.state, ranks, "a", now)
assert.equal(replay.switchTo, null)
assert.equal(replay.state.consecutive, 0)
const freshRanks = [{ ...ranks[0], observationIds: ["new-b-1", "new-b-2"] }, ranks[1]]
const fresh = decideRoutingSwitch(profile, replay.state, freshRanks, "a", now)
assert.equal(fresh.state.consecutive, 1)
assert.equal(fresh.switchTo, null)
const belowFloor = [{ ...freshRanks[0], score: 54 }, freshRanks[1]]
const reset = decideRoutingSwitch(profile, fresh.state, belowFloor, "a", now)
assert.equal(reset.state.pendingCandidateId, undefined)
assert.equal(reset.state.consecutive, 0)
const replayAfterReset = decideRoutingSwitch(profile, reset.state, freshRanks, "a", now)
assert.equal(replayAfterReset.switchTo, null)
assert.equal(replayAfterReset.state.consecutive, 0)
})
+20 -9
View File
@@ -51,36 +51,47 @@ export function rankRoutingCandidates(profile: RouteAiProfile, observations: Rou
}
export interface PathHealth { id: string; candidateId: string; status: "passed" | "failed" | "unknown"; observedAt: string }
export interface ControllerState { pendingCandidateId?: string; consecutive?: number; observationKey?: string; lastSwitchAt?: string; switches?: string[]; operationId?: string; pausedReason?: string; failureCount?: number; failureKey?: string; failingCandidateId?: string }
export interface ControllerState { pendingCandidateId?: string; consecutive?: number; observationKey?: string; lastSwitchAt?: string; switches?: string[]; operationId?: string; pausedReason?: string; failureCount?: number; failureKey?: string; failureObservedAt?: string; failingCandidateId?: string }
const pathHealthFreshnessSec = 90
export function decideRoutingSwitch(profile: RouteAiProfile, state: ControllerState, ranking: ReturnType<typeof rankRoutingCandidates>, currentCandidateId: string | null, now = Date.now(), health: PathHealth[] = [], chargedSwitchesPerHour?: number) {
const next: ControllerState = { ...state, switches: (state.switches ?? []).filter(at => now - Date.parse(at) < 3600000) }
if (chargedSwitchesPerHour !== undefined && (!Number.isSafeInteger(chargedSwitchesPerHour) || chargedSwitchesPerHour < next.switches!.length || chargedSwitchesPerHour > 4096)) throw Error("Invalid common routing charge count")
const switchCount = chargedSwitchesPerHour ?? next.switches!.length
const best = ranking.find(candidate => candidate.eligible), current = ranking.find(candidate => candidate.candidate.id === currentCandidateId)
const currentHealth = health.find(value => value.candidateId === currentCandidateId && fresh(value.observedAt, 90, now))
const currentHealth = health.find(value => value.candidateId === currentCandidateId && fresh(value.observedAt, pathHealthFreshnessSec, now))
if (currentHealth?.status === "failed" && currentCandidateId && !state.pausedReason) {
if (state.failureKey !== currentHealth.id) {
next.failureCount = state.failingCandidateId === currentCandidateId ? (state.failureCount ?? 0) + 1 : 1
next.failureKey = currentHealth.id; next.failingCandidateId = currentCandidateId
const observed = Date.parse(currentHealth.observedAt), previous = Date.parse(state.failureObservedAt ?? "")
const sameCandidate = state.failingCandidateId === currentCandidateId
if (state.failureKey === currentHealth.id) {
if (!sameCandidate || observed !== previous) return { state: next, switchTo: null, reason: "Ожидание новой проверки отказа" }
} else {
if (observed <= previous) return { state: next, switchTo: null, reason: "Ожидание новой проверки отказа" }
const continues = sameCandidate && fresh(state.failureObservedAt ?? null, pathHealthFreshnessSec, observed)
next.failureCount = continues ? (state.failureCount ?? 0) + 1 : 1
next.failureKey = currentHealth.id; next.failureObservedAt = currentHealth.observedAt; next.failingCandidateId = currentCandidateId
}
const reserve = ranking.find(value => value.eligible && value.candidate.id !== currentCandidateId && health.some(check => check.candidateId === value.candidate.id && check.status === "passed" && fresh(check.observedAt, 90, now)))
const reserve = ranking.find(value => value.eligible && value.candidate.id !== currentCandidateId && health.some(check => check.candidateId === value.candidate.id && check.status === "passed" && fresh(check.observedAt, pathHealthFreshnessSec, now)))
if ((next.failureCount ?? 0) < 3 || !reserve) return { state: next, switchTo: null, reason: "Failover ожидает три отдельных отказа и подтверждённый резерв" }
if (switchCount >= profile.maxSwitchesPerHour) return { state: { ...next, pausedReason: "Лимит переключений" }, switchTo: null, reason: "Исчерпан часовой бюджет, включая failover" }
if (next.lastSwitchAt && now - Date.parse(next.lastSwitchAt) < profile.cooldownSec * 1000) return { state: next, switchTo: null, reason: "Failover ожидает окончания cooldown" }
return { state: { ...next, failureCount: 0 }, switchTo: reserve.candidate.id, reason: "Failover: три подтверждённых отказа и проверенный резерв" }
}
next.failureCount = 0; next.failureKey = undefined; next.failingCandidateId = undefined
next.failureCount = 0; next.failureKey = undefined; next.failureObservedAt = undefined; next.failingCandidateId = undefined
if (state.pausedReason || !currentCandidateId || !best || best.candidate.id === currentCandidateId) return { state: { ...next, pendingCandidateId: undefined, consecutive: 0 }, switchTo: null, reason: "Нет подтверждённого улучшения или текущего пути" }
if (!current?.eligible || current.score === null) return { state: next, switchTo: null, reason: "Состояние текущего пути неизвестно; failover требует отдельных подтверждённых проверок отказа" }
const gain = best.score! - current.score
if (gain < Math.max(0, profile.switchThreshold - profile.hysteresis)) return { state: { ...next, consecutive: 0, pendingCandidateId: undefined }, switchTo: null, reason: "Сработал гистерезис" }
if (gain < profile.switchThreshold && next.pendingCandidateId !== best.candidate.id) return { state: next, switchTo: null, reason: "Порог улучшения не достигнут" }
if (gain < profile.switchThreshold) return { state: next, switchTo: null, reason: "Порог улучшения не достигнут" }
const observationKey = [...best.observationIds, ...current.observationIds].sort().join(":")
if (next.pendingCandidateId !== undefined && next.pendingCandidateId !== best.candidate.id) {
next.pendingCandidateId = best.candidate.id
next.consecutive = 0
}
if (observationKey !== next.observationKey) {
next.consecutive = next.pendingCandidateId === best.candidate.id ? (next.consecutive ?? 0) + 1 : 1
next.pendingCandidateId = best.candidate.id; next.observationKey = observationKey
}
if ((next.consecutive ?? 0) < profile.confirmationSamples) return { state: next, switchTo: null, reason: "Ожидание новых измерений" }
if (next.pendingCandidateId !== best.candidate.id || (next.consecutive ?? 0) < profile.confirmationSamples) return { state: next, switchTo: null, reason: "Ожидание новых измерений" }
if (next.lastSwitchAt && now - Date.parse(next.lastSwitchAt) < profile.cooldownSec * 1000) return { state: next, switchTo: null, reason: "Cooldown" }
if (switchCount >= profile.maxSwitchesPerHour) return { state: { ...next, pausedReason: "Лимит переключений" }, switchTo: null, reason: "Исчерпан часовой бюджет переключений" }
return { state: next, switchTo: best.candidate.id, reason: "Улучшение подтверждено свежими наблюдениями" }
@@ -77,7 +77,7 @@ async function main() {
}
try {
await applySqlMigrations(pool)
assert.equal(MIGRATION_MANIFEST.at(-1)?.file, "0053_server_probe_terminal_proofs.sql")
assert.equal(MIGRATION_MANIFEST.at(-1)?.file, "0055_mcp_backup_schedule_retirement.sql")
assert.equal(MIGRATION_MANIFEST.find(pin=>pin.file==="0041_server_poll_terminal_archives.sql")?.sha256,"fbb8935ce977cfc507801f01beb8f303d4817ed63c6bdd4c05799768dfaac9d4")
assert.equal((await pool.query("SELECT count(*)::int AS n FROM schema_migrations")).rows[0].n, MIGRATION_MANIFEST.length)
assert.equal((await pool.query("SELECT count(*)::int AS n FROM server_poll_storage_admissions")).rows[0].n, 0)