198 lines
17 KiB
TypeScript
198 lines
17 KiB
TypeScript
import { MIGRATION_MANIFEST } from "../db/migration-manifest.js"
|
|
/** Finite owned acceptance of the REAL MCP settings -> automatic scheduler
|
|
* handoff. Native responses come only from localhost HTTP, never fabricated SQL. */
|
|
import assert from "node:assert/strict"
|
|
import { createServer } from "node:http"
|
|
import { once } from "node:events"
|
|
import { randomUUID } from "node:crypto"
|
|
import { setTimeout as delay } from "node:timers/promises"
|
|
import { isDeepStrictEqual } from "node:util"
|
|
import { assertOwnedBandwidthSql } from "./bandwidth-admission-process.sql-helper.js"
|
|
|
|
export async function acceptAutomaticSchedulerSql(ownServerIds: number[]) {
|
|
assertOwnedBandwidthSql()
|
|
assert.equal(ownServerIds.length, 2); assert.equal(new Set(ownServerIds).size, 2)
|
|
assert(ownServerIds.every(id => Number.isSafeInteger(id) && id > 0))
|
|
const ids = [...ownServerIds], { pool } = await import("../db/index.js")
|
|
const { initializeAutomaticSchedulerRoot } = await import("./scheduler-automatic-root.js")
|
|
const { stopSchedulerForDatabaseOperation } = await import("./scheduler.js")
|
|
const { resumeSchedulerJobs, isSchedulerJobsPaused } = await import("./scheduler-running.js")
|
|
const { databaseBackgroundWork } = await import("./database-background-work.js")
|
|
const { assertNoOperationConfigurationLockContext } = await import("./operation-config-lock-lease.js")
|
|
const { assertNoConfigurationLockContext } = await import("./config-mutation-lock.js")
|
|
const { assertNoVerifiedConfigurationContext } = await import("./config-mutation-policy.js")
|
|
const { MikrotikClient } = await import("./mikrotik.js")
|
|
const { createMcpConnection, authenticateMcpToken } = await import("../modules/mcp/store.js")
|
|
const { invokeMcpTool, applyPlan, approvePlan } = await import("../modules/mcp/runtime.js")
|
|
const { findAction } = await import("../modules/mcp/actions.js")
|
|
const { mcpConnectionCreateSchema } = await import("@mmapp/contracts/mcp")
|
|
const { mcpStateDirectory } = await import("../modules/mcp/security.js")
|
|
const { readFile, lstat, unlink } = await import("node:fs/promises")
|
|
const { join } = await import("node:path")
|
|
// Bootstrap is explicit and occurs before entering ANY operation/verified scope.
|
|
initializeAutomaticSchedulerRoot()
|
|
const wasPaused = isSchedulerJobsPaused()
|
|
assert.equal(Number((await pool.query("SELECT count(*) AS n FROM public.schema_migrations")).rows[0].n), MIGRATION_MANIFEST.length)
|
|
const singletonTables = ["router_log_settings", "traffic_settings", "servers_api_ping_settings", "uptime_settings", "internet_path_settings", "certificate_renew_settings", "backup_schedule_settings", "geoip_settings", "mcp_settings", "alert_engine_cursor"] as const
|
|
const snapshots = new Map<string, Record<string, unknown> | undefined>()
|
|
for (const table of singletonTables) snapshots.set(table, (await pool.query(`SELECT to_jsonb(t) AS row FROM public.${table} t WHERE id=1`)).rows[0]?.row)
|
|
const serverRows = (await pool.query("SELECT to_jsonb(t) AS row FROM public.servers t WHERE id=ANY($1::bigint[]) ORDER BY id", [ids])).rows.map(row => row.row as Record<string, unknown>)
|
|
assert.equal(serverRows.length, 2); assert(serverRows.every(row => row.enabled === false))
|
|
assert.equal((await pool.query("SELECT id FROM public.servers WHERE enabled AND NOT(id=ANY($1::bigint[])) LIMIT 1", [ids])).rows.length, 0)
|
|
assert.equal((await pool.query("SELECT id FROM public.uptime_speed_probes WHERE enabled LIMIT 1")).rows.length, 0)
|
|
assert.equal((await pool.query("SELECT id FROM public.alert_rules LIMIT 1")).rows.length, 0)
|
|
assert.equal((await pool.query("SELECT id FROM public.alert_groups LIMIT 1")).rows.length, 0)
|
|
assert.equal((await pool.query("SELECT id FROM public.alert_outbox LIMIT 1")).rows.length, 0)
|
|
const telegram = (await pool.query("SELECT bot_token,chat_id FROM public.alert_telegram_settings WHERE id=1")).rows[0]
|
|
assert(!telegram?.bot_token && !telegram?.chat_id, "Owned scheduler fixture must have no Telegram delivery target")
|
|
let baselineRuns = new Set<string>(), baselineEvents = new Set<string>(), baselineAudits = new Set<string>(), baselineCaptured = false
|
|
const keySnapshots = new Map<string, Buffer | undefined>(), createdKeys = new Map<string, Buffer>()
|
|
for (const name of ["access-generation", "encryption-key"]) {
|
|
const file = join(mcpStateDirectory(), name)
|
|
try { const stat = await lstat(file); assert(stat.isFile() && !stat.isSymbolicLink() && stat.size === 32); keySnapshots.set(file, await readFile(file)) }
|
|
catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; keySnapshots.set(file, undefined) }
|
|
}
|
|
const probeId = "owned-automatic-speed-" + randomUUID(), protectionId = randomUUID(), connections: string[] = [], checks: string[] = []
|
|
const captureCreatedKeys = async () => {
|
|
for (const [file, previous] of keySnapshots) if (!previous && !createdKeys.has(file)) {
|
|
try { const stat = await lstat(file); assert(stat.isFile() && !stat.isSymbolicLink() && stat.size === 32); createdKeys.set(file, await readFile(file)) }
|
|
catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error }
|
|
}
|
|
}
|
|
const originalBandwidth = MikrotikClient.prototype.bandwidthTest
|
|
let bandwidthPosts = 0, pingPosts = 0, nativeGetRequests = 0, scopeChecks = 0, protocolErrors = 0, currentOperation: string | undefined
|
|
const router = createServer(async (request, response) => {
|
|
response.setHeader("content-type", "application/json")
|
|
try {
|
|
const auth = Buffer.from((request.headers.authorization ?? "").slice(6), "base64").toString()
|
|
assert(ids.some(id => auth === `scheduler-${id}:OWNED_LOCALHOST_PRIVATE`))
|
|
if (request.method === "GET") { nativeGetRequests++; response.end("[]"); return }
|
|
assert.equal(request.method, "POST"); assert.equal(auth, `scheduler-${ids[0]}:OWNED_LOCALHOST_PRIVATE`)
|
|
let text = ""; for await (const chunk of request) { text += String(chunk); assert(Buffer.byteLength(text) <= 65536) }
|
|
const body = JSON.parse(text)
|
|
if (request.url === "/rest/tool/bandwidth-test") {
|
|
assert(body.address === "127.0.0.1" && body.user === "scheduler-" + ids[1] && body.password === "OWNED_LOCALHOST_PRIVATE")
|
|
assert.equal(body.duration, "3s"); assert.equal(body.protocol, "tcp"); assert.equal(body.direction, "both")
|
|
bandwidthPosts++; response.end(JSON.stringify([{ status: "done testing", "tx-total-average": "1000000", "rx-total-average": "2000000" }]))
|
|
} else {
|
|
assert.equal(request.url, "/rest/tool/ping"); assert.equal(body.count, "4")
|
|
pingPosts++; response.end(JSON.stringify([{ sent: "4", received: "4", "packet-loss": "0%", "avg-rtt": "1ms" }]))
|
|
}
|
|
} catch { protocolErrors++; response.destroy() }
|
|
})
|
|
const restoreRow = async (table: string, row: Record<string, unknown> | undefined) => {
|
|
assert([...singletonTables, "servers"].includes(table))
|
|
if (!row) { await pool.query(`DELETE FROM public.${table} WHERE id=1`); return }
|
|
const columns = Object.keys(row).filter(name => name !== "id" && name !== "routing_endpoint_epoch")
|
|
assert(columns.every(name => /^[a-z_][a-z_0-9]*$/.test(name)))
|
|
await pool.query(`UPDATE public.${table} SET (${columns.join(",")})=(SELECT ${columns.join(",")} FROM pg_catalog.jsonb_populate_record(NULL::public.${table},$1::jsonb)) WHERE id=$2`, [JSON.stringify(row), row.id])
|
|
}
|
|
const terminal = async (operationId: string) => {
|
|
for (const until = Date.now() + 60000; Date.now() < until;) {
|
|
const row = (await pool.query("SELECT state FROM public.mcp_operations WHERE id=$1", [operationId])).rows[0]
|
|
assert(row); if (!["pending", "running"].includes(row.state)) { assert.equal(row.state, "applied"); return }
|
|
await delay(20)
|
|
}
|
|
assert.fail("Owned settings operation did not complete")
|
|
}
|
|
const waitSpeed = async (expectedCount: number, status: string) => {
|
|
for (const until = Date.now() + 60000; Date.now() < until;) {
|
|
const rows = (await pool.query("SELECT status,error,ping_error,duration_sec,src_interface,dst_interface FROM public.uptime_speed_test_runs WHERE probe_id=$1 ORDER BY created_at,id", [probeId])).rows
|
|
if (rows.length >= expectedCount) {
|
|
assert.equal(rows.length, expectedCount); const row = rows[expectedCount - 1]
|
|
assert.equal(row.status, status); assert.equal(row.duration_sec, 3); assert.equal(row.src_interface, ""); assert.equal(row.dst_interface, "")
|
|
if (status === "done") assert.equal(row.ping_error, null)
|
|
else assert(typeof row.error === "string" && row.error.length > 0)
|
|
return
|
|
}
|
|
await delay(20)
|
|
}
|
|
assert.fail("Owned automatic speed run did not publish history")
|
|
}
|
|
try {
|
|
await stopSchedulerForDatabaseOperation(); await databaseBackgroundWork.drain()
|
|
baselineRuns = new Set((await pool.query("SELECT id FROM public.scheduler_runs")).rows.map(row => row.id as string))
|
|
baselineEvents = new Set((await pool.query("SELECT id FROM public.events")).rows.map(row => row.id as string))
|
|
baselineAudits = new Set((await pool.query("SELECT id::text AS id FROM public.mcp_audit")).rows.map(row => row.id as string))
|
|
baselineCaptured = true
|
|
router.listen(0, "127.0.0.1"); await once(router, "listening")
|
|
const address = router.address(); assert(address && typeof address !== "string")
|
|
for (const table of singletonTables.filter(table => table !== "mcp_settings" && table !== "alert_engine_cursor")) {
|
|
await pool.query(`INSERT INTO public.${table}(id,enabled) VALUES(1,false) ON CONFLICT(id) DO UPDATE SET enabled=false`)
|
|
}
|
|
await pool.query("UPDATE public.uptime_settings SET resources_enabled=false,ping_enabled=false,speed_enabled=false WHERE id=1")
|
|
await pool.query("UPDATE public.mcp_settings SET enabled=true WHERE id=1")
|
|
await pool.query("UPDATE public.servers SET host='127.0.0.1',port=$2,use_ssl=false,verify_ssl=true,username='scheduler-'||id::text,password='OWNED_LOCALHOST_PRIVATE',enabled=true WHERE id=ANY($1::bigint[])", [ids, address.port])
|
|
await pool.query("INSERT INTO public.uptime_speed_probes(id,src_server_id,dst_server_id,src_interface,dst_interface,protocol,direction,duration_sec,enabled) VALUES($1,$2,$3,'','','tcp','both',3,true)", [probeId, ids[0], ids[1]])
|
|
MikrotikClient.prototype.bandwidthTest = async function (...args) {
|
|
assertNoOperationConfigurationLockContext(); assertNoConfigurationLockContext(); assertNoVerifiedConfigurationContext(); scopeChecks++
|
|
assert(currentOperation)
|
|
assert.equal((await pool.query("SELECT state FROM public.mcp_operations WHERE id=$1", [currentOperation])).rows[0]?.state, "applied")
|
|
return originalBandwidth.apply(this, args)
|
|
}
|
|
const actor = { id: randomUUID(), email: "[email protected]", name: "Owned scheduler handoff", apps: ["mm"],
|
|
permissions: ["mm:uptime:read", "mm:uptime:write", "mm:servers:read", "mm:mcp:approve"], issuedAt: Date.now() - 1000, expiresAt: Date.now() + 300000 }
|
|
const allIds = (await pool.query("SELECT id FROM public.servers ORDER BY id")).rows.map(row => Number(row.id))
|
|
const action = findAction("mm_uptime_settings_update_plan"); assert.equal(action.operation.id, "uptime.settings.update")
|
|
const access = await createMcpConnection(mcpConnectionCreateSchema.parse({ name: "Owned automatic scheduler MCP parent", mode: "approval", expiresAt: new Date(actor.expiresAt).toISOString(),
|
|
permissions: ["mm:uptime:read", "mm:uptime:write", "mm:servers:read"], scope: { serverIds: allIds, operationIds: [action.operation.id], allowGlobal: true }, limits: { callsPerMinute: 120, mutationsPerHour: 1000 } }), actor)
|
|
connections.push(access.connection.id)
|
|
await captureCreatedKeys()
|
|
const principal = await authenticateMcpToken(access.token)
|
|
const planRefresh = async () => {
|
|
const prepared = await invokeMcpTool(principal, "mm_uptime_settings_update_plan", { settings: { resourcesEnabled: false, pingEnabled: false, speedEnabled: true, speedIntervalSec: 3600 } }) as { planId: string; state: string }
|
|
await captureCreatedKeys()
|
|
assert.equal(prepared.state, "awaiting-approval"); await approvePlan(prepared.planId, actor, true)
|
|
const operation = await applyPlan(principal, prepared.planId, randomUUID()); currentOperation = operation.id
|
|
await terminal(operation.id)
|
|
}
|
|
resumeSchedulerJobs()
|
|
await planRefresh(); await waitSpeed(1, "done")
|
|
assert.equal(bandwidthPosts, 1); assert.equal(pingPosts, 1); assert.equal(protocolErrors, 0)
|
|
checks.push("real human-approved global uptime settings operation applies; immediate automatic root runs bandwidth plus after-ping once with original duration/body and no inherited operation/configuration/verified context")
|
|
await stopSchedulerForDatabaseOperation(); await databaseBackgroundWork.drain()
|
|
await pool.query("INSERT INTO public.mcp_protections(id,object_ref,reason,created_by) VALUES($1,$2::jsonb,'owned scheduler no approval inheritance',$3)", [protectionId, JSON.stringify({ kind: "server", id: String(ids[0]), serverId: ids[0], protection: "human-approval-required" }), actor.id])
|
|
await pool.query("UPDATE public.uptime_speed_probes SET last_run_at=NULL WHERE id=$1", [probeId])
|
|
resumeSchedulerJobs(); await planRefresh(); await waitSpeed(2, "error")
|
|
assert.equal(bandwidthPosts, 1); assert.equal(pingPosts, 1); assert.equal(scopeChecks, 2); assert.equal(protocolErrors, 0)
|
|
checks.push("a second approved settings parent does not grant approval to automatic speed work: protected source yields real error history before another bandwidth or ping POST")
|
|
return { checks, cases: 2, bandwidthPosts, afterPingPosts: pingPosts, automaticScopeChecks: scopeChecks, actualLocalhostGets: nativeGetRequests, realChrCalls: 0, productionActivation: false, restoredEndpointEpochPolicy: "new-genuine-epoch" }
|
|
} finally {
|
|
// Never restore/delete fixture state while a real collector still runs.
|
|
await stopSchedulerForDatabaseOperation(); await databaseBackgroundWork.drain()
|
|
try {
|
|
MikrotikClient.prototype.bandwidthTest = originalBandwidth
|
|
router.closeAllConnections(); if (router.listening) await new Promise<void>((resolve, reject) => router.close(error => error ? reject(error) : resolve()))
|
|
await pool.query("DELETE FROM public.mcp_protections WHERE id=$1", [protectionId])
|
|
await pool.query("DELETE FROM public.uptime_speed_test_runs WHERE probe_id=$1", [probeId])
|
|
await pool.query("DELETE FROM public.uptime_speed_probes WHERE id=$1", [probeId])
|
|
if (connections.length) {
|
|
for (const table of ["mcp_operation_checkpoints", "mcp_operation_steps"]) await pool.query(`DELETE FROM public.${table} WHERE operation_id IN(SELECT id FROM public.mcp_operations WHERE connection_id=ANY($1::uuid[]))`, [connections])
|
|
for (const table of ["mcp_operations", "mcp_plans", "mcp_requests", "mcp_audit"]) await pool.query(`DELETE FROM public.${table} WHERE connection_id=ANY($1::uuid[])`, [connections])
|
|
await pool.query("DELETE FROM public.mcp_connections WHERE id=ANY($1::uuid[])", [connections])
|
|
}
|
|
if (baselineCaptured) {
|
|
const ownAudits = (await pool.query("SELECT id::text AS id FROM public.mcp_audit WHERE connection_id IS NULL AND payload->>'serverId'=ANY($1::text[])", [ids.map(String)])).rows.map(row => row.id as string).filter(id => !baselineAudits.has(id))
|
|
if (ownAudits.length) await pool.query("DELETE FROM public.mcp_audit WHERE id=ANY($1::bigint[])", [ownAudits])
|
|
const runIds = (await pool.query("SELECT id FROM public.scheduler_runs")).rows.map(row => row.id as string).filter(id => !baselineRuns.has(id))
|
|
const eventIds = (await pool.query("SELECT id FROM public.events")).rows.map(row => row.id as string).filter(id => !baselineEvents.has(id))
|
|
if (runIds.length) await pool.query("DELETE FROM public.scheduler_runs WHERE id=ANY($1::text[])", [runIds])
|
|
if (eventIds.length) await pool.query("DELETE FROM public.events WHERE id=ANY($1::text[])", [eventIds])
|
|
}
|
|
for (const row of serverRows) await restoreRow("servers", row)
|
|
const restoredServers = (await pool.query("SELECT to_jsonb(t) AS row FROM public.servers t WHERE id=ANY($1::bigint[]) ORDER BY id", [ids])).rows.map(row => row.row as Record<string, unknown>)
|
|
assert.equal(restoredServers.length, serverRows.length)
|
|
for (const [index, restored] of restoredServers.entries()) {
|
|
const { routing_endpoint_epoch: oldEpoch, ...original } = serverRows[index], { routing_endpoint_epoch: freshEpoch, ...current } = restored
|
|
assert(isDeepStrictEqual(current, original), "Owned server configuration restoration mismatch")
|
|
assert(typeof freshEpoch === "string" && freshEpoch !== oldEpoch, "Restoration must retain a new genuine endpoint epoch")
|
|
}
|
|
for (const table of singletonTables) await restoreRow(table, snapshots.get(table))
|
|
for (const [file, previous] of keySnapshots) {
|
|
if (previous) { assert((await readFile(file)).equals(previous), "Original fixture MCP key unexpectedly changed"); previous.fill(0) }
|
|
else { const created = createdKeys.get(file); if (created) { assert((await readFile(file)).equals(created), "Created fixture MCP key unexpectedly changed"); await unlink(file); created.fill(0) } }
|
|
}
|
|
} finally { if (!wasPaused) resumeSchedulerJobs() }
|
|
}
|
|
}
|