336 lines
33 KiB
TypeScript
336 lines
33 KiB
TypeScript
import assert from "node:assert/strict"
|
|
import { randomUUID } from "node:crypto"
|
|
import { readFile, writeFile } from "node:fs/promises"
|
|
import { setTimeout as delay } from "node:timers/promises"
|
|
import { pool, closePool } from "../../db/index.js"
|
|
import { applySqlMigrations } from "../../db/migrate.js"
|
|
import { mcpConnectionCreateSchema } from "@mmapp/contracts/mcp"
|
|
import { createMcpConnection, authenticateMcpToken, updateMcpSettings } from "./store.js"
|
|
import { findAction, listActions } from "./actions.js"
|
|
import { invokeMcpTool, approvePlan, applyPlan } from "./runtime.js"
|
|
import { readRosSnapshot, routerClient } from "./resources.js"
|
|
import { MikrotikClient, MikrotikError } from "../../services/mikrotik.js"
|
|
import { McpError } from "./security.js"
|
|
import type { AuthUser } from "../../lib/permissions.js"
|
|
import { acceptChrOverlay, cleanupChrOverlay } from "./chr-overlay-acceptance.js"
|
|
import { cleanupChrCommunityGatewayResolvers } from "./chr-bgp-community-acceptance.js"
|
|
import { cleanupChrBgpFixture } from "./chr-bgp-acceptance.js"
|
|
import { chrNativeFailure } from "./chr-native-failure.js"
|
|
import { readChrRecoverySource } from "./chr-recovery-source.js"
|
|
import { cleanupChrOspf } from "./chr-ospf-acceptance.js"
|
|
import { chrFixtureVrf, chrOspfForwarding } from "./chr-fixture-context.js"
|
|
import { acceptChrOspfPassive } from "./chr-ospf-passive.js"
|
|
import { acceptChrWireguardInterface } from "./chr-wireguard-interface.js"
|
|
import { acceptChrWireguardPeer } from "./chr-wireguard-peer.js"
|
|
import { acceptChrWireguardInterfaceUpdate } from "./chr-wireguard-interface-update.js"
|
|
|
|
// Explicit lab allowlist: cannot accidentally point this mutating test at production.
|
|
const hosts = (process.env.MMAPP_TEST_CHR_HOSTS ?? "").split(",").map(value => value.trim()).filter(Boolean)
|
|
assert.ok(hosts.length > 0 && hosts.length <= 2 && new Set(hosts).size === hosts.length)
|
|
assert.ok(hosts.every(host => ["194.226.121.150", "104.165.244.7"].includes(host)))
|
|
const username = process.env.MMAPP_TEST_CHR_USER, password = process.env.MMAPP_TEST_CHR_PASSWORD
|
|
assert.ok(username && password, "Explicit lab credentials required")
|
|
const tunnelPorts = process.env.MMAPP_TEST_CHR_TUNNEL_PORTS?.split(",").map(Number)
|
|
if (tunnelPorts) assert.ok(tunnelPorts.length === hosts.length && tunnelPorts.every(port => Number.isInteger(port) && port >= 1024 && port <= 65535))
|
|
const actor: AuthUser = { id: "chr-acceptance-owner", apps: ["mm"], permissions: ["mm:settings:admin", "mm:network:admin", "mm:bgp:admin", "mm:servers:admin", "mm:filters:read"], expiresAt: Date.now() + 3600000 }
|
|
const marker = "mcp-test-" + randomUUID().slice(0, 8)
|
|
const routingAcceptance = process.env.MMAPP_TEST_CHR_ROUTING === "true"
|
|
const overlayAcceptance = process.env.MMAPP_TEST_CHR_OVERLAY === "true"
|
|
const ospfAcceptance = process.env.MMAPP_TEST_CHR_OSPF === "true"
|
|
const ospfSourceAcceptance = process.env.MMAPP_TEST_CHR_OSPF_SOURCES === "true"
|
|
const ospfForwarding = chrOspfForwarding()
|
|
const ospfPassiveOnly = process.env.MMAPP_TEST_CHR_OSPF_PASSIVE_ONLY === "true"
|
|
const wireguardPeerOnly = process.env.MMAPP_TEST_CHR_WG_PEER_ONLY === "true"
|
|
const wireguardPeerUpdate = process.env.MMAPP_TEST_CHR_WG_PEER_UPDATE === "true"
|
|
const wireguardInterfaceDelete = process.env.MMAPP_TEST_CHR_WG_INTERFACE_DELETE === "true"
|
|
const wireguardInterfaceUpdate = process.env.MMAPP_TEST_CHR_WG_INTERFACE_UPDATE === "true"
|
|
assert.ok(!wireguardInterfaceDelete || wireguardPeerOnly && !wireguardPeerUpdate, "WireGuard interface cascade requires its isolated peer fixture mode")
|
|
assert.ok(!wireguardPeerUpdate || wireguardPeerOnly, "WireGuard peer update requires its isolated peer scenario")
|
|
assert.ok(!ospfPassiveOnly || !ospfAcceptance && !overlayAcceptance && !routingAcceptance, "Passive-only mode must be isolated from forwarding scenarios")
|
|
assert.ok(!ospfSourceAcceptance || ospfAcceptance, "Source-bound OSPF acceptance requires OSPF")
|
|
const ecmpAcceptance = process.env.MMAPP_TEST_CHR_ECMP === "true"
|
|
const bgpAcceptance = process.env.MMAPP_TEST_CHR_BGP === "true"
|
|
const bgpExportAcceptance = process.env.MMAPP_TEST_CHR_BGP_EXPORT === "true"
|
|
const bgpForwardAcceptance = process.env.MMAPP_TEST_CHR_BGP_FORWARD === "true"
|
|
const bgpFilterBindingAcceptance = process.env.MMAPP_TEST_CHR_BGP_FILTER_BINDING === "true"
|
|
const bgpCommunityAcceptance = process.env.MMAPP_TEST_CHR_BGP_COMMUNITY === "true"
|
|
assert.ok(!wireguardPeerOnly || hosts.length === 1 && !ospfPassiveOnly && !overlayAcceptance && !ospfAcceptance && !routingAcceptance && !bgpAcceptance && !ecmpAcceptance, "WireGuard peer mode requires one isolated device")
|
|
assert.ok(!wireguardInterfaceUpdate || !wireguardPeerOnly && !wireguardInterfaceDelete && !wireguardPeerUpdate && !ospfPassiveOnly && !overlayAcceptance && !ospfAcceptance && !routingAcceptance && !bgpAcceptance && !ecmpAcceptance, "WireGuard interface update requires its isolated disabled fixture mode")
|
|
assert.ok(!ospfForwarding || ospfSourceAcceptance && !bgpAcceptance && !ecmpAcceptance, "OSPF forwarding requires its isolated source-bound scenario")
|
|
assert.ok(process.env.MMAPP_TEST_CHR_COMMUNITY_GATEWAY_RESOLVER !== "true" || bgpCommunityAcceptance && process.env.MMAPP_TEST_CHR_COMMUNITY_LITERAL_GATEWAY !== "true")
|
|
assert.ok(process.env.MMAPP_TEST_CHR_BGP_BINDING_FAULTS !== "false" || bgpCommunityAcceptance)
|
|
const overlayCleanupOnly = process.env.MMAPP_TEST_CHR_OVERLAY_CLEANUP_ONLY === "true"
|
|
assert.ok(!overlayCleanupOnly || overlayAcceptance && process.env.MMAPP_TEST_CHR_OVERLAY_CLEANUP_MARKER)
|
|
if (bgpFilterBindingAcceptance) actor.permissions.push("mm:filters:write")
|
|
assert.ok(!ospfAcceptance || overlayAcceptance, "OSPF acceptance requires the isolated overlay")
|
|
assert.ok(!ecmpAcceptance || overlayAcceptance, "ECMP acceptance requires the isolated overlay")
|
|
assert.ok(!bgpAcceptance || overlayAcceptance, "BGP acceptance requires the isolated overlay")
|
|
assert.ok(!bgpExportAcceptance || bgpAcceptance, "BGP prefix acceptance requires BGP and the isolated overlay")
|
|
assert.ok(!bgpForwardAcceptance || bgpExportAcceptance, "BGP forwarding probes require prefix acceptance")
|
|
assert.ok(!bgpFilterBindingAcceptance || bgpForwardAcceptance, "BGP filter-binding acceptance requires prefix and packet acceptance")
|
|
assert.ok(!bgpCommunityAcceptance || bgpFilterBindingAcceptance, "Community propagation requires filter-binding and packet acceptance")
|
|
const requestGapText = process.env.MMAPP_TEST_CHR_REQUEST_GAP_MS ?? "0"
|
|
assert.ok(["0", "100", "250", "500"].includes(requestGapText))
|
|
const requestGapMs = Number(requestGapText)
|
|
let pacing = Promise.resolve(), lastRequestAt = 0
|
|
const pace = async () => {
|
|
if (!requestGapMs) return
|
|
const ready = pacing.then(async () => { const wait = requestGapMs - (Date.now() - lastRequestAt); if (wait > 0) await delay(wait); lastRequestAt = Date.now() })
|
|
pacing = ready.catch(() => {}); await ready
|
|
}
|
|
const report: Record<string, unknown> = { startedAt: new Date().toISOString(), marker, tools: listActions().length, nativeRequestGapMs: requestGapMs, checks: [], complete: false, ...(ospfSourceAcceptance ? { optimizerRevision: findAction("mm_ospf_optimizer_plan").operation.version, measurementRevision: findAction("mm_routing_measurement_plan").operation.version, sourceMigration: "0029_mcp_routing_sources" } : {}) }
|
|
const checks = report.checks as Record<string, unknown>[]
|
|
if (wireguardPeerOnly) Object.assign(report, { fixtureSchema: wireguardPeerUpdate ? "wg-peer-update-v1" : "wg-peer-delete-v1", ...(wireguardPeerUpdate ? { businessUpdateRevision: findAction("mm_wireguard_peer_edit_plan").operation.version, nativeUpdateRevision: findAction("mm_wireguard_peer_update_plan").operation.version } : {}), businessDeleteRevision: findAction("mm_wireguard_peer_remove_plan").operation.version, nativeDeleteRevision: findAction("mm_wireguard_peer_delete_plan").operation.version })
|
|
if (wireguardInterfaceDelete) Object.assign(report, { fixtureSchema: "wg-interface-delete-v1", businessInterfaceDeleteRevision: findAction("mm_wireguard_interface_remove_plan").operation.version, nativeInterfaceDeleteRevision: findAction("mm_wireguard_delete_plan").operation.version })
|
|
if (ospfForwarding) Object.assign(report, { ospfForwarding: true, fixtureSchema: "ospf-forward-v1", routeProofOperation: "routing.ospf.route.proof.read", vrfNames: [1, 2].map(id => chrFixtureVrf(marker, id)) })
|
|
const reportUrl = new URL(wireguardInterfaceDelete ? "../../../../docs/mcp-chr-wireguard-interface-delete-acceptance.json" : wireguardPeerUpdate ? "../../../../docs/mcp-chr-wireguard-peer-update-acceptance.json" : wireguardPeerOnly ? "../../../../docs/mcp-chr-wireguard-peer-delete-acceptance.json" : ospfPassiveOnly ? "../../../../docs/mcp-chr-ospf-passive-toggle-acceptance.json" : ospfForwarding ? (overlayCleanupOnly ? "../../../../docs/mcp-chr-ospf-forward-recovery.json" : "../../../../docs/mcp-chr-ospf-forward-acceptance.json") : ospfSourceAcceptance ? (overlayCleanupOnly ? "../../../../docs/mcp-chr-ospf-source-recovery.json" : "../../../../docs/mcp-chr-ospf-source-acceptance.json") : bgpCommunityAcceptance ? "../../../../docs/mcp-chr-bgp-community-acceptance.json" : bgpFilterBindingAcceptance ? "../../../../docs/mcp-chr-bgp-filter-binding-acceptance.json" : bgpForwardAcceptance ? "../../../../docs/mcp-chr-bgp-forward-acceptance.json" : bgpExportAcceptance ? "../../../../docs/mcp-chr-bgp-prefix-acceptance.json" : bgpAcceptance ? "../../../../docs/mcp-chr-bgp-acceptance.json" : ecmpAcceptance ? "../../../../docs/mcp-chr-ecmp-acceptance.json" : ospfAcceptance ? "../../../../docs/mcp-chr-ospf-acceptance.json" : overlayAcceptance ? "../../../../docs/mcp-chr-overlay-acceptance.json" : routingAcceptance ? "../../../../docs/mcp-chr-path-acceptance.json" : "../../../../docs/mcp-chr-acceptance.json", import.meta.url)
|
|
let checkpointQueue = Promise.resolve()
|
|
const wgUpdateReportUrl = new URL(`../../../../docs/mcp-chr-wg-interface-update-${marker.slice(-8)}-acceptance.json`, import.meta.url)
|
|
if (wireguardInterfaceUpdate) Object.assign(report, { fixtureSchema: "wg-interface-update-v2", privateProofDomain: "wireguard-interface-update-v2", businessRevision: findAction("mm_wireguard_interface_edit_plan").operation.version, nativeAliasRevision: findAction("mm_wireguard_update_plan").operation.version, forwardingVerified: false })
|
|
const checkpoint = () => {
|
|
const payload = JSON.stringify(report, null, 2) + "\n"
|
|
const pending = checkpointQueue.catch(() => {}).then(() => writeFile(wireguardInterfaceUpdate ? wgUpdateReportUrl : overlayCleanupOnly && !ospfSourceAcceptance ? new URL("../../../../docs/mcp-chr-overlay-recovery.json", import.meta.url) : reportUrl, payload))
|
|
checkpointQueue = pending; return pending
|
|
}
|
|
const originalPost = MikrotikClient.prototype.post, originalPatch = MikrotikClient.prototype.patch
|
|
const originalGet = MikrotikClient.prototype.get, originalDelete = MikrotikClient.prototype.delete
|
|
const originalPut = MikrotikClient.prototype.put
|
|
const captureNativeFailure = async (error: unknown, method: string, path: string, startedAt: number) => {
|
|
if (checks.filter(row => row.phase === "native-request-failure").length >= 256) return
|
|
checks.push(chrNativeFailure(error, method, path, startedAt)); await checkpoint()
|
|
}
|
|
MikrotikClient.prototype.get = async function<T>(path: string, timeoutMs?: number, signal?: AbortSignal, maxResponseBytes?: number): Promise<T> {
|
|
await pace(); const startedAt = Date.now()
|
|
try { return await originalGet.call(this, path, timeoutMs, signal, maxResponseBytes) as T } catch (error) { await captureNativeFailure(error, "GET", path, startedAt); throw error }
|
|
}
|
|
MikrotikClient.prototype.delete = async function(path: string, timeoutMs?: number, beforeDispatch?:()=>Promise<unknown>): Promise<void> {
|
|
await pace(); const startedAt = Date.now()
|
|
try { const result = await originalDelete.call(this, path, timeoutMs, beforeDispatch); if (wireguardInterfaceDelete) { checks.push({ phase: "wg-cascade-native-delete-ack", path, at: new Date().toISOString() }); await checkpoint() } return result } catch (error) { await captureNativeFailure(error, "DELETE", path, startedAt); throw error }
|
|
}
|
|
MikrotikClient.prototype.put = async function<T>(path: string, body: Record<string, string>, timeoutMs?: number, beforeDispatch?:()=>Promise<unknown>): Promise<T> {
|
|
await pace(); const startedAt = Date.now()
|
|
try { return await originalPut.call(this, path, body, timeoutMs, beforeDispatch) as T } catch (error) { await captureNativeFailure(error, "PUT", path, startedAt); throw error }
|
|
}
|
|
const captureParser = async (error: unknown, path: string) => {
|
|
if (!bgpCommunityAcceptance || !(error instanceof MikrotikError) || error.statusCode !== 400 || !path.startsWith("/routing/filter/rule")) return
|
|
let detail = ""
|
|
try { const parsed = JSON.parse(error.body); if (typeof parsed.detail === "string") detail = parsed.detail } catch { /* Never save headers/raw provider payloads. */ }
|
|
detail = detail.replaceAll(password!, "[redacted]").replace(/(?:password|secret|token|private-key)\s*[=:]\s*[^\s;,]+/gi, "[redacted]").slice(0, 500)
|
|
checks.push({ phase: "community-owned-parser-error", path, nativeStatus: 400, detail }); await checkpoint()
|
|
}
|
|
{
|
|
MikrotikClient.prototype.post = async function<T>(path: string, body: Record<string, string | string[]>, timeoutMs?: number, signal?: AbortSignal): Promise<T> {
|
|
await pace(); const startedAt = Date.now()
|
|
try { return await originalPost.call(this, path, body, timeoutMs, signal) as T } catch (error) { await captureNativeFailure(error, "POST", path, startedAt); await captureParser(error, path); throw error }
|
|
}
|
|
MikrotikClient.prototype.patch = async function<T>(path: string, body: Record<string, string>, timeoutMs?: number, beforeDispatch?:()=>Promise<unknown>): Promise<T> {
|
|
await pace(); const startedAt = Date.now()
|
|
try { return await originalPatch.call(this, path, body, timeoutMs, beforeDispatch) as T } catch (error) { await captureNativeFailure(error, "PATCH", path, startedAt); await captureParser(error, path); throw error }
|
|
}
|
|
}
|
|
try {
|
|
await checkpoint()
|
|
await applySqlMigrations(pool)
|
|
assert.equal((await pool.query("SELECT count(*)::int AS n FROM servers")).rows[0].n, 0, "Acceptance requires an empty disposable database")
|
|
await updateMcpSettings({ enabled: true }, actor)
|
|
for (const [index, host] of hosts.entries()) {
|
|
const serverId = index + 1
|
|
await pool.query("INSERT INTO servers(id,name,host,port,use_ssl,username,password) VALUES($1,$2,$3,$4,false,$5,$6)", [serverId, marker, tunnelPorts ? "127.0.0.1" : host, tunnelPorts?.[index] ?? 80, username, password])
|
|
}
|
|
const operationIds = ["routing.topology.read", "routing.path.verify", "operations.recover", "operations.reconcile", "ros.bridge.list", "ros.bridge.create", "ros.bridge.update", "ros.bridge.delete", "ros.ipv4_route.list", "ros.ipv6_route.list", ...["routing_table", "ipv4_route"].flatMap(resource => ["create", "update", "delete"].map(action => `ros.${resource}.${action}`))]
|
|
if (overlayAcceptance) operationIds.push(...["gre", "vrf", "ipv4_address", "ipv6_address"].flatMap(resource => ["create", "update", "delete"].map(action => `ros.${resource}.${action}`)))
|
|
if (wireguardPeerOnly) operationIds.push("network.wireguard.peers.delete", "ros.wireguard.create", "ros.wireguard.delete", "ros.wireguard_peer.create", "ros.wireguard_peer.delete")
|
|
if (wireguardInterfaceDelete) operationIds.push("network.wireguard.interfaces.delete", ...["ipv4_address", "ipv6_address"].flatMap(resource => ["create", "delete"].map(action => `ros.${resource}.${action}`)))
|
|
if (wireguardPeerUpdate) operationIds.push("network.wireguard.peers.update", "ros.wireguard_peer.update")
|
|
if (wireguardInterfaceUpdate) operationIds.push("network.wireguard.interfaces.update", "ros.wireguard.create", "ros.wireguard.update", "ros.wireguard.delete")
|
|
if (ospfForwarding) operationIds.push("routing.ospf.route.proof.read")
|
|
if (ospfAcceptance) operationIds.push("diagnostics.bandwidth", "routing.ospf.optimize", ...["ospf_instance", "ospf_area", "ospf_template", "bfd_configuration"].flatMap(resource => ["create", "update", "delete"].map(action => `ros.${resource}.${action}`)))
|
|
if (ospfPassiveOnly) operationIds.push(...["ospf_instance", "ospf_area", "ospf_template", "vrf"].flatMap(resource => ["create", "update", "delete"].map(action => `ros.${resource}.${action}`)))
|
|
if (ecmpAcceptance) operationIds.push("routing.ecmp.update", "ros.ipv6_route.create", "ros.ipv6_route.delete")
|
|
if (bgpAcceptance) operationIds.push(...["bgp_instance", "bgp_connection"].flatMap(resource => ["create", "update", "delete"].map(action => `ros.${resource}.${action}`)))
|
|
if (bgpExportAcceptance) operationIds.push("routing.bgp.export", "routing.bgp.prefixes.read")
|
|
if (bgpFilterBindingAcceptance) operationIds.push("routing.bgp.filter.bind", "filters.rules.replace")
|
|
if (bgpCommunityAcceptance) operationIds.push("routing.bgp.community.policy")
|
|
if (process.env.MMAPP_TEST_CHR_COMMUNITY_GATEWAY_RESOLVER === "true") operationIds.push("ros.ipv6_route.create", "ros.ipv6_route.delete")
|
|
const issued = await createMcpConnection(mcpConnectionCreateSchema.parse({ name: marker, mode: "approval", expiresAt: new Date(Date.now() + (bgpCommunityAcceptance || ospfAcceptance ? 2700000 : 1800000)).toISOString(), permissions: ["mm:network:read", "mm:network:write", ...(bgpAcceptance ? ["mm:bgp:read", "mm:bgp:write"] : []), ...(bgpExportAcceptance ? ["mm:filters:read"] : []), ...(bgpFilterBindingAcceptance ? ["mm:filters:write"] : [])], scope: { serverIds: hosts.map((_, index) => index + 1), operationIds, allowGlobal: false }, limits: { callsPerMinute: 120, concurrentReads: 4, mutationsPerHour: bgpCommunityAcceptance ? 240 : ospfAcceptance || ecmpAcceptance || bgpAcceptance ? 160 : 40, serversPerPlan: 2, objectsPerPlan: 100 } }), actor)
|
|
const principal = await authenticateMcpToken(issued.token)
|
|
const execute = async (tool: string, args: Record<string, unknown>) => {
|
|
// Preserve the stage and IDs even if the disposable DB is removed after an
|
|
// interrupted acceptance. Never persist tool arguments or provider payloads.
|
|
const trace: Record<string, unknown> = { phase: "mcp-operation", tool, serverId: typeof args.serverId === "number" ? args.serverId : undefined, stage: "prepare", state: "preparing" }
|
|
checks.push(trace); await checkpoint()
|
|
try {
|
|
const plan = await invokeMcpTool(principal, tool, args) as { planId: string; state: string }
|
|
trace.planId = plan.planId; trace.stage = "approval"
|
|
assert.equal(plan.state, "awaiting-approval")
|
|
await assert.rejects(applyPlan(principal, plan.planId, randomUUID()))
|
|
await approvePlan(plan.planId, actor, true)
|
|
trace.stage = "apply"
|
|
const key = randomUUID(), operation = await applyPlan(principal, plan.planId, key)
|
|
trace.operationId = operation.id; trace.stage = "verification"; trace.state = "running"; await checkpoint()
|
|
assert.equal((await applyPlan(principal, plan.planId, key)).id, operation.id)
|
|
for (let attempt = 0; attempt < (wireguardInterfaceDelete && tool === "mm_wireguard_interface_remove_plan" ? 960 : 240); attempt++) {
|
|
const row = (await pool.query("SELECT state FROM mcp_operations WHERE id=$1", [operation.id])).rows[0]
|
|
if (!["pending", "running"].includes(row.state)) {
|
|
trace.state = row.state; trace.stage = "terminal"
|
|
if (row.state !== "applied") {
|
|
const detail = (await pool.query("SELECT error_code FROM mcp_operations WHERE id=$1", [operation.id])).rows[0]
|
|
trace.errorCode = detail.error_code
|
|
console.error({ tool, state: row.state, errorCode: detail.error_code })
|
|
if (tool === "mm_routing_table_create_plan") console.error({ tableReadBack: (await readRosSnapshot(Number(args.serverId), "routing_table")).rows.filter(row => row.name === marker) })
|
|
}
|
|
await checkpoint()
|
|
assert.equal(row.state, "applied", `Operation ${operation.id} ended ${row.state}`)
|
|
const steps = (await pool.query("SELECT state FROM mcp_operation_steps WHERE operation_id=$1", [operation.id])).rows
|
|
assert.ok(steps.length > 0 && steps.every(step => step.state === "verified"))
|
|
return operation.id
|
|
}
|
|
await delay(500)
|
|
}
|
|
throw new Error(`Operation ${operation.id} remains pending; do not automatically retry writes`)
|
|
} catch (error) {
|
|
if (["preparing", "running"].includes(String(trace.state))) trace.state = "unconfirmed"
|
|
trace.errorCode ??= error instanceof McpError ? error.code : "TEST_STAGE_UNCONFIRMED"
|
|
await checkpoint(); throw error
|
|
}
|
|
}
|
|
const cleanupMarker = process.env.MMAPP_TEST_CHR_CLEANUP_MARKER
|
|
const overlayCleanupMarker = process.env.MMAPP_TEST_CHR_OVERLAY_CLEANUP_MARKER
|
|
let recoveryBaseline: { serverId: number; resource: string; fingerprint: string }[] = []
|
|
if (overlayCleanupMarker) {
|
|
assert.ok(overlayAcceptance, "Overlay cleanup requires explicit overlay mode")
|
|
if (overlayCleanupOnly) {
|
|
const recoveryReport = process.env.MMAPP_TEST_CHR_OVERLAY_RECOVERY_REPORT ?? "mcp-chr-bgp-community-interrupted-overlay.json"
|
|
assert.match(recoveryReport, /^mcp-chr-[a-z0-9-]+\.json$/)
|
|
const previous = JSON.parse(await readFile(new URL("../../../../docs/" + recoveryReport, import.meta.url), "utf8"))
|
|
assert.equal(previous.marker, overlayCleanupMarker)
|
|
assert.equal(previous.complete, false)
|
|
assert.ok(typeof previous.finishedAt === "string" && Number.isFinite(Date.parse(previous.finishedAt)) && Date.parse(previous.finishedAt) <= Date.now(), "Recovery refuses a report whose original process has not finished")
|
|
const recoverySource = readChrRecoverySource(previous, overlayCleanupMarker)
|
|
assert.equal(recoverySource.ospfForwarding, ospfForwarding, "Recovery fixture mode must match the source report")
|
|
recoveryBaseline = recoverySource.baselines
|
|
checks.push({ phase: "recovery-source", report: recoveryReport, kind: recoverySource.kind, originalMarker: overlayCleanupMarker, originalFinishedAt: previous.finishedAt, baselines: recoveryBaseline.length }); await checkpoint()
|
|
if (recoverySource.kind === "ospf") {
|
|
assert.ok(ospfAcceptance, "OSPF recovery requires explicitly scoped OSPF permissions")
|
|
await cleanupChrOspf(overlayCleanupMarker, execute)
|
|
}
|
|
if (bgpAcceptance) await cleanupChrBgpFixture(overlayCleanupMarker, execute)
|
|
if (process.env.MMAPP_TEST_CHR_COMMUNITY_GATEWAY_RESOLVER === "true") {
|
|
const recordedRoutes = previous.checks.filter((row: { phase?: string }) => row.phase === "community-gateway-resolver-created").flatMap((row: { nativeRoutes?: unknown }) => { assert.ok(Array.isArray(row.nativeRoutes) && row.nativeRoutes.length === 1); return row.nativeRoutes })
|
|
await cleanupChrCommunityGatewayResolvers(overlayCleanupMarker, execute, recordedRoutes)
|
|
}
|
|
}
|
|
await cleanupChrOverlay(hosts, overlayCleanupMarker, execute)
|
|
checks.push({ interruptedOverlayRemoved: overlayCleanupMarker })
|
|
}
|
|
if (cleanupMarker) {
|
|
assert.match(cleanupMarker, /^mcp-test-[0-9a-f]{8}$/)
|
|
for (const [index, host] of hosts.entries()) {
|
|
const serverId = index + 1, { client } = await routerClient(serverId)
|
|
const tables = (await readRosSnapshot(serverId, "routing_table")).rows.filter(row => row.name === cleanupMarker)
|
|
if (!tables.length) continue
|
|
assert.ok(tables.every(row => row.fib === "false"), "Cleanup only accepts isolated non-FIB tables")
|
|
assert.ok(!(await client.get<Record<string, string>[]>("/routing/rule")).some(row => row.table === cleanupMarker))
|
|
for (const resource of ["ipv4_route", "ipv6_route"]) assert.ok(!(await readRosSnapshot(serverId, resource)).rows.some(row => row["routing-table"] === cleanupMarker), "Interrupted table must be empty before cleanup")
|
|
for (const row of tables) await execute("mm_routing_table_delete_plan", { serverId, id: row[".id"] })
|
|
checks.push({ host, interruptedRunCleanup: cleanupMarker, emptyNonFibTableRemoved: true })
|
|
}
|
|
}
|
|
if (overlayCleanupOnly) {
|
|
for (const baseline of recoveryBaseline) assert.equal((await readRosSnapshot(baseline.serverId, baseline.resource)).fingerprint, baseline.fingerprint, "Interrupted overlay baseline not restored")
|
|
checks.push({ phase: "interrupted-overlay-baseline-restored", marker: overlayCleanupMarker, restoredFingerprints: recoveryBaseline.length, newFixtureCreated: false }); await checkpoint()
|
|
} else if (wireguardInterfaceUpdate) await acceptChrWireguardInterfaceUpdate(marker, principal, hosts.map((_, index) => index + 1), execute, checks, checkpoint)
|
|
else if (wireguardInterfaceDelete) await acceptChrWireguardInterface(marker, principal, execute, checks, checkpoint)
|
|
else if (wireguardPeerOnly) await acceptChrWireguardPeer(marker, principal, execute, checks, checkpoint, wireguardPeerUpdate)
|
|
else if (ospfPassiveOnly) await acceptChrOspfPassive(marker, execute, checks, checkpoint)
|
|
else if (overlayAcceptance) await acceptChrOverlay(hosts, marker, principal, execute, checks, checkpoint)
|
|
else for (const [index, host] of hosts.entries()) {
|
|
const serverId = index + 1, { client } = await routerClient(serverId)
|
|
const resource = await client.get<Record<string, unknown>>("/system/resource")
|
|
const baseline = await readRosSnapshot(serverId, "bridge")
|
|
assert.ok(!baseline.rows.some(row => row.name === marker))
|
|
const routeBaseline = await Promise.all(["ipv4_route", "ipv6_route"].map(resource => readRosSnapshot(serverId, resource)))
|
|
const tableBaseline = await readRosSnapshot(serverId, "routing_table")
|
|
assert.ok(!tableBaseline.rows.some(row => row.name === marker))
|
|
const topology = await invokeMcpTool(principal, "mm_routing_topology_get", { serverId }) as { resources: Record<string, { observed: boolean; items: unknown[] }>; observedPath: boolean }
|
|
assert.equal(topology.observedPath, false)
|
|
assert.ok(topology.resources.interfaces.observed && topology.resources.ipv4.observed && topology.resources.ipv6.observed)
|
|
let pathProof: unknown
|
|
if (routingAcceptance && hosts.length === 2) {
|
|
const peerId = serverId === 1 ? 2 : 1, { client: peerClient } = await routerClient(peerId)
|
|
const addresses = await client.get<Record<string, string>[]>("/ip/address"), peerAddresses = await peerClient.get<Record<string, string>[]>("/ip/address")
|
|
const source = addresses.filter(row => row.interface === "ether1" && row.disabled !== "true" && row.address.split("/")[0] === host), peer = peerAddresses.filter(row => row.interface === "ether1" && row.disabled !== "true" && row.address.split("/")[0] === hosts[peerId - 1])
|
|
assert.equal(source.length, 1); assert.equal(peer.length, 1)
|
|
// Two physical devices validate the command/parser and direct control path.
|
|
// JumpHost/ExitNode collapse to the peer; this is explicitly not a 3-node failover lab.
|
|
const profile = { name: marker, serverId, community: "65000:65000", family: "ipv4", probeVrf: "main", candidates: [{ id: "direct-lab-path", jumpHostId: peerId, exitNodeId: peerId, wanInterface: "ether1", downstreamInterface: "ether1", gateway: peer[0].address.split("/")[0], gatewayRef: { resource: "ipv4_route", id: "not-used-by-probe" }, controlProbe: { targetInterface: "ether1", targetAddress: hosts[peerId - 1], sourceAddress: host, expectedHops: [{ serverId: peerId, interface: "ether1", address: hosts[peerId - 1] }] } }] }
|
|
pathProof = await invokeMcpTool(principal, "mm_route_path_verify", { profile, candidateId: "direct-lab-path", proofPurpose: "control-path" })
|
|
assert.equal((pathProof as { status: string }).status, "passed", JSON.stringify(pathProof))
|
|
}
|
|
await assert.rejects(invokeMcpTool(principal, "mm_bridge_list", { serverId: 99999 }))
|
|
// Never touch an existing interface: an empty disabled bridge has no ports or addresses.
|
|
let cleanup = false
|
|
try {
|
|
cleanup = true // Includes ambiguous create outcomes: finally re-reads before attempting cleanup.
|
|
await execute("mm_bridge_create_plan", { serverId, fields: { name: marker, disabled: true, comment: marker } })
|
|
let own = (await readRosSnapshot(serverId, "bridge")).rows.filter(row => row.name === marker && row.comment === marker)
|
|
assert.equal(own.length, 1); assert.equal(own[0].disabled, "true")
|
|
const id = own[0][".id"]
|
|
const updateOperationId = await execute("mm_bridge_update_plan", { serverId, id, fields: { comment: marker + ":verified" } })
|
|
own = (await readRosSnapshot(serverId, "bridge")).rows.filter(row => row[".id"] === id)
|
|
assert.equal(own[0].comment, marker + ":verified")
|
|
await execute("mm_operation_recovery_plan", { operationId: updateOperationId })
|
|
own = (await readRosSnapshot(serverId, "bridge")).rows.filter(row => row[".id"] === id)
|
|
assert.equal(own[0].comment, marker)
|
|
await execute("mm_bridge_delete_plan", { serverId, id })
|
|
cleanup = false
|
|
} finally {
|
|
if (cleanup) {
|
|
const own = (await readRosSnapshot(serverId, "bridge")).rows.filter(row => row.name === marker && [marker, marker + ":verified"].includes(String(row.comment)) && row.disabled === "true")
|
|
// Bounded cleanup uses the same approved executor; no broad name matching or direct DELETE.
|
|
for (const row of own) await execute("mm_bridge_delete_plan", { serverId, id: row[".id"] })
|
|
}
|
|
}
|
|
assert.equal((await readRosSnapshot(serverId, "bridge")).fingerprint, baseline.fingerprint, "Bridge configuration must return to its baseline")
|
|
if (routingAcceptance) {
|
|
// No rules or interfaces select this unique non-FIB table. Its routes never enter
|
|
// packet forwarding; existing main/default/management routes remain untouched.
|
|
try {
|
|
await execute("mm_routing_table_create_plan", { serverId, fields: { name: marker, fib: false } })
|
|
const tables = (await readRosSnapshot(serverId, "routing_table")).rows.filter(row => row.name === marker)
|
|
assert.equal(tables.length, 1); assert.equal(tables[0].fib, "false")
|
|
await execute("mm_ipv4_route_create_plan", { serverId, fields: { "dst-address": "192.0.2.0/24", "routing-table": marker, gateway: "ether1", distance: 10, disabled: true, comment: marker } })
|
|
const own = (await readRosSnapshot(serverId, "ipv4_route")).rows.filter(row => row.comment === marker && row["routing-table"] === marker)
|
|
assert.equal(own.length, 1)
|
|
await assert.rejects(invokeMcpTool(principal, "mm_routing_table_delete_plan", { serverId, id: tables[0][".id"] }), /Таблица используется/)
|
|
const id = own[0][".id"]
|
|
const operationId = await execute("mm_ipv4_route_update_plan", { serverId, id, fields: { distance: 20, disabled: false } })
|
|
let current = (await readRosSnapshot(serverId, "ipv4_route")).rows.find(row => row[".id"] === id)!
|
|
assert.equal(current.distance, "20"); assert.equal(current.disabled, "false")
|
|
await execute("mm_operation_recovery_plan", { operationId })
|
|
current = (await readRosSnapshot(serverId, "ipv4_route")).rows.find(row => row[".id"] === id)!
|
|
assert.equal(current.distance, "10"); assert.equal(current.disabled, "true")
|
|
checks.push({ host, isolatedRouting: { tableFib: false, prefix: "192.0.2.0/24", approvedRouteCreateUpdate: true, enabledReadBack: true, usedTableDeletionRejected: true, approvedInverseRestoredDistanceAndDisabled: true, forwardingTest: false } })
|
|
} finally {
|
|
// Exact ownership and destination checks, including ambiguous create results.
|
|
const own = (await readRosSnapshot(serverId, "ipv4_route")).rows.filter(row => row.comment === marker && row["routing-table"] === marker && row["dst-address"] === "192.0.2.0/24")
|
|
for (const row of own) await execute("mm_ipv4_route_delete_plan", { serverId, id: row[".id"] })
|
|
assert.ok(!(await readRosSnapshot(serverId, "ipv4_route")).rows.some(row => row["routing-table"] === marker))
|
|
for (const row of (await readRosSnapshot(serverId, "routing_table")).rows.filter(row => row.name === marker)) await execute("mm_routing_table_delete_plan", { serverId, id: row[".id"] })
|
|
}
|
|
assert.equal((await readRosSnapshot(serverId, "routing_table")).fingerprint, tableBaseline.fingerprint, "Routing table configuration must return to baseline")
|
|
}
|
|
for (const before of routeBaseline) assert.equal((await readRosSnapshot(serverId, before.resource)).fingerprint, before.fingerprint, "Existing route configuration changed during acceptance")
|
|
checks.push({ host, routerOs: resource.version, topology: Object.fromEntries(Object.entries(topology.resources).map(([key, value]) => [key, { observed: value.observed, count: value.items.length }])), approvedCreateUpdateDelete: true, approvedInversePlan: true, pathProof, twoDeviceControlPathOnly: routingAcceptance, idempotency: true, readBack: true, bridgeBaselineRestored: true, routeConfigurationUnchanged: true })
|
|
console.log(`✓ ${host}: actual topology, approval, create/update/delete, read-back, idempotency, cleanup`)
|
|
}
|
|
report.complete = true
|
|
} finally {
|
|
MikrotikClient.prototype.post = originalPost; MikrotikClient.prototype.patch = originalPatch
|
|
MikrotikClient.prototype.get = originalGet; MikrotikClient.prototype.delete = originalDelete
|
|
MikrotikClient.prototype.put = originalPut
|
|
report.finishedAt = new Date().toISOString()
|
|
await checkpoint()
|
|
await closePool()
|
|
}
|