366 lines
31 KiB
TypeScript
366 lines
31 KiB
TypeScript
/** Owned SQL/HTTP acceptance only; requires an exclusive disposable runner. No runtime entrypoint.
|
|
* Actual Fastify/JWT + application/control PG + signed v2 source + peer host +
|
|
* completion receipt + workflow + localhost HTTP. Enrollment Q/wire mapping and
|
|
* RouterOS responses are explicitly synthetic owned fixtures, never readiness.
|
|
*/
|
|
import assert from "node:assert/strict"
|
|
import { randomBytes, randomUUID } from "node:crypto"
|
|
import { ClientRequest, createServer } from "node:http"
|
|
import { writeFile } from "node:fs/promises"
|
|
import { join } from "node:path"
|
|
import { performance } from "node:perf_hooks"
|
|
import { AsyncLocalStorage } from "node:async_hooks"
|
|
import type { FastifyInstance } from "fastify"
|
|
import type { Client } from "pg"
|
|
import { assertOwnedBandwidthSql } from "./bandwidth-admission-process.sql-helper.js"
|
|
|
|
export async function acceptOwnedRestSpeedWorkflow() {
|
|
assertOwnedBandwidthSql(); assert.equal(process.env.MMAPP_OWN_BANDWIDTH_SPEED_WORKFLOW, "true")
|
|
assert.equal(process.env.MMAPP_OWN_BANDWIDTH_EXTERNAL_REGISTRY, "true")
|
|
await writeFile(join(process.env.MCP_STATE_DIR!, "encryption-key"), randomBytes(32), { flag: "wx", mode: 0o600 })
|
|
await writeFile(join(process.env.MCP_STATE_DIR!, "access-generation"), randomBytes(32), { flag: "wx", mode: 0o600 })
|
|
const { createOwnedExternalRegistryHarness } = await import("./bandwidth-external-node-registry.sql-helper.js")
|
|
const { pool } = await import("../db/index.js")
|
|
const { Client: PgClient } = await import("pg")
|
|
const { default: Fastify } = await import("fastify")
|
|
const { SignJWT } = await import("jose")
|
|
const { default: authPlugin, requireAuth } = await import("../plugins/auth.js")
|
|
const { env } = await import("../config.js")
|
|
const { speedTestCommandSchema } = await import("@mmapp/contracts")
|
|
const { createRestSpeedBandwidthOriginalProvider } = await import("./bandwidth-rest-speed-original.js")
|
|
const { createBandwidthPeerObservationReader } = await import("./bandwidth-peer-observation.js")
|
|
const { createBandwidthPeerExecutionHost } = await import("./bandwidth-peer-execution-host.js")
|
|
const { createBandwidthSpeedWorkflow } = await import("./bandwidth-speed-workflow.js")
|
|
const { SpeedRunPersistenceUnknown } = await import("./speed-run-persistence.js")
|
|
const { recordSpeedTestFailure } = await import("./uptime-speed-service.js")
|
|
const { createBandwidthSpeedRunIdentity } = await import("./bandwidth-speed-run-identity.js")
|
|
assert.equal(env.authRequired, true)
|
|
const f = await createOwnedExternalRegistryHarness()
|
|
const application = await pool.connect().catch(async error => { await f.close(); throw error })
|
|
let app: FastifyInstance | undefined
|
|
const checks: string[] = [], runIdChecks: string[] = [], metrics: Record<string, unknown>[] = [], runIdResults: Record<string, unknown>[] = []
|
|
const requestSql = new AsyncLocalStorage<{ applicationSql: number; controlSql: number }>()
|
|
const claims = [
|
|
{ "system-id": "OWNED-WORKFLOW-SOURCE-" + randomUUID(), "software-id": "OWNED-WORKFLOW-SOURCE" },
|
|
{ "system-id": "OWNED-WORKFLOW-DESTINATION-" + randomUUID(), "software-id": "OWNED-WORKFLOW-DESTINATION" },
|
|
]
|
|
let mode: "positive" | "zero" | "probe-edit" | "probe-delete" | "commit-loss" | "native-drop" | "runid-legacy" | "runid-history-delete" | "runid-concurrent" = "positive"
|
|
let nativePosts = 0, nativeGets = 0, pingPosts = 0, protocolErrors = 0, failureWrites = 0, sqlCalls = 0, coreSqlCalls = 0, credentialPayloadRows = 0
|
|
let beforeResponse: (() => Promise<void>) | undefined, lastNativeFinished = 0, pingDelay = 0
|
|
let actualNativeBody: unknown, actualPingBody: unknown, expectedNativeBody: unknown
|
|
let rawRows: Array<Record<string, string>> = []
|
|
const socketTimeouts: Array<{ path: string; timeoutMs: number }> = []
|
|
const ids: number[] = [], users: string[] = []
|
|
const probeId = "owned-workflow-probe-" + randomUUID()
|
|
let commitLossTarget: string | undefined, commitLossInjected = false, historyPid: number | undefined, positiveRunId = ""
|
|
const ownedRunIds = new Set<string>()
|
|
let concurrentRunId: string | undefined, concurrentArrivals = 0, releaseConcurrent: (() => void) | undefined
|
|
let concurrentGate = Promise.resolve()
|
|
let historyCommitsObserved = 0
|
|
const transactions = new WeakMap<Client, { historyRunId?: string; pid: number }>()
|
|
const query = PgClient.prototype.query, poolQuery = pool.query, setSocketTimeout = ClientRequest.prototype.setTimeout
|
|
const routers = claims.map((license, index) => createServer(async (request, response) => {
|
|
try {
|
|
assert.equal(request.socket.remoteAddress, "127.0.0.1")
|
|
assert.equal(request.headers.authorization, "Basic " + Buffer.from(users[index] + ":owned-localhost-only").toString("base64"))
|
|
response.setHeader("content-type", "application/json")
|
|
if (request.method === "GET" && request.url === "/rest/system/license") { nativeGets++; response.end(JSON.stringify(license)); return }
|
|
if (request.method === "GET" && request.url === "/rest/ip/address") {
|
|
nativeGets++
|
|
response.end(JSON.stringify(index === 0 ? [{ interface: "source-if", address: "198.51.100.1/30" }] : [
|
|
{ interface: "dest-if", address: "198.51.100.99/30", dynamic: "true" },
|
|
{ interface: "dest-if", address: "198.51.100.77/30", disabled: "true" },
|
|
{ interface: "dest-if", address: "198.51.100.66/30", invalid: "true" },
|
|
{ interface: "dest-if", address: "198.51.100.2/30", dynamic: "false" },
|
|
])); return
|
|
}
|
|
assert.equal(request.method, "POST"); assert.equal(index, 0)
|
|
let text = ""; for await (const chunk of request) { text += String(chunk); assert(Buffer.byteLength(text) <= 65536) }
|
|
const body = JSON.parse(text)
|
|
if (request.url === "/rest/tool/bandwidth-test") {
|
|
nativePosts++; actualNativeBody = body; assert.deepEqual(body, expectedNativeBody)
|
|
if (mode === "native-drop") { response.destroy(); return }
|
|
await beforeResponse?.()
|
|
rawRows = [{ status: "testing", "tx-current": mode === "zero" ? "0" : "1000000", "rx-current": mode === "zero" ? "0" : "2M", retained: "owned-extra-sample" },
|
|
{ status: "done testing", "tx-total-average": mode === "zero" ? "0" : "3000000", "rx-total-average": mode === "zero" ? "0" : "4M", retained: "owned-extra-terminal" }]
|
|
lastNativeFinished = performance.now(); response.end(JSON.stringify(rawRows)); return
|
|
}
|
|
assert.equal(request.url, "/rest/tool/ping")
|
|
pingPosts++; actualPingBody = body
|
|
assert.deepEqual(body, { address: "198.51.100.2", count: "4", interface: "source-if", interval: "1s" })
|
|
pingDelay = performance.now() - lastNativeFinished; assert(pingDelay >= 2950, "actual workflow must retain its 3-second pause")
|
|
response.end(JSON.stringify(mode === "zero" ? [] : [{ sent: "4", received: "3", "packet-loss": "25%", "avg-rtt": "2ms" }]))
|
|
} catch { protocolErrors++; response.statusCode = 502; response.end('{"error":"Owned workflow HTTP fixture protocol mismatch"}') }
|
|
}))
|
|
try {
|
|
// These are ACTUAL enrollment observations using the harness's own stable
|
|
// license vectors. Distinct source/destination claims cannot alias one node.
|
|
for (const license of claims) ids.push(await f.managedEndpoint(license))
|
|
users.push(...ids.map(id => "managed-" + id))
|
|
const receipts: Array<{ canonicalNodeId: string }> = []
|
|
for (const id of ids) receipts.push(await f.managedHost.register(await f.managedObservation(id), f.scope()))
|
|
assert.notEqual(receipts[0].canonicalNodeId, receipts[1].canonicalNodeId)
|
|
for (let index = 0; index < routers.length; index++) {
|
|
await new Promise<void>((resolve, reject) => { routers[index].once("error", reject); routers[index].listen(0, "127.0.0.1", resolve) })
|
|
// Let the actual trigger mint epochs; all post-change endpoint snapshots
|
|
// come ONLY from the genuine REST provider's real application SQL.
|
|
await application.query("UPDATE public.servers SET port=$2 WHERE id=$1", [ids[index], (routers[index].address() as { port: number }).port])
|
|
}
|
|
const signed = await f.createSignedSource(), provider = createRestSpeedBandwidthOriginalProvider()
|
|
const restoreProbe = async () => {
|
|
await application.query("DELETE FROM public.uptime_speed_probes WHERE id=$1", [probeId])
|
|
await application.query("INSERT INTO public.uptime_speed_probes(id,src_server_id,dst_server_id,src_interface,dst_interface,protocol,direction,duration_sec,enabled) VALUES($1,$2,$3,'source-if','dest-if','udp','receive',3,true)", [probeId, ids[0], ids[1]])
|
|
}
|
|
await restoreProbe()
|
|
// OBSERVATION ONLY except one explicitly selected real history COMMIT loss.
|
|
// All original SQL/configs/parameters execute unchanged on actual PG. The
|
|
// control DB can never satisfy this hook's application database/PID/INSERT.
|
|
PgClient.prototype.query = function(this: Client, ...args: unknown[]) {
|
|
const client = this as unknown as { connectionParameters: { database: string; user: string }; processID: number }
|
|
const text = typeof args[0] === "string" ? args[0] : (args[0] as { text?: string })?.text ?? ""
|
|
// Drizzle calls client.query(QueryConfig, params): the separate params
|
|
// array takes precedence, just as it does in pg's normalizeQueryConfig.
|
|
const values = (Array.isArray(args[1]) ? args[1] : (args[0] as { values?: unknown[] })?.values) as unknown[] | undefined
|
|
const applicationConnection = client.connectionParameters.database === "postgres" && client.connectionParameters.user === "mcp_test"
|
|
const counters = requestSql.getStore()
|
|
if (applicationConnection) { sqlCalls++; if (counters && typeof args.at(-1) !== "function") counters.applicationSql++ }
|
|
else if (client.connectionParameters.database === f.databaseName) { coreSqlCalls++; if (counters) counters.controlSql++ }
|
|
const result = Reflect.apply(query, this, args)
|
|
if (!result || typeof (result as Promise<unknown>).then !== "function") return result
|
|
return (result as Promise<unknown>).then(async value => {
|
|
if (!applicationConnection) return value
|
|
if (/^begin\b/i.test(text.trim())) transactions.set(this, { pid: client.processID })
|
|
const tx = transactions.get(this)
|
|
if (tx && /^insert\s+into\s+"?uptime_speed_test_runs"?/i.test(text.trim())) {
|
|
const matching = values?.filter((value): value is string => typeof value === "string" && ownedRunIds.has(value)) ?? []
|
|
assert.equal(matching.length, 1, "one exact owned history run ID per actual INSERT")
|
|
tx.historyRunId = matching[0]
|
|
}
|
|
if (/^(?:commit|rollback)\b/i.test(text.trim())) {
|
|
transactions.delete(this)
|
|
if (/^commit\b/i.test(text.trim()) && tx?.historyRunId) historyCommitsObserved++
|
|
if (/^commit\b/i.test(text.trim()) && tx && tx.historyRunId === commitLossTarget && commitLossTarget !== undefined && !commitLossInjected) {
|
|
commitLossInjected = true; historyPid = tx.pid
|
|
// COMMIT really ACKed; only its return to the history writer is lost.
|
|
throw Error("Owned application history COMMIT return intentionally lost")
|
|
}
|
|
}
|
|
return value
|
|
})
|
|
} as typeof query
|
|
// pg-pool forwards queries to Client.query with a callback, so observing
|
|
// only Client.query's Promise branch misses provider SELECT completions.
|
|
// Preserve pool SQL/config/params/results; synchronize only two real empty
|
|
// history SELECT results in the explicit concurrency case.
|
|
pool.query = function(this: typeof pool, ...args: unknown[]) {
|
|
const text = typeof args[0] === "string" ? args[0] : (args[0] as { text?: string })?.text ?? ""
|
|
const values = (Array.isArray(args[1]) ? args[1] : (args[0] as { values?: unknown[] })?.values) as unknown[] | undefined
|
|
const counters = requestSql.getStore(); if (counters) counters.applicationSql++
|
|
const result = Reflect.apply(poolQuery, this, args)
|
|
if (!result || typeof (result as Promise<unknown>).then !== "function") return result
|
|
return (result as Promise<unknown>).then(async value => {
|
|
if (text.startsWith("/* rest-speed-original:credentials */")) credentialPayloadRows += (value as { rows: unknown[] }).rows.length
|
|
if (concurrentRunId && text.startsWith("/* bandwidth-speed-run:history */") && values?.[0] === concurrentRunId) {
|
|
assert.equal((value as { rows: unknown[] }).rows.length, 0)
|
|
concurrentArrivals++; if (concurrentArrivals === 2) releaseConcurrent?.()
|
|
await concurrentGate
|
|
}
|
|
return value
|
|
})
|
|
} as typeof pool.query
|
|
ClientRequest.prototype.setTimeout = function(this: ClientRequest, ...args: Parameters<typeof setSocketTimeout>) {
|
|
if (this.path === "/rest/tool/bandwidth-test" || this.path === "/rest/tool/ping") socketTimeouts.push({ path: this.path, timeoutMs: args[0] })
|
|
return Reflect.apply(setSocketTimeout, this, args)
|
|
}
|
|
app = Fastify(); await app.register(authPlugin); app.addHook("preHandler", requireAuth)
|
|
app.setErrorHandler((_error, _request, reply) => { reply.code(403).send({ refused: true }) })
|
|
app.post("/api/uptime/speed-test", async (request, reply) => {
|
|
const counters = { applicationSql: 0, controlSql: 0 }, requestMode = mode
|
|
return requestSql.run(counters, async () => {
|
|
const command = speedTestCommandSchema.parse(request.body), scope = { deadlineAt: Date.now() + 60000 }
|
|
const began = performance.now()
|
|
let phase = "capture", failureCode: string | undefined, peerPhase: string | undefined
|
|
let observer: ReturnType<typeof createBandwidthPeerObservationReader> | undefined
|
|
try {
|
|
const token = await provider.capture(request, command, scope)
|
|
phase = "compose"
|
|
observer = createBandwidthPeerObservationReader({ read: (peer, current) => provider.readPeer(token, peer, current) })
|
|
const host = createBandwidthPeerExecutionHost({ coreSource: signed.core, nodeSource: signed.node, observer,
|
|
readOriginal: (original, current) => provider.readOriginal(original, current),
|
|
verifyRemote: async (original, intent, evidence, current) => {
|
|
assert.equal(original, token)
|
|
assert.deepEqual(intent.actualRemotePeer, { kind: "server", serverId: ids[1] })
|
|
assert.equal(evidence.sourceReceipt.canonicalNodeId, receipts[0].canonicalNodeId)
|
|
assert.equal(evidence.remoteReceipt.canonicalNodeId, receipts[1].canonicalNodeId)
|
|
assert.equal(intent.nativeBody.address, (await provider.readSpeedContext(token, current)).address)
|
|
return { proof: f.mac("SYNTHETIC-OWNED-WORKFLOW-WIRE-MAPPING", { source: receipts[0].canonicalNodeId, remote: receipts[1].canonicalNodeId,
|
|
address: intent.nativeBody.address, credentialServerId: intent.credentialServerId, generation: "owned-mapping-1" }), expiresAt: intent.expiresAt }
|
|
} })
|
|
phase = "workflow"
|
|
const result = await createBandwidthSpeedWorkflow({ originals: provider, peerHost: host }).run(token, scope)
|
|
phase = "done"; return { ok: true, result }
|
|
} catch (error) {
|
|
const cause = error instanceof Error && error.cause ? error.cause : error
|
|
const marker = cause as { code?: unknown; phase?: unknown }
|
|
if (typeof marker?.code === "string" && /^[A-Z_]{1,80}$/.test(marker.code)) failureCode = marker.code
|
|
if (typeof marker?.phase === "string" && /^[a-z-]{1,80}$/.test(marker.phase)) peerPhase = marker.phase
|
|
if (error instanceof SpeedRunPersistenceUnknown) { phase = "unknown"; return reply.code(502).send({ outcome: "unknown", runId: error.runId }) }
|
|
phase += "-refused"; failureWrites++
|
|
await recordSpeedTestFailure({ ...command, message: error instanceof Error ? error.message : String(error) })
|
|
return reply.code(502).send({ outcome: "refused" })
|
|
} finally {
|
|
observer?.close(); metrics.push({ mode: requestMode, requestId: request.id, runId: command.runId, phase, failureCode, peerPhase,
|
|
elapsedMs: Math.round(performance.now() - began), applicationSql: counters.applicationSql, controlSql: counters.controlSql })
|
|
}
|
|
})
|
|
})
|
|
const mintJwt = () => new SignJWT({ sub: "owned-workflow-actor-" + randomUUID(), apps: ["mm"], permissions: ["mm:uptime:write"], iat: Math.floor(Date.now() / 1000) - 60, exp: Math.floor(Date.now() / 1000) + 3600 })
|
|
.setProtectedHeader({ alg: "HS256" }).setIssuer(env.authIssuer).sign(new TextEncoder().encode(env.jwtSecret))
|
|
const jwt = await mintJwt()
|
|
const send = async (options: { durationSec?: number; runId?: string; jwt?: string } = {}) => {
|
|
const durationSec = options.durationSec ?? 3, runId = options.runId ?? "owned-workflow-run-" + randomUUID()
|
|
ownedRunIds.add(runId)
|
|
expectedNativeBody = { address: "198.51.100.2", user: users[1], password: "owned-localhost-only", protocol: "udp", direction: "receive", duration: durationSec + "s" }
|
|
const response = await app!.inject({ method: "POST", url: "/api/uptime/speed-test", headers: { authorization: "Bearer " + (options.jwt ?? jwt) },
|
|
payload: { runId, probeId, srcServerId: ids[0], dstServerId: ids[1], srcInterface: "source-if", dstInterface: "dest-if", protocol: "udp", direction: "receive", durationSec } })
|
|
return { response, runId }
|
|
}
|
|
const history = async (runId: string) => (await application.query("SELECT id,probe_id,status,tx_avg_mbps,rx_avg_mbps,ping_rtt_ms,ping_loss_pct,ping_error FROM public.uptime_speed_test_runs WHERE id=$1", [runId])).rows
|
|
for (const next of ["positive", "zero", "probe-edit", "probe-delete", "commit-loss"] as const) {
|
|
mode = next; await restoreProbe(); commitLossInjected = false
|
|
const caseRunId = "owned-workflow-run-" + randomUUID(); commitLossTarget = next === "commit-loss" ? caseRunId : undefined
|
|
beforeResponse = next === "probe-edit" ? async () => { await application.query("UPDATE public.uptime_speed_probes SET duration_sec=4 WHERE id=$1", [probeId]) }
|
|
: next === "probe-delete" ? async () => { await application.query("DELETE FROM public.uptime_speed_probes WHERE id=$1", [probeId]) } : undefined
|
|
const beforeNative = nativePosts, beforePing = pingPosts, beforeFailures = failureWrites, beforeCommits = historyCommitsObserved
|
|
const { response, runId } = await send({ runId: caseRunId }), rows = await history(runId), body = response.json()
|
|
console.log("Owned wholeflow case timing:", JSON.stringify(metrics.at(-1)))
|
|
assert.equal(nativePosts, beforeNative + 1, next); assert.equal(pingPosts, beforePing + 1, next)
|
|
assert.equal(failureWrites, beforeFailures, next); assert.equal(rows.length, 1, next); assert.equal(rows[0].status, "done")
|
|
assert.equal(historyCommitsObserved, beforeCommits + 1, "observe exactly one real history COMMIT for " + next)
|
|
assert.equal(rows[0].probe_id, next === "probe-edit" || next === "probe-delete" ? null : probeId)
|
|
if (next === "commit-loss") { assert.equal(response.statusCode, 502); assert.equal(body.outcome, "unknown"); assert(commitLossInjected); assert(historyPid); assert.equal(body.runId, runId) }
|
|
else {
|
|
assert.equal(response.statusCode, 200, next); assert.equal(body.ok, true); assert.deepEqual(body.result.raw, rawRows)
|
|
assert.equal(body.result.txAvgMbps, next === "zero" ? 0 : 3); assert.equal(body.result.rxAvgMbps, next === "zero" ? 0 : 4)
|
|
assert.equal(Number(rows[0].tx_avg_mbps), next === "zero" ? 0 : 3)
|
|
if (next === "zero") { assert.equal(body.result.afterBtPing.rttMs, null); assert(body.result.afterBtPing.error); assert(rows[0].ping_error) }
|
|
else { assert.equal(body.result.afterBtPing.rttMs, 2); assert.equal(body.result.afterBtPing.lossPct, 25); assert.equal(body.result.afterBtPing.error, null) }
|
|
}
|
|
assert(pingDelay >= 2950); assert.deepEqual(actualNativeBody, expectedNativeBody)
|
|
assert.deepEqual(actualPingBody, { address: "198.51.100.2", count: "4", interface: "source-if", interval: "1s" })
|
|
assert.equal(Number((await f.owner.query("SELECT count(*) AS n FROM mmapp_control.bandwidth_node_occupancy")).rows[0].n), 0)
|
|
checks.push(next + ": genuine REST + signed v2 actual broker completion receipt + exact native/ping + real atomic history; no second failure/native replay")
|
|
if (next === "positive") positiveRunId = runId
|
|
}
|
|
// Separate run-ID acceptance. Original six workflow checks remain intact.
|
|
// All fault injection is disabled before overlapping/new actor requests.
|
|
commitLossTarget = undefined; beforeResponse = undefined; await restoreProbe()
|
|
{
|
|
mode = "runid-legacy"
|
|
const legacyId = "owned-workflow-run-legacy-" + randomUUID(), identity = createBandwidthSpeedRunIdentity(legacyId)
|
|
await application.query("INSERT INTO public.uptime_speed_test_runs(id,src_server_id,dst_server_id,status) VALUES($1,$2,$3,'done')", [legacyId, ids[0], ids[1]])
|
|
assert.equal((await f.owner.query("SELECT attempt_id FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [identity.attemptId])).rows.length, 0)
|
|
const before = { nativePosts, nativeGets, pingPosts, credentialPayloadRows, failureWrites }
|
|
const { response } = await send({ runId: legacyId, jwt: await mintJwt() })
|
|
assert.equal(response.statusCode, 502); assert.equal(response.json().outcome, "unknown")
|
|
assert.deepEqual({ nativePosts, nativeGets, pingPosts, credentialPayloadRows, failureWrites }, before)
|
|
assert.equal((await history(legacyId)).length, 1)
|
|
assert.equal((await f.owner.query("SELECT attempt_id FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [identity.attemptId])).rows.length, 0)
|
|
runIdChecks.push("legacy app-history ID with no control attempt refuses before credential projection/native and writes no failure")
|
|
runIdResults.push({ case: "legacy-history", nativePosts: nativePosts - before.nativePosts, nativeGets: nativeGets - before.nativeGets,
|
|
credentialPayloadRows: credentialPayloadRows - before.credentialPayloadRows, failureWrites: failureWrites - before.failureWrites })
|
|
}
|
|
{
|
|
mode = "runid-history-delete"
|
|
assert(positiveRunId)
|
|
const identity = createBandwidthSpeedRunIdentity(positiveRunId)
|
|
const first = (await f.owner.query("SELECT caller_kind,state,resolution FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [identity.attemptId])).rows
|
|
assert.equal(first.length, 1); assert.equal(first[0].caller_kind, "rest-speed"); assert.equal(first[0].state, "released"); assert.equal(first[0].resolution, "remote_completion")
|
|
const removed = await application.query("DELETE FROM public.uptime_speed_test_runs WHERE id=$1 RETURNING id", [positiveRunId])
|
|
assert.equal(removed.rowCount, 1); assert.equal(removed.rows[0].id, positiveRunId)
|
|
const before = { nativePosts, pingPosts, failureWrites, historyCommitsObserved }
|
|
const { response } = await send({ runId: positiveRunId, jwt: await mintJwt() })
|
|
assert.equal(response.statusCode, 502); assert.equal(response.json().outcome, "unknown")
|
|
assert.deepEqual({ nativePosts, pingPosts, failureWrites, historyCommitsObserved }, before)
|
|
assert.equal((await history(positiveRunId)).length, 0)
|
|
const remaining = (await f.owner.query("SELECT caller_kind,state,resolution FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [identity.attemptId])).rows
|
|
assert.deepEqual(remaining, first)
|
|
runIdChecks.push("after DELETE of exactly one owned history row, fresh genuine JWT actor/same run ID cannot send again; permanent released control attempt remains unchanged (not a database restore claim)")
|
|
runIdResults.push({ case: "owned-history-row-delete", removedHistoryRows: removed.rowCount, nativePosts: nativePosts - before.nativePosts,
|
|
pingPosts: pingPosts - before.pingPosts, failureWrites: failureWrites - before.failureWrites, preservedControlAttempts: remaining.length })
|
|
}
|
|
{
|
|
mode = "runid-concurrent"
|
|
const sharedId = "owned-workflow-run-concurrent-" + randomUUID()
|
|
concurrentRunId = sharedId; concurrentArrivals = 0
|
|
concurrentGate = new Promise<void>(resolve => { releaseConcurrent = resolve })
|
|
const before = { nativePosts, pingPosts, failureWrites, historyCommitsObserved }
|
|
const tokens = await Promise.all([mintJwt(), mintJwt()])
|
|
let replies: Awaited<ReturnType<typeof send>>[]
|
|
try { replies = await Promise.all(tokens.map(jwt => send({ runId: sharedId, jwt }))) }
|
|
finally { releaseConcurrent?.(); concurrentRunId = undefined; releaseConcurrent = undefined }
|
|
assert.equal(concurrentArrivals, 2, "both genuine requests observed empty history before either continued")
|
|
assert.equal(nativePosts - before.nativePosts, 1); assert.equal(pingPosts - before.pingPosts, 1)
|
|
assert.equal(failureWrites, before.failureWrites); assert.equal(historyCommitsObserved - before.historyCommitsObserved, 1)
|
|
const successes = replies.filter(value => value.response.statusCode === 200).length
|
|
assert(successes <= 1)
|
|
for (const value of replies) assert(value.response.statusCode === 200 || value.response.statusCode === 502 && value.response.json().outcome === "unknown")
|
|
const rows = await history(sharedId); assert.equal(rows.length, 1); assert.equal(rows[0].status, "done")
|
|
const identity = createBandwidthSpeedRunIdentity(sharedId)
|
|
const attempts = (await f.owner.query("SELECT state FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [identity.attemptId])).rows
|
|
assert.equal(attempts.length, 1); assert.equal(attempts[0].state, "released")
|
|
assert.equal(Number((await f.owner.query("SELECT count(*) AS n FROM mmapp_control.bandwidth_node_occupancy")).rows[0].n), 0)
|
|
runIdChecks.push("two synchronized actual REST/JWT requests with one new run ID yield exactly one native POST/history COMMIT, at most one success, no failure write or automatic replay")
|
|
runIdResults.push({ case: "concurrent-real-requests", requests: replies.length, emptyHistoryArrivals: concurrentArrivals,
|
|
nativePosts: nativePosts - before.nativePosts, pingPosts: pingPosts - before.pingPosts, successes, historyRows: rows.length,
|
|
historyCommits: historyCommitsObserved - before.historyCommitsObserved, failureWrites: failureWrites - before.failureWrites })
|
|
}
|
|
{
|
|
// Actual control PG journal, explicitly synthetic fixed authority scope;
|
|
// this supplements the genuine REST cases without a fake driver/native.
|
|
const { createOwnedBandwidthHarness, fixtureBinding } = await import("./bandwidth-admission-process.sql-helper.js")
|
|
const identity = createBandwidthSpeedRunIdentity("owned-crosscaller-run-" + randomUUID())
|
|
const firstHost = await createOwnedBandwidthHarness(f.owner), secondHost = await createOwnedBandwidthHarness(f.owner)
|
|
const first = fixtureBinding("owned-crosscaller-source-a", "owned-crosscaller-dest-a", { attemptId: identity.attemptId, operationId: identity.operationId, callerKind: "rest-speed" })
|
|
const second = fixtureBinding("owned-crosscaller-source-b", "owned-crosscaller-dest-b", { attemptId: identity.attemptId, operationId: identity.operationId, callerKind: "scheduled-speed" })
|
|
firstHost.register(first); secondHost.register(second)
|
|
const before = nativePosts
|
|
assert.equal((await firstHost.journal.reserve(first, f.scope())).outcome, "new")
|
|
firstHost.markNoSend(first)
|
|
await firstHost.journal.releaseBeforeDispatch(first, { from: "reserved", proof: "private-no-send" }, f.scope())
|
|
await assert.rejects(secondHost.journal.reserve(second, f.scope()))
|
|
assert.equal(nativePosts, before)
|
|
const rows = (await f.owner.query("SELECT caller_kind,state,resolution FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [identity.attemptId])).rows
|
|
assert.equal(rows.length, 1); assert.equal(rows[0].caller_kind, "rest-speed"); assert.equal(rows[0].state, "released"); assert.equal(rows[0].resolution, "proven_not_dispatched")
|
|
assert.equal(Number((await f.owner.query("SELECT count(*) AS n FROM mmapp_control.bandwidth_node_occupancy")).rows[0].n), 0)
|
|
runIdChecks.push("actual control journal global attempt PK rejects same run ID across rest-speed/scheduled-speed after first release; explicit synthetic fixture authority, no fake SQL driver")
|
|
runIdResults.push({ case: "crosscaller-actual-control-journal", callerKinds: ["rest-speed", "scheduled-speed"], preservedAttempts: rows.length, nativePosts: nativePosts - before,
|
|
syntheticAuthority: true, actualSql: true })
|
|
}
|
|
// Domain acceptance without an impractical native wait. The actual shared
|
|
// schema retains int32 duration; CPU workflow covers its full native flow.
|
|
assert.equal(speedTestCommandSchema.parse({ srcServerId: ids[0], dstServerId: ids[1], durationSec: 2147483647 }).durationSec, 2147483647)
|
|
mode = "native-drop"; beforeResponse = undefined; commitLossTarget = undefined; await restoreProbe()
|
|
const beforeNative = nativePosts, beforePing = pingPosts, beforeFailures = failureWrites
|
|
const { response: dropped, runId: droppedRunId } = await send(); assert.equal(dropped.statusCode, 502); assert.equal(dropped.json().outcome, "unknown")
|
|
assert.equal(nativePosts, beforeNative + 1); assert.equal(pingPosts, beforePing); assert.equal(failureWrites, beforeFailures); assert.equal((await history(droppedRunId)).length, 0)
|
|
assert.equal(Number((await f.owner.query("SELECT count(*) AS n FROM mmapp_control.bandwidth_node_occupancy")).rows[0].n), 2)
|
|
checks.push("real native socket drop leaves unknown-busy pair occupancy; no ping/failure row/replay")
|
|
assert.equal(protocolErrors, 0)
|
|
assert(socketTimeouts.some(value => value.path === "/rest/tool/bandwidth-test" && value.timeoutMs === 30000))
|
|
assert(socketTimeouts.some(value => value.path === "/rest/tool/ping" && value.timeoutMs === 20000))
|
|
assert.equal(checks.length, 6); assert.equal(runIdChecks.length, 4)
|
|
return { checks, runIdChecks, runIdResults, metrics, nativePosts, nativeGets, pingPosts, protocolErrors, failureWrites, credentialPayloadRows, historyCommitsObserved, commitLossInjected: historyPid !== undefined,
|
|
actualFastifyJwt: true, actualApplicationAndControlSql: true, actualNativeHttp: true, actualPrivateCompletionReceipt: true,
|
|
syntheticEnrollmentQuiescence: true, syntheticWireMapping: true, productionMappingVerified: false, runtimeAllCallersActivated: false, realChrCalls: 0 }
|
|
} catch (error) {
|
|
console.error("Owned wholeflow failure context:", JSON.stringify({ metrics, nativePosts, pingPosts, protocolErrors, failureWrites, sqlCalls, coreSqlCalls }))
|
|
throw error
|
|
} finally {
|
|
releaseConcurrent?.(); pool.query = poolQuery; PgClient.prototype.query = query; ClientRequest.prototype.setTimeout = setSocketTimeout
|
|
await app?.close()
|
|
for (const router of routers) { router.closeAllConnections(); if (router.listening) await new Promise<void>(resolve => router.close(() => resolve())) }
|
|
application.release(true); await f.close()
|
|
}
|
|
}
|