177 lines
14 KiB
TypeScript
177 lines
14 KiB
TypeScript
/** Explicit complete speed workflow for genuine REST/scheduler originals.
|
|
* No default/runtime registration or arbitrary run/send/SQL callback injection.
|
|
* The peer host owns privileged bandwidth dispatch and durable occupancy; this
|
|
* module preserves native rows/rates, after-ping and transactional run history.
|
|
* Production activation additionally requires reviewed held mapping/all callers.
|
|
*/
|
|
import { and, eq } from "drizzle-orm"
|
|
import { isDeepStrictEqual } from "node:util"
|
|
import { performance } from "node:perf_hooks"
|
|
import { z } from "zod"
|
|
import { db } from "../db/index.js"
|
|
import { uptimeSpeedProbes, uptimeSpeedTestRuns } from "../db/schema.js"
|
|
import type { SpeedTestRunResult } from "./uptime-speed-service.js"
|
|
import type { BandwidthScope } from "./bandwidth-admission.js"
|
|
import { assertBandwidthPeerExecutionHost, readBandwidthPeerCompletion, BandwidthPeerExecutionHostError, type createBandwidthPeerExecutionHost } from "./bandwidth-peer-execution-host.js"
|
|
import { assertRestSpeedBandwidthOriginalProvider, type createRestSpeedBandwidthOriginalProvider } from "./bandwidth-rest-speed-original.js"
|
|
import { assertScheduledSpeedBandwidthOriginalProvider, type createScheduledSpeedBandwidthOriginalProvider } from "./bandwidth-scheduled-speed-original.js"
|
|
import { assertMcpSpeedBandwidthOriginalProvider, assertMcpSpeedBandwidthWorkflowRequest, type createMcpSpeedBandwidthOriginalProvider } from "../modules/mcp/speed-bandwidth-original.js"
|
|
import { MikrotikClient } from "./mikrotik.js"
|
|
import { parsePing } from "./uptime-collector.js"
|
|
import { assertBandwidthTestCompleted } from "./bandwidth-test-status.js"
|
|
import { readBandwidthTestRates } from "./bandwidth-test-rates.js"
|
|
import { persistSpeedRun, SpeedRunPersistenceUnknown, type SpeedProbeContext } from "./speed-run-persistence.js"
|
|
|
|
type Originals = ReturnType<typeof createRestSpeedBandwidthOriginalProvider> | ReturnType<typeof createScheduledSpeedBandwidthOriginalProvider> | ReturnType<typeof createMcpSpeedBandwidthOriginalProvider>
|
|
type SpeedCaller = "rest-speed" | "scheduled-speed" | "mcp-speed"
|
|
type WriteTransaction = Parameters<Parameters<typeof db.transaction>[0]>[0]
|
|
export interface BandwidthSpeedWorkflowDependencies {
|
|
originals: Originals
|
|
peerHost: ReturnType<typeof createBandwidthPeerExecutionHost>
|
|
}
|
|
/** Existing REST/scheduler catches recognize this superclass and MUST NOT write
|
|
* a second failure row or automatically replay a possibly sent native command. */
|
|
export class BandwidthSpeedWorkflowUnknown extends SpeedRunPersistenceUnknown {
|
|
readonly automaticReplay = false
|
|
constructor(runId: string, cause?: unknown) {
|
|
super(runId, cause)
|
|
this.message = `Результат сетевого теста неизвестен. Проверьте историю по runId ${runId} перед повтором`
|
|
}
|
|
}
|
|
const newRunId = () => `sr-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
|
|
const scheduledRequestSchema = z.object({ command: z.unknown(), job: z.object({
|
|
job: z.literal("uptime_speed"), generation: z.number().int().nonnegative().safe(), executionId: z.string().uuid(),
|
|
deadlineAt: z.number().int().positive().safe(), monotonicDeadline: z.number().finite().nonnegative(),
|
|
}).strict() }).strict()
|
|
function matchesOriginalRequest(caller: SpeedCaller, intent: Awaited<ReturnType<Originals["readOriginal"]>>, command: Parameters<typeof assertMcpSpeedBandwidthWorkflowRequest>[1], monotonicDeadline: number) {
|
|
if (caller === "rest-speed") return isDeepStrictEqual(intent.request, { command })
|
|
if (caller === "mcp-speed") {
|
|
try { assertMcpSpeedBandwidthWorkflowRequest(intent.request, command, intent.expiresAt); return true } catch { return false }
|
|
}
|
|
const parsed = scheduledRequestSchema.safeParse(intent.request)
|
|
// The genuine provider authenticates this exact sealed job on every read.
|
|
// Retain it here rather than projecting scheduled requests into REST shape.
|
|
return parsed.success && isDeepStrictEqual(parsed.data.command, command) && parsed.data.job.deadlineAt === intent.expiresAt
|
|
&& parsed.data.job.monotonicDeadline >= monotonicDeadline
|
|
}
|
|
function workflowScope(raw: Readonly<BandwidthScope>): Readonly<BandwidthScope> {
|
|
if (!raw || Object.getPrototypeOf(raw) !== Object.prototype || Reflect.ownKeys(raw).length !== Object.keys(raw).length
|
|
|| Object.keys(raw).some(key => key !== "deadlineAt" && key !== "signal")
|
|
|| Object.values(Object.getOwnPropertyDescriptors(raw)).some(value => !("value" in value) || !value.enumerable)
|
|
|| !Number.isSafeInteger(raw.deadlineAt) || raw.signal !== undefined && !(raw.signal instanceof AbortSignal)) throw new Error("Speed workflow scope unavailable")
|
|
return Object.freeze({ deadlineAt: raw.deadlineAt, signal: raw.signal })
|
|
}
|
|
function whereProbe(target: SpeedProbeContext) {
|
|
return and(eq(uptimeSpeedProbes.id, target.id), eq(uptimeSpeedProbes.srcServerId, target.srcServerId), eq(uptimeSpeedProbes.dstServerId, target.dstServerId),
|
|
eq(uptimeSpeedProbes.srcInterface, target.srcInterface), eq(uptimeSpeedProbes.dstInterface, target.dstInterface),
|
|
eq(uptimeSpeedProbes.protocol, target.protocol), eq(uptimeSpeedProbes.direction, target.direction), eq(uptimeSpeedProbes.durationSec, target.durationSec))
|
|
}
|
|
|
|
/** Intended service callsite: trusted bootstrap constructs ONE workflow for each
|
|
* genuine original provider and its actual peer host; REST passes its captured
|
|
* request original, scheduler passes its fixed-job selected-probe original.
|
|
* run accepts NO command JSON: every parameter comes from that genuine original.
|
|
* Failure publication remains the existing caller's recordSpeedTestFailure;
|
|
* persistence/native unknown subclasses suppress that second write as before.
|
|
*/
|
|
export function createBandwidthSpeedWorkflow(dependencies: BandwidthSpeedWorkflowDependencies) {
|
|
const originals = dependencies.originals
|
|
let callerKind: SpeedCaller
|
|
try { assertRestSpeedBandwidthOriginalProvider(originals); callerKind = "rest-speed" }
|
|
catch {
|
|
try { assertScheduledSpeedBandwidthOriginalProvider(originals); callerKind = "scheduled-speed" }
|
|
catch { assertMcpSpeedBandwidthOriginalProvider(originals); callerKind = "mcp-speed" }
|
|
}
|
|
assertBandwidthPeerExecutionHost(dependencies.peerHost)
|
|
const readOriginal = originals.readOriginal.bind(originals), readContext = originals.readSpeedContext.bind(originals), readPeer = originals.readPeer.bind(originals)
|
|
const readLifetime = originals.readLifetime.bind(originals)
|
|
const readPeerAfterCompletion = originals.readPeerAfterCompletion.bind(originals)
|
|
const execute = dependencies.peerHost.execute.bind(dependencies.peerHost)
|
|
return Object.freeze({
|
|
async run(originalContext: unknown, rawScope: Readonly<BandwidthScope>): Promise<SpeedTestRunResult> {
|
|
const input = workflowScope(rawScope)
|
|
const lifetime = await readLifetime(originalContext, input)
|
|
if (!Number.isSafeInteger(lifetime.expiresAt) || !Number.isFinite(lifetime.monotonicDeadline) || !(lifetime.signal instanceof AbortSignal)) throw new Error("Speed workflow lifetime unavailable")
|
|
const signal = input.signal ? AbortSignal.any([lifetime.signal, input.signal]) : lifetime.signal
|
|
// Translate the SAME original monotonic end into a current wall deadline.
|
|
// Recompute at dispatch boundaries so wall rollback during provider I/O
|
|
// cannot turn the original remaining lifetime into a longer native lease.
|
|
const currentScope = (): Readonly<BandwidthScope> => {
|
|
const now = Date.now(), remaining = lifetime.monotonicDeadline - performance.now()
|
|
if (signal.aborted || remaining <= 0 || now >= Math.min(input.deadlineAt, lifetime.expiresAt)) throw new Error("Speed workflow lifetime unavailable")
|
|
return Object.freeze({ deadlineAt: Math.floor(Math.min(input.deadlineAt, lifetime.expiresAt, now + remaining)), signal })
|
|
}
|
|
let scope = currentScope()
|
|
const intent = await readOriginal(originalContext, scope), context = await readContext(originalContext, scope)
|
|
const command = context.command
|
|
if (intent.callerKind !== callerKind || intent.sourceServerId !== command.srcServerId || intent.credentialServerId !== command.dstServerId
|
|
|| intent.actualRemotePeer.kind !== "server" || intent.actualRemotePeer.serverId !== command.dstServerId
|
|
|| !matchesOriginalRequest(callerKind, intent, command, lifetime.monotonicDeadline) || intent.durationSec !== command.durationSec || intent.socketTimeoutMs !== 30000
|
|
|| intent.nativeBody.address !== context.address || intent.nativeBody.duration !== `${command.durationSec}s`
|
|
|| intent.nativeBody.protocol !== command.protocol || intent.nativeBody.direction !== command.direction) throw new Error("Speed workflow original context unavailable")
|
|
const sourceRef = Object.freeze({ kind: "server" as const, serverId: command.srcServerId })
|
|
const source = await readPeer(originalContext, sourceRef, scope)
|
|
const sourceClient = new MikrotikClient(source.endpoint, command.srcServerId, { requireBandwidthPermit: true })
|
|
let native: Awaited<ReturnType<typeof execute>>
|
|
try { scope = currentScope(); native = await execute(originalContext, scope) }
|
|
catch (error) {
|
|
if (error instanceof BandwidthPeerExecutionHostError && error.code === "OUTCOME_UNKNOWN") throw new BandwidthSpeedWorkflowUnknown(command.runId || newRunId(), error)
|
|
throw error
|
|
}
|
|
// The actual peer host returns only after native terminal confirmation and
|
|
// durable completion ACK. Keep complete raw rows, including extra fields.
|
|
try {
|
|
if (!isDeepStrictEqual(readBandwidthPeerCompletion(native, originalContext), intent)) throw new Error("Speed workflow completion context changed")
|
|
} catch (error) { throw new BandwidthSpeedWorkflowUnknown(command.runId || newRunId(), error) }
|
|
if (native.remoteCompletionConfirmed !== true) throw new BandwidthSpeedWorkflowUnknown(command.runId || newRunId())
|
|
const rows = native.raw
|
|
try { assertBandwidthTestCompleted(rows) } catch (error) { throw new BandwidthSpeedWorkflowUnknown(command.runId || newRunId(), error) }
|
|
const { txAvgMbps: txFinal, rxAvgMbps: rxFinal, txSeries, rxSeries } = readBandwidthTestRates(rows)
|
|
await new Promise<void>(resolve => setTimeout(resolve, 3000))
|
|
const pingNow = new Date().toISOString()
|
|
let afterBtPing: SpeedTestRunResult["afterBtPing"] = { rttMs: null, lossPct: null, error: null }
|
|
try {
|
|
const fresh = async () => {
|
|
if (!isDeepStrictEqual(await readPeerAfterCompletion(originalContext, native, sourceRef, currentScope()), source)) throw new Error("Speed workflow source changed")
|
|
}
|
|
await fresh()
|
|
const pingRows = await sourceClient.ping(context.address, 4, command.srcInterface || undefined, { interval: "1s", signal: scope.signal, beforeDispatch: fresh })
|
|
const parsed = parsePing(pingRows)
|
|
if (!parsed.measured) throw new Error("RouterOS не вернул подтверждённые результаты контрольного Ping")
|
|
afterBtPing = { rttMs: parsed.avgRtt, lossPct: parsed.loss, error: null }
|
|
} catch (error) {
|
|
afterBtPing = { rttMs: null, lossPct: null, error: error instanceof Error ? error.message : String(error) }
|
|
}
|
|
const runId = command.runId || newRunId(), probeId = command.probeId || ""
|
|
const target: SpeedProbeContext = { id: probeId, srcServerId: command.srcServerId, dstServerId: command.dstServerId,
|
|
srcInterface: command.srcInterface, dstInterface: command.dstInterface, protocol: command.protocol, direction: command.direction, durationSec: command.durationSec }
|
|
try {
|
|
await persistSpeedRun<WriteTransaction, SpeedProbeContext>(probeId ? target : null, {
|
|
transaction: work => db.transaction(work),
|
|
updateProbe: async (tx, current) => {
|
|
const updated = await tx.update(uptimeSpeedProbes).set({ lastRunAt: pingNow, lastTxAvgMbps: txFinal, lastRxAvgMbps: rxFinal,
|
|
lastStatus: "done", lastError: "", lastPingRttMs: afterBtPing.rttMs, lastPingLossPct: afterBtPing.lossPct,
|
|
lastPingAt: pingNow, lastPingError: afterBtPing.error ?? "", updatedAt: pingNow,
|
|
}).where(whereProbe(current)).returning({ id: uptimeSpeedProbes.id })
|
|
return updated.length === 1
|
|
},
|
|
insertRun: async (tx, associatedProbeId) => {
|
|
await tx.insert(uptimeSpeedTestRuns).values({ id: runId, probeId: associatedProbeId,
|
|
srcServerId: command.srcServerId, dstServerId: command.dstServerId, srcInterface: command.srcInterface || "", dstInterface: command.dstInterface || "",
|
|
srcAddress: context.srcAddress || source.endpoint.host, dstAddress: context.address,
|
|
srcInterfaceAddress: context.srcInterfaceAddress, dstInterfaceAddress: context.dstInterfaceAddress,
|
|
protocol: command.protocol, direction: command.direction, durationSec: command.durationSec,
|
|
txAvgMbps: txFinal, rxAvgMbps: rxFinal, pingRttMs: afterBtPing.rttMs, pingLossPct: afterBtPing.lossPct,
|
|
pingError: afterBtPing.error ?? null, status: "done", error: null, createdAt: pingNow })
|
|
},
|
|
})
|
|
} catch (error) { throw new SpeedRunPersistenceUnknown(runId, error) }
|
|
return { createdAt: pingNow, runId, srcInterface: command.srcInterface || null, dstInterface: command.dstInterface || null,
|
|
address: context.address, srcAddress: context.srcAddress || source.endpoint.host, dstAddress: context.address,
|
|
srcInterfaceAddress: context.srcInterfaceAddress, dstInterfaceAddress: context.dstInterfaceAddress,
|
|
protocol: command.protocol, direction: command.direction, durationSec: command.durationSec,
|
|
txAvgMbps: txFinal, rxAvgMbps: rxFinal, txSeries, rxSeries, raw: rows as Array<Record<string, string>>, afterBtPing }
|
|
},
|
|
})
|
|
}
|