166 lines
11 KiB
TypeScript
166 lines
11 KiB
TypeScript
/** Process-bootstrap-owned execution path for fixed automatic scheduler jobs.
|
|
* No arbitrary callback, detached child or request JSON can clear a scope. */
|
|
import { AsyncLocalStorage, AsyncResource } from "node:async_hooks"
|
|
import { assertNoOperationConfigurationLockContext, captureOperationConfigurationLockHandoff } from "./operation-config-lock-lease.js"
|
|
import { assertNoConfigurationLockContext, captureConfigurationLockHandoff } from "./config-mutation-lock.js"
|
|
import { assertNoVerifiedConfigurationContext, assertConfigurationHandoffContext } from "./config-mutation-policy.js"
|
|
import { sharedDatabaseGate } from "./shared-database-gate.js"
|
|
import { databaseBackgroundWork } from "./database-background-work.js"
|
|
import { wireAlertEngineRunner } from "./alert-collector-hooks.js"
|
|
import type { SchedulerJobKey } from "./scheduler.js"
|
|
const JOBS = ["router_logs", "traffic", "uptime_resources", "uptime_ping", "uptime_speed", "servers_rest_ping", "gre_bgp", "internet_path", "certificates_renew", "backups", "geoip_update", "alert_engine", "router_logs_live", "router_logs_retention"] as const
|
|
type Job = typeof JOBS[number]
|
|
declare const ticketBrand: unique symbol
|
|
export type AutomaticSchedulerTicket = Readonly<{ [ticketBrand]: true }>
|
|
const tickets = new WeakMap<object, Readonly<{ job: Job; generation: number; handoff: Promise<void> | undefined }>>()
|
|
const pending = new WeakMap<object, Promise<void>>()
|
|
declare const authorityBrand: unique symbol
|
|
export interface AutomaticSchedulerAuthority { readonly [authorityBrand]: true }
|
|
export interface AutomaticSchedulerAuthorityScope { deadlineAt: number; signal?: AbortSignal }
|
|
export interface AutomaticSchedulerAuthoritySnapshot {
|
|
readonly job: Job
|
|
readonly generation: number
|
|
/** Process-local execution identity only. Never durable uniqueness or a
|
|
* restart-generation proof; admission still requires a fresh durable attempt. */
|
|
readonly executionId: string
|
|
readonly deadlineAt: number
|
|
/** Original process-local monotonic horizon; never reconstructed from a later
|
|
* wall-clock reading or treated as a durable cross-process timestamp. */
|
|
readonly monotonicDeadline: number
|
|
/** Always includes the private execution-completion/failure cancellation. */
|
|
readonly signal: AbortSignal
|
|
}
|
|
type Execution = { readonly job: Job; readonly generation: number; readonly executionId: string; readonly isCurrent: () => boolean;
|
|
readonly controller: AbortController; active: boolean; capability?: AutomaticSchedulerAuthority; scope?: Readonly<AutomaticSchedulerAuthorityScope>; end?: number }
|
|
const executions = new AsyncLocalStorage<Execution>()
|
|
const authorities = new WeakMap<object, Execution>()
|
|
let executionSequence = 0
|
|
export class AutomaticSchedulerAuthorityError extends Error {
|
|
readonly code = "AUTOMATIC_SCHEDULER_AUTHORITY_REFUSED"
|
|
constructor() { super("Current automatic scheduler authority unavailable") }
|
|
}
|
|
function refuseAuthority(): never { throw new AutomaticSchedulerAuthorityError() }
|
|
function authorityScope(raw: Readonly<AutomaticSchedulerAuthorityScope>): Readonly<AutomaticSchedulerAuthorityScope> {
|
|
if (!raw || typeof raw !== "object" || Array.isArray(raw)
|
|
|| !Object.hasOwn(raw, "deadlineAt")
|
|
|| "signal" in raw && !Object.hasOwn(raw, "signal")
|
|
|| Reflect.ownKeys(raw).length !== Object.keys(raw).length
|
|
|| Object.keys(raw).some(key => key !== "deadlineAt" && key !== "signal")
|
|
|| Object.values(Object.getOwnPropertyDescriptors(raw)).some(value => !("value" in value) || !value.enumerable)
|
|
|| !Number.isSafeInteger(raw.deadlineAt) || raw.signal !== undefined && !(raw.signal instanceof AbortSignal)) refuseAuthority()
|
|
return Object.freeze({ deadlineAt: raw.deadlineAt, signal: raw.signal })
|
|
}
|
|
function currentExecution(): Execution {
|
|
const state = executions.getStore()
|
|
if (!state?.active || !state.isCurrent()) return refuseAuthority()
|
|
return state
|
|
}
|
|
function liveAuthority(state: Execution, requested?: Readonly<AutomaticSchedulerAuthorityScope>, requestedEnd?: number): void {
|
|
if (currentExecution() !== state || state.controller.signal.aborted || !state.scope || state.end === undefined || state.scope.signal?.aborted
|
|
|| Date.now() >= state.scope.deadlineAt || globalThis.performance.now() >= state.end
|
|
|| requested && (requested.signal?.aborted || Date.now() >= requested.deadlineAt)
|
|
|| requestedEnd !== undefined && globalThis.performance.now() >= requestedEnd) refuseAuthority()
|
|
}
|
|
/** Capture only inside the genuine fixed-ticket execution. The first finite
|
|
* deadline/signal is immutable for that execution; recapture cannot renew it.
|
|
* No job, generation, execution ID, ticket or callback can be supplied here. */
|
|
export function captureAutomaticSchedulerAuthority(raw: Readonly<AutomaticSchedulerAuthorityScope>): AutomaticSchedulerAuthority {
|
|
const state = currentExecution(), scope = authorityScope(raw)
|
|
if (state.capability) {
|
|
liveAuthority(state)
|
|
if (scope.deadlineAt !== state.scope!.deadlineAt || scope.signal !== state.scope!.signal) refuseAuthority()
|
|
return state.capability
|
|
}
|
|
const wall = Date.now(), mono = globalThis.performance.now()
|
|
if (scope.signal?.aborted || scope.deadlineAt <= wall) refuseAuthority()
|
|
state.scope = scope; state.end = mono + (scope.deadlineAt - wall)
|
|
const capability = Object.freeze(Object.create(null)) as AutomaticSchedulerAuthority
|
|
state.capability = capability; authorities.set(capability, state)
|
|
liveAuthority(state); return capability
|
|
}
|
|
/** Inspect from the SAME live ALS execution, checking generation/lifetime on
|
|
* both sides of the await. Optional requested scope only clips this inspection;
|
|
* it never replaces/extends the original capture. Snapshot is data, not authority.
|
|
* Consumers must re-inspect at their own await/dispatch boundaries. */
|
|
export async function inspectAutomaticSchedulerAuthority(capability: unknown,
|
|
rawRequested?: Readonly<AutomaticSchedulerAuthorityScope>): Promise<Readonly<AutomaticSchedulerAuthoritySnapshot>> {
|
|
const state = capability && typeof capability === "object" ? authorities.get(capability) : undefined
|
|
if (!state) return refuseAuthority()
|
|
const requested = rawRequested === undefined ? undefined : authorityScope(rawRequested)
|
|
const end = requested ? Math.min(state.end!, globalThis.performance.now() + Math.max(0, requested.deadlineAt - Date.now())) : state.end
|
|
liveAuthority(state, requested, end)
|
|
await Promise.resolve()
|
|
liveAuthority(state, requested, end)
|
|
const signals = [...new Set([state.controller.signal, state.scope!.signal, requested?.signal].filter((value): value is AbortSignal => !!value))]
|
|
const signal = signals.length > 1 ? AbortSignal.any(signals) : signals[0]
|
|
return Object.freeze({ job: state.job, generation: state.generation, executionId: state.executionId,
|
|
deadlineAt: Math.min(state.scope!.deadlineAt, requested?.deadlineAt ?? state.scope!.deadlineAt), monotonicDeadline: state.end!, signal })
|
|
}
|
|
let root: AsyncResource | undefined
|
|
function fail(): never { throw new Error("Automatic scheduler bootstrap/ticket unavailable") }
|
|
export function initializeAutomaticSchedulerRoot(): void {
|
|
// Validate on EVERY call, including after initialization and in stale children.
|
|
assertNoOperationConfigurationLockContext(); assertNoConfigurationLockContext()
|
|
assertNoVerifiedConfigurationContext(); sharedDatabaseGate.assertNoRequestContext()
|
|
if (!root) {
|
|
root = new AsyncResource("mmapp-automatic-scheduler-root")
|
|
// Bootstrap wires the fixed hook without starting any timer. A refresh
|
|
// replaces this initial generation ticket alongside its intervals.
|
|
const alert = createAutomaticSchedulerTicket("alert_engine", 0)
|
|
wireAlertEngineRunner(() => runAutomaticSchedulerTicket(alert))
|
|
}
|
|
}
|
|
export function requireAutomaticSchedulerRoot(): void { if (!root) fail() }
|
|
export function createAutomaticSchedulerTicket(job: Job, generation: number): AutomaticSchedulerTicket {
|
|
requireAutomaticSchedulerRoot()
|
|
assertConfigurationHandoffContext()
|
|
if (!JOBS.includes(job) || !Number.isSafeInteger(generation) || generation < 0) fail()
|
|
const ticket = Object.freeze(Object.create(null)) as AutomaticSchedulerTicket
|
|
tickets.set(ticket, Object.freeze({ job, generation, handoff: Promise.all([captureOperationConfigurationLockHandoff(), captureConfigurationLockHandoff()]).then(() => {}) })); return ticket
|
|
}
|
|
export async function runAutomaticSchedulerTicket(ticket: AutomaticSchedulerTicket): Promise<void> {
|
|
const entry = ticket && typeof ticket === "object" ? tickets.get(ticket) : undefined
|
|
if (!root || !entry) fail()
|
|
const current = pending.get(ticket)
|
|
if (current) return current
|
|
const owner = root
|
|
// This root was captured ONLY at trusted bootstrap. Each job obtains a new
|
|
// DB request admission and is separately tracked for maintenance draining.
|
|
const work = owner.runInAsyncScope(async () => {
|
|
// Immediate automatic jobs must not race the still-held source operation's
|
|
// advisory locks. The parent schedules this handoff without awaiting it.
|
|
// Waiting is NOT tracked/request work: that would deadlock a later drain
|
|
// inside this same parent operation against its own completion promise.
|
|
await entry.handoff
|
|
const scheduler = await import("./scheduler.js")
|
|
// Exclusive maintenance may outlive parent completion. This wait owns no
|
|
// request/background lease. Repeated ticks coalesce on this same ticket.
|
|
while (sharedDatabaseGate.isPaused()) {
|
|
await sharedDatabaseGate.whenOpen()
|
|
if (!scheduler.isAutomaticSchedulerGenerationCurrent(entry.generation)) return
|
|
}
|
|
if (!scheduler.isAutomaticSchedulerGenerationCurrent(entry.generation)) return
|
|
// No await between paused observation and counted request admission.
|
|
return databaseBackgroundWork.track(sharedDatabaseGate.request(async () => {
|
|
if (!scheduler.isAutomaticSchedulerGenerationCurrent(entry.generation)) return
|
|
const execute = async (work: () => Promise<void>) => {
|
|
if (!scheduler.isAutomaticSchedulerGenerationCurrent(entry.generation)) return
|
|
const execution: Execution = { job: entry.job, generation: entry.generation,
|
|
executionId: `automatic-process-execution-${++executionSequence}`,
|
|
isCurrent: () => scheduler.isAutomaticSchedulerGenerationCurrent(entry.generation), controller: new AbortController(), active: true }
|
|
return executions.run(execution, async () => {
|
|
try { await work() } finally { execution.active = false; execution.controller.abort() }
|
|
})
|
|
}
|
|
if (entry.job === "router_logs_live" || entry.job === "router_logs_retention") {
|
|
const logs = await import("./router-logs-collector.js")
|
|
if (!scheduler.isAutomaticSchedulerGenerationCurrent(entry.generation)) return
|
|
if (entry.job === "router_logs_live") await execute(() => logs.runRouterLogsLiveOnce())
|
|
else await execute(() => logs.runRouterLogsRetentionOnce())
|
|
} else await execute(() => scheduler.executeAutomaticSchedulerJob(entry.job as SchedulerJobKey, entry.generation))
|
|
}))
|
|
})
|
|
pending.set(ticket, work)
|
|
try { await work } finally { if (pending.get(ticket) === work) pending.delete(ticket) }
|
|
}
|