/** Process-bootstrap-owned execution path for fixed automatic scheduler jobs. * No arbitrary callback, detached child or request JSON can clear a scope. */ import { AsyncLocalStorage, AsyncResource } from "node:async_hooks" import { assertNoOperationConfigurationLockContext, captureOperationConfigurationLockHandoff } from "./operation-config-lock-lease.js" import { assertNoConfigurationLockContext, captureConfigurationLockHandoff } from "./config-mutation-lock.js" import { assertNoVerifiedConfigurationContext, assertConfigurationHandoffContext } from "./config-mutation-policy.js" import { sharedDatabaseGate } from "./shared-database-gate.js" import { databaseBackgroundWork } from "./database-background-work.js" import { wireAlertEngineRunner } from "./alert-collector-hooks.js" import type { SchedulerJobKey } from "./scheduler.js" const JOBS = ["router_logs", "traffic", "uptime_resources", "uptime_ping", "uptime_speed", "servers_rest_ping", "gre_bgp", "internet_path", "certificates_renew", "backups", "geoip_update", "alert_engine", "router_logs_live", "router_logs_retention"] as const type Job = typeof JOBS[number] declare const ticketBrand: unique symbol export type AutomaticSchedulerTicket = Readonly<{ [ticketBrand]: true }> const tickets = new WeakMap | undefined }>>() const pending = new WeakMap>() declare const authorityBrand: unique symbol export interface AutomaticSchedulerAuthority { readonly [authorityBrand]: true } export interface AutomaticSchedulerAuthorityScope { deadlineAt: number; signal?: AbortSignal } export interface AutomaticSchedulerAuthoritySnapshot { readonly job: Job readonly generation: number /** Process-local execution identity only. Never durable uniqueness or a * restart-generation proof; admission still requires a fresh durable attempt. */ readonly executionId: string readonly deadlineAt: number /** Original process-local monotonic horizon; never reconstructed from a later * wall-clock reading or treated as a durable cross-process timestamp. */ readonly monotonicDeadline: number /** Always includes the private execution-completion/failure cancellation. */ readonly signal: AbortSignal } type Execution = { readonly job: Job; readonly generation: number; readonly executionId: string; readonly isCurrent: () => boolean; readonly controller: AbortController; active: boolean; capability?: AutomaticSchedulerAuthority; scope?: Readonly; end?: number } const executions = new AsyncLocalStorage() const authorities = new WeakMap() let executionSequence = 0 export class AutomaticSchedulerAuthorityError extends Error { readonly code = "AUTOMATIC_SCHEDULER_AUTHORITY_REFUSED" constructor() { super("Current automatic scheduler authority unavailable") } } function refuseAuthority(): never { throw new AutomaticSchedulerAuthorityError() } function authorityScope(raw: Readonly): Readonly { if (!raw || typeof raw !== "object" || Array.isArray(raw) || !Object.hasOwn(raw, "deadlineAt") || "signal" in raw && !Object.hasOwn(raw, "signal") || Reflect.ownKeys(raw).length !== Object.keys(raw).length || Object.keys(raw).some(key => key !== "deadlineAt" && key !== "signal") || Object.values(Object.getOwnPropertyDescriptors(raw)).some(value => !("value" in value) || !value.enumerable) || !Number.isSafeInteger(raw.deadlineAt) || raw.signal !== undefined && !(raw.signal instanceof AbortSignal)) refuseAuthority() return Object.freeze({ deadlineAt: raw.deadlineAt, signal: raw.signal }) } function currentExecution(): Execution { const state = executions.getStore() if (!state?.active || !state.isCurrent()) return refuseAuthority() return state } function liveAuthority(state: Execution, requested?: Readonly, requestedEnd?: number): void { if (currentExecution() !== state || state.controller.signal.aborted || !state.scope || state.end === undefined || state.scope.signal?.aborted || Date.now() >= state.scope.deadlineAt || globalThis.performance.now() >= state.end || requested && (requested.signal?.aborted || Date.now() >= requested.deadlineAt) || requestedEnd !== undefined && globalThis.performance.now() >= requestedEnd) refuseAuthority() } /** Capture only inside the genuine fixed-ticket execution. The first finite * deadline/signal is immutable for that execution; recapture cannot renew it. * No job, generation, execution ID, ticket or callback can be supplied here. */ export function captureAutomaticSchedulerAuthority(raw: Readonly): AutomaticSchedulerAuthority { const state = currentExecution(), scope = authorityScope(raw) if (state.capability) { liveAuthority(state) if (scope.deadlineAt !== state.scope!.deadlineAt || scope.signal !== state.scope!.signal) refuseAuthority() return state.capability } const wall = Date.now(), mono = globalThis.performance.now() if (scope.signal?.aborted || scope.deadlineAt <= wall) refuseAuthority() state.scope = scope; state.end = mono + (scope.deadlineAt - wall) const capability = Object.freeze(Object.create(null)) as AutomaticSchedulerAuthority state.capability = capability; authorities.set(capability, state) liveAuthority(state); return capability } /** Inspect from the SAME live ALS execution, checking generation/lifetime on * both sides of the await. Optional requested scope only clips this inspection; * it never replaces/extends the original capture. Snapshot is data, not authority. * Consumers must re-inspect at their own await/dispatch boundaries. */ export async function inspectAutomaticSchedulerAuthority(capability: unknown, rawRequested?: Readonly): Promise> { const state = capability && typeof capability === "object" ? authorities.get(capability) : undefined if (!state) return refuseAuthority() const requested = rawRequested === undefined ? undefined : authorityScope(rawRequested) const end = requested ? Math.min(state.end!, globalThis.performance.now() + Math.max(0, requested.deadlineAt - Date.now())) : state.end liveAuthority(state, requested, end) await Promise.resolve() liveAuthority(state, requested, end) const signals = [...new Set([state.controller.signal, state.scope!.signal, requested?.signal].filter((value): value is AbortSignal => !!value))] const signal = signals.length > 1 ? AbortSignal.any(signals) : signals[0] return Object.freeze({ job: state.job, generation: state.generation, executionId: state.executionId, deadlineAt: Math.min(state.scope!.deadlineAt, requested?.deadlineAt ?? state.scope!.deadlineAt), monotonicDeadline: state.end!, signal }) } let root: AsyncResource | undefined function fail(): never { throw new Error("Automatic scheduler bootstrap/ticket unavailable") } export function initializeAutomaticSchedulerRoot(): void { // Validate on EVERY call, including after initialization and in stale children. assertNoOperationConfigurationLockContext(); assertNoConfigurationLockContext() assertNoVerifiedConfigurationContext(); sharedDatabaseGate.assertNoRequestContext() if (!root) { root = new AsyncResource("mmapp-automatic-scheduler-root") // Bootstrap wires the fixed hook without starting any timer. A refresh // replaces this initial generation ticket alongside its intervals. const alert = createAutomaticSchedulerTicket("alert_engine", 0) wireAlertEngineRunner(() => runAutomaticSchedulerTicket(alert)) } } export function requireAutomaticSchedulerRoot(): void { if (!root) fail() } export function createAutomaticSchedulerTicket(job: Job, generation: number): AutomaticSchedulerTicket { requireAutomaticSchedulerRoot() assertConfigurationHandoffContext() if (!JOBS.includes(job) || !Number.isSafeInteger(generation) || generation < 0) fail() const ticket = Object.freeze(Object.create(null)) as AutomaticSchedulerTicket tickets.set(ticket, Object.freeze({ job, generation, handoff: Promise.all([captureOperationConfigurationLockHandoff(), captureConfigurationLockHandoff()]).then(() => {}) })); return ticket } export async function runAutomaticSchedulerTicket(ticket: AutomaticSchedulerTicket): Promise { const entry = ticket && typeof ticket === "object" ? tickets.get(ticket) : undefined if (!root || !entry) fail() const current = pending.get(ticket) if (current) return current const owner = root // This root was captured ONLY at trusted bootstrap. Each job obtains a new // DB request admission and is separately tracked for maintenance draining. const work = owner.runInAsyncScope(async () => { // Immediate automatic jobs must not race the still-held source operation's // advisory locks. The parent schedules this handoff without awaiting it. // Waiting is NOT tracked/request work: that would deadlock a later drain // inside this same parent operation against its own completion promise. await entry.handoff const scheduler = await import("./scheduler.js") // Exclusive maintenance may outlive parent completion. This wait owns no // request/background lease. Repeated ticks coalesce on this same ticket. while (sharedDatabaseGate.isPaused()) { await sharedDatabaseGate.whenOpen() if (!scheduler.isAutomaticSchedulerGenerationCurrent(entry.generation)) return } if (!scheduler.isAutomaticSchedulerGenerationCurrent(entry.generation)) return // No await between paused observation and counted request admission. return databaseBackgroundWork.track(sharedDatabaseGate.request(async () => { if (!scheduler.isAutomaticSchedulerGenerationCurrent(entry.generation)) return const execute = async (work: () => Promise) => { if (!scheduler.isAutomaticSchedulerGenerationCurrent(entry.generation)) return const execution: Execution = { job: entry.job, generation: entry.generation, executionId: `automatic-process-execution-${++executionSequence}`, isCurrent: () => scheduler.isAutomaticSchedulerGenerationCurrent(entry.generation), controller: new AbortController(), active: true } return executions.run(execution, async () => { try { await work() } finally { execution.active = false; execution.controller.abort() } }) } if (entry.job === "router_logs_live" || entry.job === "router_logs_retention") { const logs = await import("./router-logs-collector.js") if (!scheduler.isAutomaticSchedulerGenerationCurrent(entry.generation)) return if (entry.job === "router_logs_live") await execute(() => logs.runRouterLogsLiveOnce()) else await execute(() => logs.runRouterLogsRetentionOnce()) } else await execute(() => scheduler.executeAutomaticSchedulerJob(entry.job as SchedulerJobKey, entry.generation)) })) }) pending.set(ticket, work) try { await work } finally { if (pending.get(ticket) === work) pending.delete(ticket) } }