Files
MikrotikManager/backend/src/services/scheduler.ts
T

519 lines
19 KiB
TypeScript

/**
* In-process планировщик: отдельный интервал на job (`refreshScheduler`).
*
* **Приоритеты при конкуренции** (для дальнейшего per-server mutex; сейчас зафиксировано в дизайне):
* 1. `uptime_ping` — выше (короткий критичный сигнал).
* 2. `traffic`, `uptime_resources`, `servers_rest_ping` — средний (REST `/system/identity` по каталогу).
* 3. `uptime_speed` — ниже (BW-test, тяжёлый); локи узлов — `withBtestNodeLocks` в speed-сервисе.
*
* **Оповещения (`alert_engine`):** читают PostgreSQL после джоб сбора (см. `buildSignalSnapshot`), в т.ч.
* `gre_bgp` → snapshot в `scheduler_runs.result_json` (`greTunnels` / `bgpPeers`). После успешного завершения джоб
* `traffic`, `uptime_*`, `servers_rest_ping`, `gre_bgp` планируется **дополнительный** прогон движка
* (debounce), см. [`alert-collector-hooks.ts`](./alert-collector-hooks.ts); любой другой писатель сэмплов
* для снимка оповещений тоже должен вызывать `scheduleAlertEngineAfterDataCollectors()` после коммита.
*
* **Кастомные job без произвольного кода (MVP):** пер-тайминг ping в `uptime_probes.interval_sec`;
* таблица `scheduler_job_instances` (kind + config JSON) — при необходимости следующий этап.
*/
import { desc, eq, lt } from "drizzle-orm"
import { db } from "../db/index.js"
import { events, schedulerRuns } from "../db/schema.js"
import type { SchedulerRunSnapshot } from "../types/scheduler-run-snapshot.js"
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
import {
collectServersRestPingOnce,
getServersApiPingSettings,
} from "./servers-rest-ping-collector.js"
import { collectTrafficOnce, getTrafficSettings } from "./traffic-collector.js"
import {
collectPingProbesOnce,
collectResourceSamplesOnce,
getSettings as getUptimeSettings,
} from "./uptime-collector.js"
import { runScheduledSpeedProbesOnce } from "./uptime-speed-service.js"
import { collectGreBgpSnapshotOnce } from "./gre-bgp-snapshot-collector.js"
import { GRE_BGP_INTERVAL_MS } from "./gre-bgp-timing.js"
import { runAlertEngineOnce } from "./alert-engine/run-once.js"
import {
clearAlertCollectorHooksTimer,
scheduleAlertEngineAfterDataCollectors,
wireAlertEngineRunner,
} from "./alert-collector-hooks.js"
import {
collectInternetPathSnapshotOnce,
getInternetPathSettings,
} from "./internet-path-collector.js"
import { collectCertificatesRenewOnce } from "./certificate-renew-collector.js"
import { getCertificateRenewSettings } from "./certificates-service.js"
import { collectScheduledBackupsOnce } from "./backup-scheduler-collector.js"
import { getBackupScheduleSettings } from "./backup-service.js"
import { collectGeoipUpdateOnce } from "./geoip-update-collector.js"
import { getGeoipSettings } from "./geoip-settings.js"
import { collectRouterLogsOnce, getLogSettings, startLogsLiveTimer, stopLogsLiveTimer } from "./router-logs-collector.js"
import {
endSchedulerJob,
isSchedulerJobRunning,
tryBeginSchedulerJob,
isSchedulerJobsPaused,
pauseSchedulerJobsAndWait,
resumeSchedulerJobs,
} from "./scheduler-running.js"
import { appendEvent } from "../modules/events/service/events-service.js"
import { runRecordedSchedulerJob, type SchedulerRunReceipt } from "./scheduler-recorded-run.js"
export const JOB_KEYS = [
"router_logs",
"traffic",
"servers_rest_ping",
"uptime_resources",
"uptime_ping",
"uptime_speed",
"internet_path",
"gre_bgp",
"certificates_renew",
"backups",
"geoip_update",
"alert_engine",
] as const
export type SchedulerJobKey = (typeof JOB_KEYS)[number]
import { createAutomaticSchedulerTicket, runAutomaticSchedulerTicket, requireAutomaticSchedulerRoot } from "./scheduler-automatic-root.js"
const timers = new Map<string, ReturnType<typeof setInterval>>()
let schedulerGeneration = 0
const RUN_LOG_RETENTION_MS = 30 * 24 * 60 * 60 * 1000
const QUIET_SCHEDULER_OK_JOBS = new Set<SchedulerJobKey>([
"router_logs",
"traffic",
"servers_rest_ping",
"uptime_resources",
"uptime_ping",
"uptime_speed",
"gre_bgp",
"alert_engine",
])
export function shouldAppendSchedulerOkEvent(jobKey: SchedulerJobKey): boolean {
return !QUIET_SCHEDULER_OK_JOBS.has(jobKey)
}
function newRunId(): string {
return `sch-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`
}
async function appendSchedulerRun(row: {
jobKey: string
startedAt: string
finishedAt: string
status: "ok" | "error"
error: string | null
durationMs: number
result?: SchedulerRunSnapshot | null
}) {
const cutoff = new Date(Date.now() - RUN_LOG_RETENTION_MS).toISOString()
await db.transaction(async (tx) => {
await tx.insert(schedulerRuns).values({
id: newRunId(),
jobKey: row.jobKey,
startedAt: row.startedAt,
finishedAt: row.finishedAt,
status: row.status,
error: row.error,
durationMs: row.durationMs,
resultJson: row.result ?? null,
})
await tx.delete(schedulerRuns).where(lt(schedulerRuns.finishedAt, cutoff))
await tx.delete(events).where(lt(events.createdAt, cutoff))
})
}
async function runSchedulerJobBody(jobKey: SchedulerJobKey): Promise<SchedulerRunReceipt> {
return runRecordedSchedulerJob({
jobKey,
collect: async () => {
let snapshot: SchedulerRunSnapshot | undefined
switch (jobKey) {
case "router_logs":
snapshot = await collectRouterLogsOnce()
break
case "traffic":
snapshot = await collectTrafficOnce()
break
case "uptime_resources":
snapshot = await collectResourceSamplesOnce()
break
case "uptime_ping":
snapshot = await collectPingProbesOnce()
break
case "uptime_speed":
snapshot = await runScheduledSpeedProbesOnce()
break
case "servers_rest_ping":
snapshot = await collectServersRestPingOnce()
break
case "gre_bgp":
snapshot = await collectGreBgpSnapshotOnce()
break
case "internet_path":
snapshot = await collectInternetPathSnapshotOnce()
break
case "certificates_renew":
snapshot = await collectCertificatesRenewOnce()
break
case "backups":
snapshot = await collectScheduledBackupsOnce()
break
case "geoip_update":
snapshot = await collectGeoipUpdateOnce()
break
case "alert_engine": {
const r = await runAlertEngineOnce()
snapshot = {
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
job: "alert_engine",
sampledAt: r.sampledAt,
rulesChecked: r.rulesChecked,
standaloneFires: r.standaloneFires,
groupFires: r.groupFires,
skippedNoTelegram: r.skippedNoTelegram,
...(r.ruleDiag.length ? { ruleDiag: r.ruleDiag } : {}),
...(r.errors.length ? { errors: r.errors } : {}),
}
break
}
default:
throw new Error(`Unknown job: ${jobKey}`)
}
return snapshot
},
appendRun: appendSchedulerRun,
event: appendEvent,
emitOkEvent: shouldAppendSchedulerOkEvent(jobKey),
afterCollect: () => {
if (["traffic", "servers_rest_ping", "uptime_resources", "uptime_ping", "uptime_speed", "internet_path", "gre_bgp"].includes(jobKey)) scheduleAlertEngineAfterDataCollectors()
},
reportJournalError: (error) => console.warn("Scheduler journal/hook:", error),
})
}
/** Фоновый тик: пропуск, если предыдущий прогон ещё идёт. */
export async function executeSchedulerJob(jobKey: SchedulerJobKey): Promise<void> {
if (!tryBeginSchedulerJob(jobKey)) return
try {
await runSchedulerJobBody(jobKey)
} catch {
/* Collector result is recorded once; unavailable journal is reported separately. */
} finally {
endSchedulerJob(jobKey)
}
}
/** Ручной запуск: 409, если job уже выполняется. */
export async function runSchedulerJobNow(jobKey: string): Promise<SchedulerRunReceipt> {
if (!JOB_KEYS.includes(jobKey as SchedulerJobKey)) {
throw new Error(`Unknown job key: ${jobKey}`)
}
if (!tryBeginSchedulerJob(jobKey)) {
const err = new Error(isSchedulerJobsPaused() ? "Планировщик приостановлен для операции с базой" : "Job already running")
;(err as Error & { statusCode?: number }).statusCode = 409
if (isSchedulerJobsPaused()) (err as Error & { code?: string }).code = "SCHEDULER_PAUSED"
throw err
}
try {
return await runSchedulerJobBody(jobKey as SchedulerJobKey)
} finally {
endSchedulerJob(jobKey)
}
}
function clearAllTimers() {
stopLogsLiveTimer()
clearAlertCollectorHooksTimer()
for (const t of timers.values()) clearInterval(t)
timers.clear()
}
export function isAutomaticSchedulerGenerationCurrent(generation: number): boolean {
return generation === schedulerGeneration && !isSchedulerJobsPaused()
}
export async function executeAutomaticSchedulerJob(jobKey: SchedulerJobKey, generation: number): Promise<void> {
if (!isAutomaticSchedulerGenerationCurrent(generation)) return
let enabled: boolean
switch (jobKey) {
case "router_logs": enabled = (await getLogSettings()).enabled; break
case "traffic": enabled = (await getTrafficSettings()).enabled; break
case "servers_rest_ping": enabled = (await getServersApiPingSettings()).enabled; break
case "internet_path": enabled = (await getInternetPathSettings()).enabled; break
case "certificates_renew": enabled = (await getCertificateRenewSettings()).enabled; break
case "backups": enabled = (await getBackupScheduleSettings()).enabled; break
case "geoip_update": enabled = (await getGeoipSettings()).enabled; break
case "uptime_resources": { const current = await getUptimeSettings(); enabled = current.resourcesEnabled ?? current.enabled; break }
case "uptime_ping": { const current = await getUptimeSettings(); enabled = current.pingEnabled ?? current.enabled; break }
case "uptime_speed": { const current = await getUptimeSettings(); enabled = current.speedEnabled ?? current.enabled; break }
case "gre_bgp": case "alert_engine": enabled = true; break
default: return
}
if (!enabled || !isAutomaticSchedulerGenerationCurrent(generation)) return
await executeSchedulerJob(jobKey)
}
/** Пересоздать интервалы после смены настроек. */
export async function refreshScheduler(): Promise<void> {
if (isSchedulerJobsPaused()) return
requireAutomaticSchedulerRoot()
const generation = schedulerGeneration + 1
// Handoff preflight precedes all timer/generation side effects.
const alertTicket = createAutomaticSchedulerTicket("alert_engine", generation)
schedulerGeneration = generation
const superseded = () => generation !== schedulerGeneration || isSchedulerJobsPaused()
clearAllTimers()
const tickets = new Map<SchedulerJobKey, ReturnType<typeof createAutomaticSchedulerTicket>>()
const automatic = (job: SchedulerJobKey) => {
let ticket = tickets.get(job)
if (!ticket) { ticket = createAutomaticSchedulerTicket(job, generation); tickets.set(job, ticket) }
return runAutomaticSchedulerTicket(ticket)
}
wireAlertEngineRunner(() => runAutomaticSchedulerTicket(alertTicket))
const logs = await getLogSettings()
if (superseded()) return
startLogsLiveTimer(generation)
if (logs.enabled) {
void automatic("router_logs").catch(() => {})
timers.set("router_logs", setInterval(() => { void automatic("router_logs").catch(() => {}) }, logs.intervalSec * 1000))
}
const traffic = await getTrafficSettings()
if (superseded()) return
if (traffic.enabled) {
const ms = Math.max(5_000, traffic.intervalSec * 1000)
void automatic("traffic").catch(() => {})
timers.set(
"traffic",
setInterval(() => {
void automatic("traffic").catch(() => {})
}, ms),
)
}
const apiPing = await getServersApiPingSettings()
if (superseded()) return
const internetPath = await getInternetPathSettings()
if (superseded()) return
if (apiPing.enabled) {
const apiMs = Math.max(10_000, apiPing.intervalSec * 1000)
void automatic("servers_rest_ping").catch(() => {})
timers.set(
"servers_rest_ping",
setInterval(() => {
void automatic("servers_rest_ping").catch(() => {})
}, apiMs),
)
}
const uptime = await getUptimeSettings()
if (superseded()) return
const resOn = uptime.resourcesEnabled ?? uptime.enabled
const pingOn = uptime.pingEnabled ?? uptime.enabled
const spdOn = uptime.speedEnabled ?? uptime.enabled
if (resOn) {
const resMs = Math.max(5_000, uptime.intervalSec * 1000)
void automatic("uptime_resources").catch(() => {})
timers.set(
"uptime_resources",
setInterval(() => {
void automatic("uptime_resources").catch(() => {})
}, resMs),
)
}
if (pingOn) {
const pingMs = Math.max(1_000, uptime.probeIntervalSec * 1000)
void automatic("uptime_ping").catch(() => {})
timers.set(
"uptime_ping",
setInterval(() => {
void automatic("uptime_ping").catch(() => {})
}, pingMs),
)
}
if (spdOn) {
const spdMs = Math.max(10_000, uptime.speedIntervalSec * 1000)
void automatic("uptime_speed").catch(() => {})
timers.set(
"uptime_speed",
setInterval(() => {
void automatic("uptime_speed").catch(() => {})
}, spdMs),
)
}
if (internetPath.enabled) {
const internetPathMs = Math.max(30_000, internetPath.intervalSec * 1000)
void automatic("internet_path").catch(() => {})
timers.set(
"internet_path",
setInterval(() => {
void automatic("internet_path").catch(() => {})
}, internetPathMs),
)
}
const greBgpMs = GRE_BGP_INTERVAL_MS
void automatic("gre_bgp").catch(() => {})
timers.set(
"gre_bgp",
setInterval(() => {
void automatic("gre_bgp").catch(() => {})
}, greBgpMs),
)
const certRenew = await getCertificateRenewSettings()
if (superseded()) return
if (certRenew.enabled) {
const certRenewMs = Math.max(300_000, certRenew.intervalSec * 1000)
void automatic("certificates_renew").catch(() => {})
timers.set(
"certificates_renew",
setInterval(() => {
void automatic("certificates_renew").catch(() => {})
}, certRenewMs),
)
}
const backupSchedule = await getBackupScheduleSettings()
if (superseded()) return
if (backupSchedule.enabled) {
const backupMs = 60_000
void automatic("backups").catch(() => {})
timers.set(
"backups",
setInterval(() => {
void automatic("backups").catch(() => {})
}, backupMs),
)
}
const geoip = await getGeoipSettings()
if (superseded()) return
if (geoip.enabled) {
const geoipMs = Math.max(6 * 3600_000, geoip.updateIntervalSec * 1000)
void automatic("geoip_update").catch(() => {})
timers.set(
"geoip_update",
setInterval(() => {
void automatic("geoip_update").catch(() => {})
}, geoipMs),
)
}
const alertMs = 20_000
void automatic("alert_engine").catch(() => {})
timers.set(
"alert_engine",
setInterval(() => {
void automatic("alert_engine").catch(() => {})
}, alertMs),
)
}
export function stopScheduler(): void {
schedulerGeneration += 1
clearAllTimers()
}
export async function stopSchedulerForDatabaseOperation(): Promise<void> {
const drained = pauseSchedulerJobsAndWait()
stopScheduler()
await drained
clearAlertCollectorHooksTimer()
}
export async function restartSchedulerAfterDatabaseOperation(): Promise<void> {
resumeSchedulerJobs()
await refreshScheduler()
}
export function isJobRunning(jobKey: string): boolean {
return isSchedulerJobRunning(jobKey)
}
async function lastRunForJob(jobKey: string) {
return (await db.select().from(schedulerRuns)
.where(eq(schedulerRuns.jobKey, jobKey))
.orderBy(desc(schedulerRuns.finishedAt))
.limit(1))[0]
}
export async function getSchedulerStatus() {
const logs = await getLogSettings()
const traffic = await getTrafficSettings()
const uptime = await getUptimeSettings()
const apiPing = await getServersApiPingSettings()
const internetPath = await getInternetPathSettings()
const certRenew = await getCertificateRenewSettings()
const backupSchedule = await getBackupScheduleSettings()
const geoip = await getGeoipSettings()
const resOn = uptime.resourcesEnabled ?? uptime.enabled
const pingOn = uptime.pingEnabled ?? uptime.enabled
const spdOn = uptime.speedEnabled ?? uptime.enabled
const jobMeta: Record<SchedulerJobKey, { enabled: boolean; intervalSec: number }> = {
router_logs: { enabled: logs.enabled, intervalSec: logs.intervalSec },
traffic: { enabled: traffic.enabled, intervalSec: traffic.intervalSec },
servers_rest_ping: { enabled: apiPing.enabled, intervalSec: apiPing.intervalSec },
uptime_resources: { enabled: resOn, intervalSec: uptime.intervalSec },
uptime_ping: { enabled: pingOn, intervalSec: uptime.probeIntervalSec },
uptime_speed: { enabled: spdOn, intervalSec: uptime.speedIntervalSec },
internet_path: { enabled: internetPath.enabled, intervalSec: internetPath.intervalSec },
gre_bgp: { enabled: true, intervalSec: GRE_BGP_INTERVAL_MS / 1000 },
certificates_renew: { enabled: certRenew.enabled, intervalSec: certRenew.intervalSec },
backups: { enabled: backupSchedule.enabled, intervalSec: 60 },
geoip_update: { enabled: geoip.enabled, intervalSec: geoip.updateIntervalSec },
alert_engine: { enabled: true, intervalSec: 20 },
}
return {
jobs: await Promise.all(JOB_KEYS.map(async (jobKey) => {
const last = await lastRunForJob(jobKey)
const m = jobMeta[jobKey]
return {
jobKey,
enabled: m.enabled,
intervalSec: m.intervalSec,
running: isSchedulerJobRunning(jobKey),
lastFinishedAt: last?.finishedAt ?? null,
lastStatus: last?.status ?? null,
lastDurationMs: last?.durationMs ?? null,
lastError: last?.error ?? null,
}
})),
}
}
export async function listSchedulerRuns(opts: { jobKey?: string; limit: number; offset: number }) {
const limit = Math.min(200, Math.max(1, opts.limit))
const offset = Math.max(0, opts.offset)
if (opts.jobKey) {
return {
runs: await db.select().from(schedulerRuns)
.where(eq(schedulerRuns.jobKey, opts.jobKey))
.orderBy(desc(schedulerRuns.finishedAt))
.limit(limit)
.offset(offset),
}
}
return {
runs: await db.select().from(schedulerRuns)
.orderBy(desc(schedulerRuns.finishedAt))
.limit(limit)
.offset(offset),
}
}