chore: synchronize pending app/backend updates and repository hygiene
Includes current frontend and backend work in progress and removes generated artifacts from tracking to keep the repository clean for дальнейшая разработка. Co-authored-by: Cursor <[email protected]>
This commit is contained in:
Denozordec
@@ -0,0 +1,362 @@
|
||||
/**
|
||||
* In-process планировщик: отдельный интервал на job (`refreshScheduler`).
|
||||
*
|
||||
* **Приоритеты при конкуренции** (для дальнейшего per-server mutex; сейчас зафиксировано в дизайне):
|
||||
* 1. `uptime_ping` — выше (короткий критичный сигнал).
|
||||
* 2. `traffic`, `uptime_resources`, `servers_rest_ping` — средний (REST `/system/identity` по каталогу).
|
||||
* 3. `uptime_speed` — ниже (BW-test, тяжёлый); локи узлов — `withBtestNodeLocks` в speed-сервисе.
|
||||
*
|
||||
* **Оповещения (`alert_engine`):** читают SQLite после джоб сбора (см. `buildSignalSnapshot`), в т.ч.
|
||||
* `gre_bgp` → `alert_gre_tunnel_samples` / `alert_bgp_peer_samples`. После успешного завершения джоб
|
||||
* `traffic`, `uptime_*`, `servers_rest_ping`, `gre_bgp` планируется **дополнительный** прогон движка
|
||||
* (debounce), см. [`alert-collector-hooks.ts`](./alert-collector-hooks.ts); любой другой писатель сэмплов
|
||||
* для снимка оповещений тоже должен вызывать `scheduleAlertEngineAfterDataCollectors()` после коммита.
|
||||
*
|
||||
* **Кастомные job без произвольного кода (MVP):** пер-тайминг ping в `uptime_probes.interval_sec`;
|
||||
* таблица `scheduler_job_instances` (kind + config JSON) — при необходимости следующий этап.
|
||||
*/
|
||||
import { desc, eq, lt } from "drizzle-orm"
|
||||
import { db } from "../db/index.js"
|
||||
import { schedulerRuns } from "../db/schema.js"
|
||||
import type { SchedulerRunSnapshot } from "../types/scheduler-run-snapshot.js"
|
||||
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
|
||||
import {
|
||||
collectServersRestPingOnce,
|
||||
getServersApiPingSettings,
|
||||
} from "./servers-rest-ping-collector.js"
|
||||
import { collectTrafficOnce, getTrafficSettings } from "./traffic-collector.js"
|
||||
import {
|
||||
collectPingProbesOnce,
|
||||
collectResourceSamplesOnce,
|
||||
getSettings as getUptimeSettings,
|
||||
} from "./uptime-collector.js"
|
||||
import { runScheduledSpeedProbesOnce } from "./uptime-speed-service.js"
|
||||
import { collectGreBgpSnapshotOnce } from "./gre-bgp-snapshot-collector.js"
|
||||
import { runAlertEngineOnce } from "./alert-engine/run-once.js"
|
||||
import {
|
||||
clearAlertCollectorHooksTimer,
|
||||
scheduleAlertEngineAfterDataCollectors,
|
||||
wireAlertEngineRunner,
|
||||
} from "./alert-collector-hooks.js"
|
||||
import {
|
||||
endSchedulerJob,
|
||||
isSchedulerJobRunning,
|
||||
tryBeginSchedulerJob,
|
||||
} from "./scheduler-running.js"
|
||||
|
||||
export const JOB_KEYS = [
|
||||
"traffic",
|
||||
"servers_rest_ping",
|
||||
"uptime_resources",
|
||||
"uptime_ping",
|
||||
"uptime_speed",
|
||||
"gre_bgp",
|
||||
"alert_engine",
|
||||
] as const
|
||||
export type SchedulerJobKey = (typeof JOB_KEYS)[number]
|
||||
|
||||
const timers = new Map<string, ReturnType<typeof setInterval>>()
|
||||
|
||||
const RUN_LOG_RETENTION_MS = 30 * 24 * 60 * 60 * 1000
|
||||
|
||||
function newRunId(): string {
|
||||
return `sch-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`
|
||||
}
|
||||
|
||||
function appendSchedulerRun(row: {
|
||||
jobKey: string
|
||||
startedAt: string
|
||||
finishedAt: string
|
||||
status: "ok" | "error"
|
||||
error: string | null
|
||||
durationMs: number
|
||||
result?: SchedulerRunSnapshot | null
|
||||
}) {
|
||||
db.insert(schedulerRuns).values({
|
||||
id: newRunId(),
|
||||
jobKey: row.jobKey,
|
||||
startedAt: row.startedAt,
|
||||
finishedAt: row.finishedAt,
|
||||
status: row.status,
|
||||
error: row.error,
|
||||
durationMs: row.durationMs,
|
||||
resultJson: row.result ? JSON.stringify(row.result) : null,
|
||||
}).run()
|
||||
const cutoff = new Date(Date.now() - RUN_LOG_RETENTION_MS).toISOString()
|
||||
db.delete(schedulerRuns).where(lt(schedulerRuns.finishedAt, cutoff)).run()
|
||||
}
|
||||
|
||||
async function runSchedulerJobBody(jobKey: SchedulerJobKey): Promise<void> {
|
||||
const startedAt = Date.now()
|
||||
const startedIso = new Date().toISOString()
|
||||
let snapshot: SchedulerRunSnapshot | undefined
|
||||
try {
|
||||
switch (jobKey) {
|
||||
case "traffic":
|
||||
snapshot = await collectTrafficOnce()
|
||||
break
|
||||
case "uptime_resources":
|
||||
snapshot = await collectResourceSamplesOnce()
|
||||
break
|
||||
case "uptime_ping":
|
||||
snapshot = await collectPingProbesOnce()
|
||||
break
|
||||
case "uptime_speed":
|
||||
snapshot = await runScheduledSpeedProbesOnce()
|
||||
break
|
||||
case "servers_rest_ping":
|
||||
snapshot = await collectServersRestPingOnce()
|
||||
break
|
||||
case "gre_bgp":
|
||||
snapshot = await collectGreBgpSnapshotOnce()
|
||||
break
|
||||
case "alert_engine": {
|
||||
const r = await runAlertEngineOnce()
|
||||
snapshot = {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "alert_engine",
|
||||
sampledAt: r.sampledAt,
|
||||
rulesChecked: r.rulesChecked,
|
||||
standaloneFires: r.standaloneFires,
|
||||
groupFires: r.groupFires,
|
||||
skippedNoTelegram: r.skippedNoTelegram,
|
||||
...(r.ruleDiag.length ? { ruleDiag: r.ruleDiag } : {}),
|
||||
...(r.errors.length ? { errors: r.errors } : {}),
|
||||
}
|
||||
break
|
||||
}
|
||||
default:
|
||||
throw new Error(`Unknown job: ${jobKey}`)
|
||||
}
|
||||
const finishedIso = new Date().toISOString()
|
||||
appendSchedulerRun({
|
||||
jobKey,
|
||||
startedAt: startedIso,
|
||||
finishedAt: finishedIso,
|
||||
status: "ok",
|
||||
error: null,
|
||||
durationMs: Date.now() - startedAt,
|
||||
result: snapshot ?? null,
|
||||
})
|
||||
if (
|
||||
jobKey === "traffic" ||
|
||||
jobKey === "servers_rest_ping" ||
|
||||
jobKey === "uptime_resources" ||
|
||||
jobKey === "uptime_ping" ||
|
||||
jobKey === "uptime_speed" ||
|
||||
jobKey === "gre_bgp"
|
||||
) {
|
||||
scheduleAlertEngineAfterDataCollectors()
|
||||
}
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e)
|
||||
const finishedIso = new Date().toISOString()
|
||||
appendSchedulerRun({
|
||||
jobKey,
|
||||
startedAt: startedIso,
|
||||
finishedAt: finishedIso,
|
||||
status: "error",
|
||||
error: msg,
|
||||
durationMs: Date.now() - startedAt,
|
||||
result: snapshot ?? null,
|
||||
})
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
/** Фоновый тик: пропуск, если предыдущий прогон ещё идёт. */
|
||||
export async function executeSchedulerJob(jobKey: SchedulerJobKey): Promise<void> {
|
||||
if (!tryBeginSchedulerJob(jobKey)) return
|
||||
try {
|
||||
await runSchedulerJobBody(jobKey)
|
||||
} catch {
|
||||
/* залогировано в runSchedulerJobBody */
|
||||
} finally {
|
||||
endSchedulerJob(jobKey)
|
||||
}
|
||||
}
|
||||
|
||||
/** Ручной запуск: 409, если job уже выполняется. */
|
||||
export async function runSchedulerJobNow(jobKey: string): Promise<void> {
|
||||
if (!JOB_KEYS.includes(jobKey as SchedulerJobKey)) {
|
||||
throw new Error(`Unknown job key: ${jobKey}`)
|
||||
}
|
||||
if (!tryBeginSchedulerJob(jobKey)) {
|
||||
const err = new Error("Job already running")
|
||||
;(err as Error & { statusCode?: number }).statusCode = 409
|
||||
throw err
|
||||
}
|
||||
try {
|
||||
await runSchedulerJobBody(jobKey as SchedulerJobKey)
|
||||
} finally {
|
||||
endSchedulerJob(jobKey)
|
||||
}
|
||||
}
|
||||
|
||||
function clearAllTimers() {
|
||||
clearAlertCollectorHooksTimer()
|
||||
for (const t of timers.values()) clearInterval(t)
|
||||
timers.clear()
|
||||
}
|
||||
|
||||
/** Пересоздать интервалы после смены настроек. */
|
||||
export function refreshScheduler(): void {
|
||||
clearAllTimers()
|
||||
|
||||
const traffic = getTrafficSettings()
|
||||
if (traffic.enabled) {
|
||||
const ms = Math.max(5_000, traffic.intervalSec * 1000)
|
||||
void executeSchedulerJob("traffic").catch(() => {})
|
||||
timers.set(
|
||||
"traffic",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("traffic").catch(() => {})
|
||||
}, ms),
|
||||
)
|
||||
}
|
||||
|
||||
const apiPing = getServersApiPingSettings()
|
||||
if (apiPing.enabled) {
|
||||
const apiMs = Math.max(10_000, apiPing.intervalSec * 1000)
|
||||
void executeSchedulerJob("servers_rest_ping").catch(() => {})
|
||||
timers.set(
|
||||
"servers_rest_ping",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("servers_rest_ping").catch(() => {})
|
||||
}, apiMs),
|
||||
)
|
||||
}
|
||||
|
||||
const uptime = getUptimeSettings()
|
||||
const resOn = uptime.resourcesEnabled ?? uptime.enabled
|
||||
const pingOn = uptime.pingEnabled ?? uptime.enabled
|
||||
const spdOn = uptime.speedEnabled ?? uptime.enabled
|
||||
|
||||
if (resOn) {
|
||||
const resMs = Math.max(5_000, uptime.intervalSec * 1000)
|
||||
void executeSchedulerJob("uptime_resources").catch(() => {})
|
||||
timers.set(
|
||||
"uptime_resources",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("uptime_resources").catch(() => {})
|
||||
}, resMs),
|
||||
)
|
||||
}
|
||||
|
||||
if (pingOn) {
|
||||
const pingMs = Math.max(1_000, uptime.probeIntervalSec * 1000)
|
||||
void executeSchedulerJob("uptime_ping").catch(() => {})
|
||||
timers.set(
|
||||
"uptime_ping",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("uptime_ping").catch(() => {})
|
||||
}, pingMs),
|
||||
)
|
||||
}
|
||||
|
||||
if (spdOn) {
|
||||
const spdMs = Math.max(10_000, uptime.speedIntervalSec * 1000)
|
||||
void executeSchedulerJob("uptime_speed").catch(() => {})
|
||||
timers.set(
|
||||
"uptime_speed",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("uptime_speed").catch(() => {})
|
||||
}, spdMs),
|
||||
)
|
||||
}
|
||||
|
||||
const greBgpMs = 30_000
|
||||
void executeSchedulerJob("gre_bgp").catch(() => {})
|
||||
timers.set(
|
||||
"gre_bgp",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("gre_bgp").catch(() => {})
|
||||
}, greBgpMs),
|
||||
)
|
||||
|
||||
const alertMs = 20_000
|
||||
void executeSchedulerJob("alert_engine").catch(() => {})
|
||||
timers.set(
|
||||
"alert_engine",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("alert_engine").catch(() => {})
|
||||
}, alertMs),
|
||||
)
|
||||
}
|
||||
|
||||
export function stopScheduler(): void {
|
||||
clearAllTimers()
|
||||
}
|
||||
|
||||
export function isJobRunning(jobKey: string): boolean {
|
||||
return isSchedulerJobRunning(jobKey)
|
||||
}
|
||||
|
||||
function lastRunForJob(jobKey: string) {
|
||||
return db.select().from(schedulerRuns)
|
||||
.where(eq(schedulerRuns.jobKey, jobKey))
|
||||
.orderBy(desc(schedulerRuns.finishedAt))
|
||||
.limit(1)
|
||||
.all()[0]
|
||||
}
|
||||
|
||||
export function getSchedulerStatus() {
|
||||
const traffic = getTrafficSettings()
|
||||
const uptime = getUptimeSettings()
|
||||
const apiPing = getServersApiPingSettings()
|
||||
|
||||
const resOn = uptime.resourcesEnabled ?? uptime.enabled
|
||||
const pingOn = uptime.pingEnabled ?? uptime.enabled
|
||||
const spdOn = uptime.speedEnabled ?? uptime.enabled
|
||||
|
||||
const jobMeta: Record<SchedulerJobKey, { enabled: boolean; intervalSec: number }> = {
|
||||
traffic: { enabled: traffic.enabled, intervalSec: traffic.intervalSec },
|
||||
servers_rest_ping: { enabled: apiPing.enabled, intervalSec: apiPing.intervalSec },
|
||||
uptime_resources: { enabled: resOn, intervalSec: uptime.intervalSec },
|
||||
uptime_ping: { enabled: pingOn, intervalSec: uptime.probeIntervalSec },
|
||||
uptime_speed: { enabled: spdOn, intervalSec: uptime.speedIntervalSec },
|
||||
gre_bgp: { enabled: true, intervalSec: 30 },
|
||||
alert_engine: { enabled: true, intervalSec: 20 },
|
||||
}
|
||||
|
||||
return {
|
||||
jobs: JOB_KEYS.map((jobKey) => {
|
||||
const last = lastRunForJob(jobKey)
|
||||
const m = jobMeta[jobKey]
|
||||
return {
|
||||
jobKey,
|
||||
enabled: m.enabled,
|
||||
intervalSec: m.intervalSec,
|
||||
running: isSchedulerJobRunning(jobKey),
|
||||
lastFinishedAt: last?.finishedAt ?? null,
|
||||
lastStatus: last?.status ?? null,
|
||||
lastDurationMs: last?.durationMs ?? null,
|
||||
lastError: last?.error ?? null,
|
||||
}
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
export function listSchedulerRuns(opts: { jobKey?: string; limit: number; offset: number }) {
|
||||
const limit = Math.min(200, Math.max(1, opts.limit))
|
||||
const offset = Math.max(0, opts.offset)
|
||||
if (opts.jobKey) {
|
||||
return {
|
||||
runs: db.select().from(schedulerRuns)
|
||||
.where(eq(schedulerRuns.jobKey, opts.jobKey))
|
||||
.orderBy(desc(schedulerRuns.finishedAt))
|
||||
.limit(limit)
|
||||
.offset(offset)
|
||||
.all(),
|
||||
}
|
||||
}
|
||||
return {
|
||||
runs: db.select().from(schedulerRuns)
|
||||
.orderBy(desc(schedulerRuns.finishedAt))
|
||||
.limit(limit)
|
||||
.offset(offset)
|
||||
.all(),
|
||||
}
|
||||
}
|
||||
|
||||
wireAlertEngineRunner(() => executeSchedulerJob("alert_engine"))
|
||||
Reference in New Issue
Block a user