feat(scheduler): добавить расписание бэкапов и автообновление сертификатов
Docker images / prepare-release (push) Successful in 6s
Docker images / backend-image (push) Successful in 2m13s
Docker images / frontend-image (push) Successful in 2m11s
Docker images / updater-image (push) Successful in 50s
Docker images / notify-webhook (push) Has been skipped
Docker images / publish-release (push) Successful in 8s

This commit is contained in:
Denozordec
2026-05-12 21:46:11 +07:00
parent 4b8cf83ee9
commit 43ab17b253
33 changed files with 118807 additions and 179 deletions
+46
View File
@@ -351,6 +351,7 @@ CREATE TABLE IF NOT EXISTS certificate_issue_jobs (
id TEXT PRIMARY KEY,
status TEXT NOT NULL DEFAULT 'queued',
step TEXT NOT NULL DEFAULT 'queued',
source TEXT NOT NULL DEFAULT 'manual',
server_id TEXT NOT NULL,
cert_name TEXT NOT NULL,
domain_names TEXT NOT NULL,
@@ -362,6 +363,34 @@ CREATE TABLE IF NOT EXISTS certificate_issue_jobs (
error TEXT
);
CREATE TABLE IF NOT EXISTS certificate_renew_settings (
id INTEGER PRIMARY KEY,
enabled INTEGER NOT NULL DEFAULT 1,
interval_sec INTEGER NOT NULL DEFAULT 21600,
renew_before_days INTEGER NOT NULL DEFAULT 30,
last_collected_at TEXT,
last_duration_ms INTEGER,
last_error TEXT,
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS backup_schedule_settings (
id INTEGER PRIMARY KEY,
enabled INTEGER NOT NULL DEFAULT 1,
frequency TEXT NOT NULL DEFAULT 'daily',
hour INTEGER NOT NULL DEFAULT 3,
minute INTEGER NOT NULL DEFAULT 0,
week_day INTEGER NOT NULL DEFAULT 0,
month_day INTEGER NOT NULL DEFAULT 1,
keep_count INTEGER NOT NULL DEFAULT 7,
format TEXT NOT NULL DEFAULT 'rsc',
server_ids_json TEXT NOT NULL DEFAULT '[]',
last_run_at TEXT,
last_duration_ms INTEGER,
last_error TEXT,
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS alert_rules (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
@@ -617,6 +646,23 @@ SELECT 1, 'https://acme-v02.api.letsencrypt.org/directory', '', '', ''
WHERE NOT EXISTS (SELECT 1 FROM acme_settings WHERE id = 1);
`)
const certIssueJobCols = sqlite.prepare(`PRAGMA table_info('certificate_issue_jobs')`).all() as Array<{ name?: string }>
if (!certIssueJobCols.some((c) => c.name === "source")) {
sqlite.exec(`ALTER TABLE certificate_issue_jobs ADD COLUMN source TEXT NOT NULL DEFAULT 'manual'`)
}
sqlite.exec(`
INSERT INTO certificate_renew_settings (id, enabled, interval_sec, renew_before_days)
SELECT 1, 1, 21600, 30
WHERE NOT EXISTS (SELECT 1 FROM certificate_renew_settings WHERE id = 1);
`)
sqlite.exec(`
INSERT INTO backup_schedule_settings (id, enabled, frequency, hour, minute, week_day, month_day, keep_count, format, server_ids_json)
SELECT 1, 1, 'daily', 3, 0, 0, 1, 7, 'rsc', '[]'
WHERE NOT EXISTS (SELECT 1 FROM backup_schedule_settings WHERE id = 1);
`)
sqlite.exec(`
INSERT INTO alert_engine_cursor (id, last_source_finished_at)
SELECT 1, NULL
+31
View File
@@ -300,6 +300,7 @@ export const certificateIssueJobs = sqliteTable("certificate_issue_jobs", {
id: text("id").primaryKey(),
status: text("status", { enum: ["queued", "running", "done", "failed"] }).notNull().default("queued"),
step: text("step").notNull().default("queued"),
source: text("source", { enum: ["manual", "scheduler"] }).notNull().default("manual"),
serverId: text("server_id").notNull(),
certName: text("cert_name").notNull(),
domainNames: text("domain_names").notNull(),
@@ -311,6 +312,34 @@ export const certificateIssueJobs = sqliteTable("certificate_issue_jobs", {
error: text("error"),
})
export const certificateRenewSettings = sqliteTable("certificate_renew_settings", {
id: integer("id").primaryKey(),
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
intervalSec: integer("interval_sec").notNull().default(21600),
renewBeforeDays: integer("renew_before_days").notNull().default(30),
lastCollectedAt: text("last_collected_at"),
lastDurationMs: integer("last_duration_ms"),
lastError: text("last_error"),
updatedAt: text("updated_at").notNull().default(sql`(datetime('now'))`),
})
export const backupScheduleSettings = sqliteTable("backup_schedule_settings", {
id: integer("id").primaryKey(),
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
frequency: text("frequency", { enum: ["daily", "weekly", "monthly"] }).notNull().default("daily"),
hour: integer("hour").notNull().default(3),
minute: integer("minute").notNull().default(0),
weekDay: integer("week_day").notNull().default(0),
monthDay: integer("month_day").notNull().default(1),
keepCount: integer("keep_count").notNull().default(7),
format: text("format", { enum: ["rsc", "backup"] }).notNull().default("rsc"),
serverIdsJson: text("server_ids_json").notNull().default("[]"),
lastRunAt: text("last_run_at"),
lastDurationMs: integer("last_duration_ms"),
lastError: text("last_error"),
updatedAt: text("updated_at").notNull().default(sql`(datetime('now'))`),
})
/** Группы правил: ANY = хотя бы одно; ALL = все одновременно в окне тика. */
export const alertGroups = sqliteTable("alert_groups", {
id: text("id").primaryKey(),
@@ -531,6 +560,8 @@ export type EvobgpSettingsRow = typeof evobgpSettings.$inferSelect
export type AlertTelegramSettingsRow = typeof alertTelegramSettings.$inferSelect
export type AcmeSettingsRow = typeof acmeSettings.$inferSelect
export type CertificateIssueJobRow = typeof certificateIssueJobs.$inferSelect
export type CertificateRenewSettingsRow = typeof certificateRenewSettings.$inferSelect
export type BackupScheduleSettingsRow = typeof backupScheduleSettings.$inferSelect
export type AlertGroupRow = typeof alertGroups.$inferSelect
export type AlertRuleRow = typeof alertRules.$inferSelect
export type AlertRuleTargetRow = typeof alertRuleTargets.$inferSelect
+35 -80
View File
@@ -1,23 +1,21 @@
import { randomUUID } from "node:crypto"
import { mkdir, readFile, rm, stat, writeFile } from "node:fs/promises"
import { readFile, rm } from "node:fs/promises"
import path from "node:path"
import { z } from "zod"
import type { FastifyPluginAsyncZod } from "@fastify/type-provider-zod"
import { putBackupScheduleSettingsSchema } from "@mmapp/contracts/backups"
import { listServersRead } from "../modules/servers/service/servers-service.js"
import { getServerRowById } from "../modules/servers/repository/servers-repository.js"
import { MikrotikClient } from "../services/mikrotik.js"
import { appendEvent } from "../modules/events/service/events-service.js"
type BackupMeta = {
id: string
serverId: string
serverName: string
filename: string
sizeBytes: number
createdAt: string
kind: "manual"
notes?: string
}
import { refreshScheduler } from "../services/scheduler.js"
import {
getBackupsDir,
getBackupScheduleSettings,
readBackupIndex,
runBackupForServer,
updateBackupScheduleSettings,
writeBackupIndex,
type BackupMeta,
} from "../services/backup-service.js"
type BackupJobStatus = "queued" | "running" | "done" | "failed"
type BackupJob = {
@@ -32,36 +30,8 @@ type BackupJob = {
failures: Array<{ serverId: string; error: string }>
}
const BACKUPS_DIR = path.resolve(process.cwd(), "storage", "backups")
const INDEX_PATH = path.join(BACKUPS_DIR, "index.json")
const backupJobs = new Map<string, BackupJob>()
async function ensureStorage() {
await mkdir(BACKUPS_DIR, { recursive: true })
}
async function readIndex(): Promise<BackupMeta[]> {
await ensureStorage()
try {
const raw = await readFile(INDEX_PATH, "utf8")
const parsed = JSON.parse(raw) as unknown
if (!Array.isArray(parsed)) return []
return parsed as BackupMeta[]
} catch {
return []
}
}
async function writeIndex(rows: BackupMeta[]): Promise<void> {
await ensureStorage()
await writeFile(INDEX_PATH, JSON.stringify(rows, null, 2), "utf8")
}
function fmtTs(d = new Date()): string {
const p = (n: number) => String(n).padStart(2, "0")
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}_${p(d.getHours())}-${p(d.getMinutes())}-${p(d.getSeconds())}`
}
const CreateBackupBodySchema = z.object({
serverIds: z.array(z.union([z.string(), z.number()])).min(1),
notes: z.string().max(500).optional(),
@@ -74,42 +44,13 @@ const BackupJobIdParamSchema = z.object({
jobId: z.string().min(1),
})
async function runBackupForServer(id: string, notes?: string): Promise<BackupMeta> {
const serverIdNum = Number.parseInt(id, 10)
if (!Number.isFinite(serverIdNum)) {
throw new Error("Невалидный id сервера")
}
const row = getServerRowById(serverIdNum)
if (!row) {
throw new Error("Сервер не найден")
}
const client = MikrotikClient.fromServer(row)
const script = await client.exportConfigScript()
const ts = fmtTs()
const safeServer = row.name.replace(/[^a-zA-Z0-9._-]+/g, "_")
const filename = `${safeServer}_${ts}.rsc`
const filePath = path.join(BACKUPS_DIR, filename)
await writeFile(filePath, script, "utf8")
const st = await stat(filePath)
return {
id: randomUUID(),
serverId: String(row.id),
serverName: row.name,
filename,
sizeBytes: st.size,
createdAt: new Date().toISOString(),
kind: "manual",
notes,
}
}
async function processBackupJob(job: BackupJob, ids: string[], notes?: string) {
job.status = "running"
job.startedAt = new Date().toISOString()
const indexRows = await readIndex()
const indexRows = await readBackupIndex()
for (const id of ids) {
try {
const meta = await runBackupForServer(id, notes)
const meta = await runBackupForServer(id, "manual", notes)
indexRows.unshift(meta)
job.created.push(meta)
} catch (err) {
@@ -119,7 +60,7 @@ async function processBackupJob(job: BackupJob, ids: string[], notes?: string) {
job.completed += 1
}
}
await writeIndex(indexRows)
await writeBackupIndex(indexRows)
job.status = "done"
job.finishedAt = new Date().toISOString()
appendEvent({
@@ -141,11 +82,25 @@ async function processBackupJob(job: BackupJob, ids: string[], notes?: string) {
const backupsRoutes: FastifyPluginAsyncZod = async (app) => {
app.get("/backups", async (_req, reply) => {
const rows = await readIndex()
const rows = await readBackupIndex()
rows.sort((a, b) => b.createdAt.localeCompare(a.createdAt))
return reply.send(rows)
})
app.get("/backups/schedule", async (_req, reply) => {
return reply.send(getBackupScheduleSettings())
})
app.put("/backups/schedule", async (req, reply) => {
const parsed = putBackupScheduleSettingsSchema.safeParse(req.body ?? {})
if (!parsed.success) {
return reply.status(400).send({ error: "Некорректное тело запроса", details: parsed.error.flatten() })
}
const result = updateBackupScheduleSettings(parsed.data)
refreshScheduler()
return reply.send(result)
})
app.post("/backups/create", { schema: { body: CreateBackupBodySchema } }, async (req, reply) => {
const inputIds = req.body.serverIds.map((x) => String(x))
const notes = req.body.notes?.trim() || undefined
@@ -216,10 +171,10 @@ const backupsRoutes: FastifyPluginAsyncZod = async (app) => {
})
app.get("/backups/:id/download", { schema: { params: BackupIdParamSchema } }, async (req, reply) => {
const rows = await readIndex()
const rows = await readBackupIndex()
const hit = rows.find((r) => r.id === req.params.id)
if (!hit) return reply.status(404).send({ error: "Бэкап не найден" })
const filePath = path.join(BACKUPS_DIR, hit.filename)
const filePath = path.join(getBackupsDir(), hit.filename)
const content = await readFile(filePath, "utf8").catch(() => null)
if (content == null) return reply.status(404).send({ error: "Файл бэкапа не найден" })
reply.header("Content-Type", "text/plain; charset=utf-8")
@@ -228,12 +183,12 @@ const backupsRoutes: FastifyPluginAsyncZod = async (app) => {
})
app.delete("/backups/:id", { schema: { params: BackupIdParamSchema } }, async (req, reply) => {
const rows = await readIndex()
const rows = await readBackupIndex()
const idx = rows.findIndex((r) => r.id === req.params.id)
if (idx < 0) return reply.status(404).send({ error: "Бэкап не найден" })
const [hit] = rows.splice(idx, 1)
await writeIndex(rows)
await rm(path.join(BACKUPS_DIR, hit.filename), { force: true })
await writeBackupIndex(rows)
await rm(path.join(getBackupsDir(), hit.filename), { force: true })
return reply.status(204).send()
})
}
+22 -83
View File
@@ -3,101 +3,25 @@ import type { FastifyPluginAsyncZod } from "@fastify/type-provider-zod"
import {
certificateIssueRequestSchema,
putAcmeCloudflareSettingsSchema,
putCertificateRenewSettingsSchema,
testAcmeCloudflareSettingsSchema,
} from "@mmapp/contracts/certificates"
import { appendEvent } from "../modules/events/service/events-service.js"
import { issueCertificateWithCloudflareDns, testCloudflareToken } from "../services/acme-cloudflare.js"
import { refreshScheduler } from "../services/scheduler.js"
import { testCloudflareToken } from "../services/acme-cloudflare.js"
import { queueCertificateIssueJob } from "../services/certificate-issue-runner.js"
import {
createIssueJobRecord,
getAcmeCloudflareToken,
getAcmeSettingsPublic,
getCertificateRenewSettings,
getIssueJobRecord,
getServerRowByIdString,
listCertificatesFromServers,
toIssueJobDto,
updateAcmeSettings,
updateIssueJobRecord,
updateCertificateRenewSettings,
} from "../services/certificates-service.js"
const runningJobs = new Set<string>()
async function runIssueJob(jobId: string) {
if (runningJobs.has(jobId)) return
runningJobs.add(jobId)
const row = getIssueJobRecord(jobId)
if (!row) {
runningJobs.delete(jobId)
return
}
const server = getServerRowByIdString(row.serverId)
if (!server) {
updateIssueJobRecord(jobId, {
status: "failed",
step: "failed",
finishedAt: new Date().toISOString(),
error: "Сервер не найден",
})
runningJobs.delete(jobId)
return
}
let domainNames: string[] = []
try {
domainNames = JSON.parse(row.domainNames) as string[]
} catch {
domainNames = []
}
const trustStore = row.trustStore.split(",").map((s) => s.trim()).filter(Boolean)
const startedAt = new Date().toISOString()
updateIssueJobRecord(jobId, { status: "running", step: "acme_order", startedAt, error: null })
try {
await issueCertificateWithCloudflareDns({
server,
certName: row.certName,
domainNames,
keyType: row.keyType === "ec256" ? "ec256" : "rsa2048",
trustStore: trustStore.length > 0 ? trustStore : ["www", "api"],
onStep: (step) => updateIssueJobRecord(jobId, { step }),
})
updateIssueJobRecord(jobId, {
status: "done",
step: "done",
finishedAt: new Date().toISOString(),
error: null,
})
appendEvent({
level: "info",
eventType: "certificates.issue.done",
sourceModule: "certificates",
title: "Сертификат выпущен",
message: `${row.certName} · ${domainNames.join(", ")} · ${server.name}`,
entityType: "server",
entityId: String(server.id),
})
} catch (e) {
updateIssueJobRecord(jobId, {
status: "failed",
step: "failed",
finishedAt: new Date().toISOString(),
error: e instanceof Error ? e.message : String(e),
})
appendEvent({
level: "warning",
eventType: "certificates.issue.failed",
sourceModule: "certificates",
title: "Ошибка выпуска сертификата",
message: e instanceof Error ? e.message : String(e),
entityType: "server",
entityId: String(server.id),
})
} finally {
runningJobs.delete(jobId)
}
}
const certificatesRoutes: FastifyPluginAsyncZod = async (app) => {
app.get("/certificates", async (_req, reply) => {
return reply.send(await listCertificatesFromServers())
@@ -119,6 +43,20 @@ const certificatesRoutes: FastifyPluginAsyncZod = async (app) => {
return reply.send(updateAcmeSettings(parsed.data))
})
app.get("/certificates/renew-settings", async (_req, reply) => {
return reply.send(getCertificateRenewSettings())
})
app.put("/certificates/renew-settings", async (req, reply) => {
const parsed = putCertificateRenewSettingsSchema.safeParse(req.body ?? {})
if (!parsed.success) {
return reply.status(400).send({ error: "Некорректное тело запроса", details: parsed.error.flatten() })
}
const result = updateCertificateRenewSettings(parsed.data)
refreshScheduler()
return reply.send(result)
})
app.post("/certificates/acme-settings/test", async (req, reply) => {
const parsed = testAcmeCloudflareSettingsSchema.safeParse(req.body ?? {})
if (!parsed.success) {
@@ -157,8 +95,9 @@ const certificatesRoutes: FastifyPluginAsyncZod = async (app) => {
domainNames: parsed.data.domainNames.map((d) => d.trim()).filter(Boolean),
keyType: parsed.data.keyType ?? "rsa2048",
trustStore,
source: "manual",
})
queueMicrotask(() => { void runIssueJob(jobId) })
queueCertificateIssueJob(jobId)
return reply.send({ jobId })
})
@@ -0,0 +1,142 @@
import { appendEvent } from "../modules/events/service/events-service.js"
import type { BackupsRunSnapshot } from "../types/scheduler-run-snapshot.js"
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
import {
getBackupScheduleSettings,
isBackupDue,
pruneBackupsForServer,
readBackupIndex,
resolveBackupServerIds,
runBackupForServer,
touchBackupScheduleRunMeta,
writeBackupIndex,
} from "./backup-service.js"
let collecting = false
export function getBackupSchedulerCollectorState(): { running: boolean } {
return { running: collecting }
}
export async function collectScheduledBackupsOnce(): Promise<BackupsRunSnapshot> {
const sampledAt = new Date().toISOString()
const settings = getBackupScheduleSettings()
const snapshot: BackupsRunSnapshot = {
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
job: "backups",
sampledAt,
due: false,
created: 0,
failures: 0,
pruned: 0,
errors: [],
}
if (collecting) {
snapshot.skipped = true
return snapshot
}
if (!settings.enabled) {
snapshot.skipped = true
return snapshot
}
const due = isBackupDue(new Date(), settings, settings.lastRunAt)
snapshot.due = due
if (!due) {
return snapshot
}
if (settings.format !== "rsc") {
snapshot.skipped = true
snapshot.errors = ["Формат backup пока не поддерживается, используйте rsc"]
touchBackupScheduleRunMeta({
lastRunAt: sampledAt,
lastDurationMs: 0,
lastError: snapshot.errors[0],
})
return snapshot
}
collecting = true
const started = Date.now()
const serverIds = resolveBackupServerIds(settings)
const indexRows = await readBackupIndex()
appendEvent({
level: "info",
eventType: "backups.job.started",
sourceModule: "backups",
title: "Запущен плановый бэкап",
message: `Серверов в очереди: ${serverIds.length}`,
entityType: "backup_job",
entityId: "scheduler",
payload: { serverIds, scheduled: true },
})
try {
for (const id of serverIds) {
try {
const meta = await runBackupForServer(id, "auto")
indexRows.unshift(meta)
snapshot.created += 1
} catch (e) {
const message = e instanceof Error ? e.message : String(e)
snapshot.failures += 1
snapshot.errors?.push(`${id}: ${message}`)
}
}
await writeBackupIndex(indexRows)
for (const id of serverIds) {
snapshot.pruned += await pruneBackupsForServer(id, settings.keepCount)
}
touchBackupScheduleRunMeta({
lastRunAt: sampledAt,
lastDurationMs: Date.now() - started,
lastError: snapshot.errors?.length ? snapshot.errors.join("; ") : null,
})
appendEvent({
level: snapshot.failures > 0 ? "warning" : "info",
eventType: "backups.job.done",
sourceModule: "backups",
title: snapshot.failures > 0 ? "Плановый бэкап завершен с ошибками" : "Плановый бэкап завершен",
message: `Создано: ${snapshot.created}, ошибок: ${snapshot.failures}`,
entityType: "backup_job",
entityId: "scheduler",
payload: {
total: serverIds.length,
completed: snapshot.created + snapshot.failures,
failures: snapshot.errors,
pruned: snapshot.pruned,
},
})
return snapshot
} catch (e) {
const message = e instanceof Error ? e.message : String(e)
snapshot.fatalError = message
snapshot.errors?.push(message)
touchBackupScheduleRunMeta({
lastRunAt: sampledAt,
lastDurationMs: Date.now() - started,
lastError: message,
})
appendEvent({
level: "critical",
eventType: "backups.job.failed",
sourceModule: "backups",
title: "Плановый бэкап прерван",
message,
entityType: "backup_job",
entityId: "scheduler",
})
return snapshot
} finally {
collecting = false
}
}
+241
View File
@@ -0,0 +1,241 @@
import { randomUUID } from "node:crypto"
import { mkdir, readFile, rm, stat, writeFile } from "node:fs/promises"
import path from "node:path"
import { eq } from "drizzle-orm"
import type { BackupScheduleSettingsDto } from "@mmapp/contracts/backups"
import { db } from "../db/index.js"
import { backupScheduleSettings } from "../db/schema.js"
import { getServerRowById } from "../modules/servers/repository/servers-repository.js"
import { listServersRead } from "../modules/servers/service/servers-service.js"
import { MikrotikClient } from "./mikrotik.js"
const SETTINGS_ID = 1
const BACKUPS_DIR = path.resolve(process.cwd(), "storage", "backups")
const INDEX_PATH = path.join(BACKUPS_DIR, "index.json")
export type BackupMeta = {
id: string
serverId: string
serverName: string
filename: string
sizeBytes: number
createdAt: string
kind: "manual" | "auto"
notes?: string
}
export async function ensureBackupStorage(): Promise<void> {
await mkdir(BACKUPS_DIR, { recursive: true })
}
export async function readBackupIndex(): Promise<BackupMeta[]> {
await ensureBackupStorage()
try {
const raw = await readFile(INDEX_PATH, "utf8")
const parsed = JSON.parse(raw) as unknown
if (!Array.isArray(parsed)) return []
return parsed as BackupMeta[]
} catch {
return []
}
}
export async function writeBackupIndex(rows: BackupMeta[]): Promise<void> {
await ensureBackupStorage()
await writeFile(INDEX_PATH, JSON.stringify(rows, null, 2), "utf8")
}
function fmtTs(d = new Date()): string {
const p = (n: number) => String(n).padStart(2, "0")
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}_${p(d.getHours())}-${p(d.getMinutes())}-${p(d.getSeconds())}`
}
function parseServerIds(raw: string): string[] {
try {
const parsed = JSON.parse(raw) as unknown
if (!Array.isArray(parsed)) return []
return parsed.map(String).filter(Boolean)
} catch {
return []
}
}
function getBackupScheduleSettingsRow() {
return db.select().from(backupScheduleSettings).where(eq(backupScheduleSettings.id, SETTINGS_ID)).limit(1).all()[0]
?? {
id: SETTINGS_ID,
enabled: true,
frequency: "daily" as const,
hour: 3,
minute: 0,
weekDay: 0,
monthDay: 1,
keepCount: 7,
format: "rsc" as const,
serverIdsJson: "[]",
lastRunAt: null,
lastDurationMs: null,
lastError: null,
updatedAt: new Date().toISOString(),
}
}
export function getBackupScheduleSettings(): BackupScheduleSettingsDto {
const row = getBackupScheduleSettingsRow()
return {
enabled: row.enabled,
frequency: row.frequency,
hour: row.hour,
minute: row.minute,
weekDay: row.weekDay,
monthDay: row.monthDay,
keepCount: row.keepCount,
format: row.format,
serverIds: parseServerIds(row.serverIdsJson),
lastRunAt: row.lastRunAt ?? null,
lastDurationMs: row.lastDurationMs ?? null,
lastError: row.lastError ?? null,
updatedAt: row.updatedAt,
}
}
export function updateBackupScheduleSettings(patch: Partial<{
enabled: boolean
frequency: "daily" | "weekly" | "monthly"
hour: number
minute: number
weekDay: number
monthDay: number
keepCount: number
format: "rsc" | "backup"
serverIds: string[]
}>) {
const prev = getBackupScheduleSettingsRow()
const now = new Date().toISOString()
const next = {
enabled: patch.enabled ?? prev.enabled,
frequency: patch.frequency ?? prev.frequency,
hour: patch.hour ?? prev.hour,
minute: patch.minute ?? prev.minute,
weekDay: patch.weekDay ?? prev.weekDay,
monthDay: patch.monthDay ?? prev.monthDay,
keepCount: patch.keepCount ?? prev.keepCount,
format: patch.format ?? prev.format,
serverIdsJson: patch.serverIds ? JSON.stringify(patch.serverIds) : prev.serverIdsJson,
updatedAt: now,
}
if (db.select().from(backupScheduleSettings).where(eq(backupScheduleSettings.id, SETTINGS_ID)).limit(1).all()[0]) {
db.update(backupScheduleSettings).set(next).where(eq(backupScheduleSettings.id, SETTINGS_ID)).run()
} else {
db.insert(backupScheduleSettings).values({ id: SETTINGS_ID, ...next }).run()
}
return getBackupScheduleSettings()
}
export function touchBackupScheduleRunMeta(patch: {
lastRunAt?: string
lastDurationMs?: number
lastError?: string | null
}) {
const prev = getBackupScheduleSettingsRow()
db.update(backupScheduleSettings).set({
lastRunAt: patch.lastRunAt ?? prev.lastRunAt,
lastDurationMs: patch.lastDurationMs ?? prev.lastDurationMs,
lastError: patch.lastError === undefined ? prev.lastError : patch.lastError,
updatedAt: new Date().toISOString(),
}).where(eq(backupScheduleSettings.id, SETTINGS_ID)).run()
}
export function resolveBackupServerIds(settings: BackupScheduleSettingsDto): string[] {
const enabled = new Set(listServersRead().map((s) => String(s.id)))
const requested = settings.serverIds.length > 0 ? settings.serverIds : [...enabled]
return [...new Set(requested)].filter((id) => enabled.has(id))
}
function sameLocalSlot(a: Date, b: Date): boolean {
return a.getFullYear() === b.getFullYear()
&& a.getMonth() === b.getMonth()
&& a.getDate() === b.getDate()
&& a.getHours() === b.getHours()
&& a.getMinutes() === b.getMinutes()
}
export function isBackupDue(now: Date, settings: BackupScheduleSettingsDto, lastRunAt: string | null | undefined): boolean {
if (!settings.enabled) return false
const slot = new Date(now)
slot.setSeconds(0, 0)
slot.setHours(settings.hour, settings.minute, 0, 0)
if (settings.frequency === "weekly") {
const currentDow = (now.getDay() + 6) % 7
if (currentDow !== settings.weekDay) return false
} else if (settings.frequency === "monthly") {
if (now.getDate() !== settings.monthDay) return false
}
if (now < slot) return false
if (lastRunAt) {
const prev = new Date(lastRunAt)
if (Number.isNaN(prev.getTime())) return true
if (settings.frequency === "daily" && sameLocalSlot(prev, slot)) return false
if (settings.frequency === "weekly" && sameLocalSlot(prev, slot)) return false
if (settings.frequency === "monthly" && prev.getFullYear() === slot.getFullYear() && prev.getMonth() === slot.getMonth() && prev.getDate() === slot.getDate()) return false
}
return true
}
export async function runBackupForServer(
id: string,
kind: BackupMeta["kind"],
notes?: string,
): Promise<BackupMeta> {
const serverIdNum = Number.parseInt(id, 10)
if (!Number.isFinite(serverIdNum)) {
throw new Error("Невалидный id сервера")
}
const row = getServerRowById(serverIdNum)
if (!row) {
throw new Error("Сервер не найден")
}
const client = MikrotikClient.fromServer(row)
const script = await client.exportConfigScript()
const ts = fmtTs()
const safeServer = row.name.replace(/[^a-zA-Z0-9._-]+/g, "_")
const filename = `${safeServer}_${ts}.rsc`
const filePath = path.join(BACKUPS_DIR, filename)
await writeFile(filePath, script, "utf8")
const st = await stat(filePath)
return {
id: randomUUID(),
serverId: String(row.id),
serverName: row.name,
filename,
sizeBytes: st.size,
createdAt: new Date().toISOString(),
kind,
notes,
}
}
export async function pruneBackupsForServer(serverId: string, keepCount: number): Promise<number> {
const rows = await readBackupIndex()
const forServer = rows.filter((r) => r.serverId === serverId)
if (forServer.length <= keepCount) return 0
const sorted = [...forServer].sort((a, b) => b.createdAt.localeCompare(a.createdAt))
const toDelete = sorted.slice(keepCount)
const deleteIds = new Set(toDelete.map((r) => r.id))
for (const hit of toDelete) {
await rm(path.join(BACKUPS_DIR, hit.filename), { force: true })
}
const next = rows.filter((r) => !deleteIds.has(r.id))
await writeBackupIndex(next)
return toDelete.length
}
export function getBackupsDir(): string {
return BACKUPS_DIR
}
export function getBackupIndexPath(): string {
return INDEX_PATH
}
@@ -0,0 +1,104 @@
import { appendEvent } from "../modules/events/service/events-service.js"
import { issueCertificateWithCloudflareDns } from "./acme-cloudflare.js"
import {
getIssueJobRecord,
getServerRowByIdString,
updateIssueJobRecord,
} from "./certificates-service.js"
const runningJobs = new Set<string>()
export async function runCertificateIssueJob(jobId: string): Promise<void> {
if (runningJobs.has(jobId)) return
runningJobs.add(jobId)
const row = getIssueJobRecord(jobId)
if (!row) {
runningJobs.delete(jobId)
return
}
const server = getServerRowByIdString(row.serverId)
if (!server) {
updateIssueJobRecord(jobId, {
status: "failed",
step: "failed",
finishedAt: new Date().toISOString(),
error: "Сервер не найден",
})
runningJobs.delete(jobId)
return
}
let domainNames: string[] = []
try {
domainNames = JSON.parse(row.domainNames) as string[]
} catch {
domainNames = []
}
const trustStore = row.trustStore.split(",").map((s) => s.trim()).filter(Boolean)
const startedAt = new Date().toISOString()
const isScheduler = row.source === "scheduler"
updateIssueJobRecord(jobId, { status: "running", step: "acme_order", startedAt, error: null })
if (isScheduler) {
appendEvent({
level: "info",
eventType: "certificates.renew.started",
sourceModule: "certificates",
title: "Запущено автообновление сертификата",
message: `${row.certName} · ${domainNames.join(", ")} · ${server.name}`,
entityType: "server",
entityId: String(server.id),
})
}
try {
await issueCertificateWithCloudflareDns({
server,
certName: row.certName,
domainNames,
keyType: row.keyType === "ec256" ? "ec256" : "rsa2048",
trustStore: trustStore.length > 0 ? trustStore : ["www", "api"],
onStep: (step) => updateIssueJobRecord(jobId, { step }),
})
updateIssueJobRecord(jobId, {
status: "done",
step: "done",
finishedAt: new Date().toISOString(),
error: null,
})
appendEvent({
level: "info",
eventType: isScheduler ? "certificates.renew.done" : "certificates.issue.done",
sourceModule: "certificates",
title: isScheduler ? "Сертификат обновлён" : "Сертификат выпущен",
message: `${row.certName} · ${domainNames.join(", ")} · ${server.name}`,
entityType: "server",
entityId: String(server.id),
})
} catch (e) {
const message = e instanceof Error ? e.message : String(e)
updateIssueJobRecord(jobId, {
status: "failed",
step: "failed",
finishedAt: new Date().toISOString(),
error: message,
})
appendEvent({
level: "warning",
eventType: isScheduler ? "certificates.renew.failed" : "certificates.issue.failed",
sourceModule: "certificates",
title: isScheduler ? "Ошибка автообновления сертификата" : "Ошибка выпуска сертификата",
message,
entityType: "server",
entityId: String(server.id),
})
} finally {
runningJobs.delete(jobId)
}
}
export function queueCertificateIssueJob(jobId: string): void {
queueMicrotask(() => { void runCertificateIssueJob(jobId) })
}
@@ -0,0 +1,158 @@
import { randomUUID } from "node:crypto"
import { appendEvent } from "../modules/events/service/events-service.js"
import type { CertificatesRenewRunSnapshot } from "../types/scheduler-run-snapshot.js"
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
import { mapRosCertificates } from "./certificate-parse.js"
import { runCertificateIssueJob } from "./certificate-issue-runner.js"
import {
createIssueJobRecord,
getCertificateRenewSettings,
getServerRowByIdString,
hasActiveIssueJob,
listManagedCertificateTargets,
touchCertificateRenewRunMeta,
} from "./certificates-service.js"
import { MikrotikClient } from "./mikrotik.js"
let collecting = false
export function getCertificateRenewCollectorState(): { running: boolean } {
return { running: collecting }
}
export async function collectCertificatesRenewOnce(): Promise<CertificatesRenewRunSnapshot> {
const sampledAt = new Date().toISOString()
const settings = getCertificateRenewSettings()
if (collecting) {
return {
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
job: "certificates_renew",
sampledAt,
skipped: true,
checked: 0,
renewed: 0,
skippedTargets: 0,
errors: [],
}
}
collecting = true
const started = Date.now()
const snapshot: CertificatesRenewRunSnapshot = {
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
job: "certificates_renew",
sampledAt,
checked: 0,
renewed: 0,
skippedTargets: 0,
errors: [],
targets: [],
}
try {
if (!settings.enabled) {
snapshot.skipped = true
return snapshot
}
const targets = listManagedCertificateTargets()
for (const target of targets) {
snapshot.checked += 1
const item = {
serverId: target.serverId,
certName: target.certName,
action: "ok" as "ok" | "renewed" | "skipped" | "error",
daysLeft: null as number | null,
message: "",
}
if (hasActiveIssueJob(target.serverId, target.certName)) {
item.action = "skipped"
item.message = "Уже выполняется выпуск"
snapshot.skippedTargets += 1
snapshot.targets?.push(item)
appendEvent({
level: "info",
eventType: "certificates.renew.skipped",
sourceModule: "certificates",
title: "Автообновление пропущено",
message: `${target.certName}: уже выполняется выпуск`,
entityType: "server",
entityId: target.serverId,
})
continue
}
const server = getServerRowByIdString(target.serverId)
if (!server) {
item.action = "error"
item.message = "Сервер не найден"
snapshot.errors.push(`${target.certName}: сервер не найден`)
snapshot.targets?.push(item)
continue
}
try {
const client = MikrotikClient.fromServer(server)
const rows = await client.getCertificates()
const mapped = mapRosCertificates(server.id, server.name, rows)
const hit = mapped.find((c) => c.name === target.certName)
if (!hit) {
item.action = "error"
item.message = "Сертификат не найден на устройстве"
snapshot.errors.push(`${target.certName}: не найден на ${server.name}`)
snapshot.targets?.push(item)
continue
}
item.daysLeft = hit.daysLeft
if (hit.daysLeft > settings.renewBeforeDays) {
item.action = "ok"
item.message = `До истечения ${hit.daysLeft} дн.`
snapshot.targets?.push(item)
continue
}
const jobId = randomUUID()
createIssueJobRecord({
id: jobId,
serverId: target.serverId,
certName: target.certName,
domainNames: target.domainNames,
keyType: target.keyType,
trustStore: target.trustStore,
source: "scheduler",
})
await runCertificateIssueJob(jobId)
snapshot.renewed += 1
item.action = "renewed"
item.message = "Запущено обновление"
snapshot.targets?.push(item)
} catch (e) {
const message = e instanceof Error ? e.message : String(e)
item.action = "error"
item.message = message
snapshot.errors.push(`${target.certName}: ${message}`)
snapshot.targets?.push(item)
}
}
touchCertificateRenewRunMeta({
lastCollectedAt: sampledAt,
lastDurationMs: Date.now() - started,
lastError: snapshot.errors.length > 0 ? snapshot.errors.join("; ") : null,
})
return snapshot
} catch (e) {
const message = e instanceof Error ? e.message : String(e)
snapshot.fatalError = message
touchCertificateRenewRunMeta({
lastCollectedAt: sampledAt,
lastDurationMs: Date.now() - started,
lastError: message,
})
return snapshot
} finally {
collecting = false
}
}
+119 -3
View File
@@ -1,12 +1,26 @@
import { eq } from "drizzle-orm"
import type { CertificateDto } from "@mmapp/contracts/certificates"
import { desc, eq, inArray } from "drizzle-orm"
import type { CertificateDto, CertificateRenewSettingsDto } from "@mmapp/contracts/certificates"
import { db } from "../db/index.js"
import { acmeSettings, certificateIssueJobs, servers } from "../db/schema.js"
import {
acmeSettings,
certificateIssueJobs,
certificateRenewSettings,
servers,
} from "../db/schema.js"
import { mapRosCertificates } from "./certificate-parse.js"
import { MikrotikClient } from "./mikrotik.js"
const SETTINGS_ID = 1
export type ManagedCertificateTarget = {
serverId: string
certName: string
domainNames: string[]
keyType: string
trustStore: string
finishedAt: string
}
export async function listCertificatesFromServers(): Promise<{
certificates: CertificateDto[]
failures: Array<{ serverId: string; serverName?: string; error: string }>
@@ -109,12 +123,14 @@ export function createIssueJobRecord(input: {
domainNames: string[]
keyType: string
trustStore: string
source?: "manual" | "scheduler"
}) {
const now = new Date().toISOString()
db.insert(certificateIssueJobs).values({
id: input.id,
status: "queued",
step: "queued",
source: input.source ?? "manual",
serverId: input.serverId,
certName: input.certName,
domainNames: JSON.stringify(input.domainNames),
@@ -168,3 +184,103 @@ export function getServerRowByIdString(serverId: string) {
if (!Number.isFinite(id)) return null
return db.select().from(servers).where(eq(servers.id, id)).limit(1).all()[0] ?? null
}
function getCertificateRenewSettingsRow() {
return db.select().from(certificateRenewSettings).where(eq(certificateRenewSettings.id, SETTINGS_ID)).limit(1).all()[0]
?? {
id: SETTINGS_ID,
enabled: true,
intervalSec: 21600,
renewBeforeDays: 30,
lastCollectedAt: null,
lastDurationMs: null,
lastError: null,
updatedAt: new Date().toISOString(),
}
}
export function getCertificateRenewSettings(): CertificateRenewSettingsDto {
const row = getCertificateRenewSettingsRow()
return {
enabled: row.enabled,
intervalSec: row.intervalSec,
renewBeforeDays: row.renewBeforeDays,
lastCollectedAt: row.lastCollectedAt ?? null,
lastDurationMs: row.lastDurationMs ?? null,
lastError: row.lastError ?? null,
updatedAt: row.updatedAt,
}
}
export function updateCertificateRenewSettings(patch: {
enabled?: boolean
intervalSec?: number
renewBeforeDays?: number
}) {
const prev = getCertificateRenewSettingsRow()
const now = new Date().toISOString()
const next = {
enabled: patch.enabled ?? prev.enabled,
intervalSec: patch.intervalSec ?? prev.intervalSec,
renewBeforeDays: patch.renewBeforeDays ?? prev.renewBeforeDays,
updatedAt: now,
}
if (db.select().from(certificateRenewSettings).where(eq(certificateRenewSettings.id, SETTINGS_ID)).limit(1).all()[0]) {
db.update(certificateRenewSettings).set(next).where(eq(certificateRenewSettings.id, SETTINGS_ID)).run()
} else {
db.insert(certificateRenewSettings).values({ id: SETTINGS_ID, ...next }).run()
}
return getCertificateRenewSettings()
}
export function touchCertificateRenewRunMeta(patch: {
lastCollectedAt?: string
lastDurationMs?: number
lastError?: string | null
}) {
const prev = getCertificateRenewSettingsRow()
db.update(certificateRenewSettings).set({
lastCollectedAt: patch.lastCollectedAt ?? prev.lastCollectedAt,
lastDurationMs: patch.lastDurationMs ?? prev.lastDurationMs,
lastError: patch.lastError === undefined ? prev.lastError : patch.lastError,
updatedAt: new Date().toISOString(),
}).where(eq(certificateRenewSettings.id, SETTINGS_ID)).run()
}
export function listManagedCertificateTargets(): ManagedCertificateTarget[] {
const rows = db.select().from(certificateIssueJobs)
.where(eq(certificateIssueJobs.status, "done"))
.orderBy(desc(certificateIssueJobs.finishedAt), desc(certificateIssueJobs.requestedAt))
.all()
const byKey = new Map<string, ManagedCertificateTarget>()
for (const row of rows) {
const key = `${row.serverId}::${row.certName}`
if (byKey.has(key)) continue
let domainNames: string[] = []
try {
const parsed = JSON.parse(row.domainNames) as unknown
if (Array.isArray(parsed)) domainNames = parsed.map(String)
} catch {
domainNames = []
}
byKey.set(key, {
serverId: row.serverId,
certName: row.certName,
domainNames,
keyType: row.keyType,
trustStore: row.trustStore,
finishedAt: row.finishedAt ?? row.requestedAt,
})
}
return [...byKey.values()]
}
export function hasActiveIssueJob(serverId: string, certName: string): boolean {
const row = db.select().from(certificateIssueJobs)
.where(inArray(certificateIssueJobs.status, ["queued", "running"]))
.orderBy(desc(certificateIssueJobs.requestedAt))
.all()
.find((r) => r.serverId === serverId && r.certName === certName)
return Boolean(row)
}
+40
View File
@@ -42,6 +42,10 @@ import {
collectInternetPathSnapshotOnce,
getInternetPathSettings,
} from "./internet-path-collector.js"
import { collectCertificatesRenewOnce } from "./certificate-renew-collector.js"
import { getCertificateRenewSettings } from "./certificates-service.js"
import { collectScheduledBackupsOnce } from "./backup-scheduler-collector.js"
import { getBackupScheduleSettings } from "./backup-service.js"
import {
endSchedulerJob,
isSchedulerJobRunning,
@@ -57,6 +61,8 @@ export const JOB_KEYS = [
"uptime_speed",
"internet_path",
"gre_bgp",
"certificates_renew",
"backups",
"alert_engine",
] as const
export type SchedulerJobKey = (typeof JOB_KEYS)[number]
@@ -119,6 +125,12 @@ async function runSchedulerJobBody(jobKey: SchedulerJobKey): Promise<void> {
case "internet_path":
snapshot = await collectInternetPathSnapshotOnce()
break
case "certificates_renew":
snapshot = await collectCertificatesRenewOnce()
break
case "backups":
snapshot = await collectScheduledBackupsOnce()
break
case "alert_engine": {
const r = await runAlertEngineOnce()
snapshot = {
@@ -322,6 +334,30 @@ export function refreshScheduler(): void {
}, greBgpMs),
)
const certRenew = getCertificateRenewSettings()
if (certRenew.enabled) {
const certRenewMs = Math.max(300_000, certRenew.intervalSec * 1000)
void executeSchedulerJob("certificates_renew").catch(() => {})
timers.set(
"certificates_renew",
setInterval(() => {
void executeSchedulerJob("certificates_renew").catch(() => {})
}, certRenewMs),
)
}
const backupSchedule = getBackupScheduleSettings()
if (backupSchedule.enabled) {
const backupMs = 60_000
void executeSchedulerJob("backups").catch(() => {})
timers.set(
"backups",
setInterval(() => {
void executeSchedulerJob("backups").catch(() => {})
}, backupMs),
)
}
const alertMs = 20_000
void executeSchedulerJob("alert_engine").catch(() => {})
timers.set(
@@ -353,6 +389,8 @@ export function getSchedulerStatus() {
const uptime = getUptimeSettings()
const apiPing = getServersApiPingSettings()
const internetPath = getInternetPathSettings()
const certRenew = getCertificateRenewSettings()
const backupSchedule = getBackupScheduleSettings()
const resOn = uptime.resourcesEnabled ?? uptime.enabled
const pingOn = uptime.pingEnabled ?? uptime.enabled
@@ -366,6 +404,8 @@ export function getSchedulerStatus() {
uptime_speed: { enabled: spdOn, intervalSec: uptime.speedIntervalSec },
internet_path: { enabled: internetPath.enabled, intervalSec: internetPath.intervalSec },
gre_bgp: { enabled: true, intervalSec: 30 },
certificates_renew: { enabled: certRenew.enabled, intervalSec: certRenew.intervalSec },
backups: { enabled: backupSchedule.enabled, intervalSec: 60 },
alert_engine: { enabled: true, intervalSec: 20 },
}
@@ -192,6 +192,40 @@ export interface InternetPathRunSnapshot {
fatalError?: string
}
export interface CertificatesRenewTargetSnapshot {
serverId: string
certName: string
action: "ok" | "renewed" | "skipped" | "error"
daysLeft: number | null
message: string
}
export interface CertificatesRenewRunSnapshot {
v: typeof SCHEDULER_RUN_SNAPSHOT_VERSION
job: "certificates_renew"
sampledAt: string
skipped?: boolean
fatalError?: string
checked: number
renewed: number
skippedTargets: number
errors: string[]
targets?: CertificatesRenewTargetSnapshot[]
}
export interface BackupsRunSnapshot {
v: typeof SCHEDULER_RUN_SNAPSHOT_VERSION
job: "backups"
sampledAt: string
skipped?: boolean
due: boolean
created: number
failures: number
pruned: number
errors?: string[]
fatalError?: string
}
export type SchedulerRunSnapshot =
| TrafficRunSnapshot
| ResourcesRunSnapshot
@@ -200,4 +234,6 @@ export type SchedulerRunSnapshot =
| ServersRestPingRunSnapshot
| GreBgpSnapshotRunSnapshot
| InternetPathRunSnapshot
| CertificatesRenewRunSnapshot
| BackupsRunSnapshot
| AlertEngineRunSnapshot