feat(scheduler): добавить расписание бэкапов и автообновление сертификатов
Docker images / prepare-release (push) Successful in 6s
Docker images / backend-image (push) Successful in 2m13s
Docker images / frontend-image (push) Successful in 2m11s
Docker images / updater-image (push) Successful in 50s
Docker images / notify-webhook (push) Has been skipped
Docker images / publish-release (push) Successful in 8s
Docker images / prepare-release (push) Successful in 6s
Docker images / backend-image (push) Successful in 2m13s
Docker images / frontend-image (push) Successful in 2m11s
Docker images / updater-image (push) Successful in 50s
Docker images / notify-webhook (push) Has been skipped
Docker images / publish-release (push) Successful in 8s
This commit is contained in:
@@ -351,6 +351,7 @@ CREATE TABLE IF NOT EXISTS certificate_issue_jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
status TEXT NOT NULL DEFAULT 'queued',
|
||||
step TEXT NOT NULL DEFAULT 'queued',
|
||||
source TEXT NOT NULL DEFAULT 'manual',
|
||||
server_id TEXT NOT NULL,
|
||||
cert_name TEXT NOT NULL,
|
||||
domain_names TEXT NOT NULL,
|
||||
@@ -362,6 +363,34 @@ CREATE TABLE IF NOT EXISTS certificate_issue_jobs (
|
||||
error TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS certificate_renew_settings (
|
||||
id INTEGER PRIMARY KEY,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
interval_sec INTEGER NOT NULL DEFAULT 21600,
|
||||
renew_before_days INTEGER NOT NULL DEFAULT 30,
|
||||
last_collected_at TEXT,
|
||||
last_duration_ms INTEGER,
|
||||
last_error TEXT,
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS backup_schedule_settings (
|
||||
id INTEGER PRIMARY KEY,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
frequency TEXT NOT NULL DEFAULT 'daily',
|
||||
hour INTEGER NOT NULL DEFAULT 3,
|
||||
minute INTEGER NOT NULL DEFAULT 0,
|
||||
week_day INTEGER NOT NULL DEFAULT 0,
|
||||
month_day INTEGER NOT NULL DEFAULT 1,
|
||||
keep_count INTEGER NOT NULL DEFAULT 7,
|
||||
format TEXT NOT NULL DEFAULT 'rsc',
|
||||
server_ids_json TEXT NOT NULL DEFAULT '[]',
|
||||
last_run_at TEXT,
|
||||
last_duration_ms INTEGER,
|
||||
last_error TEXT,
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alert_rules (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
@@ -617,6 +646,23 @@ SELECT 1, 'https://acme-v02.api.letsencrypt.org/directory', '', '', ''
|
||||
WHERE NOT EXISTS (SELECT 1 FROM acme_settings WHERE id = 1);
|
||||
`)
|
||||
|
||||
const certIssueJobCols = sqlite.prepare(`PRAGMA table_info('certificate_issue_jobs')`).all() as Array<{ name?: string }>
|
||||
if (!certIssueJobCols.some((c) => c.name === "source")) {
|
||||
sqlite.exec(`ALTER TABLE certificate_issue_jobs ADD COLUMN source TEXT NOT NULL DEFAULT 'manual'`)
|
||||
}
|
||||
|
||||
sqlite.exec(`
|
||||
INSERT INTO certificate_renew_settings (id, enabled, interval_sec, renew_before_days)
|
||||
SELECT 1, 1, 21600, 30
|
||||
WHERE NOT EXISTS (SELECT 1 FROM certificate_renew_settings WHERE id = 1);
|
||||
`)
|
||||
|
||||
sqlite.exec(`
|
||||
INSERT INTO backup_schedule_settings (id, enabled, frequency, hour, minute, week_day, month_day, keep_count, format, server_ids_json)
|
||||
SELECT 1, 1, 'daily', 3, 0, 0, 1, 7, 'rsc', '[]'
|
||||
WHERE NOT EXISTS (SELECT 1 FROM backup_schedule_settings WHERE id = 1);
|
||||
`)
|
||||
|
||||
sqlite.exec(`
|
||||
INSERT INTO alert_engine_cursor (id, last_source_finished_at)
|
||||
SELECT 1, NULL
|
||||
|
||||
@@ -300,6 +300,7 @@ export const certificateIssueJobs = sqliteTable("certificate_issue_jobs", {
|
||||
id: text("id").primaryKey(),
|
||||
status: text("status", { enum: ["queued", "running", "done", "failed"] }).notNull().default("queued"),
|
||||
step: text("step").notNull().default("queued"),
|
||||
source: text("source", { enum: ["manual", "scheduler"] }).notNull().default("manual"),
|
||||
serverId: text("server_id").notNull(),
|
||||
certName: text("cert_name").notNull(),
|
||||
domainNames: text("domain_names").notNull(),
|
||||
@@ -311,6 +312,34 @@ export const certificateIssueJobs = sqliteTable("certificate_issue_jobs", {
|
||||
error: text("error"),
|
||||
})
|
||||
|
||||
export const certificateRenewSettings = sqliteTable("certificate_renew_settings", {
|
||||
id: integer("id").primaryKey(),
|
||||
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
|
||||
intervalSec: integer("interval_sec").notNull().default(21600),
|
||||
renewBeforeDays: integer("renew_before_days").notNull().default(30),
|
||||
lastCollectedAt: text("last_collected_at"),
|
||||
lastDurationMs: integer("last_duration_ms"),
|
||||
lastError: text("last_error"),
|
||||
updatedAt: text("updated_at").notNull().default(sql`(datetime('now'))`),
|
||||
})
|
||||
|
||||
export const backupScheduleSettings = sqliteTable("backup_schedule_settings", {
|
||||
id: integer("id").primaryKey(),
|
||||
enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
|
||||
frequency: text("frequency", { enum: ["daily", "weekly", "monthly"] }).notNull().default("daily"),
|
||||
hour: integer("hour").notNull().default(3),
|
||||
minute: integer("minute").notNull().default(0),
|
||||
weekDay: integer("week_day").notNull().default(0),
|
||||
monthDay: integer("month_day").notNull().default(1),
|
||||
keepCount: integer("keep_count").notNull().default(7),
|
||||
format: text("format", { enum: ["rsc", "backup"] }).notNull().default("rsc"),
|
||||
serverIdsJson: text("server_ids_json").notNull().default("[]"),
|
||||
lastRunAt: text("last_run_at"),
|
||||
lastDurationMs: integer("last_duration_ms"),
|
||||
lastError: text("last_error"),
|
||||
updatedAt: text("updated_at").notNull().default(sql`(datetime('now'))`),
|
||||
})
|
||||
|
||||
/** Группы правил: ANY = хотя бы одно; ALL = все одновременно в окне тика. */
|
||||
export const alertGroups = sqliteTable("alert_groups", {
|
||||
id: text("id").primaryKey(),
|
||||
@@ -531,6 +560,8 @@ export type EvobgpSettingsRow = typeof evobgpSettings.$inferSelect
|
||||
export type AlertTelegramSettingsRow = typeof alertTelegramSettings.$inferSelect
|
||||
export type AcmeSettingsRow = typeof acmeSettings.$inferSelect
|
||||
export type CertificateIssueJobRow = typeof certificateIssueJobs.$inferSelect
|
||||
export type CertificateRenewSettingsRow = typeof certificateRenewSettings.$inferSelect
|
||||
export type BackupScheduleSettingsRow = typeof backupScheduleSettings.$inferSelect
|
||||
export type AlertGroupRow = typeof alertGroups.$inferSelect
|
||||
export type AlertRuleRow = typeof alertRules.$inferSelect
|
||||
export type AlertRuleTargetRow = typeof alertRuleTargets.$inferSelect
|
||||
|
||||
@@ -1,23 +1,21 @@
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { mkdir, readFile, rm, stat, writeFile } from "node:fs/promises"
|
||||
import { readFile, rm } from "node:fs/promises"
|
||||
import path from "node:path"
|
||||
import { z } from "zod"
|
||||
import type { FastifyPluginAsyncZod } from "@fastify/type-provider-zod"
|
||||
import { putBackupScheduleSettingsSchema } from "@mmapp/contracts/backups"
|
||||
import { listServersRead } from "../modules/servers/service/servers-service.js"
|
||||
import { getServerRowById } from "../modules/servers/repository/servers-repository.js"
|
||||
import { MikrotikClient } from "../services/mikrotik.js"
|
||||
import { appendEvent } from "../modules/events/service/events-service.js"
|
||||
|
||||
type BackupMeta = {
|
||||
id: string
|
||||
serverId: string
|
||||
serverName: string
|
||||
filename: string
|
||||
sizeBytes: number
|
||||
createdAt: string
|
||||
kind: "manual"
|
||||
notes?: string
|
||||
}
|
||||
import { refreshScheduler } from "../services/scheduler.js"
|
||||
import {
|
||||
getBackupsDir,
|
||||
getBackupScheduleSettings,
|
||||
readBackupIndex,
|
||||
runBackupForServer,
|
||||
updateBackupScheduleSettings,
|
||||
writeBackupIndex,
|
||||
type BackupMeta,
|
||||
} from "../services/backup-service.js"
|
||||
|
||||
type BackupJobStatus = "queued" | "running" | "done" | "failed"
|
||||
type BackupJob = {
|
||||
@@ -32,36 +30,8 @@ type BackupJob = {
|
||||
failures: Array<{ serverId: string; error: string }>
|
||||
}
|
||||
|
||||
const BACKUPS_DIR = path.resolve(process.cwd(), "storage", "backups")
|
||||
const INDEX_PATH = path.join(BACKUPS_DIR, "index.json")
|
||||
const backupJobs = new Map<string, BackupJob>()
|
||||
|
||||
async function ensureStorage() {
|
||||
await mkdir(BACKUPS_DIR, { recursive: true })
|
||||
}
|
||||
|
||||
async function readIndex(): Promise<BackupMeta[]> {
|
||||
await ensureStorage()
|
||||
try {
|
||||
const raw = await readFile(INDEX_PATH, "utf8")
|
||||
const parsed = JSON.parse(raw) as unknown
|
||||
if (!Array.isArray(parsed)) return []
|
||||
return parsed as BackupMeta[]
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
async function writeIndex(rows: BackupMeta[]): Promise<void> {
|
||||
await ensureStorage()
|
||||
await writeFile(INDEX_PATH, JSON.stringify(rows, null, 2), "utf8")
|
||||
}
|
||||
|
||||
function fmtTs(d = new Date()): string {
|
||||
const p = (n: number) => String(n).padStart(2, "0")
|
||||
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}_${p(d.getHours())}-${p(d.getMinutes())}-${p(d.getSeconds())}`
|
||||
}
|
||||
|
||||
const CreateBackupBodySchema = z.object({
|
||||
serverIds: z.array(z.union([z.string(), z.number()])).min(1),
|
||||
notes: z.string().max(500).optional(),
|
||||
@@ -74,42 +44,13 @@ const BackupJobIdParamSchema = z.object({
|
||||
jobId: z.string().min(1),
|
||||
})
|
||||
|
||||
async function runBackupForServer(id: string, notes?: string): Promise<BackupMeta> {
|
||||
const serverIdNum = Number.parseInt(id, 10)
|
||||
if (!Number.isFinite(serverIdNum)) {
|
||||
throw new Error("Невалидный id сервера")
|
||||
}
|
||||
const row = getServerRowById(serverIdNum)
|
||||
if (!row) {
|
||||
throw new Error("Сервер не найден")
|
||||
}
|
||||
const client = MikrotikClient.fromServer(row)
|
||||
const script = await client.exportConfigScript()
|
||||
const ts = fmtTs()
|
||||
const safeServer = row.name.replace(/[^a-zA-Z0-9._-]+/g, "_")
|
||||
const filename = `${safeServer}_${ts}.rsc`
|
||||
const filePath = path.join(BACKUPS_DIR, filename)
|
||||
await writeFile(filePath, script, "utf8")
|
||||
const st = await stat(filePath)
|
||||
return {
|
||||
id: randomUUID(),
|
||||
serverId: String(row.id),
|
||||
serverName: row.name,
|
||||
filename,
|
||||
sizeBytes: st.size,
|
||||
createdAt: new Date().toISOString(),
|
||||
kind: "manual",
|
||||
notes,
|
||||
}
|
||||
}
|
||||
|
||||
async function processBackupJob(job: BackupJob, ids: string[], notes?: string) {
|
||||
job.status = "running"
|
||||
job.startedAt = new Date().toISOString()
|
||||
const indexRows = await readIndex()
|
||||
const indexRows = await readBackupIndex()
|
||||
for (const id of ids) {
|
||||
try {
|
||||
const meta = await runBackupForServer(id, notes)
|
||||
const meta = await runBackupForServer(id, "manual", notes)
|
||||
indexRows.unshift(meta)
|
||||
job.created.push(meta)
|
||||
} catch (err) {
|
||||
@@ -119,7 +60,7 @@ async function processBackupJob(job: BackupJob, ids: string[], notes?: string) {
|
||||
job.completed += 1
|
||||
}
|
||||
}
|
||||
await writeIndex(indexRows)
|
||||
await writeBackupIndex(indexRows)
|
||||
job.status = "done"
|
||||
job.finishedAt = new Date().toISOString()
|
||||
appendEvent({
|
||||
@@ -141,11 +82,25 @@ async function processBackupJob(job: BackupJob, ids: string[], notes?: string) {
|
||||
|
||||
const backupsRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
app.get("/backups", async (_req, reply) => {
|
||||
const rows = await readIndex()
|
||||
const rows = await readBackupIndex()
|
||||
rows.sort((a, b) => b.createdAt.localeCompare(a.createdAt))
|
||||
return reply.send(rows)
|
||||
})
|
||||
|
||||
app.get("/backups/schedule", async (_req, reply) => {
|
||||
return reply.send(getBackupScheduleSettings())
|
||||
})
|
||||
|
||||
app.put("/backups/schedule", async (req, reply) => {
|
||||
const parsed = putBackupScheduleSettingsSchema.safeParse(req.body ?? {})
|
||||
if (!parsed.success) {
|
||||
return reply.status(400).send({ error: "Некорректное тело запроса", details: parsed.error.flatten() })
|
||||
}
|
||||
const result = updateBackupScheduleSettings(parsed.data)
|
||||
refreshScheduler()
|
||||
return reply.send(result)
|
||||
})
|
||||
|
||||
app.post("/backups/create", { schema: { body: CreateBackupBodySchema } }, async (req, reply) => {
|
||||
const inputIds = req.body.serverIds.map((x) => String(x))
|
||||
const notes = req.body.notes?.trim() || undefined
|
||||
@@ -216,10 +171,10 @@ const backupsRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
})
|
||||
|
||||
app.get("/backups/:id/download", { schema: { params: BackupIdParamSchema } }, async (req, reply) => {
|
||||
const rows = await readIndex()
|
||||
const rows = await readBackupIndex()
|
||||
const hit = rows.find((r) => r.id === req.params.id)
|
||||
if (!hit) return reply.status(404).send({ error: "Бэкап не найден" })
|
||||
const filePath = path.join(BACKUPS_DIR, hit.filename)
|
||||
const filePath = path.join(getBackupsDir(), hit.filename)
|
||||
const content = await readFile(filePath, "utf8").catch(() => null)
|
||||
if (content == null) return reply.status(404).send({ error: "Файл бэкапа не найден" })
|
||||
reply.header("Content-Type", "text/plain; charset=utf-8")
|
||||
@@ -228,12 +183,12 @@ const backupsRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
})
|
||||
|
||||
app.delete("/backups/:id", { schema: { params: BackupIdParamSchema } }, async (req, reply) => {
|
||||
const rows = await readIndex()
|
||||
const rows = await readBackupIndex()
|
||||
const idx = rows.findIndex((r) => r.id === req.params.id)
|
||||
if (idx < 0) return reply.status(404).send({ error: "Бэкап не найден" })
|
||||
const [hit] = rows.splice(idx, 1)
|
||||
await writeIndex(rows)
|
||||
await rm(path.join(BACKUPS_DIR, hit.filename), { force: true })
|
||||
await writeBackupIndex(rows)
|
||||
await rm(path.join(getBackupsDir(), hit.filename), { force: true })
|
||||
return reply.status(204).send()
|
||||
})
|
||||
}
|
||||
|
||||
@@ -3,101 +3,25 @@ import type { FastifyPluginAsyncZod } from "@fastify/type-provider-zod"
|
||||
import {
|
||||
certificateIssueRequestSchema,
|
||||
putAcmeCloudflareSettingsSchema,
|
||||
putCertificateRenewSettingsSchema,
|
||||
testAcmeCloudflareSettingsSchema,
|
||||
} from "@mmapp/contracts/certificates"
|
||||
import { appendEvent } from "../modules/events/service/events-service.js"
|
||||
import { issueCertificateWithCloudflareDns, testCloudflareToken } from "../services/acme-cloudflare.js"
|
||||
import { refreshScheduler } from "../services/scheduler.js"
|
||||
import { testCloudflareToken } from "../services/acme-cloudflare.js"
|
||||
import { queueCertificateIssueJob } from "../services/certificate-issue-runner.js"
|
||||
import {
|
||||
createIssueJobRecord,
|
||||
getAcmeCloudflareToken,
|
||||
getAcmeSettingsPublic,
|
||||
getCertificateRenewSettings,
|
||||
getIssueJobRecord,
|
||||
getServerRowByIdString,
|
||||
listCertificatesFromServers,
|
||||
toIssueJobDto,
|
||||
updateAcmeSettings,
|
||||
updateIssueJobRecord,
|
||||
updateCertificateRenewSettings,
|
||||
} from "../services/certificates-service.js"
|
||||
|
||||
const runningJobs = new Set<string>()
|
||||
|
||||
async function runIssueJob(jobId: string) {
|
||||
if (runningJobs.has(jobId)) return
|
||||
runningJobs.add(jobId)
|
||||
const row = getIssueJobRecord(jobId)
|
||||
if (!row) {
|
||||
runningJobs.delete(jobId)
|
||||
return
|
||||
}
|
||||
|
||||
const server = getServerRowByIdString(row.serverId)
|
||||
if (!server) {
|
||||
updateIssueJobRecord(jobId, {
|
||||
status: "failed",
|
||||
step: "failed",
|
||||
finishedAt: new Date().toISOString(),
|
||||
error: "Сервер не найден",
|
||||
})
|
||||
runningJobs.delete(jobId)
|
||||
return
|
||||
}
|
||||
|
||||
let domainNames: string[] = []
|
||||
try {
|
||||
domainNames = JSON.parse(row.domainNames) as string[]
|
||||
} catch {
|
||||
domainNames = []
|
||||
}
|
||||
|
||||
const trustStore = row.trustStore.split(",").map((s) => s.trim()).filter(Boolean)
|
||||
const startedAt = new Date().toISOString()
|
||||
updateIssueJobRecord(jobId, { status: "running", step: "acme_order", startedAt, error: null })
|
||||
|
||||
try {
|
||||
await issueCertificateWithCloudflareDns({
|
||||
server,
|
||||
certName: row.certName,
|
||||
domainNames,
|
||||
keyType: row.keyType === "ec256" ? "ec256" : "rsa2048",
|
||||
trustStore: trustStore.length > 0 ? trustStore : ["www", "api"],
|
||||
onStep: (step) => updateIssueJobRecord(jobId, { step }),
|
||||
})
|
||||
updateIssueJobRecord(jobId, {
|
||||
status: "done",
|
||||
step: "done",
|
||||
finishedAt: new Date().toISOString(),
|
||||
error: null,
|
||||
})
|
||||
appendEvent({
|
||||
level: "info",
|
||||
eventType: "certificates.issue.done",
|
||||
sourceModule: "certificates",
|
||||
title: "Сертификат выпущен",
|
||||
message: `${row.certName} · ${domainNames.join(", ")} · ${server.name}`,
|
||||
entityType: "server",
|
||||
entityId: String(server.id),
|
||||
})
|
||||
} catch (e) {
|
||||
updateIssueJobRecord(jobId, {
|
||||
status: "failed",
|
||||
step: "failed",
|
||||
finishedAt: new Date().toISOString(),
|
||||
error: e instanceof Error ? e.message : String(e),
|
||||
})
|
||||
appendEvent({
|
||||
level: "warning",
|
||||
eventType: "certificates.issue.failed",
|
||||
sourceModule: "certificates",
|
||||
title: "Ошибка выпуска сертификата",
|
||||
message: e instanceof Error ? e.message : String(e),
|
||||
entityType: "server",
|
||||
entityId: String(server.id),
|
||||
})
|
||||
} finally {
|
||||
runningJobs.delete(jobId)
|
||||
}
|
||||
}
|
||||
|
||||
const certificatesRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
app.get("/certificates", async (_req, reply) => {
|
||||
return reply.send(await listCertificatesFromServers())
|
||||
@@ -119,6 +43,20 @@ const certificatesRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
return reply.send(updateAcmeSettings(parsed.data))
|
||||
})
|
||||
|
||||
app.get("/certificates/renew-settings", async (_req, reply) => {
|
||||
return reply.send(getCertificateRenewSettings())
|
||||
})
|
||||
|
||||
app.put("/certificates/renew-settings", async (req, reply) => {
|
||||
const parsed = putCertificateRenewSettingsSchema.safeParse(req.body ?? {})
|
||||
if (!parsed.success) {
|
||||
return reply.status(400).send({ error: "Некорректное тело запроса", details: parsed.error.flatten() })
|
||||
}
|
||||
const result = updateCertificateRenewSettings(parsed.data)
|
||||
refreshScheduler()
|
||||
return reply.send(result)
|
||||
})
|
||||
|
||||
app.post("/certificates/acme-settings/test", async (req, reply) => {
|
||||
const parsed = testAcmeCloudflareSettingsSchema.safeParse(req.body ?? {})
|
||||
if (!parsed.success) {
|
||||
@@ -157,8 +95,9 @@ const certificatesRoutes: FastifyPluginAsyncZod = async (app) => {
|
||||
domainNames: parsed.data.domainNames.map((d) => d.trim()).filter(Boolean),
|
||||
keyType: parsed.data.keyType ?? "rsa2048",
|
||||
trustStore,
|
||||
source: "manual",
|
||||
})
|
||||
queueMicrotask(() => { void runIssueJob(jobId) })
|
||||
queueCertificateIssueJob(jobId)
|
||||
return reply.send({ jobId })
|
||||
})
|
||||
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
import { appendEvent } from "../modules/events/service/events-service.js"
|
||||
import type { BackupsRunSnapshot } from "../types/scheduler-run-snapshot.js"
|
||||
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
|
||||
import {
|
||||
getBackupScheduleSettings,
|
||||
isBackupDue,
|
||||
pruneBackupsForServer,
|
||||
readBackupIndex,
|
||||
resolveBackupServerIds,
|
||||
runBackupForServer,
|
||||
touchBackupScheduleRunMeta,
|
||||
writeBackupIndex,
|
||||
} from "./backup-service.js"
|
||||
|
||||
let collecting = false
|
||||
|
||||
export function getBackupSchedulerCollectorState(): { running: boolean } {
|
||||
return { running: collecting }
|
||||
}
|
||||
|
||||
export async function collectScheduledBackupsOnce(): Promise<BackupsRunSnapshot> {
|
||||
const sampledAt = new Date().toISOString()
|
||||
const settings = getBackupScheduleSettings()
|
||||
const snapshot: BackupsRunSnapshot = {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "backups",
|
||||
sampledAt,
|
||||
due: false,
|
||||
created: 0,
|
||||
failures: 0,
|
||||
pruned: 0,
|
||||
errors: [],
|
||||
}
|
||||
|
||||
if (collecting) {
|
||||
snapshot.skipped = true
|
||||
return snapshot
|
||||
}
|
||||
|
||||
if (!settings.enabled) {
|
||||
snapshot.skipped = true
|
||||
return snapshot
|
||||
}
|
||||
|
||||
const due = isBackupDue(new Date(), settings, settings.lastRunAt)
|
||||
snapshot.due = due
|
||||
if (!due) {
|
||||
return snapshot
|
||||
}
|
||||
|
||||
if (settings.format !== "rsc") {
|
||||
snapshot.skipped = true
|
||||
snapshot.errors = ["Формат backup пока не поддерживается, используйте rsc"]
|
||||
touchBackupScheduleRunMeta({
|
||||
lastRunAt: sampledAt,
|
||||
lastDurationMs: 0,
|
||||
lastError: snapshot.errors[0],
|
||||
})
|
||||
return snapshot
|
||||
}
|
||||
|
||||
collecting = true
|
||||
const started = Date.now()
|
||||
const serverIds = resolveBackupServerIds(settings)
|
||||
const indexRows = await readBackupIndex()
|
||||
|
||||
appendEvent({
|
||||
level: "info",
|
||||
eventType: "backups.job.started",
|
||||
sourceModule: "backups",
|
||||
title: "Запущен плановый бэкап",
|
||||
message: `Серверов в очереди: ${serverIds.length}`,
|
||||
entityType: "backup_job",
|
||||
entityId: "scheduler",
|
||||
payload: { serverIds, scheduled: true },
|
||||
})
|
||||
|
||||
try {
|
||||
for (const id of serverIds) {
|
||||
try {
|
||||
const meta = await runBackupForServer(id, "auto")
|
||||
indexRows.unshift(meta)
|
||||
snapshot.created += 1
|
||||
} catch (e) {
|
||||
const message = e instanceof Error ? e.message : String(e)
|
||||
snapshot.failures += 1
|
||||
snapshot.errors?.push(`${id}: ${message}`)
|
||||
}
|
||||
}
|
||||
|
||||
await writeBackupIndex(indexRows)
|
||||
|
||||
for (const id of serverIds) {
|
||||
snapshot.pruned += await pruneBackupsForServer(id, settings.keepCount)
|
||||
}
|
||||
|
||||
touchBackupScheduleRunMeta({
|
||||
lastRunAt: sampledAt,
|
||||
lastDurationMs: Date.now() - started,
|
||||
lastError: snapshot.errors?.length ? snapshot.errors.join("; ") : null,
|
||||
})
|
||||
|
||||
appendEvent({
|
||||
level: snapshot.failures > 0 ? "warning" : "info",
|
||||
eventType: "backups.job.done",
|
||||
sourceModule: "backups",
|
||||
title: snapshot.failures > 0 ? "Плановый бэкап завершен с ошибками" : "Плановый бэкап завершен",
|
||||
message: `Создано: ${snapshot.created}, ошибок: ${snapshot.failures}`,
|
||||
entityType: "backup_job",
|
||||
entityId: "scheduler",
|
||||
payload: {
|
||||
total: serverIds.length,
|
||||
completed: snapshot.created + snapshot.failures,
|
||||
failures: snapshot.errors,
|
||||
pruned: snapshot.pruned,
|
||||
},
|
||||
})
|
||||
|
||||
return snapshot
|
||||
} catch (e) {
|
||||
const message = e instanceof Error ? e.message : String(e)
|
||||
snapshot.fatalError = message
|
||||
snapshot.errors?.push(message)
|
||||
touchBackupScheduleRunMeta({
|
||||
lastRunAt: sampledAt,
|
||||
lastDurationMs: Date.now() - started,
|
||||
lastError: message,
|
||||
})
|
||||
appendEvent({
|
||||
level: "critical",
|
||||
eventType: "backups.job.failed",
|
||||
sourceModule: "backups",
|
||||
title: "Плановый бэкап прерван",
|
||||
message,
|
||||
entityType: "backup_job",
|
||||
entityId: "scheduler",
|
||||
})
|
||||
return snapshot
|
||||
} finally {
|
||||
collecting = false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { mkdir, readFile, rm, stat, writeFile } from "node:fs/promises"
|
||||
import path from "node:path"
|
||||
import { eq } from "drizzle-orm"
|
||||
import type { BackupScheduleSettingsDto } from "@mmapp/contracts/backups"
|
||||
import { db } from "../db/index.js"
|
||||
import { backupScheduleSettings } from "../db/schema.js"
|
||||
import { getServerRowById } from "../modules/servers/repository/servers-repository.js"
|
||||
import { listServersRead } from "../modules/servers/service/servers-service.js"
|
||||
import { MikrotikClient } from "./mikrotik.js"
|
||||
|
||||
const SETTINGS_ID = 1
|
||||
const BACKUPS_DIR = path.resolve(process.cwd(), "storage", "backups")
|
||||
const INDEX_PATH = path.join(BACKUPS_DIR, "index.json")
|
||||
|
||||
export type BackupMeta = {
|
||||
id: string
|
||||
serverId: string
|
||||
serverName: string
|
||||
filename: string
|
||||
sizeBytes: number
|
||||
createdAt: string
|
||||
kind: "manual" | "auto"
|
||||
notes?: string
|
||||
}
|
||||
|
||||
export async function ensureBackupStorage(): Promise<void> {
|
||||
await mkdir(BACKUPS_DIR, { recursive: true })
|
||||
}
|
||||
|
||||
export async function readBackupIndex(): Promise<BackupMeta[]> {
|
||||
await ensureBackupStorage()
|
||||
try {
|
||||
const raw = await readFile(INDEX_PATH, "utf8")
|
||||
const parsed = JSON.parse(raw) as unknown
|
||||
if (!Array.isArray(parsed)) return []
|
||||
return parsed as BackupMeta[]
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
export async function writeBackupIndex(rows: BackupMeta[]): Promise<void> {
|
||||
await ensureBackupStorage()
|
||||
await writeFile(INDEX_PATH, JSON.stringify(rows, null, 2), "utf8")
|
||||
}
|
||||
|
||||
function fmtTs(d = new Date()): string {
|
||||
const p = (n: number) => String(n).padStart(2, "0")
|
||||
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}_${p(d.getHours())}-${p(d.getMinutes())}-${p(d.getSeconds())}`
|
||||
}
|
||||
|
||||
function parseServerIds(raw: string): string[] {
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as unknown
|
||||
if (!Array.isArray(parsed)) return []
|
||||
return parsed.map(String).filter(Boolean)
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
function getBackupScheduleSettingsRow() {
|
||||
return db.select().from(backupScheduleSettings).where(eq(backupScheduleSettings.id, SETTINGS_ID)).limit(1).all()[0]
|
||||
?? {
|
||||
id: SETTINGS_ID,
|
||||
enabled: true,
|
||||
frequency: "daily" as const,
|
||||
hour: 3,
|
||||
minute: 0,
|
||||
weekDay: 0,
|
||||
monthDay: 1,
|
||||
keepCount: 7,
|
||||
format: "rsc" as const,
|
||||
serverIdsJson: "[]",
|
||||
lastRunAt: null,
|
||||
lastDurationMs: null,
|
||||
lastError: null,
|
||||
updatedAt: new Date().toISOString(),
|
||||
}
|
||||
}
|
||||
|
||||
export function getBackupScheduleSettings(): BackupScheduleSettingsDto {
|
||||
const row = getBackupScheduleSettingsRow()
|
||||
return {
|
||||
enabled: row.enabled,
|
||||
frequency: row.frequency,
|
||||
hour: row.hour,
|
||||
minute: row.minute,
|
||||
weekDay: row.weekDay,
|
||||
monthDay: row.monthDay,
|
||||
keepCount: row.keepCount,
|
||||
format: row.format,
|
||||
serverIds: parseServerIds(row.serverIdsJson),
|
||||
lastRunAt: row.lastRunAt ?? null,
|
||||
lastDurationMs: row.lastDurationMs ?? null,
|
||||
lastError: row.lastError ?? null,
|
||||
updatedAt: row.updatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
export function updateBackupScheduleSettings(patch: Partial<{
|
||||
enabled: boolean
|
||||
frequency: "daily" | "weekly" | "monthly"
|
||||
hour: number
|
||||
minute: number
|
||||
weekDay: number
|
||||
monthDay: number
|
||||
keepCount: number
|
||||
format: "rsc" | "backup"
|
||||
serverIds: string[]
|
||||
}>) {
|
||||
const prev = getBackupScheduleSettingsRow()
|
||||
const now = new Date().toISOString()
|
||||
const next = {
|
||||
enabled: patch.enabled ?? prev.enabled,
|
||||
frequency: patch.frequency ?? prev.frequency,
|
||||
hour: patch.hour ?? prev.hour,
|
||||
minute: patch.minute ?? prev.minute,
|
||||
weekDay: patch.weekDay ?? prev.weekDay,
|
||||
monthDay: patch.monthDay ?? prev.monthDay,
|
||||
keepCount: patch.keepCount ?? prev.keepCount,
|
||||
format: patch.format ?? prev.format,
|
||||
serverIdsJson: patch.serverIds ? JSON.stringify(patch.serverIds) : prev.serverIdsJson,
|
||||
updatedAt: now,
|
||||
}
|
||||
if (db.select().from(backupScheduleSettings).where(eq(backupScheduleSettings.id, SETTINGS_ID)).limit(1).all()[0]) {
|
||||
db.update(backupScheduleSettings).set(next).where(eq(backupScheduleSettings.id, SETTINGS_ID)).run()
|
||||
} else {
|
||||
db.insert(backupScheduleSettings).values({ id: SETTINGS_ID, ...next }).run()
|
||||
}
|
||||
return getBackupScheduleSettings()
|
||||
}
|
||||
|
||||
export function touchBackupScheduleRunMeta(patch: {
|
||||
lastRunAt?: string
|
||||
lastDurationMs?: number
|
||||
lastError?: string | null
|
||||
}) {
|
||||
const prev = getBackupScheduleSettingsRow()
|
||||
db.update(backupScheduleSettings).set({
|
||||
lastRunAt: patch.lastRunAt ?? prev.lastRunAt,
|
||||
lastDurationMs: patch.lastDurationMs ?? prev.lastDurationMs,
|
||||
lastError: patch.lastError === undefined ? prev.lastError : patch.lastError,
|
||||
updatedAt: new Date().toISOString(),
|
||||
}).where(eq(backupScheduleSettings.id, SETTINGS_ID)).run()
|
||||
}
|
||||
|
||||
export function resolveBackupServerIds(settings: BackupScheduleSettingsDto): string[] {
|
||||
const enabled = new Set(listServersRead().map((s) => String(s.id)))
|
||||
const requested = settings.serverIds.length > 0 ? settings.serverIds : [...enabled]
|
||||
return [...new Set(requested)].filter((id) => enabled.has(id))
|
||||
}
|
||||
|
||||
function sameLocalSlot(a: Date, b: Date): boolean {
|
||||
return a.getFullYear() === b.getFullYear()
|
||||
&& a.getMonth() === b.getMonth()
|
||||
&& a.getDate() === b.getDate()
|
||||
&& a.getHours() === b.getHours()
|
||||
&& a.getMinutes() === b.getMinutes()
|
||||
}
|
||||
|
||||
export function isBackupDue(now: Date, settings: BackupScheduleSettingsDto, lastRunAt: string | null | undefined): boolean {
|
||||
if (!settings.enabled) return false
|
||||
const slot = new Date(now)
|
||||
slot.setSeconds(0, 0)
|
||||
slot.setHours(settings.hour, settings.minute, 0, 0)
|
||||
|
||||
if (settings.frequency === "weekly") {
|
||||
const currentDow = (now.getDay() + 6) % 7
|
||||
if (currentDow !== settings.weekDay) return false
|
||||
} else if (settings.frequency === "monthly") {
|
||||
if (now.getDate() !== settings.monthDay) return false
|
||||
}
|
||||
|
||||
if (now < slot) return false
|
||||
if (lastRunAt) {
|
||||
const prev = new Date(lastRunAt)
|
||||
if (Number.isNaN(prev.getTime())) return true
|
||||
if (settings.frequency === "daily" && sameLocalSlot(prev, slot)) return false
|
||||
if (settings.frequency === "weekly" && sameLocalSlot(prev, slot)) return false
|
||||
if (settings.frequency === "monthly" && prev.getFullYear() === slot.getFullYear() && prev.getMonth() === slot.getMonth() && prev.getDate() === slot.getDate()) return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
export async function runBackupForServer(
|
||||
id: string,
|
||||
kind: BackupMeta["kind"],
|
||||
notes?: string,
|
||||
): Promise<BackupMeta> {
|
||||
const serverIdNum = Number.parseInt(id, 10)
|
||||
if (!Number.isFinite(serverIdNum)) {
|
||||
throw new Error("Невалидный id сервера")
|
||||
}
|
||||
const row = getServerRowById(serverIdNum)
|
||||
if (!row) {
|
||||
throw new Error("Сервер не найден")
|
||||
}
|
||||
const client = MikrotikClient.fromServer(row)
|
||||
const script = await client.exportConfigScript()
|
||||
const ts = fmtTs()
|
||||
const safeServer = row.name.replace(/[^a-zA-Z0-9._-]+/g, "_")
|
||||
const filename = `${safeServer}_${ts}.rsc`
|
||||
const filePath = path.join(BACKUPS_DIR, filename)
|
||||
await writeFile(filePath, script, "utf8")
|
||||
const st = await stat(filePath)
|
||||
return {
|
||||
id: randomUUID(),
|
||||
serverId: String(row.id),
|
||||
serverName: row.name,
|
||||
filename,
|
||||
sizeBytes: st.size,
|
||||
createdAt: new Date().toISOString(),
|
||||
kind,
|
||||
notes,
|
||||
}
|
||||
}
|
||||
|
||||
export async function pruneBackupsForServer(serverId: string, keepCount: number): Promise<number> {
|
||||
const rows = await readBackupIndex()
|
||||
const forServer = rows.filter((r) => r.serverId === serverId)
|
||||
if (forServer.length <= keepCount) return 0
|
||||
const sorted = [...forServer].sort((a, b) => b.createdAt.localeCompare(a.createdAt))
|
||||
const toDelete = sorted.slice(keepCount)
|
||||
const deleteIds = new Set(toDelete.map((r) => r.id))
|
||||
for (const hit of toDelete) {
|
||||
await rm(path.join(BACKUPS_DIR, hit.filename), { force: true })
|
||||
}
|
||||
const next = rows.filter((r) => !deleteIds.has(r.id))
|
||||
await writeBackupIndex(next)
|
||||
return toDelete.length
|
||||
}
|
||||
|
||||
export function getBackupsDir(): string {
|
||||
return BACKUPS_DIR
|
||||
}
|
||||
|
||||
export function getBackupIndexPath(): string {
|
||||
return INDEX_PATH
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
import { appendEvent } from "../modules/events/service/events-service.js"
|
||||
import { issueCertificateWithCloudflareDns } from "./acme-cloudflare.js"
|
||||
import {
|
||||
getIssueJobRecord,
|
||||
getServerRowByIdString,
|
||||
updateIssueJobRecord,
|
||||
} from "./certificates-service.js"
|
||||
|
||||
const runningJobs = new Set<string>()
|
||||
|
||||
export async function runCertificateIssueJob(jobId: string): Promise<void> {
|
||||
if (runningJobs.has(jobId)) return
|
||||
runningJobs.add(jobId)
|
||||
const row = getIssueJobRecord(jobId)
|
||||
if (!row) {
|
||||
runningJobs.delete(jobId)
|
||||
return
|
||||
}
|
||||
|
||||
const server = getServerRowByIdString(row.serverId)
|
||||
if (!server) {
|
||||
updateIssueJobRecord(jobId, {
|
||||
status: "failed",
|
||||
step: "failed",
|
||||
finishedAt: new Date().toISOString(),
|
||||
error: "Сервер не найден",
|
||||
})
|
||||
runningJobs.delete(jobId)
|
||||
return
|
||||
}
|
||||
|
||||
let domainNames: string[] = []
|
||||
try {
|
||||
domainNames = JSON.parse(row.domainNames) as string[]
|
||||
} catch {
|
||||
domainNames = []
|
||||
}
|
||||
|
||||
const trustStore = row.trustStore.split(",").map((s) => s.trim()).filter(Boolean)
|
||||
const startedAt = new Date().toISOString()
|
||||
const isScheduler = row.source === "scheduler"
|
||||
updateIssueJobRecord(jobId, { status: "running", step: "acme_order", startedAt, error: null })
|
||||
|
||||
if (isScheduler) {
|
||||
appendEvent({
|
||||
level: "info",
|
||||
eventType: "certificates.renew.started",
|
||||
sourceModule: "certificates",
|
||||
title: "Запущено автообновление сертификата",
|
||||
message: `${row.certName} · ${domainNames.join(", ")} · ${server.name}`,
|
||||
entityType: "server",
|
||||
entityId: String(server.id),
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
await issueCertificateWithCloudflareDns({
|
||||
server,
|
||||
certName: row.certName,
|
||||
domainNames,
|
||||
keyType: row.keyType === "ec256" ? "ec256" : "rsa2048",
|
||||
trustStore: trustStore.length > 0 ? trustStore : ["www", "api"],
|
||||
onStep: (step) => updateIssueJobRecord(jobId, { step }),
|
||||
})
|
||||
updateIssueJobRecord(jobId, {
|
||||
status: "done",
|
||||
step: "done",
|
||||
finishedAt: new Date().toISOString(),
|
||||
error: null,
|
||||
})
|
||||
appendEvent({
|
||||
level: "info",
|
||||
eventType: isScheduler ? "certificates.renew.done" : "certificates.issue.done",
|
||||
sourceModule: "certificates",
|
||||
title: isScheduler ? "Сертификат обновлён" : "Сертификат выпущен",
|
||||
message: `${row.certName} · ${domainNames.join(", ")} · ${server.name}`,
|
||||
entityType: "server",
|
||||
entityId: String(server.id),
|
||||
})
|
||||
} catch (e) {
|
||||
const message = e instanceof Error ? e.message : String(e)
|
||||
updateIssueJobRecord(jobId, {
|
||||
status: "failed",
|
||||
step: "failed",
|
||||
finishedAt: new Date().toISOString(),
|
||||
error: message,
|
||||
})
|
||||
appendEvent({
|
||||
level: "warning",
|
||||
eventType: isScheduler ? "certificates.renew.failed" : "certificates.issue.failed",
|
||||
sourceModule: "certificates",
|
||||
title: isScheduler ? "Ошибка автообновления сертификата" : "Ошибка выпуска сертификата",
|
||||
message,
|
||||
entityType: "server",
|
||||
entityId: String(server.id),
|
||||
})
|
||||
} finally {
|
||||
runningJobs.delete(jobId)
|
||||
}
|
||||
}
|
||||
|
||||
export function queueCertificateIssueJob(jobId: string): void {
|
||||
queueMicrotask(() => { void runCertificateIssueJob(jobId) })
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { appendEvent } from "../modules/events/service/events-service.js"
|
||||
import type { CertificatesRenewRunSnapshot } from "../types/scheduler-run-snapshot.js"
|
||||
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
|
||||
import { mapRosCertificates } from "./certificate-parse.js"
|
||||
import { runCertificateIssueJob } from "./certificate-issue-runner.js"
|
||||
import {
|
||||
createIssueJobRecord,
|
||||
getCertificateRenewSettings,
|
||||
getServerRowByIdString,
|
||||
hasActiveIssueJob,
|
||||
listManagedCertificateTargets,
|
||||
touchCertificateRenewRunMeta,
|
||||
} from "./certificates-service.js"
|
||||
import { MikrotikClient } from "./mikrotik.js"
|
||||
|
||||
let collecting = false
|
||||
|
||||
export function getCertificateRenewCollectorState(): { running: boolean } {
|
||||
return { running: collecting }
|
||||
}
|
||||
|
||||
export async function collectCertificatesRenewOnce(): Promise<CertificatesRenewRunSnapshot> {
|
||||
const sampledAt = new Date().toISOString()
|
||||
const settings = getCertificateRenewSettings()
|
||||
if (collecting) {
|
||||
return {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "certificates_renew",
|
||||
sampledAt,
|
||||
skipped: true,
|
||||
checked: 0,
|
||||
renewed: 0,
|
||||
skippedTargets: 0,
|
||||
errors: [],
|
||||
}
|
||||
}
|
||||
|
||||
collecting = true
|
||||
const started = Date.now()
|
||||
const snapshot: CertificatesRenewRunSnapshot = {
|
||||
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
|
||||
job: "certificates_renew",
|
||||
sampledAt,
|
||||
checked: 0,
|
||||
renewed: 0,
|
||||
skippedTargets: 0,
|
||||
errors: [],
|
||||
targets: [],
|
||||
}
|
||||
|
||||
try {
|
||||
if (!settings.enabled) {
|
||||
snapshot.skipped = true
|
||||
return snapshot
|
||||
}
|
||||
|
||||
const targets = listManagedCertificateTargets()
|
||||
for (const target of targets) {
|
||||
snapshot.checked += 1
|
||||
const item = {
|
||||
serverId: target.serverId,
|
||||
certName: target.certName,
|
||||
action: "ok" as "ok" | "renewed" | "skipped" | "error",
|
||||
daysLeft: null as number | null,
|
||||
message: "",
|
||||
}
|
||||
|
||||
if (hasActiveIssueJob(target.serverId, target.certName)) {
|
||||
item.action = "skipped"
|
||||
item.message = "Уже выполняется выпуск"
|
||||
snapshot.skippedTargets += 1
|
||||
snapshot.targets?.push(item)
|
||||
appendEvent({
|
||||
level: "info",
|
||||
eventType: "certificates.renew.skipped",
|
||||
sourceModule: "certificates",
|
||||
title: "Автообновление пропущено",
|
||||
message: `${target.certName}: уже выполняется выпуск`,
|
||||
entityType: "server",
|
||||
entityId: target.serverId,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
const server = getServerRowByIdString(target.serverId)
|
||||
if (!server) {
|
||||
item.action = "error"
|
||||
item.message = "Сервер не найден"
|
||||
snapshot.errors.push(`${target.certName}: сервер не найден`)
|
||||
snapshot.targets?.push(item)
|
||||
continue
|
||||
}
|
||||
|
||||
try {
|
||||
const client = MikrotikClient.fromServer(server)
|
||||
const rows = await client.getCertificates()
|
||||
const mapped = mapRosCertificates(server.id, server.name, rows)
|
||||
const hit = mapped.find((c) => c.name === target.certName)
|
||||
if (!hit) {
|
||||
item.action = "error"
|
||||
item.message = "Сертификат не найден на устройстве"
|
||||
snapshot.errors.push(`${target.certName}: не найден на ${server.name}`)
|
||||
snapshot.targets?.push(item)
|
||||
continue
|
||||
}
|
||||
|
||||
item.daysLeft = hit.daysLeft
|
||||
if (hit.daysLeft > settings.renewBeforeDays) {
|
||||
item.action = "ok"
|
||||
item.message = `До истечения ${hit.daysLeft} дн.`
|
||||
snapshot.targets?.push(item)
|
||||
continue
|
||||
}
|
||||
|
||||
const jobId = randomUUID()
|
||||
createIssueJobRecord({
|
||||
id: jobId,
|
||||
serverId: target.serverId,
|
||||
certName: target.certName,
|
||||
domainNames: target.domainNames,
|
||||
keyType: target.keyType,
|
||||
trustStore: target.trustStore,
|
||||
source: "scheduler",
|
||||
})
|
||||
await runCertificateIssueJob(jobId)
|
||||
snapshot.renewed += 1
|
||||
item.action = "renewed"
|
||||
item.message = "Запущено обновление"
|
||||
snapshot.targets?.push(item)
|
||||
} catch (e) {
|
||||
const message = e instanceof Error ? e.message : String(e)
|
||||
item.action = "error"
|
||||
item.message = message
|
||||
snapshot.errors.push(`${target.certName}: ${message}`)
|
||||
snapshot.targets?.push(item)
|
||||
}
|
||||
}
|
||||
|
||||
touchCertificateRenewRunMeta({
|
||||
lastCollectedAt: sampledAt,
|
||||
lastDurationMs: Date.now() - started,
|
||||
lastError: snapshot.errors.length > 0 ? snapshot.errors.join("; ") : null,
|
||||
})
|
||||
return snapshot
|
||||
} catch (e) {
|
||||
const message = e instanceof Error ? e.message : String(e)
|
||||
snapshot.fatalError = message
|
||||
touchCertificateRenewRunMeta({
|
||||
lastCollectedAt: sampledAt,
|
||||
lastDurationMs: Date.now() - started,
|
||||
lastError: message,
|
||||
})
|
||||
return snapshot
|
||||
} finally {
|
||||
collecting = false
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,26 @@
|
||||
import { eq } from "drizzle-orm"
|
||||
import type { CertificateDto } from "@mmapp/contracts/certificates"
|
||||
import { desc, eq, inArray } from "drizzle-orm"
|
||||
import type { CertificateDto, CertificateRenewSettingsDto } from "@mmapp/contracts/certificates"
|
||||
import { db } from "../db/index.js"
|
||||
import { acmeSettings, certificateIssueJobs, servers } from "../db/schema.js"
|
||||
import {
|
||||
acmeSettings,
|
||||
certificateIssueJobs,
|
||||
certificateRenewSettings,
|
||||
servers,
|
||||
} from "../db/schema.js"
|
||||
import { mapRosCertificates } from "./certificate-parse.js"
|
||||
import { MikrotikClient } from "./mikrotik.js"
|
||||
|
||||
const SETTINGS_ID = 1
|
||||
|
||||
export type ManagedCertificateTarget = {
|
||||
serverId: string
|
||||
certName: string
|
||||
domainNames: string[]
|
||||
keyType: string
|
||||
trustStore: string
|
||||
finishedAt: string
|
||||
}
|
||||
|
||||
export async function listCertificatesFromServers(): Promise<{
|
||||
certificates: CertificateDto[]
|
||||
failures: Array<{ serverId: string; serverName?: string; error: string }>
|
||||
@@ -109,12 +123,14 @@ export function createIssueJobRecord(input: {
|
||||
domainNames: string[]
|
||||
keyType: string
|
||||
trustStore: string
|
||||
source?: "manual" | "scheduler"
|
||||
}) {
|
||||
const now = new Date().toISOString()
|
||||
db.insert(certificateIssueJobs).values({
|
||||
id: input.id,
|
||||
status: "queued",
|
||||
step: "queued",
|
||||
source: input.source ?? "manual",
|
||||
serverId: input.serverId,
|
||||
certName: input.certName,
|
||||
domainNames: JSON.stringify(input.domainNames),
|
||||
@@ -168,3 +184,103 @@ export function getServerRowByIdString(serverId: string) {
|
||||
if (!Number.isFinite(id)) return null
|
||||
return db.select().from(servers).where(eq(servers.id, id)).limit(1).all()[0] ?? null
|
||||
}
|
||||
|
||||
function getCertificateRenewSettingsRow() {
|
||||
return db.select().from(certificateRenewSettings).where(eq(certificateRenewSettings.id, SETTINGS_ID)).limit(1).all()[0]
|
||||
?? {
|
||||
id: SETTINGS_ID,
|
||||
enabled: true,
|
||||
intervalSec: 21600,
|
||||
renewBeforeDays: 30,
|
||||
lastCollectedAt: null,
|
||||
lastDurationMs: null,
|
||||
lastError: null,
|
||||
updatedAt: new Date().toISOString(),
|
||||
}
|
||||
}
|
||||
|
||||
export function getCertificateRenewSettings(): CertificateRenewSettingsDto {
|
||||
const row = getCertificateRenewSettingsRow()
|
||||
return {
|
||||
enabled: row.enabled,
|
||||
intervalSec: row.intervalSec,
|
||||
renewBeforeDays: row.renewBeforeDays,
|
||||
lastCollectedAt: row.lastCollectedAt ?? null,
|
||||
lastDurationMs: row.lastDurationMs ?? null,
|
||||
lastError: row.lastError ?? null,
|
||||
updatedAt: row.updatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
export function updateCertificateRenewSettings(patch: {
|
||||
enabled?: boolean
|
||||
intervalSec?: number
|
||||
renewBeforeDays?: number
|
||||
}) {
|
||||
const prev = getCertificateRenewSettingsRow()
|
||||
const now = new Date().toISOString()
|
||||
const next = {
|
||||
enabled: patch.enabled ?? prev.enabled,
|
||||
intervalSec: patch.intervalSec ?? prev.intervalSec,
|
||||
renewBeforeDays: patch.renewBeforeDays ?? prev.renewBeforeDays,
|
||||
updatedAt: now,
|
||||
}
|
||||
if (db.select().from(certificateRenewSettings).where(eq(certificateRenewSettings.id, SETTINGS_ID)).limit(1).all()[0]) {
|
||||
db.update(certificateRenewSettings).set(next).where(eq(certificateRenewSettings.id, SETTINGS_ID)).run()
|
||||
} else {
|
||||
db.insert(certificateRenewSettings).values({ id: SETTINGS_ID, ...next }).run()
|
||||
}
|
||||
return getCertificateRenewSettings()
|
||||
}
|
||||
|
||||
export function touchCertificateRenewRunMeta(patch: {
|
||||
lastCollectedAt?: string
|
||||
lastDurationMs?: number
|
||||
lastError?: string | null
|
||||
}) {
|
||||
const prev = getCertificateRenewSettingsRow()
|
||||
db.update(certificateRenewSettings).set({
|
||||
lastCollectedAt: patch.lastCollectedAt ?? prev.lastCollectedAt,
|
||||
lastDurationMs: patch.lastDurationMs ?? prev.lastDurationMs,
|
||||
lastError: patch.lastError === undefined ? prev.lastError : patch.lastError,
|
||||
updatedAt: new Date().toISOString(),
|
||||
}).where(eq(certificateRenewSettings.id, SETTINGS_ID)).run()
|
||||
}
|
||||
|
||||
export function listManagedCertificateTargets(): ManagedCertificateTarget[] {
|
||||
const rows = db.select().from(certificateIssueJobs)
|
||||
.where(eq(certificateIssueJobs.status, "done"))
|
||||
.orderBy(desc(certificateIssueJobs.finishedAt), desc(certificateIssueJobs.requestedAt))
|
||||
.all()
|
||||
|
||||
const byKey = new Map<string, ManagedCertificateTarget>()
|
||||
for (const row of rows) {
|
||||
const key = `${row.serverId}::${row.certName}`
|
||||
if (byKey.has(key)) continue
|
||||
let domainNames: string[] = []
|
||||
try {
|
||||
const parsed = JSON.parse(row.domainNames) as unknown
|
||||
if (Array.isArray(parsed)) domainNames = parsed.map(String)
|
||||
} catch {
|
||||
domainNames = []
|
||||
}
|
||||
byKey.set(key, {
|
||||
serverId: row.serverId,
|
||||
certName: row.certName,
|
||||
domainNames,
|
||||
keyType: row.keyType,
|
||||
trustStore: row.trustStore,
|
||||
finishedAt: row.finishedAt ?? row.requestedAt,
|
||||
})
|
||||
}
|
||||
return [...byKey.values()]
|
||||
}
|
||||
|
||||
export function hasActiveIssueJob(serverId: string, certName: string): boolean {
|
||||
const row = db.select().from(certificateIssueJobs)
|
||||
.where(inArray(certificateIssueJobs.status, ["queued", "running"]))
|
||||
.orderBy(desc(certificateIssueJobs.requestedAt))
|
||||
.all()
|
||||
.find((r) => r.serverId === serverId && r.certName === certName)
|
||||
return Boolean(row)
|
||||
}
|
||||
|
||||
@@ -42,6 +42,10 @@ import {
|
||||
collectInternetPathSnapshotOnce,
|
||||
getInternetPathSettings,
|
||||
} from "./internet-path-collector.js"
|
||||
import { collectCertificatesRenewOnce } from "./certificate-renew-collector.js"
|
||||
import { getCertificateRenewSettings } from "./certificates-service.js"
|
||||
import { collectScheduledBackupsOnce } from "./backup-scheduler-collector.js"
|
||||
import { getBackupScheduleSettings } from "./backup-service.js"
|
||||
import {
|
||||
endSchedulerJob,
|
||||
isSchedulerJobRunning,
|
||||
@@ -57,6 +61,8 @@ export const JOB_KEYS = [
|
||||
"uptime_speed",
|
||||
"internet_path",
|
||||
"gre_bgp",
|
||||
"certificates_renew",
|
||||
"backups",
|
||||
"alert_engine",
|
||||
] as const
|
||||
export type SchedulerJobKey = (typeof JOB_KEYS)[number]
|
||||
@@ -119,6 +125,12 @@ async function runSchedulerJobBody(jobKey: SchedulerJobKey): Promise<void> {
|
||||
case "internet_path":
|
||||
snapshot = await collectInternetPathSnapshotOnce()
|
||||
break
|
||||
case "certificates_renew":
|
||||
snapshot = await collectCertificatesRenewOnce()
|
||||
break
|
||||
case "backups":
|
||||
snapshot = await collectScheduledBackupsOnce()
|
||||
break
|
||||
case "alert_engine": {
|
||||
const r = await runAlertEngineOnce()
|
||||
snapshot = {
|
||||
@@ -322,6 +334,30 @@ export function refreshScheduler(): void {
|
||||
}, greBgpMs),
|
||||
)
|
||||
|
||||
const certRenew = getCertificateRenewSettings()
|
||||
if (certRenew.enabled) {
|
||||
const certRenewMs = Math.max(300_000, certRenew.intervalSec * 1000)
|
||||
void executeSchedulerJob("certificates_renew").catch(() => {})
|
||||
timers.set(
|
||||
"certificates_renew",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("certificates_renew").catch(() => {})
|
||||
}, certRenewMs),
|
||||
)
|
||||
}
|
||||
|
||||
const backupSchedule = getBackupScheduleSettings()
|
||||
if (backupSchedule.enabled) {
|
||||
const backupMs = 60_000
|
||||
void executeSchedulerJob("backups").catch(() => {})
|
||||
timers.set(
|
||||
"backups",
|
||||
setInterval(() => {
|
||||
void executeSchedulerJob("backups").catch(() => {})
|
||||
}, backupMs),
|
||||
)
|
||||
}
|
||||
|
||||
const alertMs = 20_000
|
||||
void executeSchedulerJob("alert_engine").catch(() => {})
|
||||
timers.set(
|
||||
@@ -353,6 +389,8 @@ export function getSchedulerStatus() {
|
||||
const uptime = getUptimeSettings()
|
||||
const apiPing = getServersApiPingSettings()
|
||||
const internetPath = getInternetPathSettings()
|
||||
const certRenew = getCertificateRenewSettings()
|
||||
const backupSchedule = getBackupScheduleSettings()
|
||||
|
||||
const resOn = uptime.resourcesEnabled ?? uptime.enabled
|
||||
const pingOn = uptime.pingEnabled ?? uptime.enabled
|
||||
@@ -366,6 +404,8 @@ export function getSchedulerStatus() {
|
||||
uptime_speed: { enabled: spdOn, intervalSec: uptime.speedIntervalSec },
|
||||
internet_path: { enabled: internetPath.enabled, intervalSec: internetPath.intervalSec },
|
||||
gre_bgp: { enabled: true, intervalSec: 30 },
|
||||
certificates_renew: { enabled: certRenew.enabled, intervalSec: certRenew.intervalSec },
|
||||
backups: { enabled: backupSchedule.enabled, intervalSec: 60 },
|
||||
alert_engine: { enabled: true, intervalSec: 20 },
|
||||
}
|
||||
|
||||
|
||||
@@ -192,6 +192,40 @@ export interface InternetPathRunSnapshot {
|
||||
fatalError?: string
|
||||
}
|
||||
|
||||
export interface CertificatesRenewTargetSnapshot {
|
||||
serverId: string
|
||||
certName: string
|
||||
action: "ok" | "renewed" | "skipped" | "error"
|
||||
daysLeft: number | null
|
||||
message: string
|
||||
}
|
||||
|
||||
export interface CertificatesRenewRunSnapshot {
|
||||
v: typeof SCHEDULER_RUN_SNAPSHOT_VERSION
|
||||
job: "certificates_renew"
|
||||
sampledAt: string
|
||||
skipped?: boolean
|
||||
fatalError?: string
|
||||
checked: number
|
||||
renewed: number
|
||||
skippedTargets: number
|
||||
errors: string[]
|
||||
targets?: CertificatesRenewTargetSnapshot[]
|
||||
}
|
||||
|
||||
export interface BackupsRunSnapshot {
|
||||
v: typeof SCHEDULER_RUN_SNAPSHOT_VERSION
|
||||
job: "backups"
|
||||
sampledAt: string
|
||||
skipped?: boolean
|
||||
due: boolean
|
||||
created: number
|
||||
failures: number
|
||||
pruned: number
|
||||
errors?: string[]
|
||||
fatalError?: string
|
||||
}
|
||||
|
||||
export type SchedulerRunSnapshot =
|
||||
| TrafficRunSnapshot
|
||||
| ResourcesRunSnapshot
|
||||
@@ -200,4 +234,6 @@ export type SchedulerRunSnapshot =
|
||||
| ServersRestPingRunSnapshot
|
||||
| GreBgpSnapshotRunSnapshot
|
||||
| InternetPathRunSnapshot
|
||||
| CertificatesRenewRunSnapshot
|
||||
| BackupsRunSnapshot
|
||||
| AlertEngineRunSnapshot
|
||||
|
||||
Reference in New Issue
Block a user