diff --git a/backend/package.json b/backend/package.json index e47db51c..996a7ecb 100644 --- a/backend/package.json +++ b/backend/package.json @@ -37,7 +37,7 @@ "test:geoip": "tsx src/services/geoip-download.test.ts && tsx src/services/traffic-flow-geoip.test.ts && tsx src/plugins/auth.geoip.test.ts", "test:ospf-optimizer": "tsx src/services/ospf-optimizer-identity.test.ts && tsx src/services/ospf-optimizer-metrics.test.ts && tsx src/services/ospf-optimizer-plan.test.ts && tsx src/services/ospf-effective-disabled.test.ts && tsx src/routes/ospf-optimizer.test.ts", "test:containers": "tsx src/services/containers-live.test.ts && tsx src/services/containers-state.test.ts && tsx src/services/container-removal.test.ts && tsx src/routes/containers-state.test.ts", - "test:certificates": "tsx src/services/certificate-issue-execution.test.ts && tsx src/services/certificate-issue-request.test.ts && tsx src/routes/certificates-issue.test.ts && tsx src/services/certificate-issue-dto.test.ts && tsx src/services/certificate-parse.test.ts && tsx src/services/certificates-state.test.ts && tsx src/routes/certificates-state.test.ts", + "test:certificates": "tsx src/services/certificate-renew-collector.test.ts && tsx src/services/certificate-issue-execution.test.ts && tsx src/services/certificate-issue-request.test.ts && tsx src/routes/certificates-issue.test.ts && tsx src/services/certificate-issue-dto.test.ts && tsx src/services/certificate-parse.test.ts && tsx src/services/certificates-state.test.ts && tsx src/routes/certificates-state.test.ts", "test:system-database": "tsx src/services/system-database-partition-restore.test.ts && tsx src/services/system-database-cache.test.ts && tsx src/services/database-background-work.test.ts && tsx src/services/database-request-gate.test.ts && tsx src/services/traffic-flow-worker-shutdown.test.ts && tsx src/services/system-database-restore-execution.test.ts && tsx src/services/system-database-operation.test.ts && tsx src/routes/system-database.test.ts && tsx src/plugins/auth.system-database.test.ts", "test:all": "npm run test:probes && npm run test:exec && npm run test:alert-engine && npm run test:alerts-telegram && npm run test:auth && npm run test:wireguard && npm run test:ipsec && npm run test:traffic-rate && npm run test:traffic-flow && npm run test:users && npm run test:pg && npm run test:backups && npm run test:scheduler && npm run test:live-maps && npm run test:ospf-optimizer && npm run test:recursive-routes && npm run test:config-sync && npm run test:ros-file && npm run test:containers && npm run test:certificates && npm run test:netflow-settings && npm run test:netflow-purge && npm run test:evolution && npm run test:system-database && npm run test:server-privacy", "test:server-privacy": "tsx src/routes/server-secrets-sql.test.ts" diff --git a/backend/src/services/certificate-issue-execution.test.ts b/backend/src/services/certificate-issue-execution.test.ts index 4de7896c..5b9d4d7b 100644 --- a/backend/src/services/certificate-issue-execution.test.ts +++ b/backend/src/services/certificate-issue-execution.test.ts @@ -91,3 +91,9 @@ test("queue handles rejected setup without unhandled rejection", async () => { await new Promise(resolve => setImmediate(resolve)) assert.equal(f.warnings.length, 1); assert.equal(f.calls(), 0) }) + +test("disabled server fails before DNS or device issue", async () => { + const f = fixture(); f.deps.getServer = async () => ({ id: 1, name: "router", enabled: false }) + await createCertificateIssueRunner(f.deps).run("job") + assert.equal(f.calls(), 0); assert.equal(f.row.status, "failed"); assert.match(f.row.error ?? "", /выключен/) +}) diff --git a/backend/src/services/certificate-issue-execution.ts b/backend/src/services/certificate-issue-execution.ts index 1529926c..f65e6867 100644 --- a/backend/src/services/certificate-issue-execution.ts +++ b/backend/src/services/certificate-issue-execution.ts @@ -17,7 +17,7 @@ type Dependencies = { } /** Separate the network outcome from progress/event persistence failures. */ -export function createCertificateIssueRunner(deps: Dependencies) { +export function createCertificateIssueRunner(deps: Dependencies) { const runningJobs = new Set() const report = (error: unknown) => { deps.warn(error) } const event = async (input: Parameters[0]) => { @@ -31,8 +31,8 @@ export function createCertificateIssueRunner> = { id: 1, name: "fixture", enabled: true } as NonNullable>> + let rows: RosCertificateRow[] = [{ ".id": "*A", name: "cert", flags: "KT", "invalid-before": new Date(Date.now() - 86400000).toISOString(), "invalid-after": new Date(Date.now() + 172800000).toISOString() }] + let job: CertificateIssueJobRow | null = null, outcome: CertificateIssueJobRow["status"] = "done", readFailure = false, active = false + let resultPatch: Partial = {}, missingResult = false, targetGate: Promise | undefined + let reads = 0, creates = 0, issues = 0 + const collector = createCertificateRenewCollector({ + getSettings: async () => settings, getTargets: async () => { await targetGate; return [target] }, hasActiveJob: async () => active, getServer: async () => server, + read: async () => { reads++; if (readFailure) throw new Error("RouterOS offline"); return rows }, + createJob: async input => { creates++; job = { ...input, serverId: Number(input.serverId), status: "queued", step: "queued", source: "scheduler", requestedAt: new Date().toISOString(), startedAt: null, finishedAt: null, error: null } }, + runJob: async () => { issues++; assert(job); job.status = outcome; job.step = outcome; job.error = outcome === "failed" ? "import refused" : null }, + getJob: async () => missingResult || !job ? null : { ...job, ...resultPatch }, touchMeta: async () => {}, event: async () => {}, + }) + return { collector, settings, target, counts: () => ({ reads, creates, issues }), setServer: (v: typeof server) => { server = v }, setRows: (v: typeof rows) => { rows = v }, setOutcome: (v: typeof outcome) => { outcome = v }, failRead: () => { readFailure = true }, setActive: () => { active = true }, rows: () => rows, patchResult: (v: typeof resultPatch) => { resultPatch = v }, hideResult: () => { missingResult = true }, blockTargets: (v: Promise) => { targetGate = v } } +} + +for (const outcome of ["done", "failed", "running", "queued"] as const) { + const f = fixture(); f.setOutcome(outcome); const result = await f.collector.run() + assert.equal(result.renewed, outcome === "done" ? 1 : 0) + assert.equal(result.errors.length, outcome === "failed" ? 1 : 0) + assert.equal(result.skippedTargets, outcome === "running" || outcome === "queued" ? 1 : 0) + assert.equal(f.counts().issues, 1); assert.equal(f.collector.getState().running, false) +} +for (const disabled of [null, { id: 1, name: "off", enabled: false }]) { + const f = fixture(); f.setServer(disabled as Awaited>) + const result = await f.collector.run(); assert.equal(result.renewed, 0); assert.equal(result.errors.length, 1); assert.deepEqual(f.counts(), { reads: 0, creates: 0, issues: 0 }) +} +{ + const f = fixture(); f.setRows([{ ...f.rows()[0], "invalid-after": undefined }]); const result = await f.collector.run() + assert.equal(result.targets?.[0].daysLeft, null); assert.equal(result.skippedTargets, 1); assert.equal(f.counts().creates, 0) +} +for (const kind of ["offline", "malformed", "ambiguous"] as const) { + const f = fixture(); if (kind === "offline") f.failRead(); else if (kind === "malformed") f.setRows([{}]); else f.setRows([...f.rows(), { ...f.rows()[0], ".id": "*B" }]) + const result = await f.collector.run(); assert.equal(result.renewed, 0); assert.equal(result.errors.length, 1); assert.equal(f.counts().creates, 0) +} +{ + const f = fixture(); f.settings.enabled = false; assert.equal((await f.collector.run()).skipped, true); assert.equal(f.counts().reads, 0) +} +{ + const f = fixture(); f.setActive(); const result = await f.collector.run(); assert.equal(result.skippedTargets, 1); assert.equal(f.counts().reads, 0) +} +for (const invalid of ["1abc", "", "0", "-1", "1.5", "9007199254740993"]) assert.equal(await getServerRowByIdString(invalid), null) +for (const patch of [{ serverId: 2 }, { certName: "another" }, { source: "manual" as const }, { domainNames: ["another.test"] }, { step: "cleanup" }]) { + const f = fixture(); f.patchResult(patch); const result = await f.collector.run(); assert.equal(result.renewed, 0) +} +{ + const f = fixture(); f.hideResult(); const result = await f.collector.run(); assert.equal(result.renewed, 0); assert.match(result.errors[0], /не подтверждён/) +} +{ + const f = fixture(); let release!: () => void; f.blockTargets(new Promise(resolve => { release = resolve })) + const first = f.collector.run(); await new Promise(resolve => setImmediate(resolve)); assert.equal(f.collector.getState().running, true) + assert.equal((await f.collector.run()).skipped, true); release(); assert.equal((await first).renewed, 1); assert.equal(f.counts().issues, 1) +} +console.log("Renewal: only matching done counts as renewed; failed/pending/disabled/unknown/ambiguous/read error never dispatch or claim success; invalid IDs do not query DB") diff --git a/backend/src/services/certificate-renew-collector.ts b/backend/src/services/certificate-renew-collector.ts index 2dd2fa59..62d58c92 100644 --- a/backend/src/services/certificate-renew-collector.ts +++ b/backend/src/services/certificate-renew-collector.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto" import { appendEvent } from "../modules/events/service/events-service.js" import type { CertificatesRenewRunSnapshot } from "../types/scheduler-run-snapshot.js" import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js" -import { mapRosCertificates } from "./certificate-parse.js" +import { readCertificateServerState } from "./certificates-state.js" import { runCertificateIssueJob } from "./certificate-issue-runner.js" import { createIssueJobRecord, @@ -11,157 +11,198 @@ import { hasActiveIssueJob, listManagedCertificateTargets, touchCertificateRenewRunMeta, + getIssueJobRecord, } from "./certificates-service.js" import { MikrotikClient } from "./mikrotik.js" +import { parseJsonArray } from "../db/json.js" -let collecting = false - -export function getCertificateRenewCollectorState(): { running: boolean } { - return { running: collecting } +const defaults = { + getSettings: getCertificateRenewSettings, + getTargets: listManagedCertificateTargets, + hasActiveJob: hasActiveIssueJob, + getServer: getServerRowByIdString, + read: (server: NonNullable>>) => MikrotikClient.fromServer(server).getCertificates(), + createJob: createIssueJobRecord, + runJob: runCertificateIssueJob, + getJob: getIssueJobRecord, + touchMeta: touchCertificateRenewRunMeta, + event: appendEvent, } -export async function collectCertificatesRenewOnce(): Promise { - const sampledAt = new Date().toISOString() - const settings = await getCertificateRenewSettings() - if (collecting) { - return { +export function createCertificateRenewCollector(overrides: Partial = {}) { + const deps = { ...defaults, ...overrides } + let collecting = false + + function getState(): { running: boolean } { + return { running: collecting } + } + + async function run(): Promise { + const sampledAt = new Date().toISOString() + const settings = await deps.getSettings() + if (collecting) { + return { + v: SCHEDULER_RUN_SNAPSHOT_VERSION, + job: "certificates_renew", + sampledAt, + skipped: true, + checked: 0, + renewed: 0, + skippedTargets: 0, + errors: [], + } + } + + collecting = true + const started = Date.now() + const snapshot: CertificatesRenewRunSnapshot = { v: SCHEDULER_RUN_SNAPSHOT_VERSION, job: "certificates_renew", sampledAt, - skipped: true, checked: 0, renewed: 0, skippedTargets: 0, errors: [], - } - } - - collecting = true - const started = Date.now() - const snapshot: CertificatesRenewRunSnapshot = { - v: SCHEDULER_RUN_SNAPSHOT_VERSION, - job: "certificates_renew", - sampledAt, - checked: 0, - renewed: 0, - skippedTargets: 0, - errors: [], - targets: [], - } - - try { - if (!settings.enabled) { - snapshot.skipped = true - return snapshot + targets: [], } - const targets = await listManagedCertificateTargets() - for (const target of targets) { - snapshot.checked += 1 - const item = { - serverId: target.serverId, - certName: target.certName, - action: "ok" as "ok" | "renewed" | "skipped" | "error", - daysLeft: null as number | null, - message: "", + try { + if (!settings.enabled) { + snapshot.skipped = true + return snapshot } - if (await hasActiveIssueJob(target.serverId, target.certName)) { - item.action = "skipped" - item.message = "Уже выполняется выпуск" - snapshot.skippedTargets += 1 - snapshot.targets?.push(item) - await appendEvent({ - level: "info", - eventType: "certificates.renew.skipped", - sourceModule: "certificates", - title: "Автообновление пропущено", - message: `${target.certName}: уже выполняется выпуск`, - entityType: "server", - entityId: target.serverId, - }) - continue - } - - const server = await getServerRowByIdString(target.serverId) - if (!server) { - item.action = "error" - item.message = "Сервер не найден" - snapshot.errors.push(`${target.certName}: сервер не найден`) - snapshot.targets?.push(item) - continue - } - - try { - const client = MikrotikClient.fromServer(server) - const rows = await client.getCertificates() - const mapped = mapRosCertificates(server.id, server.name, rows) - const hit = mapped.find((c) => c.name === target.certName) - if (!hit) { - item.action = "error" - item.message = "Сертификат не найден на устройстве" - snapshot.errors.push(`${target.certName}: не найден на ${server.name}`) - snapshot.targets?.push(item) - continue - } - - item.daysLeft = hit.daysLeft - if (hit.daysLeft > settings.renewBeforeDays) { - item.action = "ok" - item.message = `До истечения ${hit.daysLeft} дн.` - snapshot.targets?.push(item) - continue - } - - const stillOn = await getCertificateRenewSettings() - if (!stillOn.enabled) { - item.action = "skipped" - item.message = "Автообновление выключено" - snapshot.skippedTargets += 1 - snapshot.targets?.push(item) - continue - } - - const jobId = randomUUID() - await createIssueJobRecord({ - id: jobId, + const targets = await deps.getTargets() + for (const target of targets) { + snapshot.checked += 1 + const item = { serverId: target.serverId, certName: target.certName, - domainNames: target.domainNames, - keyType: target.keyType, - trustStore: target.trustStore, - source: "scheduler", - }) - await runCertificateIssueJob(jobId) - snapshot.renewed += 1 - item.action = "renewed" - item.message = "Запущено обновление" - snapshot.targets?.push(item) - } catch (e) { - const message = e instanceof Error ? e.message : String(e) - item.action = "error" - item.message = message - snapshot.errors.push(`${target.certName}: ${message}`) - snapshot.targets?.push(item) - } - } + action: "ok" as "ok" | "renewed" | "skipped" | "error", + daysLeft: null as number | null, + message: "", + } - await touchCertificateRenewRunMeta({ - lastCollectedAt: sampledAt, - lastDurationMs: Date.now() - started, - lastError: snapshot.errors.length > 0 ? snapshot.errors.join("; ") : null, - }) - return snapshot - } catch (e) { - const message = e instanceof Error ? e.message : String(e) - snapshot.fatalError = message - await touchCertificateRenewRunMeta({ - lastCollectedAt: sampledAt, - lastDurationMs: Date.now() - started, - lastError: message, - }) - return snapshot - } finally { - collecting = false + if (await deps.hasActiveJob(target.serverId, target.certName)) { + item.action = "skipped" + item.message = "Уже выполняется выпуск" + snapshot.skippedTargets += 1 + snapshot.targets?.push(item) + await deps.event({ + level: "info", + eventType: "certificates.renew.skipped", + sourceModule: "certificates", + title: "Автообновление пропущено", + message: `${target.certName}: уже выполняется выпуск`, + entityType: "server", + entityId: target.serverId, + }) + continue + } + + const server = await deps.getServer(target.serverId) + if (!server || !server.enabled) { + item.action = "error" + item.message = "Сервер не найден или выключен" + snapshot.errors.push(`${target.certName}: сервер не найден или выключен`) + snapshot.targets?.push(item) + continue + } + + try { + const state = await readCertificateServerState(server, () => deps.read(server)) + if (state.error) throw new Error(state.error) + const matches = state.certificates.filter(c => c.name === target.certName) + if (matches.length > 1) throw new Error("Имя сертификата неоднозначно") + const hit = matches[0] + if (!hit) { + item.action = "error" + item.message = "Сертификат не найден на устройстве" + snapshot.errors.push(`${target.certName}: не найден на ${server.name}`) + snapshot.targets?.push(item) + continue + } + + item.daysLeft = hit.daysLeft + if (hit.daysLeft === null || hit.status === "unknown") { + item.action = "skipped" + item.message = "Срок или состояние сертификата не подтверждены. Обновление не выполнялось." + snapshot.skippedTargets += 1 + snapshot.targets?.push(item) + continue + } + if (hit.daysLeft > settings.renewBeforeDays) { + item.action = "ok" + item.message = `До истечения ${hit.daysLeft} дн.` + snapshot.targets?.push(item) + continue + } + + const stillOn = await deps.getSettings() + if (!stillOn.enabled) { + item.action = "skipped" + item.message = "Автообновление выключено" + snapshot.skippedTargets += 1 + snapshot.targets?.push(item) + continue + } + + const jobId = randomUUID() + await deps.createJob({ + id: jobId, + serverId: target.serverId, + certName: target.certName, + domainNames: target.domainNames, + keyType: target.keyType, + trustStore: target.trustStore, + source: "scheduler", + }) + await deps.runJob(jobId) + const result = await deps.getJob(jobId) + if (!result || result.id !== jobId || String(result.serverId) !== target.serverId || result.certName !== target.certName || result.source !== "scheduler" || JSON.stringify(parseJsonArray(result.domainNames).map(String)) !== JSON.stringify(target.domainNames) || result.keyType !== target.keyType || result.trustStore !== target.trustStore) throw new Error("Результат обновления сертификата не подтверждён") + if (result.status === "failed") throw new Error(result.error || "Задача обновления сертификата завершилась ошибкой") + if (result.status !== "done" || result.step !== "done") { + item.action = "skipped" + item.message = "Результат задачи пока не подтверждён; новый выпуск не запускался." + snapshot.skippedTargets += 1 + snapshot.targets?.push(item) + continue + } + snapshot.renewed += 1 + item.action = "renewed" + item.message = "Задача обновления завершена" + snapshot.targets?.push(item) + } catch (e) { + const message = e instanceof Error ? e.message : String(e) + item.action = "error" + item.message = message + snapshot.errors.push(`${target.certName}: ${message}`) + snapshot.targets?.push(item) + } + } + + await deps.touchMeta({ + lastCollectedAt: sampledAt, + lastDurationMs: Date.now() - started, + lastError: snapshot.errors.length > 0 ? snapshot.errors.join("; ") : null, + }) + return snapshot + } catch (e) { + const message = e instanceof Error ? e.message : String(e) + snapshot.fatalError = message + await deps.touchMeta({ + lastCollectedAt: sampledAt, + lastDurationMs: Date.now() - started, + lastError: message, + }) + return snapshot + } finally { + collecting = false + } } + return { run, getState } } + +const collector = createCertificateRenewCollector() +export const collectCertificatesRenewOnce = collector.run +export const getCertificateRenewCollectorState = collector.getState diff --git a/backend/src/services/certificates-service.ts b/backend/src/services/certificates-service.ts index 0d7f1b39..64be8bb6 100644 --- a/backend/src/services/certificates-service.ts +++ b/backend/src/services/certificates-service.ts @@ -1,4 +1,5 @@ -import { desc, eq, inArray } from "drizzle-orm" +import { and, desc, eq, inArray } from "drizzle-orm" +import { parseDbServerId } from "../utils/server-id.js" import type { CertificateDto, CertificateRenewSettingsDto } from "@mmapp/contracts/certificates" import { db } from "../db/index.js" import { parseJsonArray } from "../db/json.js" @@ -170,9 +171,9 @@ export function toIssueJobDto(row: NonNullable