Hub stage 1: apps across servers, read-only

hub/apps/matrix reads apps/list and each app's config on every server (the read
token first, the admin token once if a server refuses) and returns per app and server
the version, available update and a config hash, plus whether versions and configs
match and how many JSON leaf paths differ. Config texts never leave the hub. In hub
mode the Apps section has the server selector: All servers shows the matrix with
filters for differing versions, differing configs and updates; one server shows its
list without write actions. The dev mock takes --variant a|b with differing apps.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

hub/apps-matrix.test.ts

@@ -0,0 +1,280 @@
+// @vitest-environment node
+import http from 'node:http'
+import type { AddressInfo } from 'node:net'
+import { mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
+import { parseConfig } from './config.ts'
+import { createHub } from './server.ts'
+import { countDifferences } from './apps-matrix.ts'
+import { jsonDiff } from './json-diff.ts'
+
+interface FakeApp {
+  version: string
+  config: string | null
+  updateAvailable?: boolean
+  updateVersion?: string
+}
+
+interface Fake {
+  server: http.Server
+  url: string
+  apps: Record<string, FakeApp>
+  down: boolean
+  /** Config reads with the read token are refused with a permission error. */
+  denyRead: boolean
+  tokens: string[]
+}
+
+function fake(apps: Fake['apps']): Promise<Fake> {
+  return new Promise((resolve) => {
+    const f: Fake = { server: http.createServer(), url: '', apps, down: false, denyRead: false, tokens: [] }
+    f.server.on('request', (req, res) => {
+      if (f.down) return void req.socket.destroy()
+      const url = new URL(req.url ?? '/', 'http://fake')
+      const apiPath = url.pathname.replace(/^\/api\//, '')
+      const token = url.searchParams.get('token') ?? ''
+      res.setHeader('content-type', 'application/json')
+      if (apiPath === 'apps/list') {
+        const list = Object.entries(f.apps).map(([name, a]) => ({
+          name,
+          description: `${name} description`,
+          version: a.version,
+          ...(a.updateAvailable ? { updateAvailable: true, updateVersion: a.updateVersion } : {}),
+        }))
+        return void res.end(JSON.stringify({ status: 'ok', response: { apps: list } }))
+      }
+      if (apiPath === 'apps/config/get') {
+        f.tokens.push(token)
+        if (f.denyRead && !token.startsWith('admin-')) {
+          return void res.end(JSON.stringify({ status: 'error', errorMessage: "Access was denied to user 'reader' for apps." }))
+        }
+        const app = f.apps[url.searchParams.get('name') ?? '']
+        return void res.end(JSON.stringify({ status: 'ok', response: { config: app?.config ?? null } }))
+      }
+      res.end(JSON.stringify({ status: 'ok', response: {} }))
+    })
+    f.server.listen(0, '127.0.0.1', () => {
+      f.url = `http://127.0.0.1:${(f.server.address() as AddressInfo).port}`
+      resolve(f)
+    })
+  })
+}
+
+const json = (o: unknown): string => JSON.stringify(o, null, 2)
+
+let a: Fake
+let b: Fake
+let dataDir: string
+let staticDir: string
+let hub: http.Server
+let base: string
+let token: string
+
+function reset(): void {
+  a.down = b.down = false
+  a.denyRead = b.denyRead = false
+  a.tokens = []
+  b.tokens = []
+  a.apps = {
+    Same: { version: '1.0', config: json({ x: 1 }) },
+    Differs: { version: '2.0', config: json({ x: 1, y: { z: 2 }, list: [1, 2] }) },
+    'Only A': { version: '3.0', config: json({ secret: 'top-secret-value' }) },
+    Plain: { version: '4.0', config: 'line one\nline two\nline three' },
+    Empty: { version: '5.0', config: null },
+  }
+  b.apps = {
+    Same: { version: '1.0', config: json({ x: 1 }) },
+    Differs: { version: '2.1', updateAvailable: true, updateVersion: '2.2', config: json({ x: 2, y: { z: 3 }, list: [1, 2, 3] }) },
+    Plain: { version: '4.0', config: 'line one\nline TWO\nline three' },
+    Empty: { version: '5.0', config: null },
+  }
+}
+
+async function start(extra: Record<string, string> = {}): Promise<void> {
+  hub?.close()
+  hub = createHub(
+    parseConfig({
+      HUB_SERVERS: `a=${a.url},b=${b.url}`,
+      A_TOKEN: 'read-a',
+      B_TOKEN: 'read-b',
+      HUB_USER: 'admin',
+      HUB_PASSWORD: 'secret',
+      HUB_DATA: dataDir,
+      HUB_STATIC: staticDir,
+      HUB_CACHE_SECONDS: '0',
+      HUB_TIMEOUT_MS: '2000',
+      ...extra,
+    }),
+  )
+  await new Promise<void>((resolve) => hub.listen(0, '127.0.0.1', resolve))
+  base = `http://127.0.0.1:${(hub.address() as AddressInfo).port}`
+  const res = await fetch(`${base}/api/user/login`, {
+    method: 'POST',
+    headers: { 'content-type': 'application/x-www-form-urlencoded' },
+    body: 'user=admin&pass=secret',
+  })
+  token = ((await res.json()) as { token: string }).token
+}
+
+async function matrix() {
+  const res = await fetch(`${base}/api/hub/apps/matrix`, { headers: { authorization: `Bearer ${token}` } })
+  const text = await res.text()
+  return { text, body: JSON.parse(text) as Record<string, any> }
+}
+
+const byName = (body: Record<string, any>, name: string) => (body.response.apps as any[]).find((x) => x.name === name)
+
+beforeAll(async () => {
+  a = await fake({})
+  b = await fake({})
+  dataDir = mkdtempSync(path.join(tmpdir(), 'hub-matrix-data-'))
+  staticDir = mkdtempSync(path.join(tmpdir(), 'hub-matrix-static-'))
+})
+
+afterAll(() => {
+  hub?.close()
+  a.server.close()
+  b.server.close()
+  rmSync(dataDir, { recursive: true, force: true })
+  rmSync(staticDir, { recursive: true, force: true })
+})
+
+beforeEach(async () => {
+  reset()
+  await start()
+})
+
+describe('jsonDiff', () => {
+  it('compares objects by key and arrays by index', () => {
+    expect(jsonDiff({ a: 1 }, { a: 1 })).toEqual([])
+    expect(jsonDiff({ a: 1, b: { c: 2 } }, { a: 2, b: { c: 3 } })).toEqual([
+      { path: 'a', a: 1, b: 2 },
+      { path: 'b.c', a: 2, b: 3 },
+    ])
+    expect(jsonDiff([1, 2], [1, 3, 4])).toEqual([
+      { path: '[1]', a: 2, b: 3 },
+      { path: '[2]', a: undefined, b: 4 },
+    ])
+  })
+
+  it('reports type changes once and missing keys on either side', () => {
+    expect(jsonDiff({ a: { x: 1 } }, { a: [1] })).toEqual([{ path: 'a', a: { x: 1 }, b: [1] }])
+    expect(jsonDiff({ a: null }, { a: 0 })).toEqual([{ path: 'a', a: null, b: 0 }])
+    expect(jsonDiff({ a: 1 }, { b: 1 })).toEqual([
+      { path: 'a', a: 1, b: undefined },
+      { path: 'b', a: undefined, b: 1 },
+    ])
+  })
+})
+
+describe('countDifferences', () => {
+  it('counts leaf paths for JSON and lines otherwise', () => {
+    expect(countDifferences([json({ a: 1, b: 2 }), json({ a: 2, b: 3 })])).toBe(2)
+    expect(countDifferences(['one\ntwo', 'one\nTWO'])).toBe(2)
+    expect(countDifferences([json({ a: 1 }), 'not json'])).toBeGreaterThan(0)
+    expect(countDifferences([json({ a: 1 })])).toBe(0)
+  })
+})
+
+describe('hub/apps/matrix', () => {
+  it('has the documented shape and classifies each app', async () => {
+    const { body } = await matrix()
+    expect(body.status).toBe('ok')
+    expect(body.hubWarnings).toEqual([])
+    expect(body.response.servers).toEqual([{ id: 'a', ok: true }, { id: 'b', ok: true }])
+    expect((body.response.apps as any[]).map((x) => x.name)).toEqual(['Differs', 'Empty', 'Only A', 'Plain', 'Same'])
+
+    const same = byName(body, 'Same')
+    expect(same).toMatchObject({ configs: 'same', versions: 'same', description: 'Same description' })
+    expect(same.diffCount).toBeUndefined()
+    expect(same.servers.a).toMatchObject({ installed: true, version: '1.0', configStatus: 'read' })
+    expect(same.servers.a.configHash).toBe(same.servers.b.configHash)
+    expect(same.servers.a.configHash).toMatch(/^[0-9a-f]{16}$/)
+
+    const differs = byName(body, 'Differs')
+    expect(differs).toMatchObject({ configs: 'different', versions: 'different', diffCount: 3 })
+    expect(differs.servers.b).toMatchObject({ updateAvailable: true, updateVersion: '2.2', version: '2.1' })
+    expect(differs.servers.a.updateAvailable).toBeUndefined()
+
+    expect(byName(body, 'Plain')).toMatchObject({ configs: 'different', diffCount: 2 })
+
+    const only = byName(body, 'Only A')
+    expect(only).toMatchObject({ configs: 'single', versions: 'single' })
+    expect(only.servers.b).toEqual({ installed: false, configStatus: 'none' })
+
+    const empty = byName(body, 'Empty')
+    expect(empty).toMatchObject({ configs: 'same' })
+    expect(empty.servers.a).toMatchObject({ configStatus: 'none' })
+    expect(empty.servers.a.configHash).toBeUndefined()
+  })
+
+  it('never returns a config text', async () => {
+    const { text } = await matrix()
+    expect(text).not.toContain('top-secret-value')
+    expect(text).not.toContain('line one')
+    expect(text).not.toContain('read-a')
+  })
+
+  it('marks a config unreadable when the read token is refused and there is no admin token', async () => {
+    a.denyRead = true
+    const { body } = await matrix()
+    const same = byName(body, 'Same')
+    expect(same.servers.a).toMatchObject({ installed: true, configStatus: 'unreadable' })
+    expect(same.servers.a.configError).toMatch(/denied/)
+    expect(same.servers.a.configHash).toBeUndefined()
+    expect(same.configs).toBe('unknown')
+    expect(byName(body, 'Only A').configs).toBe('unknown')
+    // Versions are still known.
+    expect(same.versions).toBe('same')
+  })
+
+  it('retries with the admin token when the read token is refused', async () => {
+    a.denyRead = true
+    await start({ A_ADMIN_TOKEN: 'admin-a' })
+    const { body, text } = await matrix()
+    expect(byName(body, 'Same')).toMatchObject({ configs: 'same' })
+    expect(byName(body, 'Same').servers.a.configStatus).toBe('read')
+    expect(a.tokens).toContain('read-a')
+    expect(a.tokens).toContain('admin-a')
+    expect(b.tokens.every((t) => t === 'read-b')).toBe(true)
+    expect(text).not.toContain('admin-a')
+  })
+
+  it('shows a server that is down and still renders the rest', async () => {
+    b.down = true
+    const { body } = await matrix()
+    expect(body.status).toBe('ok')
+    expect(body.response.servers[0]).toEqual({ id: 'a', ok: true })
+    expect(body.response.servers[1]).toMatchObject({ id: 'b', ok: false })
+    expect(body.response.servers[1].error).toBeTruthy()
+    expect(body.hubWarnings).toHaveLength(1)
+    expect(body.hubWarnings[0]).toMatch(/^b:/)
+    const same = byName(body, 'Same')
+    expect(same.servers.a.installed).toBe(true)
+    expect(same.servers.b).toMatchObject({ installed: false, configStatus: 'unreadable' })
+    expect(same.configs).toBe('unknown')
+  })
+
+  it('fails when no server answers', async () => {
+    a.down = true
+    b.down = true
+    const { body } = await matrix()
+    expect(body.status).toBe('error')
+    expect(body.errorMessage).toMatch(/No server answered/)
+  })
+
+  it('is open to every role that can read', async () => {
+    const res = await fetch(`${base}/api/hub/apps/matrix`)
+    expect(((await res.json()) as { status: string }).status).toBe('invalid-token')
+  })
+})
+
+describe('session permissions', () => {
+  it('lets every role view Apps without modifying it', async () => {
+    const res = await fetch(`${base}/api/user/session/get`, { headers: { authorization: `Bearer ${token}` } })
+    const info = ((await res.json()) as any).info
+    expect(info.permissions.Apps).toEqual({ canView: true, canModify: false, canDelete: false })
+  })
+})

hub/apps-matrix.ts

@@ -0,0 +1,166 @@
+/*
+`hub/apps/matrix`: which DNS apps are installed on which server, with what version, and
+whether their configs agree. Read-only. The config texts are compared here and never
+leave: the answer carries a short hash per copy and the comparison, because a config may
+hold secrets.
+*/
+import { createHash } from 'node:crypto'
+import type { HubConfig, HubServer } from './config.ts'
+import { jsonDiff } from './json-diff.ts'
+import { Upstream, UpstreamError } from './upstream.ts'
+
+export interface ServerCell {
+  installed: boolean
+  version?: string
+  updateAvailable?: boolean
+  updateVersion?: string
+  configStatus: 'none' | 'read' | 'unreadable'
+  configHash?: string
+  configError?: string
+}
+
+export interface MatrixApp {
+  name: string
+  description: string
+  servers: Record<string, ServerCell>
+  configs: 'same' | 'different' | 'single' | 'unknown'
+  diffCount?: number
+  versions: 'same' | 'different' | 'single'
+}
+
+export interface Matrix {
+  servers: { id: string; ok: boolean; error?: string }[]
+  apps: MatrixApp[]
+}
+
+interface Listed {
+  name: string
+  description?: string
+  version?: string
+  updateAvailable?: boolean
+  updateVersion?: string
+}
+
+const hash = (text: string): string => createHash('sha256').update(text).digest('hex').slice(0, 16)
+
+const reason = (e: unknown): string => (e instanceof Error ? e.message : 'failed')
+
+/* The read token first; when the server refuses it, once more with the admin token if there is one. */
+async function readConfig(upstream: Upstream, server: HubServer, name: string): Promise<string | null> {
+  const params = { name }
+  let response: unknown
+  try {
+    response = await upstream.call(server, 'apps/config/get', params)
+  } catch (e) {
+    if (!(e instanceof UpstreamError) || e.kind !== 'rejected' || server.adminToken === undefined) throw e
+    response = await upstream.call(server, 'apps/config/get', params, server.adminToken)
+  }
+  const config = (response as { config?: unknown } | null)?.config
+  return typeof config === 'string' ? config : null
+}
+
+/* How many places differ between the readable configs. JSON: differing leaf paths; else differing lines. */
+export function countDifferences(texts: string[]): number {
+  if (texts.length < 2) return 0
+  let parsed: unknown[] | null = []
+  for (const text of texts) {
+    try {
+      parsed.push(JSON.parse(text))
+    } catch {
+      parsed = null
+      break
+    }
+  }
+  if (parsed !== null) {
+    const paths = new Set<string>()
+    for (let i = 1; i < parsed.length; i++) for (const d of jsonDiff(parsed[0], parsed[i])) paths.add(d.path)
+    return paths.size
+  }
+  const lines = texts.map((t) => t.split(/\r?\n/).map((l) => l.trim()))
+  let most = 0
+  for (let i = 1; i < lines.length; i++) {
+    const [a, b] = [new Set(lines[0]), new Set(lines[i])]
+    let n = 0
+    for (const l of a) if (!b.has(l)) n++
+    for (const l of b) if (!a.has(l)) n++
+    most = Math.max(most, n)
+  }
+  return most
+}
+
+/* The matrix and the servers that did not answer, as `hubWarnings` lines. */
+export async function buildMatrix(config: HubConfig, upstream: Upstream): Promise<{ matrix: Matrix; warnings: string[] }> {
+  type Copy = { cell: ServerCell; text: string | null }
+  const perServer = await Promise.all(
+    config.servers.map(async (server) => {
+      let list: Listed[]
+      try {
+        const response = (await upstream.call(server, 'apps/list')) as { apps?: Listed[] } | null
+        list = response?.apps ?? []
+      } catch (e) {
+        return { server, error: reason(e), apps: new Map<string, { listed: Listed } & Copy>() }
+      }
+      const entries = await Promise.all(
+        list.map(async (listed): Promise<[string, { listed: Listed } & Copy]> => {
+          const cell: ServerCell = {
+            installed: true,
+            ...(listed.version === undefined ? {} : { version: listed.version }),
+            ...(listed.updateAvailable === undefined ? {} : { updateAvailable: listed.updateAvailable }),
+            ...(listed.updateVersion === undefined ? {} : { updateVersion: listed.updateVersion }),
+            configStatus: 'none',
+          }
+          let text: string | null = null
+          try {
+            text = await readConfig(upstream, server, listed.name)
+            if (text !== null && text !== '') {
+              cell.configStatus = 'read'
+              cell.configHash = hash(text)
+            }
+          } catch (e) {
+            cell.configStatus = 'unreadable'
+            cell.configError = reason(e)
+            text = null
+          }
+          return [listed.name, { listed, cell, text }]
+        }),
+      )
+      return { server, error: undefined, apps: new Map(entries) }
+    }),
+  )
+
+  const servers = perServer.map((p) => (p.error === undefined ? { id: p.server.id, ok: true } : { id: p.server.id, ok: false, error: p.error }))
+  const warnings = perServer.filter((p) => p.error !== undefined).map((p) => `${p.server.id}: ${p.error}`)
+  if (warnings.length === perServer.length) throw new Error(`No server answered. ${warnings.join('; ')}`.trim())
+
+  const names = [...new Set(perServer.flatMap((p) => [...p.apps.keys()]))].sort((a, b) => a.localeCompare(b))
+  const apps: MatrixApp[] = names.map((name) => {
+    const cells: Record<string, ServerCell> = {}
+    const copies: { id: string; cell: ServerCell; text: string | null }[] = []
+    let description = ''
+    for (const p of perServer) {
+      const found = p.apps.get(name)
+      if (found !== undefined) {
+        cells[p.server.id] = found.cell
+        copies.push({ id: p.server.id, cell: found.cell, text: found.text })
+        if (description === '' && found.listed.description) description = found.listed.description
+      } else if (p.error !== undefined) {
+        cells[p.server.id] = { installed: false, configStatus: 'unreadable', configError: `${p.server.id} did not answer` }
+      } else {
+        cells[p.server.id] = { installed: false, configStatus: 'none' }
+      }
+    }
+
+    const versionSet = new Set(copies.map((c) => c.cell.version ?? ''))
+    const versions = copies.length < 2 ? 'single' : versionSet.size === 1 ? 'same' : 'different'
+
+    const readable = copies.filter((c) => c.cell.configStatus !== 'unreadable').map((c) => c.text ?? '')
+    const hasUnknown = copies.some((c) => c.cell.configStatus === 'unreadable') || (warnings.length > 0 && copies.length < 2)
+    const differs = new Set(readable).size > 1
+    const configs: MatrixApp['configs'] = differs ? 'different' : hasUnknown ? 'unknown' : copies.length < 2 ? 'single' : 'same'
+    const app: MatrixApp = { name, description, servers: cells, configs, versions }
+    if (differs) app.diffCount = countDifferences(readable)
+    return app
+  })
+
+  return { matrix: { servers, apps }, warnings }
+}

hub/json-diff.ts

@@ -0,0 +1,38 @@
+/*
+A small structural diff of two JSON values: objects by key, arrays by index. A leaf that
+differs, a key present on one side only, and a change of type each give one entry.
+*/
+export interface JsonDifference {
+  path: string
+  a: unknown
+  b: unknown
+}
+
+const kind = (v: unknown): string => (v === null ? 'null' : Array.isArray(v) ? 'array' : typeof v)
+
+export function jsonDiff(a: unknown, b: unknown, path = ''): JsonDifference[] {
+  const ka = kind(a)
+  if (ka !== kind(b)) return [{ path, a, b }]
+  if (ka === 'array') {
+    const [xs, ys] = [a as unknown[], b as unknown[]]
+    const out: JsonDifference[] = []
+    for (let i = 0; i < Math.max(xs.length, ys.length); i++) {
+      const here = `${path}[${i}]`
+      if (i >= xs.length || i >= ys.length) out.push({ path: here, a: xs[i], b: ys[i] })
+      else out.push(...jsonDiff(xs[i], ys[i], here))
+    }
+    return out
+  }
+  if (ka === 'object') {
+    const [x, y] = [a as Record<string, unknown>, b as Record<string, unknown>]
+    const out: JsonDifference[] = []
+    for (const key of new Set([...Object.keys(x), ...Object.keys(y)])) {
+      const here = path === '' ? key : `${path}.${key}`
+      const [inX, inY] = [Object.hasOwn(x, key), Object.hasOwn(y, key)]
+      if (inX && inY) out.push(...jsonDiff(x[key], y[key], here))
+      else out.push({ path: here, a: x[key], b: y[key] })
+    }
+    return out
+  }
+  return a === b || (Number.isNaN(a) && Number.isNaN(b)) ? [] : [{ path, a, b }]
+}

hub/routes.ts

@@ -29,6 +29,7 @@ export const ROUTES: Record<string, Route> = {
   'admin/cluster/state': read(),
   // The hub's own namespace; never forwarded to a server.
   'hub/audit/list': read('audit.read'),
+  'hub/apps/matrix': read(),
 }
 
 export function routeFor(apiPath: string): Route | undefined {

hub/server.ts

@@ -21,6 +21,7 @@ import { Users, can } from './users.ts'
 import { LoginLimiter, lockedMessage } from './ratelimit.ts'
 import { Audit } from './audit.ts'
 import { routeFor } from './routes.ts'
+import { buildMatrix } from './apps-matrix.ts'
 import {
   mergeQueryLogs,
   mergeSettled,
@@ -248,6 +249,15 @@ export function createHub(config: HubConfig, users: Users = Users.load(config.da
       send(res, 200, ok({ entries: audit.list(Number(params.limit ?? 200), params.before) }))
       return
     }
+    if (apiPath === 'hub/apps/matrix') {
+      try {
+        const { matrix, warnings } = await buildMatrix(config, upstream)
+        send(res, 200, ok(matrix, warnings))
+      } catch (e) {
+        send(res, 200, fail(e instanceof Error ? e.message : 'failed'))
+      }
+      return
+    }
     if (apiPath === 'user/checkForUpdate') {
       send(res, 200, ok({ updateAvailable: false }))
       return

hub/session.ts

@@ -23,13 +23,13 @@ interface Session extends Identity {
 /*
 Every key `src/app/sections.ts` reads, spelled out: the console hides a section only
 when `canView === false`, so a missing key would leave it visible. The hub shows the
-Dashboard and the Logs; About has no permission and is always there. The hub is
+Dashboard, the Logs and the Apps; About has no permission and is always there. The hub is
 read-only, so nothing can be modified or deleted.
 */
 const PERMISSION_KEYS = [
   'Dashboard', 'Zones', 'Cache', 'Allowed', 'Blocked', 'Apps', 'DnsClient', 'Settings', 'DhcpServer', 'Administration', 'Logs',
 ]
-const VIEWABLE = new Set(['Dashboard', 'Logs'])
+const VIEWABLE = new Set(['Dashboard', 'Logs', 'Apps'])
 
 export class Sessions {
   #config: HubConfig

hub/upstream.ts

@@ -41,14 +41,14 @@ export class Upstream {
   }
 
   /** The `response` of the server's envelope, or an `UpstreamError`. */
-  async call(server: HubServer, path: string, params: Params = {}): Promise<unknown> {
+  async call(server: HubServer, path: string, params: Params = {}, token: string = server.token): Promise<unknown> {
     const sorted = Object.entries(params).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
     const key = `${server.id}\n${path}\n${new URLSearchParams(sorted).toString()}`
     const hit = this.#cache.get(key)
     if (hit !== undefined && hit.until > this.#now()) return hit.response
 
     const query = new URLSearchParams(sorted)
-    query.set('token', server.token)
+    query.set('token', token)
     const controller = new AbortController()
     const timer = setTimeout(() => controller.abort(), this.#config.timeoutMs)
 
@@ -60,7 +60,7 @@ export class Upstream {
       })
       envelope = (await res.json()) as typeof envelope
     } catch (e) {
-      throw new UpstreamError('unreachable', this.#describe(e, server))
+      throw new UpstreamError('unreachable', this.#describe(e, [server.token, token]))
     } finally {
       clearTimeout(timer)
     }
@@ -88,10 +88,10 @@ export class Upstream {
   }
 
   /* A short reason without anything that could carry the token. */
-  #describe(e: unknown, server: HubServer): string {
+  #describe(e: unknown, secrets: string[]): string {
     if (e instanceof Error && e.name === 'AbortError') return `timed out after ${this.#config.timeoutMs} ms`
     const cause = e instanceof Error ? (e.cause as { code?: unknown } | undefined) : undefined
     const text = typeof cause?.code === 'string' ? cause.code : e instanceof Error ? e.message : 'request failed'
-    return text.split(server.token).join('***')
+    return secrets.reduce((out, secret) => (secret === '' ? out : out.split(secret).join('***')), text)
   }
 }