Hub stage 1: apps across servers, read-only

hub/apps/matrix reads apps/list and each app's config on every server (the read
token first, the admin token once if a server refuses) and returns per app and server
the version, available update and a config hash, plus whether versions and configs
match and how many JSON leaf paths differ. Config texts never leave the hub. In hub
mode the Apps section has the server selector: All servers shows the matrix with
filters for differing versions, differing configs and updates; one server shows its
list without write actions. The dev mock takes --variant a|b with differing apps.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

dev/mock-api.mjs

@@ -15,10 +15,12 @@ behaviour: it exists so a design can be seen with data in it. The real harness i
 */
 import http from 'node:http'
 
-// `--port <n>` and `--scale <x>`: two instances with different numbers stand in for two servers.
+// `--port <n>`, `--scale <x>` and `--variant a|b`: two instances with different numbers (and apps) stand in for two servers.
 const arg = (name) => { const i = process.argv.indexOf(`--${name}`); return i >= 0 ? process.argv[i + 1] : undefined }
 const PORT = Number(arg('port') ?? process.env.PORT ?? 5380)
 const SCALE = Number(arg('scale') ?? 1)
+// `--variant b`: the second server of the Apps matrix (a missing app, another version, other config fields).
+const VARIANT = arg('variant') === 'b' ? 'b' : 'a'
 const now = Date.now()
 const iso = (ms) => new Date(ms).toISOString()
 const ago = (min) => iso(now - min * 60_000)
@@ -189,6 +191,17 @@ let apps = [
   { name: 'Advanced Blocking', description: 'Group based blocking with regex', version: '10.1', updateAvailable: true, updateVersion: '10.2', updateUrl: 'https://download.technitium.com/dns/apps/AdvancedBlockingApp-v10.2.zip', dnsApps: [{ ...app(false), classPath: 'AdvancedBlocking.App' }] },
   { name: 'Split Horizon', description: 'Different answers for different networks', version: '9.0', dnsApps: [{ ...app(false), classPath: 'SplitHorizon.SimpleAddress' }, { ...app(false), classPath: 'SplitHorizon.SimpleCNAME' }] },
 ]
+if (VARIANT === 'b') {
+  apps = apps.filter((a) => a.name !== 'Split Horizon')
+  apps = apps.map((a) => (a.name === 'Query Logs (Sqlite)' ? { ...a, version: '7.1', updateAvailable: true, updateVersion: '7.2', updateUrl: 'https://download.technitium.com/dns/apps/QueryLogsSqliteApp-v7.2.zip' } : a))
+}
+const appConfigs = {
+  'Advanced Blocking': VARIANT === 'b'
+    ? { enableBlocking: true, blockingAnswerTtl: 60, networkGroupMap: { '0.0.0.0/0': 'default' }, groups: [{ name: 'default', enableBlocking: false, blockAsNxDomain: false }] }
+    : { enableBlocking: true, blockingAnswerTtl: 30, networkGroupMap: { '0.0.0.0/0': 'default' }, groups: [{ name: 'default', enableBlocking: true, blockAsNxDomain: false }] },
+  'Query Logs (Sqlite)': { enableLogging: true, maxLogDays: 365, sqliteDbPath: VARIANT === 'b' ? '/var/lib/dns/ndns/querylogs.db' : '/var/lib/dns/mdns/querylogs.db' },
+  'Split Horizon': { domainGroupMap: { 'lan.example.net': 'internal' }, networkGroupMap: { '10.0.0.0/8': 'internal' } },
+}
 const store = [
   ['Failover', 'Fails over to a backup address when the primary is down', '8.0', '58 KB'], ['Geo Country', 'Answers by the country of the client', '7.2', '1.2 MB'],
   ['Wild IP', 'Answers with the IP address in the name', '5.1', '22 KB'], ['Auto PTR', 'Answers PTR queries from the forward zones', '6.0', '31 KB'],
@@ -267,7 +280,7 @@ function route(path, q) {
     case 'dhcp/scopes/delete': scopes = scopes.filter((s) => s.name !== q.get('name')); return ok()
     case 'apps/list': return ok({ apps })
     case 'apps/listStoreApps': return ok({ storeApps: store })
-    case 'apps/config/get': return ok({ config: '{\n  "enabled": true\n}' })
+    case 'apps/config/get': return ok({ config: appConfigs[q.get('name')] ? JSON.stringify(appConfigs[q.get('name')], null, 2) : JSON.stringify({ enabled: true }, null, 2) })
     case 'apps/uninstall': apps = apps.filter((a) => a.name !== q.get('name')); return ok()
     case 'logs/list': return ok({ logFiles: [{ fileName: '2026-10-26', size: '412.8 KB' }, { fileName: '2026-10-25', size: '1.2 MB' }, { fileName: '2026-10-24', size: '980.4 KB' }] })
     case 'logs/query': {

hub/apps-matrix.test.ts

@@ -0,0 +1,280 @@
+// @vitest-environment node
+import http from 'node:http'
+import type { AddressInfo } from 'node:net'
+import { mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
+import { parseConfig } from './config.ts'
+import { createHub } from './server.ts'
+import { countDifferences } from './apps-matrix.ts'
+import { jsonDiff } from './json-diff.ts'
+
+interface FakeApp {
+  version: string
+  config: string | null
+  updateAvailable?: boolean
+  updateVersion?: string
+}
+
+interface Fake {
+  server: http.Server
+  url: string
+  apps: Record<string, FakeApp>
+  down: boolean
+  /** Config reads with the read token are refused with a permission error. */
+  denyRead: boolean
+  tokens: string[]
+}
+
+function fake(apps: Fake['apps']): Promise<Fake> {
+  return new Promise((resolve) => {
+    const f: Fake = { server: http.createServer(), url: '', apps, down: false, denyRead: false, tokens: [] }
+    f.server.on('request', (req, res) => {
+      if (f.down) return void req.socket.destroy()
+      const url = new URL(req.url ?? '/', 'http://fake')
+      const apiPath = url.pathname.replace(/^\/api\//, '')
+      const token = url.searchParams.get('token') ?? ''
+      res.setHeader('content-type', 'application/json')
+      if (apiPath === 'apps/list') {
+        const list = Object.entries(f.apps).map(([name, a]) => ({
+          name,
+          description: `${name} description`,
+          version: a.version,
+          ...(a.updateAvailable ? { updateAvailable: true, updateVersion: a.updateVersion } : {}),
+        }))
+        return void res.end(JSON.stringify({ status: 'ok', response: { apps: list } }))
+      }
+      if (apiPath === 'apps/config/get') {
+        f.tokens.push(token)
+        if (f.denyRead && !token.startsWith('admin-')) {
+          return void res.end(JSON.stringify({ status: 'error', errorMessage: "Access was denied to user 'reader' for apps." }))
+        }
+        const app = f.apps[url.searchParams.get('name') ?? '']
+        return void res.end(JSON.stringify({ status: 'ok', response: { config: app?.config ?? null } }))
+      }
+      res.end(JSON.stringify({ status: 'ok', response: {} }))
+    })
+    f.server.listen(0, '127.0.0.1', () => {
+      f.url = `http://127.0.0.1:${(f.server.address() as AddressInfo).port}`
+      resolve(f)
+    })
+  })
+}
+
+const json = (o: unknown): string => JSON.stringify(o, null, 2)
+
+let a: Fake
+let b: Fake
+let dataDir: string
+let staticDir: string
+let hub: http.Server
+let base: string
+let token: string
+
+function reset(): void {
+  a.down = b.down = false
+  a.denyRead = b.denyRead = false
+  a.tokens = []
+  b.tokens = []
+  a.apps = {
+    Same: { version: '1.0', config: json({ x: 1 }) },
+    Differs: { version: '2.0', config: json({ x: 1, y: { z: 2 }, list: [1, 2] }) },
+    'Only A': { version: '3.0', config: json({ secret: 'top-secret-value' }) },
+    Plain: { version: '4.0', config: 'line one\nline two\nline three' },
+    Empty: { version: '5.0', config: null },
+  }
+  b.apps = {
+    Same: { version: '1.0', config: json({ x: 1 }) },
+    Differs: { version: '2.1', updateAvailable: true, updateVersion: '2.2', config: json({ x: 2, y: { z: 3 }, list: [1, 2, 3] }) },
+    Plain: { version: '4.0', config: 'line one\nline TWO\nline three' },
+    Empty: { version: '5.0', config: null },
+  }
+}
+
+async function start(extra: Record<string, string> = {}): Promise<void> {
+  hub?.close()
+  hub = createHub(
+    parseConfig({
+      HUB_SERVERS: `a=${a.url},b=${b.url}`,
+      A_TOKEN: 'read-a',
+      B_TOKEN: 'read-b',
+      HUB_USER: 'admin',
+      HUB_PASSWORD: 'secret',
+      HUB_DATA: dataDir,
+      HUB_STATIC: staticDir,
+      HUB_CACHE_SECONDS: '0',
+      HUB_TIMEOUT_MS: '2000',
+      ...extra,
+    }),
+  )
+  await new Promise<void>((resolve) => hub.listen(0, '127.0.0.1', resolve))
+  base = `http://127.0.0.1:${(hub.address() as AddressInfo).port}`
+  const res = await fetch(`${base}/api/user/login`, {
+    method: 'POST',
+    headers: { 'content-type': 'application/x-www-form-urlencoded' },
+    body: 'user=admin&pass=secret',
+  })
+  token = ((await res.json()) as { token: string }).token
+}
+
+async function matrix() {
+  const res = await fetch(`${base}/api/hub/apps/matrix`, { headers: { authorization: `Bearer ${token}` } })
+  const text = await res.text()
+  return { text, body: JSON.parse(text) as Record<string, any> }
+}
+
+const byName = (body: Record<string, any>, name: string) => (body.response.apps as any[]).find((x) => x.name === name)
+
+beforeAll(async () => {
+  a = await fake({})
+  b = await fake({})
+  dataDir = mkdtempSync(path.join(tmpdir(), 'hub-matrix-data-'))
+  staticDir = mkdtempSync(path.join(tmpdir(), 'hub-matrix-static-'))
+})
+
+afterAll(() => {
+  hub?.close()
+  a.server.close()
+  b.server.close()
+  rmSync(dataDir, { recursive: true, force: true })
+  rmSync(staticDir, { recursive: true, force: true })
+})
+
+beforeEach(async () => {
+  reset()
+  await start()
+})
+
+describe('jsonDiff', () => {
+  it('compares objects by key and arrays by index', () => {
+    expect(jsonDiff({ a: 1 }, { a: 1 })).toEqual([])
+    expect(jsonDiff({ a: 1, b: { c: 2 } }, { a: 2, b: { c: 3 } })).toEqual([
+      { path: 'a', a: 1, b: 2 },
+      { path: 'b.c', a: 2, b: 3 },
+    ])
+    expect(jsonDiff([1, 2], [1, 3, 4])).toEqual([
+      { path: '[1]', a: 2, b: 3 },
+      { path: '[2]', a: undefined, b: 4 },
+    ])
+  })
+
+  it('reports type changes once and missing keys on either side', () => {
+    expect(jsonDiff({ a: { x: 1 } }, { a: [1] })).toEqual([{ path: 'a', a: { x: 1 }, b: [1] }])
+    expect(jsonDiff({ a: null }, { a: 0 })).toEqual([{ path: 'a', a: null, b: 0 }])
+    expect(jsonDiff({ a: 1 }, { b: 1 })).toEqual([
+      { path: 'a', a: 1, b: undefined },
+      { path: 'b', a: undefined, b: 1 },
+    ])
+  })
+})
+
+describe('countDifferences', () => {
+  it('counts leaf paths for JSON and lines otherwise', () => {
+    expect(countDifferences([json({ a: 1, b: 2 }), json({ a: 2, b: 3 })])).toBe(2)
+    expect(countDifferences(['one\ntwo', 'one\nTWO'])).toBe(2)
+    expect(countDifferences([json({ a: 1 }), 'not json'])).toBeGreaterThan(0)
+    expect(countDifferences([json({ a: 1 })])).toBe(0)
+  })
+})
+
+describe('hub/apps/matrix', () => {
+  it('has the documented shape and classifies each app', async () => {
+    const { body } = await matrix()
+    expect(body.status).toBe('ok')
+    expect(body.hubWarnings).toEqual([])
+    expect(body.response.servers).toEqual([{ id: 'a', ok: true }, { id: 'b', ok: true }])
+    expect((body.response.apps as any[]).map((x) => x.name)).toEqual(['Differs', 'Empty', 'Only A', 'Plain', 'Same'])
+
+    const same = byName(body, 'Same')
+    expect(same).toMatchObject({ configs: 'same', versions: 'same', description: 'Same description' })
+    expect(same.diffCount).toBeUndefined()
+    expect(same.servers.a).toMatchObject({ installed: true, version: '1.0', configStatus: 'read' })
+    expect(same.servers.a.configHash).toBe(same.servers.b.configHash)
+    expect(same.servers.a.configHash).toMatch(/^[0-9a-f]{16}$/)
+
+    const differs = byName(body, 'Differs')
+    expect(differs).toMatchObject({ configs: 'different', versions: 'different', diffCount: 3 })
+    expect(differs.servers.b).toMatchObject({ updateAvailable: true, updateVersion: '2.2', version: '2.1' })
+    expect(differs.servers.a.updateAvailable).toBeUndefined()
+
+    expect(byName(body, 'Plain')).toMatchObject({ configs: 'different', diffCount: 2 })
+
+    const only = byName(body, 'Only A')
+    expect(only).toMatchObject({ configs: 'single', versions: 'single' })
+    expect(only.servers.b).toEqual({ installed: false, configStatus: 'none' })
+
+    const empty = byName(body, 'Empty')
+    expect(empty).toMatchObject({ configs: 'same' })
+    expect(empty.servers.a).toMatchObject({ configStatus: 'none' })
+    expect(empty.servers.a.configHash).toBeUndefined()
+  })
+
+  it('never returns a config text', async () => {
+    const { text } = await matrix()
+    expect(text).not.toContain('top-secret-value')
+    expect(text).not.toContain('line one')
+    expect(text).not.toContain('read-a')
+  })
+
+  it('marks a config unreadable when the read token is refused and there is no admin token', async () => {
+    a.denyRead = true
+    const { body } = await matrix()
+    const same = byName(body, 'Same')
+    expect(same.servers.a).toMatchObject({ installed: true, configStatus: 'unreadable' })
+    expect(same.servers.a.configError).toMatch(/denied/)
+    expect(same.servers.a.configHash).toBeUndefined()
+    expect(same.configs).toBe('unknown')
+    expect(byName(body, 'Only A').configs).toBe('unknown')
+    // Versions are still known.
+    expect(same.versions).toBe('same')
+  })
+
+  it('retries with the admin token when the read token is refused', async () => {
+    a.denyRead = true
+    await start({ A_ADMIN_TOKEN: 'admin-a' })
+    const { body, text } = await matrix()
+    expect(byName(body, 'Same')).toMatchObject({ configs: 'same' })
+    expect(byName(body, 'Same').servers.a.configStatus).toBe('read')
+    expect(a.tokens).toContain('read-a')
+    expect(a.tokens).toContain('admin-a')
+    expect(b.tokens.every((t) => t === 'read-b')).toBe(true)
+    expect(text).not.toContain('admin-a')
+  })
+
+  it('shows a server that is down and still renders the rest', async () => {
+    b.down = true
+    const { body } = await matrix()
+    expect(body.status).toBe('ok')
+    expect(body.response.servers[0]).toEqual({ id: 'a', ok: true })
+    expect(body.response.servers[1]).toMatchObject({ id: 'b', ok: false })
+    expect(body.response.servers[1].error).toBeTruthy()
+    expect(body.hubWarnings).toHaveLength(1)
+    expect(body.hubWarnings[0]).toMatch(/^b:/)
+    const same = byName(body, 'Same')
+    expect(same.servers.a.installed).toBe(true)
+    expect(same.servers.b).toMatchObject({ installed: false, configStatus: 'unreadable' })
+    expect(same.configs).toBe('unknown')
+  })
+
+  it('fails when no server answers', async () => {
+    a.down = true
+    b.down = true
+    const { body } = await matrix()
+    expect(body.status).toBe('error')
+    expect(body.errorMessage).toMatch(/No server answered/)
+  })
+
+  it('is open to every role that can read', async () => {
+    const res = await fetch(`${base}/api/hub/apps/matrix`)
+    expect(((await res.json()) as { status: string }).status).toBe('invalid-token')
+  })
+})
+
+describe('session permissions', () => {
+  it('lets every role view Apps without modifying it', async () => {
+    const res = await fetch(`${base}/api/user/session/get`, { headers: { authorization: `Bearer ${token}` } })
+    const info = ((await res.json()) as any).info
+    expect(info.permissions.Apps).toEqual({ canView: true, canModify: false, canDelete: false })
+  })
+})

hub/apps-matrix.ts

@@ -0,0 +1,166 @@
+/*
+`hub/apps/matrix`: which DNS apps are installed on which server, with what version, and
+whether their configs agree. Read-only. The config texts are compared here and never
+leave: the answer carries a short hash per copy and the comparison, because a config may
+hold secrets.
+*/
+import { createHash } from 'node:crypto'
+import type { HubConfig, HubServer } from './config.ts'
+import { jsonDiff } from './json-diff.ts'
+import { Upstream, UpstreamError } from './upstream.ts'
+
+export interface ServerCell {
+  installed: boolean
+  version?: string
+  updateAvailable?: boolean
+  updateVersion?: string
+  configStatus: 'none' | 'read' | 'unreadable'
+  configHash?: string
+  configError?: string
+}
+
+export interface MatrixApp {
+  name: string
+  description: string
+  servers: Record<string, ServerCell>
+  configs: 'same' | 'different' | 'single' | 'unknown'
+  diffCount?: number
+  versions: 'same' | 'different' | 'single'
+}
+
+export interface Matrix {
+  servers: { id: string; ok: boolean; error?: string }[]
+  apps: MatrixApp[]
+}
+
+interface Listed {
+  name: string
+  description?: string
+  version?: string
+  updateAvailable?: boolean
+  updateVersion?: string
+}
+
+const hash = (text: string): string => createHash('sha256').update(text).digest('hex').slice(0, 16)
+
+const reason = (e: unknown): string => (e instanceof Error ? e.message : 'failed')
+
+/* The read token first; when the server refuses it, once more with the admin token if there is one. */
+async function readConfig(upstream: Upstream, server: HubServer, name: string): Promise<string | null> {
+  const params = { name }
+  let response: unknown
+  try {
+    response = await upstream.call(server, 'apps/config/get', params)
+  } catch (e) {
+    if (!(e instanceof UpstreamError) || e.kind !== 'rejected' || server.adminToken === undefined) throw e
+    response = await upstream.call(server, 'apps/config/get', params, server.adminToken)
+  }
+  const config = (response as { config?: unknown } | null)?.config
+  return typeof config === 'string' ? config : null
+}
+
+/* How many places differ between the readable configs. JSON: differing leaf paths; else differing lines. */
+export function countDifferences(texts: string[]): number {
+  if (texts.length < 2) return 0
+  let parsed: unknown[] | null = []
+  for (const text of texts) {
+    try {
+      parsed.push(JSON.parse(text))
+    } catch {
+      parsed = null
+      break
+    }
+  }
+  if (parsed !== null) {
+    const paths = new Set<string>()
+    for (let i = 1; i < parsed.length; i++) for (const d of jsonDiff(parsed[0], parsed[i])) paths.add(d.path)
+    return paths.size
+  }
+  const lines = texts.map((t) => t.split(/\r?\n/).map((l) => l.trim()))
+  let most = 0
+  for (let i = 1; i < lines.length; i++) {
+    const [a, b] = [new Set(lines[0]), new Set(lines[i])]
+    let n = 0
+    for (const l of a) if (!b.has(l)) n++
+    for (const l of b) if (!a.has(l)) n++
+    most = Math.max(most, n)
+  }
+  return most
+}
+
+/* The matrix and the servers that did not answer, as `hubWarnings` lines. */
+export async function buildMatrix(config: HubConfig, upstream: Upstream): Promise<{ matrix: Matrix; warnings: string[] }> {
+  type Copy = { cell: ServerCell; text: string | null }
+  const perServer = await Promise.all(
+    config.servers.map(async (server) => {
+      let list: Listed[]
+      try {
+        const response = (await upstream.call(server, 'apps/list')) as { apps?: Listed[] } | null
+        list = response?.apps ?? []
+      } catch (e) {
+        return { server, error: reason(e), apps: new Map<string, { listed: Listed } & Copy>() }
+      }
+      const entries = await Promise.all(
+        list.map(async (listed): Promise<[string, { listed: Listed } & Copy]> => {
+          const cell: ServerCell = {
+            installed: true,
+            ...(listed.version === undefined ? {} : { version: listed.version }),
+            ...(listed.updateAvailable === undefined ? {} : { updateAvailable: listed.updateAvailable }),
+            ...(listed.updateVersion === undefined ? {} : { updateVersion: listed.updateVersion }),
+            configStatus: 'none',
+          }
+          let text: string | null = null
+          try {
+            text = await readConfig(upstream, server, listed.name)
+            if (text !== null && text !== '') {
+              cell.configStatus = 'read'
+              cell.configHash = hash(text)
+            }
+          } catch (e) {
+            cell.configStatus = 'unreadable'
+            cell.configError = reason(e)
+            text = null
+          }
+          return [listed.name, { listed, cell, text }]
+        }),
+      )
+      return { server, error: undefined, apps: new Map(entries) }
+    }),
+  )
+
+  const servers = perServer.map((p) => (p.error === undefined ? { id: p.server.id, ok: true } : { id: p.server.id, ok: false, error: p.error }))
+  const warnings = perServer.filter((p) => p.error !== undefined).map((p) => `${p.server.id}: ${p.error}`)
+  if (warnings.length === perServer.length) throw new Error(`No server answered. ${warnings.join('; ')}`.trim())
+
+  const names = [...new Set(perServer.flatMap((p) => [...p.apps.keys()]))].sort((a, b) => a.localeCompare(b))
+  const apps: MatrixApp[] = names.map((name) => {
+    const cells: Record<string, ServerCell> = {}
+    const copies: { id: string; cell: ServerCell; text: string | null }[] = []
+    let description = ''
+    for (const p of perServer) {
+      const found = p.apps.get(name)
+      if (found !== undefined) {
+        cells[p.server.id] = found.cell
+        copies.push({ id: p.server.id, cell: found.cell, text: found.text })
+        if (description === '' && found.listed.description) description = found.listed.description
+      } else if (p.error !== undefined) {
+        cells[p.server.id] = { installed: false, configStatus: 'unreadable', configError: `${p.server.id} did not answer` }
+      } else {
+        cells[p.server.id] = { installed: false, configStatus: 'none' }
+      }
+    }
+
+    const versionSet = new Set(copies.map((c) => c.cell.version ?? ''))
+    const versions = copies.length < 2 ? 'single' : versionSet.size === 1 ? 'same' : 'different'
+
+    const readable = copies.filter((c) => c.cell.configStatus !== 'unreadable').map((c) => c.text ?? '')
+    const hasUnknown = copies.some((c) => c.cell.configStatus === 'unreadable') || (warnings.length > 0 && copies.length < 2)
+    const differs = new Set(readable).size > 1
+    const configs: MatrixApp['configs'] = differs ? 'different' : hasUnknown ? 'unknown' : copies.length < 2 ? 'single' : 'same'
+    const app: MatrixApp = { name, description, servers: cells, configs, versions }
+    if (differs) app.diffCount = countDifferences(readable)
+    return app
+  })
+
+  return { matrix: { servers, apps }, warnings }
+}

hub/json-diff.ts

@@ -0,0 +1,38 @@
+/*
+A small structural diff of two JSON values: objects by key, arrays by index. A leaf that
+differs, a key present on one side only, and a change of type each give one entry.
+*/
+export interface JsonDifference {
+  path: string
+  a: unknown
+  b: unknown
+}
+
+const kind = (v: unknown): string => (v === null ? 'null' : Array.isArray(v) ? 'array' : typeof v)
+
+export function jsonDiff(a: unknown, b: unknown, path = ''): JsonDifference[] {
+  const ka = kind(a)
+  if (ka !== kind(b)) return [{ path, a, b }]
+  if (ka === 'array') {
+    const [xs, ys] = [a as unknown[], b as unknown[]]
+    const out: JsonDifference[] = []
+    for (let i = 0; i < Math.max(xs.length, ys.length); i++) {
+      const here = `${path}[${i}]`
+      if (i >= xs.length || i >= ys.length) out.push({ path: here, a: xs[i], b: ys[i] })
+      else out.push(...jsonDiff(xs[i], ys[i], here))
+    }
+    return out
+  }
+  if (ka === 'object') {
+    const [x, y] = [a as Record<string, unknown>, b as Record<string, unknown>]
+    const out: JsonDifference[] = []
+    for (const key of new Set([...Object.keys(x), ...Object.keys(y)])) {
+      const here = path === '' ? key : `${path}.${key}`
+      const [inX, inY] = [Object.hasOwn(x, key), Object.hasOwn(y, key)]
+      if (inX && inY) out.push(...jsonDiff(x[key], y[key], here))
+      else out.push({ path: here, a: x[key], b: y[key] })
+    }
+    return out
+  }
+  return a === b || (Number.isNaN(a) && Number.isNaN(b)) ? [] : [{ path, a, b }]
+}

hub/routes.ts

@@ -29,6 +29,7 @@ export const ROUTES: Record<string, Route> = {
   'admin/cluster/state': read(),
   // The hub's own namespace; never forwarded to a server.
   'hub/audit/list': read('audit.read'),
+  'hub/apps/matrix': read(),
 }
 
 export function routeFor(apiPath: string): Route | undefined {

hub/server.ts

@@ -21,6 +21,7 @@ import { Users, can } from './users.ts'
 import { LoginLimiter, lockedMessage } from './ratelimit.ts'
 import { Audit } from './audit.ts'
 import { routeFor } from './routes.ts'
+import { buildMatrix } from './apps-matrix.ts'
 import {
   mergeQueryLogs,
   mergeSettled,
@@ -248,6 +249,15 @@ export function createHub(config: HubConfig, users: Users = Users.load(config.da
       send(res, 200, ok({ entries: audit.list(Number(params.limit ?? 200), params.before) }))
       return
     }
+    if (apiPath === 'hub/apps/matrix') {
+      try {
+        const { matrix, warnings } = await buildMatrix(config, upstream)
+        send(res, 200, ok(matrix, warnings))
+      } catch (e) {
+        send(res, 200, fail(e instanceof Error ? e.message : 'failed'))
+      }
+      return
+    }
     if (apiPath === 'user/checkForUpdate') {
       send(res, 200, ok({ updateAvailable: false }))
       return

hub/session.ts

@@ -23,13 +23,13 @@ interface Session extends Identity {
 /*
 Every key `src/app/sections.ts` reads, spelled out: the console hides a section only
 when `canView === false`, so a missing key would leave it visible. The hub shows the
-Dashboard and the Logs; About has no permission and is always there. The hub is
+Dashboard, the Logs and the Apps; About has no permission and is always there. The hub is
 read-only, so nothing can be modified or deleted.
 */
 const PERMISSION_KEYS = [
   'Dashboard', 'Zones', 'Cache', 'Allowed', 'Blocked', 'Apps', 'DnsClient', 'Settings', 'DhcpServer', 'Administration', 'Logs',
 ]
-const VIEWABLE = new Set(['Dashboard', 'Logs'])
+const VIEWABLE = new Set(['Dashboard', 'Logs', 'Apps'])
 
 export class Sessions {
   #config: HubConfig

hub/upstream.ts

@@ -41,14 +41,14 @@ export class Upstream {
   }
 
   /** The `response` of the server's envelope, or an `UpstreamError`. */
-  async call(server: HubServer, path: string, params: Params = {}): Promise<unknown> {
+  async call(server: HubServer, path: string, params: Params = {}, token: string = server.token): Promise<unknown> {
     const sorted = Object.entries(params).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
     const key = `${server.id}\n${path}\n${new URLSearchParams(sorted).toString()}`
     const hit = this.#cache.get(key)
     if (hit !== undefined && hit.until > this.#now()) return hit.response
 
     const query = new URLSearchParams(sorted)
-    query.set('token', server.token)
+    query.set('token', token)
     const controller = new AbortController()
     const timer = setTimeout(() => controller.abort(), this.#config.timeoutMs)
 
@@ -60,7 +60,7 @@ export class Upstream {
       })
       envelope = (await res.json()) as typeof envelope
     } catch (e) {
-      throw new UpstreamError('unreachable', this.#describe(e, server))
+      throw new UpstreamError('unreachable', this.#describe(e, [server.token, token]))
     } finally {
       clearTimeout(timer)
     }
@@ -88,10 +88,10 @@ export class Upstream {
   }
 
   /* A short reason without anything that could carry the token. */
-  #describe(e: unknown, server: HubServer): string {
+  #describe(e: unknown, secrets: string[]): string {
     if (e instanceof Error && e.name === 'AbortError') return `timed out after ${this.#config.timeoutMs} ms`
     const cause = e instanceof Error ? (e.cause as { code?: unknown } | undefined) : undefined
     const text = typeof cause?.code === 'string' ? cause.code : e instanceof Error ? e.message : 'request failed'
-    return text.split(server.token).join('***')
+    return secrets.reduce((out, secret) => (secret === '' ? out : out.split(secret).join('***')), text)
   }
 }

src/api/apps.ts

@@ -187,3 +187,35 @@ export function updateApp(
   const { path, form } = buildUpload('apps/update', name, file)
   return apiRequest(path, { token, form })
 }
+
+/*
+The hub's matrix of apps over servers (`hub/apps/matrix`, hub/apps-matrix.ts). Only on a
+hub. A config is never in it: just a hash per server and the comparison.
+*/
+export interface MatrixCell {
+  installed: boolean
+  version?: string
+  updateAvailable?: boolean
+  updateVersion?: string
+  configStatus: 'none' | 'read' | 'unreadable'
+  configHash?: string
+  configError?: string
+}
+
+export interface MatrixApp {
+  name: string
+  description: string
+  servers: Record<string, MatrixCell>
+  configs: 'same' | 'different' | 'single' | 'unknown'
+  diffCount?: number
+  versions: 'same' | 'different' | 'single'
+}
+
+export interface AppsMatrixData {
+  servers: { id: string; ok: boolean; error?: string }[]
+  apps: MatrixApp[]
+}
+
+export function getAppsMatrix(token: string | null): Promise<ApiOutcome<{ response: AppsMatrixData }>> {
+  return apiRequest('hub/apps/matrix', { token })
+}

src/app/Shell.tsx

@@ -649,7 +649,7 @@ export function Shell({
         ) : current?.id === 'about' ? (
           <About info={info} />
         ) : current?.id === 'apps' ? (
-          <Apps token={session.token} />
+          <Apps token={session.token} nodes={session.info?.clusterNodes ?? []} clusterInitialised={session.info?.clusterInitialized === true} />
         ) : current?.id === 'cache' ? (
           <Cache token={session.token} nodes={session.info?.clusterNodes ?? []} clusterInitialised={session.info?.clusterInitialized === true} />
         ) : current?.id === 'blocking' ? (

src/screens/apps/AppCard.tsx

@@ -21,6 +21,7 @@ export function AppCard({
   onUpdate,
   onStoreUpdate,
   onUninstall,
+  readOnly = false,
 }: {
   app: InstalledApp
   busy: boolean
@@ -28,6 +29,8 @@ export function AppCard({
   onUpdate: () => void
   onStoreUpdate: () => void
   onUninstall: () => void
+  /** Behind a hub nothing can be changed yet: no action buttons. */
+  readOnly?: boolean
 }) {
   const hasUpdate = app.updateAvailable === true && Boolean(app.updateUrl)
 
@@ -54,6 +57,9 @@ export function AppCard({
         <Tag tone={hasUpdate ? 'warn' : 'ok'}>{hasUpdate ? 'Update available' : 'Up to date'}</Tag>
       </div>
 
+      {readOnly ? (
+        <div />
+      ) : (
       <div className={styles.rowActs}>
         <Button disabled={busy} onClick={onConfig}>
           Config
@@ -94,6 +100,7 @@ export function AppCard({
           )}
         </Menu>
       </div>
+      )}
 
       {app.dnsApps.length > 0 && (
         <Details className={styles.rowDetails} summary="More Details">

src/screens/apps/Apps.tsx

@@ -1,5 +1,5 @@
 import { KpiStrip } from '../../ui/KpiStrip'
-import { useCallback, useEffect, useState } from 'react'
+import { useCallback, useContext, useEffect, useState } from 'react'
 import {
   downloadAndUpdate,
   getAppConfig,
@@ -25,6 +25,9 @@ import styles from './Apps.module.css'
 import { noticeFromFailure } from '../../lib/notice'
 import { Notifier } from '../../ui/Notifier'
 import { Confirm } from '../../ui/Confirm'
+import { InChrome } from '../../app/ChromeSlot'
+import { AGGREGATE, ClusterNodeSelect, HubMode, type ClusterNode } from '../../ui/ClusterNodeSelect'
+import { AppsMatrix } from './AppsMatrix'
 
 /*
 A replica of the Apps tab (apps.js + index.html:807-835).
@@ -56,7 +59,41 @@ export function error(outcome: { kind: string; message?: string }): AlertState {
   return noticeFromFailure(outcome)
 }
 
-export function Apps({ token }: { token: string | null }) {
+export function Apps({
+  token,
+  nodes = [],
+  clusterInitialised = false,
+}: {
+  token: string | null
+  /** Only used behind a hub: its servers. */
+  nodes?: ClusterNode[]
+  clusterInitialised?: boolean
+}) {
+  /*
+  Behind a hub (`info.hub`) this screen is read-only and has a server selector: "All
+  servers" shows the matrix of apps over servers, one server shows that server's list.
+  Anywhere else none of this exists.
+  */
+  const hub = useContext(HubMode)
+  const [node, setNode] = useState<string>(() => {
+    try {
+      return localStorage.getItem('hubAppsNode') || AGGREGATE
+    } catch {
+      return AGGREGATE
+    }
+  })
+  const chosen = node === AGGREGATE || nodes.some((n) => n.name === node) ? node : AGGREGATE
+  const matrixMode = hub && chosen === AGGREGATE
+  const requestNode = hub && !matrixMode ? chosen : ''
+  useEffect(() => {
+    if (!hub) return
+    try {
+      localStorage.setItem('hubAppsNode', node)
+    } catch {
+      // Not remembered; the choice still works.
+    }
+  }, [hub, node])
+
   const [apps, setApps] = useState<InstalledApp[] | null>(null)
   const [alert, setAlert] = useState<AlertState | null>(null)
   const [modal, setModal] = useState<Modal | null>(null)
@@ -67,7 +104,7 @@ export function Apps({ token }: { token: string | null }) {
   const [pendingUninstall, setPendingUninstall] = useState<string | null>(null)
 
   const reload = useCallback(async () => {
-    const outcome = await listApps(token)
+    const outcome = await listApps(token, requestNode)
     if (outcome.kind === 'ok') {
       setApps(outcome.data.response.apps)
       setFailure(false)
@@ -87,11 +124,11 @@ export function Apps({ token }: { token: string | null }) {
     setApps([])
     setFailure(true)
     setAlert(error(outcome))
-  }, [token])
+  }, [token, requestNode])
 
   useEffect(() => {
-    void reload()
-  }, [reload])
+    if (!matrixMode) void reload()
+  }, [reload, matrixMode])
 
   /*
   apps.js:425-449. The confirmation and the alert are upstream literals.
@@ -169,13 +206,30 @@ export function Apps({ token }: { token: string | null }) {
   */
   const { rows: sortedApps, toggle } = useSort({ name: (a: InstalledApp) => a.name }, apps ?? [])
 
+  const selector = hub ? (
+    <InChrome>
+      <ClusterNodeSelect nodes={nodes} initialised={clusterInitialised} aggregate value={chosen} onChange={setNode} />
+    </InChrome>
+  ) : null
+
+  if (matrixMode) {
+    return (
+      <>
+        {selector}
+        <SectionHeader title="Apps" subtitle="DNS apps on every server" />
+        <AppsMatrix token={token} />
+      </>
+    )
+  }
+
   return (
     <>
+      {selector}
       <Notifier notice={alert} onClose={() => setAlert(null)} />
 
       <SectionHeader
         title="Apps"
-        subtitle="DNS apps installed on this server"
+        subtitle={hub ? `DNS apps installed on ${chosen}` : 'DNS apps installed on this server'}
         labels={
           withUpdate > 0 ? (
             <Tag tone="warn">
@@ -184,12 +238,14 @@ export function Apps({ token }: { token: string | null }) {
           ) : undefined
         }
         actions={
-          <>
-            <Button variant="primary" onClick={() => setTab('store')}>
-              App Store
-            </Button>
-            <Button onClick={() => setModal({ kind: 'install' })}>Install from file</Button>
-          </>
+          hub ? undefined : (
+            <>
+              <Button variant="primary" onClick={() => setTab('store')}>
+                App Store
+              </Button>
+              <Button onClick={() => setModal({ kind: 'install' })}>Install from file</Button>
+            </>
+          )
         }
       />
 
@@ -204,17 +260,19 @@ export function Apps({ token }: { token: string | null }) {
         />
       )}
 
-      <PageTabs
-        label="Apps"
-        tabs={[
-          { id: 'installed', label: 'Installed', count: apps?.length },
-          { id: 'store', label: 'App Store' },
-        ]}
-        active={tab}
-        onChoose={setTab}
-      />
+      {!hub && (
+        <PageTabs
+          label="Apps"
+          tabs={[
+            { id: 'installed', label: 'Installed', count: apps?.length },
+            { id: 'store', label: 'App Store' },
+          ]}
+          active={tab}
+          onChoose={setTab}
+        />
+      )}
 
-      {tab === 'store' ? (
+      {tab === 'store' && !hub ? (
         <StoreApps token={token} onChanged={() => void reload()} />
       ) : apps === null ? (
         <Loading />
@@ -227,13 +285,16 @@ export function Apps({ token }: { token: string | null }) {
         <Empty
           title="No apps installed"
           actions={
-            <Button variant="primary" onClick={() => setTab('store')}>
-              Open App Store
-            </Button>
+            hub ? undefined : (
+              <Button variant="primary" onClick={() => setTab('store')}>
+                Open App Store
+              </Button>
+            )
           }
         >
           DNS Apps add behaviour to the server (query logging, advanced blocking, split horizon)
-          without touching the base configuration. Open the store to see what is available.
+          without touching the base configuration.
+          {hub ? ' Install apps in the console of this server.' : ' Open the store to see what is available.'}
         </Empty>
       ) : (
         <Panel title="Installed" description="Version, DNS apps and status">
@@ -258,6 +319,7 @@ export function Apps({ token }: { token: string | null }) {
                 key={app.name}
                 app={app}
                 busy={busy === app.name}
+                readOnly={hub}
                 onConfig={() => void openConfig(app.name)}
                 onUpdate={() => setModal({ kind: 'update', name: app.name })}
                 onStoreUpdate={() => void updateFromStore(app)}

src/screens/apps/AppsMatrix.test.tsx

@@ -0,0 +1,243 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { render, screen, within } from '@testing-library/react'
+import userEvent from '@testing-library/user-event'
+import { AppsMatrix } from './AppsMatrix'
+import { Apps } from './Apps'
+import { SlotProvider } from '../../app/ChromeSlot'
+import { HubMode } from '../../ui/ClusterNodeSelect'
+import * as api from '../../api/apps'
+import type { AppsMatrixData } from '../../api/apps'
+
+afterEach(() => {
+  vi.restoreAllMocks()
+  localStorage.clear()
+})
+
+const DATA: AppsMatrixData = {
+  servers: [
+    { id: 'mdns', ok: true },
+    { id: 'ndns', ok: true },
+  ],
+  apps: [
+    {
+      name: 'Advanced Blocking',
+      description: 'Group based blocking',
+      configs: 'different',
+      diffCount: 2,
+      versions: 'same',
+      servers: {
+        mdns: { installed: true, version: '10.1', configStatus: 'read', configHash: 'aaaa' },
+        ndns: { installed: true, version: '10.1', configStatus: 'read', configHash: 'bbbb' },
+      },
+    },
+    {
+      name: 'Query Logs (Sqlite)',
+      description: 'Logs queries',
+      configs: 'same',
+      versions: 'different',
+      servers: {
+        mdns: { installed: true, version: '7.0', configStatus: 'read', configHash: 'cccc' },
+        ndns: { installed: true, version: '7.1', updateAvailable: true, updateVersion: '7.2', configStatus: 'read', configHash: 'cccc' },
+      },
+    },
+    {
+      name: 'Split Horizon',
+      description: '',
+      configs: 'single',
+      versions: 'single',
+      servers: {
+        mdns: { installed: true, version: '9.0', configStatus: 'read', configHash: 'dddd' },
+        ndns: { installed: false, configStatus: 'none' },
+      },
+    },
+    {
+      name: 'Wild IP',
+      description: '',
+      configs: 'unknown',
+      versions: 'same',
+      servers: {
+        mdns: { installed: true, version: '5.1', configStatus: 'unreadable', configError: 'Access was denied' },
+        ndns: { installed: true, version: '5.1', configStatus: 'read', configHash: 'eeee' },
+      },
+    },
+  ],
+}
+
+function mockMatrix(data: AppsMatrixData = DATA) {
+  return vi.spyOn(api, 'getAppsMatrix').mockResolvedValue({ kind: 'ok', data: { response: data } } as never)
+}
+
+const row = (name: string) => screen.getByText(name).closest('tr') as HTMLElement
+
+describe('AppsMatrix', () => {
+  it('draws a row per app and a column per server', async () => {
+    mockMatrix()
+    render(<AppsMatrix token="t" />)
+    expect(await screen.findByText('Advanced Blocking')).toBeInTheDocument()
+    expect(screen.getByRole('columnheader', { name: 'mdns' })).toBeInTheDocument()
+    expect(screen.getByRole('columnheader', { name: 'ndns' })).toBeInTheDocument()
+    expect(within(row('Split Horizon')).getByText('v9.0')).toBeInTheDocument()
+    expect(within(row('Split Horizon')).getByText('Not installed')).toBeInTheDocument()
+    expect(within(row('Query Logs (Sqlite)')).getByText('v7.1')).toBeInTheDocument()
+    expect(within(row('Query Logs (Sqlite)')).getByText('Update v7.2')).toBeInTheDocument()
+  })
+
+  it('shows the config state of each app', async () => {
+    mockMatrix()
+    render(<AppsMatrix token="t" />)
+    await screen.findByText('Advanced Blocking')
+    expect(within(row('Advanced Blocking')).getByText('Different · 2')).toBeInTheDocument()
+    expect(within(row('Query Logs (Sqlite)')).getByText('Same')).toBeInTheDocument()
+    expect(within(row('Split Horizon')).getByText('Single server')).toBeInTheDocument()
+    expect(within(row('Wild IP')).getByText('Unknown')).toBeInTheDocument()
+  })
+
+  it('explains an unknown config in a tooltip', async () => {
+    mockMatrix()
+    render(<AppsMatrix token="t" />)
+    await screen.findByText('Advanced Blocking')
+    await userEvent.hover(within(row('Wild IP')).getByText('Unknown'))
+    expect(await screen.findByRole('tooltip')).toHaveTextContent('mdns: Access was denied')
+  })
+
+  it('filters with counts', async () => {
+    mockMatrix()
+    render(<AppsMatrix token="t" />)
+    await screen.findByText('Advanced Blocking')
+    const filters = screen.getByRole('group', { name: 'Filter apps' })
+    expect(within(filters).getByRole('button', { name: 'All · 4' })).toBeInTheDocument()
+    expect(within(filters).getByRole('button', { name: 'Version differs · 1' })).toBeInTheDocument()
+    expect(within(filters).getByRole('button', { name: 'Config differs · 1' })).toBeInTheDocument()
+    expect(within(filters).getByRole('button', { name: 'Update available · 1' })).toBeInTheDocument()
+
+    await userEvent.click(within(filters).getByRole('button', { name: 'Config differs · 1' }))
+    expect(screen.getByText('Advanced Blocking')).toBeInTheDocument()
+    expect(screen.queryByText('Split Horizon')).not.toBeInTheDocument()
+
+    await userEvent.click(within(filters).getByRole('button', { name: 'Update available · 1' }))
+    expect(screen.getByText('Query Logs (Sqlite)')).toBeInTheDocument()
+    expect(screen.queryByText('Advanced Blocking')).not.toBeInTheDocument()
+  })
+
+  it('marks a server that did not answer as unknown', async () => {
+    mockMatrix({
+      servers: [{ id: 'mdns', ok: true }, { id: 'ndns', ok: false, error: 'timed out' }],
+      apps: [
+        {
+          name: 'Split Horizon',
+          description: '',
+          configs: 'unknown',
+          versions: 'single',
+          servers: {
+            mdns: { installed: true, version: '9.0', configStatus: 'read' },
+            ndns: { installed: false, configStatus: 'unreadable', configError: 'ndns did not answer' },
+          },
+        },
+      ],
+    })
+    render(<AppsMatrix token="t" />)
+    await screen.findByText('Split Horizon')
+    expect(within(row('Split Horizon')).getByText('v9.0')).toBeInTheDocument()
+    expect(within(row('Split Horizon')).getAllByText('Unknown')).toHaveLength(2)
+  })
+
+  it('says so when no filter matches, with a way back', async () => {
+    mockMatrix({ ...DATA, apps: [DATA.apps[2]] })
+    render(<AppsMatrix token="t" />)
+    await screen.findByText('Split Horizon')
+    await userEvent.click(screen.getByRole('button', { name: 'Config differs · 0' }))
+    expect(screen.getByText('No apps match')).toBeInTheDocument()
+    await userEvent.click(screen.getByRole('button', { name: 'Show all' }))
+    expect(screen.getByText('Split Horizon')).toBeInTheDocument()
+  })
+
+  it('has an empty state with an action', async () => {
+    const spy = mockMatrix({ servers: DATA.servers, apps: [] })
+    render(<AppsMatrix token="t" />)
+    expect(await screen.findByText('No apps installed')).toBeInTheDocument()
+    await userEvent.click(screen.getByRole('button', { name: 'Refresh' }))
+    expect(spy).toHaveBeenCalledTimes(2)
+  })
+
+  it('shows loading and then an error with Retry', async () => {
+    const spy = vi.spyOn(api, 'getAppsMatrix').mockResolvedValueOnce({ kind: 'error', message: 'No server answered.' } as never)
+    render(<AppsMatrix token="t" />)
+    expect(screen.getByRole('status')).toHaveTextContent('Loading')
+    expect(await screen.findByText(/Could not load the apps/)).toHaveTextContent('No server answered.')
+    spy.mockResolvedValue({ kind: 'ok', data: { response: DATA } } as never)
+    await userEvent.click(screen.getByRole('button', { name: 'Retry' }))
+    expect(await screen.findByText('Advanced Blocking')).toBeInTheDocument()
+  })
+})
+
+describe('Apps behind a hub', () => {
+  const INSTALLED = {
+    name: 'Split Horizon',
+    description: 'Different answers',
+    version: '9.0',
+    dnsApps: [],
+  }
+  const NODES = [{ name: 'mdns', type: 'Primary' }, { name: 'ndns', type: 'Secondary' }]
+
+  function mount(hub: boolean) {
+    return render(
+      <HubMode.Provider value={hub}>
+        <SlotProvider>
+          {(ref) => (
+            <>
+              <div ref={ref} />
+              <Apps token="t" nodes={NODES} clusterInitialised />
+            </>
+          )}
+        </SlotProvider>
+      </HubMode.Provider>,
+    )
+  }
+
+  it('shows the selector and the matrix for All servers', async () => {
+    const matrix = mockMatrix()
+    const list = vi.spyOn(api, 'listApps')
+    mount(true)
+    expect(await screen.findByText('Advanced Blocking')).toBeInTheDocument()
+    expect(await screen.findByRole('combobox', { name: 'Server' })).toHaveTextContent('All servers')
+    expect(matrix).toHaveBeenCalled()
+    expect(list).not.toHaveBeenCalled()
+    expect(screen.queryByRole('button', { name: 'App Store' })).not.toBeInTheDocument()
+  })
+
+  it('shows one server read-only and remembers the choice', async () => {
+    mockMatrix()
+    const list = vi.spyOn(api, 'listApps').mockResolvedValue({ kind: 'ok', data: { response: { apps: [INSTALLED] } } } as never)
+    mount(true)
+    await userEvent.click(await screen.findByRole('combobox', { name: 'Server' }))
+    await userEvent.click(screen.getByRole('option', { name: 'ndns' }))
+    expect(await screen.findByText('Split Horizon')).toBeInTheDocument()
+    expect(list).toHaveBeenCalledWith('t', 'ndns')
+    expect(localStorage.getItem('hubAppsNode')).toBe('ndns')
+    for (const name of ['Config', 'Update', 'Store Update', 'App Store', 'Install from file']) {
+      expect(screen.queryByRole('button', { name })).not.toBeInTheDocument()
+    }
+    expect(screen.queryByRole('button', { name: /Actions: Split Horizon/ })).not.toBeInTheDocument()
+    expect(screen.queryByRole('tab', { name: /App Store/ })).not.toBeInTheDocument()
+  })
+
+  it('starts on the remembered server', async () => {
+    localStorage.setItem('hubAppsNode', 'mdns')
+    const list = vi.spyOn(api, 'listApps').mockResolvedValue({ kind: 'ok', data: { response: { apps: [INSTALLED] } } } as never)
+    mount(true)
+    expect(await screen.findByText('Split Horizon')).toBeInTheDocument()
+    expect(list).toHaveBeenCalledWith('t', 'mdns')
+  })
+
+  it('is unchanged outside a hub', async () => {
+    const matrix = mockMatrix()
+    const list = vi.spyOn(api, 'listApps').mockResolvedValue({ kind: 'ok', data: { response: { apps: [INSTALLED] } } } as never)
+    mount(false)
+    expect(await screen.findByText('Split Horizon')).toBeInTheDocument()
+    expect(list).toHaveBeenCalledWith('t', '')
+    expect(matrix).not.toHaveBeenCalled()
+    expect(screen.queryByRole('combobox', { name: 'Server' })).not.toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'App Store' })).toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'Config' })).toBeInTheDocument()
+  })
+})

src/screens/apps/AppsMatrix.tsx

@@ -0,0 +1,193 @@
+import { useCallback, useEffect, useMemo, useState } from 'react'
+import { getAppsMatrix, type AppsMatrixData, type MatrixApp } from '../../api/apps'
+import { Badge } from '@/components/reui/badge'
+import { Button } from '../../ui/Button'
+import { DataTable, type Col } from '../../ui/DataTable'
+import { Empty, Failure, Loading } from '../../ui/Empty'
+import { Panel } from '../../ui/Panel'
+import { Segmented } from '../../ui/Segmented'
+import { Tag } from '../../ui/Tag'
+import { Tooltip } from '../../ui/Tooltip'
+import { head } from '../../ui/Table'
+
+/*
+Behind a hub: one row per app, one column per server, and whether the configs agree.
+Read-only. The numbers come from `hub/apps/matrix`, which never sends a config, only its
+hash and the comparison, so nothing here can show a secret.
+*/
+
+type Filter = 'all' | 'version' | 'config' | 'update'
+
+const hasUpdate = (app: MatrixApp): boolean => Object.values(app.servers).some((c) => c.installed && c.updateAvailable === true)
+
+const FILTERS: Record<Filter, (app: MatrixApp) => boolean> = {
+  all: () => true,
+  version: (app) => app.versions === 'different',
+  config: (app) => app.configs === 'different',
+  update: hasUpdate,
+}
+
+/* Why a config's state is not known, from the cells and the servers that did not answer. */
+function unknownReason(app: MatrixApp, data: AppsMatrixData): string {
+  const reasons = new Set<string>()
+  for (const [id, cell] of Object.entries(app.servers)) {
+    if (cell.installed && cell.configStatus === 'unreadable') reasons.add(`${id}: ${cell.configError ?? 'config not readable'}`)
+  }
+  for (const s of data.servers) if (!s.ok) reasons.add(`${s.id}: ${s.error ?? 'did not answer'}`)
+  return [...reasons].join('; ') || 'The configs could not be compared.'
+}
+
+function ConfigState({ app, data }: { app: MatrixApp; data: AppsMatrixData }) {
+  switch (app.configs) {
+    case 'same':
+      return <Tag tone="ok">Same</Tag>
+    case 'different':
+      return <Tag tone="warn">{app.diffCount === undefined ? 'Different' : `Different · ${app.diffCount}`}</Tag>
+    case 'single':
+      return <Badge variant="outline">Single server</Badge>
+    default:
+      return (
+        <Tooltip text={unknownReason(app, data)} placement="top">
+          <Badge variant="outline" tabIndex={0}>
+            Unknown
+          </Badge>
+        </Tooltip>
+      )
+  }
+}
+
+function ServerCell({ app, id, data }: { app: MatrixApp; id: string; data: AppsMatrixData }) {
+  const cell = app.servers[id]
+  if (data.servers.find((s) => s.id === id)?.ok === false) return <span className="text-sm text-muted-foreground">Unknown</span>
+  if (cell === undefined || !cell.installed) return <span className="text-sm text-muted-foreground">Not installed</span>
+  return (
+    <span className="flex flex-wrap items-center gap-2 text-sm">
+      <span>{cell.version === undefined ? 'Installed' : `v${cell.version}`}</span>
+      {cell.updateAvailable === true && (
+        <Tag tone="warn">{cell.updateVersion === undefined ? 'Update' : `Update v${cell.updateVersion}`}</Tag>
+      )}
+    </span>
+  )
+}
+
+export function AppsMatrix({ token }: { token: string | null }) {
+  const [data, setData] = useState<AppsMatrixData | null>(null)
+  const [failure, setFailure] = useState<string | null>(null)
+  const [filter, setFilter] = useState<Filter>('all')
+
+  const load = useCallback(async () => {
+    const outcome = await getAppsMatrix(token)
+    if (outcome.kind === 'ok') {
+      setData(outcome.data.response)
+      setFailure(null)
+      return
+    }
+    setFailure(outcome.kind === 'error' ? outcome.message : 'The session ended.')
+  }, [token])
+
+  useEffect(() => {
+    void load()
+  }, [load])
+
+  const counts = useMemo(() => {
+    const apps = data?.apps ?? []
+    return (Object.keys(FILTERS) as Filter[]).reduce(
+      (acc, f) => ({ ...acc, [f]: apps.filter(FILTERS[f]).length }),
+      {} as Record<Filter, number>,
+    )
+  }, [data])
+
+  const retry = (
+    <Button
+      variant="primary"
+      onClick={() => {
+        setFailure(null)
+        setData(null)
+        void load()
+      }}
+    >
+      Retry
+    </Button>
+  )
+
+  if (failure !== null && data === null) {
+    return (
+      <Failure>
+        <span className="flex flex-col items-center gap-3">
+          <span>Could not load the apps of the servers. {failure}</span>
+          {retry}
+        </span>
+      </Failure>
+    )
+  }
+  if (data === null) return <Loading>Loading the apps of every server…</Loading>
+
+  if (data.apps.length === 0) {
+    return (
+      <Empty
+        title="No apps installed"
+        actions={
+          <Button variant="primary" onClick={() => void load()}>
+            Refresh
+          </Button>
+        }
+      >
+        None of the servers that answered has a DNS app installed. Install apps in a server's own console.
+      </Empty>
+    )
+  }
+
+  const rows = data.apps.filter(FILTERS[filter])
+  const columns: Col<MatrixApp>[] = [
+    {
+      id: 'name',
+      ...head({ children: 'App' }),
+      cell: (a) => (
+        <span className="flex flex-col gap-1">
+          <span className="text-sm font-medium">{a.name}</span>
+          {a.description !== '' && <span className="text-xs text-muted-foreground">{a.description}</span>}
+        </span>
+      ),
+    },
+    ...data.servers.map(
+      (s): Col<MatrixApp> => ({
+        id: `server-${s.id}`,
+        ...head({ children: s.id }),
+        cell: (a) => <ServerCell app={a} id={s.id} data={data} />,
+      }),
+    ),
+    { id: 'config', ...head({ children: 'Config' }), cell: (a) => <ConfigState app={a} data={data} /> },
+  ]
+
+  return (
+    <div className="flex flex-col gap-4">
+      <Segmented
+        label="Filter apps"
+        active={filter}
+        onChoose={setFilter}
+        options={[
+          { id: 'all', label: `All · ${counts.all}` },
+          { id: 'version', label: `Version differs · ${counts.version}` },
+          { id: 'config', label: `Config differs · ${counts.config}` },
+          { id: 'update', label: `Update available · ${counts.update}` },
+        ]}
+      />
+      <Panel title="Apps by server" description="Installed version and whether the configs match">
+        {rows.length === 0 ? (
+          <Empty
+            title="No apps match"
+            actions={
+              <Button variant="primary" onClick={() => setFilter('all')}>
+                Show all
+              </Button>
+            }
+          >
+            No app fits this filter.
+          </Empty>
+        ) : (
+          <DataTable data={rows} columns={columns} getRowId={(a) => a.name} />
+        )}
+      </Panel>
+    </div>
+  )
+}