technitium-console/hub/app-lists.ts

/*
How many domains the block lists of a server's Advanced Blocking app hold (stage 9). The app has
no API that says it, so the hub reads the app's config from the server (with the READ token, like
any `apps/config/get`), downloads each http(s) block list and counts the distinct domains.

Counted: a hosts line (`0.0.0.0 example.com`, every name after the address), a plain domain
(`example.com`, `*.example.com`) and an Adblock Plus domain rule (`||example.com^`). Not counted:
blank lines, comments (`#`, `!`, `[`), Adblock exceptions (`@@`) and regular expressions. The
console has the same rules for the no-hub case (`src/lib/list-count.ts`); the two are kept in step
by hand and tested on the same sample.

The hub fetches a URL an administrator typed into a config, so it is careful where it goes:
- only `http:` and `https:`, no credentials in the URL;
- the host must resolve to public addresses only (no loopback, private, link-local, carrier-grade
  NAT, multicast or unspecified ones, IPv4 or IPv6). Redirects are followed by hand, up to three,
  and checked at every step. `HUB_LISTS_ALLOW_PRIVATE=1` lifts this for lists kept on the LAN.
  The check resolves the name and the fetch resolves it again, so a name that changes between the
  two is not caught; the hub is not a place an attacker can write DNS for;
- a timeout and a size cap per list, a cap on how many lists, and four downloads at a time.

Results are cached per URL for several hours (a failure for minutes), so opening the Overview
does not download a hundred megabytes again.
*/
import { lookup as dnsLookup } from 'node:dns/promises'
import { isIP } from 'node:net'
import type { HubServer } from './config.ts'
import type { Upstream } from './upstream.ts'

export interface ListCount {
  url: string
  count: number | null
  error?: string
}

export interface ListCounts {
  lists: ListCount[]
  total: number
}

export interface AppListsOptions {
  fetch?: typeof fetch
  /** Resolves a host to its addresses. */
  lookup?: (host: string) => Promise<string[]>
  now?: () => number
  allowPrivate?: boolean
  /** How long a counted list is remembered (default 6 hours). */
  ttlMs?: number
  /** How long a failure is remembered (default 5 minutes). */
  failureTtlMs?: number
  timeoutMs?: number
  maxBytes?: number
  maxLists?: number
}

const HOURS = 3_600_000
const MAX_CACHED = 500
const MAX_REDIRECTS = 3
const CONCURRENCY = 4

/* --- counting --- */

const ADDRESS = /^(?:\d{1,3}(?:\.\d{1,3}){3}|[0-9a-f:]*:[0-9a-f:]*)$/i
const NAME = /^[a-z0-9_]([a-z0-9_.-]*[a-z0-9_])?$/i
const OWN = new Set(['localhost', 'localhost.localdomain', 'local', 'broadcasthost', '0.0.0.0'])

function accept(raw: string, into: Set<string>): void {
  const name = raw.replace(/^\*\./, '').replace(/\.$/, '').toLowerCase()
  if (name === '' || OWN.has(name) || name.startsWith('ip6-') || !NAME.test(name)) return
  into.add(name)
}

function countLine(line: string, into: Set<string>): void {
  const t = line.trim()
  if (t === '' || t.startsWith('#') || t.startsWith('!') || t.startsWith('[') || t.startsWith('@@') || t.startsWith('/')) return
  if (t.startsWith('||')) {
    const end = t.search(/[\^$/]/)
    accept(end < 0 ? t.slice(2) : t.slice(2, end), into)
    return
  }
  const tokens = t.split('#')[0].trim().split(/\s+/)
  if (tokens.length > 1 && ADDRESS.test(tokens[0])) for (const name of tokens.slice(1)) accept(name, into)
  else if (tokens.length === 1) accept(tokens[0], into)
}

/* Counts as text arrives in pieces, so a big list is never held whole. */
export class DomainCounter {
  #domains = new Set<string>()
  #rest = ''

  push(chunk: string): void {
    const lines = (this.#rest + chunk).split(/\r?\n/)
    this.#rest = lines.pop() ?? ''
    for (const line of lines) countLine(line, this.#domains)
  }

  end(): number {
    countLine(this.#rest, this.#domains)
    this.#rest = ''
    return this.#domains.size
  }
}

export function countDomains(text: string): number {
  const c = new DomainCounter()
  c.push(text)
  return c.end()
}

/* --- where a fetch may go --- */

function ipv4Allowed(a: number, b: number): boolean {
  if (a === 0 || a === 10 || a === 127 || a >= 224) return false
  if (a === 100 && b >= 64 && b <= 127) return false
  if (a === 169 && b === 254) return false
  if (a === 172 && b >= 16 && b <= 31) return false
  if (a === 192 && b === 168) return false
  if (a === 192 && b === 0) return false
  if (a === 198 && (b === 18 || b === 19)) return false
  return true
}

/** Whether an address is one a list may be fetched from: public unicast only. */
export function publicAddress(address: string): boolean {
  const kind = isIP(address)
  if (kind === 4) {
    const [a, b] = address.split('.').map(Number)
    return ipv4Allowed(a, b)
  }
  if (kind === 6) {
    const a = address.toLowerCase()
    const mapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/.exec(a)
    if (mapped !== null) return publicAddress(mapped[1])
    if (a === '::' || a === '::1') return false
    const first = parseInt(a.split(':')[0] || '0', 16)
    if ((first & 0xfe00) === 0xfc00) return false // fc00::/7 unique local
    if ((first & 0xffc0) === 0xfe80) return false // fe80::/10 link local
    if ((first & 0xff00) === 0xff00) return false // multicast
    return true
  }
  return false
}

async function defaultLookup(host: string): Promise<string[]> {
  return (await dnsLookup(host, { all: true })).map((r) => r.address)
}

class Refused extends Error {}

/* --- the service --- */

interface Cached {
  until: number
  result: ListCount
}

export class AppLists {
  #upstream: Upstream
  #fetch: typeof fetch
  #lookup: (host: string) => Promise<string[]>
  #now: () => number
  #allowPrivate: boolean
  #ttl: number
  #failureTtl: number
  #timeout: number
  #maxBytes: number
  #maxLists: number
  #cache = new Map<string, Cached>()
  #inflight = new Map<string, Promise<ListCount>>()

  constructor(upstream: Upstream, options: AppListsOptions = {}) {
    this.#upstream = upstream
    this.#fetch = options.fetch ?? fetch
    this.#lookup = options.lookup ?? defaultLookup
    this.#now = options.now ?? Date.now
    this.#allowPrivate = options.allowPrivate ?? false
    this.#ttl = options.ttlMs ?? 6 * HOURS
    this.#failureTtl = options.failureTtlMs ?? 5 * 60_000
    this.#timeout = options.timeoutMs ?? 20_000
    this.#maxBytes = options.maxBytes ?? 64 * 1024 * 1024
    this.#maxLists = options.maxLists ?? 100
  }

  /** The distinct block list URLs (plain and Adblock) of the app's config on one server. */
  async urlsOf(server: HubServer, appName: string): Promise<string[]> {
    const response = (await this.#upstream.call(server, 'apps/config/get', { name: appName })) as { config?: string | null } | null
    const text = response?.config
    if (typeof text !== 'string' || text.trim() === '') return []
    let root: unknown
    try {
      root = JSON.parse(text)
    } catch {
      return []
    }
    const groups = root !== null && typeof root === 'object' && Array.isArray((root as { groups?: unknown }).groups) ? ((root as { groups: unknown[] }).groups) : []
    const urls = new Set<string>()
    for (const g of groups) {
      if (g === null || typeof g !== 'object') continue
      for (const field of ['blockListUrls', 'adblockListUrls']) {
        const list = (g as Record<string, unknown>)[field]
        if (!Array.isArray(list)) continue
        for (const e of list) {
          const url = typeof e === 'string' ? e : e !== null && typeof e === 'object' && typeof (e as { url?: unknown }).url === 'string' ? (e as { url: string }).url : ''
          if (/^https?:\/\//i.test(url.trim())) urls.add(url.trim())
        }
      }
    }
    return [...urls]
  }

  async count(server: HubServer, appName: string): Promise<ListCounts> {
    const urls = (await this.urlsOf(server, appName)).slice(0, this.#maxLists)
    const lists: ListCount[] = new Array(urls.length)
    let next = 0
    const worker = async () => {
      while (next < urls.length) {
        const i = next++
        lists[i] = await this.countUrl(urls[i])
      }
    }
    await Promise.all(Array.from({ length: Math.min(CONCURRENCY, urls.length) }, worker))
    return { lists, total: lists.reduce((n, l) => n + (l.count ?? 0), 0) }
  }

  countUrl(url: string): Promise<ListCount> {
    const hit = this.#cache.get(url)
    if (hit !== undefined && hit.until > this.#now()) return Promise.resolve(hit.result)
    const running = this.#inflight.get(url)
    if (running !== undefined) return running
    const job = this.#download(url).then((result) => {
      if (this.#cache.size >= MAX_CACHED) this.#prune()
      this.#cache.set(url, { until: this.#now() + (result.count === null ? this.#failureTtl : this.#ttl), result })
      return result
    })
    this.#inflight.set(url, job)
    void job.finally(() => this.#inflight.delete(url))
    return job
  }

  #prune(): void {
    const now = this.#now()
    for (const [key, entry] of this.#cache) if (entry.until <= now) this.#cache.delete(key)
    while (this.#cache.size >= MAX_CACHED) this.#cache.delete(this.#cache.keys().next().value as string)
  }

  async #allowed(url: URL): Promise<void> {
    if (url.protocol !== 'http:' && url.protocol !== 'https:') throw new Refused('only http and https lists are read')
    if (url.username !== '' || url.password !== '') throw new Refused('a list address may not carry credentials')
    if (this.#allowPrivate) return
    const host = url.hostname.replace(/^\[|\]$/g, '')
    const addresses = isIP(host) !== 0 ? [host] : await this.#lookup(host)
    if (addresses.length === 0 || !addresses.every(publicAddress)) throw new Refused('the address of this list is not a public one')
  }

  async #download(start: string): Promise<ListCount> {
    const controller = new AbortController()
    const timer = setTimeout(() => controller.abort(), this.#timeout)
    try {
      let url = new URL(start)
      for (let hop = 0; ; hop++) {
        await this.#allowed(url)
        const res = await this.#fetch(url, { redirect: 'manual', signal: controller.signal, headers: { accept: 'text/plain, */*' } })
        if (res.status >= 300 && res.status < 400 && res.headers.get('location') !== null) {
          void res.body?.cancel()
          if (hop >= MAX_REDIRECTS) throw new Refused('too many redirects')
          url = new URL(res.headers.get('location') as string, url)
          continue
        }
        if (!res.ok) {
          void res.body?.cancel()
          throw new Refused(`the list server answered ${res.status}`)
        }
        const declared = Number(res.headers.get('content-length') ?? 0)
        if (declared > this.#maxBytes) {
          void res.body?.cancel()
          throw new Refused('the list is larger than the limit')
        }
        const counter = new DomainCounter()
        const decoder = new TextDecoder()
        let bytes = 0
        if (res.body !== null) {
          const reader = res.body.getReader()
          for (;;) {
            const { done, value } = await reader.read()
            if (done) break
            bytes += value.byteLength
            if (bytes > this.#maxBytes) {
              void reader.cancel()
              throw new Refused('the list is larger than the limit')
            }
            counter.push(decoder.decode(value, { stream: true }))
          }
          counter.push(decoder.decode())
        }
        return { url: start, count: counter.end() }
      }
    } catch (e) {
      const message =
        e instanceof Refused ? e.message : e instanceof Error && e.name === 'AbortError' ? 'timed out' : 'could not be downloaded'
      return { url: start, count: null, error: message }
    } finally {
      clearTimeout(timer)
    }
  }
}