technitium-console/hub/app-lists.test.ts

// @vitest-environment node
/*
Stage 9: counting the domains in an Advanced Blocking app's block lists. The unit tests inject
the fetch and the DNS lookup; the hub tests run against fake servers and a fake list server.
*/
import http from 'node:http'
import type { AddressInfo } from 'node:net'
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { AppLists, DomainCounter, countDomains, publicAddress } from './app-lists.ts'
import type { HubServer } from './config.ts'
import type { Upstream } from './upstream.ts'
import { fake, startHub, type Fake, type TestHub } from './fake-servers.ts'

const server: HubServer = { id: 'a', url: 'http://a.invalid', token: 't' }

const config = (groups: unknown[]) => JSON.stringify({ enableBlocking: true, groups })

function upstreamWith(text: string | null): Upstream {
  return { call: async () => ({ config: text }) } as unknown as Upstream
}

const HOSTS = '# hosts\n127.0.0.1 localhost\n0.0.0.0 a.example\n0.0.0.0 b.example\n0.0.0.0 a.example\n'

function listFetch(files: Record<string, string>, seen: string[] = []): typeof fetch {
  return (async (input: URL | string) => {
    const url = String(input)
    seen.push(url)
    const body = files[url]
    return body === undefined ? new Response('nope', { status: 404 }) : new Response(body, { status: 200 })
  }) as typeof fetch
}

const PUBLIC = async () => ['93.184.216.34']

describe('countDomains', () => {
  it('counts hosts, plain and adblock lines once each', () => {
    expect(countDomains(HOSTS)).toBe(2)
    expect(countDomains('x.example\n*.y.example\n! c\n||z.example^$third-party\n@@||ok.example^\n[Adblock Plus]\n')).toBe(3)
  })

  it('counts across chunks that cut a line in two', () => {
    const c = new DomainCounter()
    c.push('0.0.0.0 a.exa')
    c.push('mple\n0.0.0.0 b.example')
    expect(c.end()).toBe(2)
  })
})

describe('publicAddress', () => {
  it('refuses loopback, private, link-local, shared and unspecified addresses', () => {
    for (const a of ['127.0.0.1', '10.1.2.3', '172.16.0.1', '192.168.1.1', '169.254.169.254', '100.64.0.1', '0.0.0.0', '224.0.0.1', '::1', '::', 'fe80::1', 'fd00::1', '::ffff:10.0.0.1']) {
      expect(publicAddress(a), a).toBe(false)
    }
  })

  it('accepts public ones', () => {
    for (const a of ['93.184.216.34', '8.8.8.8', '2606:4700:4700::1111']) expect(publicAddress(a), a).toBe(true)
  })
})

describe('AppLists', () => {
  it('reads the block and adblock URLs of every group, strings or objects, each once, http(s) only', async () => {
    const text = config([
      { name: 'one', blockListUrls: ['https://x.example/a', { url: 'https://x.example/b' }, 'file:///etc/hosts'], adblockListUrls: ['https://x.example/c'], regexBlockListUrls: ['https://x.example/re'] },
      { name: 'two', blockListUrls: ['https://x.example/a'], allowListUrls: ['https://x.example/allow'] },
    ])
    const lists = new AppLists(upstreamWith(text))
    expect(await lists.urlsOf(server, 'Advanced Blocking')).toEqual(['https://x.example/a', 'https://x.example/b', 'https://x.example/c'])
  })

  it('has no URLs for a missing, empty or broken config', async () => {
    expect(await new AppLists(upstreamWith(null)).urlsOf(server, 'x')).toEqual([])
    expect(await new AppLists(upstreamWith('{nope')).urlsOf(server, 'x')).toEqual([])
  })

  it('counts each list and totals them, and caches by URL', async () => {
    const seen: string[] = []
    const files = { 'https://x.example/a': HOSTS, 'https://x.example/b': 'c.example\n' }
    let now = 1_000
    const lists = new AppLists(upstreamWith(config([{ name: 'g', blockListUrls: ['https://x.example/a', 'https://x.example/b'] }])), {
      fetch: listFetch(files, seen), lookup: PUBLIC, now: () => now,
    })
    const first = await lists.count(server, 'Advanced Blocking')
    expect(first.lists).toEqual([{ url: 'https://x.example/a', count: 2 }, { url: 'https://x.example/b', count: 1 }])
    expect(first.total).toBe(3)
    await lists.count(server, 'Advanced Blocking')
    expect(seen).toHaveLength(2)
    now += 7 * 3_600_000
    await lists.count(server, 'Advanced Blocking')
    expect(seen).toHaveLength(4)
  })

  it('reports a list that fails, with a reason, and keeps the others', async () => {
    const lists = new AppLists(upstreamWith(config([{ name: 'g', blockListUrls: ['https://x.example/a', 'https://x.example/gone'] }])), {
      fetch: listFetch({ 'https://x.example/a': HOSTS }), lookup: PUBLIC,
    })
    const out = await lists.count(server, 'x')
    expect(out.lists[0].count).toBe(2)
    expect(out.lists[1]).toEqual({ url: 'https://x.example/gone', count: null, error: 'the list server answered 404' })
    expect(out.total).toBe(2)
  })

  it('never fetches an address that is not public, or a name that resolves to one', async () => {
    const seen: string[] = []
    const lists = new AppLists(
      upstreamWith(config([{ name: 'g', blockListUrls: ['http://127.0.0.1/a', 'http://[::1]/a', 'http://169.254.169.254/latest', 'https://internal.example/a', 'ftp://x.example/a'] }])),
      { fetch: listFetch({}, seen), lookup: async (h) => (h === 'internal.example' ? ['10.0.0.5'] : ['93.184.216.34']) },
    )
    const out = await lists.count(server, 'x')
    expect(seen).toEqual([])
    expect(out.lists.every((l) => l.count === null)).toBe(true)
    expect(out.lists[0].error).toBe('the address of this list is not a public one')
  })

  it('checks every redirect, and stops after three', async () => {
    const seen: string[] = []
    const fetcher = (async (input: URL | string) => {
      const url = String(input)
      seen.push(url)
      if (url === 'https://x.example/start') return new Response(null, { status: 302, headers: { location: 'http://10.0.0.1/inner' } })
      if (url.startsWith('https://x.example/loop')) return new Response(null, { status: 302, headers: { location: 'https://x.example/loop' } })
      return new Response('a.example\n')
    }) as typeof fetch
    const lists = new AppLists(upstreamWith(config([{ name: 'g', blockListUrls: ['https://x.example/start', 'https://x.example/loop'] }])), { fetch: fetcher, lookup: async (h) => (h === 'x.example' ? ['93.184.216.34'] : ['10.0.0.1']) })
    const out = await lists.count(server, 'x')
    expect(seen).not.toContain('http://10.0.0.1/inner')
    expect(out.lists[0].error).toBe('the address of this list is not a public one')
    expect(out.lists[1].error).toBe('too many redirects')
  })

  it('refuses a list over the size cap, declared or streamed', async () => {
    const big = (async (input: URL | string) =>
      String(input).endsWith('/declared')
        ? new Response('a.example\n', { headers: { 'content-length': '5000' } })
        : new Response('a.example\n'.repeat(1000))) as typeof fetch
    const lists = new AppLists(upstreamWith(config([{ name: 'g', blockListUrls: ['https://x.example/declared', 'https://x.example/streamed'] }])), { fetch: big, lookup: PUBLIC, maxBytes: 1000 })
    const out = await lists.count(server, 'x')
    expect(out.lists.map((l) => l.error)).toEqual(['the list is larger than the limit', 'the list is larger than the limit'])
  })

  it('times a slow list out', async () => {
    const slow = ((_: URL | string, init?: RequestInit) =>
      new Promise((_resolve, reject) => init?.signal?.addEventListener('abort', () => reject(Object.assign(new Error('x'), { name: 'AbortError' }))))) as typeof fetch
    const lists = new AppLists(upstreamWith(config([{ name: 'g', blockListUrls: ['https://x.example/slow'] }])), { fetch: slow, lookup: PUBLIC, timeoutMs: 20 })
    expect((await lists.count(server, 'x')).lists[0].error).toBe('timed out')
  })

  it('takes at most maxLists lists', async () => {
    const urls = Array.from({ length: 10 }, (_, i) => `https://x.example/${i}`)
    const lists = new AppLists(upstreamWith(config([{ name: 'g', blockListUrls: urls }])), { fetch: listFetch({}), lookup: PUBLIC, maxLists: 3 })
    expect((await lists.count(server, 'x')).lists).toHaveLength(3)
  })
})

describe('hub/blocking/app-lists/count', () => {
  let a: Fake
  let b: Fake
  let hub: TestHub
  let files: http.Server
  let base = ''

  beforeAll(async () => {
    files = http.createServer((req, res) => {
      res.setHeader('content-type', 'text/plain')
      res.end(req.url === '/hosts' ? HOSTS : 'p.example\nq.example\nr.example\n')
    })
    await new Promise<void>((resolve) => files.listen(0, '127.0.0.1', resolve))
    base = `http://127.0.0.1:${(files.address() as AddressInfo).port}`
    a = await fake()
    b = await fake()
    a.on('apps/config/get', () => ({
      response: { config: config([{ name: 'g', enableBlocking: true, blockListUrls: [`${base}/hosts`], adblockListUrls: [`${base}/other`] }]) },
    }))
    b.on('apps/config/get', () => ({ response: { config: null } }))
    hub = await startHub(a, b, { HUB_LISTS_ALLOW_PRIVATE: '1' })
  })

  afterAll(() => {
    hub.close()
    a.server.close()
    b.server.close()
    files.close()
  })

  it('counts the lists of the chosen server with its read token', async () => {
    const r = await hub.call('viewer', 'hub/blocking/app-lists/count', { node: 'a', name: 'Advanced Blocking' }, 'GET')
    expect(r.code).toBe(200)
    expect(r.body.response.total).toBe(5)
    expect(r.body.response.lists).toEqual([{ url: `${base}/hosts`, count: 2 }, { url: `${base}/other`, count: 3 }])
    const read = a.calls.find((c) => c.path === 'apps/config/get')
    expect(read?.token).toBe('read-a')
    expect(read?.query.get('name')).toBe('Advanced Blocking')
  })

  it('answers an empty list for a server whose app has no config', async () => {
    const r = await hub.call('viewer', 'hub/blocking/app-lists/count', { node: 'b' }, 'GET')
    expect(r.body.response).toEqual({ lists: [], total: 0 })
  })

  it('needs a server', async () => {
    for (const node of ['', 'cluster', 'nope']) {
      const r = await hub.call('viewer', 'hub/blocking/app-lists/count', { node }, 'GET')
      expect(r.code).toBe(400)
    }
  })

  it('is refused without a session', async () => {
    const res = await fetch(`${hub.base}/api/hub/blocking/app-lists/count?node=a`)
    expect(((await res.json()) as { status: string }).status).toBe('invalid-token')
  })

  it('does not fetch private addresses unless the hub is told to', async () => {
    const strict = await startHub(a, b)
    const r = await strict.call('viewer', 'hub/blocking/app-lists/count', { node: 'a' }, 'GET')
    strict.close()
    expect(r.body.response.lists.map((l: { error?: string }) => l.error)).toEqual(['the address of this list is not a public one', 'the address of this list is not a public one'])
    expect(r.body.response.total).toBe(0)
  })
})