fix(ci): support ipset sockets and deployed image smoke checks

scripts/bench/fixture.mjs

@@ -3,6 +3,10 @@ import { createRequire } from 'node:module'
 import { createHash } from 'node:crypto'
 import { fileURLToPath } from 'node:url'
 import { resolve } from 'node:path'
+import { realpathSync } from 'node:fs'
+export function fixtureDatabaseConstructor() {
+  return createRequire(realpathSync(resolve('node_modules/@evofw/db/package.json')))('better-sqlite3')
+}
 export const benchmarkPrefix = (i, first = 10) => `${first}.${i >>> 16}.${(i >>> 8) & 255}.${i & 255}/32`
 export function seedFixture(db) {
   db.pragma('foreign_keys = ON')
@@ -26,8 +30,7 @@ export function seedFixture(db) {
   return { fixture: { agents: 100, prefixes: 50_000 }, integrity: db.pragma('integrity_check', { simple: true }) }
 }
 if (!process.argv[1] || resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
-  const require = createRequire('/app/node_modules/@evofw/db/package.json')
-  const Database = require('better-sqlite3')
+  const Database = fixtureDatabaseConstructor()
   const db = new Database('/data/app.db')
   try { console.log(JSON.stringify(seedFixture(db))) } finally { db.close() }
 }

scripts/bench/runtime.sh

@@ -7,9 +7,11 @@ out="${2:?OUTPUT_DIR required}"
 mkdir -p "$out"
 id="evofw-bench-$(date +%s)-$$"
 volume="$id-data"
+load_id="$id-load"
 cleanup() {
   docker logs "$id" >"$out/container.log" 2>&1 || true
   [ -z "${sampler:-}" ] || { kill "$sampler" 2>/dev/null || true; wait "$sampler" 2>/dev/null || true; }
+  docker rm -fv "$load_id" >/dev/null 2>&1 || true
   docker rm -f "$id" >/dev/null 2>&1 || true
   docker volume rm "$volume" >/dev/null 2>&1 || true
 }
@@ -20,7 +22,7 @@ docker image inspect "$image" >"$out/image.json"
 if docker volume inspect "$volume" >/dev/null 2>&1; then echo 'Refusing reused benchmark volume' >&2; exit 1; fi
 docker volume create "$volume" >/dev/null
 started="$(date +%s)"
-docker run -d --name "$id" --cpus 2 --memory 4g --publish 127.0.0.1::8080 \
+docker run -d --name "$id" --cpus 2 --memory 4g --network none \
   --mount "type=volume,src=$volume,dst=/data" -e EVOFW_ALLOW_UNSAFE=true -e AUTH_REQUIRED=false "$image" >/dev/null
 attempts=0
 until docker exec "$id" node -e "fetch('http://127.0.0.1:8080/ready').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" >/dev/null 2>&1; do
@@ -31,8 +33,24 @@ docker exec -i "$id" node --input-type=module <scripts/bench/fixture.mjs >"$out/
 # Includes CPU, memory, block IO and PIDs. Metrics are raw observations, no claimed gains.
 (while docker inspect -f '{{.State.Running}}' "$id" 2>/dev/null | grep -q true; do docker stats --no-stream --format '{{json .}}' "$id"; sleep 1; done) >"$out/stats.jsonl" &
 sampler=$!
-port="$(docker port "$id" 8080/tcp | sed 's/.*://')"
-BENCH_P95_LIMIT_MS="${BENCH_P95_LIMIT_MS:-500}" BENCH_URL="http://127.0.0.1:$port" node scripts/bench/load.mjs >"$out/load.json" || load_failed=true
+# The daemon can be remote or reached from a Gitea job container: runner
+# localhost is not the API container. A separate nonroot driver joins only the
+# API network namespace, so its CPU work is outside the app's 2-CPU quota.
+docker create --name "$load_id" --network "container:$id" \
+  --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges:true \
+  --pids-limit 128 --cpus 2 --memory 1g --entrypoint node \
+  -e BENCH_URL=http://127.0.0.1:8080 \
+  -e "BENCH_P95_LIMIT_MS=${BENCH_P95_LIMIT_MS:-500}" \
+  -e "BENCH_CONCURRENCY=${BENCH_CONCURRENCY:-10}" \
+  -e "BENCH_STEADY_CONCURRENCY=${BENCH_STEADY_CONCURRENCY:-2}" \
+  -e "BENCH_STEADY_INTERVAL_MS=${BENCH_STEADY_INTERVAL_MS:-1200}" \
+  "$image" /tmp/load.mjs >/dev/null
+# docker cp transfers bytes through the Docker API; no runner bind mount or
+# container Docker socket is needed. Both modules are readable by the driver UID.
+docker cp scripts/bench/load.mjs "$load_id:/tmp/load.mjs"
+docker cp scripts/bench/fixture.mjs "$load_id:/tmp/fixture.mjs"
+docker start --attach "$load_id" >"$out/load.json" || load_failed=true
+[ "$(docker wait "$load_id")" = 0 ] || load_failed=true
 cat "$out/load.json"
 docker exec "$id" sh -c 'ls -ln /data/app.db*; cat /proc/1/status; cat /proc/1/io' >"$out/db-process.txt"
 sh deploy/docker/backup.sh "$id" "$out/backup.db"

scripts/bench/runtime.test.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env sh
+# Remote-daemon wiring regression; no Docker daemon, app, or network load.
+set -eu
+wrapper="$(pwd)/scripts/bench/runtime.sh"
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT HUP INT TERM
+mkdir "$tmp/bin"
+cat >"$tmp/bin/docker" <<'DOCKER'
+#!/usr/bin/env sh
+set -eu
+printf '%s\n' "$*" >>"$FAKE_LOG"
+case "$1" in
+  version|info) printf 'fake remote daemon\n' ;;
+  image) printf '{}\n' ;;
+  volume) case "$2" in inspect) exit 1;; *) exit 0;; esac ;;
+  run|create) printf 'fake-container-id\n' ;;
+  exec)
+    case "$*" in
+      *--input-type=module*) cat >/dev/null; printf '{"fixture":true}\n' ;;
+      *) exit 0 ;;
+    esac ;;
+  inspect) printf 'false\n' ;;
+  stats) printf '{}\n' ;;
+  start) printf '{"assertions":{"passed":true}}\n' ;;
+  wait) printf '%s\n' "$FAKE_LOAD_EXIT" ;;
+  cp) case "$2" in *:/tmp/*) printf 'fake-db' >"$3";; *) exit 0;; esac ;;
+  logs|rm) exit 0 ;;
+  *) echo 'Unexpected Docker operation' >&2; exit 99 ;;
+esac
+DOCKER
+cat >"$tmp/bin/node" <<'NODE'
+#!/usr/bin/env sh
+echo 'Benchmark must never execute runner-host Node' >&2
+exit 97
+NODE
+chmod +x "$tmp/bin/docker" "$tmp/bin/node"
+for expected in 0 23; do
+  log="$tmp/docker-$expected.log"
+  if PATH="$tmp/bin:$PATH" FAKE_LOG="$log" FAKE_LOAD_EXIT="$expected" BENCH_P95_LIMIT_MS=321 BENCH_CONCURRENCY=7 BENCH_STEADY_CONCURRENCY=3 BENCH_STEADY_INTERVAL_MS=1350 sh "$wrapper" fixture:immutable "$tmp/out-$expected" >/dev/null; then actual=0; else actual=$?; fi
+  if [ "$expected" = 0 ]; then [ "$actual" = 0 ]; else [ "$actual" != 0 ]; fi
+  grep -E '^run -d --name evofw-bench-[^ ]+ --cpus 2 --memory 4g --network none ' "$log" >/dev/null
+  grep -E '^create --name evofw-bench-[^ ]+-load --network container:evofw-bench-[^ ]+ --user 65534:65534 --cap-drop ALL --security-opt no-new-privileges:true ' "$log" >/dev/null
+  grep -F -- ' --entrypoint node -e BENCH_URL=http://127.0.0.1:8080 -e BENCH_P95_LIMIT_MS=321 -e BENCH_CONCURRENCY=7 -e BENCH_STEADY_CONCURRENCY=3 -e BENCH_STEADY_INTERVAL_MS=1350 fixture:immutable /tmp/load.mjs' "$log" >/dev/null
+  grep -E '^cp scripts/bench/load.mjs evofw-bench-[^ ]+-load:/tmp/load.mjs$' "$log" >/dev/null
+  grep -E '^cp scripts/bench/fixture.mjs evofw-bench-[^ ]+-load:/tmp/fixture.mjs$' "$log" >/dev/null
+  grep -E '^start --attach evofw-bench-[^ ]+-load$' "$log" >/dev/null
+  grep -E '^wait evofw-bench-[^ ]+-load$' "$log" >/dev/null
+  grep -E '^rm -fv evofw-bench-[^ ]+-load$' "$log" >/dev/null
+  grep -E '^volume rm evofw-bench-[^ ]+-data$' "$log" >/dev/null
+  if grep -E -- '(--publish|--privileged|--network host|--pid host|type=bind|docker.sock|^port )' "$log" >/dev/null; then echo 'Unsafe or daemon-local benchmark wiring' >&2; exit 1; fi
+  [ -f "$tmp/out-$expected/backup.db" ]
+done
+echo 'Runtime benchmark wrapper regression passed (remote daemon, separate driver, failure, cleanup)'

scripts/ci/image-smoke-marker.test.mjs

@@ -0,0 +1,61 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import { readFileSync, writeFileSync, mkdtempSync, rmSync } from 'node:fs'
+import { join, resolve } from 'node:path'
+import { tmpdir } from 'node:os'
+import { spawnSync } from 'node:child_process'
+import { pathToFileURL } from 'node:url'
+
+test('actual image smoke marker uses the old/new public list API and never recreates missing persisted data', () => {
+  const script = readFileSync(resolve('scripts/ci/image-smoke.sh'), 'utf8')
+  const program = script.match(/<<'MARKER'\r?\n([\s\S]*?)\r?\nMARKER/)[1]
+  const directory = mkdtempSync(join(tmpdir(), 'evofw-marker-'))
+  const marker = join(directory, 'marker.mjs'), mock = join(directory, 'fetch.mjs'), state = join(directory, 'state.json')
+  writeFileSync(marker, program)
+  writeFileSync(mock, `
+import assert from 'node:assert/strict'
+import { existsSync, readFileSync, writeFileSync } from 'node:fs'
+globalThis.fetch = async (url, options = {}) => {
+  const state = existsSync(process.env.MARKER_STATE) ? JSON.parse(readFileSync(process.env.MARKER_STATE, 'utf8')) : { creates: 0, entity: null }
+  let status = 200, body
+  if (options.method === 'POST') {
+    assert.equal(url, 'http://127.0.0.1:8080/api/v1/lists')
+    const payload = JSON.parse(options.body)
+    state.creates++
+    state.entity = { id: 'persisted-list-id', name: payload.name, type: payload.type, entries: payload.entries, entry_count: payload.entries.length }
+    writeFileSync(process.env.MARKER_STATE, JSON.stringify(state))
+    status = Number(process.env.CREATE_STATUS)
+    body = { id: state.entity.id, name: state.entity.name, type: state.entity.type }
+  } else {
+    assert.equal(url, 'http://127.0.0.1:8080/api/v1/lists/persisted-list-id')
+    status = state.entity ? 200 : 404
+    body = state.entity || { error: 'NOT_FOUND' }
+  }
+  return { ok: status >= 200 && status < 300, status, json: async () => body }
+}
+`)
+  const run = (mode, id, status) => spawnSync(process.execPath, ['--import', pathToFileURL(mock).href, marker, mode, id, 'unique-smoke-marker'], { encoding: 'utf8', env: { ...process.env, MARKER_STATE: state, CREATE_STATUS: String(status) } })
+  try {
+    // Previous aa47584 returns 200 for POST /lists; newer implementations may use 201.
+    for (const status of [200, 201]) {
+      rmSync(state, { force: true })
+      const created = run('write', '', status)
+      assert.equal(created.status, 0, created.stderr)
+      assert.equal(created.stdout, 'persisted-list-id')
+      for (const phase of ['read', 'write', 'read', 'read']) {
+        const verified = run(phase, created.stdout, status)
+        assert.equal(verified.status, 0, verified.stderr)
+        assert.equal(verified.stdout, created.stdout)
+      }
+      const saved = JSON.parse(readFileSync(state, 'utf8'))
+      assert.equal(saved.creates, 1, 'Upgrade/restart/restore verification must never create replacement data')
+      saved.entity.entries = []; saved.entity.entry_count = 0
+      writeFileSync(state, JSON.stringify(saved))
+      assert.notEqual(run('read', created.stdout, status).status, 0, 'Lost marker entries must fail smoke')
+      saved.entity = null
+      writeFileSync(state, JSON.stringify(saved))
+      assert.notEqual(run('read', created.stdout, status).status, 0, 'Missing marker must fail smoke')
+      assert.equal(JSON.parse(readFileSync(state, 'utf8')).creates, 1)
+    }
+  } finally { rmSync(directory, { recursive: true, force: true }) }
+})

scripts/ci/image-smoke.sh

@@ -13,6 +13,8 @@ id="evofw-smoke-$(date +%s)-$$"
 volume="$id-data"
 restore_volume="$id-restore"
 container="$id"
+marker_id=''
+marker_name="$id-persistence"
 tmp="$(mktemp -d)"
 cleanup() {
   result=$?
@@ -38,7 +40,32 @@ start() {
   done
 }
 marker() {
-  docker exec "$container" node -e "const D=require('node:module').createRequire('/app/node_modules/@evofw/db/package.json')('better-sqlite3'); const db=new D('/data/app.db'); if(process.argv[1]==='write') db.prepare('INSERT OR REPLACE INTO settings(key,value) VALUES (?,?)').run('ci-smoke','survives'); else if(db.prepare('SELECT value FROM settings WHERE key=?').get('ci-smoke')?.value!=='survives')process.exitCode=1; db.close()" "$1"
+  value="$(docker exec -i "$container" node --input-type=module - "$1" "$marker_id" "$marker_name" <<'MARKER'
+import assert from 'node:assert/strict'
+const [mode, existingId, name] = process.argv.slice(2)
+assert.ok(mode === 'write' || mode === 'read')
+const call = async (path, options = {}) => {
+  const response = await fetch(`http://127.0.0.1:8080/api/v1${path}`, { ...options, signal: AbortSignal.timeout(10000) })
+  assert.ok(response.ok, `Persistence marker HTTP ${response.status}`)
+  return response.json()
+}
+let id = existingId
+if (mode === 'write' && !id) {
+  const created = await call('/lists', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ name, type: 'static', entries: ['192.0.2.123/32'] }) })
+  assert.ok(typeof created.id === 'string' && created.id.length > 0, 'Marker creation did not return an ID')
+  id = created.id
+}
+assert.ok(id, 'Persistence marker ID is missing')
+const stored = await call(`/lists/${encodeURIComponent(id)}`)
+assert.equal(stored.id, id)
+assert.equal(stored.name, name)
+assert.equal(stored.type, 'static')
+assert.deepEqual(stored.entries, ['192.0.2.123/32'], 'Marker entries did not survive')
+assert.equal(stored.entry_count, 1)
+process.stdout.write(id)
+MARKER
+)"
+  if [ "$1" = write ]; then marker_id="$value"; fi
 }
 stop() {
   docker stop -t 30 "$container" >/dev/null

scripts/ci/linux-agent-container-smoke.sh

@@ -22,13 +22,15 @@ docker build --file scripts/ci/linux-agent-smoke.Dockerfile --iidfile "$tmp/imag
 image="$(cat "$tmp/image-id")"
 printf '%s\n' "$image" | grep -Eq '^sha256:[0-9a-f]{64}$' || { echo 'Missing immutable helper image ID' >&2; exit 1; }
 set -- docker run --rm --name "$name" --network none \
-  --cap-drop ALL --cap-add SYS_ADMIN --cap-add NET_ADMIN \
+  --cap-drop ALL --cap-add SYS_ADMIN --cap-add NET_ADMIN --cap-add NET_RAW \
   --security-opt no-new-privileges:true \
   --read-only --tmpfs /tmp:rw,nosuid,nodev,noexec,size=64m \
   --pids-limit 256 --cpus 2 --memory 512m
 # docker-default AppArmor denies the mount operations used by unshare/ip-netns.
 # Only this disposable, networkless helper gets the mount-capable exception.
-# Default seccomp remains enabled and adapts to the two granted capabilities.
+# libxt_set opens an AF_INET/SOCK_RAW/IPPROTO_RAW socket to query ipset;
+# NET_RAW permits that query inside the isolated helper network namespace.
+# Default seccomp remains enabled and adapts to the granted capabilities.
 security="$(docker info --format '{{json .SecurityOptions}}')"
 if printf '%s' "$security" | grep -q apparmor; then
   set -- "$@" --security-opt apparmor=unconfined

scripts/ci/linux-agent-container-smoke.test.sh

@@ -32,7 +32,9 @@ for expected in 0 23; do
   if PATH="$tmp/bin:$PATH" FAKE_LOG="$log" FAKE_APPARMOR="$apparmor" FAKE_RUN_EXIT="$expected" sh "$wrapper"; then actual=0; else actual=$?; fi
   [ "$actual" = "$expected" ] || { echo 'Wrapper did not preserve the container exit code' >&2; exit 1; }
   grep -F 'build --file scripts/ci/linux-agent-smoke.Dockerfile --iidfile ' "$log" >/dev/null
-  grep -F ' --network none --cap-drop ALL --cap-add SYS_ADMIN --cap-add NET_ADMIN ' "$log" >/dev/null
+  grep -F ' --network none --cap-drop ALL --cap-add SYS_ADMIN --cap-add NET_ADMIN --cap-add NET_RAW ' "$log" >/dev/null
+  capabilities="$(grep '^run ' "$log" | awk '{ for (i=1; i<=NF; i++) if ($i == "--cap-add") print $(i+1) }' | sort | tr '\n' ' ')"
+  [ "$capabilities" = 'NET_ADMIN NET_RAW SYS_ADMIN ' ] || { echo 'Unexpected helper capability set' >&2; exit 1; }
   grep -F ' --security-opt no-new-privileges:true --read-only --tmpfs /tmp:' "$log" >/dev/null
   grep -F 'image rm sha256:' "$log" >/dev/null
   grep -E '^rm -f evofw-linux-smoke-' "$log" >/dev/null

scripts/ci/sqlite-backup.test.mjs

@@ -1,10 +1,11 @@
 import test from 'node:test'
 import assert from 'node:assert/strict'
 import { createRequire } from 'node:module'
-import { mkdtempSync, rmSync, writeFileSync, existsSync } from 'node:fs'
+import { mkdtempSync, rmSync, writeFileSync, existsSync, mkdirSync, symlinkSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { resolve, join } from 'node:path'
 import { spawnSync } from 'node:child_process'
+import { fixtureDatabaseConstructor } from '../bench/fixture.mjs'
 const root = resolve(import.meta.dirname, '../..')
 const Database = createRequire(join(root, 'packages/db/package.json'))('better-sqlite3')
 const script = join(root, 'deploy/docker/evofw/sqlite-backup.mjs')
@@ -12,8 +13,23 @@ const script = join(root, 'deploy/docker/evofw/sqlite-backup.mjs')
 test('online backup includes committed WAL and restore verifies before offline replacement', () => {
   const directory = mkdtempSync(join(tmpdir(), 'evofw-backup-test-'))
   let db
-  const run = (...args) => spawnSync(process.execPath, [script, ...args], { cwd: directory, encoding: 'utf8', env: { ...process.env, NODE_PATH: join(root, 'packages/db/node_modules') } })
   try {
+  // Isolated pnpm-shaped tree: the dependency is a sibling of the real package,
+  // never a top-level dependency reachable from its logical symlink location.
+  const store = join(directory, 'virtual-store/node_modules')
+  const realPackage = join(store, '@evofw/db')
+  mkdirSync(realPackage, { recursive: true })
+  writeFileSync(join(realPackage, 'package.json'), '{"name":"@evofw/db"}')
+  mkdirSync(join(directory, 'node_modules/@evofw'), { recursive: true })
+  const actualDependency = createRequire(join(root, 'packages/db/package.json')).resolve('better-sqlite3/package.json')
+  symlinkSync(resolve(actualDependency, '..'), join(store, 'better-sqlite3'), 'junction')
+  symlinkSync(realPackage, join(directory, 'node_modules/@evofw/db'), 'junction')
+  const env = { ...process.env }
+  delete env.NODE_PATH
+  const run = (...args) => spawnSync(process.execPath, [script, ...args], { cwd: directory, encoding: 'utf8', env })
+    assert.throws(() => createRequire(join(directory, 'node_modules/@evofw/db/package.json'))('better-sqlite3'), { code: 'MODULE_NOT_FOUND' })
+    const previousDirectory = process.cwd()
+    try { process.chdir(directory); assert.equal(fixtureDatabaseConstructor(), Database) } finally { process.chdir(previousDirectory) }
     const source = join(directory, 'source.db')
     const backup = join(directory, 'backup.db')
     const restored = join(directory, 'restored.db')