fix(ci): restore backups through remote Docker API

.gitea/workflows/quality.yaml

@@ -273,6 +273,7 @@ jobs:
           pnpm exec turbo run test --filter=@evofw/api --filter=@evofw/shared
           node --test scripts/ci/sqlite-backup.test.mjs
           node --test scripts/ci/image-smoke-marker.test.mjs
+          sh scripts/ci/restore-wrapper.test.sh
           pnpm exec turbo run build --filter=@evofw/api
           node --test scripts/ci/benchmark-fixture.test.mjs
           node --test scripts/ci/manifest-platform.test.mjs

deploy/docker/README.md

@@ -8,7 +8,7 @@
 
 Bake читает переменные из **окружения** (`REGISTRY`, `IMAGE_TAG`, `CACHE_REF_NODE`, `BASE_NODE`). Скрипт [write-bake-override.sh](write-bake-override.sh) — опциональный helper для локальной отладки.
 
-BuildKit кэширует pnpm store через `RUN --mount=type=cache`. На CI mounts живут, пока named builder `evofw` не удаляют (`cleanup: false`).
+BuildKit кэширует pnpm store через `RUN --mount=type=cache`. В CI каждый запуск использует отдельный builder с `cleanup: true`; долговременный кэш хранится в registry (`mode=max`). Общий постоянный builder `evofw` больше не используется.
 
 ### Слои и runtime
 
@@ -33,6 +33,8 @@ BuildKit кэширует pnpm store через `RUN --mount=type=cache`. На C
 
 Runtime smoke, backup/restore, baseline и оставшиеся release gates: [runtime-validation.md](../../docs/runtime-validation.md).
 
+Restore работает и с удалённым Docker daemon: локальная резервная копия передаётся через `docker cp` в остановленный одноразовый helper, без bind mount пути runner. Непосредственно перед запуском повторно проверяется, что целевой named volume не используется. Helper удаляется с его анонимными volumes; целевой named volume сохраняется. На всё время восстановления остальные writers и их автоматический перезапуск должны оставаться остановленными.
+
 ## Локальная сборка
 
 Из корня репозитория:

deploy/docker/restore.sh

@@ -7,7 +7,29 @@ backup="${3:?BACKUP_FILE required}"
 backup="$(cd "$(dirname "$backup")" && pwd)/$(basename "$backup")"
 # Named volumes only. All users must remain stopped for the whole operation.
 docker volume inspect "$volume" >/dev/null
-[ -z "$(docker ps -q --filter "volume=$volume")" ] || { echo 'Stop every container using the volume before restore' >&2; exit 1; }
-docker run --rm --network none --mount "type=volume,src=$volume,dst=/data" \
-  --mount "type=bind,src=$backup,dst=/backup.db,readonly" --entrypoint node \
-  "$image" tools/sqlite-backup.mjs restore /backup.db /data/app.db --offline
+active="$(docker ps -q --filter "volume=$volume")"
+[ -z "$active" ] || { echo 'Stop every container using the volume before restore' >&2; exit 1; }
+helper="evofw-restore-$(date +%s)-$$"
+cleanup() {
+  result=$?
+  trap - EXIT HUP INT TERM
+  # -v removes anonymous helper volumes only, never the target named volume.
+  docker rm -fv "$helper" >/dev/null 2>&1 || true
+  exit "$result"
+}
+trap cleanup EXIT
+trap 'exit 129' HUP
+trap 'exit 130' INT
+trap 'exit 143' TERM
+# Transfer input through the Docker API, including when its daemon is remote.
+docker create --name "$helper" --network none --cap-drop ALL \
+  --security-opt no-new-privileges:true --mount "type=volume,src=$volume,dst=/data" \
+  --entrypoint node "$image" tools/sqlite-backup.mjs restore /tmp/evofw-restore-input.db /data/app.db --offline >/dev/null
+docker cp "$backup" "$helper:/tmp/evofw-restore-input.db"
+active="$(docker ps -q --filter "volume=$volume")"
+[ -z "$active" ] || { echo 'Target volume became active before restore; helper was not started' >&2; exit 1; }
+docker start --attach "$helper"
+result="$(docker wait "$helper")"
+case "$result" in ''|*[!0-9]*) echo 'Invalid restore container exit status' >&2; exit 1;; esac
+[ "$result" -le 255 ] || { echo 'Invalid restore container exit status' >&2; exit 1; }
+exit "$result"

docs/runtime-validation.md

@@ -55,7 +55,9 @@ docker compose up -d evofirewall
 
 Backup uses `better-sqlite3`'s SQLite online backup API and verifies the output, including foreign keys. It includes committed WAL data while the API is running; copying only `app.db` from a live WAL database is not a backup. The host output must be new. Keep backups on a separate durable filesystem and protect them like credentials.
 
-Restore supports a named Docker volume with `/data/app.db`. Stop **all** writers and prevent supervisors/other operators from starting them for the entire restore. The wrapper rejects a volume currently mounted by any running container, mounts the backup read-only, verifies it, prepares a temporary database, removes old sidecars and replaces the offline DB. This exclusion check is a guard, not a cross-host lock. Take a backup before replacing an existing DB. Rehearse restoration and upgrade on a cloned volume before production. Downgrading application code after new migrations requires restoring the matching pre-upgrade backup; down migrations are not supplied.
+Restore supports a named Docker volume with `/data/app.db`. Stop **all** writers and prevent supervisors/other operators from starting them for the entire restore. The wrapper first verifies the volume exists and has no running writers. It creates a stopped, networkless helper with the named volume, copies the local backup into its private `/tmp` through `docker cp`, then checks again for running writers immediately before starting the helper. Failed Docker writer enumeration is an error, never evidence that the volume is offline. The helper verifies the input, prepares a temporary database, removes old sidecars and replaces the offline DB. Start/wait failures and the restore process exit status propagate; cleanup removes only the helper and its anonymous volumes, never the target named volume. No runner-local path is used as a daemon-host bind mount, so a remote Docker daemon is supported. The offline checks are guards, not a cross-host lock. Take a backup before replacing an existing DB. Rehearse restoration and upgrade on a cloned volume before production. Downgrading application code after new migrations requires restoring the matching pre-upgrade backup; down migrations are not supplied.
+
+The subsequent pushed image smoke confirmed upgrade persistence, API/SPA/script checks, online backup integrity and restart-marker persistence. Restore then failed because a runner-local temporary backup path was passed as a bind source to a different Docker daemon host. The stopped-helper/copy correction addresses that transport failure without weakening SQLite verification. A fake-Docker regression passed for successful restore, copy/start/process failures, initially active and newly active volumes, failed writer enumeration at both guards, cleanup and target-volume preservation; CI now runs it beside the backup/marker tests. Actual restore completion and the downstream constrained benchmark still require the corrected CI run. Image-smoke cleanup also checks container existence before collecting final logs, avoiding misleading missing-container errors after a successful stop/remove.
 
 ## Reproducible baseline
 

scripts/ci/image-smoke.sh

@@ -18,7 +18,7 @@ marker_name="$id-persistence"
 tmp="$(mktemp -d)"
 cleanup() {
   result=$?
-  docker logs "$container" >>"$tmp/container.log" 2>&1 || true
+  if docker inspect "$container" >/dev/null 2>&1; then docker logs "$container" >>"$tmp/container.log" 2>&1 || true; fi
   if [ "$result" -ne 0 ]; then cat "$tmp/container.log" >&2; fi
   docker rm -f "$container" >/dev/null 2>&1 || true
   docker volume rm "$volume" "$restore_volume" >/dev/null 2>&1 || true

scripts/ci/restore-wrapper.test.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env sh
+# No Docker daemon: verify actual restore transport, guards and exit propagation.
+set -eu
+wrapper="$(pwd)/deploy/docker/restore.sh"
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT HUP INT TERM
+mkdir "$tmp/bin"
+backup="$tmp/input backup.db"
+printf 'synthetic backup' >"$backup"
+cat >"$tmp/bin/docker" <<'DOCKER'
+#!/usr/bin/env sh
+set -eu
+printf '%s\n' "$*" >>"$FAKE_LOG"
+case "$1" in
+  volume) [ "$2" = inspect ]; exit 0 ;;
+  ps)
+    count=0; [ ! -f "$FAKE_COUNT" ] || count="$(cat "$FAKE_COUNT")"
+    count=$((count + 1)); printf '%s' "$count" >"$FAKE_COUNT"
+    if [ "$FAKE_CASE" = ps-initial-failure ] || { [ "$FAKE_CASE" = ps-later-failure ] && [ "$count" = 2 ]; }; then exit 29; fi
+    if [ "$FAKE_CASE" = live-initial ] || { [ "$FAKE_CASE" = live-later ] && [ "$count" = 2 ]; }; then printf 'other-writer\n'; fi ;;
+  create) printf 'helper-id\n' ;;
+  cp) [ "$2" = "$FAKE_BACKUP" ]; [ "$FAKE_CASE" != cp-failure ] || exit 17 ;;
+  start) [ "$FAKE_CASE" != start-failure ] || exit 19 ;;
+  wait) if [ "$FAKE_CASE" = restore-failure ]; then printf '23\n'; else printf '0\n'; fi ;;
+  rm) exit 0 ;;
+  *) echo 'Unexpected Docker operation in restore' >&2; exit 98 ;;
+esac
+DOCKER
+chmod +x "$tmp/bin/docker"
+for scenario in success cp-failure start-failure restore-failure live-initial live-later ps-initial-failure ps-later-failure; do
+  log="$tmp/$scenario.log"
+  if PATH="$tmp/bin:$PATH" FAKE_LOG="$log" FAKE_COUNT="$tmp/$scenario.count" FAKE_CASE="$scenario" FAKE_BACKUP="$backup" sh "$wrapper" image:immutable target-named-volume "$backup"; then actual=0; else actual=$?; fi
+  case "$scenario" in success) expected=0;; cp-failure) expected=17;; start-failure) expected=19;; restore-failure) expected=23;; ps-*-failure) expected=29;; *) expected=1;; esac
+  [ "$actual" = "$expected" ] || { echo "Unexpected status for $scenario: $actual" >&2; exit 1; }
+  grep -F 'volume inspect target-named-volume' "$log" >/dev/null
+  if grep -E -- '(type=bind|--privileged|--network host|docker.sock|^volume rm)' "$log" >/dev/null; then echo 'Restore must not use a host bind or remove the target volume' >&2; exit 1; fi
+  if [ "$scenario" = live-initial ] || [ "$scenario" = ps-initial-failure ]; then
+    if grep -E '^(create|cp|start) ' "$log" >/dev/null; then echo 'Active target must fail before helper creation' >&2; exit 1; fi
+    continue
+  fi
+  grep -E '^create --name evofw-restore-[^ ]+ --network none --cap-drop ALL --security-opt no-new-privileges:true --mount type=volume,src=target-named-volume,dst=/data --entrypoint node image:immutable tools/sqlite-backup.mjs restore /tmp/evofw-restore-input.db /data/app.db --offline$' "$log" >/dev/null
+  grep -E '^cp .*input backup.db evofw-restore-[^ ]+:/tmp/evofw-restore-input.db$' "$log" >/dev/null
+  grep -E '^rm -fv evofw-restore-' "$log" >/dev/null
+  if [ "$scenario" = cp-failure ] || [ "$scenario" = live-later ] || [ "$scenario" = ps-later-failure ]; then
+    if grep -E '^start ' "$log" >/dev/null; then echo 'Incomplete input or newly active target must never start restore' >&2; exit 1; fi
+  elif [ "$scenario" != start-failure ]; then
+    grep -E '^wait evofw-restore-' "$log" >/dev/null
+    [ "$(cat "$tmp/$scenario.count")" = 2 ]
+  fi
+done
+echo 'Restore wrapper regression passed (remote copy, offline guards, failures, cleanup)'