fix(geoip): проверить локальный кеш и ограничить загрузку баз

backend/scripts/acceptance/geoip-mirror.mjs

@@ -0,0 +1,66 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import path from 'node:path';
+import { createHash } from 'node:crypto';
+const database = process.env.MMAPP_OWN_SQL_DB;
+assert(/^mm_accept_166_\d+$/.test(database));
+const url = new URL(process.env.DATABASE_URL); assert.equal(url.pathname, '/' + database); assert.equal(url.username, database);
+assert.equal(process.env.GEOIP_DIR, '/tmp/' + database + '/geoip');
+fs.mkdirSync('/tmp/' + database, { recursive: true }); process.chdir('/tmp/' + database);
+const { pool, closePool } = await import('/app/backend/dist/db/index.js');
+const { initDatabase } = await import('/app/backend/dist/db/bootstrap.js');
+const role = (await pool.query('SELECT rolsuper,rolcreatedb,rolcreaterole,rolbypassrls FROM pg_roles WHERE rolname=current_user')).rows[0];
+assert(Object.values(role).every(v => v === false)); await initDatabase();
+const { collectGeoipUpdateOnce, setGeoipFetchForTests, resetGeoipUpdateForTests } = await import('/app/backend/dist/services/geoip-update-collector.js');
+const { getGeoipSettings, updateGeoipSettings } = await import('/app/backend/dist/services/geoip-settings.js');
+const { geoipReadersStatus, lookupGeoip, GEOIP_COUNTRY_FILE, GEOIP_ASN_FILE } = await import('/app/backend/dist/services/traffic-flow-geoip.js');
+const { databaseBackgroundWork } = await import('/app/backend/dist/services/database-background-work.js');
+const report = { date: '2026-10-14', block: '12.5.242', passed: false, scope: 'Compiled GeoIP collector + owned PostgreSQL/files + actual public mirror; controlled negative bodies separately marked', mirrorRequests: [], checks: [], chrContacts: 0, mainDatabaseWrites: 0 };
+const file = name => path.join(process.env.GEOIP_DIR, name);
+const sha = name => createHash('sha256').update(fs.readFileSync(file(name))).digest('hex');
+const save = () => fs.writeFileSync('/checks/geoip-mirror242.json', JSON.stringify(report, null, 2) + '\n');
+const check = (name, detail = {}) => { report.checks.push({ name, ...detail }); save(); };
+const realFetch = globalThis.fetch.bind(globalThis);
+async function observedFetch(url, options) {
+ const started = Date.now(), response = await realFetch(url, options);
+ report.mirrorRequests.push({ file: new URL(String(url)).pathname.split('/').at(-1), conditional: Boolean(options?.headers?.['If-None-Match']), status: response.status, headersMs: Date.now() - started }); save(); return response;
+}
+try {
+ await updateGeoipSettings({ enabled: false }); setGeoipFetchForTests(observedFetch);
+ let snapshot = await collectGeoipUpdateOnce({ force: true });
+ assert(!snapshot.fatalError); assert.deepEqual(snapshot.errors, []); assert.equal(snapshot.downloaded, 2);
+ assert(geoipReadersStatus().countryLoaded && geoipReadersStatus().asnLoaded);
+ const initial = { country: sha(GEOIP_COUNTRY_FILE), asn: sha(GEOIP_ASN_FILE) };
+ const lookup = lookupGeoip('8.8.8.8'); assert.equal(lookup?.country, 'US'); assert.equal(lookup?.asn, 15169);
+ check('actual-two-downloads-valid-mmdb-and-lookup', { bytes: snapshot.bytes, hashes: initial, settings: await getGeoipSettings(), lookup: { country: lookup.country, asn: lookup.asn } });
+ const first = collectGeoipUpdateOnce({ force: true });
+ await assert.rejects(collectGeoipUpdateOnce({ force: true }), e => e.statusCode === 409);
+ assert.equal((await collectGeoipUpdateOnce()).skipped, true);
+ snapshot = await first; assert.deepEqual(snapshot.errors, []); assert.equal(snapshot.skippedUnchanged, 2); assert.equal(snapshot.downloaded, 0);
+ check('actual-conditional304-and-concurrent409', { skippedUnchanged: 2, hashesUnchanged: sha(GEOIP_COUNTRY_FILE) === initial.country && sha(GEOIP_ASN_FILE) === initial.asn });
+ fs.unlinkSync(file(GEOIP_COUNTRY_FILE));
+ const beforeMissing = report.mirrorRequests.length; snapshot = await collectGeoipUpdateOnce({ force: true });
+ assert.deepEqual(snapshot.errors, []); assert.equal(snapshot.downloaded, 1); assert.equal(snapshot.skippedUnchanged, 1);
+ assert(report.mirrorRequests.slice(beforeMissing).some(r => r.file === GEOIP_COUNTRY_FILE && !r.conditional && r.status === 200));
+ assert.equal(sha(GEOIP_COUNTRY_FILE), initial.country);
+ check('missing-country-refetched-without-conditional-header');
+ fs.writeFileSync(file(GEOIP_COUNTRY_FILE), 'Own corrupt GeoIP fixture');
+ snapshot = await collectGeoipUpdateOnce({ force: true }); assert.deepEqual(snapshot.errors, []); assert.equal(snapshot.downloaded, 1); assert.equal(sha(GEOIP_COUNTRY_FILE), initial.country);
+ check('corrupt-local-country-refetched-and-validated');
+ const lastSuccess = (await getGeoipSettings()).lastSuccessAt;
+ setGeoipFetchForTests(async (url, options) => String(url).endsWith(GEOIP_COUNTRY_FILE) ? new Response('Own invalid MMDB body', { status: 200 }) : observedFetch(url, options));
+ snapshot = await collectGeoipUpdateOnce({ force: true }); assert.equal(snapshot.errors.length, 1); assert.equal(snapshot.downloaded, 0); assert.equal(snapshot.skippedUnchanged, 1);
+ assert.equal(sha(GEOIP_COUNTRY_FILE), initial.country); assert.equal((await getGeoipSettings()).lastSuccessAt, lastSuccess);
+ assert((await getGeoipSettings()).lastError); assert(!fs.existsSync(file(GEOIP_COUNTRY_FILE) + '.tmp'));
+ check('controlled-invalid-body-preserves-file-and-last-success', { negativeCountryResponse: 'controlled-200-invalid-MMDB', realAsnStatus: 304 });
+ fs.unlinkSync(file(GEOIP_COUNTRY_FILE));
+ setGeoipFetchForTests(async (url, options) => String(url).endsWith(GEOIP_COUNTRY_FILE) ? new Response(null, { status: 304 }) : observedFetch(url, options));
+ snapshot = await collectGeoipUpdateOnce({ force: true }); assert.equal(snapshot.errors.length, 1); assert.equal(snapshot.skippedUnchanged, 1); assert(snapshot.errors[0].includes('304 без проверенной локальной базы'));
+ assert.equal((await getGeoipSettings()).lastSuccessAt, lastSuccess);
+ check('controlled-uncached304-is-error-with-last-success-retained');
+ setGeoipFetchForTests(observedFetch); snapshot = await collectGeoipUpdateOnce({ force: true }); assert.deepEqual(snapshot.errors, []); assert.equal(snapshot.downloaded, 1);
+ assert.equal(sha(GEOIP_COUNTRY_FILE), initial.country); assert.equal(sha(GEOIP_ASN_FILE), initial.asn);
+ const beforeSkip = report.mirrorRequests.length; assert.equal((await collectGeoipUpdateOnce()).skipped, true); assert.equal(report.mirrorRequests.length, beforeSkip);
+ check('actual-recovery-and-disabled-background-no-fetch'); report.passed = true;
+} catch (e) { report.failure = { name: e.name }; process.exitCode = 1; }
+finally { resetGeoipUpdateForTests(); await databaseBackgroundWork.drain(); await closePool(); save(); console.log(JSON.stringify(report)); }

backend/scripts/acceptance/run-geoip-mirror.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -euo pipefail
+umask 077
+cd /opt/mmapp-test
+mm242_db="mm_accept_166_$(date +%s%3N)"
+mm242_pass="$(openssl rand -hex 24)"
+mm242_role_created=false
+mm242_db_created=false
+cleanup242() {
+  docker rm mmapp-s3app242-owned >/dev/null 2>&1 || true
+  if $mm242_db_created; then docker exec mmapp-test-postgres psql -U mmapp -d mmapp -v ON_ERROR_STOP=1 -c "DROP DATABASE \"$mm242_db\" WITH (FORCE)"; fi
+  if $mm242_role_created; then docker exec mmapp-test-postgres psql -U mmapp -d mmapp -v ON_ERROR_STOP=1 -c "DROP ROLE \"$mm242_db\""; fi
+  remaining=$(docker exec mmapp-test-postgres psql -U mmapp -d mmapp -Atc "SELECT (SELECT count(*) FROM pg_database WHERE datname='$mm242_db')+(SELECT count(*) FROM pg_roles WHERE rolname='$mm242_db')")
+  [ "$remaining" = 0 ]
+  python3 - <<'PY'
+import json,pathlib
+p=pathlib.Path('/opt/mmapp-test/artifacts/backup242-checks/geoip-mirror242.json')
+if p.exists():
+ r=json.loads(p.read_text());r['ownedDatabaseAndRoleRemoved']=True;p.write_text(json.dumps(r,indent=2)+'\n')
+PY
+  printf 'OWNED_DATABASE_AND_ROLE_REMOVED\n'
+  unset mm242_pass
+}
+trap cleanup242 EXIT
+[[ "$mm242_db" =~ ^mm_accept_166_[0-9]+$ ]]
+existing=$(docker exec mmapp-test-postgres psql -U mmapp -d mmapp -Atc "SELECT (SELECT count(*) FROM pg_database WHERE datname='$mm242_db')+(SELECT count(*) FROM pg_roles WHERE rolname='$mm242_db')")
+[ "$existing" = 0 ]
+printf "CREATE ROLE \"%s\" LOGIN PASSWORD '%s' NOSUPERUSER NOCREATEDB NOCREATEROLE NOBYPASSRLS;\n" "$mm242_db" "$mm242_pass" | docker exec -i mmapp-test-postgres psql -U mmapp -d mmapp -v ON_ERROR_STOP=1
+mm242_role_created=true
+docker exec mmapp-test-postgres psql -U mmapp -d mmapp -v ON_ERROR_STOP=1 -c "CREATE DATABASE \"$mm242_db\" OWNER \"$mm242_db\""
+mm242_db_created=true
+docker run --name mmapp-s3app242-owned --network mmapp-test_default --cap-drop ALL --security-opt no-new-privileges -e "MMAPP_ORIGIN_BASELINE=${2:-false}" -e NODE_ENV=test -e "DATABASE_URL=postgres://$mm242_db:$mm242_pass@postgres:5432/$mm242_db" -e "MMAPP_OWN_SQL_DB=$mm242_db" -e "DATABASE_PATH=/tmp/$mm242_db/legacy.db" -e "GEOIP_DIR=/tmp/$mm242_db/geoip" -v /opt/mmapp-test/artifacts/backup242-checks:/checks --entrypoint node "${1:-mmapp-test/backend:ux-geoip242}" /checks/geoip-mirror.mjs