Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
930e42b0b0 | ||
|
|
16b4923bd7 | ||
|
|
1cfd062835 | ||
|
|
21233bd578 | ||
|
|
990cc739df | ||
|
|
3500bd4624 | ||
|
|
374575ec01 | ||
|
|
f57b430052 | ||
|
|
8a9d60a5a7 | ||
|
|
b963311b43 | ||
|
|
50bdb8232b | ||
|
|
ee8e24ffc6 | ||
|
|
44b94caacf | ||
|
|
cbb4b467ad | ||
|
|
e65cf0d958 | ||
|
|
4a57c91e29 | ||
|
|
2289107911 | ||
|
|
782097420d | ||
|
|
82382d90f2 | ||
|
|
5a16a45922 | ||
|
|
6a6f6cedbc | ||
|
|
9639a03bfe | ||
|
|
48c10b7436 | ||
|
|
4db6438245 | ||
|
|
fb108ec5ab |
@@ -1,3 +0,0 @@
|
|||||||
{
|
|
||||||
"extends": ["@commitlint/config-conventional"]
|
|
||||||
}
|
|
||||||
@@ -142,6 +142,8 @@ feat(web): add module create dialog on /modules
|
|||||||
| `.cursor/` | `chore` |
|
| `.cursor/` | `chore` |
|
||||||
| прочее в корне | `chore` |
|
| прочее в корне | `chore` |
|
||||||
|
|
||||||
|
**Запрещено:** несколько scope через запятую (`refactor(web, httpapi): …`) — semantic-release не распознает `type`, релиз не будет (см. [docs/releasing.md](../../docs/releasing.md)).
|
||||||
|
|
||||||
`type` определять по **содержимому diff**, не только по пути.
|
`type` определять по **содержимому diff**, не только по пути.
|
||||||
|
|
||||||
## Multi-change
|
## Multi-change
|
||||||
|
|||||||
@@ -303,6 +303,8 @@ jobs:
|
|||||||
cache-dependency-path: package-lock.json
|
cache-dependency-path: package-lock.json
|
||||||
- name: Install release tooling
|
- name: Install release tooling
|
||||||
run: npm ci
|
run: npm ci
|
||||||
|
- name: Verify releasable commit messages
|
||||||
|
run: node scripts/commit/verify-release-commits.mjs
|
||||||
- name: Semantic release
|
- name: Semantic release
|
||||||
run: npx semantic-release
|
run: npx semantic-release
|
||||||
env:
|
env:
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
/** @type {import('@commitlint/types').UserConfig} */
|
||||||
|
module.exports = {
|
||||||
|
extends: ['@commitlint/config-conventional'],
|
||||||
|
plugins: [
|
||||||
|
{
|
||||||
|
rules: {
|
||||||
|
'scope-no-commas': ({ scope }) => {
|
||||||
|
if (scope && scope.includes(',')) {
|
||||||
|
return [
|
||||||
|
false,
|
||||||
|
'scope must not contain commas (semantic-release will not parse the commit type)'
|
||||||
|
];
|
||||||
|
}
|
||||||
|
return [true];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
rules: {
|
||||||
|
'scope-no-commas': [2, 'always']
|
||||||
|
}
|
||||||
|
};
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
# Диагностика схемы PostgreSQL (EvoBGP)
|
||||||
|
|
||||||
|
Runbook для оценки объёма БД и узких мест **перед** и **после** миграций оптимизации схемы. Выполнять на staging или production read-only сессией.
|
||||||
|
|
||||||
|
## Подключение
|
||||||
|
|
||||||
|
```bash
|
||||||
|
psql "$EVOBGP_DATABASE_URL"
|
||||||
|
```
|
||||||
|
|
||||||
|
## 1. Размеры таблиц и индексов
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT relname,
|
||||||
|
pg_size_pretty(pg_total_relation_size(relid)) AS total,
|
||||||
|
pg_size_pretty(pg_relation_size(relid)) AS heap,
|
||||||
|
pg_size_pretty(pg_indexes_size(relid)) AS indexes
|
||||||
|
FROM pg_catalog.pg_statio_user_tables
|
||||||
|
ORDER BY pg_total_relation_size(relid) DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
**Ожидание:** лидеры — `revision_materialized_prefix`, `config_revision` (TOAST от preview), JSONB-кэши.
|
||||||
|
|
||||||
|
## 2. Seq scan (горячие таблицы)
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT schemaname, relname, seq_scan, seq_tup_read, idx_scan
|
||||||
|
FROM pg_stat_user_tables
|
||||||
|
WHERE schemaname = 'public'
|
||||||
|
ORDER BY seq_tup_read DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
Сброс статистики после деплоя: `SELECT pg_stat_reset();` (только осознанно, теряется baseline).
|
||||||
|
|
||||||
|
## 3. Неиспользуемые индексы
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT indexrelname, idx_scan, pg_size_pretty(pg_relation_size(indexrelid)) AS size
|
||||||
|
FROM pg_stat_user_indexes
|
||||||
|
WHERE schemaname = 'public' AND idx_scan = 0
|
||||||
|
ORDER BY pg_relation_size(indexrelid) DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
## 4. Дубликаты в materialized prefixes
|
||||||
|
|
||||||
|
Перед UNIQUE `(revision_id, prefix, community_id, source)`:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT revision_id, prefix, community_id, source, COUNT(*) AS n
|
||||||
|
FROM revision_materialized_prefix
|
||||||
|
GROUP BY 1, 2, 3, 4
|
||||||
|
HAVING COUNT(*) > 1
|
||||||
|
LIMIT 20;
|
||||||
|
```
|
||||||
|
|
||||||
|
## 5. Шаблон отчёта staging
|
||||||
|
|
||||||
|
| Метрика | До | После | Дата |
|
||||||
|
|---------|-----|-------|------|
|
||||||
|
| `revision_materialized_prefix` total | | | |
|
||||||
|
| `config_revision` total | | | |
|
||||||
|
| `module_prefix_snapshot` total | | | |
|
||||||
|
| `asn_prefix_cache` total | | | |
|
||||||
|
| Top seq_scan table | | | |
|
||||||
|
| Unused indexes (count) | | | |
|
||||||
|
|
||||||
|
## 6. EXPLAIN для типовых запросов
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Список префиксов ревизии (keyset)
|
||||||
|
EXPLAIN (ANALYZE, BUFFERS)
|
||||||
|
SELECT prefix::text, community_id::text, source
|
||||||
|
FROM revision_materialized_prefix
|
||||||
|
WHERE revision_id = '<revision-uuid>'::uuid
|
||||||
|
ORDER BY id
|
||||||
|
LIMIT 51;
|
||||||
|
|
||||||
|
-- Diff added (anti-join)
|
||||||
|
EXPLAIN (ANALYZE, BUFFERS)
|
||||||
|
SELECT b.prefix::text
|
||||||
|
FROM revision_materialized_prefix b
|
||||||
|
LEFT JOIN revision_materialized_prefix a
|
||||||
|
ON a.revision_id = '<rev-a>'::uuid AND a.prefix = b.prefix
|
||||||
|
WHERE b.revision_id = '<rev-b>'::uuid
|
||||||
|
AND a.prefix IS NULL
|
||||||
|
ORDER BY b.prefix
|
||||||
|
LIMIT 5001;
|
||||||
|
```
|
||||||
|
|
||||||
|
Цель: Index Scan / Bitmap Index Scan по `(revision_id, …)`, без Seq Scan на больших таблицах.
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
# Production checklist
|
||||||
|
|
||||||
|
Краткий чеклист перед выводом EvoBGP в production (10+ клиентов, нестабильная сеть).
|
||||||
|
|
||||||
|
## Обязательно
|
||||||
|
|
||||||
|
- `EVOBGP_SEED_DEMO=0` — отключить demo-tenant и токен `Bearer dev`.
|
||||||
|
- `EVOBGP_DEV_INSECURE` не задавать или `0` — не использовать lab-флаги в prod.
|
||||||
|
- `EVOBGP_BUNDLE_SEED_HEX` — задать стабильный hex-ключ подписи бандлов; сохранить pubkey для нод.
|
||||||
|
- PostgreSQL с TLS (`sslmode` не `disable`) при доступе вне private network.
|
||||||
|
- `EVOBGP_CORS_ORIGINS` — явный whitelist origin веб-панели.
|
||||||
|
- `EVOBGP_STALE_ON_UPSTREAM_ERROR=1` (по умолчанию) — stale snapshot при сбоях CDN/ASN/DoH.
|
||||||
|
|
||||||
|
## Рекомендуется
|
||||||
|
|
||||||
|
- `EVOBGP_JOB_MAX_CONCURRENT=16`, `EVOBGP_DB_MAX_CONNS=25`, `EVOBGP_COLLECT_CONCURRENCY=16` при росте tenants.
|
||||||
|
- `EVOBGP_NODE_DISPATCH_INSECURE_TLS=0` — только валидный TLS к agent.
|
||||||
|
- Ограничить `/metrics` сетевой политикой или reverse proxy.
|
||||||
|
- Профиль `evobgp-all` или HA API + персистентная `job_audit` (PostgreSQL).
|
||||||
|
- Мониторинг drift: `evobgp-deploy`, `last_applied_revision_id` vs published.
|
||||||
|
|
||||||
|
## Не использовать в prod
|
||||||
|
|
||||||
|
- `EVOBGP_CDN_ALLOW_PRIVATE=1` — только тесты/lab.
|
||||||
|
- Plaintext `EVOBGP_API_KEYS` без ротации (break-glass — временно).
|
||||||
|
- Ручное редактирование `evobgp_*.conf` на нодах без ревизии.
|
||||||
@@ -11,6 +11,8 @@ EvoBGP использует [Conventional Commits](https://www.conventionalcommi
|
|||||||
| `feat!`, `fix!` или `BREAKING CHANGE:` в теле | major (1.0.0 → 2.0.0) |
|
| `feat!`, `fix!` или `BREAKING CHANGE:` в теле | major (1.0.0 → 2.0.0) |
|
||||||
| `docs`, `chore`, `test` | без релиза |
|
| `docs`, `chore`, `test` | без релиза |
|
||||||
|
|
||||||
|
**Scope:** один идентификатор **без запятых** (`web`, `httpapi`, `api`). Заголовок `refactor(a, b): …` **не парсится** semantic-release → релиз не создаётся (commitlint на PR это тоже отклонит). Подробнее — раздел «Scope и semantic-release» ниже.
|
||||||
|
|
||||||
`refactor` — patch без новых функций: перестройка кода/UI при том же поведении для пользователя. По semver на одном уровне с `fix`, но семантически «мельче» `feat` (не minor).
|
`refactor` — patch без новых функций: перестройка кода/UI при том же поведении для пользователя. По semver на одном уровне с `fix`, но семантически «мельче» `feat` (не minor).
|
||||||
|
|
||||||
Отдельного суффикса `1.x.y.fix` в semver нет: «fix» в Conventional Commits означает **patch** (третья цифра). Для починки пайплайна без смены продукта — `fix(ci):` или `ci:` (оба дают patch после настройки `.releaserc.json`).
|
Отдельного суффикса `1.x.y.fix` в semver нет: «fix» в Conventional Commits означает **patch** (третья цифра). Для починки пайплайна без смены продукта — `fix(ci):` или `ci:` (оба дают patch после настройки `.releaserc.json`).
|
||||||
@@ -62,6 +64,21 @@ API: `GET /version`, `GET /v1/version` — поля `version`, `git_sha`, `build
|
|||||||
|
|
||||||
Web UI показывает версию из API (footer sidebar, страница «Мониторинг»).
|
Web UI показывает версию из API (footer sidebar, страница «Мониторинг»).
|
||||||
|
|
||||||
|
## Scope и semantic-release
|
||||||
|
|
||||||
|
Парсер [conventional-commits-parser](https://github.com/conventional-changelog/conventional-changelog/tree/master/packages/conventional-commits-parser) (его использует semantic-release) **не понимает запятые в scope**:
|
||||||
|
|
||||||
|
| Заголовок | Парсится | Релиз |
|
||||||
|
|-----------|----------|-------|
|
||||||
|
| `refactor(web): fix layout` | да, `refactor` | patch |
|
||||||
|
| `refactor(NetworkOverviewTab, NetworkSpeakersCard): fix layout` | **нет**, `type: null` | **нет** |
|
||||||
|
|
||||||
|
Правило: **один scope** из таблицы в [.cursor/rules/conventional-commits.mdc](../.cursor/rules/conventional-commits.mdc) (`web`, `httpapi`, `api`, …).
|
||||||
|
|
||||||
|
На push в `main` job **release** запускает `scripts/commit/verify-release-commits.mjs` — в логе будут предупреждения о непарсящихся коммитах.
|
||||||
|
|
||||||
|
Если релиз «не создался», а CI зелёный: смотрите лог release — часто `No releasable commits`. Исправление: новый коммит с корректным заголовком (например `refactor(web): …`).
|
||||||
|
|
||||||
## CHANGELOG
|
## CHANGELOG
|
||||||
|
|
||||||
Release notes — в Gitea Release; файл `CHANGELOG.md` генерируется в CI и прикрепляется как asset, **не** попадает в git history.
|
Release notes — в Gitea Release; файл `CHANGELOG.md` генерируется в CI и прикрепляется как asset, **не** попадает в git history.
|
||||||
|
|||||||
@@ -14,6 +14,8 @@ import (
|
|||||||
"evobgp/internal/nodecli"
|
"evobgp/internal/nodecli"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const upstreamErrorDetail = "upstream request failed"
|
||||||
|
|
||||||
// Config holds evobgp-agent serve settings.
|
// Config holds evobgp-agent serve settings.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Listen string
|
Listen string
|
||||||
@@ -85,7 +87,7 @@ func (s *Server) handleBirdProtocols(w http.ResponseWriter, r *http.Request) {
|
|||||||
out, err := birdfmt.ShowProtocols(ctx, sock, strings.TrimSpace(s.cfg.BirdcBin))
|
out, err := birdfmt.ShowProtocols(ctx, sock, strings.TrimSpace(s.cfg.BirdcBin))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("agentserver: bird protocols: %v", err)
|
log.Printf("agentserver: bird protocols: %v", err)
|
||||||
writeProblem(w, http.StatusBadGateway, err.Error())
|
writeProblem(w, http.StatusBadGateway, upstreamErrorDetail)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
writeJSON(w, http.StatusOK, map[string]any{
|
writeJSON(w, http.StatusOK, map[string]any{
|
||||||
@@ -126,7 +128,7 @@ func (s *Server) handleSync(w http.ResponseWriter, r *http.Request) {
|
|||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("agentserver: sync: %v", err)
|
log.Printf("agentserver: sync: %v", err)
|
||||||
writeProblem(w, http.StatusBadGateway, err.Error())
|
writeProblem(w, http.StatusBadGateway, upstreamErrorDetail)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if s.cfg.OnSyncSuccess != nil {
|
if s.cfg.OnSyncSuccess != nil {
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/httpclient"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DefaultRIPEStatURL is the RIPEstat announced-prefixes data call (no API key).
|
// DefaultRIPEStatURL is the RIPEstat announced-prefixes data call (no API key).
|
||||||
@@ -24,7 +26,7 @@ const DefaultASOverviewURL = "https://stat.ripe.net/data/as-overview/data.json"
|
|||||||
// AnnouncedPrefixes returns currently announced IPv4/IPv6 prefixes for the ASN (best-effort via RIPEstat).
|
// AnnouncedPrefixes returns currently announced IPv4/IPv6 prefixes for the ASN (best-effort via RIPEstat).
|
||||||
func AnnouncedPrefixes(ctx context.Context, hc *http.Client, asn int64) ([]netip.Prefix, error) {
|
func AnnouncedPrefixes(ctx context.Context, hc *http.Client, asn int64) ([]netip.Prefix, error) {
|
||||||
if hc == nil {
|
if hc == nil {
|
||||||
hc = http.DefaultClient
|
hc = httpclient.New(httpclient.DefaultTimeout)
|
||||||
}
|
}
|
||||||
base := strings.TrimSpace(os.Getenv("EVOBGP_RIPESTAT_ANNOUNCED_PREFIXES_URL"))
|
base := strings.TrimSpace(os.Getenv("EVOBGP_RIPESTAT_ANNOUNCED_PREFIXES_URL"))
|
||||||
if base == "" {
|
if base == "" {
|
||||||
@@ -38,7 +40,7 @@ func AnnouncedPrefixes(ctx context.Context, hc *http.Client, asn int64) ([]netip
|
|||||||
req.Header.Set("Accept", "application/json")
|
req.Header.Set("Accept", "application/json")
|
||||||
req.Header.Set("User-Agent", "evobgp-asnresolve/1.0")
|
req.Header.Set("User-Agent", "evobgp-asnresolve/1.0")
|
||||||
|
|
||||||
resp, err := hc.Do(req)
|
resp, err := httpclient.DoWithBreaker(ctx, hc, req, 3)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("ripestat fetch AS%d: %w", asn, err)
|
return nil, fmt.Errorf("ripestat fetch AS%d: %w", asn, err)
|
||||||
}
|
}
|
||||||
@@ -86,7 +88,7 @@ func AnnouncedPrefixes(ctx context.Context, hc *http.Client, asn int64) ([]netip
|
|||||||
// ASHolderName returns the holder / organization label for the ASN from RIPEstat as-overview (best-effort).
|
// ASHolderName returns the holder / organization label for the ASN from RIPEstat as-overview (best-effort).
|
||||||
func ASHolderName(ctx context.Context, hc *http.Client, asn int64) (string, error) {
|
func ASHolderName(ctx context.Context, hc *http.Client, asn int64) (string, error) {
|
||||||
if hc == nil {
|
if hc == nil {
|
||||||
hc = http.DefaultClient
|
hc = httpclient.New(httpclient.DefaultTimeout)
|
||||||
}
|
}
|
||||||
base := strings.TrimSpace(os.Getenv("EVOBGP_RIPESTAT_AS_OVERVIEW_URL"))
|
base := strings.TrimSpace(os.Getenv("EVOBGP_RIPESTAT_AS_OVERVIEW_URL"))
|
||||||
if base == "" {
|
if base == "" {
|
||||||
@@ -100,7 +102,7 @@ func ASHolderName(ctx context.Context, hc *http.Client, asn int64) (string, erro
|
|||||||
req.Header.Set("Accept", "application/json")
|
req.Header.Set("Accept", "application/json")
|
||||||
req.Header.Set("User-Agent", "evobgp-asnresolve/1.0")
|
req.Header.Set("User-Agent", "evobgp-asnresolve/1.0")
|
||||||
|
|
||||||
resp, err := hc.Do(req)
|
resp, err := httpclient.DoWithBreaker(ctx, hc, req, 3)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("ripestat as-overview AS%d: %w", asn, err)
|
return "", fmt.Errorf("ripestat as-overview AS%d: %w", asn, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/db"
|
"evobgp/internal/db"
|
||||||
|
"evobgp/internal/httpclient"
|
||||||
"evobgp/internal/jobs"
|
"evobgp/internal/jobs"
|
||||||
"evobgp/internal/observability"
|
"evobgp/internal/observability"
|
||||||
"evobgp/internal/repository"
|
"evobgp/internal/repository"
|
||||||
@@ -17,7 +18,7 @@ import (
|
|||||||
|
|
||||||
// NewCDNHTTPClient returns the shared HTTP client for CDN and preview fetches (PERF-02 / ERR-03).
|
// NewCDNHTTPClient returns the shared HTTP client for CDN and preview fetches (PERF-02 / ERR-03).
|
||||||
func NewCDNHTTPClient() *http.Client {
|
func NewCDNHTTPClient() *http.Client {
|
||||||
return &http.Client{Timeout: 45 * time.Second}
|
return httpclient.New(httpclient.DefaultTimeout)
|
||||||
}
|
}
|
||||||
|
|
||||||
// BootstrapWorkers opens the same store.Backend and jobs.Registry as New (without HTTP or bundle keys).
|
// BootstrapWorkers opens the same store.Backend and jobs.Registry as New (without HTTP or bundle keys).
|
||||||
@@ -55,18 +56,40 @@ func BootstrapWorkers(ctx context.Context, opts Options) (store.Backend, *jobs.R
|
|||||||
wk.Registry = reg
|
wk.Registry = reg
|
||||||
if pool != nil {
|
if pool != nil {
|
||||||
audit := repository.NewJobAuditWriter(pool)
|
audit := repository.NewJobAuditWriter(pool)
|
||||||
reg.SetTerminalHook(func(j *jobs.Job) {
|
jobMeta := func(j *jobs.Job) map[string]any {
|
||||||
if j == nil {
|
if j == nil {
|
||||||
return
|
return nil
|
||||||
}
|
}
|
||||||
st := j.Snapshot()
|
st := j.Snapshot()
|
||||||
status, _ := st["status"].(string)
|
meta, _ := st["meta"].(map[string]any)
|
||||||
var errMsg *string
|
return meta
|
||||||
if e, ok := st["error"].(string); ok && e != "" {
|
}
|
||||||
errMsg = &e
|
reg.SetPersistHooks(
|
||||||
}
|
func(j *jobs.Job) {
|
||||||
audit.MarkTerminal(context.Background(), j.TenantID, j.ID, status, errMsg, time.Now().UTC())
|
if j == nil {
|
||||||
})
|
return
|
||||||
|
}
|
||||||
|
audit.UpsertQueued(context.Background(), j.TenantID, j.ID, j.Kind, j.IdempotencyKey, j.ModuleID, jobMeta(j))
|
||||||
|
},
|
||||||
|
func(j *jobs.Job) {
|
||||||
|
if j == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
audit.UpsertRunning(context.Background(), j.TenantID, j.ID, j.Kind, j.IdempotencyKey, jobMeta(j))
|
||||||
|
},
|
||||||
|
func(j *jobs.Job) {
|
||||||
|
if j == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
st := j.Snapshot()
|
||||||
|
status, _ := st["status"].(string)
|
||||||
|
var errMsg *string
|
||||||
|
if e, ok := st["error"].(string); ok && e != "" {
|
||||||
|
errMsg = &e
|
||||||
|
}
|
||||||
|
audit.MarkTerminal(context.Background(), j.TenantID, j.ID, status, errMsg, time.Now().UTC())
|
||||||
|
},
|
||||||
|
)
|
||||||
}
|
}
|
||||||
observability.RegisterStoreBackend(backend)
|
observability.RegisterStoreBackend(backend)
|
||||||
return backend, reg, pool, nil
|
return backend, reg, pool, nil
|
||||||
|
|||||||
@@ -579,7 +579,7 @@ func (s *Server) handleGetRevision(w http.ResponseWriter, r *http.Request) {
|
|||||||
if !s.requireAtLeast(w, a, "viewer") {
|
if !s.requireAtLeast(w, a, "viewer") {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
rev, err := s.store.GetRevision(a.TenantID, r.PathValue("revision_id"))
|
rev, err := s.store.GetRevisionSummary(a.TenantID, r.PathValue("revision_id"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeProblem(w, http.StatusNotFound, "Not Found", "revision not found")
|
writeProblem(w, http.StatusNotFound, "Not Found", "revision not found")
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -242,6 +242,14 @@ func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request)
|
|||||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "url is required")
|
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "url is required")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if _, err := pipeline.ValidateCDNURL(u); err != nil {
|
||||||
|
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := pipeline.ResolveCDNURLHost(r.Context(), u); err != nil {
|
||||||
|
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
|
||||||
|
return
|
||||||
|
}
|
||||||
mod, err := s.store.GetModule(a.TenantID, r.PathValue("module_id"))
|
mod, err := s.store.GetModule(a.TenantID, r.PathValue("module_id"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeStoreErr(w, err)
|
writeStoreErr(w, err)
|
||||||
@@ -304,6 +312,16 @@ func (s *Server) handlePostCDNSource(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeProblem(w, http.StatusBadRequest, "Bad Request", "invalid json")
|
writeProblem(w, http.StatusBadRequest, "Bad Request", "invalid json")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if body.URL != "" {
|
||||||
|
if _, err := pipeline.ValidateCDNURL(body.URL); err != nil {
|
||||||
|
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := pipeline.ResolveCDNURLHost(r.Context(), body.URL); err != nil {
|
||||||
|
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
mid := r.PathValue("module_id")
|
mid := r.PathValue("module_id")
|
||||||
x, err := s.store.CreateCDNSource(a.TenantID, mid, &body)
|
x, err := s.store.CreateCDNSource(a.TenantID, mid, &body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -324,6 +342,16 @@ func (s *Server) handlePatchCDNSource(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeProblem(w, http.StatusBadRequest, "Bad Request", "invalid json")
|
writeProblem(w, http.StatusBadRequest, "Bad Request", "invalid json")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if body.URL != nil && strings.TrimSpace(*body.URL) != "" {
|
||||||
|
if _, err := pipeline.ValidateCDNURL(*body.URL); err != nil {
|
||||||
|
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := pipeline.ResolveCDNURLHost(r.Context(), *body.URL); err != nil {
|
||||||
|
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
mid := r.PathValue("module_id")
|
mid := r.PathValue("module_id")
|
||||||
x, err := s.store.UpdateCDNSource(a.TenantID, mid, r.PathValue("source_id"), &body)
|
x, err := s.store.UpdateCDNSource(a.TenantID, mid, r.PathValue("source_id"), &body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -39,7 +39,10 @@ func speakerJSONFromStore(st store.Backend, sp *store.Speaker) map[string]any {
|
|||||||
if strings.TrimSpace(sp.MetaJSON) != "" && sp.MetaJSON != "{}" {
|
if strings.TrimSpace(sp.MetaJSON) != "" && sp.MetaJSON != "{}" {
|
||||||
var raw map[string]any
|
var raw map[string]any
|
||||||
if json.Unmarshal([]byte(sp.MetaJSON), &raw) == nil {
|
if json.Unmarshal([]byte(sp.MetaJSON), &raw) == nil {
|
||||||
m["meta_json"] = raw
|
delete(raw, "agent_secret")
|
||||||
|
if len(raw) > 0 {
|
||||||
|
m["meta_json"] = raw
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if meta.AgentDomain != "" {
|
if meta.AgentDomain != "" {
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGetSpeaker_redactsAgentSecret(t *testing.T) {
|
||||||
|
srv, err := New(Options{InsecureDev: true, SeedDemo: true, BundleSeedHex: testBundleSeed})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer srv.Close()
|
||||||
|
tenant, _, _, _, demoSpk := srv.Store().DemoIDs()
|
||||||
|
mustSetTestAPIKeys(t, srv, "vwkey|"+tenant+"|viewer")
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/v1/speakers/"+demoSpk, nil)
|
||||||
|
req.Header.Set("Authorization", "Bearer vwkey")
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
srv.Handler().ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status %d body %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
var out map[string]any
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if out["agent_secret"] != nil {
|
||||||
|
t.Fatalf("agent_secret must not appear at top level: %#v", out["agent_secret"])
|
||||||
|
}
|
||||||
|
meta, _ := out["meta_json"].(map[string]any)
|
||||||
|
if meta != nil {
|
||||||
|
if v, ok := meta["agent_secret"]; ok && v != nil && v != "" {
|
||||||
|
t.Fatalf("agent_secret must be redacted from meta_json: %#v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListSpeakers_redactsAgentSecret(t *testing.T) {
|
||||||
|
srv, err := New(Options{InsecureDev: true, SeedDemo: true, BundleSeedHex: testBundleSeed})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer srv.Close()
|
||||||
|
tenant, _, _, _, _ := srv.Store().DemoIDs()
|
||||||
|
mustSetTestAPIKeys(t, srv, "vwkey|"+tenant+"|viewer")
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/v1/speakers", nil)
|
||||||
|
req.Header.Set("Authorization", "Bearer vwkey")
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
srv.Handler().ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status %d body %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
body := rec.Body.String()
|
||||||
|
if strings.Contains(body, "agent_secret") {
|
||||||
|
t.Fatalf("list response must not contain agent_secret: %s", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
package httpclient
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
defaultBreakerThreshold = 5
|
||||||
|
defaultBreakerCooldown = 30 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
type hostBreaker struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
failures int
|
||||||
|
openUntil time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
var hostBreakers sync.Map // string -> *hostBreaker
|
||||||
|
|
||||||
|
func breakerForHost(host string) *hostBreaker {
|
||||||
|
if host == "" {
|
||||||
|
host = "_"
|
||||||
|
}
|
||||||
|
v, _ := hostBreakers.LoadOrStore(host, &hostBreaker{})
|
||||||
|
return v.(*hostBreaker)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *hostBreaker) allow() bool {
|
||||||
|
b.mu.Lock()
|
||||||
|
defer b.mu.Unlock()
|
||||||
|
return time.Now().After(b.openUntil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *hostBreaker) recordSuccess() {
|
||||||
|
b.mu.Lock()
|
||||||
|
defer b.mu.Unlock()
|
||||||
|
b.failures = 0
|
||||||
|
b.openUntil = time.Time{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *hostBreaker) recordFailure() {
|
||||||
|
b.mu.Lock()
|
||||||
|
defer b.mu.Unlock()
|
||||||
|
b.failures++
|
||||||
|
if b.failures >= defaultBreakerThreshold {
|
||||||
|
b.openUntil = time.Now().Add(defaultBreakerCooldown)
|
||||||
|
b.failures = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResetHostBreakers clears all circuit breakers (tests only).
|
||||||
|
func ResetHostBreakers() {
|
||||||
|
hostBreakers = sync.Map{}
|
||||||
|
}
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
package httpclient
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDoWithBreaker_opensAfterFailures(t *testing.T) {
|
||||||
|
ResetHostBreakers()
|
||||||
|
var calls atomic.Int32
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
calls.Add(1)
|
||||||
|
http.Error(w, "fail", http.StatusBadGateway)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
hc := New(5 * time.Second)
|
||||||
|
for i := 0; i < defaultBreakerThreshold*3; i++ {
|
||||||
|
req, _ := http.NewRequest(http.MethodGet, srv.URL, nil)
|
||||||
|
_, _ = DoWithBreaker(context.Background(), hc, req, 1)
|
||||||
|
}
|
||||||
|
req, _ := http.NewRequest(http.MethodGet, srv.URL, nil)
|
||||||
|
_, err := DoWithBreaker(context.Background(), hc, req, 1)
|
||||||
|
if err == nil || err.Error() == "" {
|
||||||
|
t.Fatal("expected circuit open error")
|
||||||
|
}
|
||||||
|
if got := calls.Load(); got == 0 {
|
||||||
|
t.Fatal("expected at least one upstream call")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
// Package httpclient provides shared HTTP clients and retry helpers for outbound calls.
|
||||||
|
package httpclient
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const DefaultTimeout = 45 * time.Second
|
||||||
|
|
||||||
|
// New returns an HTTP client with timeout and tuned idle connection pooling.
|
||||||
|
func New(timeout time.Duration) *http.Client {
|
||||||
|
if timeout <= 0 {
|
||||||
|
timeout = DefaultTimeout
|
||||||
|
}
|
||||||
|
tr := http.DefaultTransport.(*http.Transport).Clone()
|
||||||
|
tr.MaxIdleConns = 100
|
||||||
|
tr.MaxIdleConnsPerHost = 10
|
||||||
|
return &http.Client{Timeout: timeout, Transport: tr}
|
||||||
|
}
|
||||||
|
|
||||||
|
// DoWithRetry executes hc.Do(req) up to maxAttempts times with linear backoff.
|
||||||
|
func DoWithRetry(ctx context.Context, hc *http.Client, req *http.Request, maxAttempts int) (*http.Response, error) {
|
||||||
|
if maxAttempts <= 0 {
|
||||||
|
maxAttempts = 3
|
||||||
|
}
|
||||||
|
var lastErr error
|
||||||
|
for attempt := 0; attempt < maxAttempts; attempt++ {
|
||||||
|
if attempt > 0 {
|
||||||
|
wait := time.Duration(attempt) * 2 * time.Second
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
case <-time.After(wait):
|
||||||
|
}
|
||||||
|
if req.GetBody != nil {
|
||||||
|
body, err := req.GetBody()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
req.Body = body
|
||||||
|
}
|
||||||
|
}
|
||||||
|
reqClone := req.Clone(ctx)
|
||||||
|
resp, err := hc.Do(reqClone)
|
||||||
|
if err != nil {
|
||||||
|
lastErr = err
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if resp.StatusCode >= 500 {
|
||||||
|
_, _ = io.Copy(io.Discard, resp.Body)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
lastErr = fmt.Errorf("httpclient: upstream %s", resp.Status)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
if lastErr != nil {
|
||||||
|
return nil, lastErr
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("httpclient: request failed after %d attempts", maxAttempts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DoWithBreaker applies per-host circuit breaking then retries transient failures.
|
||||||
|
func DoWithBreaker(ctx context.Context, hc *http.Client, req *http.Request, maxAttempts int) (*http.Response, error) {
|
||||||
|
if req == nil || req.URL == nil {
|
||||||
|
return nil, fmt.Errorf("httpclient: nil request")
|
||||||
|
}
|
||||||
|
br := breakerForHost(req.URL.Hostname())
|
||||||
|
if !br.allow() {
|
||||||
|
return nil, fmt.Errorf("httpclient: circuit open for %s", req.URL.Hostname())
|
||||||
|
}
|
||||||
|
resp, err := DoWithRetry(ctx, hc, req, maxAttempts)
|
||||||
|
if err != nil {
|
||||||
|
br.recordFailure()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if resp.StatusCode >= 500 {
|
||||||
|
br.recordFailure()
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
br.recordSuccess()
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package httpclient
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDoWithRetry_retriesOn500(t *testing.T) {
|
||||||
|
var calls int
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
calls++
|
||||||
|
if calls < 3 {
|
||||||
|
http.Error(w, "fail", http.StatusBadGateway)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
_, _ = w.Write([]byte("ok"))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
req, err := http.NewRequest(http.MethodGet, srv.URL, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
resp, err := DoWithRetry(context.Background(), New(5*time.Second), req, 3)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("status %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
if calls != 3 {
|
||||||
|
t.Fatalf("want 3 calls, got %d", calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
+12
-3
@@ -3,11 +3,11 @@ package ingest
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/broker"
|
"evobgp/internal/broker"
|
||||||
"evobgp/internal/config"
|
"evobgp/internal/config"
|
||||||
|
"evobgp/internal/httpclient"
|
||||||
"evobgp/internal/pipeline"
|
"evobgp/internal/pipeline"
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
)
|
)
|
||||||
@@ -17,6 +17,8 @@ type Deps struct {
|
|||||||
Store store.Backend
|
Store store.Backend
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var lastMaintenance time.Time
|
||||||
|
|
||||||
// Run blocks until ctx is cancelled.
|
// Run blocks until ctx is cancelled.
|
||||||
func Run(ctx context.Context, deps *Deps) {
|
func Run(ctx context.Context, deps *Deps) {
|
||||||
cfg := config.Load()
|
cfg := config.Load()
|
||||||
@@ -24,7 +26,7 @@ func Run(ctx context.Context, deps *Deps) {
|
|||||||
if deps == nil || deps.Store == nil {
|
if deps == nil || deps.Store == nil {
|
||||||
log.Fatalf("evobgp-ingest: missing store (pass ingest.Deps from BootstrapWorkers or evobgp-all)")
|
log.Fatalf("evobgp-ingest: missing store (pass ingest.Deps from BootstrapWorkers or evobgp-all)")
|
||||||
}
|
}
|
||||||
hc := &http.Client{Timeout: 45 * time.Second}
|
hc := httpclient.New(httpclient.DefaultTimeout)
|
||||||
t := time.NewTicker(60 * time.Second)
|
t := time.NewTicker(60 * time.Second)
|
||||||
defer t.Stop()
|
defer t.Stop()
|
||||||
log.Printf("evobgp-ingest: active (CDN conditional GET / ETag prefetch)")
|
log.Printf("evobgp-ingest: active (CDN conditional GET / ETag prefetch)")
|
||||||
@@ -34,7 +36,14 @@ func Run(ctx context.Context, deps *Deps) {
|
|||||||
log.Printf("evobgp-ingest: stopped")
|
log.Printf("evobgp-ingest: stopped")
|
||||||
return
|
return
|
||||||
case <-t.C:
|
case <-t.C:
|
||||||
if err := pipeline.PrefetchCDNSourceETags(context.Background(), deps.Store, hc); err != nil {
|
if deps.Store != nil && time.Since(lastMaintenance) > time.Hour {
|
||||||
|
deps.Store.RunPeriodicMaintenance(ctx)
|
||||||
|
lastMaintenance = time.Now()
|
||||||
|
}
|
||||||
|
prefetchCtx, cancel := context.WithTimeout(ctx, 50*time.Second)
|
||||||
|
err := pipeline.PrefetchCDNSourceETags(prefetchCtx, deps.Store, hc)
|
||||||
|
cancel()
|
||||||
|
if err != nil {
|
||||||
log.Printf("evobgp-ingest: prefetch: %v", err)
|
log.Printf("evobgp-ingest: prefetch: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+49
-4
@@ -182,6 +182,8 @@ type Registry struct {
|
|||||||
workerStart func(j *Job)
|
workerStart func(j *Job)
|
||||||
workerSem chan struct{}
|
workerSem chan struct{}
|
||||||
onTerminal func(j *Job)
|
onTerminal func(j *Job)
|
||||||
|
onEnqueued func(j *Job)
|
||||||
|
onRunning func(j *Job)
|
||||||
}
|
}
|
||||||
|
|
||||||
type idempoKey struct {
|
type idempoKey struct {
|
||||||
@@ -209,6 +211,44 @@ func (r *Registry) SetTerminalHook(fn func(j *Job)) {
|
|||||||
r.onTerminal = fn
|
r.onTerminal = fn
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetPersistHooks registers best-effort callbacks for job lifecycle persistence.
|
||||||
|
func (r *Registry) SetPersistHooks(onEnqueued, onRunning, onTerminal func(j *Job)) {
|
||||||
|
if r == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
r.onEnqueued = onEnqueued
|
||||||
|
r.onRunning = onRunning
|
||||||
|
if onTerminal != nil {
|
||||||
|
r.onTerminal = onTerminal
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Registry) fireEnqueued(j *Job) {
|
||||||
|
if r == nil || j == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r.mu.RLock()
|
||||||
|
fn := r.onEnqueued
|
||||||
|
r.mu.RUnlock()
|
||||||
|
if fn != nil {
|
||||||
|
fn(j)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Registry) fireRunning(j *Job) {
|
||||||
|
if r == nil || j == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r.mu.RLock()
|
||||||
|
fn := r.onRunning
|
||||||
|
r.mu.RUnlock()
|
||||||
|
if fn != nil {
|
||||||
|
fn(j)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (r *Registry) fireTerminal(j *Job) {
|
func (r *Registry) fireTerminal(j *Job) {
|
||||||
if r == nil || j == nil {
|
if r == nil || j == nil {
|
||||||
return
|
return
|
||||||
@@ -271,8 +311,6 @@ func (r *Registry) pruneTerminalIfOver(maxJobs int) {
|
|||||||
// Enqueue creates a job or returns an existing one for the same idempotency key.
|
// Enqueue creates a job or returns an existing one for the same idempotency key.
|
||||||
func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, moduleID *string, meta map[string]any) (*Job, bool, error) {
|
func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, moduleID *string, meta map[string]any) (*Job, bool, error) {
|
||||||
r.mu.Lock()
|
r.mu.Lock()
|
||||||
defer r.mu.Unlock()
|
|
||||||
|
|
||||||
maxJobs := registryMaxJobsFromEnv()
|
maxJobs := registryMaxJobsFromEnv()
|
||||||
r.pruneTerminalIfOver(maxJobs)
|
r.pruneTerminalIfOver(maxJobs)
|
||||||
|
|
||||||
@@ -281,6 +319,7 @@ func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, module
|
|||||||
if existing, ok := r.byIdempo[k]; ok {
|
if existing, ok := r.byIdempo[k]; ok {
|
||||||
st := existing.statusLocked()
|
st := existing.statusLocked()
|
||||||
if st == StatusQueued || st == StatusRunning {
|
if st == StatusQueued || st == StatusRunning {
|
||||||
|
r.mu.Unlock()
|
||||||
return existing, false, nil
|
return existing, false, nil
|
||||||
}
|
}
|
||||||
delete(r.byIdempo, k)
|
delete(r.byIdempo, k)
|
||||||
@@ -302,8 +341,14 @@ func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, module
|
|||||||
}
|
}
|
||||||
r.byID[j.ID] = j
|
r.byID[j.ID] = j
|
||||||
r.pruneTerminalIfOver(maxJobs)
|
r.pruneTerminalIfOver(maxJobs)
|
||||||
|
enqueuedHook := r.onEnqueued
|
||||||
|
workerStart := r.workerStart
|
||||||
|
r.mu.Unlock()
|
||||||
|
|
||||||
if r.workerStart != nil {
|
if enqueuedHook != nil {
|
||||||
|
enqueuedHook(j)
|
||||||
|
}
|
||||||
|
if workerStart != nil {
|
||||||
go func() {
|
go func() {
|
||||||
r.workerSem <- struct{}{}
|
r.workerSem <- struct{}{}
|
||||||
active := len(r.workerSem)
|
active := len(r.workerSem)
|
||||||
@@ -313,7 +358,7 @@ func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, module
|
|||||||
<-r.workerSem
|
<-r.workerSem
|
||||||
observability.RecordJobQueueDepth(len(r.workerSem), capacity)
|
observability.RecordJobQueueDepth(len(r.workerSem), capacity)
|
||||||
}()
|
}()
|
||||||
r.workerStart(j)
|
workerStart(j)
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
return j, true, nil
|
return j, true, nil
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
|
|
||||||
"evobgp/internal/birddeploy"
|
"evobgp/internal/birddeploy"
|
||||||
"evobgp/internal/birdfmt"
|
"evobgp/internal/birdfmt"
|
||||||
|
"evobgp/internal/httpclient"
|
||||||
"evobgp/internal/nodedispatch"
|
"evobgp/internal/nodedispatch"
|
||||||
"evobgp/internal/observability"
|
"evobgp/internal/observability"
|
||||||
"evobgp/internal/pipeline"
|
"evobgp/internal/pipeline"
|
||||||
@@ -70,7 +71,7 @@ type revisionLogEntry struct {
|
|||||||
Message string `json:"message"`
|
Message string `json:"message"`
|
||||||
}
|
}
|
||||||
|
|
||||||
var defaultWorkerHTTP = &http.Client{Timeout: 45 * time.Second}
|
var defaultWorkerHTTP = httpclient.New(httpclient.DefaultTimeout)
|
||||||
|
|
||||||
func (w *Worker) httpClient() *http.Client {
|
func (w *Worker) httpClient() *http.Client {
|
||||||
if w != nil && w.HTTPClient != nil {
|
if w != nil && w.HTTPClient != nil {
|
||||||
@@ -94,6 +95,9 @@ func (w *Worker) Process(j *Job) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
j.MarkRunning()
|
j.MarkRunning()
|
||||||
|
if w != nil && w.Registry != nil {
|
||||||
|
w.Registry.fireRunning(j)
|
||||||
|
}
|
||||||
if j.IsCancelRequested() {
|
if j.IsCancelRequested() {
|
||||||
j.MarkCancelled()
|
j.MarkCancelled()
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import (
|
|||||||
|
|
||||||
"evobgp/internal/birdfmt"
|
"evobgp/internal/birdfmt"
|
||||||
"evobgp/internal/bundle"
|
"evobgp/internal/bundle"
|
||||||
|
"evobgp/internal/httpclient"
|
||||||
"evobgp/internal/signing"
|
"evobgp/internal/signing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -55,6 +56,10 @@ func CmdPullBundle(args []string) int {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func nodeHTTPClient() *http.Client {
|
||||||
|
return httpclient.New(60 * time.Second)
|
||||||
|
}
|
||||||
|
|
||||||
func fetchLatestRevision(base, token, speaker string) (string, error) {
|
func fetchLatestRevision(base, token, speaker string) (string, error) {
|
||||||
u := strings.TrimRight(base, "/") + "/v1/speakers/" + speaker + "/revisions/latest"
|
u := strings.TrimRight(base, "/") + "/v1/speakers/" + speaker + "/revisions/latest"
|
||||||
req, err := http.NewRequest(http.MethodGet, u, nil)
|
req, err := http.NewRequest(http.MethodGet, u, nil)
|
||||||
@@ -62,7 +67,9 @@ func fetchLatestRevision(base, token, speaker string) (string, error) {
|
|||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
req.Header.Set("Authorization", "Bearer "+token)
|
req.Header.Set("Authorization", "Bearer "+token)
|
||||||
resp, err := http.DefaultClient.Do(req)
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
resp, err := httpclient.DoWithRetry(ctx, nodeHTTPClient(), req, 3)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -90,7 +97,9 @@ func fetchBundle(base, token, speaker, revision string) ([]byte, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
req.Header.Set("Authorization", "Bearer "+token)
|
req.Header.Set("Authorization", "Bearer "+token)
|
||||||
resp, err := http.DefaultClient.Do(req)
|
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
resp, err := httpclient.DoWithRetry(ctx, nodeHTTPClient(), req, 3)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestCollectModulePrefixRows_CDNSendsIfNoneMatch(t *testing.T) {
|
func TestCollectModulePrefixRows_CDNSendsIfNoneMatch(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
|
||||||
m := store.NewMemory()
|
m := store.NewMemory()
|
||||||
m.SeedDemo()
|
m.SeedDemo()
|
||||||
tenant, _, _, _, _ := m.DemoIDs()
|
tenant, _, _, _, _ := m.DemoIDs()
|
||||||
@@ -25,7 +26,7 @@ func TestCollectModulePrefixRows_CDNSendsIfNoneMatch(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var gotIfNoneMatch string
|
var gotIfNoneMatch string
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
gotIfNoneMatch = strings.TrimSpace(r.Header.Get("If-None-Match"))
|
gotIfNoneMatch = strings.TrimSpace(r.Header.Get("If-None-Match"))
|
||||||
w.Header().Set("ETag", "etag-new")
|
w.Header().Set("ETag", "etag-new")
|
||||||
_, _ = w.Write([]byte("198.51.100.0/24\n"))
|
_, _ = w.Write([]byte("198.51.100.0/24\n"))
|
||||||
@@ -54,6 +55,7 @@ func TestCollectModulePrefixRows_CDNSendsIfNoneMatch(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCollectModulePrefixRows_CDN304UsesSnapshot(t *testing.T) {
|
func TestCollectModulePrefixRows_CDN304UsesSnapshot(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
|
||||||
m := store.NewMemory()
|
m := store.NewMemory()
|
||||||
m.SeedDemo()
|
m.SeedDemo()
|
||||||
tenant, _, _, _, _ := m.DemoIDs()
|
tenant, _, _, _, _ := m.DemoIDs()
|
||||||
@@ -67,7 +69,7 @@ func TestCollectModulePrefixRows_CDN304UsesSnapshot(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
w.WriteHeader(http.StatusNotModified)
|
w.WriteHeader(http.StatusNotModified)
|
||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
@@ -99,6 +101,7 @@ func TestCollectModulePrefixRows_CDN304UsesSnapshot(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRefreshModuleIngest_CDN304UsesStoredSnapshot(t *testing.T) {
|
func TestRefreshModuleIngest_CDN304UsesStoredSnapshot(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
|
||||||
m := store.NewMemory()
|
m := store.NewMemory()
|
||||||
m.SeedDemo()
|
m.SeedDemo()
|
||||||
tenant, _, _, _, _ := m.DemoIDs()
|
tenant, _, _, _, _ := m.DemoIDs()
|
||||||
@@ -112,7 +115,7 @@ func TestRefreshModuleIngest_CDN304UsesStoredSnapshot(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
w.WriteHeader(http.StatusNotModified)
|
w.WriteHeader(http.StatusNotModified)
|
||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
@@ -140,6 +143,7 @@ func TestRefreshModuleIngest_CDN304UsesStoredSnapshot(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCollectModulePrefixRows_CDN304RetriesWithoutETag(t *testing.T) {
|
func TestCollectModulePrefixRows_CDN304RetriesWithoutETag(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
|
||||||
m := store.NewMemory()
|
m := store.NewMemory()
|
||||||
m.SeedDemo()
|
m.SeedDemo()
|
||||||
tenant, _, _, _, _ := m.DemoIDs()
|
tenant, _, _, _, _ := m.DemoIDs()
|
||||||
@@ -154,7 +158,7 @@ func TestCollectModulePrefixRows_CDN304RetriesWithoutETag(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var calls int
|
var calls int
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
calls++
|
calls++
|
||||||
if calls == 1 {
|
if calls == 1 {
|
||||||
if got := strings.TrimSpace(r.Header.Get("If-None-Match")); got != "etag-stable" {
|
if got := strings.TrimSpace(r.Header.Get("If-None-Match")); got != "etag-stable" {
|
||||||
|
|||||||
@@ -119,6 +119,12 @@ func applyCDNSourceHTTPResult(ctx context.Context, st store.Backend, hc *http.Cl
|
|||||||
if u == "" {
|
if u == "" {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
if _, err := ValidateCDNURL(u); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := ResolveCDNURLHost(ctx, u); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
sourceKey := cdnSourceKey(src.ID)
|
sourceKey := cdnSourceKey(src.ID)
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -127,7 +133,7 @@ func applyCDNSourceHTTPResult(ctx context.Context, st store.Backend, hc *http.Cl
|
|||||||
if etag := strings.TrimSpace(src.Etag); etag != "" {
|
if etag := strings.TrimSpace(src.Etag); etag != "" {
|
||||||
req.Header.Set("If-None-Match", etag)
|
req.Header.Set("If-None-Match", etag)
|
||||||
}
|
}
|
||||||
resp, err := hc.Do(req)
|
resp, err := upstreamHTTPDo(ctx, hc, req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
|
return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
|
||||||
}
|
}
|
||||||
@@ -143,7 +149,7 @@ func applyCDNSourceHTTPResult(ctx context.Context, st store.Backend, hc *http.Cl
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
resp, err = hc.Do(req2)
|
resp, err = upstreamHTTPDo(ctx, hc, req2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
|
return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
|
||||||
}
|
}
|
||||||
@@ -189,6 +195,12 @@ func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client,
|
|||||||
if u == "" {
|
if u == "" {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
if _, err := ValidateCDNURL(u); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := ResolveCDNURLHost(ctx, u); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
sourceKey := cdnSourceKey(src.ID)
|
sourceKey := cdnSourceKey(src.ID)
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -197,7 +209,7 @@ func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client,
|
|||||||
if etag := strings.TrimSpace(src.Etag); etag != "" {
|
if etag := strings.TrimSpace(src.Etag); etag != "" {
|
||||||
req.Header.Set("If-None-Match", etag)
|
req.Header.Set("If-None-Match", etag)
|
||||||
}
|
}
|
||||||
resp, err := hc.Do(req)
|
resp, err := upstreamHTTPDo(ctx, hc, req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
|
return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
|
||||||
}
|
}
|
||||||
@@ -212,7 +224,7 @@ func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
resp, err = hc.Do(req2)
|
resp, err = upstreamHTTPDo(ctx, hc, req2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
|
return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,116 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"net/netip"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func isBlockedCDNIP(ip netip.Addr) bool {
|
||||||
|
if allowPrivateCDNURLs() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if !ip.IsValid() {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsMulticast() ||
|
||||||
|
ip.IsUnspecified() || ip == netip.MustParseAddr("169.254.169.254")
|
||||||
|
}
|
||||||
|
|
||||||
|
func allowPrivateCDNURLs() bool {
|
||||||
|
v := strings.TrimSpace(os.Getenv("EVOBGP_CDN_ALLOW_PRIVATE"))
|
||||||
|
return v == "1" || strings.EqualFold(v, "true")
|
||||||
|
}
|
||||||
|
|
||||||
|
func isBlockedCDNHostname(host string) bool {
|
||||||
|
if allowPrivateCDNURLs() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
h := strings.ToLower(strings.TrimSpace(host))
|
||||||
|
if h == "" || h == "localhost" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if strings.HasSuffix(h, ".local") || strings.HasSuffix(h, ".internal") || strings.HasSuffix(h, ".localhost") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateCDNURL checks CDN source URLs for SSRF-safe HTTPS endpoints (hostname only; no DNS resolve).
|
||||||
|
func ValidateCDNURL(raw string) (string, error) {
|
||||||
|
raw = strings.TrimSpace(raw)
|
||||||
|
if raw == "" {
|
||||||
|
return "", fmt.Errorf("pipeline: cdn url is required")
|
||||||
|
}
|
||||||
|
u, err := url.Parse(raw)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("pipeline: cdn url invalid: %w", err)
|
||||||
|
}
|
||||||
|
if u.Scheme != "https" {
|
||||||
|
return "", fmt.Errorf("pipeline: cdn url must use https")
|
||||||
|
}
|
||||||
|
if u.User != nil {
|
||||||
|
return "", fmt.Errorf("pipeline: cdn url must not include credentials")
|
||||||
|
}
|
||||||
|
host := strings.TrimSpace(u.Hostname())
|
||||||
|
if host == "" {
|
||||||
|
return "", fmt.Errorf("pipeline: cdn url missing host")
|
||||||
|
}
|
||||||
|
if isBlockedCDNHostname(host) {
|
||||||
|
return "", fmt.Errorf("pipeline: cdn url blocked host")
|
||||||
|
}
|
||||||
|
if ip, err := netip.ParseAddr(host); err == nil {
|
||||||
|
if isBlockedCDNIP(ip) {
|
||||||
|
return "", fmt.Errorf("pipeline: cdn url blocked host")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return u.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolveCDNURLHost resolves a CDN hostname and rejects private/link-local targets (SSRF at fetch time).
|
||||||
|
func ResolveCDNURLHost(ctx context.Context, raw string) error {
|
||||||
|
u, err := url.Parse(strings.TrimSpace(raw))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
host := strings.TrimSpace(u.Hostname())
|
||||||
|
if host == "" {
|
||||||
|
return fmt.Errorf("pipeline: cdn url missing host")
|
||||||
|
}
|
||||||
|
if ip, err := netip.ParseAddr(host); err == nil {
|
||||||
|
if isBlockedCDNIP(ip) {
|
||||||
|
return fmt.Errorf("pipeline: cdn url blocked host")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if isBlockedCDNHostname(host) {
|
||||||
|
return fmt.Errorf("pipeline: cdn url blocked host")
|
||||||
|
}
|
||||||
|
if ctx == nil {
|
||||||
|
ctx = context.Background()
|
||||||
|
}
|
||||||
|
resolveCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
ips, err := net.DefaultResolver.LookupIP(resolveCtx, "ip", host)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("pipeline: cdn url dns lookup: %w", err)
|
||||||
|
}
|
||||||
|
if len(ips) == 0 {
|
||||||
|
return fmt.Errorf("pipeline: cdn url dns lookup: no addresses")
|
||||||
|
}
|
||||||
|
for _, ip := range ips {
|
||||||
|
addr, ok := netip.AddrFromSlice(ip)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if isBlockedCDNIP(addr) {
|
||||||
|
return fmt.Errorf("pipeline: cdn url resolves to blocked address")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestValidateCDNURL(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
raw string
|
||||||
|
ok bool
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"https://cdn.example.com/prefixes.txt", true, "https://cdn.example.com/prefixes.txt"},
|
||||||
|
{"http://cdn.example.com/x", false, ""},
|
||||||
|
{"https://127.0.0.1/x", false, ""},
|
||||||
|
{"https://10.0.0.1/x", false, ""},
|
||||||
|
{"https://169.254.169.254/latest/meta-data", false, ""},
|
||||||
|
{"https://localhost/x", false, ""},
|
||||||
|
{"file:///etc/passwd", false, ""},
|
||||||
|
{"https://user:[email protected]/x", false, ""},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
got, err := ValidateCDNURL(tc.raw)
|
||||||
|
if tc.ok && err != nil {
|
||||||
|
t.Errorf("%q: unexpected err %v", tc.raw, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !tc.ok && err == nil {
|
||||||
|
t.Errorf("%q: expected error", tc.raw)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if tc.ok && got != tc.want {
|
||||||
|
t.Errorf("%q: got %q want %q", tc.raw, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -25,7 +25,7 @@ func prefixRowsForSource(rows []store.PrefixRow, sourceKey string) []store.Prefi
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func collectASPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module, list []*store.ASEntry) ([]store.PrefixRow, error) {
|
func collectASPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module, list []*store.ASEntry, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) {
|
||||||
moduleID := mod.ID
|
moduleID := mod.ID
|
||||||
legacy := strings.TrimSpace(os.Getenv("EVOBGP_ASN_RESOLVE")) == "0"
|
legacy := strings.TrimSpace(os.Getenv("EVOBGP_ASN_RESOLVE")) == "0"
|
||||||
if legacy {
|
if legacy {
|
||||||
@@ -76,6 +76,41 @@ func collectASPrefixRows(ctx context.Context, st store.Backend, hc *http.Client,
|
|||||||
}
|
}
|
||||||
pfxs, holder, err := resolveASNForEntry(ctx, st, hc, entry.ASN)
|
pfxs, holder, err := resolveASNForEntry(ctx, st, hc, entry.ASN)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if staleOnUpstreamError() {
|
||||||
|
if staleRows, staleHolder, ok := staleASNPrefixes(st, priorSnapshot, entry.ASN); ok {
|
||||||
|
logStaleUpstream("asn", fmt.Sprintf("AS%d: %v", entry.ASN, err))
|
||||||
|
src := fmt.Sprintf("as:%d", entry.ASN)
|
||||||
|
rows := append([]store.PrefixRow(nil), staleRows...)
|
||||||
|
for i := range rows {
|
||||||
|
rows[i].CommunityID = comm
|
||||||
|
rows[i].Source = src
|
||||||
|
}
|
||||||
|
results[idx] = entryResult{
|
||||||
|
rows: rows,
|
||||||
|
metaID: entry.ID,
|
||||||
|
asn: entry.ASN,
|
||||||
|
holder: staleHolder,
|
||||||
|
count: int64(len(rows)),
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if pfxs2, holder2, ok := asnCacheExpired(st, entry.ASN); ok {
|
||||||
|
logStaleUpstream("asn", fmt.Sprintf("AS%d expired cache: %v", entry.ASN, err))
|
||||||
|
src := fmt.Sprintf("as:%d", entry.ASN)
|
||||||
|
var rows []store.PrefixRow
|
||||||
|
for _, pfx := range pfxs2 {
|
||||||
|
rows = append(rows, store.PrefixRow{Prefix: pfx.String(), CommunityID: comm, Source: src})
|
||||||
|
}
|
||||||
|
results[idx] = entryResult{
|
||||||
|
rows: rows,
|
||||||
|
metaID: entry.ID,
|
||||||
|
asn: entry.ASN,
|
||||||
|
holder: holder2,
|
||||||
|
count: int64(len(pfxs2)),
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
results[idx] = entryResult{err: fmt.Errorf("resolve AS%d: %w", entry.ASN, err)}
|
results[idx] = entryResult{err: fmt.Errorf("resolve AS%d: %w", entry.ASN, err)}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -167,6 +202,13 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client
|
|||||||
}
|
}
|
||||||
rows, err := fetchCDNSourceRows(ctx, st, hc, tenantID, moduleID, mod, src, priorSnapshot, now)
|
rows, err := fetchCDNSourceRows(ctx, st, hc, tenantID, moduleID, mod, src, priorSnapshot, now)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if staleOnUpstreamError() {
|
||||||
|
if cached, ok := staleCDNPrefixes(st, tenantID, moduleID, priorSnapshot, src.ID); ok {
|
||||||
|
logStaleUpstream("cdn", fmt.Sprintf("source %s: %v", src.ID, err))
|
||||||
|
results[idx] = srcResult{rows: cached}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
results[idx] = srcResult{err: err}
|
results[idx] = srcResult{err: err}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -190,7 +232,7 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func collectDomainPrefixRows(ctx context.Context, hc *http.Client, mod *store.Module, profiles []*store.DohProfile, policy string, entries []*store.DomainEntry) ([]store.PrefixRow, error) {
|
func collectDomainPrefixRows(ctx context.Context, hc *http.Client, mod *store.Module, profiles []*store.DohProfile, policy string, entries []*store.DomainEntry, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) {
|
||||||
var validDom []*store.DomainEntry
|
var validDom []*store.DomainEntry
|
||||||
for _, e := range entries {
|
for _, e := range entries {
|
||||||
if e != nil {
|
if e != nil {
|
||||||
@@ -219,6 +261,19 @@ func collectDomainPrefixRows(ctx context.Context, hc *http.Client, mod *store.Mo
|
|||||||
}
|
}
|
||||||
addrs, err := resolveDomainIPsWithPolicy(ctx, hc, profiles, policy, entry.FQDN)
|
addrs, err := resolveDomainIPsWithPolicy(ctx, hc, profiles, policy, entry.FQDN)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if staleOnUpstreamError() {
|
||||||
|
if cached, ok := staleDomainPrefixes(priorSnapshot, entry.FQDN); ok {
|
||||||
|
logStaleUpstream("domain", fmt.Sprintf("%q: %v", entry.FQDN, err))
|
||||||
|
rows := append([]store.PrefixRow(nil), cached...)
|
||||||
|
for i := range rows {
|
||||||
|
if rows[i].CommunityID == nil {
|
||||||
|
rows[i].CommunityID = comm
|
||||||
|
}
|
||||||
|
}
|
||||||
|
results[idx] = domResult{rows: rows}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
results[idx] = domResult{err: fmt.Errorf("resolve domain %q: %w", entry.FQDN, err)}
|
results[idx] = domResult{err: fmt.Errorf("resolve domain %q: %w", entry.FQDN, err)}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,86 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"net/netip"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// staleOnUpstreamError reports whether ingest should keep last-known prefixes when an upstream fetch fails.
|
||||||
|
// Enabled by default; set EVOBGP_STALE_ON_UPSTREAM_ERROR=0 to restore fail-fast behavior.
|
||||||
|
func staleOnUpstreamError() bool {
|
||||||
|
v := strings.TrimSpace(os.Getenv("EVOBGP_STALE_ON_UPSTREAM_ERROR"))
|
||||||
|
if v == "" || v == "1" || strings.EqualFold(v, "true") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func logStaleUpstream(kind, detail string) {
|
||||||
|
log.Printf("pipeline: stale upstream fallback (%s): %s", kind, detail)
|
||||||
|
}
|
||||||
|
|
||||||
|
func staleASNPrefixes(st store.Backend, priorSnapshot []store.PrefixRow, asn int64) ([]store.PrefixRow, string, bool) {
|
||||||
|
sourceKey := fmt.Sprintf("as:%d", asn)
|
||||||
|
if cached := prefixRowsForSource(priorSnapshot, sourceKey); len(cached) > 0 {
|
||||||
|
return cached, "", true
|
||||||
|
}
|
||||||
|
if st == nil {
|
||||||
|
return nil, "", false
|
||||||
|
}
|
||||||
|
ent, ok, err := st.GetASNPrefixCache(asn)
|
||||||
|
if err != nil || !ok || ent == nil || len(ent.Prefixes) == 0 {
|
||||||
|
return nil, "", false
|
||||||
|
}
|
||||||
|
var rows []store.PrefixRow
|
||||||
|
for _, p := range ent.Prefixes {
|
||||||
|
pfx, perr := netip.ParsePrefix(strings.TrimSpace(p))
|
||||||
|
if perr != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rows = append(rows, store.PrefixRow{Prefix: pfx.Masked().String(), Source: sourceKey})
|
||||||
|
}
|
||||||
|
if len(rows) == 0 {
|
||||||
|
return nil, "", false
|
||||||
|
}
|
||||||
|
return rows, ent.Holder, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func staleDomainPrefixes(priorSnapshot []store.PrefixRow, fqdn string) ([]store.PrefixRow, bool) {
|
||||||
|
sourceKey := "domain:" + strings.TrimSpace(fqdn)
|
||||||
|
cached := prefixRowsForSource(priorSnapshot, sourceKey)
|
||||||
|
return cached, len(cached) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func staleCDNPrefixes(st store.Backend, tenantID, moduleID string, priorSnapshot []store.PrefixRow, sourceID string) ([]store.PrefixRow, bool) {
|
||||||
|
sourceKey := cdnSourceKey(sourceID)
|
||||||
|
cached := cachedCDNPrefixRows(st, tenantID, moduleID, priorSnapshot, sourceKey)
|
||||||
|
return cached, len(cached) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// asnCacheExpired returns cached ASN prefixes even past TTL (for stale fallback only).
|
||||||
|
func asnCacheExpired(st store.Backend, asn int64) ([]netip.Prefix, string, bool) {
|
||||||
|
if st == nil {
|
||||||
|
return nil, "", false
|
||||||
|
}
|
||||||
|
ent, ok, err := st.GetASNPrefixCache(asn)
|
||||||
|
if err != nil || !ok || ent == nil || len(ent.Prefixes) == 0 {
|
||||||
|
return nil, "", false
|
||||||
|
}
|
||||||
|
out := make([]netip.Prefix, 0, len(ent.Prefixes))
|
||||||
|
for _, p := range ent.Prefixes {
|
||||||
|
pfx, perr := netip.ParsePrefix(strings.TrimSpace(p))
|
||||||
|
if perr != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, pfx.Masked())
|
||||||
|
}
|
||||||
|
if len(out) == 0 {
|
||||||
|
return nil, "", false
|
||||||
|
}
|
||||||
|
return out, ent.Holder, true
|
||||||
|
}
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCollectCDNPrefixRows_StaleOnFetchError(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_STALE_ON_UPSTREAM_ERROR", "1")
|
||||||
|
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
|
||||||
|
|
||||||
|
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
http.Error(w, "upstream down", http.StatusServiceUnavailable)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
st := store.NewMemory()
|
||||||
|
st.SeedDemo()
|
||||||
|
tenant, _, _, _, _ := st.DemoIDs()
|
||||||
|
mod, err := st.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn", Enabled: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := st.CreateCDNSource(tenant, mod.ID, &store.CDNSource{
|
||||||
|
ID: "s1", URL: srv.URL, SourceKind: "plain",
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prior := []store.PrefixRow{
|
||||||
|
{Prefix: "203.0.113.0/24", Source: "cdn:s1"},
|
||||||
|
}
|
||||||
|
rows, err := collectCDNPrefixRows(context.Background(), st, srv.Client(), tenant, mod, []*store.CDNSource{{ID: "s1", URL: srv.URL, SourceKind: "plain"}}, prior)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected stale fallback, got err: %v", err)
|
||||||
|
}
|
||||||
|
if len(rows) != 1 || rows[0].Prefix != "203.0.113.0/24" {
|
||||||
|
t.Fatalf("unexpected rows: %+v", rows)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollectCDNPrefixRows_FailFastWhenNoStale(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_STALE_ON_UPSTREAM_ERROR", "0")
|
||||||
|
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
|
||||||
|
|
||||||
|
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
http.Error(w, "upstream down", http.StatusServiceUnavailable)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
st := store.NewMemory()
|
||||||
|
st.SeedDemo()
|
||||||
|
tenant, _, _, _, _ := st.DemoIDs()
|
||||||
|
mod, err := st.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn", Enabled: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := st.CreateCDNSource(tenant, mod.ID, &store.CDNSource{
|
||||||
|
ID: "s1", URL: srv.URL, SourceKind: "plain",
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = collectCDNPrefixRows(context.Background(), st, srv.Client(), tenant, mod, []*store.CDNSource{{ID: "s1", URL: srv.URL, SourceKind: "plain"}}, nil)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error when stale disabled and no cache")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,86 +5,116 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/httpclient"
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type prefetchTask struct {
|
||||||
|
tenantID string
|
||||||
|
mod *store.Module
|
||||||
|
src *store.CDNSource
|
||||||
|
}
|
||||||
|
|
||||||
// PrefetchCDNSourceETags performs conditional GETs for CDN sources; on 200 parses CIDRs into module_prefix_snapshot.
|
// PrefetchCDNSourceETags performs conditional GETs for CDN sources; on 200 parses CIDRs into module_prefix_snapshot.
|
||||||
func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Client) error {
|
func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Client) error {
|
||||||
if hc == nil {
|
if hc == nil {
|
||||||
hc = http.DefaultClient
|
hc = httpclient.New(httpclient.DefaultTimeout)
|
||||||
|
}
|
||||||
|
if ctx == nil {
|
||||||
|
ctx = context.Background()
|
||||||
}
|
}
|
||||||
tenants, err := st.ListTenantIDs()
|
tenants, err := st.ListTenantIDs()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
now := time.Now().UTC()
|
var tasks []prefetchTask
|
||||||
for _, tid := range tenants {
|
for _, tid := range tenants {
|
||||||
for _, mod := range st.ListModules(tid) {
|
for _, mod := range st.ListModules(tid) {
|
||||||
if mod == nil || !mod.Enabled || mod.Type != "CDN_CIDRS" {
|
if mod == nil || !mod.Enabled || mod.Type != "CDN_CIDRS" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
omod, err := st.GetModule(tid, mod.ID)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
sources, err := st.ListCDNSources(tid, mod.ID)
|
sources, err := st.ListCDNSources(tid, mod.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
var prior []store.PrefixRow
|
|
||||||
if snap, ok, _ := st.GetModulePrefixSnapshot(tid, mod.ID); ok && snap != nil {
|
|
||||||
prior = snap.Prefixes
|
|
||||||
}
|
|
||||||
for _, src := range sources {
|
for _, src := range sources {
|
||||||
if src == nil || strings.TrimSpace(src.URL) == "" {
|
if src != nil && strings.TrimSpace(src.URL) != "" {
|
||||||
continue
|
tasks = append(tasks, prefetchTask{tenantID: tid, mod: mod, src: src})
|
||||||
}
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimSpace(src.URL), nil)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if etag := strings.TrimSpace(src.Etag); etag != "" {
|
|
||||||
req.Header.Set("If-None-Match", etag)
|
|
||||||
}
|
|
||||||
resp, err := hc.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if resp.StatusCode == http.StatusNotModified {
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
_, _ = io.Copy(io.Discard, resp.Body)
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
prefixStrs, err := parseCDNBody(string(body), src)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
newEtag := strings.TrimSpace(resp.Header.Get("ETag"))
|
|
||||||
patch := &store.CDNSourcePatch{LastRefreshedAt: &now}
|
|
||||||
if newEtag != "" && newEtag != strings.TrimSpace(src.Etag) {
|
|
||||||
e := newEtag
|
|
||||||
patch.Etag = &e
|
|
||||||
}
|
|
||||||
_, _ = st.UpdateCDNSource(tid, mod.ID, src.ID, patch)
|
|
||||||
rows := cdnRowsFromParsed(omod, src, prefixStrs)
|
|
||||||
_ = mergeCDNSourceIntoModuleSnapshot(st, tid, omod, src.ID, rows)
|
|
||||||
_ = prior // prior may be stale after merge; refresh for next source in loop
|
|
||||||
if snap, ok, _ := st.GetModulePrefixSnapshot(tid, mod.ID); ok && snap != nil {
|
|
||||||
prior = snap.Prefixes
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if len(tasks) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
sem := make(chan struct{}, collectConcurrency())
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for _, task := range tasks {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(t prefetchTask) {
|
||||||
|
defer wg.Done()
|
||||||
|
sem <- struct{}{}
|
||||||
|
defer func() { <-sem }()
|
||||||
|
prefetchOneCDNSource(ctx, st, hc, t)
|
||||||
|
}(task)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func prefetchOneCDNSource(ctx context.Context, st store.Backend, hc *http.Client, t prefetchTask) {
|
||||||
|
now := time.Now().UTC()
|
||||||
|
tid, mod, src := t.tenantID, t.mod, t.src
|
||||||
|
u := strings.TrimSpace(src.URL)
|
||||||
|
if _, err := ValidateCDNURL(u); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := ResolveCDNURLHost(ctx, u); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
omod, err := st.GetModule(tid, mod.ID)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if etag := strings.TrimSpace(src.Etag); etag != "" {
|
||||||
|
req.Header.Set("If-None-Match", etag)
|
||||||
|
}
|
||||||
|
resp, err := upstreamHTTPDo(ctx, hc, req)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if resp.StatusCode == http.StatusNotModified {
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
_, _ = io.Copy(io.Discard, resp.Body)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
prefixStrs, err := parseCDNBody(string(body), src)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
newEtag := strings.TrimSpace(resp.Header.Get("ETag"))
|
||||||
|
patch := &store.CDNSourcePatch{LastRefreshedAt: &now}
|
||||||
|
if newEtag != "" && newEtag != strings.TrimSpace(src.Etag) {
|
||||||
|
e := newEtag
|
||||||
|
patch.Etag = &e
|
||||||
|
}
|
||||||
|
_, _ = st.UpdateCDNSource(tid, mod.ID, src.ID, patch)
|
||||||
|
rows := cdnRowsFromParsed(omod, src, prefixStrs)
|
||||||
|
_ = mergeCDNSourceIntoModuleSnapshot(st, tid, omod, src.ID, rows)
|
||||||
|
}
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/birdfmt"
|
"evobgp/internal/birdfmt"
|
||||||
|
"evobgp/internal/httpclient"
|
||||||
"evobgp/internal/observability"
|
"evobgp/internal/observability"
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
|
|
||||||
@@ -50,7 +51,7 @@ func MaterializedASPrefixKey(asn int64) string {
|
|||||||
// It does not create a new config revision.
|
// It does not create a new config revision.
|
||||||
func RefreshModuleIngest(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string) error {
|
func RefreshModuleIngest(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string) error {
|
||||||
if hc == nil {
|
if hc == nil {
|
||||||
hc = http.DefaultClient
|
hc = httpclient.New(httpclient.DefaultTimeout)
|
||||||
}
|
}
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
mod, err := st.GetModule(tenantID, moduleID)
|
mod, err := st.GetModule(tenantID, moduleID)
|
||||||
@@ -84,7 +85,7 @@ func RefreshModuleIngest(ctx context.Context, st store.Backend, hc *http.Client,
|
|||||||
// If materialized prefixes are unchanged, returns latest revision id without creating a duplicate.
|
// If materialized prefixes are unchanged, returns latest revision id without creating a duplicate.
|
||||||
func RenderTenantRevision(ctx context.Context, st store.Backend, hc *http.Client, tenantID, triggerModuleID string) (revisionID string, err error) {
|
func RenderTenantRevision(ctx context.Context, st store.Backend, hc *http.Client, tenantID, triggerModuleID string) (revisionID string, err error) {
|
||||||
if hc == nil {
|
if hc == nil {
|
||||||
hc = http.DefaultClient
|
hc = httpclient.New(httpclient.DefaultTimeout)
|
||||||
}
|
}
|
||||||
agg, err := aggregateTenantPrefixRowsAll(ctx, st, hc, tenantID)
|
agg, err := aggregateTenantPrefixRowsAll(ctx, st, hc, tenantID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -117,7 +118,7 @@ func RenderTenantRevision(ctx context.Context, st store.Backend, hc *http.Client
|
|||||||
func RenderTenantRevisionFromPrefixes(ctx context.Context, st store.Backend, hc *http.Client, tenantID, triggerModuleID string, rows []store.PrefixRow) (revisionID string, err error) {
|
func RenderTenantRevisionFromPrefixes(ctx context.Context, st store.Backend, hc *http.Client, tenantID, triggerModuleID string, rows []store.PrefixRow) (revisionID string, err error) {
|
||||||
_ = ctx
|
_ = ctx
|
||||||
if hc == nil {
|
if hc == nil {
|
||||||
hc = http.DefaultClient
|
hc = httpclient.New(httpclient.DefaultTimeout)
|
||||||
}
|
}
|
||||||
agg := append([]store.PrefixRow(nil), rows...)
|
agg := append([]store.PrefixRow(nil), rows...)
|
||||||
rawCount := len(agg)
|
rawCount := len(agg)
|
||||||
@@ -176,7 +177,7 @@ func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Cli
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
sort.Slice(list, func(i, j int) bool { return list[i].ASN < list[j].ASN })
|
sort.Slice(list, func(i, j int) bool { return list[i].ASN < list[j].ASN })
|
||||||
return collectASPrefixRows(ctx, st, hc, tenantID, mod, list)
|
return collectASPrefixRows(ctx, st, hc, tenantID, mod, list, priorSnapshot)
|
||||||
case "CDN_CIDRS":
|
case "CDN_CIDRS":
|
||||||
sources, err := st.ListCDNSources(tenantID, moduleID)
|
sources, err := st.ListCDNSources(tenantID, moduleID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -192,7 +193,7 @@ func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Cli
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return collectDomainPrefixRows(ctx, hc, mod, profiles, policy, entries)
|
return collectDomainPrefixRows(ctx, hc, mod, profiles, policy, entries, priorSnapshot)
|
||||||
default:
|
default:
|
||||||
return nil, fmt.Errorf("pipeline: unknown module type %q", mod.Type)
|
return nil, fmt.Errorf("pipeline: unknown module type %q", mod.Type)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,13 +7,14 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"evobgp/internal/httpclient"
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
// RefreshTenantModules ingests all listed modules in parallel and updates per-module snapshots.
|
// RefreshTenantModules ingests all listed modules in parallel and updates per-module snapshots.
|
||||||
func RefreshTenantModules(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, moduleIDs []string) error {
|
func RefreshTenantModules(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, moduleIDs []string) error {
|
||||||
if hc == nil {
|
if hc == nil {
|
||||||
hc = http.DefaultClient
|
hc = httpclient.New(httpclient.DefaultTimeout)
|
||||||
}
|
}
|
||||||
var ids []string
|
var ids []string
|
||||||
seen := make(map[string]struct{})
|
seen := make(map[string]struct{})
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"evobgp/internal/httpclient"
|
||||||
|
)
|
||||||
|
|
||||||
|
func upstreamHTTPDo(ctx context.Context, hc *http.Client, req *http.Request) (*http.Response, error) {
|
||||||
|
if hc == nil {
|
||||||
|
hc = httpclient.New(httpclient.DefaultTimeout)
|
||||||
|
}
|
||||||
|
resp, err := httpclient.DoWithBreaker(ctx, hc, req, 3)
|
||||||
|
if err != nil {
|
||||||
|
if req.URL != nil {
|
||||||
|
return nil, fmt.Errorf("cdn fetch %s: %w", req.URL.String(), err)
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
@@ -11,14 +11,22 @@ import (
|
|||||||
"github.com/jackc/pgx/v5"
|
"github.com/jackc/pgx/v5"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func asnCacheRowTableExists(ctx context.Context, q queryRower) bool {
|
||||||
|
var n int
|
||||||
|
err := q.QueryRow(ctx, `
|
||||||
|
SELECT 1 FROM information_schema.tables
|
||||||
|
WHERE table_schema = 'public' AND table_name = 'asn_prefix_cache_row'
|
||||||
|
LIMIT 1`).Scan(&n)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
func (p *Postgres) GetASNPrefixCache(asn int64) (*store.ASNPrefixCacheEntry, bool, error) {
|
func (p *Postgres) GetASNPrefixCache(asn int64) (*store.ASNPrefixCacheEntry, bool, error) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
var holder string
|
var holder string
|
||||||
var fetchedAt time.Time
|
var fetchedAt time.Time
|
||||||
var raw []byte
|
|
||||||
err := p.pool.QueryRow(ctx, `
|
err := p.pool.QueryRow(ctx, `
|
||||||
SELECT holder, fetched_at, prefixes_json FROM asn_prefix_cache WHERE asn = $1`, asn).
|
SELECT holder, fetched_at FROM asn_prefix_cache WHERE asn = $1`, asn).
|
||||||
Scan(&holder, &fetchedAt, &raw)
|
Scan(&holder, &fetchedAt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, pgx.ErrNoRows) {
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
return nil, false, nil
|
return nil, false, nil
|
||||||
@@ -26,8 +34,25 @@ func (p *Postgres) GetASNPrefixCache(asn int64) (*store.ASNPrefixCacheEntry, boo
|
|||||||
return nil, false, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
var prefixes []string
|
var prefixes []string
|
||||||
if len(raw) > 0 {
|
if asnCacheRowTableExists(ctx, p.pool) {
|
||||||
_ = json.Unmarshal(raw, &prefixes)
|
rows, qerr := p.pool.Query(ctx, `
|
||||||
|
SELECT prefix::text FROM asn_prefix_cache_row WHERE asn = $1 ORDER BY prefix`, asn)
|
||||||
|
if qerr != nil {
|
||||||
|
return nil, false, qerr
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
var s string
|
||||||
|
if err := rows.Scan(&s); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
prefixes = append(prefixes, s)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
var raw []byte
|
||||||
|
if err := p.pool.QueryRow(ctx, `SELECT prefixes_json FROM asn_prefix_cache WHERE asn = $1`, asn).Scan(&raw); err == nil && len(raw) > 0 {
|
||||||
|
_ = json.Unmarshal(raw, &prefixes)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return &store.ASNPrefixCacheEntry{
|
return &store.ASNPrefixCacheEntry{
|
||||||
ASN: asn,
|
ASN: asn,
|
||||||
@@ -38,18 +63,40 @@ func (p *Postgres) GetASNPrefixCache(asn int64) (*store.ASNPrefixCacheEntry, boo
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (p *Postgres) SetASNPrefixCache(asn int64, holder string, prefixes []string) error {
|
func (p *Postgres) SetASNPrefixCache(asn int64, holder string, prefixes []string) error {
|
||||||
raw, err := json.Marshal(prefixes)
|
ctx := context.Background()
|
||||||
|
tx, err := p.pool.Begin(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
ctx := context.Background()
|
defer func() { _ = tx.Rollback(ctx) }()
|
||||||
_, err = p.pool.Exec(ctx, `
|
_, err = tx.Exec(ctx, `
|
||||||
INSERT INTO asn_prefix_cache (asn, holder, prefixes_json, fetched_at)
|
INSERT INTO asn_prefix_cache (asn, holder, fetched_at)
|
||||||
VALUES ($1, $2, $3::jsonb, now())
|
VALUES ($1, $2, now())
|
||||||
ON CONFLICT (asn) DO UPDATE SET
|
ON CONFLICT (asn) DO UPDATE SET
|
||||||
holder = EXCLUDED.holder,
|
holder = EXCLUDED.holder,
|
||||||
prefixes_json = EXCLUDED.prefixes_json,
|
fetched_at = EXCLUDED.fetched_at`, asn, holder)
|
||||||
fetched_at = EXCLUDED.fetched_at`,
|
if err != nil {
|
||||||
asn, holder, string(raw))
|
return err
|
||||||
return err
|
}
|
||||||
|
if asnCacheRowTableExists(ctx, tx) {
|
||||||
|
if _, err := tx.Exec(ctx, `DELETE FROM asn_prefix_cache_row WHERE asn = $1`, asn); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, pfx := range prefixes {
|
||||||
|
if _, err := tx.Exec(ctx, `
|
||||||
|
INSERT INTO asn_prefix_cache_row (asn, prefix) VALUES ($1, $2::cidr)`, asn, pfx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
raw, err := json.Marshal(prefixes)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(ctx, `
|
||||||
|
UPDATE asn_prefix_cache SET prefixes_json = $2::jsonb WHERE asn = $1`, asn, string(raw)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return tx.Commit(ctx)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
package repository
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const defaultRepoTimeout = 60 * time.Second
|
||||||
|
|
||||||
|
// boundedRepoCtx returns a context with default repository I/O timeout.
|
||||||
|
func boundedRepoCtx(parent context.Context) (context.Context, context.CancelFunc) {
|
||||||
|
if parent == nil {
|
||||||
|
parent = context.Background()
|
||||||
|
}
|
||||||
|
if _, ok := parent.Deadline(); ok {
|
||||||
|
return parent, func() {}
|
||||||
|
}
|
||||||
|
return context.WithTimeout(parent, defaultRepoTimeout)
|
||||||
|
}
|
||||||
@@ -20,6 +20,28 @@ func NewJobAuditWriter(pool *pgxpool.Pool) *JobAuditWriter {
|
|||||||
return &JobAuditWriter{pool: pool}
|
return &JobAuditWriter{pool: pool}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UpsertQueued inserts a queued job row (best-effort).
|
||||||
|
func (w *JobAuditWriter) UpsertQueued(ctx context.Context, tenantID, jobID, kind string, idempotencyKey *string, moduleID *string, meta map[string]any) {
|
||||||
|
if w == nil || w.pool == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
metaJSON, _ := json.Marshal(meta)
|
||||||
|
var idem any
|
||||||
|
if idempotencyKey != nil && *idempotencyKey != "" {
|
||||||
|
idem = *idempotencyKey
|
||||||
|
}
|
||||||
|
var mod any
|
||||||
|
if moduleID != nil && *moduleID != "" {
|
||||||
|
mod = *moduleID
|
||||||
|
}
|
||||||
|
_, _ = w.pool.Exec(ctx, `
|
||||||
|
INSERT INTO job_audit (id, tenant_id, kind, status, idempotency_key, module_id, meta_json, created_at)
|
||||||
|
VALUES ($1::uuid, $2::uuid, $3, 'queued', $4, $5::uuid, $6::jsonb, now())
|
||||||
|
ON CONFLICT (tenant_id, idempotency_key) WHERE idempotency_key IS NOT NULL
|
||||||
|
DO UPDATE SET status='queued', meta_json=EXCLUDED.meta_json, module_id=EXCLUDED.module_id`,
|
||||||
|
jobID, tenantID, kind, idem, mod, metaJSON)
|
||||||
|
}
|
||||||
|
|
||||||
// UpsertRunning inserts or updates a running job row (best-effort).
|
// UpsertRunning inserts or updates a running job row (best-effort).
|
||||||
func (w *JobAuditWriter) UpsertRunning(ctx context.Context, tenantID, jobID, kind string, idempotencyKey *string, meta map[string]any) {
|
func (w *JobAuditWriter) UpsertRunning(ctx context.Context, tenantID, jobID, kind string, idempotencyKey *string, meta map[string]any) {
|
||||||
if w == nil || w.pool == nil {
|
if w == nil || w.pool == nil {
|
||||||
@@ -33,8 +55,7 @@ func (w *JobAuditWriter) UpsertRunning(ctx context.Context, tenantID, jobID, kin
|
|||||||
_, _ = w.pool.Exec(ctx, `
|
_, _ = w.pool.Exec(ctx, `
|
||||||
INSERT INTO job_audit (id, tenant_id, kind, status, idempotency_key, meta_json, created_at, started_at)
|
INSERT INTO job_audit (id, tenant_id, kind, status, idempotency_key, meta_json, created_at, started_at)
|
||||||
VALUES ($1::uuid, $2::uuid, $3, 'running', $4, $5::jsonb, now(), now())
|
VALUES ($1::uuid, $2::uuid, $3, 'running', $4, $5::jsonb, now(), now())
|
||||||
ON CONFLICT (tenant_id, idempotency_key) WHERE idempotency_key IS NOT NULL
|
ON CONFLICT (id) DO UPDATE SET status='running', started_at=COALESCE(job_audit.started_at, now()), meta_json=EXCLUDED.meta_json`,
|
||||||
DO UPDATE SET status='running', started_at=now(), meta_json=EXCLUDED.meta_json`,
|
|
||||||
jobID, tenantID, kind, idem, metaJSON)
|
jobID, tenantID, kind, idem, metaJSON)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package repository
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
jobAuditRetentionDays = 90
|
||||||
|
asnCacheRetentionDays = 7
|
||||||
|
)
|
||||||
|
|
||||||
|
// RunPeriodicMaintenance prunes stale job_audit and asn_prefix_cache rows (PostgreSQL).
|
||||||
|
func (p *Postgres) RunPeriodicMaintenance(ctx context.Context) {
|
||||||
|
if p == nil || p.pool == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if ctx == nil {
|
||||||
|
ctx = context.Background()
|
||||||
|
}
|
||||||
|
jobCutoff := time.Now().UTC().Add(-time.Duration(jobAuditRetentionDays) * 24 * time.Hour)
|
||||||
|
_, _ = p.pool.Exec(ctx, `
|
||||||
|
DELETE FROM job_audit
|
||||||
|
WHERE created_at < $1
|
||||||
|
AND status IN ('succeeded', 'failed', 'cancelled')`, jobCutoff)
|
||||||
|
asnCutoff := time.Now().UTC().Add(-time.Duration(asnCacheRetentionDays) * 24 * time.Hour)
|
||||||
|
_, _ = p.pool.Exec(ctx, `
|
||||||
|
DELETE FROM asn_prefix_cache WHERE fetched_at < $1`, asnCutoff)
|
||||||
|
}
|
||||||
@@ -12,16 +12,24 @@ import (
|
|||||||
"github.com/jackc/pgx/v5"
|
"github.com/jackc/pgx/v5"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func moduleSnapshotRowTableExists(ctx context.Context, q queryRower) bool {
|
||||||
|
var n int
|
||||||
|
err := q.QueryRow(ctx, `
|
||||||
|
SELECT 1 FROM information_schema.tables
|
||||||
|
WHERE table_schema = 'public' AND table_name = 'module_prefix_snapshot_row'
|
||||||
|
LIMIT 1`).Scan(&n)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
func (p *Postgres) GetModulePrefixSnapshot(tenantID, moduleID string) (*store.ModulePrefixSnapshot, bool, error) {
|
func (p *Postgres) GetModulePrefixSnapshot(tenantID, moduleID string) (*store.ModulePrefixSnapshot, bool, error) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
var inputHash string
|
var inputHash string
|
||||||
var collectedAt time.Time
|
var collectedAt time.Time
|
||||||
var raw []byte
|
|
||||||
err := p.pool.QueryRow(ctx, `
|
err := p.pool.QueryRow(ctx, `
|
||||||
SELECT input_hash, collected_at, prefixes_json
|
SELECT input_hash, collected_at
|
||||||
FROM module_prefix_snapshot
|
FROM module_prefix_snapshot
|
||||||
WHERE tenant_id = $1 AND module_id = $2`,
|
WHERE tenant_id = $1 AND module_id = $2`,
|
||||||
tenantID, moduleID).Scan(&inputHash, &collectedAt, &raw)
|
tenantID, moduleID).Scan(&inputHash, &collectedAt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, pgx.ErrNoRows) {
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
return nil, false, nil
|
return nil, false, nil
|
||||||
@@ -29,9 +37,31 @@ func (p *Postgres) GetModulePrefixSnapshot(tenantID, moduleID string) (*store.Mo
|
|||||||
return nil, false, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
var prefixes []store.PrefixRow
|
var prefixes []store.PrefixRow
|
||||||
if len(raw) > 0 {
|
if moduleSnapshotRowTableExists(ctx, p.pool) {
|
||||||
if err := json.Unmarshal(raw, &prefixes); err != nil {
|
rows, qerr := p.pool.Query(ctx, `
|
||||||
return nil, false, err
|
SELECT prefix, community_id::text, source
|
||||||
|
FROM module_prefix_snapshot_row
|
||||||
|
WHERE tenant_id = $1::uuid AND module_id = $2::uuid
|
||||||
|
ORDER BY ord`, tenantID, moduleID)
|
||||||
|
if qerr != nil {
|
||||||
|
return nil, false, qerr
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
var pr store.PrefixRow
|
||||||
|
var comm *string
|
||||||
|
if err := rows.Scan(&pr.Prefix, &comm, &pr.Source); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pr.CommunityID = comm
|
||||||
|
prefixes = append(prefixes, pr)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
var raw []byte
|
||||||
|
if err := p.pool.QueryRow(ctx, `
|
||||||
|
SELECT prefixes_json FROM module_prefix_snapshot
|
||||||
|
WHERE tenant_id = $1 AND module_id = $2`, tenantID, moduleID).Scan(&raw); err == nil && len(raw) > 0 {
|
||||||
|
_ = json.Unmarshal(raw, &prefixes)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return &store.ModulePrefixSnapshot{
|
return &store.ModulePrefixSnapshot{
|
||||||
@@ -45,20 +75,57 @@ func (p *Postgres) SetModulePrefixSnapshot(tenantID, moduleID, inputHash string,
|
|||||||
if strings.TrimSpace(tenantID) == "" || strings.TrimSpace(moduleID) == "" || strings.TrimSpace(inputHash) == "" {
|
if strings.TrimSpace(tenantID) == "" || strings.TrimSpace(moduleID) == "" || strings.TrimSpace(inputHash) == "" {
|
||||||
return store.ErrInvalidInput
|
return store.ErrInvalidInput
|
||||||
}
|
}
|
||||||
raw, err := json.Marshal(prefixes)
|
ctx := context.Background()
|
||||||
|
tx, err := p.pool.Begin(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
ctx := context.Background()
|
defer func() { _ = tx.Rollback(ctx) }()
|
||||||
_, err = p.pool.Exec(ctx, `
|
_, err = tx.Exec(ctx, `
|
||||||
INSERT INTO module_prefix_snapshot (tenant_id, module_id, input_hash, collected_at, prefixes_json)
|
INSERT INTO module_prefix_snapshot (tenant_id, module_id, input_hash, collected_at)
|
||||||
VALUES ($1::uuid, $2::uuid, $3, now(), $4::jsonb)
|
VALUES ($1::uuid, $2::uuid, $3, now())
|
||||||
ON CONFLICT (tenant_id, module_id) DO UPDATE SET
|
ON CONFLICT (tenant_id, module_id) DO UPDATE SET
|
||||||
input_hash = EXCLUDED.input_hash,
|
input_hash = EXCLUDED.input_hash,
|
||||||
collected_at = EXCLUDED.collected_at,
|
collected_at = EXCLUDED.collected_at`,
|
||||||
prefixes_json = EXCLUDED.prefixes_json`,
|
tenantID, moduleID, inputHash)
|
||||||
tenantID, moduleID, inputHash, string(raw))
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
}
|
||||||
|
if moduleSnapshotRowTableExists(ctx, tx) {
|
||||||
|
if _, err := tx.Exec(ctx, `
|
||||||
|
DELETE FROM module_prefix_snapshot_row
|
||||||
|
WHERE tenant_id = $1::uuid AND module_id = $2::uuid`, tenantID, moduleID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for i, pr := range prefixes {
|
||||||
|
var comm any
|
||||||
|
if pr.CommunityID != nil && strings.TrimSpace(*pr.CommunityID) != "" {
|
||||||
|
comm = strings.TrimSpace(*pr.CommunityID)
|
||||||
|
}
|
||||||
|
src := pr.Source
|
||||||
|
if strings.TrimSpace(src) == "" {
|
||||||
|
src = "render"
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(ctx, `
|
||||||
|
INSERT INTO module_prefix_snapshot_row (tenant_id, module_id, ord, prefix, community_id, source)
|
||||||
|
VALUES ($1::uuid, $2::uuid, $3, $4, $5::uuid, $6)`,
|
||||||
|
tenantID, moduleID, i, strings.TrimSpace(pr.Prefix), comm, src); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
raw, err := json.Marshal(prefixes)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(ctx, `
|
||||||
|
UPDATE module_prefix_snapshot SET prefixes_json = $3::jsonb
|
||||||
|
WHERE tenant_id = $1::uuid AND module_id = $2::uuid`,
|
||||||
|
tenantID, moduleID, string(raw)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return tx.Commit(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *Postgres) DeleteModulePrefixSnapshot(tenantID, moduleID string) error {
|
func (p *Postgres) DeleteModulePrefixSnapshot(tenantID, moduleID string) error {
|
||||||
|
|||||||
+106
-67
@@ -659,7 +659,8 @@ func (p *Postgres) DeleteSpeaker(tenantID, id string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (p *Postgres) GetRevision(tenantID, revisionID string) (*store.Revision, error) {
|
func (p *Postgres) GetRevision(tenantID, revisionID string) (*store.Revision, error) {
|
||||||
ctx := context.Background()
|
ctx, cancel := boundedRepoCtx(context.Background())
|
||||||
|
defer cancel()
|
||||||
var r store.Revision
|
var r store.Revision
|
||||||
var mod *string
|
var mod *string
|
||||||
var parent *string
|
var parent *string
|
||||||
@@ -686,11 +687,38 @@ func (p *Postgres) GetRevision(tenantID, revisionID string) (*store.Revision, er
|
|||||||
if mj.PreviewFragments == nil {
|
if mj.PreviewFragments == nil {
|
||||||
mj.PreviewFragments = map[string]string{}
|
mj.PreviewFragments = map[string]string{}
|
||||||
}
|
}
|
||||||
r.PreviewFragments = mj.PreviewFragments
|
r.PreviewFragments = loadRevisionPreview(ctx, p.pool, revisionID, mj.PreviewFragments)
|
||||||
r.MaterializedPrefixCount = mj.MaterializedPrefixCount
|
r.MaterializedPrefixCount = mj.MaterializedPrefixCount
|
||||||
return &r, nil
|
return &r, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *Postgres) GetRevisionSummary(tenantID, revisionID string) (*store.Revision, error) {
|
||||||
|
ctx, cancel := boundedRepoCtx(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
var r store.Revision
|
||||||
|
var mod *string
|
||||||
|
var parent *string
|
||||||
|
var prefixCount int
|
||||||
|
err := p.pool.QueryRow(ctx, `
|
||||||
|
SELECT id::text, tenant_id::text, module_id::text, content_hash, parent_revision_id::text,
|
||||||
|
COALESCE((meta_json->>'materialized_prefix_count')::int, 0), created_at
|
||||||
|
FROM config_revision WHERE id=$1 AND tenant_id=$2`, revisionID, tenantID).Scan(
|
||||||
|
&r.ID, &r.TenantID, &mod, &r.ContentHash, &parent, &prefixCount, &r.CreatedAt)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return nil, store.ErrNotFound
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if mod != nil {
|
||||||
|
r.ModuleID = *mod
|
||||||
|
}
|
||||||
|
r.ParentRevisionID = strOrNil(parent)
|
||||||
|
r.MaterializedPrefixCount = prefixCount
|
||||||
|
r.PreviewFragments = map[string]string{}
|
||||||
|
return &r, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (p *Postgres) ListRevisions(tenantID, moduleID string, cursor string, limit int) ([]*store.Revision, string, bool) {
|
func (p *Postgres) ListRevisions(tenantID, moduleID string, cursor string, limit int) ([]*store.Revision, string, bool) {
|
||||||
if limit <= 0 {
|
if limit <= 0 {
|
||||||
limit = 50
|
limit = 50
|
||||||
@@ -761,12 +789,7 @@ func (p *Postgres) ListRevisionPrefixes(tenantID, revisionID string, cursor stri
|
|||||||
if limit <= 0 {
|
if limit <= 0 {
|
||||||
limit = 50
|
limit = 50
|
||||||
}
|
}
|
||||||
off := 0
|
afterID, off, useOffset := store.ParsePrefixPageCursor(cursor)
|
||||||
if cursor != "" {
|
|
||||||
if n, err := strconv.Atoi(cursor); err == nil && n >= 0 {
|
|
||||||
off = n
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
var one int
|
var one int
|
||||||
if err := p.pool.QueryRow(ctx, `
|
if err := p.pool.QueryRow(ctx, `
|
||||||
@@ -777,34 +800,55 @@ func (p *Postgres) ListRevisionPrefixes(tenantID, revisionID string, cursor stri
|
|||||||
}
|
}
|
||||||
return nil, "", false
|
return nil, "", false
|
||||||
}
|
}
|
||||||
rows, err := p.pool.Query(ctx, `
|
if snapID, ok := p.revisionPrefixSnapshotID(ctx, revisionID); ok {
|
||||||
SELECT prefix::text, community_id::text, source FROM revision_materialized_prefix
|
return p.listSnapshotPrefixes(ctx, snapID, cursor, limit)
|
||||||
WHERE revision_id=$1 ORDER BY id
|
}
|
||||||
LIMIT $2 OFFSET $3`, revisionID, limit+1, off)
|
var rows pgx.Rows
|
||||||
|
var err error
|
||||||
|
if useOffset {
|
||||||
|
rows, err = p.pool.Query(ctx, `
|
||||||
|
SELECT id, prefix::text, community_id::text, source FROM revision_materialized_prefix
|
||||||
|
WHERE revision_id=$1::uuid ORDER BY id
|
||||||
|
LIMIT $2 OFFSET $3`, revisionID, limit+1, off)
|
||||||
|
} else {
|
||||||
|
var afterArg any
|
||||||
|
if afterID != nil {
|
||||||
|
afterArg = *afterID
|
||||||
|
}
|
||||||
|
rows, err = p.pool.Query(ctx, `
|
||||||
|
SELECT id, prefix::text, community_id::text, source FROM revision_materialized_prefix
|
||||||
|
WHERE revision_id=$1::uuid AND ($2::bigint IS NULL OR id > $2::bigint)
|
||||||
|
ORDER BY id
|
||||||
|
LIMIT $3`, revisionID, afterArg, limit+1)
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", false
|
return nil, "", false
|
||||||
}
|
}
|
||||||
defer rows.Close()
|
defer rows.Close()
|
||||||
var all []store.PrefixRow
|
var all []store.PrefixRow
|
||||||
|
var ids []int64
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
|
var rowID int64
|
||||||
var pr store.PrefixRow
|
var pr store.PrefixRow
|
||||||
var comm *string
|
var comm *string
|
||||||
if err := rows.Scan(&pr.Prefix, &comm, &pr.Source); err != nil {
|
if err := rows.Scan(&rowID, &pr.Prefix, &comm, &pr.Source); err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
pr.CommunityID = comm
|
pr.CommunityID = comm
|
||||||
|
ids = append(ids, rowID)
|
||||||
all = append(all, pr)
|
all = append(all, pr)
|
||||||
}
|
}
|
||||||
agentDebugNDJSON3214("B", "repository/postgres.go:ListRevisionPrefixes", "list_prefixes_fetched", map[string]any{
|
agentDebugNDJSON3214("B", "repository/postgres.go:ListRevisionPrefixes", "list_prefixes_fetched", map[string]any{
|
||||||
"rows": len(all), "limit": limit, "offset": off,
|
"rows": len(all), "limit": limit, "keyset": !useOffset,
|
||||||
})
|
})
|
||||||
more := len(all) > limit
|
more := len(all) > limit
|
||||||
if more {
|
if more {
|
||||||
all = all[:limit]
|
all = all[:limit]
|
||||||
|
ids = ids[:limit]
|
||||||
}
|
}
|
||||||
next := ""
|
next := ""
|
||||||
if more {
|
if more && len(ids) > 0 {
|
||||||
next = fmt.Sprintf("%d", off+limit)
|
next = store.FormatPrefixPageCursor(ids[len(ids)-1])
|
||||||
}
|
}
|
||||||
if len(all) == 0 {
|
if len(all) == 0 {
|
||||||
return nil, "", false
|
return nil, "", false
|
||||||
@@ -820,26 +864,35 @@ func (p *Postgres) CreateRollbackRevision(tenantID, sourceRevisionID string) (st
|
|||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
newID := uuid.NewString()
|
newID := uuid.NewString()
|
||||||
parent := sourceRevisionID
|
parent := sourceRevisionID
|
||||||
meta, _ := json.Marshal(map[string]any{
|
meta, err := revisionMetaWithoutPreview(src.MaterializedPrefixCount)
|
||||||
"preview_fragments": src.PreviewFragments,
|
if err != nil {
|
||||||
"materialized_prefix_count": src.MaterializedPrefixCount,
|
return "", err
|
||||||
})
|
}
|
||||||
var modArg any
|
var modArg any
|
||||||
if strings.TrimSpace(src.ModuleID) != "" {
|
if strings.TrimSpace(src.ModuleID) != "" {
|
||||||
modArg = src.ModuleID
|
modArg = src.ModuleID
|
||||||
}
|
}
|
||||||
_, err = p.pool.Exec(ctx, `
|
tx, err := p.pool.Begin(ctx)
|
||||||
INSERT INTO config_revision (id, tenant_id, module_id, content_hash, parent_revision_id, meta_json)
|
|
||||||
VALUES ($1,$2,$3,$4,$5::uuid,$6::jsonb)`,
|
|
||||||
newID, tenantID, modArg, src.ContentHash+":rollback", parent, string(meta))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
// copy materialized prefixes
|
defer func() { _ = tx.Rollback(ctx) }()
|
||||||
_, _ = p.pool.Exec(ctx, `
|
_, err = tx.Exec(ctx, `
|
||||||
INSERT INTO revision_materialized_prefix (revision_id, prefix, community_id, source, meta_json)
|
INSERT INTO config_revision (id, tenant_id, module_id, content_hash, parent_revision_id, meta_json)
|
||||||
SELECT $1::uuid, prefix, community_id, source, meta_json FROM revision_materialized_prefix WHERE revision_id=$2::uuid`,
|
VALUES ($1,$2,$3,$4,$5::uuid,$6::jsonb)`,
|
||||||
newID, sourceRevisionID)
|
newID, tenantID, modArg, src.ContentHash+":rollback", parent, meta)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := copyRevisionPreview(ctx, tx, newID, sourceRevisionID); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := p.copyRevisionPrefixSnapshotRef(ctx, tx, newID, sourceRevisionID); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := tx.Commit(ctx); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
return newID, nil
|
return newID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -856,20 +909,18 @@ func (p *Postgres) RevisionDiff(tenantID, aID, bID string) (map[string]any, erro
|
|||||||
var unchanged int
|
var unchanged int
|
||||||
err := p.pool.QueryRow(ctx, `
|
err := p.pool.QueryRow(ctx, `
|
||||||
SELECT COUNT(*)::int FROM (
|
SELECT COUNT(*)::int FROM (
|
||||||
SELECT prefix FROM revision_materialized_prefix WHERE revision_id=$1::uuid
|
SELECT b.prefix FROM (`+sqlRevisionPrefixes("$2")+`) b
|
||||||
INTERSECT
|
INNER JOIN (`+sqlRevisionPrefixes("$1")+`) a ON a.prefix = b.prefix
|
||||||
SELECT prefix FROM revision_materialized_prefix WHERE revision_id=$2::uuid
|
|
||||||
) t`, aID, bID).Scan(&unchanged)
|
) t`, aID, bID).Scan(&unchanged)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// added: в B, нет в A; removed: в A, нет в B — без загрузки полных снапшотов в память.
|
|
||||||
rowsAdded, err := p.pool.Query(ctx, `
|
rowsAdded, err := p.pool.Query(ctx, `
|
||||||
SELECT prefix::text FROM (
|
SELECT b.prefix::text FROM (`+sqlRevisionPrefixes("$2")+`) b
|
||||||
SELECT prefix FROM revision_materialized_prefix WHERE revision_id=$1::uuid
|
LEFT JOIN (`+sqlRevisionPrefixes("$1")+`) a ON a.prefix = b.prefix
|
||||||
EXCEPT
|
WHERE a.prefix IS NULL
|
||||||
SELECT prefix FROM revision_materialized_prefix WHERE revision_id=$2::uuid
|
ORDER BY b.prefix
|
||||||
) s ORDER BY 1 LIMIT $3`, bID, aID, maxRevisionDiffRows+1)
|
LIMIT $3`, aID, bID, maxRevisionDiffRows+1)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -888,11 +939,11 @@ func (p *Postgres) RevisionDiff(tenantID, aID, bID string) (map[string]any, erro
|
|||||||
}
|
}
|
||||||
addedTruncated := len(added) >= maxRevisionDiffRows
|
addedTruncated := len(added) >= maxRevisionDiffRows
|
||||||
rowsRem, err := p.pool.Query(ctx, `
|
rowsRem, err := p.pool.Query(ctx, `
|
||||||
SELECT prefix::text FROM (
|
SELECT a.prefix::text FROM (`+sqlRevisionPrefixes("$1")+`) a
|
||||||
SELECT prefix FROM revision_materialized_prefix WHERE revision_id=$1::uuid
|
LEFT JOIN (`+sqlRevisionPrefixes("$2")+`) b ON b.prefix = a.prefix
|
||||||
EXCEPT
|
WHERE b.prefix IS NULL
|
||||||
SELECT prefix FROM revision_materialized_prefix WHERE revision_id=$2::uuid
|
ORDER BY a.prefix
|
||||||
) s ORDER BY 1 LIMIT $3`, aID, bID, maxRevisionDiffRows+1)
|
LIMIT $3`, aID, bID, maxRevisionDiffRows+1)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -1028,10 +1079,7 @@ func (p *Postgres) CreateRenderRevision(revisionID, tenantID, moduleID string, p
|
|||||||
if previewFragments == nil {
|
if previewFragments == nil {
|
||||||
previewFragments = map[string]string{}
|
previewFragments = map[string]string{}
|
||||||
}
|
}
|
||||||
meta, err := json.Marshal(map[string]any{
|
meta, err := revisionMetaWithoutPreview(len(prefixes))
|
||||||
"preview_fragments": previewFragments,
|
|
||||||
"materialized_prefix_count": len(prefixes),
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -1045,32 +1093,23 @@ func (p *Postgres) CreateRenderRevision(revisionID, tenantID, moduleID string, p
|
|||||||
if parentRevisionID != nil && strings.TrimSpace(*parentRevisionID) != "" {
|
if parentRevisionID != nil && strings.TrimSpace(*parentRevisionID) != "" {
|
||||||
parent = strings.TrimSpace(*parentRevisionID)
|
parent = strings.TrimSpace(*parentRevisionID)
|
||||||
}
|
}
|
||||||
|
revID := strings.TrimSpace(revisionID)
|
||||||
_, err = tx.Exec(ctx, `
|
_, err = tx.Exec(ctx, `
|
||||||
INSERT INTO config_revision (id, tenant_id, module_id, content_hash, parent_revision_id, meta_json)
|
INSERT INTO config_revision (id, tenant_id, module_id, content_hash, parent_revision_id, meta_json)
|
||||||
VALUES ($1::uuid, $2::uuid, $3::uuid, $4, $5::uuid, $6::jsonb)`,
|
VALUES ($1::uuid, $2::uuid, $3::uuid, $4, $5::uuid, $6::jsonb)`,
|
||||||
strings.TrimSpace(revisionID), tenantID, moduleID, strings.TrimSpace(contentHash), parent, string(meta))
|
revID, tenantID, moduleID, strings.TrimSpace(contentHash), parent, meta)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if len(prefixes) > 0 {
|
if err := insertRevisionPreview(ctx, tx, revID, previewFragments); err != nil {
|
||||||
_, err = tx.CopyFrom(ctx,
|
return err
|
||||||
pgx.Identifier{"revision_materialized_prefix"},
|
}
|
||||||
[]string{"revision_id", "prefix", "community_id", "source"},
|
snapID, err := p.ensurePrefixSnapshot(ctx, tx, contentHash, prefixes)
|
||||||
pgx.CopyFromSlice(len(prefixes), func(i int) ([]any, error) {
|
if err != nil {
|
||||||
pr := prefixes[i]
|
return err
|
||||||
var comm any
|
}
|
||||||
if pr.CommunityID != nil && strings.TrimSpace(*pr.CommunityID) != "" {
|
if err := p.linkRevisionPrefixSnapshot(ctx, tx, revID, snapID); err != nil {
|
||||||
comm = strings.TrimSpace(*pr.CommunityID)
|
return err
|
||||||
}
|
|
||||||
src := pr.Source
|
|
||||||
if strings.TrimSpace(src) == "" {
|
|
||||||
src = "render"
|
|
||||||
}
|
|
||||||
return []any{strings.TrimSpace(revisionID), strings.TrimSpace(pr.Prefix), comm, src}, nil
|
|
||||||
}))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if err := tx.Commit(ctx); err != nil {
|
if err := tx.Commit(ctx); err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -0,0 +1,186 @@
|
|||||||
|
package repository
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"evobgp/internal/store"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
func prefixSnapshotTableExists(ctx context.Context, q queryRower) bool {
|
||||||
|
var n int
|
||||||
|
err := q.QueryRow(ctx, `
|
||||||
|
SELECT 1 FROM information_schema.tables
|
||||||
|
WHERE table_schema = 'public' AND table_name = 'prefix_snapshot'
|
||||||
|
LIMIT 1`).Scan(&n)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeSnapshotHash(contentHash string) string {
|
||||||
|
h := strings.TrimSpace(contentHash)
|
||||||
|
if strings.HasPrefix(h, "sha256:") {
|
||||||
|
h = strings.TrimPrefix(h, "sha256:")
|
||||||
|
}
|
||||||
|
if len(h) > 64 {
|
||||||
|
h = h[:64]
|
||||||
|
}
|
||||||
|
if len(h) < 64 {
|
||||||
|
h = h + strings.Repeat("0", 64-len(h))
|
||||||
|
}
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Postgres) revisionPrefixSnapshotID(ctx context.Context, revisionID string) (string, bool) {
|
||||||
|
if !prefixSnapshotTableExists(ctx, p.pool) {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
var snap *string
|
||||||
|
err := p.pool.QueryRow(ctx, `
|
||||||
|
SELECT prefix_snapshot_id::text FROM config_revision
|
||||||
|
WHERE id = $1::uuid AND prefix_snapshot_id IS NOT NULL`, revisionID).Scan(&snap)
|
||||||
|
if err != nil || snap == nil || strings.TrimSpace(*snap) == "" {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return *snap, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Postgres) ensurePrefixSnapshot(ctx context.Context, db execQuerier, contentHash string, prefixes []store.PrefixRow) (string, error) {
|
||||||
|
if !prefixSnapshotTableExists(ctx, db) {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
hash := normalizeSnapshotHash(contentHash)
|
||||||
|
var existing string
|
||||||
|
err := db.QueryRow(ctx, `SELECT id::text FROM prefix_snapshot WHERE content_hash = $1`, hash).Scan(&existing)
|
||||||
|
if err == nil && existing != "" {
|
||||||
|
return existing, nil
|
||||||
|
}
|
||||||
|
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
snapID := uuid.NewString()
|
||||||
|
if _, err := db.Exec(ctx, `
|
||||||
|
INSERT INTO prefix_snapshot (id, content_hash) VALUES ($1::uuid, $2)
|
||||||
|
ON CONFLICT (content_hash) DO NOTHING`, snapID, hash); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := db.QueryRow(ctx, `SELECT id::text FROM prefix_snapshot WHERE content_hash = $1`, hash).Scan(&snapID); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
var rowCount int
|
||||||
|
_ = db.QueryRow(ctx, `SELECT COUNT(*)::int FROM prefix_snapshot_row WHERE snapshot_id = $1::uuid`, snapID).Scan(&rowCount)
|
||||||
|
if rowCount > 0 {
|
||||||
|
return snapID, nil
|
||||||
|
}
|
||||||
|
for i, pr := range prefixes {
|
||||||
|
var comm any
|
||||||
|
if pr.CommunityID != nil && strings.TrimSpace(*pr.CommunityID) != "" {
|
||||||
|
comm = strings.TrimSpace(*pr.CommunityID)
|
||||||
|
}
|
||||||
|
src := pr.Source
|
||||||
|
if strings.TrimSpace(src) == "" {
|
||||||
|
src = "render"
|
||||||
|
}
|
||||||
|
if _, err := db.Exec(ctx, `
|
||||||
|
INSERT INTO prefix_snapshot_row (snapshot_id, ord, prefix, community_id, source)
|
||||||
|
VALUES ($1::uuid, $2, $3, $4::uuid, $5)`,
|
||||||
|
snapID, i, strings.TrimSpace(pr.Prefix), comm, src); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return snapID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Postgres) listSnapshotPrefixes(ctx context.Context, snapshotID, cursor string, limit int) ([]store.PrefixRow, string, bool) {
|
||||||
|
afterOrd, off, useOffset := store.ParsePrefixPageCursor(cursor)
|
||||||
|
var rows pgx.Rows
|
||||||
|
var err error
|
||||||
|
if useOffset {
|
||||||
|
rows, err = p.pool.Query(ctx, `
|
||||||
|
SELECT ord, prefix::text, community_id::text, source
|
||||||
|
FROM prefix_snapshot_row
|
||||||
|
WHERE snapshot_id = $1::uuid
|
||||||
|
ORDER BY ord
|
||||||
|
LIMIT $2 OFFSET $3`, snapshotID, limit+1, off)
|
||||||
|
} else {
|
||||||
|
var afterArg any
|
||||||
|
if afterOrd != nil {
|
||||||
|
afterArg = int(*afterOrd)
|
||||||
|
}
|
||||||
|
rows, err = p.pool.Query(ctx, `
|
||||||
|
SELECT ord, prefix::text, community_id::text, source
|
||||||
|
FROM prefix_snapshot_row
|
||||||
|
WHERE snapshot_id = $1::uuid AND ($2::int IS NULL OR ord > $2::int)
|
||||||
|
ORDER BY ord
|
||||||
|
LIMIT $3`, snapshotID, afterArg, limit+1)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", false
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var all []store.PrefixRow
|
||||||
|
var ords []int64
|
||||||
|
for rows.Next() {
|
||||||
|
var ord int
|
||||||
|
var pr store.PrefixRow
|
||||||
|
var comm *string
|
||||||
|
if err := rows.Scan(&ord, &pr.Prefix, &comm, &pr.Source); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pr.CommunityID = comm
|
||||||
|
ords = append(ords, int64(ord))
|
||||||
|
all = append(all, pr)
|
||||||
|
}
|
||||||
|
more := len(all) > limit
|
||||||
|
if more {
|
||||||
|
all = all[:limit]
|
||||||
|
ords = ords[:limit]
|
||||||
|
}
|
||||||
|
next := ""
|
||||||
|
if more && len(ords) > 0 {
|
||||||
|
next = store.FormatPrefixPageCursor(ords[len(ords)-1])
|
||||||
|
}
|
||||||
|
if len(all) == 0 {
|
||||||
|
return nil, "", false
|
||||||
|
}
|
||||||
|
return all, next, more
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Postgres) linkRevisionPrefixSnapshot(ctx context.Context, db execQuerier, revisionID, snapshotID string) error {
|
||||||
|
if snapshotID == "" || !prefixSnapshotTableExists(ctx, db) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
_, err := db.Exec(ctx, `
|
||||||
|
UPDATE config_revision SET prefix_snapshot_id = $2::uuid WHERE id = $1::uuid`,
|
||||||
|
revisionID, snapshotID)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Postgres) copyRevisionPrefixSnapshotRef(ctx context.Context, db execQuerier, dstRevisionID, srcRevisionID string) error {
|
||||||
|
if !prefixSnapshotTableExists(ctx, db) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
_, err := db.Exec(ctx, `
|
||||||
|
UPDATE config_revision dst
|
||||||
|
SET prefix_snapshot_id = src.prefix_snapshot_id
|
||||||
|
FROM config_revision src
|
||||||
|
WHERE dst.id = $1::uuid AND src.id = $2::uuid AND src.prefix_snapshot_id IS NOT NULL`,
|
||||||
|
dstRevisionID, srcRevisionID)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func sqlRevisionPrefixes(revParam string) string {
|
||||||
|
return `SELECT psr.prefix FROM config_revision cr
|
||||||
|
JOIN prefix_snapshot_row psr ON psr.snapshot_id = cr.prefix_snapshot_id
|
||||||
|
WHERE cr.id = ` + revParam + `::uuid AND cr.prefix_snapshot_id IS NOT NULL
|
||||||
|
UNION ALL
|
||||||
|
SELECT rmp.prefix FROM revision_materialized_prefix rmp
|
||||||
|
WHERE rmp.revision_id = ` + revParam + `::uuid
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM config_revision cr2
|
||||||
|
WHERE cr2.id = ` + revParam + `::uuid AND cr2.prefix_snapshot_id IS NOT NULL
|
||||||
|
)`
|
||||||
|
}
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
package repository
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
"github.com/jackc/pgx/v5/pgconn"
|
||||||
|
)
|
||||||
|
|
||||||
|
func revisionPreviewTableExists(ctx context.Context, q queryRower) bool {
|
||||||
|
var n int
|
||||||
|
err := q.QueryRow(ctx, `
|
||||||
|
SELECT 1 FROM information_schema.tables
|
||||||
|
WHERE table_schema = 'public' AND table_name = 'config_revision_preview'
|
||||||
|
LIMIT 1`).Scan(&n)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type queryRower interface {
|
||||||
|
QueryRow(ctx context.Context, sql string, args ...any) pgx.Row
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadRevisionPreview(ctx context.Context, q queryRower, revisionID string, metaPreview map[string]string) map[string]string {
|
||||||
|
if revisionPreviewTableExists(ctx, q) {
|
||||||
|
var raw []byte
|
||||||
|
err := q.QueryRow(ctx, `
|
||||||
|
SELECT fragments FROM config_revision_preview WHERE revision_id = $1::uuid`,
|
||||||
|
revisionID).Scan(&raw)
|
||||||
|
if err == nil {
|
||||||
|
out := map[string]string{}
|
||||||
|
_ = json.Unmarshal(raw, &out)
|
||||||
|
if out == nil {
|
||||||
|
out = map[string]string{}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
if !errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return metaPreview
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if metaPreview == nil {
|
||||||
|
return map[string]string{}
|
||||||
|
}
|
||||||
|
return metaPreview
|
||||||
|
}
|
||||||
|
|
||||||
|
type execQuerier interface {
|
||||||
|
queryRower
|
||||||
|
Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func insertRevisionPreview(ctx context.Context, db execQuerier, revisionID string, preview map[string]string) error {
|
||||||
|
if !revisionPreviewTableExists(ctx, db) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if preview == nil {
|
||||||
|
preview = map[string]string{}
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(preview)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = db.Exec(ctx, `
|
||||||
|
INSERT INTO config_revision_preview (revision_id, fragments)
|
||||||
|
VALUES ($1::uuid, $2::jsonb)
|
||||||
|
ON CONFLICT (revision_id) DO UPDATE SET fragments = EXCLUDED.fragments`,
|
||||||
|
revisionID, string(raw))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyRevisionPreview(ctx context.Context, db execQuerier, dstRevisionID, srcRevisionID string) error {
|
||||||
|
if !revisionPreviewTableExists(ctx, db) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
_, err := db.Exec(ctx, `
|
||||||
|
INSERT INTO config_revision_preview (revision_id, fragments)
|
||||||
|
SELECT $1::uuid, fragments FROM config_revision_preview WHERE revision_id = $2::uuid
|
||||||
|
ON CONFLICT (revision_id) DO UPDATE SET fragments = EXCLUDED.fragments`,
|
||||||
|
dstRevisionID, srcRevisionID)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func revisionMetaWithoutPreview(materializedPrefixCount int) (string, error) {
|
||||||
|
raw, err := json.Marshal(map[string]any{
|
||||||
|
"materialized_prefix_count": materializedPrefixCount,
|
||||||
|
})
|
||||||
|
return string(raw), err
|
||||||
|
}
|
||||||
@@ -26,9 +26,8 @@ func (p *Postgres) seedDemo(ctx context.Context) error {
|
|||||||
p1 := uuid.NewString()
|
p1 := uuid.NewString()
|
||||||
p2 := uuid.NewString()
|
p2 := uuid.NewString()
|
||||||
|
|
||||||
preview := map[string]any{
|
previewFrags := map[string]string{
|
||||||
"preview_fragments": map[string]string{
|
"bird.conf": `# EvoBGP demo bundle
|
||||||
"bird.conf": `# EvoBGP demo bundle
|
|
||||||
router id 192.0.2.1;
|
router id 192.0.2.1;
|
||||||
|
|
||||||
protocol device {
|
protocol device {
|
||||||
@@ -39,15 +38,12 @@ protocol direct {
|
|||||||
ipv6;
|
ipv6;
|
||||||
}
|
}
|
||||||
`,
|
`,
|
||||||
"bird.d/evobgp_demo.conf": "# static demo fragment\n",
|
"bird.d/evobgp_demo.conf": "# static demo fragment\n",
|
||||||
},
|
|
||||||
"materialized_prefix_count": 128,
|
|
||||||
}
|
}
|
||||||
previewB, _ := json.Marshal(preview)
|
previewMeta, _ := json.Marshal(map[string]any{"materialized_prefix_count": 128})
|
||||||
parentMeta, _ := json.Marshal(map[string]any{
|
parentMeta, _ := json.Marshal(map[string]any{"materialized_prefix_count": 0})
|
||||||
"preview_fragments": map[string]string{"bird.conf": "# parent revision\n"},
|
parentPreview, _ := json.Marshal(map[string]string{"bird.conf": "# parent revision\n"})
|
||||||
"materialized_prefix_count": 0,
|
previewFragsB, _ := json.Marshal(previewFrags)
|
||||||
})
|
|
||||||
|
|
||||||
tx, err := p.pool.Begin(ctx)
|
tx, err := p.pool.Begin(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -77,13 +73,32 @@ protocol direct {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if _, err := tx.Exec(ctx, `
|
if _, err := tx.Exec(ctx, `
|
||||||
INSERT INTO config_revision (id, tenant_id, content_hash, parent_revision_id, meta_json)
|
INSERT INTO config_revision_preview (revision_id, fragments) VALUES ($1::uuid, $2::jsonb)`, parent, string(parentPreview)); err != nil {
|
||||||
VALUES ($1,$2,'sha256:demo-rev-1',$3::uuid,$4::jsonb)`, rid, tid, parent, string(previewB)); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if _, err := tx.Exec(ctx, `
|
if _, err := tx.Exec(ctx, `
|
||||||
INSERT INTO revision_materialized_prefix (revision_id, prefix, community_id, source)
|
INSERT INTO config_revision (id, tenant_id, content_hash, parent_revision_id, meta_json)
|
||||||
VALUES ($1::uuid,'203.0.113.0/24',$2::uuid,'demo'), ($1::uuid,'2001:db8::/32',$2::uuid,'demo')`, rid, cid); err != nil {
|
VALUES ($1,$2,'sha256:demo-rev-1',$3::uuid,$4::jsonb)`, rid, tid, parent, string(previewMeta)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(ctx, `
|
||||||
|
INSERT INTO config_revision_preview (revision_id, fragments) VALUES ($1::uuid, $2::jsonb)`, rid, string(previewFragsB)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
snapID := uuid.NewString()
|
||||||
|
demoHash := normalizeSnapshotHash("sha256:demo-rev-1")
|
||||||
|
if _, err := tx.Exec(ctx, `
|
||||||
|
INSERT INTO prefix_snapshot (id, content_hash) VALUES ($1::uuid, $2)`, snapID, demoHash); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(ctx, `
|
||||||
|
UPDATE config_revision SET prefix_snapshot_id = $2::uuid WHERE id = $1::uuid`, rid, snapID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(ctx, `
|
||||||
|
INSERT INTO prefix_snapshot_row (snapshot_id, ord, prefix, community_id, source)
|
||||||
|
VALUES ($1::uuid, 0, '203.0.113.0/24', $2::uuid, 'demo'),
|
||||||
|
($1::uuid, 1, '2001:db8::/32', $2::uuid, 'demo')`, snapID, cid); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if _, err := tx.Exec(ctx, `
|
if _, err := tx.Exec(ctx, `
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
|
|
||||||
"evobgp/internal/broker"
|
"evobgp/internal/broker"
|
||||||
"evobgp/internal/config"
|
"evobgp/internal/config"
|
||||||
|
"evobgp/internal/httpclient"
|
||||||
"evobgp/internal/jobs"
|
"evobgp/internal/jobs"
|
||||||
"evobgp/internal/pipeline"
|
"evobgp/internal/pipeline"
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
@@ -108,6 +109,9 @@ func postTenantRefresh(ctx context.Context, deps *Deps, moduleIDs []string, idem
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
req.GetBody = func() (io.ReadCloser, error) {
|
||||||
|
return io.NopCloser(bytes.NewReader(body)), nil
|
||||||
|
}
|
||||||
req.Header.Set("Authorization", "Bearer "+strings.TrimSpace(deps.APIToken))
|
req.Header.Set("Authorization", "Bearer "+strings.TrimSpace(deps.APIToken))
|
||||||
req.Header.Set("Content-Type", "application/json")
|
req.Header.Set("Content-Type", "application/json")
|
||||||
if idempotencyKey != "" {
|
if idempotencyKey != "" {
|
||||||
@@ -115,9 +119,9 @@ func postTenantRefresh(ctx context.Context, deps *Deps, moduleIDs []string, idem
|
|||||||
}
|
}
|
||||||
hc := deps.HTTP
|
hc := deps.HTTP
|
||||||
if hc == nil {
|
if hc == nil {
|
||||||
hc = http.DefaultClient
|
hc = httpclient.New(httpclient.DefaultTimeout)
|
||||||
}
|
}
|
||||||
resp, err := hc.Do(req)
|
resp, err := httpclient.DoWithRetry(ctx, hc, req, 3)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -76,6 +76,8 @@ type Backend interface {
|
|||||||
DeleteSpeaker(tenantID, id string) error
|
DeleteSpeaker(tenantID, id string) error
|
||||||
|
|
||||||
GetRevision(tenantID, revisionID string) (*Revision, error)
|
GetRevision(tenantID, revisionID string) (*Revision, error)
|
||||||
|
// GetRevisionSummary returns revision metadata without preview_fragments payloads.
|
||||||
|
GetRevisionSummary(tenantID, revisionID string) (*Revision, error)
|
||||||
ListRevisions(tenantID, moduleID string, cursor string, limit int) (items []*Revision, nextCursor string, hasMore bool)
|
ListRevisions(tenantID, moduleID string, cursor string, limit int) (items []*Revision, nextCursor string, hasMore bool)
|
||||||
ListRevisionPrefixes(tenantID, revisionID string, cursor string, limit int) (prefixes []PrefixRow, next string, more bool)
|
ListRevisionPrefixes(tenantID, revisionID string, cursor string, limit int) (prefixes []PrefixRow, next string, more bool)
|
||||||
CreateRollbackRevision(tenantID, sourceRevisionID string) (newID string, err error)
|
CreateRollbackRevision(tenantID, sourceRevisionID string) (newID string, err error)
|
||||||
@@ -110,6 +112,9 @@ type Backend interface {
|
|||||||
|
|
||||||
// Ping verifies backend connectivity (no-op for in-memory).
|
// Ping verifies backend connectivity (no-op for in-memory).
|
||||||
Ping(ctx context.Context) error
|
Ping(ctx context.Context) error
|
||||||
|
|
||||||
|
// RunPeriodicMaintenance prunes stale DB rows (no-op for in-memory).
|
||||||
|
RunPeriodicMaintenance(ctx context.Context)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ASNPrefixCacheEntry is a cached RIPEstat response for one ASN.
|
// ASNPrefixCacheEntry is a cached RIPEstat response for one ASN.
|
||||||
@@ -309,7 +314,7 @@ type SpeakerPatch struct {
|
|||||||
|
|
||||||
// PrefixRow is one materialized prefix for GET /revisions/.../prefixes.
|
// PrefixRow is one materialized prefix for GET /revisions/.../prefixes.
|
||||||
type PrefixRow struct {
|
type PrefixRow struct {
|
||||||
Prefix string
|
Prefix string `json:"prefix"`
|
||||||
CommunityID *string
|
CommunityID *string `json:"community_id,omitempty"`
|
||||||
Source string
|
Source string `json:"source,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -305,6 +305,11 @@ func (m *Memory) Ping(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RunPeriodicMaintenance is a no-op for the in-memory backend.
|
||||||
|
func (m *Memory) RunPeriodicMaintenance(ctx context.Context) {
|
||||||
|
_ = ctx
|
||||||
|
}
|
||||||
|
|
||||||
// ListTenantIDs returns tenant ids sorted lexicographically.
|
// ListTenantIDs returns tenant ids sorted lexicographically.
|
||||||
func (m *Memory) ListTenantIDs() ([]string, error) {
|
func (m *Memory) ListTenantIDs() ([]string, error) {
|
||||||
m.mu.RLock()
|
m.mu.RLock()
|
||||||
@@ -424,14 +429,19 @@ func (m *Memory) GetModule(tenantID, moduleID string) (*Module, error) {
|
|||||||
func (m *Memory) GetRevision(tenantID, revisionID string) (*Revision, error) {
|
func (m *Memory) GetRevision(tenantID, revisionID string) (*Revision, error) {
|
||||||
m.mu.RLock()
|
m.mu.RLock()
|
||||||
defer m.mu.RUnlock()
|
defer m.mu.RUnlock()
|
||||||
rev, ok := m.revisions[revisionID]
|
return m.getRevisionLocked(tenantID, revisionID)
|
||||||
if !ok {
|
}
|
||||||
return nil, ErrNotFound
|
|
||||||
|
func (m *Memory) GetRevisionSummary(tenantID, revisionID string) (*Revision, error) {
|
||||||
|
m.mu.RLock()
|
||||||
|
defer m.mu.RUnlock()
|
||||||
|
rev, err := m.getRevisionLocked(tenantID, revisionID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
if rev.TenantID != tenantID {
|
cp := *rev
|
||||||
return nil, ErrTenantScope
|
cp.PreviewFragments = nil
|
||||||
}
|
return &cp, nil
|
||||||
return rev, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Memory) GetSpeaker(tenantID, speakerID string) (*Speaker, error) {
|
func (m *Memory) GetSpeaker(tenantID, speakerID string) (*Speaker, error) {
|
||||||
|
|||||||
@@ -1,8 +1,6 @@
|
|||||||
package store
|
package store
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -795,31 +793,37 @@ func (m *Memory) ListRevisionPrefixes(tenantID, revisionID string, cursor string
|
|||||||
if limit <= 0 {
|
if limit <= 0 {
|
||||||
limit = 50
|
limit = 50
|
||||||
}
|
}
|
||||||
|
afterID, off, useOffset := ParsePrefixPageCursor(cursor)
|
||||||
m.mu.RLock()
|
m.mu.RLock()
|
||||||
defer m.mu.RUnlock()
|
defer m.mu.RUnlock()
|
||||||
if _, err := m.getRevisionLocked(tenantID, revisionID); err != nil {
|
if _, err := m.getRevisionLocked(tenantID, revisionID); err != nil {
|
||||||
return nil, "", false
|
return nil, "", false
|
||||||
}
|
}
|
||||||
all := m.revPrefixes[revisionID]
|
allRows := m.revPrefixes[revisionID]
|
||||||
off := 0
|
start := 0
|
||||||
if cursor != "" {
|
if useOffset {
|
||||||
if n, err := strconv.Atoi(cursor); err == nil && n >= 0 {
|
start = off
|
||||||
off = n
|
} else if afterID != nil {
|
||||||
}
|
start = int(*afterID) + 1
|
||||||
}
|
}
|
||||||
end := off + limit
|
if start > len(allRows) {
|
||||||
next := ""
|
|
||||||
more := false
|
|
||||||
if end > len(all) {
|
|
||||||
end = len(all)
|
|
||||||
} else {
|
|
||||||
more = true
|
|
||||||
next = fmt.Sprintf("%d", end)
|
|
||||||
}
|
|
||||||
if off >= len(all) {
|
|
||||||
return nil, "", false
|
return nil, "", false
|
||||||
}
|
}
|
||||||
return all[off:end], next, more
|
end := start + limit
|
||||||
|
next := ""
|
||||||
|
more := false
|
||||||
|
if end > len(allRows) {
|
||||||
|
end = len(allRows)
|
||||||
|
} else {
|
||||||
|
more = true
|
||||||
|
next = FormatPrefixPageCursor(int64(end - 1))
|
||||||
|
}
|
||||||
|
if start >= end {
|
||||||
|
return nil, "", false
|
||||||
|
}
|
||||||
|
out := make([]PrefixRow, end-start)
|
||||||
|
copy(out, allRows[start:end])
|
||||||
|
return out, next, more
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Memory) ListGlobalSettings(tenantID string) (map[string]any, error) {
|
func (m *Memory) ListGlobalSettings(tenantID string) (map[string]any, error) {
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ParsePrefixPageCursor decodes opaque cursors for revision prefix pagination.
|
||||||
|
func ParsePrefixPageCursor(cursor string) (afterID *int64, offset int, useOffset bool) {
|
||||||
|
cursor = strings.TrimSpace(cursor)
|
||||||
|
if cursor == "" {
|
||||||
|
return nil, 0, false
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(cursor, "o:") {
|
||||||
|
n, err := strconv.Atoi(strings.TrimPrefix(cursor, "o:"))
|
||||||
|
if err != nil || n < 0 {
|
||||||
|
return nil, 0, false
|
||||||
|
}
|
||||||
|
return nil, n, true
|
||||||
|
}
|
||||||
|
if n, err := strconv.ParseInt(cursor, 10, 64); err == nil && n >= 0 {
|
||||||
|
return &n, 0, false
|
||||||
|
}
|
||||||
|
return nil, 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// FormatPrefixPageCursor encodes the keyset cursor (last row id or slice index).
|
||||||
|
func FormatPrefixPageCursor(lastID int64) string {
|
||||||
|
return strconv.FormatInt(lastID, 10)
|
||||||
|
}
|
||||||
@@ -72,7 +72,9 @@ func MergeSpeakerMetaJSON(existing string, patch SpeakerMeta) string {
|
|||||||
if patch.LastDispatchAt != "" {
|
if patch.LastDispatchAt != "" {
|
||||||
cur.LastDispatchAt = patch.LastDispatchAt
|
cur.LastDispatchAt = patch.LastDispatchAt
|
||||||
}
|
}
|
||||||
if patch.LastDispatchError != "" {
|
if patch.LastDispatchStatus == "ok" {
|
||||||
|
cur.LastDispatchError = ""
|
||||||
|
} else if patch.LastDispatchError != "" {
|
||||||
cur.LastDispatchError = patch.LastDispatchError
|
cur.LastDispatchError = patch.LastDispatchError
|
||||||
}
|
}
|
||||||
if patch.LastDispatchStatus != "" {
|
if patch.LastDispatchStatus != "" {
|
||||||
|
|||||||
@@ -34,3 +34,24 @@ func TestAgentSyncURL(t *testing.T) {
|
|||||||
t.Fatalf("got %q", u)
|
t.Fatalf("got %q", u)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMergeSpeakerMetaJSON_clearsDispatchErrorOnOk(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
existing := store.SpeakerMetaJSON(store.SpeakerMeta{
|
||||||
|
LastDispatchError: "HTTP 502: bundle 403",
|
||||||
|
LastDispatchStatus: "error",
|
||||||
|
SyncStatus: "error",
|
||||||
|
})
|
||||||
|
merged := store.MergeSpeakerMetaJSON(existing, store.SpeakerMeta{
|
||||||
|
LastDispatchStatus: "ok",
|
||||||
|
SyncStatus: "synced",
|
||||||
|
LastDispatchAt: "2026-05-21T15:06:43Z",
|
||||||
|
})
|
||||||
|
m := store.ParseSpeakerMeta(merged)
|
||||||
|
if m.LastDispatchError != "" {
|
||||||
|
t.Fatalf("LastDispatchError should clear on ok dispatch, got %q", m.LastDispatchError)
|
||||||
|
}
|
||||||
|
if m.LastDispatchStatus != "ok" || m.SyncStatus != "synced" {
|
||||||
|
t.Fatalf("status: dispatch=%q sync=%q", m.LastDispatchStatus, m.SyncStatus)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
DROP INDEX IF EXISTS idx_rev_mat_prefix_uniq;
|
||||||
|
DROP INDEX IF EXISTS idx_rev_mat_prefix_rev_cover;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_rev_mat_prefix_rev_id
|
||||||
|
ON revision_materialized_prefix (revision_id, id);
|
||||||
|
DROP INDEX IF EXISTS idx_module_tenant_active_sort;
|
||||||
|
DROP INDEX IF EXISTS idx_rev_mat_prefix_rev_prefix;
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
-- Query indexes: revision diff/list and module sort (H4 + M4).
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_rev_mat_prefix_rev_prefix
|
||||||
|
ON revision_materialized_prefix (revision_id, prefix);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_module_tenant_active_sort
|
||||||
|
ON module (tenant_id, priority, name)
|
||||||
|
WHERE deleted_at IS NULL;
|
||||||
|
|
||||||
|
-- Covering index for prefix list pagination (PostgreSQL 11+ INCLUDE).
|
||||||
|
DROP INDEX IF EXISTS idx_rev_mat_prefix_rev_id;
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_rev_mat_prefix_rev_cover
|
||||||
|
ON revision_materialized_prefix (revision_id, id)
|
||||||
|
INCLUDE (prefix, community_id, source);
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_rev_mat_prefix_uniq
|
||||||
|
ON revision_materialized_prefix (revision_id, prefix, community_id, source);
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
UPDATE config_revision cr
|
||||||
|
SET meta_json = cr.meta_json || jsonb_build_object('preview_fragments', COALESCE(p.fragments, '{}'::jsonb))
|
||||||
|
FROM config_revision_preview p
|
||||||
|
WHERE p.revision_id = cr.id;
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS config_revision_preview;
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
-- Split BIRD preview fragments out of config_revision.meta_json (H1).
|
||||||
|
|
||||||
|
CREATE TABLE config_revision_preview (
|
||||||
|
revision_id UUID PRIMARY KEY REFERENCES config_revision (id) ON DELETE CASCADE,
|
||||||
|
fragments JSONB NOT NULL DEFAULT '{}'
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT INTO config_revision_preview (revision_id, fragments)
|
||||||
|
SELECT id, COALESCE(meta_json->'preview_fragments', '{}'::jsonb)
|
||||||
|
FROM config_revision
|
||||||
|
WHERE meta_json ? 'preview_fragments';
|
||||||
|
|
||||||
|
UPDATE config_revision
|
||||||
|
SET meta_json = meta_json - 'preview_fragments'
|
||||||
|
WHERE meta_json ? 'preview_fragments';
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
ALTER TABLE config_revision DROP COLUMN IF EXISTS prefix_snapshot_id;
|
||||||
|
DROP TABLE IF EXISTS prefix_snapshot_row;
|
||||||
|
DROP TABLE IF EXISTS prefix_snapshot;
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
-- Content-addressed prefix snapshots (H2 expand).
|
||||||
|
|
||||||
|
CREATE TABLE prefix_snapshot (
|
||||||
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
content_hash CHAR(64) NOT NULL,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
CONSTRAINT prefix_snapshot_hash_uniq UNIQUE (content_hash)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE prefix_snapshot_row (
|
||||||
|
snapshot_id UUID NOT NULL REFERENCES prefix_snapshot (id) ON DELETE CASCADE,
|
||||||
|
ord INTEGER NOT NULL,
|
||||||
|
prefix TEXT NOT NULL,
|
||||||
|
community_id UUID REFERENCES bgp_community (id) ON DELETE SET NULL,
|
||||||
|
source TEXT NOT NULL DEFAULT '',
|
||||||
|
PRIMARY KEY (snapshot_id, ord)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_prefix_snapshot_row_snapshot ON prefix_snapshot_row (snapshot_id);
|
||||||
|
|
||||||
|
ALTER TABLE config_revision
|
||||||
|
ADD COLUMN prefix_snapshot_id UUID REFERENCES prefix_snapshot (id) ON DELETE RESTRICT;
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
DELETE FROM prefix_snapshot_row;
|
||||||
|
UPDATE config_revision SET prefix_snapshot_id = NULL WHERE prefix_snapshot_id IS NOT NULL;
|
||||||
|
DELETE FROM prefix_snapshot;
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
-- Backfill prefix snapshots from revision_materialized_prefix.
|
||||||
|
-- prefix_snapshot_row.prefix is TEXT (revision_materialized_prefix.prefix since 000003).
|
||||||
|
|
||||||
|
ALTER TABLE prefix_snapshot_row
|
||||||
|
ALTER COLUMN prefix TYPE TEXT USING prefix::text;
|
||||||
|
|
||||||
|
WITH new_snaps AS (
|
||||||
|
INSERT INTO prefix_snapshot (id, content_hash)
|
||||||
|
SELECT gen_random_uuid(),
|
||||||
|
substr(replace(cr.id::text, '-', '') || replace(cr.id::text, '-', ''), 1, 64)
|
||||||
|
FROM config_revision cr
|
||||||
|
WHERE cr.prefix_snapshot_id IS NULL
|
||||||
|
AND EXISTS (
|
||||||
|
SELECT 1 FROM revision_materialized_prefix rmp WHERE rmp.revision_id = cr.id
|
||||||
|
)
|
||||||
|
RETURNING id, content_hash
|
||||||
|
)
|
||||||
|
UPDATE config_revision cr
|
||||||
|
SET prefix_snapshot_id = ns.id
|
||||||
|
FROM new_snaps ns
|
||||||
|
WHERE cr.prefix_snapshot_id IS NULL
|
||||||
|
AND ns.content_hash = substr(replace(cr.id::text, '-', '') || replace(cr.id::text, '-', ''), 1, 64);
|
||||||
|
|
||||||
|
INSERT INTO prefix_snapshot_row (snapshot_id, ord, prefix, community_id, source)
|
||||||
|
SELECT cr.prefix_snapshot_id,
|
||||||
|
(row_number() OVER (PARTITION BY cr.id ORDER BY rmp.id) - 1)::int,
|
||||||
|
rmp.prefix,
|
||||||
|
rmp.community_id,
|
||||||
|
rmp.source
|
||||||
|
FROM config_revision cr
|
||||||
|
JOIN revision_materialized_prefix rmp ON rmp.revision_id = cr.id
|
||||||
|
WHERE cr.prefix_snapshot_id IS NOT NULL
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM prefix_snapshot_row psr WHERE psr.snapshot_id = cr.prefix_snapshot_id
|
||||||
|
);
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
COMMENT ON COLUMN config_revision.prefix_snapshot_id IS NULL;
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
-- H2 contract marker: new revisions use prefix_snapshot_id only (enforced in application code).
|
||||||
|
|
||||||
|
COMMENT ON COLUMN config_revision.prefix_snapshot_id IS 'Materialized prefixes; revision_materialized_prefix deprecated for new rows';
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
ALTER TABLE module_prefix_snapshot ADD COLUMN prefixes_json JSONB NOT NULL DEFAULT '[]';
|
||||||
|
|
||||||
|
UPDATE module_prefix_snapshot mps
|
||||||
|
SET prefixes_json = COALESCE((
|
||||||
|
SELECT jsonb_agg(
|
||||||
|
jsonb_build_object(
|
||||||
|
'prefix', psr.prefix::text,
|
||||||
|
'community_id', psr.community_id,
|
||||||
|
'source', psr.source
|
||||||
|
) ORDER BY psr.ord
|
||||||
|
)
|
||||||
|
FROM module_prefix_snapshot_row psr
|
||||||
|
WHERE psr.tenant_id = mps.tenant_id AND psr.module_id = mps.module_id
|
||||||
|
), '[]'::jsonb);
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS module_prefix_snapshot_row;
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
CREATE TABLE module_prefix_snapshot_row (
|
||||||
|
tenant_id UUID NOT NULL,
|
||||||
|
module_id UUID NOT NULL,
|
||||||
|
ord INTEGER NOT NULL,
|
||||||
|
prefix TEXT NOT NULL,
|
||||||
|
community_id UUID,
|
||||||
|
source TEXT NOT NULL DEFAULT '',
|
||||||
|
PRIMARY KEY (tenant_id, module_id, ord),
|
||||||
|
FOREIGN KEY (tenant_id, module_id)
|
||||||
|
REFERENCES module_prefix_snapshot (tenant_id, module_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- prefixes_json from Go json.Marshal(PrefixRow) used "Prefix"/"CommunityID"/"Source" before json tags.
|
||||||
|
INSERT INTO module_prefix_snapshot_row (tenant_id, module_id, ord, prefix, community_id, source)
|
||||||
|
SELECT tenant_id,
|
||||||
|
module_id,
|
||||||
|
(row_number() OVER (PARTITION BY tenant_id, module_id ORDER BY ordinality) - 1)::int,
|
||||||
|
prefix,
|
||||||
|
NULLIF(community_id, '')::uuid,
|
||||||
|
COALESCE(source, '')
|
||||||
|
FROM (
|
||||||
|
SELECT mps.tenant_id,
|
||||||
|
mps.module_id,
|
||||||
|
t.ordinality,
|
||||||
|
COALESCE(
|
||||||
|
NULLIF(trim(t.elem->>'prefix'), ''),
|
||||||
|
NULLIF(trim(t.elem->>'Prefix'), '')
|
||||||
|
) AS prefix,
|
||||||
|
COALESCE(
|
||||||
|
NULLIF(trim(t.elem->>'community_id'), ''),
|
||||||
|
NULLIF(trim(t.elem->>'CommunityID'), '')
|
||||||
|
) AS community_id,
|
||||||
|
COALESCE(
|
||||||
|
NULLIF(trim(t.elem->>'source'), ''),
|
||||||
|
NULLIF(trim(t.elem->>'Source'), ''),
|
||||||
|
''
|
||||||
|
) AS source
|
||||||
|
FROM module_prefix_snapshot mps
|
||||||
|
CROSS JOIN LATERAL jsonb_array_elements(mps.prefixes_json) WITH ORDINALITY AS t(elem, ordinality)
|
||||||
|
WHERE jsonb_typeof(mps.prefixes_json) = 'array'
|
||||||
|
AND jsonb_array_length(mps.prefixes_json) > 0
|
||||||
|
) parsed
|
||||||
|
WHERE parsed.prefix IS NOT NULL
|
||||||
|
AND trim(parsed.prefix) <> '';
|
||||||
|
|
||||||
|
ALTER TABLE module_prefix_snapshot DROP COLUMN prefixes_json;
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
ALTER TABLE asn_prefix_cache ADD COLUMN prefixes_json JSONB NOT NULL DEFAULT '[]';
|
||||||
|
|
||||||
|
UPDATE asn_prefix_cache apc
|
||||||
|
SET prefixes_json = COALESCE((
|
||||||
|
SELECT jsonb_agg(apcr.prefix::text ORDER BY apcr.prefix::text)
|
||||||
|
FROM asn_prefix_cache_row apcr
|
||||||
|
WHERE apcr.asn = apc.asn
|
||||||
|
), '[]'::jsonb);
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS asn_prefix_cache_row;
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
CREATE TABLE asn_prefix_cache_row (
|
||||||
|
asn BIGINT NOT NULL REFERENCES asn_prefix_cache (asn) ON DELETE CASCADE,
|
||||||
|
prefix CIDR NOT NULL,
|
||||||
|
PRIMARY KEY (asn, prefix)
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT INTO asn_prefix_cache_row (asn, prefix)
|
||||||
|
SELECT apc.asn, t.elem::cidr
|
||||||
|
FROM asn_prefix_cache apc
|
||||||
|
CROSS JOIN LATERAL jsonb_array_elements_text(apc.prefixes_json) AS t(elem)
|
||||||
|
WHERE jsonb_typeof(apc.prefixes_json) = 'array'
|
||||||
|
AND jsonb_array_length(apc.prefixes_json) > 0;
|
||||||
|
|
||||||
|
ALTER TABLE asn_prefix_cache DROP COLUMN prefixes_json;
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
DROP INDEX IF EXISTS idx_job_audit_created_brin;
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
CREATE INDEX IF NOT EXISTS idx_job_audit_created_brin
|
||||||
|
ON job_audit USING BRIN (created_at);
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
CREATE TABLE IF NOT EXISTS module_cdn_fetch_log (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
module_id UUID NOT NULL REFERENCES module (id) ON DELETE CASCADE,
|
||||||
|
source_id UUID REFERENCES module_cdn_source (id) ON DELETE SET NULL,
|
||||||
|
http_status INTEGER,
|
||||||
|
bytes BIGINT,
|
||||||
|
error TEXT,
|
||||||
|
fetched_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_module_cdn_fetch_log_module ON module_cdn_fetch_log (module_id, fetched_at DESC);
|
||||||
|
|
||||||
|
ALTER TABLE revision_materialized_prefix ADD COLUMN IF NOT EXISTS meta_json JSONB NOT NULL DEFAULT '{}';
|
||||||
|
ALTER TABLE module_domain_entry ADD COLUMN IF NOT EXISTS resolve_meta JSONB NOT NULL DEFAULT '{}';
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_rev_mat_prefix_revision ON revision_materialized_prefix (revision_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_module_tenant ON module (tenant_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_module_prefix_snapshot_collected ON module_prefix_snapshot (collected_at);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_rev_mat_prefix_value ON revision_materialized_prefix (prefix);
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
ALTER TABLE revision_materialized_prefix DROP COLUMN IF EXISTS meta_json;
|
||||||
|
ALTER TABLE module_domain_entry DROP COLUMN IF EXISTS resolve_meta;
|
||||||
|
DROP TABLE IF EXISTS module_cdn_fetch_log;
|
||||||
|
|
||||||
|
DROP INDEX IF EXISTS idx_rev_mat_prefix_revision;
|
||||||
|
DROP INDEX IF EXISTS idx_module_tenant;
|
||||||
|
DROP INDEX IF EXISTS idx_module_prefix_snapshot_collected;
|
||||||
|
DROP INDEX IF EXISTS idx_rev_mat_prefix_value;
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
DROP INDEX IF EXISTS idx_rev_mat_prefix_uniq;
|
||||||
|
DROP INDEX IF EXISTS idx_module_tenant_active_sort;
|
||||||
|
DROP INDEX IF EXISTS idx_rev_mat_prefix_rev_prefix;
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
CREATE INDEX IF NOT EXISTS idx_rev_mat_prefix_rev_prefix
|
||||||
|
ON revision_materialized_prefix (revision_id, prefix);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_module_tenant_active_sort
|
||||||
|
ON module (tenant_id, priority, name)
|
||||||
|
WHERE deleted_at IS NULL;
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_rev_mat_prefix_uniq
|
||||||
|
ON revision_materialized_prefix (revision_id, prefix, community_id, source);
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
UPDATE config_revision
|
||||||
|
SET meta_json = json_set(
|
||||||
|
meta_json,
|
||||||
|
'$.preview_fragments',
|
||||||
|
json(COALESCE((SELECT fragments FROM config_revision_preview p WHERE p.revision_id = config_revision.id), '{}'))
|
||||||
|
)
|
||||||
|
WHERE id IN (SELECT revision_id FROM config_revision_preview);
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS config_revision_preview;
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
CREATE TABLE config_revision_preview (
|
||||||
|
revision_id TEXT NOT NULL PRIMARY KEY REFERENCES config_revision (id) ON DELETE CASCADE,
|
||||||
|
fragments TEXT NOT NULL DEFAULT '{}'
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT INTO config_revision_preview (revision_id, fragments)
|
||||||
|
SELECT id, json(COALESCE(json_extract(meta_json, '$.preview_fragments'), '{}'))
|
||||||
|
FROM config_revision
|
||||||
|
WHERE json_extract(meta_json, '$.preview_fragments') IS NOT NULL;
|
||||||
|
|
||||||
|
UPDATE config_revision
|
||||||
|
SET meta_json = json_remove(meta_json, '$.preview_fragments')
|
||||||
|
WHERE json_extract(meta_json, '$.preview_fragments') IS NOT NULL;
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
ALTER TABLE config_revision DROP COLUMN prefix_snapshot_id;
|
||||||
|
DROP TABLE IF EXISTS prefix_snapshot_row;
|
||||||
|
DROP TABLE IF EXISTS prefix_snapshot;
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
CREATE TABLE prefix_snapshot (
|
||||||
|
id TEXT NOT NULL PRIMARY KEY,
|
||||||
|
content_hash TEXT NOT NULL UNIQUE,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE prefix_snapshot_row (
|
||||||
|
snapshot_id TEXT NOT NULL REFERENCES prefix_snapshot (id) ON DELETE CASCADE,
|
||||||
|
ord INTEGER NOT NULL,
|
||||||
|
prefix TEXT NOT NULL,
|
||||||
|
community_id TEXT,
|
||||||
|
source TEXT NOT NULL DEFAULT '',
|
||||||
|
PRIMARY KEY (snapshot_id, ord)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_prefix_snapshot_row_snapshot ON prefix_snapshot_row (snapshot_id);
|
||||||
|
|
||||||
|
ALTER TABLE config_revision ADD COLUMN prefix_snapshot_id TEXT REFERENCES prefix_snapshot (id) ON DELETE RESTRICT;
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
DELETE FROM prefix_snapshot_row;
|
||||||
|
UPDATE config_revision SET prefix_snapshot_id = NULL;
|
||||||
|
DELETE FROM prefix_snapshot;
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
-- SQLite backfill: one snapshot per revision with materialized prefixes.
|
||||||
|
|
||||||
|
INSERT INTO prefix_snapshot (id, content_hash)
|
||||||
|
SELECT lower(hex(randomblob(16))),
|
||||||
|
substr(replace(cr.id, '-', '') || replace(cr.id, '-', ''), 1, 64)
|
||||||
|
FROM config_revision cr
|
||||||
|
WHERE cr.prefix_snapshot_id IS NULL
|
||||||
|
AND EXISTS (SELECT 1 FROM revision_materialized_prefix rmp WHERE rmp.revision_id = cr.id);
|
||||||
|
|
||||||
|
UPDATE config_revision
|
||||||
|
SET prefix_snapshot_id = (
|
||||||
|
SELECT ps.id FROM prefix_snapshot ps
|
||||||
|
WHERE ps.content_hash = substr(replace(config_revision.id, '-', '') || replace(config_revision.id, '-', ''), 1, 64)
|
||||||
|
)
|
||||||
|
WHERE prefix_snapshot_id IS NULL
|
||||||
|
AND EXISTS (SELECT 1 FROM revision_materialized_prefix rmp WHERE rmp.revision_id = config_revision.id);
|
||||||
|
|
||||||
|
INSERT INTO prefix_snapshot_row (snapshot_id, ord, prefix, community_id, source)
|
||||||
|
SELECT cr.prefix_snapshot_id,
|
||||||
|
(SELECT COUNT(*) FROM revision_materialized_prefix r2
|
||||||
|
WHERE r2.revision_id = cr.id AND r2.id <= rmp.id) - 1,
|
||||||
|
rmp.prefix,
|
||||||
|
rmp.community_id,
|
||||||
|
rmp.source
|
||||||
|
FROM config_revision cr
|
||||||
|
JOIN revision_materialized_prefix rmp ON rmp.revision_id = cr.id
|
||||||
|
WHERE cr.prefix_snapshot_id IS NOT NULL
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM prefix_snapshot_row psr WHERE psr.snapshot_id = cr.prefix_snapshot_id
|
||||||
|
);
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
-- H2 contract marker (application stops writing revision_materialized_prefix for new revisions).
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
ALTER TABLE module_prefix_snapshot ADD COLUMN prefixes_json TEXT NOT NULL DEFAULT '[]';
|
||||||
|
DROP TABLE IF EXISTS module_prefix_snapshot_row;
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
CREATE TABLE module_prefix_snapshot_row (
|
||||||
|
tenant_id TEXT NOT NULL,
|
||||||
|
module_id TEXT NOT NULL,
|
||||||
|
ord INTEGER NOT NULL,
|
||||||
|
prefix TEXT NOT NULL,
|
||||||
|
community_id TEXT,
|
||||||
|
source TEXT NOT NULL DEFAULT '',
|
||||||
|
PRIMARY KEY (tenant_id, module_id, ord),
|
||||||
|
FOREIGN KEY (tenant_id, module_id)
|
||||||
|
REFERENCES module_prefix_snapshot (tenant_id, module_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
ALTER TABLE module_prefix_snapshot DROP COLUMN prefixes_json;
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
ALTER TABLE asn_prefix_cache ADD COLUMN prefixes_json TEXT NOT NULL DEFAULT '[]';
|
||||||
|
DROP TABLE IF EXISTS asn_prefix_cache_row;
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
CREATE TABLE asn_prefix_cache_row (
|
||||||
|
asn INTEGER NOT NULL REFERENCES asn_prefix_cache (asn) ON DELETE CASCADE,
|
||||||
|
prefix TEXT NOT NULL,
|
||||||
|
PRIMARY KEY (asn, prefix)
|
||||||
|
);
|
||||||
|
|
||||||
|
ALTER TABLE asn_prefix_cache DROP COLUMN prefixes_json;
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
DROP INDEX IF EXISTS idx_job_audit_created_brin;
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
CREATE INDEX IF NOT EXISTS idx_job_audit_created_brin ON job_audit (created_at);
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
ALTER TABLE revision_materialized_prefix ADD COLUMN meta_json TEXT NOT NULL DEFAULT '{}';
|
||||||
|
ALTER TABLE module_domain_entry ADD COLUMN resolve_meta TEXT NOT NULL DEFAULT '{}';
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS module_cdn_fetch_log (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
module_id TEXT NOT NULL REFERENCES module (id) ON DELETE CASCADE,
|
||||||
|
source_id TEXT REFERENCES module_cdn_source (id) ON DELETE SET NULL,
|
||||||
|
http_status INTEGER,
|
||||||
|
bytes INTEGER,
|
||||||
|
error TEXT,
|
||||||
|
fetched_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_module_cdn_fetch_log_module ON module_cdn_fetch_log (module_id, fetched_at DESC);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_rev_mat_prefix_revision ON revision_materialized_prefix (revision_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_module_tenant ON module (tenant_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_module_prefix_snapshot_collected ON module_prefix_snapshot (collected_at);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_rev_mat_prefix_value ON revision_materialized_prefix (prefix);
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
ALTER TABLE revision_materialized_prefix DROP COLUMN meta_json;
|
||||||
|
ALTER TABLE module_domain_entry DROP COLUMN resolve_meta;
|
||||||
|
DROP TABLE IF EXISTS module_cdn_fetch_log;
|
||||||
|
|
||||||
|
DROP INDEX IF EXISTS idx_rev_mat_prefix_revision;
|
||||||
|
DROP INDEX IF EXISTS idx_module_tenant;
|
||||||
|
DROP INDEX IF EXISTS idx_module_prefix_snapshot_collected;
|
||||||
|
DROP INDEX IF EXISTS idx_rev_mat_prefix_value;
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* Warns about commits since the last tag that semantic-release cannot parse.
|
||||||
|
* Exit 0 always — semantic-release still decides release/no-op.
|
||||||
|
*/
|
||||||
|
import { execSync } from 'node:child_process';
|
||||||
|
import parser from 'conventional-commits-parser';
|
||||||
|
|
||||||
|
const RELEASABLE = new Set(['feat', 'fix', 'perf', 'ci', 'refactor']);
|
||||||
|
|
||||||
|
function lastTag() {
|
||||||
|
try {
|
||||||
|
return execSync('git describe --tags --abbrev=0', { encoding: 'utf8' }).trim();
|
||||||
|
} catch {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function commitsSince(ref) {
|
||||||
|
const range = ref ? `${ref}..HEAD` : 'HEAD';
|
||||||
|
const out = execSync(`git log ${range} --format=%H%x09%s`, { encoding: 'utf8' }).trim();
|
||||||
|
if (!out) return [];
|
||||||
|
return out.split('\n').map((line) => {
|
||||||
|
const [hash, subject] = line.split('\t');
|
||||||
|
return { hash: hash.trim(), subject: subject.trim() };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const tag = lastTag();
|
||||||
|
const commits = commitsSince(tag);
|
||||||
|
const unparseable = [];
|
||||||
|
const releasable = [];
|
||||||
|
|
||||||
|
for (const { hash, subject } of commits) {
|
||||||
|
const parsed = parser.sync(subject);
|
||||||
|
if (!parsed.type) {
|
||||||
|
unparseable.push({ hash: hash.slice(0, 7), subject });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (RELEASABLE.has(parsed.type)) {
|
||||||
|
releasable.push({ hash: hash.slice(0, 7), subject, type: parsed.type });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (commits.length === 0) {
|
||||||
|
console.log(`No new commits since ${tag || 'initial'}.`);
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (unparseable.length > 0) {
|
||||||
|
console.warn('::warning:: Commits not parseable by semantic-release (no version bump):');
|
||||||
|
for (const b of unparseable) {
|
||||||
|
console.warn(` ${b.hash} ${b.subject}`);
|
||||||
|
}
|
||||||
|
console.warn('Fix: single scope without commas, e.g. refactor(web): summary');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (releasable.length > 0) {
|
||||||
|
console.log(`Releasable since ${tag}: ${releasable.length} commit(s).`);
|
||||||
|
} else {
|
||||||
|
console.warn('::warning:: No releasable commits since last tag — release job will no-op.');
|
||||||
|
}
|
||||||
|
|
||||||
|
process.exit(0);
|
||||||
@@ -9,7 +9,6 @@
|
|||||||
networkOverallStatusLabel
|
networkOverallStatusLabel
|
||||||
} from '$lib/network/network-metrics.js';
|
} from '$lib/network/network-metrics.js';
|
||||||
import { Alert, AlertDescription, AlertTitle } from '$lib/ui/core/alert/index.js';
|
import { Alert, AlertDescription, AlertTitle } from '$lib/ui/core/alert/index.js';
|
||||||
import { Badge } from '$lib/ui/core/badge/index.js';
|
|
||||||
import { Button } from '$lib/ui/core/button/index.js';
|
import { Button } from '$lib/ui/core/button/index.js';
|
||||||
import KpiMetricsGrid from '$lib/ui/patterns/kpi/kpi-metrics-grid.svelte';
|
import KpiMetricsGrid from '$lib/ui/patterns/kpi/kpi-metrics-grid.svelte';
|
||||||
import NetworkSpeakerStatusCard from '$lib/components/network/NetworkSpeakerStatusCard.svelte';
|
import NetworkSpeakerStatusCard from '$lib/components/network/NetworkSpeakerStatusCard.svelte';
|
||||||
@@ -166,71 +165,76 @@
|
|||||||
]);
|
]);
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
{#if !initialLoading && !loading}
|
<div class="flex min-w-0 flex-col gap-6">
|
||||||
{#if overallStatus === 'ok'}
|
{#if !initialLoading && !loading}
|
||||||
<Alert class="border-success/30 bg-success/5">
|
{#if overallStatus === 'ok'}
|
||||||
<CheckCircle class="text-success" />
|
<Alert class="border-success/30 bg-success/5">
|
||||||
<AlertTitle>{networkOverallStatusLabel(overallStatus)}</AlertTitle>
|
<CheckCircle class="text-success" />
|
||||||
<AlertDescription>{overallHint}</AlertDescription>
|
<AlertTitle>{networkOverallStatusLabel(overallStatus)}</AlertTitle>
|
||||||
</Alert>
|
<AlertDescription>{overallHint}</AlertDescription>
|
||||||
{:else if overallStatus === 'warn'}
|
</Alert>
|
||||||
<Alert class="border-warning/30 bg-warning/5">
|
{:else if overallStatus === 'warn'}
|
||||||
<AlertTriangle class="text-warning" />
|
<Alert class="border-warning/30 bg-warning/5">
|
||||||
<AlertTitle>{networkOverallStatusLabel(overallStatus)}</AlertTitle>
|
<AlertTriangle class="text-warning" />
|
||||||
<AlertDescription>
|
<AlertTitle>{networkOverallStatusLabel(overallStatus)}</AlertTitle>
|
||||||
{overallHint}
|
<AlertDescription>
|
||||||
{#if issues.length > 0}
|
{overallHint}
|
||||||
<ul class="mt-2 list-inside list-disc text-sm">
|
{#if issues.length > 0}
|
||||||
{#each issues as issue (issue.id)}
|
<ul class="mt-2 list-inside list-disc text-sm">
|
||||||
<li>{issue.message}</li>
|
{#each issues as issue (issue.id)}
|
||||||
{/each}
|
<li>{issue.message}</li>
|
||||||
</ul>
|
{/each}
|
||||||
{/if}
|
</ul>
|
||||||
</AlertDescription>
|
{/if}
|
||||||
</Alert>
|
</AlertDescription>
|
||||||
{:else}
|
</Alert>
|
||||||
<Alert variant="destructive">
|
{:else}
|
||||||
<XCircle />
|
<Alert variant="destructive">
|
||||||
<AlertTitle>{networkOverallStatusLabel(overallStatus)}</AlertTitle>
|
<XCircle />
|
||||||
<AlertDescription>
|
<AlertTitle>{networkOverallStatusLabel(overallStatus)}</AlertTitle>
|
||||||
{overallHint}
|
<AlertDescription>
|
||||||
{#if issues.length > 0}
|
{overallHint}
|
||||||
<ul class="mt-2 list-inside list-disc text-sm">
|
{#if issues.length > 0}
|
||||||
{#each issues as issue (issue.id)}
|
<ul class="mt-2 list-inside list-disc text-sm">
|
||||||
<li>{issue.message}</li>
|
{#each issues as issue (issue.id)}
|
||||||
{/each}
|
<li>{issue.message}</li>
|
||||||
</ul>
|
{/each}
|
||||||
{/if}
|
</ul>
|
||||||
</AlertDescription>
|
{/if}
|
||||||
</Alert>
|
</AlertDescription>
|
||||||
|
</Alert>
|
||||||
|
{/if}
|
||||||
{/if}
|
{/if}
|
||||||
{/if}
|
|
||||||
|
|
||||||
<KpiMetricsGrid
|
<KpiMetricsGrid
|
||||||
cards={kpiCards}
|
cards={kpiCards}
|
||||||
loading={initialLoading || loading}
|
loading={initialLoading || loading}
|
||||||
skeletonCount={6}
|
skeletonCount={6}
|
||||||
class="sm:grid-cols-2 xl:grid-cols-3"
|
class="sm:grid-cols-2 xl:grid-cols-3"
|
||||||
/>
|
/>
|
||||||
|
|
||||||
<div class="flex items-center justify-between gap-2">
|
<section class="flex min-w-0 flex-col gap-4">
|
||||||
<h2 class="text-base font-semibold">Ноды</h2>
|
<div class="flex flex-wrap items-center justify-between gap-2">
|
||||||
<Button variant="outline" size="sm" href={resolve('/monitoring')}>
|
<h2 class="text-base font-semibold">Ноды</h2>
|
||||||
<Gauge class="size-3.5" />
|
<Button variant="outline" size="sm" href={resolve('/monitoring')}>
|
||||||
Мониторинг API
|
<Gauge class="size-3.5" />
|
||||||
</Button>
|
Мониторинг API
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{#if speakers.length === 0 && !initialLoading && !loading}
|
||||||
|
<p class="text-sm text-muted-foreground">Спикеры не зарегистрированы.</p>
|
||||||
|
{:else}
|
||||||
|
<div class="grid auto-rows-fr gap-4 sm:grid-cols-2 xl:grid-cols-3">
|
||||||
|
{#each speakers as speaker (speaker.id)}
|
||||||
|
<NetworkSpeakerStatusCard
|
||||||
|
{speaker}
|
||||||
|
{peers}
|
||||||
|
class="h-full"
|
||||||
|
onclick={onSpeakerSelect ? () => onSpeakerSelect(speaker) : undefined}
|
||||||
|
/>
|
||||||
|
{/each}
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</section>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{#if speakers.length === 0 && !initialLoading && !loading}
|
|
||||||
<p class="text-sm text-muted-foreground">Спикеры не зарегистрированы.</p>
|
|
||||||
{:else}
|
|
||||||
<div class="grid gap-4 sm:grid-cols-2 xl:grid-cols-3">
|
|
||||||
{#each speakers as speaker (speaker.id)}
|
|
||||||
<NetworkSpeakerStatusCard
|
|
||||||
{speaker}
|
|
||||||
{peers}
|
|
||||||
onclick={onSpeakerSelect ? () => onSpeakerSelect(speaker) : undefined}
|
|
||||||
/>
|
|
||||||
{/each}
|
|
||||||
</div>
|
|
||||||
{/if}
|
|
||||||
|
|||||||
@@ -3,9 +3,13 @@
|
|||||||
import {
|
import {
|
||||||
peersForSpeaker,
|
peersForSpeaker,
|
||||||
speakerDisplayStatus,
|
speakerDisplayStatus,
|
||||||
|
speakerDispatchError,
|
||||||
speakerHasDrift,
|
speakerHasDrift,
|
||||||
speakerLabel
|
speakerLabel,
|
||||||
|
speakerLiveAgentError,
|
||||||
|
speakerLiveBgpError
|
||||||
} from '$lib/network/network-metrics.js';
|
} from '$lib/network/network-metrics.js';
|
||||||
|
import { Alert, AlertDescription, AlertTitle } from '$lib/ui/core/alert/index.js';
|
||||||
import { Badge } from '$lib/ui/core/badge/index.js';
|
import { Badge } from '$lib/ui/core/badge/index.js';
|
||||||
import { Button } from '$lib/ui/core/button/index.js';
|
import { Button } from '$lib/ui/core/button/index.js';
|
||||||
import { Separator } from '$lib/ui/core/separator/index.js';
|
import { Separator } from '$lib/ui/core/separator/index.js';
|
||||||
@@ -24,6 +28,7 @@
|
|||||||
TableHeader,
|
TableHeader,
|
||||||
TableRow
|
TableRow
|
||||||
} from '$lib/ui/core/table/index.js';
|
} from '$lib/ui/core/table/index.js';
|
||||||
|
import AlertTriangle from '@lucide/svelte/icons/alert-triangle';
|
||||||
|
|
||||||
type Props = {
|
type Props = {
|
||||||
speaker: SpeakerRow | null;
|
speaker: SpeakerRow | null;
|
||||||
@@ -39,6 +44,9 @@
|
|||||||
const label = $derived(speaker ? speakerLabel(speaker) : '');
|
const label = $derived(speaker ? speakerLabel(speaker) : '');
|
||||||
const relatedPeers = $derived(speaker ? peersForSpeaker(peers, speaker.id) : []);
|
const relatedPeers = $derived(speaker ? peersForSpeaker(peers, speaker.id) : []);
|
||||||
const sessions = $derived(speaker?.live?.sessions ?? []);
|
const sessions = $derived(speaker?.live?.sessions ?? []);
|
||||||
|
const dispatchError = $derived(speaker ? speakerDispatchError(speaker) : null);
|
||||||
|
const agentError = $derived(speaker ? speakerLiveAgentError(speaker) : null);
|
||||||
|
const bgpError = $derived(speaker ? speakerLiveBgpError(speaker) : null);
|
||||||
|
|
||||||
function driftLabel(s: SpeakerRow): string {
|
function driftLabel(s: SpeakerRow): string {
|
||||||
const pub = s.published_revision_id?.slice(0, 8) ?? '—';
|
const pub = s.published_revision_id?.slice(0, 8) ?? '—';
|
||||||
@@ -46,22 +54,28 @@
|
|||||||
return `${app} / ${pub}`;
|
return `${app} / ${pub}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function formatSyncAt(iso: string | undefined): string {
|
||||||
|
if (!iso) return '—';
|
||||||
|
const d = new Date(iso);
|
||||||
|
return Number.isNaN(d.getTime()) ? iso : d.toLocaleString('ru-RU');
|
||||||
|
}
|
||||||
|
|
||||||
$effect(() => {
|
$effect(() => {
|
||||||
onOpenChange?.(open);
|
onOpenChange?.(open);
|
||||||
});
|
});
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<Sheet bind:open>
|
<Sheet bind:open>
|
||||||
<SheetContent class="flex w-full flex-col overflow-y-auto sm:max-w-lg">
|
<SheetContent class="flex w-full flex-col gap-0 overflow-y-auto p-0 sm:max-w-md">
|
||||||
{#if speaker}
|
{#if speaker}
|
||||||
<SheetHeader>
|
<div class="flex min-w-0 flex-col gap-4 px-4 pt-4 pb-6">
|
||||||
<SheetTitle class="truncate">{label}</SheetTitle>
|
<SheetHeader class="space-y-1 pr-8 text-left">
|
||||||
<SheetDescription>
|
<SheetTitle class="truncate">{label}</SheetTitle>
|
||||||
{speaker.role} · {speaker.agent_domain ?? speaker.endpoint}
|
<SheetDescription class="truncate">
|
||||||
</SheetDescription>
|
{speaker.role} · {speaker.agent_domain ?? speaker.endpoint}
|
||||||
</SheetHeader>
|
</SheetDescription>
|
||||||
|
</SheetHeader>
|
||||||
|
|
||||||
<div class="mt-4 space-y-4">
|
|
||||||
<div class="flex flex-wrap items-center gap-2">
|
<div class="flex flex-wrap items-center gap-2">
|
||||||
{#if status}
|
{#if status}
|
||||||
<Badge variant={status.variant}>{status.label}</Badge>
|
<Badge variant={status.variant}>{status.label}</Badge>
|
||||||
@@ -71,101 +85,118 @@
|
|||||||
{/if}
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="grid gap-2 text-sm">
|
<dl class="grid grid-cols-[minmax(0,9rem)_1fr] gap-x-3 gap-y-2 text-sm">
|
||||||
<div class="flex justify-between gap-2">
|
<dt class="text-muted-foreground">BGP Established</dt>
|
||||||
<span class="text-muted-foreground">BGP Established</span>
|
<dd class="text-right font-medium tabular-nums">
|
||||||
<span class="font-medium tabular-nums">
|
{speaker.live?.bgp_established ?? '—'} / {speaker.live?.bgp_sessions_total ?? '—'}
|
||||||
{speaker.live?.bgp_established ?? '—'} / {speaker.live?.bgp_sessions_total ?? '—'}
|
</dd>
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
{#if speaker.live?.agent_last_sync_at}
|
{#if speaker.live?.agent_last_sync_at}
|
||||||
<div class="flex justify-between gap-2">
|
<dt class="text-muted-foreground">Последний sync</dt>
|
||||||
<span class="text-muted-foreground">Последний sync</span>
|
<dd class="text-right text-xs tabular-nums">
|
||||||
<span class="text-xs">{speaker.live.agent_last_sync_at}</span>
|
{formatSyncAt(speaker.live.agent_last_sync_at)}
|
||||||
</div>
|
</dd>
|
||||||
{/if}
|
{/if}
|
||||||
<div class="flex justify-between gap-2">
|
<dt class="text-muted-foreground">Drift (app / pub)</dt>
|
||||||
<span class="text-muted-foreground">Drift (app / pub)</span>
|
<dd class="truncate text-right font-mono text-xs">{driftLabel(speaker)}</dd>
|
||||||
<span class="font-mono text-xs">{driftLabel(speaker)}</span>
|
|
||||||
</div>
|
|
||||||
{#if speaker.last_dispatch_at}
|
{#if speaker.last_dispatch_at}
|
||||||
<div class="flex justify-between gap-2">
|
<dt class="text-muted-foreground">Dispatch</dt>
|
||||||
<span class="text-muted-foreground">Dispatch</span>
|
<dd class="text-right text-xs tabular-nums">
|
||||||
<span class="text-xs">{speaker.last_dispatch_at}</span>
|
{formatSyncAt(speaker.last_dispatch_at)}
|
||||||
</div>
|
</dd>
|
||||||
{/if}
|
{/if}
|
||||||
{#if speaker.last_dispatch_error}
|
</dl>
|
||||||
<p class="text-xs text-destructive">{speaker.last_dispatch_error}</p>
|
|
||||||
{/if}
|
{#if dispatchError}
|
||||||
{#if speaker.live?.agent_error}
|
<Alert class="border-warning/30 bg-warning/5">
|
||||||
<p class="text-xs text-destructive">Agent: {speaker.live.agent_error}</p>
|
<AlertTriangle class="text-warning" />
|
||||||
{/if}
|
<AlertTitle class="text-sm">{dispatchError.title}</AlertTitle>
|
||||||
{#if speaker.live?.bgp_poll_error}
|
<AlertDescription class="text-xs leading-relaxed"
|
||||||
<p class="text-xs text-destructive">BGP poll: {speaker.live.bgp_poll_error}</p>
|
>{dispatchError.detail}</AlertDescription
|
||||||
{/if}
|
>
|
||||||
</div>
|
</Alert>
|
||||||
|
{/if}
|
||||||
|
{#if agentError}
|
||||||
|
<Alert variant="destructive">
|
||||||
|
<AlertTitle class="text-sm">{agentError.title}</AlertTitle>
|
||||||
|
<AlertDescription class="text-xs">{agentError.detail}</AlertDescription>
|
||||||
|
</Alert>
|
||||||
|
{/if}
|
||||||
|
{#if bgpError}
|
||||||
|
<Alert variant="destructive">
|
||||||
|
<AlertTitle class="text-sm">{bgpError.title}</AlertTitle>
|
||||||
|
<AlertDescription class="text-xs">{bgpError.detail}</AlertDescription>
|
||||||
|
</Alert>
|
||||||
|
{/if}
|
||||||
|
|
||||||
{#if onApply && speaker.published_revision_id}
|
{#if onApply && speaker.published_revision_id}
|
||||||
<Button variant="outline" size="sm" onclick={() => onApply(speaker)}
|
<Button variant="outline" size="sm" class="w-fit" onclick={() => onApply(speaker)}>
|
||||||
>Apply revision</Button
|
Apply revision
|
||||||
>
|
</Button>
|
||||||
{/if}
|
{/if}
|
||||||
|
|
||||||
<Separator />
|
<Separator />
|
||||||
|
|
||||||
<div class="space-y-2">
|
<section class="min-w-0 space-y-2">
|
||||||
<h3 class="text-sm font-medium">BGP-сессии (live)</h3>
|
<h3 class="text-sm font-medium">BGP-сессии (live)</h3>
|
||||||
{#if sessions.length === 0}
|
{#if sessions.length === 0}
|
||||||
<p class="text-sm text-muted-foreground">Нет данных или сессий нет.</p>
|
<p class="text-sm text-muted-foreground">Нет данных или сессий нет.</p>
|
||||||
{:else}
|
{:else}
|
||||||
<Table>
|
<div class="rounded-md border">
|
||||||
<TableHeader>
|
<Table class="table-fixed">
|
||||||
<TableRow>
|
<TableHeader>
|
||||||
<TableHead>Имя</TableHead>
|
|
||||||
<TableHead>Состояние</TableHead>
|
|
||||||
</TableRow>
|
|
||||||
</TableHeader>
|
|
||||||
<TableBody>
|
|
||||||
{#each sessions as sess, i (sess.name + i)}
|
|
||||||
<TableRow>
|
<TableRow>
|
||||||
<TableCell class="font-mono text-xs">
|
<TableHead class="w-[65%]">Имя</TableHead>
|
||||||
{sess.name}
|
<TableHead class="w-[35%] text-right">Состояние</TableHead>
|
||||||
{#if sess.neighbor}
|
|
||||||
<div class="text-muted-foreground">{sess.neighbor}</div>
|
|
||||||
{/if}
|
|
||||||
</TableCell>
|
|
||||||
<TableCell>
|
|
||||||
<Badge variant="outline">{sess.state}</Badge>
|
|
||||||
</TableCell>
|
|
||||||
</TableRow>
|
</TableRow>
|
||||||
{/each}
|
</TableHeader>
|
||||||
</TableBody>
|
<TableBody>
|
||||||
</Table>
|
{#each sessions as sess, i (sess.name + i)}
|
||||||
|
<TableRow>
|
||||||
|
<TableCell class="align-top">
|
||||||
|
<p class="truncate font-mono text-xs" title={sess.name}>{sess.name}</p>
|
||||||
|
{#if sess.neighbor}
|
||||||
|
<p class="truncate text-xs text-muted-foreground" title={sess.neighbor}>
|
||||||
|
{sess.neighbor}
|
||||||
|
</p>
|
||||||
|
{/if}
|
||||||
|
</TableCell>
|
||||||
|
<TableCell class="text-right align-top">
|
||||||
|
<Badge variant="outline" class="shrink-0">{sess.state}</Badge>
|
||||||
|
</TableCell>
|
||||||
|
</TableRow>
|
||||||
|
{/each}
|
||||||
|
</TableBody>
|
||||||
|
</Table>
|
||||||
|
</div>
|
||||||
{/if}
|
{/if}
|
||||||
</div>
|
</section>
|
||||||
|
|
||||||
<Separator />
|
<Separator />
|
||||||
|
|
||||||
<div class="space-y-2">
|
<section class="min-w-0 space-y-2">
|
||||||
<h3 class="text-sm font-medium">Пиры на ноде</h3>
|
<h3 class="text-sm font-medium">Пиры на ноде</h3>
|
||||||
{#if relatedPeers.length === 0}
|
{#if relatedPeers.length === 0}
|
||||||
<p class="text-sm text-muted-foreground">Нет привязанных пиров.</p>
|
<p class="text-sm text-muted-foreground">Нет привязанных пиров.</p>
|
||||||
{:else}
|
{:else}
|
||||||
<ul class="space-y-2">
|
<ul class="divide-y rounded-md border">
|
||||||
{#each relatedPeers as p (p.id)}
|
{#each relatedPeers as p (p.id)}
|
||||||
<li class="rounded-lg border px-3 py-2 text-sm">
|
<li class="flex min-w-0 items-start justify-between gap-3 px-3 py-2.5 text-sm">
|
||||||
<div class="flex items-center justify-between gap-2">
|
<div class="min-w-0 flex-1">
|
||||||
<span class="font-medium">{p.name?.trim() || p.neighbor}</span>
|
<p class="truncate font-medium" title={p.name?.trim() || p.neighbor}>
|
||||||
<Badge variant="outline">{p.session_state || '—'}</Badge>
|
{p.name?.trim() || p.neighbor}
|
||||||
|
</p>
|
||||||
|
{#if p.session_mismatch}
|
||||||
|
<p class="mt-0.5 text-xs text-warning">
|
||||||
|
Mismatch: сессия не на назначенной ноде
|
||||||
|
</p>
|
||||||
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
{#if p.session_mismatch}
|
<Badge variant="outline" class="shrink-0">{p.session_state || '—'}</Badge>
|
||||||
<p class="mt-1 text-xs text-warning">Mismatch: сессия не на назначенной ноде</p>
|
|
||||||
{/if}
|
|
||||||
</li>
|
</li>
|
||||||
{/each}
|
{/each}
|
||||||
</ul>
|
</ul>
|
||||||
{/if}
|
{/if}
|
||||||
</div>
|
</section>
|
||||||
</div>
|
</div>
|
||||||
{/if}
|
{/if}
|
||||||
</SheetContent>
|
</SheetContent>
|
||||||
|
|||||||
@@ -42,8 +42,8 @@
|
|||||||
<!-- svelte-ignore a11y_no_noninteractive_tabindex -->
|
<!-- svelte-ignore a11y_no_noninteractive_tabindex -->
|
||||||
<Card
|
<Card
|
||||||
class={cn(
|
class={cn(
|
||||||
'cursor-pointer transition-colors hover:border-primary/35',
|
'flex h-full flex-col transition-colors',
|
||||||
onclick ? 'cursor-pointer' : '',
|
onclick ? 'cursor-pointer hover:border-primary/35' : '',
|
||||||
className
|
className
|
||||||
)}
|
)}
|
||||||
role={onclick ? 'button' : undefined}
|
role={onclick ? 'button' : undefined}
|
||||||
@@ -57,31 +57,29 @@
|
|||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
<CardHeader class="pb-2">
|
<CardHeader class="pb-2">
|
||||||
<div class="flex items-start justify-between gap-2">
|
<div class="flex items-start gap-2">
|
||||||
<div class="min-w-0">
|
<div class="min-w-0 flex-1">
|
||||||
<CardTitle class="flex items-center gap-2 truncate text-sm">
|
<CardTitle class="flex items-center gap-2 text-sm">
|
||||||
<Server class="size-4 shrink-0 text-muted-foreground" />
|
<Server class="size-4 shrink-0 text-muted-foreground" aria-hidden="true" />
|
||||||
<span class="truncate">{label}</span>
|
<span class="truncate" title={label}>{label}</span>
|
||||||
</CardTitle>
|
</CardTitle>
|
||||||
<CardDescription class="truncate font-mono text-xs">{speaker.role}</CardDescription>
|
<CardDescription class="truncate font-mono text-xs">{speaker.role}</CardDescription>
|
||||||
</div>
|
</div>
|
||||||
<Badge variant={status.variant}>{status.label}</Badge>
|
<Badge variant={status.variant} class="shrink-0">{status.label}</Badge>
|
||||||
</div>
|
</div>
|
||||||
</CardHeader>
|
</CardHeader>
|
||||||
<CardContent class="space-y-2 pt-0 text-sm">
|
<CardContent class="mt-auto pt-0">
|
||||||
<div class="flex justify-between gap-2">
|
<dl class="grid grid-cols-[1fr_auto] gap-x-3 gap-y-2 text-sm">
|
||||||
<span class="text-muted-foreground">BGP</span>
|
<dt class="text-muted-foreground">BGP</dt>
|
||||||
<span class="font-medium tabular-nums">{speakerBgpText(speaker)}</span>
|
<dd class="font-medium tabular-nums">{speakerBgpText(speaker)}</dd>
|
||||||
</div>
|
<dt class="text-muted-foreground">Пиры</dt>
|
||||||
<div class="flex justify-between gap-2">
|
<dd class="tabular-nums">{peerCount}</dd>
|
||||||
<span class="text-muted-foreground">Пиры</span>
|
<dt class="text-muted-foreground">Drift</dt>
|
||||||
<span class="tabular-nums">{peerCount}</span>
|
<dd>
|
||||||
</div>
|
<Badge variant={drift ? 'secondary' : 'outline'} class="text-xs">
|
||||||
<div class="flex justify-between gap-2">
|
{drift ? 'есть' : 'нет'}
|
||||||
<span class="text-muted-foreground">Drift</span>
|
</Badge>
|
||||||
<Badge variant={drift ? 'secondary' : 'outline'} class="text-xs">
|
</dd>
|
||||||
{drift ? 'есть' : 'нет'}
|
</dl>
|
||||||
</Badge>
|
|
||||||
</div>
|
|
||||||
</CardContent>
|
</CardContent>
|
||||||
</Card>
|
</Card>
|
||||||
|
|||||||
@@ -197,6 +197,13 @@ export function collectNetworkIssues(
|
|||||||
message: `Ошибка BGP-опроса: ${speakerLabel(s)}`,
|
message: `Ошибка BGP-опроса: ${speakerLabel(s)}`,
|
||||||
severity: 'warn'
|
severity: 'warn'
|
||||||
});
|
});
|
||||||
|
} else if (s.last_dispatch_error) {
|
||||||
|
const err = formatSpeakerError(s.last_dispatch_error);
|
||||||
|
issues.push({
|
||||||
|
id: `speaker-dispatch-${s.id}`,
|
||||||
|
message: `Dispatch: ${speakerLabel(s)}${err ? ` — ${err.detail.slice(0, 80)}` : ''}`,
|
||||||
|
severity: 'warn'
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -226,3 +233,63 @@ export function writeNetworkAutoRefresh(enabled: boolean): void {
|
|||||||
if (typeof localStorage === 'undefined') return;
|
if (typeof localStorage === 'undefined') return;
|
||||||
localStorage.setItem(NETWORK_AUTO_REFRESH_KEY, enabled ? '1' : '0');
|
localStorage.setItem(NETWORK_AUTO_REFRESH_KEY, enabled ? '1' : '0');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type FormattedSpeakerError = {
|
||||||
|
title: string;
|
||||||
|
detail: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Humanize stored dispatch/agent errors (avoid raw JSON in UI). */
|
||||||
|
export function formatSpeakerError(raw: string | null | undefined): FormattedSpeakerError | null {
|
||||||
|
if (!raw?.trim()) return null;
|
||||||
|
const text = raw.trim();
|
||||||
|
|
||||||
|
const jsonMatch = text.match(/\{[\s\S]*\}/);
|
||||||
|
if (jsonMatch) {
|
||||||
|
try {
|
||||||
|
const obj = JSON.parse(jsonMatch[0]) as {
|
||||||
|
detail?: string;
|
||||||
|
title?: string;
|
||||||
|
status?: number;
|
||||||
|
};
|
||||||
|
const detail = String(obj.detail ?? text);
|
||||||
|
if (/403/.test(detail) && /bundle/i.test(detail)) {
|
||||||
|
return {
|
||||||
|
title: 'Dispatch: доступ к бандлу',
|
||||||
|
detail:
|
||||||
|
'Нода не смогла скачать бандл с CP (403). Проверьте node API-ключ (роль node) и EVOBGP_NODE_TOKEN на реплике — см. docs/access.md.'
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const httpPrefix = text.match(/^HTTP \d+:\s*/)?.[0] ?? '';
|
||||||
|
return {
|
||||||
|
title: obj.title && obj.title !== 'Bad Gateway' ? obj.title : 'Ошибка dispatch',
|
||||||
|
detail: httpPrefix ? `${httpPrefix.trim()} ${detail}`.trim() : detail
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
/* fall through */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (/^HTTP \d+:/.test(text)) {
|
||||||
|
return { title: 'Ошибка HTTP', detail: text };
|
||||||
|
}
|
||||||
|
return { title: 'Ошибка', detail: text };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function speakerDispatchError(s: SpeakerRow): FormattedSpeakerError | null {
|
||||||
|
const liveRev = s.live?.agent_last_applied_revision_id?.trim();
|
||||||
|
const pub = s.published_revision_id?.trim();
|
||||||
|
// CP meta can keep a stale dispatch error after a later successful agent sync.
|
||||||
|
if (s.live?.agent_ok && liveRev && pub && liveRev === pub) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return formatSpeakerError(s.last_dispatch_error);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function speakerLiveAgentError(s: SpeakerRow): FormattedSpeakerError | null {
|
||||||
|
return formatSpeakerError(s.live?.agent_error);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function speakerLiveBgpError(s: SpeakerRow): FormattedSpeakerError | null {
|
||||||
|
return formatSpeakerError(s.live?.bgp_poll_error);
|
||||||
|
}
|
||||||
|
|||||||
@@ -45,7 +45,7 @@
|
|||||||
let { cards, loading = false, skeletonCount = 3, class: className }: Props = $props();
|
let { cards, loading = false, skeletonCount = 3, class: className }: Props = $props();
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<div class={cn('grid gap-4', className)}>
|
<div class={cn('grid auto-rows-fr gap-4', className)}>
|
||||||
{#if loading}
|
{#if loading}
|
||||||
{#each Array(skeletonCount) as _, i (i)}
|
{#each Array(skeletonCount) as _, i (i)}
|
||||||
<CardSkeleton />
|
<CardSkeleton />
|
||||||
@@ -56,7 +56,7 @@
|
|||||||
{@const a = card.accent}
|
{@const a = card.accent}
|
||||||
<Card
|
<Card
|
||||||
class={cn(
|
class={cn(
|
||||||
'overflow-hidden border-l-4 shadow-sm',
|
'flex h-full flex-col overflow-hidden border-l-4 shadow-sm',
|
||||||
card.href ? 'transition-colors hover:border-primary/35' : '',
|
card.href ? 'transition-colors hover:border-primary/35' : '',
|
||||||
a.border,
|
a.border,
|
||||||
a.bg
|
a.bg
|
||||||
@@ -96,7 +96,7 @@
|
|||||||
>{card.value}</CardTitle
|
>{card.value}</CardTitle
|
||||||
>
|
>
|
||||||
</CardHeader>
|
</CardHeader>
|
||||||
<CardContent class="space-y-2">
|
<CardContent class="mt-auto space-y-2">
|
||||||
<Badge variant={card.badgeVariant ?? 'outline'} class={card.badgeClass}
|
<Badge variant={card.badgeVariant ?? 'outline'} class={card.badgeClass}
|
||||||
>{card.badge}</Badge
|
>{card.badge}</Badge
|
||||||
>
|
>
|
||||||
|
|||||||
@@ -204,7 +204,7 @@
|
|||||||
<TabsTrigger value="control-plane">Control plane</TabsTrigger>
|
<TabsTrigger value="control-plane">Control plane</TabsTrigger>
|
||||||
</TabsList>
|
</TabsList>
|
||||||
|
|
||||||
<TabsContent value="overview" class="mt-4">
|
<TabsContent value="overview" class="mt-4 min-w-0">
|
||||||
<NetworkOverviewTab
|
<NetworkOverviewTab
|
||||||
{peers}
|
{peers}
|
||||||
{speakers}
|
{speakers}
|
||||||
|
|||||||
Reference in New Issue
Block a user