feat: add tenant refresh functionality to HTTP API and job processing

Implemented a new endpoint for tenant refresh in the HTTP API, allowing for the refresh of modules associated with a tenant. Enhanced job processing to handle tenant refresh jobs, including logic for managing module IDs and job status updates. Updated the scheduler to enqueue tenant refresh jobs based on module due dates, improving the overall efficiency of module management. Additionally, introduced caching for ASN prefix data to optimize performance during refresh operations.

internal/pipeline/asn_cache.go

@@ -0,0 +1,58 @@
+package pipeline
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/netip"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"evobgp/internal/asnresolve"
+	"evobgp/internal/store"
+)
+
+func asnCacheTTL() time.Duration {
+	sec := 1800
+	if s := strings.TrimSpace(os.Getenv("EVOBGP_ASN_CACHE_TTL_SEC")); s != "" {
+		if v, err := strconv.Atoi(s); err == nil && v > 0 {
+			sec = v
+		}
+	}
+	return time.Duration(sec) * time.Second
+}
+
+// resolveASNForEntry fetches prefixes and holder with shared TTL cache (asn_prefix_cache).
+func resolveASNForEntry(ctx context.Context, st store.Backend, hc *http.Client, asn int64) ([]netip.Prefix, string, error) {
+	ttl := asnCacheTTL()
+	if st != nil {
+		if ent, ok, err := st.GetASNPrefixCache(asn); err == nil && ok && ent != nil && time.Since(ent.FetchedAt) < ttl {
+			out := make([]netip.Prefix, 0, len(ent.Prefixes))
+			for _, p := range ent.Prefixes {
+				pfx, perr := netip.ParsePrefix(strings.TrimSpace(p))
+				if perr != nil {
+					continue
+				}
+				out = append(out, pfx.Masked())
+			}
+			return out, ent.Holder, nil
+		}
+	}
+	pfxs, err := asnresolve.AnnouncedPrefixes(ctx, hc, asn)
+	if err != nil {
+		return nil, "", err
+	}
+	holder, _ := asnresolve.ASHolderName(ctx, hc, asn)
+	if st != nil {
+		strs := make([]string, len(pfxs))
+		for i, p := range pfxs {
+			strs[i] = p.String()
+		}
+		if err := st.SetASNPrefixCache(asn, holder, strs); err != nil {
+			return nil, "", fmt.Errorf("asn cache AS%d: %w", asn, err)
+		}
+	}
+	return pfxs, holder, nil
+}

internal/pipeline/asn_cache_test.go

@@ -0,0 +1,64 @@
+package pipeline
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"evobgp/internal/store"
+)
+
+func TestResolveASNForEntry_UsesTTLCache(t *testing.T) {
+	m := store.NewMemory()
+	t.Setenv("EVOBGP_ASN_CACHE_TTL_SEC", "3600")
+
+	var calls atomic.Int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calls.Add(1)
+		if strings.Contains(r.URL.Path, "/announced") {
+			_, _ = w.Write([]byte(`{"status":"ok","data":{"prefixes":[{"prefix":"203.0.113.0/24"}]}}`))
+			return
+		}
+		_, _ = w.Write([]byte(`{"status":"ok","data":{"holder":"Test AS"}}`))
+	}))
+	defer srv.Close()
+	t.Setenv("EVOBGP_RIPESTAT_ANNOUNCED_PREFIXES_URL", srv.URL+"/announced")
+	t.Setenv("EVOBGP_RIPESTAT_AS_OVERVIEW_URL", srv.URL+"/overview")
+
+	ctx := context.Background()
+	hc := srv.Client()
+	p1, h1, err := resolveASNForEntry(ctx, m, hc, 64512)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(p1) != 1 || h1 != "Test AS" {
+		t.Fatalf("unexpected first resolve: %+v holder=%q", p1, h1)
+	}
+	if calls.Load() < 2 {
+		t.Fatalf("expected ripestat calls on first resolve, got %d", calls.Load())
+	}
+	firstCalls := calls.Load()
+	_, _, err = resolveASNForEntry(ctx, m, hc, 64512)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if calls.Load() != firstCalls {
+		t.Fatalf("expected cache hit (no new HTTP), calls went from %d to %d", firstCalls, calls.Load())
+	}
+}
+
+func TestModuleDueForScheduler_BucketRollover(t *testing.T) {
+	mod := &store.Module{Enabled: true, Type: "CDN_CIDRS", RefreshIntervalSec: 300}
+	boundary := time.Unix(300, 0)
+	if !ModuleDueForScheduler(mod, boundary) {
+		t.Fatal("expected due when refresh bucket rolls")
+	}
+	mid := time.Unix(330, 0)
+	if ModuleDueForScheduler(mod, mid) {
+		t.Fatal("expected not due within same bucket")
+	}
+}

internal/pipeline/cdn_prefetch_test.go

@@ -0,0 +1,99 @@
+package pipeline
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"evobgp/internal/store"
+)
+
+func TestCollectModulePrefixRows_CDNSendsIfNoneMatch(t *testing.T) {
+	m := store.NewMemory()
+	m.SeedDemo()
+	tenant, _, _, _, _ := m.DemoIDs()
+
+	mod, err := m.CreateModule(tenant, &store.Module{
+		Type:    "CDN_CIDRS",
+		Name:    "cdn-conditional",
+		Enabled: true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var gotIfNoneMatch string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotIfNoneMatch = strings.TrimSpace(r.Header.Get("If-None-Match"))
+		w.Header().Set("ETag", "etag-new")
+		_, _ = w.Write([]byte("198.51.100.0/24\n"))
+	}))
+	defer srv.Close()
+
+	etag := "etag-old"
+	if _, err := m.CreateCDNSource(tenant, mod.ID, &store.CDNSource{
+		SourceKind: "txt",
+		URL:        srv.URL,
+		Etag:       etag,
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if gotIfNoneMatch != etag {
+		t.Fatalf("want If-None-Match %q, got %q", etag, gotIfNoneMatch)
+	}
+	if len(collected) != 1 || collected[0].Prefix != "198.51.100.0/24" {
+		t.Fatalf("unexpected collected rows: %+v", collected)
+	}
+}
+
+func TestCollectModulePrefixRows_CDN304UsesSnapshot(t *testing.T) {
+	m := store.NewMemory()
+	m.SeedDemo()
+	tenant, _, _, _, _ := m.DemoIDs()
+
+	mod, err := m.CreateModule(tenant, &store.Module{
+		Type:    "CDN_CIDRS",
+		Name:    "cdn-304",
+		Enabled: true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNotModified)
+	}))
+	defer srv.Close()
+
+	etag := "etag-stable"
+	src, err := m.CreateCDNSource(tenant, mod.ID, &store.CDNSource{
+		SourceKind: "txt",
+		URL:        srv.URL,
+		Etag:       etag,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	prior := []store.PrefixRow{{
+		Prefix: "203.0.113.0/24",
+		Source: cdnSourceKey(src.ID),
+	}}
+	if err := mergeCDNSourceIntoModuleSnapshot(m, tenant, mod, src.ID, prior); err != nil {
+		t.Fatal(err)
+	}
+
+	collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, prior)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(collected) != 1 || collected[0].Prefix != "203.0.113.0/24" {
+		t.Fatalf("want cached prefix on 304, got %+v", collected)
+	}
+}

internal/pipeline/cdn_snapshot.go

@@ -0,0 +1,126 @@
+package pipeline
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"evobgp/internal/store"
+)
+
+func cdnSourceKey(sourceID string) string {
+	return "cdn:" + strings.TrimSpace(sourceID)
+}
+
+func mergeSnapshotDropSource(rows []store.PrefixRow, sourceKey string) []store.PrefixRow {
+	if len(rows) == 0 {
+		return nil
+	}
+	out := make([]store.PrefixRow, 0, len(rows))
+	for _, row := range rows {
+		if row.Source != sourceKey {
+			out = append(out, row)
+		}
+	}
+	return out
+}
+
+func cdnRowsFromParsed(mod *store.Module, src *store.CDNSource, pfxStrings []string) []store.PrefixRow {
+	var rows []store.PrefixRow
+	for _, p := range pfxStrings {
+		p = strings.TrimSpace(p)
+		if p == "" {
+			continue
+		}
+		comm := src.CommunityID
+		if comm == nil && mod != nil && mod.DefaultCommunityID != nil {
+			c := *mod.DefaultCommunityID
+			comm = &c
+		}
+		rows = append(rows, store.PrefixRow{Prefix: p, CommunityID: comm, Source: cdnSourceKey(src.ID)})
+	}
+	return rows
+}
+
+// mergeCDNSourceIntoModuleSnapshot replaces rows for one CDN source in the module snapshot.
+func mergeCDNSourceIntoModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, sourceID string, newRows []store.PrefixRow) error {
+	if st == nil || mod == nil {
+		return nil
+	}
+	sourceKey := cdnSourceKey(sourceID)
+	var base []store.PrefixRow
+	if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil {
+		base = mergeSnapshotDropSource(snap.Prefixes, sourceKey)
+	}
+	merged := append(base, newRows...)
+	return persistModuleSnapshot(st, tenantID, mod, merged)
+}
+
+func parseCDNBody(body string, src *store.CDNSource) ([]string, error) {
+	pfxs, err := ExtractCIDRs(body, src.SourceKind, src.PrefixPath)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]string, 0, len(pfxs))
+	for _, pfx := range pfxs {
+		out = append(out, pfx.String())
+	}
+	return out, nil
+}
+
+func applyCDNSourceHTTPResult(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, priorSnapshot []store.PrefixRow, now time.Time) ([]store.PrefixRow, error) {
+	u := strings.TrimSpace(src.URL)
+	if u == "" {
+		return nil, nil
+	}
+	sourceKey := cdnSourceKey(src.ID)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
+	if err != nil {
+		return nil, err
+	}
+	if etag := strings.TrimSpace(src.Etag); etag != "" {
+		req.Header.Set("If-None-Match", etag)
+	}
+	resp, err := hc.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusNotModified {
+		if cached := prefixRowsForSource(priorSnapshot, sourceKey); len(cached) > 0 {
+			return cached, nil
+		}
+		return nil, fmt.Errorf("cdn url %s: 304 without cached prefixes", u)
+	}
+	if resp.StatusCode != http.StatusOK {
+		_, _ = io.Copy(io.Discard, resp.Body)
+		return nil, fmt.Errorf("cdn url %s: %s", u, resp.Status)
+	}
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
+	if err != nil {
+		return nil, err
+	}
+	prefixStrs, err := parseCDNBody(string(body), src)
+	if err != nil {
+		return nil, fmt.Errorf("cdn parse %s: %w", u, err)
+	}
+	etag := strings.TrimSpace(resp.Header.Get("ETag"))
+	patch := &store.CDNSourcePatch{}
+	if etag != "" && etag != strings.TrimSpace(src.Etag) {
+		e := etag
+		patch.Etag = &e
+	}
+	refreshedAt := now
+	patch.LastRefreshedAt = &refreshedAt
+	_, _ = st.UpdateCDNSource(tenantID, moduleID, src.ID, patch)
+
+	rows := cdnRowsFromParsed(mod, src, prefixStrs)
+	if err := mergeCDNSourceIntoModuleSnapshot(st, tenantID, mod, src.ID, rows); err != nil {
+		return nil, err
+	}
+	return rows, nil
+}

internal/pipeline/collect_parallel.go

@@ -3,14 +3,12 @@ package pipeline
 import (
 	"context"
 	"fmt"
-	"io"
 	"net/http"
 	"os"
 	"strings"
 	"sync"
 	"time"
 
-	"evobgp/internal/asnresolve"
 	"evobgp/internal/store"
 )
 
@@ -76,15 +74,11 @@ func collectASPrefixRows(ctx context.Context, st store.Backend, hc *http.Client,
 				c := *mod.DefaultCommunityID
 				comm = &c
 			}
-			pfxs, err := asnresolve.AnnouncedPrefixes(ctx, hc, entry.ASN)
+			pfxs, holder, err := resolveASNForEntry(ctx, st, hc, entry.ASN)
 			if err != nil {
 				results[idx] = entryResult{err: fmt.Errorf("resolve AS%d: %w", entry.ASN, err)}
 				return
 			}
-			holder := ""
-			if h, err := asnresolve.ASHolderName(ctx, hc, entry.ASN); err == nil {
-				holder = h
-			}
 			src := fmt.Sprintf("as:%d", entry.ASN)
 			var rows []store.PrefixRow
 			for _, pfx := range pfxs {
@@ -161,18 +155,11 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client
 					return
 				}
 			}
-			u := strings.TrimSpace(src.URL)
-			if u == "" {
-				return
-			}
-			rows, err := fetchAndParseCDNSource(ctx, hc, st, tenantID, moduleID, mod, src, now)
+			rows, err := applyCDNSourceHTTPResult(ctx, st, hc, tenantID, moduleID, mod, src, priorSnapshot, now)
 			if err != nil {
 				results[idx] = srcResult{err: err}
 				return
 			}
-			for i := range rows {
-				rows[i].Source = sourceKey
-			}
 			results[idx] = srcResult{rows: rows}
 		}(i, src)
 	}
@@ -188,51 +175,6 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client
 	return out, nil
 }
 
-func fetchAndParseCDNSource(ctx context.Context, hc *http.Client, st store.Backend, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, now time.Time) ([]store.PrefixRow, error) {
-	u := strings.TrimSpace(src.URL)
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
-	if err != nil {
-		return nil, err
-	}
-	resp, err := hc.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
-	}
-	if resp.StatusCode != http.StatusOK {
-		_, _ = io.Copy(io.Discard, resp.Body)
-		_ = resp.Body.Close()
-		return nil, fmt.Errorf("cdn url %s: %s", u, resp.Status)
-	}
-	body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
-	_ = resp.Body.Close()
-	if err != nil {
-		return nil, err
-	}
-	etag := strings.TrimSpace(resp.Header.Get("ETag"))
-	patch := &store.CDNSourcePatch{}
-	if etag != "" && etag != strings.TrimSpace(src.Etag) {
-		e := etag
-		patch.Etag = &e
-	}
-	refreshedAt := now
-	patch.LastRefreshedAt = &refreshedAt
-	_, _ = st.UpdateCDNSource(tenantID, moduleID, src.ID, patch)
-	pfxs, err := ExtractCIDRs(string(body), src.SourceKind, src.PrefixPath)
-	if err != nil {
-		return nil, fmt.Errorf("cdn parse %s: %w", u, err)
-	}
-	var rows []store.PrefixRow
-	for _, pfx := range pfxs {
-		comm := src.CommunityID
-		if comm == nil && mod.DefaultCommunityID != nil {
-			c := *mod.DefaultCommunityID
-			comm = &c
-		}
-		rows = append(rows, store.PrefixRow{Prefix: pfx.String(), CommunityID: comm})
-	}
-	return rows, nil
-}
-
 func collectDomainPrefixRows(ctx context.Context, hc *http.Client, mod *store.Module, profile *store.DohProfile, entries []*store.DomainEntry) ([]store.PrefixRow, error) {
 	var validDom []*store.DomainEntry
 	for _, e := range entries {

internal/pipeline/materialize_regression_test.go

@@ -1,9 +1,6 @@
 package pipeline
 
 import (
-	"context"
-	"net/http"
-	"net/http/httptest"
 	"strings"
 	"testing"
 
@@ -50,45 +47,3 @@ func TestBuildPreviewFragments_SamePrefixDifferentCommunity(t *testing.T) {
 	}
 }
 
-func TestCollectModulePrefixRows_CDNRefreshForcesFullGet(t *testing.T) {
-	m := store.NewMemory()
-	m.SeedDemo()
-	tenant, _, _, _, _ := m.DemoIDs()
-
-	mod, err := m.CreateModule(tenant, &store.Module{
-		Type:    "CDN_CIDRS",
-		Name:    "cdn-refresh-force-full",
-		Enabled: true,
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var gotIfNoneMatch string
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		gotIfNoneMatch = strings.TrimSpace(r.Header.Get("If-None-Match"))
-		w.Header().Set("ETag", "etag-new")
-		_, _ = w.Write([]byte("198.51.100.0/24\n"))
-	}))
-	defer srv.Close()
-
-	etag := "etag-old"
-	if _, err := m.CreateCDNSource(tenant, mod.ID, &store.CDNSource{
-		SourceKind: "txt",
-		URL:        srv.URL,
-		Etag:       etag,
-	}); err != nil {
-		t.Fatal(err)
-	}
-
-	collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if gotIfNoneMatch != "" {
-		t.Fatalf("refresh path must not send If-None-Match, got %q", gotIfNoneMatch)
-	}
-	if len(collected) != 1 || collected[0].Prefix != "198.51.100.0/24" {
-		t.Fatalf("unexpected collected rows: %+v", collected)
-	}
-}

internal/pipeline/prefetch.go

@@ -5,11 +5,12 @@ import (
 	"io"
 	"net/http"
 	"strings"
+	"time"
 
 	"evobgp/internal/store"
 )
 
-// PrefetchCDNSourceETags performs conditional GETs for CDN module sources and updates stored ETags when the origin responds 200.
+// PrefetchCDNSourceETags performs conditional GETs for CDN sources; on 200 parses CIDRs into module_prefix_snapshot.
 func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Client) error {
 	if hc == nil {
 		hc = http.DefaultClient
@@ -18,42 +19,70 @@ func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Clie
 	if err != nil {
 		return err
 	}
+	now := time.Now().UTC()
 	for _, tid := range tenants {
 		for _, mod := range st.ListModules(tid) {
-			if !mod.Enabled || mod.Type != "CDN_CIDRS" {
+			if mod == nil || !mod.Enabled || mod.Type != "CDN_CIDRS" {
+				continue
+			}
+			omod, err := st.GetModule(tid, mod.ID)
+			if err != nil {
 				continue
 			}
 			sources, err := st.ListCDNSources(tid, mod.ID)
 			if err != nil {
 				continue
 			}
+			var prior []store.PrefixRow
+			if snap, ok, _ := st.GetModulePrefixSnapshot(tid, mod.ID); ok && snap != nil {
+				prior = snap.Prefixes
+			}
 			for _, src := range sources {
-				u := strings.TrimSpace(src.URL)
-				if u == "" {
+				if src == nil || strings.TrimSpace(src.URL) == "" {
 					continue
 				}
-				req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
+				req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimSpace(src.URL), nil)
 				if err != nil {
 					continue
 				}
-				if strings.TrimSpace(src.Etag) != "" {
-					req.Header.Set("If-None-Match", strings.TrimSpace(src.Etag))
+				if etag := strings.TrimSpace(src.Etag); etag != "" {
+					req.Header.Set("If-None-Match", etag)
 				}
 				resp, err := hc.Do(req)
 				if err != nil {
 					continue
 				}
-				_, _ = io.Copy(io.Discard, resp.Body)
-				_ = resp.Body.Close()
+				if resp.StatusCode == http.StatusNotModified {
+					_ = resp.Body.Close()
+					continue
+				}
 				if resp.StatusCode != http.StatusOK {
+					_, _ = io.Copy(io.Discard, resp.Body)
+					_ = resp.Body.Close()
 					continue
 				}
-				etag := strings.TrimSpace(resp.Header.Get("ETag"))
-				if etag == "" || etag == strings.TrimSpace(src.Etag) {
+				body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
+				_ = resp.Body.Close()
+				if err != nil {
 					continue
 				}
-				e := etag
-				_, _ = st.UpdateCDNSource(tid, mod.ID, src.ID, &store.CDNSourcePatch{Etag: &e})
+				prefixStrs, err := parseCDNBody(string(body), src)
+				if err != nil {
+					continue
+				}
+				newEtag := strings.TrimSpace(resp.Header.Get("ETag"))
+				patch := &store.CDNSourcePatch{LastRefreshedAt: &now}
+				if newEtag != "" && newEtag != strings.TrimSpace(src.Etag) {
+					e := newEtag
+					patch.Etag = &e
+				}
+				_, _ = st.UpdateCDNSource(tid, mod.ID, src.ID, patch)
+				rows := cdnRowsFromParsed(omod, src, prefixStrs)
+				_ = mergeCDNSourceIntoModuleSnapshot(st, tid, omod, src.ID, rows)
+				_ = prior // prior may be stale after merge; refresh for next source in loop
+				if snap, ok, _ := st.GetModulePrefixSnapshot(tid, mod.ID); ok && snap != nil {
+					prior = snap.Prefixes
+				}
 			}
 		}
 	}

internal/pipeline/scheduler_due.go

@@ -0,0 +1,24 @@
+package pipeline
+
+import (
+	"time"
+
+	"evobgp/internal/store"
+)
+
+// SchedulerTickSec matches the scheduler ticker interval (internal/scheduler).
+const SchedulerTickSec = 30
+
+// ModuleDueForScheduler reports whether a module's refresh interval bucket rolled since the last scheduler tick.
+func ModuleDueForScheduler(mod *store.Module, now time.Time) bool {
+	if mod == nil || !mod.Enabled || mod.Type == "IP_RANGES" || mod.RefreshIntervalSec <= 0 {
+		return false
+	}
+	win := int64(mod.RefreshIntervalSec)
+	if win < 60 {
+		win = 60
+	}
+	cur := now.Unix() / win
+	prev := (now.Unix() - SchedulerTickSec) / win
+	return cur != prev
+}

internal/pipeline/tenant_refresh.go

@@ -0,0 +1,80 @@
+package pipeline
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+
+	"evobgp/internal/store"
+)
+
+// RefreshTenantModules ingests all listed modules in parallel and updates per-module snapshots.
+func RefreshTenantModules(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, moduleIDs []string) error {
+	if hc == nil {
+		hc = http.DefaultClient
+	}
+	var ids []string
+	seen := make(map[string]struct{})
+	for _, id := range moduleIDs {
+		id = strings.TrimSpace(id)
+		if id == "" {
+			continue
+		}
+		if _, ok := seen[id]; ok {
+			continue
+		}
+		seen[id] = struct{}{}
+		ids = append(ids, id)
+	}
+	if len(ids) == 0 {
+		return nil
+	}
+	if len(ids) == 1 {
+		return RefreshModuleIngest(ctx, st, hc, tenantID, ids[0])
+	}
+
+	sem := make(chan struct{}, collectConcurrency())
+	errs := make([]error, len(ids))
+	var wg sync.WaitGroup
+	for i, mid := range ids {
+		wg.Add(1)
+		go func(idx int, moduleID string) {
+			defer wg.Done()
+			sem <- struct{}{}
+			defer func() { <-sem }()
+			errs[idx] = RefreshModuleIngest(ctx, st, hc, tenantID, moduleID)
+		}(i, mid)
+	}
+	wg.Wait()
+	for _, err := range errs {
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// PickTenantRefreshTriggerModule returns a module id for revision metadata (first enabled due module).
+func PickTenantRefreshTriggerModule(st store.Backend, tenantID string, moduleIDs []string) (string, error) {
+	for _, id := range moduleIDs {
+		id = strings.TrimSpace(id)
+		if id == "" {
+			continue
+		}
+		mod, err := st.GetModule(tenantID, id)
+		if err != nil {
+			return "", err
+		}
+		if mod.Enabled {
+			return mod.ID, nil
+		}
+	}
+	for _, mod := range st.ListModules(tenantID) {
+		if mod != nil && mod.Enabled {
+			return mod.ID, nil
+		}
+	}
+	return "", fmt.Errorf("no enabled module for tenant refresh")
+}