feat: update collectModulePrefixRows to support prior snapshots and enhance CDN/domain prefix collection

Modified the collectModulePrefixRows function to accept an optional priorSnapshot parameter, allowing for more efficient data retrieval by skipping unnecessary CDN fetches. Refactored the logic for collecting prefix rows from AS, CDN, and domain sources to utilize dedicated functions, improving code organization and maintainability. Additionally, introduced caching for module prefix snapshots to optimize performance during refresh operations.

internal/pipeline/refresh.go

@@ -12,13 +12,11 @@ import (
 	"net/http"
 	"net/netip"
 	"net/url"
-	"os"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
 
-	"evobgp/internal/asnresolve"
 	"evobgp/internal/birdfmt"
 	"evobgp/internal/store"
 
@@ -55,10 +53,13 @@ func RefreshModuleIngest(ctx context.Context, st store.Backend, hc *http.Client,
 		return fmt.Errorf("module disabled")
 	}
 
-	_, err = collectModulePrefixRows(ctx, st, hc, tenantID, mod)
+	rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, mod, nil)
 	if err != nil {
 		return err
 	}
+	if err := persistModuleSnapshot(st, tenantID, mod, rows); err != nil {
+		return err
+	}
 	refreshedAt := time.Now().UTC()
 	_, _ = st.UpdateModule(tenantID, moduleID, &store.ModulePatch{LastRefreshedAt: &refreshedAt})
 	return nil
@@ -129,7 +130,8 @@ func RefreshModule(ctx context.Context, st store.Backend, hc *http.Client, tenan
 }
 
 // collectModulePrefixRows returns materialized prefix rows for a single module (source of truth from store / ASN resolve / CDN fetch).
-func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module) ([]store.PrefixRow, error) {
+// priorSnapshot is the last stored module snapshot (used to skip CDN fetches when refresh interval has not elapsed).
+func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) {
 	moduleID := mod.ID
 	switch mod.Type {
 	case "IP_RANGES":
@@ -153,110 +155,13 @@ func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Cli
 			return nil, err
 		}
 		sort.Slice(list, func(i, j int) bool { return list[i].ASN < list[j].ASN })
-		legacy := strings.TrimSpace(os.Getenv("EVOBGP_ASN_RESOLVE")) == "0"
-		seenPfx := make(map[string]struct{})
-		var rows []store.PrefixRow
-		for i, e := range list {
-			if !store.ValidASN(e.ASN) {
-				continue
-			}
-			comm := e.CommunityID
-			if comm == nil && mod.DefaultCommunityID != nil {
-				c := *mod.DefaultCommunityID
-				comm = &c
-			}
-			if legacy {
-				rows = append(rows, store.PrefixRow{Prefix: MaterializedASPrefixKey(e.ASN), CommunityID: comm, Source: "as_entry"})
-				continue
-			}
-			if i > 0 {
-				asnresolve.PolitePause()
-			}
-			pfxs, err := asnresolve.AnnouncedPrefixes(ctx, hc, e.ASN)
-			if err != nil {
-				return nil, fmt.Errorf("resolve AS%d: %w", e.ASN, err)
-			}
-			holder := ""
-			asnresolve.PolitePause()
-			if h, err := asnresolve.ASHolderName(ctx, hc, e.ASN); err == nil {
-				holder = h
-			}
-			now := time.Now().UTC()
-			if err := st.UpdateASEntryResolveMeta(tenantID, moduleID, e.ID, holder, int64(len(pfxs)), now); err != nil {
-				return nil, fmt.Errorf("as entry meta AS%d: %w", e.ASN, err)
-			}
-			src := fmt.Sprintf("as:%d", e.ASN)
-			for _, pfx := range pfxs {
-				k := pfx.String()
-				if _, ok := seenPfx[k]; ok {
-					continue
-				}
-				seenPfx[k] = struct{}{}
-				rows = append(rows, store.PrefixRow{Prefix: k, CommunityID: comm, Source: src})
-			}
-		}
-		return rows, nil
+		return collectASPrefixRows(ctx, st, hc, tenantID, mod, list)
 	case "CDN_CIDRS":
 		sources, err := st.ListCDNSources(tenantID, moduleID)
 		if err != nil {
 			return nil, err
 		}
-		var rows []store.PrefixRow
-		latestCDNRows := latestCDNRowsBySource(st, tenantID)
-		for _, src := range sources {
-			sourceKey := "cdn:" + src.ID
-			now := time.Now().UTC()
-			if shouldSkipCDNSourceFetch(src, now) {
-				if cached := latestCDNRows[sourceKey]; len(cached) > 0 {
-					rows = append(rows, cached...)
-					continue
-				}
-			}
-			u := strings.TrimSpace(src.URL)
-			if u == "" {
-				continue
-			}
-			req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
-			if err != nil {
-				return nil, err
-			}
-			resp, err := hc.Do(req)
-			if err != nil {
-				return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
-			}
-			if resp.StatusCode != http.StatusOK {
-				_, _ = io.Copy(io.Discard, resp.Body)
-				_ = resp.Body.Close()
-				return nil, fmt.Errorf("cdn url %s: %s", u, resp.Status)
-			}
-			body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
-			_ = resp.Body.Close()
-			if err != nil {
-				return nil, err
-			}
-			etag := strings.TrimSpace(resp.Header.Get("ETag"))
-			patch := &store.CDNSourcePatch{}
-			if etag != "" && etag != strings.TrimSpace(src.Etag) {
-				e := etag
-				patch.Etag = &e
-			}
-			refreshedAt := now
-			patch.LastRefreshedAt = &refreshedAt
-			_, _ = st.UpdateCDNSource(tenantID, moduleID, src.ID, patch)
-			pfxs, err := ExtractCIDRs(string(body), src.SourceKind, src.PrefixPath)
-			if err != nil {
-				return nil, fmt.Errorf("cdn parse %s: %w", u, err)
-			}
-			for _, pfx := range pfxs {
-				comm := src.CommunityID
-				if comm == nil && mod.DefaultCommunityID != nil {
-					c := *mod.DefaultCommunityID
-					comm = &c
-				}
-				rows = append(rows, store.PrefixRow{Prefix: pfx.String(), CommunityID: comm, Source: sourceKey})
-			}
-		}
-		return rows, nil
+		return collectCDNPrefixRows(ctx, st, hc, tenantID, mod, sources, priorSnapshot)
 	case "DOMAINS":
 		entries, err := st.ListDomainEntries(tenantID, moduleID)
 		if err != nil {
@@ -269,40 +174,7 @@ func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Cli
 				return nil, fmt.Errorf("get doh profile: %w", err)
 			}
 		}
-		var rows []store.PrefixRow
-		seen := make(map[string]struct{})
-		for _, e := range entries {
-			if e == nil {
-				continue
-			}
-			comm := e.CommunityID
-			if comm == nil && mod.DefaultCommunityID != nil {
-				c := *mod.DefaultCommunityID
-				comm = &c
-			}
-			addrs, err := resolveDomainIPs(ctx, hc, profile, e.FQDN)
-			if err != nil {
-				return nil, fmt.Errorf("resolve domain %q: %w", e.FQDN, err)
-			}
-			src := "domain:" + strings.TrimSpace(e.FQDN)
-			for _, ip := range addrs {
-				cidr := ipToHostPrefix(ip)
-				if cidr == "" {
-					continue
-				}
-				key := cidr + "|" + src
-				if _, ok := seen[key]; ok {
-					continue
-				}
-				seen[key] = struct{}{}
-				rows = append(rows, store.PrefixRow{
-					Prefix:      cidr,
-					CommunityID: comm,
-					Source:      src,
-				})
-			}
-		}
-		return rows, nil
+		return collectDomainPrefixRows(ctx, hc, mod, profile, entries)
 	default:
 		return nil, fmt.Errorf("unknown module type %q", mod.Type)
 	}
@@ -548,28 +420,6 @@ func ipToHostPrefix(ip netip.Addr) string {
 	return netip.PrefixFrom(ip, bits).Masked().String()
 }
 
-// aggregateTenantPrefixRowsAll builds the union of materialized prefixes for all enabled modules
-// using current source data from store/external resolvers.
-func aggregateTenantPrefixRowsAll(ctx context.Context, st store.Backend, hc *http.Client, tenantID string) ([]store.PrefixRow, error) {
-	mods := st.ListModules(tenantID)
-	var out []store.PrefixRow
-	for _, m := range mods {
-		if m == nil || !m.Enabled {
-			continue
-		}
-		omod, err := st.GetModule(tenantID, m.ID)
-		if err != nil {
-			return nil, err
-		}
-		rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, omod)
-		if err != nil {
-			return nil, fmt.Errorf("module %s: %w", m.ID, err)
-		}
-		out = append(out, rows...)
-	}
-	return out, nil
-}
-
 type prefixGroupKey struct {
 	community string
 	source    string