feat: update collectModulePrefixRows to support prior snapshots and enhance CDN/domain prefix collection

Modified the collectModulePrefixRows function to accept an optional priorSnapshot parameter, allowing for more efficient data retrieval by skipping unnecessary CDN fetches. Refactored the logic for collecting prefix rows from AS, CDN, and domain sources to utilize dedicated functions, improving code organization and maintainability. Additionally, introduced caching for module prefix snapshots to optimize performance during refresh operations.

internal/store/backend.go

@@ -79,6 +79,18 @@ type Backend interface {
 
 	ListGlobalSettings(tenantID string) (map[string]any, error)
 	PatchGlobalSettings(tenantID string, patch map[string]any) error
+
+	// Module prefix snapshots cache last successful collect per module (pipeline ingest/render).
+	GetModulePrefixSnapshot(tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error)
+	SetModulePrefixSnapshot(tenantID, moduleID, inputHash string, prefixes []PrefixRow) error
+	DeleteModulePrefixSnapshot(tenantID, moduleID string) error
+}
+
+// ModulePrefixSnapshot is the cached materialization for one module between refreshes.
+type ModulePrefixSnapshot struct {
+	InputHash   string
+	CollectedAt time.Time
+	Prefixes    []PrefixRow
 }
 
 // ModulePatch is a partial update for module.

internal/store/memory.go

@@ -39,7 +39,8 @@ type Memory struct {
 	domainEnt   map[string]*DomainEntry
 	ipRanges    map[string]*IPRangeEntry
 	settings    map[string]map[string]any // tenantID -> key -> JSON-compatible value
-	revPrefixes map[string][]PrefixRow
+	revPrefixes     map[string][]PrefixRow
+	moduleSnapshots map[string]*moduleSnapshotRec
 
 	// DemoIDs valid after SeedDemo()
 	demoTenantID   string
@@ -126,6 +127,7 @@ func NewMemory() *Memory {
 		ipRanges:          make(map[string]*IPRangeEntry),
 		settings:          make(map[string]map[string]any),
 		revPrefixes:       make(map[string][]PrefixRow),
+		moduleSnapshots:   make(map[string]*moduleSnapshotRec),
 	}
 }
 

internal/store/memory_crud.go

@@ -89,6 +89,9 @@ func (m *Memory) UpdateModule(tenantID, moduleID string, patch *ModulePatch) (*M
 func (m *Memory) SoftDeleteModule(tenantID, moduleID string) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
+	if m.moduleSnapshots != nil {
+		delete(m.moduleSnapshots, moduleSnapshotKey(tenantID, moduleID))
+	}
 	mod, ok := m.modules[moduleID]
 	if !ok || mod.TenantID != tenantID {
 		return ErrNotFound

internal/store/memory_snapshot.go

@@ -0,0 +1,62 @@
+package store
+
+import (
+	"strings"
+	"time"
+)
+
+func (m *Memory) GetModulePrefixSnapshot(tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	key := moduleSnapshotKey(tenantID, moduleID)
+	snap, ok := m.moduleSnapshots[key]
+	if !ok || snap == nil {
+		return nil, false, nil
+	}
+	cp := make([]PrefixRow, len(snap.Prefixes))
+	copy(cp, snap.Prefixes)
+	return &ModulePrefixSnapshot{
+		InputHash:   snap.InputHash,
+		CollectedAt: snap.CollectedAt,
+		Prefixes:    cp,
+	}, true, nil
+}
+
+func (m *Memory) SetModulePrefixSnapshot(tenantID, moduleID, inputHash string, prefixes []PrefixRow) error {
+	if strings.TrimSpace(tenantID) == "" || strings.TrimSpace(moduleID) == "" || strings.TrimSpace(inputHash) == "" {
+		return ErrInvalidInput
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.moduleSnapshots == nil {
+		m.moduleSnapshots = make(map[string]*moduleSnapshotRec)
+	}
+	cp := make([]PrefixRow, len(prefixes))
+	copy(cp, prefixes)
+	key := moduleSnapshotKey(tenantID, moduleID)
+	m.moduleSnapshots[key] = &moduleSnapshotRec{
+		InputHash:   inputHash,
+		CollectedAt: time.Now().UTC(),
+		Prefixes:    cp,
+	}
+	return nil
+}
+
+func (m *Memory) DeleteModulePrefixSnapshot(tenantID, moduleID string) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.moduleSnapshots != nil {
+		delete(m.moduleSnapshots, moduleSnapshotKey(tenantID, moduleID))
+	}
+	return nil
+}
+
+type moduleSnapshotRec struct {
+	InputHash   string
+	CollectedAt time.Time
+	Prefixes    []PrefixRow
+}
+
+func moduleSnapshotKey(tenantID, moduleID string) string {
+	return strings.TrimSpace(tenantID) + "\x00" + strings.TrimSpace(moduleID)
+}