feat: add tenant refresh functionality to HTTP API and job processing

Implemented a new endpoint for tenant refresh in the HTTP API, allowing for the refresh of modules associated with a tenant. Enhanced job processing to handle tenant refresh jobs, including logic for managing module IDs and job status updates. Updated the scheduler to enqueue tenant refresh jobs based on module due dates, improving the overall efficiency of module management. Additionally, introduced caching for ASN prefix data to optimize performance during refresh operations.

internal/pipeline/cdn_snapshot.go

@@ -0,0 +1,126 @@
+package pipeline
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"evobgp/internal/store"
+)
+
+func cdnSourceKey(sourceID string) string {
+	return "cdn:" + strings.TrimSpace(sourceID)
+}
+
+func mergeSnapshotDropSource(rows []store.PrefixRow, sourceKey string) []store.PrefixRow {
+	if len(rows) == 0 {
+		return nil
+	}
+	out := make([]store.PrefixRow, 0, len(rows))
+	for _, row := range rows {
+		if row.Source != sourceKey {
+			out = append(out, row)
+		}
+	}
+	return out
+}
+
+func cdnRowsFromParsed(mod *store.Module, src *store.CDNSource, pfxStrings []string) []store.PrefixRow {
+	var rows []store.PrefixRow
+	for _, p := range pfxStrings {
+		p = strings.TrimSpace(p)
+		if p == "" {
+			continue
+		}
+		comm := src.CommunityID
+		if comm == nil && mod != nil && mod.DefaultCommunityID != nil {
+			c := *mod.DefaultCommunityID
+			comm = &c
+		}
+		rows = append(rows, store.PrefixRow{Prefix: p, CommunityID: comm, Source: cdnSourceKey(src.ID)})
+	}
+	return rows
+}
+
+// mergeCDNSourceIntoModuleSnapshot replaces rows for one CDN source in the module snapshot.
+func mergeCDNSourceIntoModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, sourceID string, newRows []store.PrefixRow) error {
+	if st == nil || mod == nil {
+		return nil
+	}
+	sourceKey := cdnSourceKey(sourceID)
+	var base []store.PrefixRow
+	if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil {
+		base = mergeSnapshotDropSource(snap.Prefixes, sourceKey)
+	}
+	merged := append(base, newRows...)
+	return persistModuleSnapshot(st, tenantID, mod, merged)
+}
+
+func parseCDNBody(body string, src *store.CDNSource) ([]string, error) {
+	pfxs, err := ExtractCIDRs(body, src.SourceKind, src.PrefixPath)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]string, 0, len(pfxs))
+	for _, pfx := range pfxs {
+		out = append(out, pfx.String())
+	}
+	return out, nil
+}
+
+func applyCDNSourceHTTPResult(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, priorSnapshot []store.PrefixRow, now time.Time) ([]store.PrefixRow, error) {
+	u := strings.TrimSpace(src.URL)
+	if u == "" {
+		return nil, nil
+	}
+	sourceKey := cdnSourceKey(src.ID)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
+	if err != nil {
+		return nil, err
+	}
+	if etag := strings.TrimSpace(src.Etag); etag != "" {
+		req.Header.Set("If-None-Match", etag)
+	}
+	resp, err := hc.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusNotModified {
+		if cached := prefixRowsForSource(priorSnapshot, sourceKey); len(cached) > 0 {
+			return cached, nil
+		}
+		return nil, fmt.Errorf("cdn url %s: 304 without cached prefixes", u)
+	}
+	if resp.StatusCode != http.StatusOK {
+		_, _ = io.Copy(io.Discard, resp.Body)
+		return nil, fmt.Errorf("cdn url %s: %s", u, resp.Status)
+	}
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
+	if err != nil {
+		return nil, err
+	}
+	prefixStrs, err := parseCDNBody(string(body), src)
+	if err != nil {
+		return nil, fmt.Errorf("cdn parse %s: %w", u, err)
+	}
+	etag := strings.TrimSpace(resp.Header.Get("ETag"))
+	patch := &store.CDNSourcePatch{}
+	if etag != "" && etag != strings.TrimSpace(src.Etag) {
+		e := etag
+		patch.Etag = &e
+	}
+	refreshedAt := now
+	patch.LastRefreshedAt = &refreshedAt
+	_, _ = st.UpdateCDNSource(tenantID, moduleID, src.ID, patch)
+
+	rows := cdnRowsFromParsed(mod, src, prefixStrs)
+	if err := mergeCDNSourceIntoModuleSnapshot(st, tenantID, mod, src.ID, rows); err != nil {
+		return nil, err
+	}
+	return rows, nil
+}