feat: add tenant refresh functionality to HTTP API and job processing

Implemented a new endpoint for tenant refresh in the HTTP API, allowing for the refresh of modules associated with a tenant. Enhanced job processing to handle tenant refresh jobs, including logic for managing module IDs and job status updates. Updated the scheduler to enqueue tenant refresh jobs based on module due dates, improving the overall efficiency of module management. Additionally, introduced caching for ASN prefix data to optimize performance during refresh operations.

internal/pipeline/collect_parallel.go

@@ -3,14 +3,12 @@ package pipeline
 import (
 	"context"
 	"fmt"
-	"io"
 	"net/http"
 	"os"
 	"strings"
 	"sync"
 	"time"
 
-	"evobgp/internal/asnresolve"
 	"evobgp/internal/store"
 )
 
@@ -76,15 +74,11 @@ func collectASPrefixRows(ctx context.Context, st store.Backend, hc *http.Client,
 				c := *mod.DefaultCommunityID
 				comm = &c
 			}
-			pfxs, err := asnresolve.AnnouncedPrefixes(ctx, hc, entry.ASN)
+			pfxs, holder, err := resolveASNForEntry(ctx, st, hc, entry.ASN)
 			if err != nil {
 				results[idx] = entryResult{err: fmt.Errorf("resolve AS%d: %w", entry.ASN, err)}
 				return
 			}
-			holder := ""
-			if h, err := asnresolve.ASHolderName(ctx, hc, entry.ASN); err == nil {
-				holder = h
-			}
 			src := fmt.Sprintf("as:%d", entry.ASN)
 			var rows []store.PrefixRow
 			for _, pfx := range pfxs {
@@ -161,18 +155,11 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client
 					return
 				}
 			}
-			u := strings.TrimSpace(src.URL)
-			if u == "" {
-				return
-			}
-			rows, err := fetchAndParseCDNSource(ctx, hc, st, tenantID, moduleID, mod, src, now)
+			rows, err := applyCDNSourceHTTPResult(ctx, st, hc, tenantID, moduleID, mod, src, priorSnapshot, now)
 			if err != nil {
 				results[idx] = srcResult{err: err}
 				return
 			}
-			for i := range rows {
-				rows[i].Source = sourceKey
-			}
 			results[idx] = srcResult{rows: rows}
 		}(i, src)
 	}
@@ -188,51 +175,6 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client
 	return out, nil
 }
 
-func fetchAndParseCDNSource(ctx context.Context, hc *http.Client, st store.Backend, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, now time.Time) ([]store.PrefixRow, error) {
-	u := strings.TrimSpace(src.URL)
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
-	if err != nil {
-		return nil, err
-	}
-	resp, err := hc.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
-	}
-	if resp.StatusCode != http.StatusOK {
-		_, _ = io.Copy(io.Discard, resp.Body)
-		_ = resp.Body.Close()
-		return nil, fmt.Errorf("cdn url %s: %s", u, resp.Status)
-	}
-	body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
-	_ = resp.Body.Close()
-	if err != nil {
-		return nil, err
-	}
-	etag := strings.TrimSpace(resp.Header.Get("ETag"))
-	patch := &store.CDNSourcePatch{}
-	if etag != "" && etag != strings.TrimSpace(src.Etag) {
-		e := etag
-		patch.Etag = &e
-	}
-	refreshedAt := now
-	patch.LastRefreshedAt = &refreshedAt
-	_, _ = st.UpdateCDNSource(tenantID, moduleID, src.ID, patch)
-	pfxs, err := ExtractCIDRs(string(body), src.SourceKind, src.PrefixPath)
-	if err != nil {
-		return nil, fmt.Errorf("cdn parse %s: %w", u, err)
-	}
-	var rows []store.PrefixRow
-	for _, pfx := range pfxs {
-		comm := src.CommunityID
-		if comm == nil && mod.DefaultCommunityID != nil {
-			c := *mod.DefaultCommunityID
-			comm = &c
-		}
-		rows = append(rows, store.PrefixRow{Prefix: pfx.String(), CommunityID: comm})
-	}
-	return rows, nil
-}
-
 func collectDomainPrefixRows(ctx context.Context, hc *http.Client, mod *store.Module, profile *store.DohProfile, entries []*store.DomainEntry) ([]store.PrefixRow, error) {
 	var validDom []*store.DomainEntry
 	for _, e := range entries {