perf(pipeline): parallel prefetch and lean revision GET

GetRevisionSummary без preview_fragments; boundedRepoCtx для GetRevision;
parallel CDN prefetch с ctx из ingest.

Co-authored-by: Cursor <[email protected]>

internal/pipeline/prefetch.go

@@ -5,87 +5,116 @@ import (
 	"io"
 	"net/http"
 	"strings"
+	"sync"
 	"time"
 
 	"evobgp/internal/httpclient"
 	"evobgp/internal/store"
 )
 
+type prefetchTask struct {
+	tenantID string
+	mod      *store.Module
+	src      *store.CDNSource
+}
+
 // PrefetchCDNSourceETags performs conditional GETs for CDN sources; on 200 parses CIDRs into module_prefix_snapshot.
 func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Client) error {
 	if hc == nil {
 		hc = httpclient.New(httpclient.DefaultTimeout)
 	}
+	if ctx == nil {
+		ctx = context.Background()
+	}
 	tenants, err := st.ListTenantIDs()
 	if err != nil {
 		return err
 	}
-	now := time.Now().UTC()
+	var tasks []prefetchTask
 	for _, tid := range tenants {
 		for _, mod := range st.ListModules(tid) {
 			if mod == nil || !mod.Enabled || mod.Type != "CDN_CIDRS" {
 				continue
 			}
-			omod, err := st.GetModule(tid, mod.ID)
-			if err != nil {
-				continue
-			}
 			sources, err := st.ListCDNSources(tid, mod.ID)
 			if err != nil {
 				continue
 			}
-			var prior []store.PrefixRow
-			if snap, ok, _ := st.GetModulePrefixSnapshot(tid, mod.ID); ok && snap != nil {
-				prior = snap.Prefixes
-			}
 			for _, src := range sources {
-				if src == nil || strings.TrimSpace(src.URL) == "" {
-					continue
-				}
-				req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimSpace(src.URL), nil)
-				if err != nil {
-					continue
-				}
-				if etag := strings.TrimSpace(src.Etag); etag != "" {
-					req.Header.Set("If-None-Match", etag)
-				}
-				resp, err := hc.Do(req)
-				if err != nil {
-					continue
-				}
-				if resp.StatusCode == http.StatusNotModified {
-					_ = resp.Body.Close()
-					continue
-				}
-				if resp.StatusCode != http.StatusOK {
-					_, _ = io.Copy(io.Discard, resp.Body)
-					_ = resp.Body.Close()
-					continue
-				}
-				body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
-				_ = resp.Body.Close()
-				if err != nil {
-					continue
-				}
-				prefixStrs, err := parseCDNBody(string(body), src)
-				if err != nil {
-					continue
-				}
-				newEtag := strings.TrimSpace(resp.Header.Get("ETag"))
-				patch := &store.CDNSourcePatch{LastRefreshedAt: &now}
-				if newEtag != "" && newEtag != strings.TrimSpace(src.Etag) {
-					e := newEtag
-					patch.Etag = &e
-				}
-				_, _ = st.UpdateCDNSource(tid, mod.ID, src.ID, patch)
-				rows := cdnRowsFromParsed(omod, src, prefixStrs)
-				_ = mergeCDNSourceIntoModuleSnapshot(st, tid, omod, src.ID, rows)
-				_ = prior // prior may be stale after merge; refresh for next source in loop
-				if snap, ok, _ := st.GetModulePrefixSnapshot(tid, mod.ID); ok && snap != nil {
-					prior = snap.Prefixes
+				if src != nil && strings.TrimSpace(src.URL) != "" {
+					tasks = append(tasks, prefetchTask{tenantID: tid, mod: mod, src: src})
 				}
 			}
 		}
 	}
+	if len(tasks) == 0 {
+		return nil
+	}
+	sem := make(chan struct{}, collectConcurrency())
+	var wg sync.WaitGroup
+	for _, task := range tasks {
+		wg.Add(1)
+		go func(t prefetchTask) {
+			defer wg.Done()
+			sem <- struct{}{}
+			defer func() { <-sem }()
+			prefetchOneCDNSource(ctx, st, hc, t)
+		}(task)
+	}
+	wg.Wait()
 	return nil
 }
+
+func prefetchOneCDNSource(ctx context.Context, st store.Backend, hc *http.Client, t prefetchTask) {
+	now := time.Now().UTC()
+	tid, mod, src := t.tenantID, t.mod, t.src
+	u := strings.TrimSpace(src.URL)
+	if _, err := ValidateCDNURL(u); err != nil {
+		return
+	}
+	if err := ResolveCDNURLHost(ctx, u); err != nil {
+		return
+	}
+	omod, err := st.GetModule(tid, mod.ID)
+	if err != nil {
+		return
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
+	if err != nil {
+		return
+	}
+	if etag := strings.TrimSpace(src.Etag); etag != "" {
+		req.Header.Set("If-None-Match", etag)
+	}
+	resp, err := upstreamHTTPDo(ctx, hc, req)
+	if err != nil {
+		return
+	}
+	if resp.StatusCode == http.StatusNotModified {
+		_ = resp.Body.Close()
+		return
+	}
+	if resp.StatusCode != http.StatusOK {
+		_, _ = io.Copy(io.Discard, resp.Body)
+		_ = resp.Body.Close()
+		return
+	}
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
+	_ = resp.Body.Close()
+	if err != nil {
+		return
+	}
+	prefixStrs, err := parseCDNBody(string(body), src)
+	if err != nil {
+		return
+	}
+	newEtag := strings.TrimSpace(resp.Header.Get("ETag"))
+	patch := &store.CDNSourcePatch{LastRefreshedAt: &now}
+	if newEtag != "" && newEtag != strings.TrimSpace(src.Etag) {
+		e := newEtag
+		patch.Etag = &e
+	}
+	_, _ = st.UpdateCDNSource(tid, mod.ID, src.ID, patch)
+	rows := cdnRowsFromParsed(omod, src, prefixStrs)
+	_ = mergeCDNSourceIntoModuleSnapshot(st, tid, omod, src.ID, rows)
+}