Пакет httpclient: timeout 45s, idle pool, DoWithRetry. Scheduler и nodecli используют retry; pipeline/asnresolve/jobs — httpclient.New вместо DefaultClient. Co-authored-by: Cursor <[email protected]>
92 lines
2.4 KiB
Go
92 lines
2.4 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"evobgp/internal/httpclient"
|
|
"evobgp/internal/store"
|
|
)
|
|
|
|
// PrefetchCDNSourceETags performs conditional GETs for CDN sources; on 200 parses CIDRs into module_prefix_snapshot.
|
|
func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Client) error {
|
|
if hc == nil {
|
|
hc = httpclient.New(httpclient.DefaultTimeout)
|
|
}
|
|
tenants, err := st.ListTenantIDs()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
now := time.Now().UTC()
|
|
for _, tid := range tenants {
|
|
for _, mod := range st.ListModules(tid) {
|
|
if mod == nil || !mod.Enabled || mod.Type != "CDN_CIDRS" {
|
|
continue
|
|
}
|
|
omod, err := st.GetModule(tid, mod.ID)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
sources, err := st.ListCDNSources(tid, mod.ID)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
var prior []store.PrefixRow
|
|
if snap, ok, _ := st.GetModulePrefixSnapshot(tid, mod.ID); ok && snap != nil {
|
|
prior = snap.Prefixes
|
|
}
|
|
for _, src := range sources {
|
|
if src == nil || strings.TrimSpace(src.URL) == "" {
|
|
continue
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimSpace(src.URL), nil)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if etag := strings.TrimSpace(src.Etag); etag != "" {
|
|
req.Header.Set("If-None-Match", etag)
|
|
}
|
|
resp, err := hc.Do(req)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if resp.StatusCode == http.StatusNotModified {
|
|
_ = resp.Body.Close()
|
|
continue
|
|
}
|
|
if resp.StatusCode != http.StatusOK {
|
|
_, _ = io.Copy(io.Discard, resp.Body)
|
|
_ = resp.Body.Close()
|
|
continue
|
|
}
|
|
body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
|
_ = resp.Body.Close()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
prefixStrs, err := parseCDNBody(string(body), src)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
newEtag := strings.TrimSpace(resp.Header.Get("ETag"))
|
|
patch := &store.CDNSourcePatch{LastRefreshedAt: &now}
|
|
if newEtag != "" && newEtag != strings.TrimSpace(src.Etag) {
|
|
e := newEtag
|
|
patch.Etag = &e
|
|
}
|
|
_, _ = st.UpdateCDNSource(tid, mod.ID, src.ID, patch)
|
|
rows := cdnRowsFromParsed(omod, src, prefixStrs)
|
|
_ = mergeCDNSourceIntoModuleSnapshot(st, tid, omod, src.ID, rows)
|
|
_ = prior // prior may be stale after merge; refresh for next source in loop
|
|
if snap, ok, _ := st.GetModulePrefixSnapshot(tid, mod.ID); ok && snap != nil {
|
|
prior = snap.Prefixes
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|