perf: quick wins for pipeline, jobs, httpapi and web ops
- CDN snapshot: batch merge после parallel fetch, без race на persist - ListRevisionPrefixes: лёгкая проверка revision вместо GetRevision - Jobs: timeout/cancel context для pipeline и deploy - HTTP server timeouts; кэш birdc для GET /peers - ListModules: batch DoH profiles одним запросом - Web: tab-scoped load на /operations, debounce job search, меньше over-fetch на dashboard Co-authored-by: Cursor <[email protected]>
This commit is contained in:
@@ -45,6 +45,35 @@ func mergeSnapshotDropSource(rows []store.PrefixRow, sourceKey string) []store.P
|
||||
return out
|
||||
}
|
||||
|
||||
// mergeSnapshotDropCDNSources removes all cdn:* rows (used before batch CDN merge).
|
||||
func mergeSnapshotDropCDNSources(rows []store.PrefixRow) []store.PrefixRow {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]store.PrefixRow, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if !strings.HasPrefix(strings.TrimSpace(row.Source), "cdn:") {
|
||||
out = append(out, row)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// mergeAllCDNSourcesIntoModuleSnapshot replaces all CDN rows in one write (avoids parallel read-modify-write races).
|
||||
func mergeAllCDNSourcesIntoModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, priorSnapshot []store.PrefixRow, cdnRows []store.PrefixRow) error {
|
||||
if st == nil || mod == nil {
|
||||
return nil
|
||||
}
|
||||
var base []store.PrefixRow
|
||||
if len(priorSnapshot) > 0 {
|
||||
base = mergeSnapshotDropCDNSources(priorSnapshot)
|
||||
} else if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil {
|
||||
base = mergeSnapshotDropCDNSources(snap.Prefixes)
|
||||
}
|
||||
merged := append(base, cdnRows...)
|
||||
return persistModuleSnapshot(st, tenantID, mod, merged)
|
||||
}
|
||||
|
||||
func cdnRowsFromParsed(mod *store.Module, src *store.CDNSource, pfxStrings []string) []store.PrefixRow {
|
||||
var rows []store.PrefixRow
|
||||
for _, p := range pfxStrings {
|
||||
@@ -156,3 +185,68 @@ func applyCDNSourceHTTPResult(ctx context.Context, st store.Backend, hc *http.Cl
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// fetchCDNSourceRows loads CDN prefixes without persisting the module snapshot (caller merges once).
|
||||
func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, priorSnapshot []store.PrefixRow, now time.Time) ([]store.PrefixRow, error) {
|
||||
u := strings.TrimSpace(src.URL)
|
||||
if u == "" {
|
||||
return nil, nil
|
||||
}
|
||||
sourceKey := cdnSourceKey(src.ID)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if etag := strings.TrimSpace(src.Etag); etag != "" {
|
||||
req.Header.Set("If-None-Match", etag)
|
||||
}
|
||||
resp, err := hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
|
||||
}
|
||||
|
||||
if resp.StatusCode == http.StatusNotModified {
|
||||
if cached := cachedCDNPrefixRows(st, tenantID, moduleID, priorSnapshot, sourceKey); len(cached) > 0 {
|
||||
_ = resp.Body.Close()
|
||||
return cached, nil
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
req2, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err = hc.Do(req2)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
|
||||
}
|
||||
if resp.StatusCode == http.StatusNotModified {
|
||||
_ = resp.Body.Close()
|
||||
return nil, fmt.Errorf("cdn url %s: 304 without cached prefixes", u)
|
||||
}
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
return nil, fmt.Errorf("cdn url %s: %s", u, resp.Status)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
prefixStrs, err := parseCDNBody(string(body), src)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdn parse %s: %w", u, err)
|
||||
}
|
||||
etag := strings.TrimSpace(resp.Header.Get("ETag"))
|
||||
patch := &store.CDNSourcePatch{}
|
||||
if etag != "" && etag != strings.TrimSpace(src.Etag) {
|
||||
e := etag
|
||||
patch.Etag = &e
|
||||
}
|
||||
refreshedAt := now
|
||||
patch.LastRefreshedAt = &refreshedAt
|
||||
_, _ = st.UpdateCDNSource(tenantID, moduleID, src.ID, patch)
|
||||
|
||||
return cdnRowsFromParsed(mod, src, prefixStrs), nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user