fix(pipeline): harden upstream resilience and production shutdown

DoH через DoWithRetry; CDN preview через UpstreamHTTPDo; частичный fail CDN (EVOBGP_CDN_PARTIAL_OK); безопасный доступ к Job.Meta; drain jobs при SIGTERM; ValidateProductionEnforce при EVOBGP_PRODUCTION=1.

Co-authored-by: Cursor <[email protected]>

internal/pipeline/collect_parallel.go

@@ -3,6 +3,7 @@ package pipeline
 import (
 	"context"
 	"fmt"
+	"log"
 	"net/http"
 	"os"
 	"strings"
@@ -218,12 +219,21 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client
 	wg.Wait()
 
 	var out []store.PrefixRow
+	var skipped int
 	for _, r := range results {
 		if r.err != nil {
+			if cdnPartialOK() {
+				log.Printf("pipeline: CDN partial skip source error: %v", r.err)
+				skipped++
+				continue
+			}
 			return nil, r.err
 		}
 		out = append(out, r.rows...)
 	}
+	if skipped > 0 && len(out) == 0 && len(valid) > 0 {
+		return nil, fmt.Errorf("cdn: all %d source(s) failed (partial ok)", len(valid))
+	}
 	if len(valid) > 0 {
 		if err := mergeAllCDNSourcesIntoModuleSnapshot(st, tenantID, mod, priorSnapshot, out); err != nil {
 			return nil, err

internal/pipeline/collect_stale.go

@@ -20,6 +20,13 @@ func staleOnUpstreamError() bool {
 	return false
 }
 
+// cdnPartialOK reports whether a failed CDN source without stale cache should be skipped
+// instead of failing the whole module refresh. Opt-in: EVOBGP_CDN_PARTIAL_OK=1.
+func cdnPartialOK() bool {
+	v := strings.TrimSpace(os.Getenv("EVOBGP_CDN_PARTIAL_OK"))
+	return v == "1" || strings.EqualFold(v, "true")
+}
+
 func logStaleUpstream(kind, detail string) {
 	log.Printf("pipeline: stale upstream fallback (%s): %s", kind, detail)
 }

internal/pipeline/refresh.go

@@ -282,7 +282,7 @@ func resolveDomainWithDOHMessage(ctx context.Context, hc *http.Client, baseURL,
 		return nil, err
 	}
 	req.Header.Set("Accept", "application/dns-message")
-	resp, err := hc.Do(req)
+	resp, err := httpclient.DoWithRetry(ctx, hc, req, 3)
 	if err != nil {
 		return nil, err
 	}
@@ -354,7 +354,7 @@ func resolveDomainWithDOHJSON(ctx context.Context, hc *http.Client, baseURL, hos
 	}
 	req.Header.Set("Accept", "application/dns-json")
 
-	resp, err := hc.Do(req)
+	resp, err := httpclient.DoWithRetry(ctx, hc, req, 3)
 	if err != nil {
 		return nil, err
 	}

internal/pipeline/upstream_http.go

@@ -8,6 +8,11 @@ import (
 	"evobgp/internal/httpclient"
 )
 
+// UpstreamHTTPDo performs an outbound GET/POST with circuit breaker + retries (CDN, previews).
+func UpstreamHTTPDo(ctx context.Context, hc *http.Client, req *http.Request) (*http.Response, error) {
+	return upstreamHTTPDo(ctx, hc, req)
+}
+
 func upstreamHTTPDo(ctx context.Context, hc *http.Client, req *http.Request) (*http.Response, error) {
 	if hc == nil {
 		hc = httpclient.New(httpclient.DefaultTimeout)