fix(pipeline): harden upstream resilience and production shutdown

DoH через DoWithRetry; CDN preview через UpstreamHTTPDo; частичный fail CDN (EVOBGP_CDN_PARTIAL_OK); безопасный доступ к Job.Meta; drain jobs при SIGTERM; ValidateProductionEnforce при EVOBGP_PRODUCTION=1.

Co-authored-by: Cursor <[email protected]>

internal/jobs/job.go

@@ -1,6 +1,7 @@
 package jobs
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"sort"
@@ -111,6 +112,42 @@ func (j *Job) mergeMeta(kv map[string]any) {
 	}
 }
 
+// metaString returns a string meta field under lock (safe vs concurrent mergeMeta).
+func (j *Job) metaString(key string) string {
+	j.mu.Lock()
+	defer j.mu.Unlock()
+	if j.Meta == nil {
+		return ""
+	}
+	s, _ := j.Meta[key].(string)
+	return s
+}
+
+// metaBool returns a bool meta field under lock.
+func (j *Job) metaBool(key string) bool {
+	j.mu.Lock()
+	defer j.mu.Unlock()
+	if j.Meta == nil {
+		return false
+	}
+	b, _ := j.Meta[key].(bool)
+	return b
+}
+
+// metaCopy returns a shallow copy of job meta under lock.
+func (j *Job) metaCopy() map[string]any {
+	j.mu.Lock()
+	defer j.mu.Unlock()
+	if j.Meta == nil {
+		return nil
+	}
+	out := make(map[string]any, len(j.Meta))
+	for k, v := range j.Meta {
+		out[k] = v
+	}
+	return out
+}
+
 // statusLocked is used by the worker defer for metrics (any stable terminal or in-flight status).
 func (j *Job) statusLocked() string {
 	j.mu.Lock()
@@ -528,3 +565,65 @@ func (r *Registry) RequestCancel(tenantID, jobID string) (*Job, error) {
 	j.RequestCancel()
 	return j, nil
 }
+
+// RequestCancelAll requests cancellation of all non-terminal jobs (all tenants).
+func (r *Registry) RequestCancelAll() int {
+	if r == nil {
+		return 0
+	}
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	n := 0
+	for _, j := range r.byID {
+		if j == nil {
+			continue
+		}
+		st := j.statusLocked()
+		if st == StatusSucceeded || st == StatusFailed || st == StatusCancelled {
+			continue
+		}
+		j.RequestCancel()
+		n++
+	}
+	return n
+}
+
+// ActiveCount returns the number of queued or running jobs.
+func (r *Registry) ActiveCount() int {
+	if r == nil {
+		return 0
+	}
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	n := 0
+	for _, j := range r.byID {
+		if j == nil {
+			continue
+		}
+		st := j.statusLocked()
+		if st == StatusQueued || st == StatusRunning {
+			n++
+		}
+	}
+	return n
+}
+
+// Drain waits until no queued/running jobs remain or ctx is done.
+// Call RequestCancelAll first for a cooperative shutdown.
+func (r *Registry) Drain(ctx context.Context) error {
+	if r == nil {
+		return nil
+	}
+	ticker := time.NewTicker(50 * time.Millisecond)
+	defer ticker.Stop()
+	for {
+		if r.ActiveCount() == 0 {
+			return nil
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+		}
+	}
+}