perf: quick wins for pipeline, jobs, httpapi and web ops

- CDN snapshot: batch merge после parallel fetch, без race на persist - ListRevisionPrefixes: лёгкая проверка revision вместо GetRevision - Jobs: timeout/cancel context для pipeline и deploy - HTTP server timeouts; кэш birdc для GET /peers - ListModules: batch DoH profiles одним запросом - Web: tab-scoped load на /operations, debounce job search, меньше over-fetch на dashboard Co-authored-by: Cursor <[email protected]>
2026-05-21 10:42:13 +07:00
parent 5fca165c69
commit be3d73f374
13 changed files with 425 additions and 34 deletions
@@ -0,0 +1,77 @@
+package jobs
+
+import (
+	"context"
+	"os"
+	"strconv"
+	"time"
+)
+
+const (
+	defaultJobTimeoutModuleRefresh = 10 * time.Minute
+	defaultJobTimeoutTenantRefresh = 15 * time.Minute
+	defaultJobTimeoutDeployApply   = 5 * time.Minute
+	defaultJobTimeoutPeerReconcile = 10 * time.Minute
+	defaultJobTimeoutRollback      = 5 * time.Minute
+	defaultJobTimeoutBirdReload    = 2 * time.Minute
+)
+
+func jobTimeout(kind string) time.Duration {
+	envKey := map[string]string{
+		KindModuleRefresh:    "EVOBGP_JOB_TIMEOUT_MODULE_REFRESH",
+		KindTenantRefresh:    "EVOBGP_JOB_TIMEOUT_TENANT_REFRESH",
+		KindDeployApply:      "EVOBGP_JOB_TIMEOUT_DEPLOY_APPLY",
+		KindPeerReconcile:    "EVOBGP_JOB_TIMEOUT_PEER_RECONCILE",
+		KindRevisionRollback: "EVOBGP_JOB_TIMEOUT_ROLLBACK",
+		KindBirdReload:       "EVOBGP_JOB_TIMEOUT_BIRD_RELOAD",
+	}[kind]
+	if envKey != "" {
+		if d, err := time.ParseDuration(os.Getenv(envKey)); err == nil && d > 0 {
+			return d
+		}
+	}
+	switch kind {
+	case KindModuleRefresh:
+		return defaultJobTimeoutModuleRefresh
+	case KindTenantRefresh:
+		return defaultJobTimeoutTenantRefresh
+	case KindDeployApply:
+		return defaultJobTimeoutDeployApply
+	case KindPeerReconcile:
+		return defaultJobTimeoutPeerReconcile
+	case KindRevisionRollback:
+		return defaultJobTimeoutRollback
+	case KindBirdReload:
+		return defaultJobTimeoutBirdReload
+	default:
+		if n, err := strconv.Atoi(os.Getenv("EVOBGP_JOB_TIMEOUT_SEC")); err == nil && n > 0 {
+			return time.Duration(n) * time.Second
+		}
+		return defaultJobTimeoutModuleRefresh
+	}
+}
+
+// workContext returns a timeout context that also cancels when the job is cancelled.
+func (j *Job) workContext() (context.Context, context.CancelFunc) {
+	if j == nil {
+		return context.Background(), func() {}
+	}
+	timeout := jobTimeout(j.Kind)
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	go func() {
+		ticker := time.NewTicker(500 * time.Millisecond)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				if j.IsCancelRequested() {
+					cancel()
+					return
+				}
+			}
+		}
+	}()
+	return ctx, cancel
+}
@@ -102,7 +102,17 @@ func (w *Worker) Process(j *Job) {
 			j.Fail("missing module_id in job meta")
 			return
 		}
-		if err := pipeline.RefreshModuleIngest(context.Background(), w.Store, w.httpClient(), j.TenantID, mid); err != nil {
+		ctx, cancel := j.workContext()
+		defer cancel()
+		if ctx.Err() != nil {
+			j.MarkCancelled()
+			return
+		}
+		if err := pipeline.RefreshModuleIngest(ctx, w.Store, w.httpClient(), j.TenantID, mid); err != nil {
+			if ctx.Err() != nil {
+				j.MarkCancelled()
+				return
+			}
 			j.Fail(err.Error())
 			return
 		}
@@ -121,11 +131,17 @@ func (w *Worker) Process(j *Job) {
 			j.Succeed()
 			return
 		}
+		ctx, cancel := j.workContext()
+		defer cancel()
 		ctl := &birdfmt.BirdCtl{
 			Socket: sock,
 			Birdc:  strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_BIN")),
 		}
-		if err := ctl.Configure(context.Background()); err != nil {
+		if err := ctl.Configure(ctx); err != nil {
+			if ctx.Err() != nil {
+				j.MarkCancelled()
+				return
+			}
 			j.Fail(err.Error())
 			return
 		}
@@ -152,9 +168,14 @@ func (w *Worker) runPeerReconcile(j *Job) {
 		return
 	}
 	if len(latest) == 0 {
-		// First run fallback: render full tenant state once if no baseline revision exists yet.
-		rid, err := pipeline.RenderTenantRevision(context.Background(), w.Store, w.httpClient(), j.TenantID, triggerModuleID)
+		ctx, cancel := j.workContext()
+		defer cancel()
+		rid, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
 		if err != nil {
+			if ctx.Err() != nil {
+				j.MarkCancelled()
+				return
+			}
 			j.Fail(err.Error())
 			return
 		}
@@ -171,8 +192,14 @@ func (w *Worker) runPeerReconcile(j *Job) {
 			}
 			cursor = next
 		}
-		rid, err := pipeline.RenderTenantRevisionFromPrefixes(context.Background(), w.Store, w.httpClient(), j.TenantID, triggerModuleID, rows)
+		ctx, cancel := j.workContext()
+		defer cancel()
+		rid, err := pipeline.RenderTenantRevisionFromPrefixes(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID, rows)
 		if err != nil {
+			if ctx.Err() != nil {
+				j.MarkCancelled()
+				return
+			}
 			j.Fail(err.Error())
 			return
 		}
@@ -230,7 +257,13 @@ func (w *Worker) runTenantRefresh(j *Job) {
 		j.Fail("missing module_ids in job meta")
 		return
 	}
-	if err := pipeline.RefreshTenantModules(context.Background(), w.Store, w.httpClient(), j.TenantID, moduleIDs); err != nil {
+	ctx, cancel := j.workContext()
+	defer cancel()
+	if err := pipeline.RefreshTenantModules(ctx, w.Store, w.httpClient(), j.TenantID, moduleIDs); err != nil {
+		if ctx.Err() != nil {
+			j.MarkCancelled()
+			return
+		}
 		j.Fail(err.Error())
 		return
 	}
@@ -275,6 +308,8 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) {
 	mu := w.tenantRefreshMu(j.TenantID)
 	mu.Lock()
 	defer mu.Unlock()
+	ctx, cancel := j.workContext()
+	defer cancel()
 	deferDeploy := false
 	if w.Registry != nil {
 		deferDeploy = w.Registry.CountOtherActiveRefresh(j.TenantID, j.ID) > 0
@@ -288,8 +323,12 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) {
 		return
 	}

-	rev, err := pipeline.RenderTenantRevision(context.Background(), w.Store, w.httpClient(), j.TenantID, triggerModuleID)
+	rev, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
 	if err != nil {
+		if ctx.Err() != nil {
+			j.MarkCancelled()
+			return
+		}
 		j.Fail(err.Error())
 		return
 	}
@@ -335,6 +374,8 @@ func (w *Worker) runDeployApply(j *Job) {
 		j.Fail("missing revision_id in job meta")
 		return
 	}
+	ctx, cancel := j.workContext()
+	defer cancel()
 	activeDir := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_ACTIVE_DIR"))
 	if activeDir != "" {
 		revObj, err := w.Store.GetRevision(j.TenantID, revID)
@@ -354,7 +395,11 @@ func (w *Worker) runDeployApply(j *Job) {
 			Socket:     strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_SOCKET")),
 		}
 		ctl := &birdfmt.BirdCtl{Bird: cfg.BirdBin, Birdc: cfg.BirdcBin, Socket: cfg.Socket}
-		if err := birddeploy.ApplyRevision(context.Background(), ctl, revObj, cfg); err != nil {
+		if err := birddeploy.ApplyRevision(ctx, ctl, revObj, cfg); err != nil {
+			if ctx.Err() != nil {
+				j.MarkCancelled()
+				return
+			}
 			j.Fail(err.Error())
 			return
 		}