EvoBGP/internal/jobs/deployment_reconcile.go

package jobs

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"strconv"
	"time"

	"evobgp/internal/logging"
	"evobgp/internal/nodedispatch"
	"evobgp/internal/store"
)

func deploymentFenceForJob(j *Job) store.JobFence {
	generation, _ := strconv.ParseInt(j.metaString("reconcile_generation"), 10, 64)
	round, _ := strconv.Atoi(j.metaString("reconcile_round"))
	return store.JobFence{JobID: j.ID, TenantID: j.TenantID, OwnerID: j.OwnerID, Token: j.FencingToken, ExpectedGeneration: generation, RetryRound: round}
}

func beginDeployment(ctx context.Context, st store.Deployments, f store.JobFence, speaker, revision string) (store.DeploymentSpeaker, error) {
	for {
		d, err := st.BeginDeploymentSpeaker(ctx, f, speaker, revision)
		if !errors.Is(err, store.ErrDeploymentBusy) {
			return d, err
		}
		timer := time.NewTimer(time.Second)
		select {
		case <-ctx.Done():
			timer.Stop()
			return store.DeploymentSpeaker{}, ctx.Err()
		case <-timer.C:
		}
	}
}

// ReconcileDeployments schedules at most three recovery rounds for current desired
// generations. Stable keys coalesce API replicas; the worker CAS checks generation
// again before publishing, and preserves the original desired request sequence.
func ReconcileDeployments(ctx context.Context, r *Registry, st store.Deployments) error {
	rows, err := st.ListPendingDeployments(ctx, 100)
	if err != nil {
		return err
	}
	for _, d := range rows {
		key := fmt.Sprintf("deployment_reconcile:%s:%s:%d", d.JobID, d.SpeakerID, d.RetryRound+1)
		_, _, err = r.EnqueueContext(ctx, d.TenantID, KindDeployApply, &key, nil, map[string]any{"revision_id": d.RevisionID, "speaker_id": d.SpeakerID, "reconcile_generation": strconv.FormatInt(d.Generation, 10), "reconcile_round": strconv.Itoa(d.RetryRound + 1), "source_deployment_job_id": d.JobID})
		if err != nil {
			return err
		}
	}
	return nil
}

// StartDeploymentReconciliationLoop belongs to the API server lifecycle. Static
// ingest/deploy/scheduler processes must not start this job-producing loop.
func StartDeploymentReconciliationLoop(ctx context.Context, r *Registry, st store.Backend) {
	if r == nil || r.queue == nil || st == nil {
		return
	}
	r.durableMu.Lock()
	if r.durableStop {
		r.durableMu.Unlock()
		return
	}
	r.durableLoopWG.Add(1)
	r.durableMu.Unlock()
	go func() {
		defer r.durableLoopWG.Done()
		ticker := time.NewTicker(10 * time.Second)
		defer ticker.Stop()
		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
			}
			r.durableMu.Lock()
			stopped := r.durableStop
			r.durableMu.Unlock()
			if stopped {
				return
			}
			qctx, cancel := context.WithTimeout(ctx, 40*time.Second)
			err := ObserveDeployments(qctx, st, nil)
			if err == nil {
				err = ReconcileDeployments(qctx, r, st)
			}
			cancel()
			if err != nil && ctx.Err() == nil {
				logging.With("component", "jobs.deployment_reconcile").Warn("reconciliation failed", "err", err)
			}
		}
	}()
}

// ObserveDeployments samples compatible health before reconciling drift. The CP
// receipt time and optional agent source time are stored separately; missing
// legacy revision fields never manufacture an applied/verified confirmation.
func ObserveDeployments(ctx context.Context, st store.Backend, client *http.Client) error {
	rows, err := st.ListDeploymentObservations(ctx, 10)
	if err != nil {
		return err
	}
	for _, d := range rows {
		sp, e := st.GetSpeaker(d.TenantID, d.SpeakerID)
		if e != nil {
			return e
		}
		observation := store.DeploymentObservation{Error: "remote health unavailable for this speaker"}
		if nodedispatch.Enabled() && store.SpeakerNeedsRemoteDispatch(sp.Role, store.ParseSpeakerMeta(sp.MetaJSON)) {
			hctx, cancel := context.WithTimeout(ctx, 3*time.Second)
			h := nodedispatch.FetchAgentHealth(hctx, sp, nodedispatch.Options{HTTPClient: client, Timeout: 3 * time.Second})
			cancel()
			observation = store.DeploymentObservation{RevisionID: h.LastAppliedRevisionID, SourceAt: h.LastSyncAt, Error: h.Error}
			if h.OK && observation.RevisionID == "" {
				observation.Error = "legacy health did not report an applied revision"
			}
		}
		if err = st.ObserveDeployment(ctx, d, observation); err != nil && !errors.Is(err, store.ErrJobLeaseLost) {
			return err
		}
	}
	return nil
}