Files
EvoBGP/internal/store/memory_deployment.go
DenozordecandCursor f8e0520817 feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь.

Co-authored-by: Cursor <[email protected]>
2026-11-02 17:23:59 +07:00

236 lines
7.2 KiB
Go

package store
import (
"context"
"sort"
"time"
)
type deploymentReservation struct {
Owner string
Token int64
Until time.Time
}
func (m *Memory) deploymentFence(f JobFence) (JobRecord, error) {
j, ok := m.jobs[f.JobID]
if !ok || j.TenantID != f.TenantID || j.Status != "running" || j.OwnerID != f.OwnerID || j.FencingToken != f.Token || j.CancelRequested || j.LeaseUntil == nil || !j.LeaseUntil.After(time.Now()) {
return JobRecord{}, ErrJobLeaseLost
}
return j, nil
}
func (m *Memory) BeginDeploymentSpeaker(ctx context.Context, f JobFence, speaker, revision string) (DeploymentSpeaker, error) {
if err := ctx.Err(); err != nil {
return DeploymentSpeaker{}, err
}
m.jobMu.Lock()
defer m.jobMu.Unlock()
j, err := m.deploymentFence(f)
if err != nil {
return DeploymentSpeaker{}, err
}
m.mu.Lock()
defer m.mu.Unlock()
sp, ok := m.speakers[speaker]
if !ok || sp.TenantID != f.TenantID {
return DeploymentSpeaker{}, ErrNotFound
}
r, ok := m.revisions[revision]
if !ok || r.TenantID != f.TenantID {
return DeploymentSpeaker{}, ErrNotFound
}
if m.deployments == nil {
m.deployments = make(map[string]DeploymentSpeaker)
m.deploymentCurrent = make(map[string]DeploymentSpeaker)
}
key := f.JobID + ":" + speaker
cur := m.deploymentCurrent[speaker]
if f.ExpectedGeneration != 0 && cur.JobID != f.JobID && (cur.Generation != f.ExpectedGeneration || cur.RevisionID != revision) {
return DeploymentSpeaker{}, ErrJobLeaseLost
}
reserve := func() error {
if m.deploymentReservations == nil {
m.deploymentReservations = make(map[string]deploymentReservation)
}
if r := m.deploymentReservations[speaker]; r.Until.After(time.Now()) {
return ErrDeploymentBusy
}
m.deploymentReservations[speaker] = deploymentReservation{Owner: f.OwnerID, Token: f.Token, Until: time.Now().Add(DeploymentDeliveryWindow)}
return nil
}
if existing, ok := m.deployments[key]; ok {
if cur.JobID != f.JobID || cur.Generation != existing.Generation || cur.RevisionID != revision {
return DeploymentSpeaker{}, ErrJobLeaseLost
}
if existing.State != "applied" {
if err := reserve(); err != nil {
return DeploymentSpeaker{}, err
}
}
return existing, nil
}
if f.ExpectedGeneration != 0 {
j.RequestSequence = cur.RequestSequence
}
if cur.RequestSequence >= j.RequestSequence && cur.JobID != "" && f.ExpectedGeneration == 0 {
return DeploymentSpeaker{}, ErrJobLeaseLost
}
if err := reserve(); err != nil {
return DeploymentSpeaker{}, err
}
d := DeploymentSpeaker{JobID: f.JobID, TenantID: f.TenantID, SpeakerID: speaker, RevisionID: revision, Generation: cur.Generation + 1, RequestSequence: j.RequestSequence, State: "published", RetryRound: f.RetryRound}
m.deployments[key] = d
m.deploymentCurrent[speaker] = d
m.publishedRevision[speaker] = publishedInfo{RevisionID: revision, PublishedAt: time.Now().UTC()}
return d, nil
}
func (m *Memory) RecordDeploymentSpeaker(ctx context.Context, f JobFence, d DeploymentSpeaker, state, confirmation, message string) error {
if err := ctx.Err(); err != nil {
return err
}
m.jobMu.Lock()
defer m.jobMu.Unlock()
if _, err := m.deploymentFence(f); err != nil {
return err
}
m.mu.Lock()
defer m.mu.Unlock()
cur := m.deploymentCurrent[d.SpeakerID]
if d.JobID != f.JobID || d.TenantID != f.TenantID || cur.JobID != d.JobID || cur.Generation != d.Generation || cur.RevisionID != d.RevisionID || m.publishedRevision[d.SpeakerID].RevisionID != d.RevisionID {
return ErrJobLeaseLost
}
r := m.deploymentReservations[d.SpeakerID]
if r.Owner != f.OwnerID || r.Token != f.Token || !r.Until.After(time.Now()) {
return ErrJobLeaseLost
}
if state == "applied" && confirmation != "agent_reported" && confirmation != "local_applied" {
return ErrInvalidInput
}
if state != "dispatched" && state != "unknown" {
delete(m.deploymentReservations, d.SpeakerID)
}
cur.State, cur.Confirmation, cur.Error = state, confirmation, message
if state == "dispatched" {
cur.DispatchAttempts++
}
if state == "applied" {
revision := d.RevisionID
m.speakers[d.SpeakerID].LastAppliedRevisionID = &revision
}
m.deploymentCurrent[d.SpeakerID] = cur
m.deployments[d.JobID+":"+d.SpeakerID] = cur
return nil
}
func (m *Memory) ListDeploymentSpeakers(ctx context.Context, tenant, job string) ([]DeploymentSpeaker, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
m.mu.RLock()
defer m.mu.RUnlock()
var out []DeploymentSpeaker
for _, d := range m.deployments {
if d.TenantID == tenant && d.JobID == job {
out = append(out, d)
}
}
return out, nil
}
func (m *Memory) ListPendingDeployments(ctx context.Context, limit int) ([]DeploymentSpeaker, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
m.jobMu.Lock()
defer m.jobMu.Unlock()
m.mu.RLock()
defer m.mu.RUnlock()
var out []DeploymentSpeaker
for speaker, d := range m.deploymentCurrent {
j := m.jobs[d.JobID]
if d.State != "applied" && d.RetryRound < 3 && !j.CancelRequested && (j.Status == "failed" || j.Status == "succeeded") && !m.deploymentReservations[speaker].Until.After(time.Now()) {
out = append(out, d)
if len(out) >= limit {
break
}
}
}
return out, nil
}
func (m *Memory) ListDeploymentObservations(ctx context.Context, limit int) ([]DeploymentSpeaker, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if limit <= 0 || limit > 100 {
limit = 10
}
m.jobMu.Lock()
defer m.jobMu.Unlock()
m.mu.Lock()
defer m.mu.Unlock()
var out []DeploymentSpeaker
for speaker, d := range m.deploymentCurrent {
j := m.jobs[d.JobID]
if !j.CancelRequested && (j.Status == "failed" || j.Status == "succeeded") && !m.deploymentReservations[speaker].Until.After(time.Now()) {
out = append(out, d)
}
}
sort.Slice(out, func(i, j int) bool {
a, b := out[i].HealthReceivedAt, out[j].HealthReceivedAt
if a == nil && b == nil {
return out[i].SpeakerID < out[j].SpeakerID
}
if a == nil {
return true
}
if b == nil {
return false
}
return a.Before(*b)
})
if len(out) > limit {
out = out[:limit]
}
for i := range out {
out[i].ObservationToken++
m.deploymentCurrent[out[i].SpeakerID] = out[i]
m.deployments[out[i].JobID+":"+out[i].SpeakerID] = out[i]
}
return out, nil
}
func (m *Memory) ObserveDeployment(ctx context.Context, d DeploymentSpeaker, o DeploymentObservation) error {
if err := ctx.Err(); err != nil {
return err
}
m.mu.Lock()
defer m.mu.Unlock()
cur := m.deploymentCurrent[d.SpeakerID]
if cur.TenantID != d.TenantID || cur.JobID != d.JobID || cur.Generation != d.Generation || cur.RevisionID != d.RevisionID || cur.ObservationToken != d.ObservationToken || m.publishedRevision[d.SpeakerID].RevisionID != d.RevisionID || m.deploymentReservations[d.SpeakerID].Until.After(time.Now()) {
return ErrJobLeaseLost
}
now := time.Now().UTC()
cur.ObservationToken++
cur.HealthReceivedAt = &now
cur.HealthRevisionID = o.RevisionID
cur.HealthSourceAt = o.SourceAt
cur.HealthError = o.Error
if o.Error == "" && o.RevisionID != "" {
if o.RevisionID == d.RevisionID {
cur.State = "applied"
cur.Confirmation = "agent_reported"
revision := d.RevisionID
m.speakers[d.SpeakerID].LastAppliedRevisionID = &revision
} else {
cur.State = "unknown"
cur.Confirmation = "unconfirmed"
}
}
m.deploymentCurrent[d.SpeakerID] = cur
m.deployments[d.JobID+":"+d.SpeakerID] = cur
return nil
}