Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <[email protected]>
236 lines
7.2 KiB
Go
236 lines
7.2 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"sort"
|
|
"time"
|
|
)
|
|
|
|
type deploymentReservation struct {
|
|
Owner string
|
|
Token int64
|
|
Until time.Time
|
|
}
|
|
|
|
func (m *Memory) deploymentFence(f JobFence) (JobRecord, error) {
|
|
j, ok := m.jobs[f.JobID]
|
|
if !ok || j.TenantID != f.TenantID || j.Status != "running" || j.OwnerID != f.OwnerID || j.FencingToken != f.Token || j.CancelRequested || j.LeaseUntil == nil || !j.LeaseUntil.After(time.Now()) {
|
|
return JobRecord{}, ErrJobLeaseLost
|
|
}
|
|
return j, nil
|
|
}
|
|
|
|
func (m *Memory) BeginDeploymentSpeaker(ctx context.Context, f JobFence, speaker, revision string) (DeploymentSpeaker, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return DeploymentSpeaker{}, err
|
|
}
|
|
m.jobMu.Lock()
|
|
defer m.jobMu.Unlock()
|
|
j, err := m.deploymentFence(f)
|
|
if err != nil {
|
|
return DeploymentSpeaker{}, err
|
|
}
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
sp, ok := m.speakers[speaker]
|
|
if !ok || sp.TenantID != f.TenantID {
|
|
return DeploymentSpeaker{}, ErrNotFound
|
|
}
|
|
r, ok := m.revisions[revision]
|
|
if !ok || r.TenantID != f.TenantID {
|
|
return DeploymentSpeaker{}, ErrNotFound
|
|
}
|
|
if m.deployments == nil {
|
|
m.deployments = make(map[string]DeploymentSpeaker)
|
|
m.deploymentCurrent = make(map[string]DeploymentSpeaker)
|
|
}
|
|
key := f.JobID + ":" + speaker
|
|
cur := m.deploymentCurrent[speaker]
|
|
if f.ExpectedGeneration != 0 && cur.JobID != f.JobID && (cur.Generation != f.ExpectedGeneration || cur.RevisionID != revision) {
|
|
return DeploymentSpeaker{}, ErrJobLeaseLost
|
|
}
|
|
reserve := func() error {
|
|
if m.deploymentReservations == nil {
|
|
m.deploymentReservations = make(map[string]deploymentReservation)
|
|
}
|
|
if r := m.deploymentReservations[speaker]; r.Until.After(time.Now()) {
|
|
return ErrDeploymentBusy
|
|
}
|
|
m.deploymentReservations[speaker] = deploymentReservation{Owner: f.OwnerID, Token: f.Token, Until: time.Now().Add(DeploymentDeliveryWindow)}
|
|
return nil
|
|
}
|
|
if existing, ok := m.deployments[key]; ok {
|
|
if cur.JobID != f.JobID || cur.Generation != existing.Generation || cur.RevisionID != revision {
|
|
return DeploymentSpeaker{}, ErrJobLeaseLost
|
|
}
|
|
if existing.State != "applied" {
|
|
if err := reserve(); err != nil {
|
|
return DeploymentSpeaker{}, err
|
|
}
|
|
}
|
|
return existing, nil
|
|
}
|
|
if f.ExpectedGeneration != 0 {
|
|
j.RequestSequence = cur.RequestSequence
|
|
}
|
|
if cur.RequestSequence >= j.RequestSequence && cur.JobID != "" && f.ExpectedGeneration == 0 {
|
|
return DeploymentSpeaker{}, ErrJobLeaseLost
|
|
}
|
|
if err := reserve(); err != nil {
|
|
return DeploymentSpeaker{}, err
|
|
}
|
|
d := DeploymentSpeaker{JobID: f.JobID, TenantID: f.TenantID, SpeakerID: speaker, RevisionID: revision, Generation: cur.Generation + 1, RequestSequence: j.RequestSequence, State: "published", RetryRound: f.RetryRound}
|
|
m.deployments[key] = d
|
|
m.deploymentCurrent[speaker] = d
|
|
m.publishedRevision[speaker] = publishedInfo{RevisionID: revision, PublishedAt: time.Now().UTC()}
|
|
return d, nil
|
|
}
|
|
|
|
func (m *Memory) RecordDeploymentSpeaker(ctx context.Context, f JobFence, d DeploymentSpeaker, state, confirmation, message string) error {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
m.jobMu.Lock()
|
|
defer m.jobMu.Unlock()
|
|
if _, err := m.deploymentFence(f); err != nil {
|
|
return err
|
|
}
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
cur := m.deploymentCurrent[d.SpeakerID]
|
|
if d.JobID != f.JobID || d.TenantID != f.TenantID || cur.JobID != d.JobID || cur.Generation != d.Generation || cur.RevisionID != d.RevisionID || m.publishedRevision[d.SpeakerID].RevisionID != d.RevisionID {
|
|
return ErrJobLeaseLost
|
|
}
|
|
r := m.deploymentReservations[d.SpeakerID]
|
|
if r.Owner != f.OwnerID || r.Token != f.Token || !r.Until.After(time.Now()) {
|
|
return ErrJobLeaseLost
|
|
}
|
|
if state == "applied" && confirmation != "agent_reported" && confirmation != "local_applied" {
|
|
return ErrInvalidInput
|
|
}
|
|
if state != "dispatched" && state != "unknown" {
|
|
delete(m.deploymentReservations, d.SpeakerID)
|
|
}
|
|
cur.State, cur.Confirmation, cur.Error = state, confirmation, message
|
|
if state == "dispatched" {
|
|
cur.DispatchAttempts++
|
|
}
|
|
if state == "applied" {
|
|
revision := d.RevisionID
|
|
m.speakers[d.SpeakerID].LastAppliedRevisionID = &revision
|
|
}
|
|
m.deploymentCurrent[d.SpeakerID] = cur
|
|
m.deployments[d.JobID+":"+d.SpeakerID] = cur
|
|
return nil
|
|
}
|
|
|
|
func (m *Memory) ListDeploymentSpeakers(ctx context.Context, tenant, job string) ([]DeploymentSpeaker, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
var out []DeploymentSpeaker
|
|
for _, d := range m.deployments {
|
|
if d.TenantID == tenant && d.JobID == job {
|
|
out = append(out, d)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (m *Memory) ListPendingDeployments(ctx context.Context, limit int) ([]DeploymentSpeaker, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
m.jobMu.Lock()
|
|
defer m.jobMu.Unlock()
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
var out []DeploymentSpeaker
|
|
for speaker, d := range m.deploymentCurrent {
|
|
j := m.jobs[d.JobID]
|
|
if d.State != "applied" && d.RetryRound < 3 && !j.CancelRequested && (j.Status == "failed" || j.Status == "succeeded") && !m.deploymentReservations[speaker].Until.After(time.Now()) {
|
|
out = append(out, d)
|
|
if len(out) >= limit {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (m *Memory) ListDeploymentObservations(ctx context.Context, limit int) ([]DeploymentSpeaker, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
if limit <= 0 || limit > 100 {
|
|
limit = 10
|
|
}
|
|
m.jobMu.Lock()
|
|
defer m.jobMu.Unlock()
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
var out []DeploymentSpeaker
|
|
for speaker, d := range m.deploymentCurrent {
|
|
j := m.jobs[d.JobID]
|
|
if !j.CancelRequested && (j.Status == "failed" || j.Status == "succeeded") && !m.deploymentReservations[speaker].Until.After(time.Now()) {
|
|
out = append(out, d)
|
|
}
|
|
}
|
|
sort.Slice(out, func(i, j int) bool {
|
|
a, b := out[i].HealthReceivedAt, out[j].HealthReceivedAt
|
|
if a == nil && b == nil {
|
|
return out[i].SpeakerID < out[j].SpeakerID
|
|
}
|
|
if a == nil {
|
|
return true
|
|
}
|
|
if b == nil {
|
|
return false
|
|
}
|
|
return a.Before(*b)
|
|
})
|
|
if len(out) > limit {
|
|
out = out[:limit]
|
|
}
|
|
for i := range out {
|
|
out[i].ObservationToken++
|
|
m.deploymentCurrent[out[i].SpeakerID] = out[i]
|
|
m.deployments[out[i].JobID+":"+out[i].SpeakerID] = out[i]
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (m *Memory) ObserveDeployment(ctx context.Context, d DeploymentSpeaker, o DeploymentObservation) error {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
cur := m.deploymentCurrent[d.SpeakerID]
|
|
if cur.TenantID != d.TenantID || cur.JobID != d.JobID || cur.Generation != d.Generation || cur.RevisionID != d.RevisionID || cur.ObservationToken != d.ObservationToken || m.publishedRevision[d.SpeakerID].RevisionID != d.RevisionID || m.deploymentReservations[d.SpeakerID].Until.After(time.Now()) {
|
|
return ErrJobLeaseLost
|
|
}
|
|
now := time.Now().UTC()
|
|
cur.ObservationToken++
|
|
cur.HealthReceivedAt = &now
|
|
cur.HealthRevisionID = o.RevisionID
|
|
cur.HealthSourceAt = o.SourceAt
|
|
cur.HealthError = o.Error
|
|
if o.Error == "" && o.RevisionID != "" {
|
|
if o.RevisionID == d.RevisionID {
|
|
cur.State = "applied"
|
|
cur.Confirmation = "agent_reported"
|
|
revision := d.RevisionID
|
|
m.speakers[d.SpeakerID].LastAppliedRevisionID = &revision
|
|
} else {
|
|
cur.State = "unknown"
|
|
cur.Confirmation = "unconfirmed"
|
|
}
|
|
}
|
|
m.deploymentCurrent[d.SpeakerID] = cur
|
|
m.deployments[d.JobID+":"+d.SpeakerID] = cur
|
|
return nil
|
|
}
|