DenozordecandCursor
f8e0520817
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <[email protected]>
106 lines
3.1 KiB
Go
106 lines
3.1 KiB
Go
package httpapi
|
|
|
|
import (
|
|
"net/http"
|
|
"os"
|
|
"strconv"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
type expensiveOperation int
|
|
|
|
const (
|
|
lookupOperation expensiveOperation = iota
|
|
cdnPreviewOperation
|
|
revisionPreviewOperation
|
|
)
|
|
|
|
type tenantWork struct {
|
|
start time.Time
|
|
counts [3]int
|
|
active int
|
|
}
|
|
|
|
// operationLimiter is deliberately local to one API process. It never queues
|
|
// waiters or evicts active tenants to admit new identities.
|
|
type operationLimiter struct {
|
|
mu sync.Mutex
|
|
tenants map[string]*tenantWork
|
|
active, maxActive, maxTenantActive, maxTenants int
|
|
rates [3]int
|
|
nextSweep time.Time
|
|
}
|
|
|
|
func newOperationLimiter() *operationLimiter {
|
|
return &operationLimiter{
|
|
tenants: make(map[string]*tenantWork), maxTenants: 4096,
|
|
maxActive: operationEnvLimit("EVOBGP_API_EXPENSIVE_MAX_CONCURRENT", 4, 64),
|
|
maxTenantActive: operationEnvLimit("EVOBGP_API_EXPENSIVE_TENANT_CONCURRENT", 2, 64),
|
|
rates: [3]int{operationEnvLimit("EVOBGP_API_LOOKUP_PER_MINUTE", 120, 100000), operationEnvLimit("EVOBGP_API_CDN_PREVIEW_PER_MINUTE", 30, 100000), operationEnvLimit("EVOBGP_API_REVISION_PREVIEW_PER_MINUTE", 30, 100000)},
|
|
}
|
|
}
|
|
|
|
func operationEnvLimit(name string, fallback, maximum int) int {
|
|
n, err := strconv.Atoi(os.Getenv(name))
|
|
if err != nil || n < 1 || n > maximum {
|
|
return fallback
|
|
}
|
|
return n
|
|
}
|
|
|
|
func (l *operationLimiter) acquire(tenant string, op expensiveOperation, now time.Time) (func(), int) {
|
|
l.mu.Lock()
|
|
defer l.mu.Unlock()
|
|
if !now.Before(l.nextSweep) {
|
|
for key, entry := range l.tenants {
|
|
if entry.active == 0 && !now.Before(entry.start.Add(time.Minute)) {
|
|
delete(l.tenants, key)
|
|
}
|
|
}
|
|
l.nextSweep = now.Add(time.Second)
|
|
}
|
|
if l.active >= l.maxActive {
|
|
return nil, 1
|
|
}
|
|
entry := l.tenants[tenant]
|
|
if entry == nil {
|
|
if len(l.tenants) >= l.maxTenants {
|
|
return nil, 1
|
|
}
|
|
entry = &tenantWork{start: now}
|
|
l.tenants[tenant] = entry
|
|
}
|
|
if !now.Before(entry.start.Add(time.Minute)) {
|
|
entry.start = now
|
|
entry.counts = [3]int{}
|
|
}
|
|
if entry.active >= l.maxTenantActive {
|
|
return nil, 1
|
|
}
|
|
if entry.counts[op] >= l.rates[op] {
|
|
wait := entry.start.Add(time.Minute).Sub(now)
|
|
return nil, max(1, int((wait+time.Second-1)/time.Second))
|
|
}
|
|
entry.counts[op]++
|
|
entry.active++
|
|
l.active++
|
|
var once sync.Once
|
|
return func() { once.Do(func() { l.mu.Lock(); defer l.mu.Unlock(); entry.active--; l.active-- }) }, 0
|
|
}
|
|
|
|
func (s *Server) acquireOperation(w http.ResponseWriter, r *http.Request, tenant string, op expensiveOperation) (func(), bool) {
|
|
if r.Context().Err() != nil {
|
|
writeProblem(w, http.StatusRequestTimeout, "Request Timeout", "request cancelled")
|
|
return nil, false
|
|
}
|
|
s.operationOnce.Do(func() { s.operationLimiter = newOperationLimiter() })
|
|
release, retry := s.operationLimiter.acquire(tenant, op, time.Now())
|
|
if release == nil {
|
|
w.Header().Set("Retry-After", strconv.Itoa(retry))
|
|
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "preview/lookup quota exceeded; retry later")
|
|
return nil, false
|
|
}
|
|
return release, true
|
|
}
|