Add backend capacity policy configuration

This commit is contained in:
2026-07-29 20:48:10 +00:00
parent ffe6d261a9
commit 238fa90bfa
6 changed files with 268 additions and 17 deletions

View File

@@ -163,10 +163,20 @@ type PromptMessageTemplate struct {
// Backend describes reusable OpenAI-compatible connection defaults.
type Backend struct {
ID string
Endpoint string
APIKeyEnv string
ExtraParams map[string]any
ID string
Endpoint string
APIKeyEnv string
ExtraParams map[string]any
ConcurrencyLimit int
QueueCapacity int
QueueCapacitySet bool
}
// BackendCapacityPolicy describes normalized run and generation capacity for
// one limited backend.
type BackendCapacityPolicy struct {
ConcurrencyLimit int
QueueCapacity int
}
// ExecutionProfile describes how and where to execute a model.