Add backend capacity policy configuration
This commit is contained in:
@@ -163,10 +163,20 @@ type PromptMessageTemplate struct {
|
||||
|
||||
// Backend describes reusable OpenAI-compatible connection defaults.
|
||||
type Backend struct {
|
||||
ID string
|
||||
Endpoint string
|
||||
APIKeyEnv string
|
||||
ExtraParams map[string]any
|
||||
ID string
|
||||
Endpoint string
|
||||
APIKeyEnv string
|
||||
ExtraParams map[string]any
|
||||
ConcurrencyLimit int
|
||||
QueueCapacity int
|
||||
QueueCapacitySet bool
|
||||
}
|
||||
|
||||
// BackendCapacityPolicy describes normalized run and generation capacity for
|
||||
// one limited backend.
|
||||
type BackendCapacityPolicy struct {
|
||||
ConcurrencyLimit int
|
||||
QueueCapacity int
|
||||
}
|
||||
|
||||
// ExecutionProfile describes how and where to execute a model.
|
||||
|
||||
Reference in New Issue
Block a user