Add backend capacity policy configuration

backends.go

@@ -31,6 +31,17 @@ type Backend struct {
 	// service_tier, reasoning_effort, or response_format. An empty map supplies
 	// no defaults. NewEngine deeply copies the map.
 	ExtraParams map[string]any
+	// ConcurrencyLimit is the maximum number of simultaneous model-generation
+	// calls allowed for this backend within one Engine. Zero leaves the backend
+	// unlimited. A negative value makes NewEngine fail with ErrInvalidConfig.
+	ConcurrencyLimit int
+	// QueueCapacity controls how many additional Run calls may be admitted
+	// beyond ConcurrencyLimit. Nil uses 1024 when ConcurrencyLimit is positive;
+	// a pointer uses its exact value, including zero. The pointed-to value must
+	// be non-negative, and QueueCapacity must be nil when ConcurrencyLimit is
+	// zero. Their sum must fit in an int. WithBackend copies the value and does
+	// not retain the pointer.
+	QueueCapacity *int
 }
 
 // WithBackend adds one Backend registration to the constructed Engine.
@@ -43,12 +54,20 @@ type Backend struct {
 // mutated after construction. WithBackend does not install package-global
 // state.
 func WithBackend(backend Backend) Option {
+	queueCapacity := 0
+	queueCapacitySet := backend.QueueCapacity != nil
+	if queueCapacitySet {
+		queueCapacity = *backend.QueueCapacity
+	}
 	return optionFunc(func(options *engineOptions) error {
 		options.backends = append(options.backends, domain.Backend{
-			ID:          backend.ID,
-			Endpoint:    backend.Endpoint,
-			APIKeyEnv:   backend.APIKeyEnv,
-			ExtraParams: backend.ExtraParams,
+			ID:               backend.ID,
+			Endpoint:         backend.Endpoint,
+			APIKeyEnv:        backend.APIKeyEnv,
+			ExtraParams:      backend.ExtraParams,
+			ConcurrencyLimit: backend.ConcurrencyLimit,
+			QueueCapacity:    queueCapacity,
+			QueueCapacitySet: queueCapacitySet,
 		})
 		return nil
 	})