eric/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Wire backend capacity into engine runtime

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-29 21:11:08 +00:00
parent 861da355d8
commit d2c4051dd0
5 changed files with 720 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
engine.go
View File

@@ -13,6 +13,7 @@ import (
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
@@ -62,6 +63,10 @@ var (
// ErrPromptRender identifies a failure to render prompt messages or the
// session ID from the resolved inputs and variables.
ErrPromptRender = errors.New("failed to render prompt")
// ErrCapacityExceeded identifies a Run rejected because the selected backend
// already admitted ConcurrencyLimit + QueueCapacity calls. It is not an
// invalid request, provider rate limit, or ErrLLMGenerate.
ErrCapacityExceeded = errors.New("backend capacity exceeded")
// ErrLLMGenerate identifies a model-client failure or a nil successful
// response. Errors returned by an injected LLMClient remain available
// through errors.Is.
@@ -75,7 +80,9 @@ var (
// Engine prepares and runs Promptkit prompt requests.
//
// An Engine is safe for concurrent calls to [Engine.Prepare] and [Engine.Run].
// Injected collaborators may consequently be invoked concurrently.
// Each Engine owns independent backend-capacity pools that coordinate Run
// admission and model generation. Injected collaborators may still be invoked
// concurrently across different backend pools or for unlimited backends.
type Engine struct {
runner *usecase.Runner
}
@@ -141,8 +148,10 @@ type engineOptions struct {
// WithLLMClient replaces the built-in model client used by [Engine.Run].
//
// A nil client makes NewEngine fail with ErrInvalidConfig. The client may be
// called concurrently and is not used by [Engine.Prepare].
// A nil client makes NewEngine fail with ErrInvalidConfig. The Engine schedules
// Generate calls according to the selected backend's capacity policy, but the
// client may still be called concurrently across different backend pools or for
// unlimited backends. The client is not used by [Engine.Prepare].
func WithLLMClient(client LLMClient) Option {
return optionFunc(func(options *engineOptions) error {
if client == nil {
@@ -313,8 +322,9 @@ func WithSchemaFile(path string) Option {
// or Run needs them.
//
// NewEngine returns an error matching ErrInvalidConfig for invalid
// configuration or options. It does not perform model requests or require
// credentials.
// configuration, options, or backend-capacity policies. Each constructed
// Engine has independent backend-capacity pools. Construction does not perform
// model requests or require credentials.
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
var options engineOptions
for _, opt := range opts {
@@ -347,6 +357,11 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err)
}
capacityManager, err := capacity.NewManager(backendRegistry.CapacityPolicies())
if err != nil {
return nil, fmt.Errorf("%w: failed to construct backend capacity manager: %v", ErrInvalidConfig, err)
}
validator := options.validator
if !options.validatorSource {
schemaDir := cfg.SchemaDir
@@ -367,6 +382,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
}
}
llmClient = capacity.NewClient(capacityManager, llmClient)
artifacts := options.artifactReader
if !options.artifactSource {
@@ -382,7 +398,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
prompt.NewGoRenderer(),
llmClient,
validator,
nil,
capacityManager,
),
}, nil
}
@@ -451,11 +467,14 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
// single-pass even when OutputContract.RepairAttempts is positive.
//
// Run can return every error category documented by [Engine.Prepare], plus
// ErrLLMGenerate. Errors from injected clients remain available through
// errors.Is. Cancellation is passed through the active collaborator and is
// reported in the applicable operation category; no general errors.Is
// relationship to ctx.Err is promised. A nil Engine returns ErrInvalidConfig.
// Run returns no partial result on error.
// ErrCapacityExceeded and ErrLLMGenerate. ErrCapacityExceeded identifies
// rejection before artifacts, schemas, rendering, or model generation because
// the selected backend's admission capacity is full; it does not match
// ErrInvalidRequest or ErrLLMGenerate. Errors from injected clients remain
// available through errors.Is. Cancellation while waiting for model-generation
// capacity matches both ErrLLMGenerate and the context error. Cancellation
// otherwise follows the active collaborator's documented behavior. A nil
// Engine returns ErrInvalidConfig. Run returns no partial result on error.
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)