From 157209f097715eddf253ea79f341ede5835cc327 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 29 Aug 2026 14:26:04 +0000 Subject: [PATCH] Expose backend identity and capacity errors --- docs/api.md | 10 ++- docs/cli.md | 11 ++- docs/operations.md | 8 ++ docs/roadmap/implementation.md | 2 + internal/adapter/cli/run.go | 12 ++- internal/adapter/cli/run_test.go | 34 +++++++- internal/adapter/http/dto.go | 2 + internal/adapter/http/handler.go | 4 + internal/adapter/http/handler_test.go | 111 ++++++++++++++++++++++++++ internal/format/prepared_run.go | 3 + internal/format/prepared_run_test.go | 3 + 11 files changed, 194 insertions(+), 6 deletions(-) diff --git a/docs/api.md b/docs/api.md index 06bebd8..e1f50f3 100644 --- a/docs/api.md +++ b/docs/api.md @@ -100,13 +100,13 @@ validation contract. The response contains: optional `uri`; - `validation`: `status`, `mode`, `repair_attempts`, `is_valid`, plus optional `errors` and `schema_path`; -- `metadata`: run, prompt, rendered-prompt, profile, model, input-hash, usage, +- `metadata`: run, prompt, rendered-prompt, profile, optional backend identity, model, input-hash, usage, timing, validation, and repair-attempt metadata; and - optional `raw_model_output` when requested. `metadata.model_params` has `endpoint`, `model`, `temperature`, `max_tokens`, `top_p`, and `timeout_seconds`, plus optional -`service_tier`, `reasoning_effort`, `api_key_env`, and `extra_params`. +`backend_id`, `service_tier`, `reasoning_effort`, `api_key_env`, and `extra_params`. `metadata.usage` always includes `prompt_tokens`, `completion_tokens`, `total_tokens`, `cached_tokens`, and `cache_write_tokens`; unavailable cache usage is reported as zero. @@ -115,6 +115,11 @@ When Promptkit resolves a direct or definition-rendered session ID, `metadata.session_id` contains that effective result value. It is omitted when no effective session ID exists. +`metadata.selected_backend_id` and `metadata.model_params.backend_id` report +the corresponding Promptkit result fields independently when present. Both are +omitted for an endpoint-only profile; Scriptorium does not infer backend +identity from an endpoint. + A validation failure has `validation.status: "failed"`, `is_valid: false`, and any available diagnostic errors, while still returning the artifact and metadata. @@ -150,6 +155,7 @@ Messages are concise and do not expose wrapped internal causes. | `500` | `validation_runtime_failed` | Schema or validator runtime failure. | | `500` | `internal_error` | Unclassified server failure. | | `502` | `llm_failed` | Outbound model request failed. | +| `503` | `capacity_exceeded` | The selected model backend has no admission capacity. No retry timing is supplied. | ## Retry And Idempotency diff --git a/docs/cli.md b/docs/cli.md index 2f3a326..d0cf847 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -150,9 +150,12 @@ CLI inputs are file references. HTTP inline inputs are defined by the ## Output And Exit Behavior - `run` writes generated content to stdout, or to `--out` when supplied, and - writes a concise summary to stderr. + writes a concise summary to stderr. The summary includes `backend=` when + Promptkit selected a backend; endpoint-only profiles omit it. - `render` writes prepared-run output to stdout, or to `--out` when supplied, - without a success summary. + without a success summary. Text output includes `selected_backend_id` after + `selected_profile_id` when Promptkit selected one; endpoint-only profiles + omit it. - `serve` writes startup and server errors to stderr. Exit statuses: @@ -163,6 +166,10 @@ Exit statuses: | `1` | Parse, configuration, loading, rendering, generation, output-write, or other runtime error. | | `2` | `run` generated and wrote output, but validation failed. | +A backend admission rejection is a runtime error and prints `run error: model +backend capacity is exhausted`. The HTTP capacity response is defined in the +[HTTP API reference](api.md). + ## Workflows And Examples The [maintained render script](../examples/render-markdown-summary.sh) is a diff --git a/docs/operations.md b/docs/operations.md index d4defd8..a90eea2 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -94,6 +94,14 @@ protect request bodies, HTTP file artifacts, and encoded responses; configure them through the [configuration reference](config.md) and rely on the [HTTP API reference](api.md) for their response effects. +Configured backend concurrency and queue capacity are enforced per constructed +Promptkit engine. A `serve` process constructs one engine for its handler, so +concurrent HTTP requests share that transient admission state. Scriptorium does +not retain workflow state: capacity is neither durable nor a queue of resumable +runs. When admission is exhausted, HTTP returns `503 capacity_exceeded` without +retry timing; callers choose any retry policy that is safe for another model +call. + Before increasing a limit: 1. measure representative input, generated-output, and optional raw-output diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index d1fcbb8..d789d88 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -278,6 +278,8 @@ validation ownership. ## Stage 5: Present Backend Identity And Map Capacity Outcomes +**Completion: Complete.** + Expose Promptkit's selected routing identity and make overload behavior a stable application contract. diff --git a/internal/adapter/cli/run.go b/internal/adapter/cli/run.go index b2bb7f7..cc297ed 100644 --- a/internal/adapter/cli/run.go +++ b/internal/adapter/cli/run.go @@ -148,7 +148,7 @@ func runCommand(args []string, stdout, stderr io.Writer) int { res, runErr := engine.Run(context.Background(), req) if runErr != nil { - fmt.Fprintf(stderr, "run error: %v\n", runErr) + fmt.Fprintln(stderr, runErrorMessage(runErr)) return ExitRuntimeError } @@ -732,9 +732,19 @@ func printSummary(stderr io.Writer, res *promptkit.RunResult) { if res.Usage.CachedTokens != 0 || res.Usage.CacheWriteTokens != 0 { fmt.Fprintf(stderr, " cached_tokens=%d cache_write_tokens=%d", res.Usage.CachedTokens, res.Usage.CacheWriteTokens) } + if res.SelectedBackendID != "" { + fmt.Fprintf(stderr, " backend=%s", res.SelectedBackendID) + } fmt.Fprintln(stderr) } +func runErrorMessage(err error) string { + if errors.Is(err, promptkit.ErrCapacityExceeded) { + return "run error: model backend capacity is exhausted" + } + return fmt.Sprintf("run error: %v", err) +} + func printUsage(w io.Writer) { fmt.Fprintln(w, "usage: scriptorium ...") fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID [--prompt-version VERSION] [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--timeout 10m]") diff --git a/internal/adapter/cli/run_test.go b/internal/adapter/cli/run_test.go index 1678ee2..3ab138e 100644 --- a/internal/adapter/cli/run_test.go +++ b/internal/adapter/cli/run_test.go @@ -1033,13 +1033,16 @@ backends: queue_capacity: 0 `, lib.promptDir, lib.profileDir)) - code, _, stderr := runCLICommand(t, renderCommand, []string{ + code, stdout, stderr := runCLICommand(t, renderCommand, []string{ "--config", configPath, "--prompt", "custom", }) if code != ExitOK { t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) } + if !strings.Contains(stdout, "selected_backend_id: local-gpu") { + t.Fatalf("expected configured backend in prepared output, got:\n%s", stdout) + } } func TestRenderCommandMapsReasoningEffortAndSessionID(t *testing.T) { @@ -1624,6 +1627,9 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) { if strings.Contains(stderr.String(), "cached_tokens=") || strings.Contains(stderr.String(), "cache_write_tokens=") { t.Fatalf("expected zero cache usage to be omitted from summary, got %q", stderr.String()) } + if strings.Contains(stderr.String(), "backend=") { + t.Fatalf("expected endpoint-only backend to be omitted from summary, got %q", stderr.String()) + } } func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) { @@ -1653,6 +1659,32 @@ func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) { if !strings.Contains(summary, "cached_tokens=0 cache_write_tokens=3") { t.Fatalf("expected cache usage in summary, got %q", summary) } + if strings.Contains(summary, "backend=") { + t.Fatalf("expected endpoint-only backend to be omitted from summary, got %q", summary) + } +} + +func TestPrintSummaryIncludesBackendWhenPresent(t *testing.T) { + var stderr bytes.Buffer + printSummary(&stderr, &promptkit.RunResult{ + PromptID: "p", + PromptVersion: "1", + SelectedProfileID: "exec", + SelectedBackendID: "local", + ModelName: "m", + Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic}, + RenderedPromptHash: "h", + }) + if !strings.Contains(stderr.String(), "backend=local") { + t.Fatalf("expected backend in summary, got %q", stderr.String()) + } +} + +func TestRunErrorMessageDoesNotExposeCapacityDetails(t *testing.T) { + got := runErrorMessage(&promptkit.CapacityError{BackendID: "private-backend"}) + if got != "run error: model backend capacity is exhausted" { + t.Fatalf("unexpected capacity diagnostic: %q", got) + } } type cliTestLibrary struct { diff --git a/internal/adapter/http/dto.go b/internal/adapter/http/dto.go index e841803..fd652f2 100644 --- a/internal/adapter/http/dto.go +++ b/internal/adapter/http/dto.go @@ -57,6 +57,7 @@ type metadataDTO struct { PromptHash string `json:"prompt_hash"` RenderedPromptHash string `json:"rendered_prompt_hash"` SelectedProfileID string `json:"selected_profile_id"` + SelectedBackendID string `json:"selected_backend_id,omitempty"` SessionID string `json:"session_id,omitempty"` ModelName string `json:"model_name"` Endpoint string `json:"endpoint"` @@ -73,6 +74,7 @@ type metadataDTO struct { type modelParamsDTO struct { Endpoint string `json:"endpoint"` + BackendID string `json:"backend_id,omitempty"` Model string `json:"model"` Temperature float64 `json:"temperature"` MaxTokens int `json:"max_tokens"` diff --git a/internal/adapter/http/handler.go b/internal/adapter/http/handler.go index 957c3f0..68bc1af 100644 --- a/internal/adapter/http/handler.go +++ b/internal/adapter/http/handler.go @@ -125,6 +125,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { PromptHash: res.PromptHash, RenderedPromptHash: res.RenderedPromptHash, SelectedProfileID: res.SelectedProfileID, + SelectedBackendID: res.SelectedBackendID, SessionID: res.SessionID, ModelName: res.ModelName, Endpoint: res.Endpoint, @@ -173,6 +174,7 @@ func executionTargetOverrideFromModelOverrideDTO(dto *modelOverrideRequestDTO) * func modelParamsDTOFromExecutionTarget(target promptkit.ExecutionTarget) modelParamsDTO { return modelParamsDTO{ Endpoint: target.Endpoint, + BackendID: target.BackendID, Model: target.Model, Temperature: target.Temperature, MaxTokens: target.MaxTokens, @@ -220,6 +222,8 @@ func mapRunError(err error) (int, string, string) { return http.StatusBadRequest, "artifact_read_failed", "failed to read input artifact" case errors.Is(err, promptkit.ErrPromptRender): return http.StatusBadRequest, "prompt_render_failed", "failed to render prompt" + case errors.Is(err, promptkit.ErrCapacityExceeded): + return http.StatusServiceUnavailable, "capacity_exceeded", "model backend capacity is exhausted" case errors.Is(err, promptkit.ErrLLMGenerate): return http.StatusBadGateway, "llm_failed", "model generation request failed" case errors.Is(err, promptkit.ErrValidation): diff --git a/internal/adapter/http/handler_test.go b/internal/adapter/http/handler_test.go index 47afe05..e944c81 100644 --- a/internal/adapter/http/handler_test.go +++ b/internal/adapter/http/handler_test.go @@ -11,6 +11,7 @@ import ( "path/filepath" "reflect" "strings" + "sync/atomic" "testing" "time" @@ -69,6 +70,34 @@ func (handlerLLMClient) Generate(ctx context.Context, req promptkit.GenerateRequ return &promptkit.GenerateResponse{Content: "ok"}, nil } +type blockingLLMClient struct { + started chan struct{} + release chan struct{} + current int32 + peak int32 +} + +func (c *blockingLLMClient) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + current := atomic.AddInt32(&c.current, 1) + defer atomic.AddInt32(&c.current, -1) + for { + peak := atomic.LoadInt32(&c.peak) + if current <= peak || atomic.CompareAndSwapInt32(&c.peak, peak, current) { + break + } + } + select { + case c.started <- struct{}{}: + default: + } + select { + case <-c.release: + return &promptkit.GenerateResponse{Content: "ok"}, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) { start := time.Now().UTC() end := start.Add(2 * time.Second) @@ -90,9 +119,11 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) { PromptHash: "phash", RenderedPromptHash: "rhash", SelectedProfileID: "exec-default", + SelectedBackendID: "local", ModelName: "m1", Endpoint: "http://llm/v1", EffectiveModelParams: promptkit.ExecutionTarget{ + BackendID: "local", Endpoint: "http://llm/v1", Model: "m1", Temperature: 0.2, @@ -151,6 +182,9 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) { if metadata["selected_profile_id"] != "exec-default" { t.Fatalf("unexpected metadata.selected_profile_id: %#v", metadata["selected_profile_id"]) } + if metadata["selected_backend_id"] != "local" { + t.Fatalf("unexpected metadata.selected_backend_id: %#v", metadata["selected_backend_id"]) + } if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" { t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"]) } @@ -162,6 +196,9 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) { t.Fatalf("unexpected cache usage metadata: %#v", usage) } modelParams := metadata["model_params"].(map[string]any) + if modelParams["backend_id"] != "local" { + t.Fatalf("expected model_params.backend_id=local, got %#v", modelParams["backend_id"]) + } if modelParams["api_key_env"] != envName { t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"]) } @@ -664,6 +701,79 @@ output: } } +func TestHandlerSharesBackendCapacityAcrossConcurrentRequests(t *testing.T) { + promptDir := t.TempDir() + profileDir := t.TempDir() + if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`id: p +version: "1" +default_profile: limited +messages: + - role: user + content: "hi" +output: + format: text + validation_mode: none +`), 0o644); err != nil { + t.Fatalf("write prompt fixture: %v", err) + } + if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`id: limited +backend: limited +model: test +`), 0o644); err != nil { + t.Fatalf("write profile fixture: %v", err) + } + queueCapacity := 0 + client := &blockingLLMClient{started: make(chan struct{}, 1), release: make(chan struct{})} + engine, err := promptkit.NewEngine( + promptkit.Config{PromptDir: promptDir, ProfileDir: profileDir}, + promptkit.WithBackend(promptkit.Backend{ + ID: "limited", + Endpoint: "http://127.0.0.1:1/v1", + ConcurrencyLimit: 1, + QueueCapacity: &queueCapacity, + }), + promptkit.WithLLMClient(client), + ) + if err != nil { + t.Fatalf("new engine: %v", err) + } + h := NewHandler(engine) + first := httptest.NewRecorder() + firstDone := make(chan struct{}) + go func() { + defer close(firstDone) + h.ServeHTTP(first, httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p"}`))) + }() + select { + case <-client.started: + case <-time.After(time.Second): + t.Fatal("first request did not reach generation") + } + + second := httptest.NewRecorder() + h.ServeHTTP(second, httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p"}`))) + assertHTTPErrorCode(t, second, http.StatusServiceUnavailable, "capacity_exceeded") + if second.Header().Get("Retry-After") != "" { + t.Fatalf("expected no Retry-After header, got %q", second.Header().Get("Retry-After")) + } + if strings.Contains(second.Body.String(), "limited") { + t.Fatalf("capacity response leaked backend details: %s", second.Body.String()) + } + + close(client.release) + select { + case <-firstDone: + case <-time.After(time.Second): + t.Fatal("first request did not complete") + } + if first.Code != http.StatusOK { + t.Fatalf("expected first request to succeed, got %d body=%s", first.Code, first.Body.String()) + } + if atomic.LoadInt32(&client.peak) != 1 { + t.Fatalf("expected peak generation concurrency of one, got %d", atomic.LoadInt32(&client.peak)) + } +} + func TestHandlerRejectsOverlongSessionIDAndNonStringReasoningEffort(t *testing.T) { engine := newHandlerEngine(t) for _, body := range []string{ @@ -1014,6 +1124,7 @@ func TestHandlerPublicErrorMapping(t *testing.T) { {name: "file too large", err: ErrFileTooLarge, status: http.StatusRequestEntityTooLarge, code: "artifact_too_large", message: "file input artifact is too large"}, {name: "artifact", err: wrap(promptkit.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"}, {name: "prompt render", err: wrap(promptkit.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"}, + {name: "capacity", err: &promptkit.CapacityError{BackendID: "private-backend"}, status: http.StatusServiceUnavailable, code: "capacity_exceeded", message: "model backend capacity is exhausted", avoidCause: "private-backend"}, {name: "llm", err: wrap(promptkit.ErrLLMGenerate, fmt.Errorf("llm failed")), status: http.StatusBadGateway, code: "llm_failed", message: "model generation request failed", avoidCause: "llm failed"}, {name: "validation runtime", err: wrap(promptkit.ErrValidation, fmt.Errorf("validator broke")), status: http.StatusInternalServerError, code: "validation_runtime_failed", message: "validation runtime failed", avoidCause: "validator broke"}, } diff --git a/internal/format/prepared_run.go b/internal/format/prepared_run.go index 5aec67b..e4e8c56 100644 --- a/internal/format/prepared_run.go +++ b/internal/format/prepared_run.go @@ -93,6 +93,9 @@ func (textPreparedRunFormatter) Format(prepared *promptkit.PreparedRun) ([]byte, fmt.Fprintf(&b, "prompt: %s\n", prepared.PromptID) fmt.Fprintf(&b, "prompt_version: %s\n", prepared.PromptVersion) fmt.Fprintf(&b, "selected_profile_id: %s\n", prepared.SelectedProfileID) + if prepared.SelectedBackendID != "" { + fmt.Fprintf(&b, "selected_backend_id: %s\n", prepared.SelectedBackendID) + } if prepared.PromptHash != "" { fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash) } diff --git a/internal/format/prepared_run_test.go b/internal/format/prepared_run_test.go index 25390f1..2668253 100644 --- a/internal/format/prepared_run_test.go +++ b/internal/format/prepared_run_test.go @@ -22,6 +22,7 @@ func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) { "prompt: prompt.id", "prompt_version: v1", "selected_profile_id: local-fast", + "selected_backend_id: local", "endpoint: http://llm/v1", "model: gpt-test", "temperature: 0.4", @@ -346,7 +347,9 @@ func samplePreparedRun() *promptkit.PreparedRun { PromptVersion: "v1", PromptHash: "prompt-hash", SelectedProfileID: "local-fast", + SelectedBackendID: "local", EffectiveModelParams: promptkit.ExecutionTarget{ + BackendID: "local", Endpoint: "http://llm/v1", Model: "gpt-test", Temperature: 0.4,