From f5e12c00f5570da5aa1311ce86988ed94ad47e23 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 30 Jul 2026 18:20:35 +0000 Subject: [PATCH] Expose prepared execution handles --- engine.go | 86 +++- prepared_execution.go | 55 +++ prepared_execution_contract_test.go | 702 ++++++++++++++++++++++++++++ types.go | 34 +- 4 files changed, 857 insertions(+), 20 deletions(-) create mode 100644 prepared_execution.go create mode 100644 prepared_execution_contract_test.go diff --git a/engine.go b/engine.go index 4689a59..647e8e6 100644 --- a/engine.go +++ b/engine.go @@ -63,9 +63,10 @@ var ( // ErrPromptRender identifies a failure to render prompt messages or the // session ID from the resolved inputs and variables. ErrPromptRender = errors.New("failed to render prompt") - // ErrCapacityExceeded identifies a Run rejected because the selected backend - // already admitted ConcurrencyLimit + QueueCapacity calls. It is not an - // invalid request, an LLM or provider rate-limit response, or ErrLLMGenerate. + // ErrCapacityExceeded identifies a Run or RunPrepared rejected because the + // selected backend already admitted ConcurrencyLimit + QueueCapacity calls. + // It is not an invalid request, an LLM or provider rate-limit response, or + // ErrLLMGenerate. ErrCapacityExceeded = errors.New("backend capacity exceeded") // ErrLLMGenerate identifies a model-client failure or a nil successful // response. Errors returned by an injected LLMClient remain available @@ -79,10 +80,12 @@ var ( // Engine prepares and runs Promptkit prompt requests. // -// An Engine is safe for concurrent calls to [Engine.Prepare] and [Engine.Run]. -// Each Engine owns independent backend-capacity pools that coordinate Run -// admission and model generation. Injected collaborators may still be invoked -// concurrently across different backend pools or for unlimited backends. +// An Engine is safe for concurrent calls to [Engine.Prepare], +// [Engine.PrepareExecution], [Engine.Run], and [Engine.RunPrepared]. Each +// Engine owns independent backend-capacity pools that coordinate Run and +// RunPrepared admission and model generation. Injected collaborators may still +// be invoked concurrently across different backend pools or for unlimited +// backends. type Engine struct { runner *usecase.Runner } @@ -457,6 +460,38 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err return fromDomainPreparedRun(prepared), nil } +// PrepareExecution completely prepares a prompt request without calling the +// configured LLMClient or reserving backend admission capacity. +// +// The returned opaque handle is bound to this Engine and permits one +// [Engine.RunPrepared] invocation. Preparation freezes the selected sources, +// rendered messages, effective settings, inputs, provider structured-output +// metadata, and validation resources needed by that invocation. The handle +// retains a direct RunRequest.APIKey only in private execution state; +// [PreparedExecution.Details] is credential-redacted. +// +// The context governs preparation only. Cancellation after this method +// returns does not invalidate the handle or propagate to RunPrepared. +// PrepareExecution returns the same error categories as [Engine.Prepare] and +// returns no handle on error. A nil Engine returns an error matching +// ErrInvalidConfig. +func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) { + if e == nil || e.runner == nil { + return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig) + } + + domainReq, err := toDomainRunRequest(req) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err) + } + + prepared, err := e.runner.PrepareExecution(ctx, domainReq) + if err != nil { + return nil, mapPublicError(err) + } + return &PreparedExecution{internal: prepared}, nil +} + // Run prepares a request, invokes the configured LLMClient, and validates the // generated output. // @@ -491,3 +526,40 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) { } return fromDomainRunResult(result), nil } + +// RunPrepared atomically claims and executes a handle created by +// [Engine.PrepareExecution]. +// +// A valid owning-Engine invocation consumes the handle's one attempt before +// credential revalidation, backend admission, generation, or validation. +// Cancellation, capacity rejection, generation failure, operational +// validation failure, and success all leave the handle unusable. A nil, +// zero-value, foreign-Engine, discarded, claimed, or used handle returns an +// error matching ErrInvalidRequest; a nil Engine returns ErrInvalidConfig and +// does not claim the handle. +// +// The supplied context governs this execution attempt independently of the +// preparation context. It covers credential revalidation, admission, +// generation, validation, and any internal repair. Result timing begins after +// the claim and excludes preparation and consumer-held delay. +// +// RunPrepared can return ErrInvalidRequest, ErrAPIKeyEnvMissing, +// ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while +// preserving documented collaborator and context identities. A completed +// content-validation rejection is returned in RunResult, not as an +// operational error. An operational error returns no partial RunResult. +func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) { + if e == nil || e.runner == nil { + return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig) + } + + var internal *usecase.PreparedExecution + if prepared != nil { + internal = prepared.internal + } + result, err := e.runner.RunPrepared(ctx, internal) + if err != nil { + return nil, mapPublicError(err) + } + return fromDomainRunResult(result), nil +} diff --git a/prepared_execution.go b/prepared_execution.go new file mode 100644 index 0000000..06b584d --- /dev/null +++ b/prepared_execution.go @@ -0,0 +1,55 @@ +package promptkit + +import "gitea.maximumdirect.net/eric/promptkit/internal/usecase" + +const preparedExecutionString = "promptkit.PreparedExecution{opaque}" + +// PreparedExecution is an opaque, in-process handle for one completely +// prepared execution. A handle is bound to the [Engine] that created it and +// permits one [Engine.RunPrepared] invocation. +// +// PreparedExecution contains no supported serializable state and cannot be +// used as a restartable job. Copying the value preserves the same shared +// lifecycle; it does not create another execution attempt. +type PreparedExecution struct { + internal *usecase.PreparedExecution +} + +// Details returns a fresh caller-owned, credential-redacted copy of the +// prepared request details. Mutating the result cannot affect execution or a +// later Details call. Details remains available after execution or discard. +// +// A nil receiver or zero-value PreparedExecution returns a zero [PreparedRun]. +func (p *PreparedExecution) Details() PreparedRun { + if p == nil || p.internal == nil { + return PreparedRun{} + } + details := fromDomainPreparedRun(p.internal.Details()) + if details == nil { + return PreparedRun{} + } + return *details +} + +// Discard invalidates an unclaimed handle and drops Promptkit's references to +// its execution-only state. Discard is nil-safe and idempotent. It does not +// cancel an execution that has already claimed the handle; use the +// [Engine.RunPrepared] context for cancellation. +func (p *PreparedExecution) Discard() { + if p == nil || p.internal == nil { + return + } + p.internal.Discard() +} + +// String returns a constant representation that exposes no retained request, +// rendered content, or credential data. +func (p *PreparedExecution) String() string { + return preparedExecutionString +} + +// GoString returns a constant Go-syntax representation that exposes no +// retained request, rendered content, or credential data. +func (p *PreparedExecution) GoString() string { + return preparedExecutionString +} diff --git a/prepared_execution_contract_test.go b/prepared_execution_contract_test.go new file mode 100644 index 0000000..7e84d3f --- /dev/null +++ b/prepared_execution_contract_test.go @@ -0,0 +1,702 @@ +package promptkit_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "reflect" + "strings" + "sync" + "testing" + "testing/fstest" + "time" + + "gitea.maximumdirect.net/eric/promptkit" +) + +func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.T) { + promptSource := preparedPromptSource("original") + profileSource := preparedProfileSource("original-model") + schemaSource := preparedSchemaSource() + reader := &mutablePreparedArtifactReader{ + body: "original artifact", + hash: "original-input-hash", + } + client := &preparedRecordingClient{ + response: &promptkit.GenerateResponse{Content: `{"value":3}`}, + } + engine, err := promptkit.NewEngine( + promptkit.Config{}, + promptkit.WithPromptFS(promptSource, "."), + promptkit.WithProfileFS(profileSource, "."), + promptkit.WithSchemaFS(schemaSource, "."), + promptkit.WithArtifactReader(reader), + promptkit.WithLLMClient(client), + ) + if err != nil { + t.Fatalf("construct engine: %v", err) + } + + temperature := 0.25 + extraParams := map[string]any{ + "nested": map[string]any{"source": "original"}, + } + request := promptkit.RunRequest{ + PromptID: "prepared", + Inputs: map[string]promptkit.ArtifactRef{ + "input": promptkit.Inline("original request input"), + }, + Vars: map[string]string{"label": "original variable"}, + Execution: &promptkit.ExecutionTargetOverride{ + Temperature: &temperature, + ExtraParams: extraParams, + }, + } + preparationContext, cancelPreparation := context.WithCancel(context.Background()) + prepared, err := engine.PrepareExecution(preparationContext, request) + if err != nil { + t.Fatalf("prepare execution: %v", err) + } + cancelPreparation() + + request.PromptID = "changed" + request.Inputs["input"] = promptkit.Inline("changed request input") + request.Vars["label"] = "changed variable" + temperature = 1.5 + extraParams["nested"].(map[string]any)["source"] = "changed" + promptSource["prompt.yaml"] = &fstest.MapFile{Data: []byte(`id: changed`)} + profileSource["profile.yaml"] = &fstest.MapFile{Data: []byte(`id: changed`)} + schemaSource["schema.json"] = &fstest.MapFile{Data: []byte(`{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "changed root", + "type": "string" +}`)} + schemaSource["value.json"] = &fstest.MapFile{Data: []byte(`{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string" +}`)} + reader.set("changed artifact", "changed-input-hash") + + first := prepared.Details() + first.Messages[0].Content = "changed details" + first.InputHashes["input"] = "changed-details-hash" + first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["source"] = "changed details" + first.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] = "changed details" + + second := prepared.Details() + if second.Messages[0].Content != "Input=original artifact Label=original variable" { + t.Fatalf("details message changed: %q", second.Messages[0].Content) + } + if second.InputHashes["input"] != "original-input-hash" { + t.Fatalf("details input hash changed: %q", second.InputHashes["input"]) + } + if second.EffectiveModelParams.Model != "original-model" || + second.EffectiveModelParams.Temperature != 0.25 || + second.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["source"] != "original" { + t.Fatalf("details target changed: %+v", second.EffectiveModelParams) + } + schema := second.StructuredOutput.JSONSchema.Schema.(map[string]any) + if schema["title"] != "original root" { + t.Fatalf("details schema changed: %#v", schema) + } + + result, err := engine.RunPrepared(context.Background(), prepared) + if err != nil { + t.Fatalf("run prepared after preparation-context cancellation: %v", err) + } + if result.Validation.Status != promptkit.ValidationPassed || !result.Validation.IsValid { + t.Fatalf("frozen schema did not validate original output: %+v", result.Validation) + } + if reader.callCount() != 1 { + t.Fatalf("execution reopened artifact source: calls=%d", reader.callCount()) + } + + requests := client.snapshot() + if len(requests) != 1 { + t.Fatalf("generation calls=%d, want 1", len(requests)) + } + generated := requests[0] + if generated.Prompt.Messages[0].Content != second.Messages[0].Content || + generated.Target.Model != second.EffectiveModelParams.Model || + !reflect.DeepEqual(generated.Target.ExtraParams, second.EffectiveModelParams.ExtraParams) || + !reflect.DeepEqual(generated.StructuredOutput, second.StructuredOutput) { + t.Fatalf("generation did not use frozen details:\nrequest=%+v\ndetails=%+v", generated, second) + } + if result.PromptID != second.PromptID || + result.PromptVersion != second.PromptVersion || + result.PromptHash != second.PromptHash || + result.SessionID != second.SessionID || + result.RenderedPromptHash != second.RenderedPromptHash || + result.SelectedProfileID != second.SelectedProfileID || + result.SelectedBackendID != second.SelectedBackendID || + !reflect.DeepEqual(result.EffectiveModelParams, second.EffectiveModelParams) || + !reflect.DeepEqual(result.InputHashes, second.InputHashes) { + t.Fatalf("result provenance does not match details:\nresult=%+v\ndetails=%+v", result, second) + } +} + +func TestPreparedExecutionLifecycleAndEngineBinding(t *testing.T) { + ownerClient := &preparedRecordingClient{ + response: &promptkit.GenerateResponse{Content: "ok"}, + } + owner := newPreparedContractEngine(t, ownerClient, "owner content") + foreign := newPreparedContractEngine(t, &preparedRecordingClient{ + response: &promptkit.GenerateResponse{Content: "unexpected"}, + }, "foreign content") + + prepared, err := owner.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"}) + if err != nil { + t.Fatalf("prepare execution: %v", err) + } + copied := *prepared + + var nilEngine *promptkit.Engine + if result, err := nilEngine.RunPrepared(context.Background(), prepared); result != nil || + !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("nil engine result=(%+v, %v), want ErrInvalidConfig", result, err) + } + if result, err := foreign.RunPrepared(context.Background(), prepared); result != nil || + !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("foreign engine result=(%+v, %v), want ErrInvalidRequest", result, err) + } + if result, err := owner.RunPrepared(context.Background(), nil); result != nil || + !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("nil handle result=(%+v, %v), want ErrInvalidRequest", result, err) + } + if result, err := owner.RunPrepared(context.Background(), &promptkit.PreparedExecution{}); result != nil || + !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("zero handle result=(%+v, %v), want ErrInvalidRequest", result, err) + } + + result, err := owner.RunPrepared(context.Background(), &copied) + if err != nil || result == nil { + t.Fatalf("owner run prepared=(%+v, %v), want success", result, err) + } + for name, handle := range map[string]*promptkit.PreparedExecution{ + "original": prepared, + "copy": &copied, + } { + if result, err := owner.RunPrepared(context.Background(), handle); result != nil || + !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("%s reused handle result=(%+v, %v), want ErrInvalidRequest", name, result, err) + } + if handle.Details().PromptID != "prepared" { + t.Fatalf("%s details unavailable after execution", name) + } + } + if len(ownerClient.snapshot()) != 1 { + t.Fatalf("owner generation calls=%d, want 1", len(ownerClient.snapshot())) + } + + collaboratorFailure := errors.New("prepared collaborator failure") + failingClient := &preparedRecordingClient{err: collaboratorFailure} + failingEngine := newPreparedContractEngine(t, failingClient, "failure content") + failing, err := failingEngine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"}) + if err != nil { + t.Fatalf("prepare failing execution: %v", err) + } + if result, err := failingEngine.RunPrepared(context.Background(), failing); result != nil || + !errors.Is(err, promptkit.ErrLLMGenerate) || + !errors.Is(err, collaboratorFailure) { + t.Fatalf("generation failure result=(%+v, %v), want public and collaborator identities", result, err) + } + if result, err := failingEngine.RunPrepared(context.Background(), failing); result != nil || + !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("failed execution was reusable: result=(%+v, %v)", result, err) + } + + cancellationRelease := make(chan struct{}) + cancellationStarted := make(chan struct{}, 1) + cancelingEngine := newPreparedContractEngine(t, &preparedRecordingClient{ + response: &promptkit.GenerateResponse{Content: "unexpected"}, + started: cancellationStarted, + release: cancellationRelease, + }, "cancellation content") + canceling, err := cancelingEngine.PrepareExecution( + context.Background(), + promptkit.RunRequest{PromptID: "prepared"}, + ) + if err != nil { + t.Fatalf("prepare canceled execution: %v", err) + } + executionContext, cancelExecution := context.WithCancel(context.Background()) + type canceledOutcome struct { + result *promptkit.RunResult + err error + } + canceledResult := make(chan canceledOutcome, 1) + go func() { + result, runErr := cancelingEngine.RunPrepared(executionContext, canceling) + canceledResult <- canceledOutcome{result: result, err: runErr} + }() + select { + case <-cancellationStarted: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for cancelable generation") + } + cancelExecution() + select { + case outcome := <-canceledResult: + if outcome.result != nil || + !errors.Is(outcome.err, promptkit.ErrLLMGenerate) || + !errors.Is(outcome.err, context.Canceled) { + t.Fatalf( + "canceled execution=(%+v, %v), want generation and context identities", + outcome.result, + outcome.err, + ) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for canceled execution") + } + if result, err := cancelingEngine.RunPrepared(context.Background(), canceling); result != nil || + !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("canceled execution was reusable: result=(%+v, %v)", result, err) + } +} + +func TestPreparedExecutionConcurrentClaimAllowsOneGeneration(t *testing.T) { + release := make(chan struct{}) + client := &preparedRecordingClient{ + response: &promptkit.GenerateResponse{Content: "ok"}, + started: make(chan struct{}, 1), + release: release, + } + engine := newPreparedContractEngine(t, client, "concurrent content") + prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"}) + if err != nil { + t.Fatalf("prepare execution: %v", err) + } + + type outcome struct { + result *promptkit.RunResult + err error + } + outcomes := make(chan outcome, 2) + for i := 0; i < 2; i++ { + go func() { + result, runErr := engine.RunPrepared(context.Background(), prepared) + outcomes <- outcome{result: result, err: runErr} + }() + } + + select { + case <-client.started: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for generation") + } + select { + case loser := <-outcomes: + if loser.result != nil || !errors.Is(loser.err, promptkit.ErrInvalidRequest) { + t.Fatalf("concurrent loser=(%+v, %v), want ErrInvalidRequest", loser.result, loser.err) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for rejected concurrent claim") + } + + close(release) + select { + case winner := <-outcomes: + if winner.err != nil || winner.result == nil { + t.Fatalf("concurrent winner=(%+v, %v), want success", winner.result, winner.err) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for successful concurrent claim") + } + if len(client.snapshot()) != 1 { + t.Fatalf("generation calls=%d, want 1", len(client.snapshot())) + } +} + +func TestPreparedExecutionDiscardAndFormattingDoNotExposePrivateState(t *testing.T) { + const ( + directCredential = "pk-test-direct-credential-41f7" + renderedContent = "rendered-content-sentinel-98d2" + ) + client := &preparedRecordingClient{ + response: &promptkit.GenerateResponse{Content: "generated output"}, + } + engine, err := promptkit.NewEngine( + promptkit.Config{}, + promptkit.WithPromptFS(contractPromptFS("prepared", "profile", renderedContent), "."), + promptkit.WithProfiles(promptkit.Profile{ + ID: "profile", + Endpoint: "http://example.test/v1", + Model: "model", + APIKeyRequired: true, + }), + promptkit.WithLLMClient(client), + ) + if err != nil { + t.Fatalf("construct engine: %v", err) + } + prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ + PromptID: "prepared", + APIKey: directCredential, + }) + if err != nil { + t.Fatalf("prepare execution: %v", err) + } + + formattedValues := []string{ + fmt.Sprint(prepared), + fmt.Sprintf("%+v", prepared), + fmt.Sprintf("%#v", prepared), + } + for _, formatted := range formattedValues { + if formatted != "promptkit.PreparedExecution{opaque}" { + t.Fatalf("unexpected opaque formatting: %q", formatted) + } + assertPreparedPrivateValuesAbsent(t, formatted, directCredential, renderedContent) + } + payload, err := json.Marshal(prepared) + if err != nil { + t.Fatalf("marshal opaque handle: %v", err) + } + if string(payload) != "{}" { + t.Fatalf("opaque handle JSON=%s, want {}", payload) + } + assertPreparedPrivateValuesAbsent(t, string(payload), directCredential, renderedContent) + + detailsBefore := prepared.Details() + detailsJSON, err := json.Marshal(detailsBefore) + if err != nil { + t.Fatalf("marshal prepared details: %v", err) + } + assertPreparedPrivateValuesAbsent(t, string(detailsJSON), directCredential) + + prepared.Discard() + prepared.Discard() + result, lifecycleErr := engine.RunPrepared(context.Background(), prepared) + if result != nil || !errors.Is(lifecycleErr, promptkit.ErrInvalidRequest) { + t.Fatalf("discarded execution result=(%+v, %v), want ErrInvalidRequest", result, lifecycleErr) + } + assertPreparedPrivateValuesAbsent(t, lifecycleErr.Error(), directCredential, renderedContent) + if !reflect.DeepEqual(prepared.Details(), detailsBefore) { + t.Fatal("details changed after discard") + } + + executed, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ + PromptID: "prepared", + APIKey: directCredential, + }) + if err != nil { + t.Fatalf("prepare execution for request inspection: %v", err) + } + executionResult, err := engine.RunPrepared(context.Background(), executed) + if err != nil { + t.Fatalf("run execution for request inspection: %v", err) + } + requests := client.snapshot() + if len(requests) != 1 || requests[0].APIKey != directCredential { + t.Fatalf("direct credential did not reach only the client credential field: %#v", requests) + } + requestJSON, err := json.Marshal(requests[0]) + if err != nil { + t.Fatalf("marshal captured generate request: %v", err) + } + for _, value := range []string{ + fmt.Sprint(requests[0]), + fmt.Sprintf("%+v", requests[0]), + fmt.Sprintf("%#v", requests[0]), + string(requestJSON), + fmt.Sprint(executionResult), + } { + assertPreparedPrivateValuesAbsent(t, value, directCredential) + } + resultJSON, err := json.Marshal(executionResult) + if err != nil { + t.Fatalf("marshal execution result: %v", err) + } + assertPreparedPrivateValuesAbsent(t, string(resultJSON), directCredential) + + var nilHandle *promptkit.PreparedExecution + nilHandle.Discard() + if !reflect.DeepEqual(nilHandle.Details(), promptkit.PreparedRun{}) { + t.Fatalf("nil handle details=%+v, want zero value", nilHandle.Details()) + } + zeroHandle := &promptkit.PreparedExecution{} + zeroHandle.Discard() + if !reflect.DeepEqual(zeroHandle.Details(), promptkit.PreparedRun{}) { + t.Fatalf("zero handle details=%+v, want zero value", zeroHandle.Details()) + } +} + +func TestPreparedExecutionCredentialCapacityAndTimingBoundaries(t *testing.T) { + t.Run("credential is rechecked before generation", func(t *testing.T) { + const ( + environmentName = "PROMPTKIT_PREPARED_CONTRACT_KEY" + environmentKey = "environment-credential-sentinel" + ) + t.Setenv(environmentName, environmentKey) + client := &preparedRecordingClient{ + response: &promptkit.GenerateResponse{Content: "unexpected"}, + } + engine, err := promptkit.NewEngine( + promptkit.Config{}, + promptkit.WithPromptFS(contractPromptFS("prepared", "profile", "content"), "."), + promptkit.WithProfileFS(preparedCredentialProfileSource(environmentName), "."), + promptkit.WithLLMClient(client), + ) + if err != nil { + t.Fatalf("construct credential engine: %v", err) + } + prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"}) + if err != nil { + t.Fatalf("prepare credential execution: %v", err) + } + if err := os.Unsetenv(environmentName); err != nil { + t.Fatalf("unset credential environment: %v", err) + } + + result, err := engine.RunPrepared(context.Background(), prepared) + if result != nil || + !errors.Is(err, promptkit.ErrInvalidRequest) || + !errors.Is(err, promptkit.ErrAPIKeyEnvMissing) { + t.Fatalf("credential execution=(%+v, %v), want credential identities", result, err) + } + if len(client.snapshot()) != 0 { + t.Fatalf("credential failure reached generation: %d calls", len(client.snapshot())) + } + assertPreparedPrivateValuesAbsent(t, err.Error(), environmentKey) + if result, err := engine.RunPrepared(context.Background(), prepared); result != nil || + !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("credential failure did not consume handle: result=(%+v, %v)", result, err) + } + }) + + t.Run("preparation does not admit and execution timing starts after retention", func(t *testing.T) { + release := make(chan struct{}) + client := newCapacityGateClient(release, 4) + engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil) + + activeRun := make(chan capacityRunResult, 1) + go runCapacityRequest( + engine, + context.Background(), + promptkit.RunRequest{PromptID: "prompt"}, + activeRun, + ) + awaitCapacityRequest(t, client.started) + + prepared, err := engine.PrepareExecution( + context.Background(), + promptkit.RunRequest{PromptID: "prompt"}, + ) + if err != nil { + t.Fatalf("prepare while capacity is full: %v", err) + } + if _, _, calls := client.snapshot(); calls != 1 { + t.Fatalf("preparation invoked generation: calls=%d", calls) + } + if result, err := engine.RunPrepared(context.Background(), prepared); result != nil || + !errors.Is(err, promptkit.ErrCapacityExceeded) { + t.Fatalf("capacity execution=(%+v, %v), want ErrCapacityExceeded", result, err) + } + if result, err := engine.RunPrepared(context.Background(), prepared); result != nil || + !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("capacity rejection did not consume handle: result=(%+v, %v)", result, err) + } + if prepared.Details().PromptID != "prompt" { + t.Fatal("details unavailable after capacity rejection") + } + + close(release) + activeOutcome := awaitCapacityRun(t, activeRun) + if activeOutcome.err != nil || activeOutcome.result == nil { + t.Fatalf("active run outcome=(%+v, %v), want success", activeOutcome.result, activeOutcome.err) + } + + timed, err := engine.PrepareExecution( + context.Background(), + promptkit.RunRequest{PromptID: "prompt"}, + ) + if err != nil { + t.Fatalf("prepare timed execution: %v", err) + } + details := timed.Details() + time.Sleep(25 * time.Millisecond) + executionFloor := time.Now().UTC() + result, err := engine.RunPrepared(context.Background(), timed) + if err != nil { + t.Fatalf("run timed execution: %v", err) + } + if result.StartTime.Before(executionFloor) || + !result.StartTime.After(details.EndTime) || + result.EndTime.Before(result.StartTime) || + result.Duration != result.EndTime.Sub(result.StartTime) { + t.Fatalf( + "execution timing includes preparation or retention: details_end=%s floor=%s result=%+v", + details.EndTime, + executionFloor, + result, + ) + } + if _, _, calls := client.snapshot(); calls != 2 { + t.Fatalf("generation calls=%d, want active and timed executions only", calls) + } + }) +} + +type mutablePreparedArtifactReader struct { + mu sync.Mutex + body string + hash string + calls int +} + +func (r *mutablePreparedArtifactReader) Read( + _ context.Context, + _ promptkit.ArtifactRef, +) (*promptkit.Artifact, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.calls++ + return &promptkit.Artifact{ + Body: []byte(r.body), + Hash: r.hash, + }, nil +} + +func (r *mutablePreparedArtifactReader) set(body, hash string) { + r.mu.Lock() + defer r.mu.Unlock() + r.body = body + r.hash = hash +} + +func (r *mutablePreparedArtifactReader) callCount() int { + r.mu.Lock() + defer r.mu.Unlock() + return r.calls +} + +type preparedRecordingClient struct { + mu sync.Mutex + response *promptkit.GenerateResponse + err error + requests []promptkit.GenerateRequest + started chan struct{} + release <-chan struct{} +} + +func (c *preparedRecordingClient) Generate( + ctx context.Context, + request promptkit.GenerateRequest, +) (*promptkit.GenerateResponse, error) { + c.mu.Lock() + c.requests = append(c.requests, request) + c.mu.Unlock() + + if c.started != nil { + c.started <- struct{}{} + } + if c.release != nil { + select { + case <-c.release: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + if c.err != nil { + return nil, c.err + } + return c.response, nil +} + +func (c *preparedRecordingClient) snapshot() []promptkit.GenerateRequest { + c.mu.Lock() + defer c.mu.Unlock() + return append([]promptkit.GenerateRequest(nil), c.requests...) +} + +func newPreparedContractEngine( + t *testing.T, + client promptkit.LLMClient, + message string, +) *promptkit.Engine { + t.Helper() + engine, err := promptkit.NewEngine( + promptkit.Config{}, + promptkit.WithPromptFS(contractPromptFS("prepared", "profile", message), "."), + promptkit.WithProfiles(promptkit.Profile{ + ID: "profile", + Endpoint: "http://example.test/v1", + Model: "model", + }), + promptkit.WithLLMClient(client), + ) + if err != nil { + t.Fatalf("construct prepared execution engine: %v", err) + } + return engine +} + +func preparedPromptSource(label string) fstest.MapFS { + return fstest.MapFS{ + "prompt.yaml": &fstest.MapFile{Data: []byte(`id: prepared +version: "1" +default_profile: profile +inputs: + - name: input + required: true +messages: + - role: user + content: 'Input={{input "input"}} Label={{.label}}' +description: ` + label + ` +output: + format: json + validation_mode: json_schema + schema_path: schema.json +`)}, + } +} + +func preparedProfileSource(model string) fstest.MapFS { + return fstest.MapFS{ + "profile.yaml": &fstest.MapFile{Data: []byte(`id: profile +endpoint: http://example.test/v1 +model: ` + model + ` +`)}, + } +} + +func preparedCredentialProfileSource(environmentName string) fstest.MapFS { + return fstest.MapFS{ + "profile.yaml": &fstest.MapFile{Data: []byte(`id: profile +endpoint: http://example.test/v1 +model: model +api_key_env: ` + environmentName + ` +`)}, + } +} + +func preparedSchemaSource() fstest.MapFS { + return fstest.MapFS{ + "schema.json": &fstest.MapFile{Data: []byte(`{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "original root", + "type": "object", + "required": ["value"], + "properties": { + "value": {"$ref": "value.json"} + } +}`)}, + "value.json": &fstest.MapFile{Data: []byte(`{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer", + "minimum": 2 +}`)}, + } +} + +func assertPreparedPrivateValuesAbsent(t *testing.T, value string, privateValues ...string) { + t.Helper() + for _, privateValue := range privateValues { + if strings.Contains(value, privateValue) { + t.Fatalf("value exposed private data %q: %s", privateValue, value) + } + } +} diff --git a/types.go b/types.go index f26dc8d..41de243 100644 --- a/types.go +++ b/types.go @@ -78,9 +78,10 @@ const ( // RunRequest selects one prompt execution. It has no stable JSON // representation. // -// Prepare and Run copy the request's maps, pointers, and nested -// JSON-compatible values before using them. The caller may mutate the request -// after either method returns. +// Prepare, PrepareExecution, and Run copy the request's maps, pointers, and +// nested JSON-compatible values before using them. The caller may mutate the +// request after any method returns. A successful PrepareExecution retains its +// own private execution snapshot for RunPrepared. type RunRequest struct { // PromptID is the required non-empty prompt identifier. PromptID string @@ -98,12 +99,14 @@ type RunRequest struct { // opaque consumer metadata, not a credential, and may be exposed in // prepared values, results, collaborator requests, provider requests, and // provider observability. Callers should use stable, non-sensitive - // identifiers. An overlong direct value makes Prepare or Run return an - // error matching ErrInvalidRequest. + // identifiers. An overlong direct value makes Prepare, PrepareExecution, or + // Run return an error matching ErrInvalidRequest. SessionID string // APIKey is a request-scoped direct credential. It takes precedence over // APIKeyEnv, is passed to the selected LLMClient, and is never included in - // prepared values, results, hashes, JSON, String, or GoString output. + // prepared values, results, hashes, JSON, String, or GoString output. A + // successful PrepareExecution retains it only in the opaque handle until + // RunPrepared claims the handle or Discard invalidates it. APIKey string `json:"-"` // Inputs maps prompt input names to references. A nil or empty map is valid // only when the selected prompt and its templates require no inputs. @@ -119,9 +122,10 @@ type RunRequest struct { Validation *OutputContract } -// PreparedRun contains prepared prompt execution state. It does not include -// resolved API key values, model output, validation results, or internal target -// presence metadata. PreparedRun has a stable JSON representation. +// PreparedRun contains prepared prompt execution state returned by +// [Engine.Prepare] or [PreparedExecution.Details]. It does not include resolved +// API key values, model output, validation results, or internal target presence +// metadata. PreparedRun has a stable JSON representation. // // All maps, slices, pointers, and schema values are caller-owned copies. JSON // timestamps use RFC 3339 and zero timing values are omitted. Hash formats are @@ -154,7 +158,8 @@ type PreparedRun struct { SessionID string `json:"session_id,omitempty"` // RenderedPromptHash is an opaque equality value for SessionID and Messages. RenderedPromptHash string `json:"rendered_prompt_hash"` - // Messages are the rendered messages that Run passes to the LLM client. + // Messages are the rendered messages that Run or RunPrepared passes to the + // LLM client. Messages []RenderedMessage `json:"messages"` // StartTime is the UTC time at which preparation began. StartTime time.Time `json:"start_time,omitempty"` @@ -214,12 +219,15 @@ type RunResult struct { InputHashes map[string]string `json:"input_hashes,omitempty"` // Usage is the token accounting reported by the LLM client. Usage TokenUsage `json:"usage"` - // StartTime is the UTC time immediately before preparation begins. + // StartTime is the UTC time immediately before ordinary Run preparation or + // after RunPrepared claims its handle. StartTime time.Time `json:"start_time,omitempty"` // EndTime is the UTC time after generation and validation complete. EndTime time.Time `json:"end_time,omitempty"` - // Duration covers preparation, generation, and validation. JSON represents - // it as integer milliseconds in duration_ms and omits a zero value. + // Duration covers preparation, generation, and validation for Run. For + // RunPrepared it covers only the execution attempt after claim and excludes + // preparation and consumer-held delay. JSON represents it as integer + // milliseconds in duration_ms and omits a zero value. Duration time.Duration `json:"-"` }