Expose prepared execution handles
This commit is contained in:
86
engine.go
86
engine.go
@@ -63,9 +63,10 @@ var (
|
||||
// ErrPromptRender identifies a failure to render prompt messages or the
|
||||
// session ID from the resolved inputs and variables.
|
||||
ErrPromptRender = errors.New("failed to render prompt")
|
||||
// ErrCapacityExceeded identifies a Run rejected because the selected backend
|
||||
// already admitted ConcurrencyLimit + QueueCapacity calls. It is not an
|
||||
// invalid request, an LLM or provider rate-limit response, or ErrLLMGenerate.
|
||||
// ErrCapacityExceeded identifies a Run or RunPrepared rejected because the
|
||||
// selected backend already admitted ConcurrencyLimit + QueueCapacity calls.
|
||||
// It is not an invalid request, an LLM or provider rate-limit response, or
|
||||
// ErrLLMGenerate.
|
||||
ErrCapacityExceeded = errors.New("backend capacity exceeded")
|
||||
// ErrLLMGenerate identifies a model-client failure or a nil successful
|
||||
// response. Errors returned by an injected LLMClient remain available
|
||||
@@ -79,10 +80,12 @@ var (
|
||||
|
||||
// Engine prepares and runs Promptkit prompt requests.
|
||||
//
|
||||
// An Engine is safe for concurrent calls to [Engine.Prepare] and [Engine.Run].
|
||||
// Each Engine owns independent backend-capacity pools that coordinate Run
|
||||
// admission and model generation. Injected collaborators may still be invoked
|
||||
// concurrently across different backend pools or for unlimited backends.
|
||||
// An Engine is safe for concurrent calls to [Engine.Prepare],
|
||||
// [Engine.PrepareExecution], [Engine.Run], and [Engine.RunPrepared]. Each
|
||||
// Engine owns independent backend-capacity pools that coordinate Run and
|
||||
// RunPrepared admission and model generation. Injected collaborators may still
|
||||
// be invoked concurrently across different backend pools or for unlimited
|
||||
// backends.
|
||||
type Engine struct {
|
||||
runner *usecase.Runner
|
||||
}
|
||||
@@ -457,6 +460,38 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
||||
return fromDomainPreparedRun(prepared), nil
|
||||
}
|
||||
|
||||
// PrepareExecution completely prepares a prompt request without calling the
|
||||
// configured LLMClient or reserving backend admission capacity.
|
||||
//
|
||||
// The returned opaque handle is bound to this Engine and permits one
|
||||
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources,
|
||||
// rendered messages, effective settings, inputs, provider structured-output
|
||||
// metadata, and validation resources needed by that invocation. The handle
|
||||
// retains a direct RunRequest.APIKey only in private execution state;
|
||||
// [PreparedExecution.Details] is credential-redacted.
|
||||
//
|
||||
// The context governs preparation only. Cancellation after this method
|
||||
// returns does not invalidate the handle or propagate to RunPrepared.
|
||||
// PrepareExecution returns the same error categories as [Engine.Prepare] and
|
||||
// returns no handle on error. A nil Engine returns an error matching
|
||||
// ErrInvalidConfig.
|
||||
func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
domainReq, err := toDomainRunRequest(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
prepared, err := e.runner.PrepareExecution(ctx, domainReq)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return &PreparedExecution{internal: prepared}, nil
|
||||
}
|
||||
|
||||
// Run prepares a request, invokes the configured LLMClient, and validates the
|
||||
// generated output.
|
||||
//
|
||||
@@ -491,3 +526,40 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
}
|
||||
return fromDomainRunResult(result), nil
|
||||
}
|
||||
|
||||
// RunPrepared atomically claims and executes a handle created by
|
||||
// [Engine.PrepareExecution].
|
||||
//
|
||||
// A valid owning-Engine invocation consumes the handle's one attempt before
|
||||
// credential revalidation, backend admission, generation, or validation.
|
||||
// Cancellation, capacity rejection, generation failure, operational
|
||||
// validation failure, and success all leave the handle unusable. A nil,
|
||||
// zero-value, foreign-Engine, discarded, claimed, or used handle returns an
|
||||
// error matching ErrInvalidRequest; a nil Engine returns ErrInvalidConfig and
|
||||
// does not claim the handle.
|
||||
//
|
||||
// The supplied context governs this execution attempt independently of the
|
||||
// preparation context. It covers credential revalidation, admission,
|
||||
// generation, validation, and any internal repair. Result timing begins after
|
||||
// the claim and excludes preparation and consumer-held delay.
|
||||
//
|
||||
// RunPrepared can return ErrInvalidRequest, ErrAPIKeyEnvMissing,
|
||||
// ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while
|
||||
// preserving documented collaborator and context identities. A completed
|
||||
// content-validation rejection is returned in RunResult, not as an
|
||||
// operational error. An operational error returns no partial RunResult.
|
||||
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
|
||||
var internal *usecase.PreparedExecution
|
||||
if prepared != nil {
|
||||
internal = prepared.internal
|
||||
}
|
||||
result, err := e.runner.RunPrepared(ctx, internal)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return fromDomainRunResult(result), nil
|
||||
}
|
||||
|
||||
55
prepared_execution.go
Normal file
55
prepared_execution.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package promptkit
|
||||
|
||||
import "gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||
|
||||
const preparedExecutionString = "promptkit.PreparedExecution{opaque}"
|
||||
|
||||
// PreparedExecution is an opaque, in-process handle for one completely
|
||||
// prepared execution. A handle is bound to the [Engine] that created it and
|
||||
// permits one [Engine.RunPrepared] invocation.
|
||||
//
|
||||
// PreparedExecution contains no supported serializable state and cannot be
|
||||
// used as a restartable job. Copying the value preserves the same shared
|
||||
// lifecycle; it does not create another execution attempt.
|
||||
type PreparedExecution struct {
|
||||
internal *usecase.PreparedExecution
|
||||
}
|
||||
|
||||
// Details returns a fresh caller-owned, credential-redacted copy of the
|
||||
// prepared request details. Mutating the result cannot affect execution or a
|
||||
// later Details call. Details remains available after execution or discard.
|
||||
//
|
||||
// A nil receiver or zero-value PreparedExecution returns a zero [PreparedRun].
|
||||
func (p *PreparedExecution) Details() PreparedRun {
|
||||
if p == nil || p.internal == nil {
|
||||
return PreparedRun{}
|
||||
}
|
||||
details := fromDomainPreparedRun(p.internal.Details())
|
||||
if details == nil {
|
||||
return PreparedRun{}
|
||||
}
|
||||
return *details
|
||||
}
|
||||
|
||||
// Discard invalidates an unclaimed handle and drops Promptkit's references to
|
||||
// its execution-only state. Discard is nil-safe and idempotent. It does not
|
||||
// cancel an execution that has already claimed the handle; use the
|
||||
// [Engine.RunPrepared] context for cancellation.
|
||||
func (p *PreparedExecution) Discard() {
|
||||
if p == nil || p.internal == nil {
|
||||
return
|
||||
}
|
||||
p.internal.Discard()
|
||||
}
|
||||
|
||||
// String returns a constant representation that exposes no retained request,
|
||||
// rendered content, or credential data.
|
||||
func (p *PreparedExecution) String() string {
|
||||
return preparedExecutionString
|
||||
}
|
||||
|
||||
// GoString returns a constant Go-syntax representation that exposes no
|
||||
// retained request, rendered content, or credential data.
|
||||
func (p *PreparedExecution) GoString() string {
|
||||
return preparedExecutionString
|
||||
}
|
||||
702
prepared_execution_contract_test.go
Normal file
702
prepared_execution_contract_test.go
Normal file
@@ -0,0 +1,702 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.T) {
|
||||
promptSource := preparedPromptSource("original")
|
||||
profileSource := preparedProfileSource("original-model")
|
||||
schemaSource := preparedSchemaSource()
|
||||
reader := &mutablePreparedArtifactReader{
|
||||
body: "original artifact",
|
||||
hash: "original-input-hash",
|
||||
}
|
||||
client := &preparedRecordingClient{
|
||||
response: &promptkit.GenerateResponse{Content: `{"value":3}`},
|
||||
}
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFS(promptSource, "."),
|
||||
promptkit.WithProfileFS(profileSource, "."),
|
||||
promptkit.WithSchemaFS(schemaSource, "."),
|
||||
promptkit.WithArtifactReader(reader),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
|
||||
temperature := 0.25
|
||||
extraParams := map[string]any{
|
||||
"nested": map[string]any{"source": "original"},
|
||||
}
|
||||
request := promptkit.RunRequest{
|
||||
PromptID: "prepared",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"input": promptkit.Inline("original request input"),
|
||||
},
|
||||
Vars: map[string]string{"label": "original variable"},
|
||||
Execution: &promptkit.ExecutionTargetOverride{
|
||||
Temperature: &temperature,
|
||||
ExtraParams: extraParams,
|
||||
},
|
||||
}
|
||||
preparationContext, cancelPreparation := context.WithCancel(context.Background())
|
||||
prepared, err := engine.PrepareExecution(preparationContext, request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
cancelPreparation()
|
||||
|
||||
request.PromptID = "changed"
|
||||
request.Inputs["input"] = promptkit.Inline("changed request input")
|
||||
request.Vars["label"] = "changed variable"
|
||||
temperature = 1.5
|
||||
extraParams["nested"].(map[string]any)["source"] = "changed"
|
||||
promptSource["prompt.yaml"] = &fstest.MapFile{Data: []byte(`id: changed`)}
|
||||
profileSource["profile.yaml"] = &fstest.MapFile{Data: []byte(`id: changed`)}
|
||||
schemaSource["schema.json"] = &fstest.MapFile{Data: []byte(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "changed root",
|
||||
"type": "string"
|
||||
}`)}
|
||||
schemaSource["value.json"] = &fstest.MapFile{Data: []byte(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "string"
|
||||
}`)}
|
||||
reader.set("changed artifact", "changed-input-hash")
|
||||
|
||||
first := prepared.Details()
|
||||
first.Messages[0].Content = "changed details"
|
||||
first.InputHashes["input"] = "changed-details-hash"
|
||||
first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["source"] = "changed details"
|
||||
first.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] = "changed details"
|
||||
|
||||
second := prepared.Details()
|
||||
if second.Messages[0].Content != "Input=original artifact Label=original variable" {
|
||||
t.Fatalf("details message changed: %q", second.Messages[0].Content)
|
||||
}
|
||||
if second.InputHashes["input"] != "original-input-hash" {
|
||||
t.Fatalf("details input hash changed: %q", second.InputHashes["input"])
|
||||
}
|
||||
if second.EffectiveModelParams.Model != "original-model" ||
|
||||
second.EffectiveModelParams.Temperature != 0.25 ||
|
||||
second.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["source"] != "original" {
|
||||
t.Fatalf("details target changed: %+v", second.EffectiveModelParams)
|
||||
}
|
||||
schema := second.StructuredOutput.JSONSchema.Schema.(map[string]any)
|
||||
if schema["title"] != "original root" {
|
||||
t.Fatalf("details schema changed: %#v", schema)
|
||||
}
|
||||
|
||||
result, err := engine.RunPrepared(context.Background(), prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("run prepared after preparation-context cancellation: %v", err)
|
||||
}
|
||||
if result.Validation.Status != promptkit.ValidationPassed || !result.Validation.IsValid {
|
||||
t.Fatalf("frozen schema did not validate original output: %+v", result.Validation)
|
||||
}
|
||||
if reader.callCount() != 1 {
|
||||
t.Fatalf("execution reopened artifact source: calls=%d", reader.callCount())
|
||||
}
|
||||
|
||||
requests := client.snapshot()
|
||||
if len(requests) != 1 {
|
||||
t.Fatalf("generation calls=%d, want 1", len(requests))
|
||||
}
|
||||
generated := requests[0]
|
||||
if generated.Prompt.Messages[0].Content != second.Messages[0].Content ||
|
||||
generated.Target.Model != second.EffectiveModelParams.Model ||
|
||||
!reflect.DeepEqual(generated.Target.ExtraParams, second.EffectiveModelParams.ExtraParams) ||
|
||||
!reflect.DeepEqual(generated.StructuredOutput, second.StructuredOutput) {
|
||||
t.Fatalf("generation did not use frozen details:\nrequest=%+v\ndetails=%+v", generated, second)
|
||||
}
|
||||
if result.PromptID != second.PromptID ||
|
||||
result.PromptVersion != second.PromptVersion ||
|
||||
result.PromptHash != second.PromptHash ||
|
||||
result.SessionID != second.SessionID ||
|
||||
result.RenderedPromptHash != second.RenderedPromptHash ||
|
||||
result.SelectedProfileID != second.SelectedProfileID ||
|
||||
result.SelectedBackendID != second.SelectedBackendID ||
|
||||
!reflect.DeepEqual(result.EffectiveModelParams, second.EffectiveModelParams) ||
|
||||
!reflect.DeepEqual(result.InputHashes, second.InputHashes) {
|
||||
t.Fatalf("result provenance does not match details:\nresult=%+v\ndetails=%+v", result, second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedExecutionLifecycleAndEngineBinding(t *testing.T) {
|
||||
ownerClient := &preparedRecordingClient{
|
||||
response: &promptkit.GenerateResponse{Content: "ok"},
|
||||
}
|
||||
owner := newPreparedContractEngine(t, ownerClient, "owner content")
|
||||
foreign := newPreparedContractEngine(t, &preparedRecordingClient{
|
||||
response: &promptkit.GenerateResponse{Content: "unexpected"},
|
||||
}, "foreign content")
|
||||
|
||||
prepared, err := owner.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
copied := *prepared
|
||||
|
||||
var nilEngine *promptkit.Engine
|
||||
if result, err := nilEngine.RunPrepared(context.Background(), prepared); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("nil engine result=(%+v, %v), want ErrInvalidConfig", result, err)
|
||||
}
|
||||
if result, err := foreign.RunPrepared(context.Background(), prepared); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("foreign engine result=(%+v, %v), want ErrInvalidRequest", result, err)
|
||||
}
|
||||
if result, err := owner.RunPrepared(context.Background(), nil); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("nil handle result=(%+v, %v), want ErrInvalidRequest", result, err)
|
||||
}
|
||||
if result, err := owner.RunPrepared(context.Background(), &promptkit.PreparedExecution{}); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("zero handle result=(%+v, %v), want ErrInvalidRequest", result, err)
|
||||
}
|
||||
|
||||
result, err := owner.RunPrepared(context.Background(), &copied)
|
||||
if err != nil || result == nil {
|
||||
t.Fatalf("owner run prepared=(%+v, %v), want success", result, err)
|
||||
}
|
||||
for name, handle := range map[string]*promptkit.PreparedExecution{
|
||||
"original": prepared,
|
||||
"copy": &copied,
|
||||
} {
|
||||
if result, err := owner.RunPrepared(context.Background(), handle); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("%s reused handle result=(%+v, %v), want ErrInvalidRequest", name, result, err)
|
||||
}
|
||||
if handle.Details().PromptID != "prepared" {
|
||||
t.Fatalf("%s details unavailable after execution", name)
|
||||
}
|
||||
}
|
||||
if len(ownerClient.snapshot()) != 1 {
|
||||
t.Fatalf("owner generation calls=%d, want 1", len(ownerClient.snapshot()))
|
||||
}
|
||||
|
||||
collaboratorFailure := errors.New("prepared collaborator failure")
|
||||
failingClient := &preparedRecordingClient{err: collaboratorFailure}
|
||||
failingEngine := newPreparedContractEngine(t, failingClient, "failure content")
|
||||
failing, err := failingEngine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare failing execution: %v", err)
|
||||
}
|
||||
if result, err := failingEngine.RunPrepared(context.Background(), failing); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrLLMGenerate) ||
|
||||
!errors.Is(err, collaboratorFailure) {
|
||||
t.Fatalf("generation failure result=(%+v, %v), want public and collaborator identities", result, err)
|
||||
}
|
||||
if result, err := failingEngine.RunPrepared(context.Background(), failing); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("failed execution was reusable: result=(%+v, %v)", result, err)
|
||||
}
|
||||
|
||||
cancellationRelease := make(chan struct{})
|
||||
cancellationStarted := make(chan struct{}, 1)
|
||||
cancelingEngine := newPreparedContractEngine(t, &preparedRecordingClient{
|
||||
response: &promptkit.GenerateResponse{Content: "unexpected"},
|
||||
started: cancellationStarted,
|
||||
release: cancellationRelease,
|
||||
}, "cancellation content")
|
||||
canceling, err := cancelingEngine.PrepareExecution(
|
||||
context.Background(),
|
||||
promptkit.RunRequest{PromptID: "prepared"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare canceled execution: %v", err)
|
||||
}
|
||||
executionContext, cancelExecution := context.WithCancel(context.Background())
|
||||
type canceledOutcome struct {
|
||||
result *promptkit.RunResult
|
||||
err error
|
||||
}
|
||||
canceledResult := make(chan canceledOutcome, 1)
|
||||
go func() {
|
||||
result, runErr := cancelingEngine.RunPrepared(executionContext, canceling)
|
||||
canceledResult <- canceledOutcome{result: result, err: runErr}
|
||||
}()
|
||||
select {
|
||||
case <-cancellationStarted:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for cancelable generation")
|
||||
}
|
||||
cancelExecution()
|
||||
select {
|
||||
case outcome := <-canceledResult:
|
||||
if outcome.result != nil ||
|
||||
!errors.Is(outcome.err, promptkit.ErrLLMGenerate) ||
|
||||
!errors.Is(outcome.err, context.Canceled) {
|
||||
t.Fatalf(
|
||||
"canceled execution=(%+v, %v), want generation and context identities",
|
||||
outcome.result,
|
||||
outcome.err,
|
||||
)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for canceled execution")
|
||||
}
|
||||
if result, err := cancelingEngine.RunPrepared(context.Background(), canceling); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("canceled execution was reusable: result=(%+v, %v)", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedExecutionConcurrentClaimAllowsOneGeneration(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
client := &preparedRecordingClient{
|
||||
response: &promptkit.GenerateResponse{Content: "ok"},
|
||||
started: make(chan struct{}, 1),
|
||||
release: release,
|
||||
}
|
||||
engine := newPreparedContractEngine(t, client, "concurrent content")
|
||||
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
|
||||
type outcome struct {
|
||||
result *promptkit.RunResult
|
||||
err error
|
||||
}
|
||||
outcomes := make(chan outcome, 2)
|
||||
for i := 0; i < 2; i++ {
|
||||
go func() {
|
||||
result, runErr := engine.RunPrepared(context.Background(), prepared)
|
||||
outcomes <- outcome{result: result, err: runErr}
|
||||
}()
|
||||
}
|
||||
|
||||
select {
|
||||
case <-client.started:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for generation")
|
||||
}
|
||||
select {
|
||||
case loser := <-outcomes:
|
||||
if loser.result != nil || !errors.Is(loser.err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("concurrent loser=(%+v, %v), want ErrInvalidRequest", loser.result, loser.err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for rejected concurrent claim")
|
||||
}
|
||||
|
||||
close(release)
|
||||
select {
|
||||
case winner := <-outcomes:
|
||||
if winner.err != nil || winner.result == nil {
|
||||
t.Fatalf("concurrent winner=(%+v, %v), want success", winner.result, winner.err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for successful concurrent claim")
|
||||
}
|
||||
if len(client.snapshot()) != 1 {
|
||||
t.Fatalf("generation calls=%d, want 1", len(client.snapshot()))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedExecutionDiscardAndFormattingDoNotExposePrivateState(t *testing.T) {
|
||||
const (
|
||||
directCredential = "pk-test-direct-credential-41f7"
|
||||
renderedContent = "rendered-content-sentinel-98d2"
|
||||
)
|
||||
client := &preparedRecordingClient{
|
||||
response: &promptkit.GenerateResponse{Content: "generated output"},
|
||||
}
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", renderedContent), "."),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "profile",
|
||||
Endpoint: "http://example.test/v1",
|
||||
Model: "model",
|
||||
APIKeyRequired: true,
|
||||
}),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "prepared",
|
||||
APIKey: directCredential,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution: %v", err)
|
||||
}
|
||||
|
||||
formattedValues := []string{
|
||||
fmt.Sprint(prepared),
|
||||
fmt.Sprintf("%+v", prepared),
|
||||
fmt.Sprintf("%#v", prepared),
|
||||
}
|
||||
for _, formatted := range formattedValues {
|
||||
if formatted != "promptkit.PreparedExecution{opaque}" {
|
||||
t.Fatalf("unexpected opaque formatting: %q", formatted)
|
||||
}
|
||||
assertPreparedPrivateValuesAbsent(t, formatted, directCredential, renderedContent)
|
||||
}
|
||||
payload, err := json.Marshal(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal opaque handle: %v", err)
|
||||
}
|
||||
if string(payload) != "{}" {
|
||||
t.Fatalf("opaque handle JSON=%s, want {}", payload)
|
||||
}
|
||||
assertPreparedPrivateValuesAbsent(t, string(payload), directCredential, renderedContent)
|
||||
|
||||
detailsBefore := prepared.Details()
|
||||
detailsJSON, err := json.Marshal(detailsBefore)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal prepared details: %v", err)
|
||||
}
|
||||
assertPreparedPrivateValuesAbsent(t, string(detailsJSON), directCredential)
|
||||
|
||||
prepared.Discard()
|
||||
prepared.Discard()
|
||||
result, lifecycleErr := engine.RunPrepared(context.Background(), prepared)
|
||||
if result != nil || !errors.Is(lifecycleErr, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("discarded execution result=(%+v, %v), want ErrInvalidRequest", result, lifecycleErr)
|
||||
}
|
||||
assertPreparedPrivateValuesAbsent(t, lifecycleErr.Error(), directCredential, renderedContent)
|
||||
if !reflect.DeepEqual(prepared.Details(), detailsBefore) {
|
||||
t.Fatal("details changed after discard")
|
||||
}
|
||||
|
||||
executed, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "prepared",
|
||||
APIKey: directCredential,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare execution for request inspection: %v", err)
|
||||
}
|
||||
executionResult, err := engine.RunPrepared(context.Background(), executed)
|
||||
if err != nil {
|
||||
t.Fatalf("run execution for request inspection: %v", err)
|
||||
}
|
||||
requests := client.snapshot()
|
||||
if len(requests) != 1 || requests[0].APIKey != directCredential {
|
||||
t.Fatalf("direct credential did not reach only the client credential field: %#v", requests)
|
||||
}
|
||||
requestJSON, err := json.Marshal(requests[0])
|
||||
if err != nil {
|
||||
t.Fatalf("marshal captured generate request: %v", err)
|
||||
}
|
||||
for _, value := range []string{
|
||||
fmt.Sprint(requests[0]),
|
||||
fmt.Sprintf("%+v", requests[0]),
|
||||
fmt.Sprintf("%#v", requests[0]),
|
||||
string(requestJSON),
|
||||
fmt.Sprint(executionResult),
|
||||
} {
|
||||
assertPreparedPrivateValuesAbsent(t, value, directCredential)
|
||||
}
|
||||
resultJSON, err := json.Marshal(executionResult)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal execution result: %v", err)
|
||||
}
|
||||
assertPreparedPrivateValuesAbsent(t, string(resultJSON), directCredential)
|
||||
|
||||
var nilHandle *promptkit.PreparedExecution
|
||||
nilHandle.Discard()
|
||||
if !reflect.DeepEqual(nilHandle.Details(), promptkit.PreparedRun{}) {
|
||||
t.Fatalf("nil handle details=%+v, want zero value", nilHandle.Details())
|
||||
}
|
||||
zeroHandle := &promptkit.PreparedExecution{}
|
||||
zeroHandle.Discard()
|
||||
if !reflect.DeepEqual(zeroHandle.Details(), promptkit.PreparedRun{}) {
|
||||
t.Fatalf("zero handle details=%+v, want zero value", zeroHandle.Details())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedExecutionCredentialCapacityAndTimingBoundaries(t *testing.T) {
|
||||
t.Run("credential is rechecked before generation", func(t *testing.T) {
|
||||
const (
|
||||
environmentName = "PROMPTKIT_PREPARED_CONTRACT_KEY"
|
||||
environmentKey = "environment-credential-sentinel"
|
||||
)
|
||||
t.Setenv(environmentName, environmentKey)
|
||||
client := &preparedRecordingClient{
|
||||
response: &promptkit.GenerateResponse{Content: "unexpected"},
|
||||
}
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", "content"), "."),
|
||||
promptkit.WithProfileFS(preparedCredentialProfileSource(environmentName), "."),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct credential engine: %v", err)
|
||||
}
|
||||
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare credential execution: %v", err)
|
||||
}
|
||||
if err := os.Unsetenv(environmentName); err != nil {
|
||||
t.Fatalf("unset credential environment: %v", err)
|
||||
}
|
||||
|
||||
result, err := engine.RunPrepared(context.Background(), prepared)
|
||||
if result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) ||
|
||||
!errors.Is(err, promptkit.ErrAPIKeyEnvMissing) {
|
||||
t.Fatalf("credential execution=(%+v, %v), want credential identities", result, err)
|
||||
}
|
||||
if len(client.snapshot()) != 0 {
|
||||
t.Fatalf("credential failure reached generation: %d calls", len(client.snapshot()))
|
||||
}
|
||||
assertPreparedPrivateValuesAbsent(t, err.Error(), environmentKey)
|
||||
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("credential failure did not consume handle: result=(%+v, %v)", result, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("preparation does not admit and execution timing starts after retention", func(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
client := newCapacityGateClient(release, 4)
|
||||
engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil)
|
||||
|
||||
activeRun := make(chan capacityRunResult, 1)
|
||||
go runCapacityRequest(
|
||||
engine,
|
||||
context.Background(),
|
||||
promptkit.RunRequest{PromptID: "prompt"},
|
||||
activeRun,
|
||||
)
|
||||
awaitCapacityRequest(t, client.started)
|
||||
|
||||
prepared, err := engine.PrepareExecution(
|
||||
context.Background(),
|
||||
promptkit.RunRequest{PromptID: "prompt"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare while capacity is full: %v", err)
|
||||
}
|
||||
if _, _, calls := client.snapshot(); calls != 1 {
|
||||
t.Fatalf("preparation invoked generation: calls=%d", calls)
|
||||
}
|
||||
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrCapacityExceeded) {
|
||||
t.Fatalf("capacity execution=(%+v, %v), want ErrCapacityExceeded", result, err)
|
||||
}
|
||||
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
|
||||
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("capacity rejection did not consume handle: result=(%+v, %v)", result, err)
|
||||
}
|
||||
if prepared.Details().PromptID != "prompt" {
|
||||
t.Fatal("details unavailable after capacity rejection")
|
||||
}
|
||||
|
||||
close(release)
|
||||
activeOutcome := awaitCapacityRun(t, activeRun)
|
||||
if activeOutcome.err != nil || activeOutcome.result == nil {
|
||||
t.Fatalf("active run outcome=(%+v, %v), want success", activeOutcome.result, activeOutcome.err)
|
||||
}
|
||||
|
||||
timed, err := engine.PrepareExecution(
|
||||
context.Background(),
|
||||
promptkit.RunRequest{PromptID: "prompt"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare timed execution: %v", err)
|
||||
}
|
||||
details := timed.Details()
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
executionFloor := time.Now().UTC()
|
||||
result, err := engine.RunPrepared(context.Background(), timed)
|
||||
if err != nil {
|
||||
t.Fatalf("run timed execution: %v", err)
|
||||
}
|
||||
if result.StartTime.Before(executionFloor) ||
|
||||
!result.StartTime.After(details.EndTime) ||
|
||||
result.EndTime.Before(result.StartTime) ||
|
||||
result.Duration != result.EndTime.Sub(result.StartTime) {
|
||||
t.Fatalf(
|
||||
"execution timing includes preparation or retention: details_end=%s floor=%s result=%+v",
|
||||
details.EndTime,
|
||||
executionFloor,
|
||||
result,
|
||||
)
|
||||
}
|
||||
if _, _, calls := client.snapshot(); calls != 2 {
|
||||
t.Fatalf("generation calls=%d, want active and timed executions only", calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
type mutablePreparedArtifactReader struct {
|
||||
mu sync.Mutex
|
||||
body string
|
||||
hash string
|
||||
calls int
|
||||
}
|
||||
|
||||
func (r *mutablePreparedArtifactReader) Read(
|
||||
_ context.Context,
|
||||
_ promptkit.ArtifactRef,
|
||||
) (*promptkit.Artifact, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.calls++
|
||||
return &promptkit.Artifact{
|
||||
Body: []byte(r.body),
|
||||
Hash: r.hash,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *mutablePreparedArtifactReader) set(body, hash string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.body = body
|
||||
r.hash = hash
|
||||
}
|
||||
|
||||
func (r *mutablePreparedArtifactReader) callCount() int {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.calls
|
||||
}
|
||||
|
||||
type preparedRecordingClient struct {
|
||||
mu sync.Mutex
|
||||
response *promptkit.GenerateResponse
|
||||
err error
|
||||
requests []promptkit.GenerateRequest
|
||||
started chan struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (c *preparedRecordingClient) Generate(
|
||||
ctx context.Context,
|
||||
request promptkit.GenerateRequest,
|
||||
) (*promptkit.GenerateResponse, error) {
|
||||
c.mu.Lock()
|
||||
c.requests = append(c.requests, request)
|
||||
c.mu.Unlock()
|
||||
|
||||
if c.started != nil {
|
||||
c.started <- struct{}{}
|
||||
}
|
||||
if c.release != nil {
|
||||
select {
|
||||
case <-c.release:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
if c.err != nil {
|
||||
return nil, c.err
|
||||
}
|
||||
return c.response, nil
|
||||
}
|
||||
|
||||
func (c *preparedRecordingClient) snapshot() []promptkit.GenerateRequest {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return append([]promptkit.GenerateRequest(nil), c.requests...)
|
||||
}
|
||||
|
||||
func newPreparedContractEngine(
|
||||
t *testing.T,
|
||||
client promptkit.LLMClient,
|
||||
message string,
|
||||
) *promptkit.Engine {
|
||||
t.Helper()
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", message), "."),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "profile",
|
||||
Endpoint: "http://example.test/v1",
|
||||
Model: "model",
|
||||
}),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct prepared execution engine: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func preparedPromptSource(label string) fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prepared
|
||||
version: "1"
|
||||
default_profile: profile
|
||||
inputs:
|
||||
- name: input
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content: 'Input={{input "input"}} Label={{.label}}'
|
||||
description: ` + label + `
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: schema.json
|
||||
`)},
|
||||
}
|
||||
}
|
||||
|
||||
func preparedProfileSource(model string) fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"profile.yaml": &fstest.MapFile{Data: []byte(`id: profile
|
||||
endpoint: http://example.test/v1
|
||||
model: ` + model + `
|
||||
`)},
|
||||
}
|
||||
}
|
||||
|
||||
func preparedCredentialProfileSource(environmentName string) fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"profile.yaml": &fstest.MapFile{Data: []byte(`id: profile
|
||||
endpoint: http://example.test/v1
|
||||
model: model
|
||||
api_key_env: ` + environmentName + `
|
||||
`)},
|
||||
}
|
||||
}
|
||||
|
||||
func preparedSchemaSource() fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"schema.json": &fstest.MapFile{Data: []byte(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "original root",
|
||||
"type": "object",
|
||||
"required": ["value"],
|
||||
"properties": {
|
||||
"value": {"$ref": "value.json"}
|
||||
}
|
||||
}`)},
|
||||
"value.json": &fstest.MapFile{Data: []byte(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "integer",
|
||||
"minimum": 2
|
||||
}`)},
|
||||
}
|
||||
}
|
||||
|
||||
func assertPreparedPrivateValuesAbsent(t *testing.T, value string, privateValues ...string) {
|
||||
t.Helper()
|
||||
for _, privateValue := range privateValues {
|
||||
if strings.Contains(value, privateValue) {
|
||||
t.Fatalf("value exposed private data %q: %s", privateValue, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
34
types.go
34
types.go
@@ -78,9 +78,10 @@ const (
|
||||
// RunRequest selects one prompt execution. It has no stable JSON
|
||||
// representation.
|
||||
//
|
||||
// Prepare and Run copy the request's maps, pointers, and nested
|
||||
// JSON-compatible values before using them. The caller may mutate the request
|
||||
// after either method returns.
|
||||
// Prepare, PrepareExecution, and Run copy the request's maps, pointers, and
|
||||
// nested JSON-compatible values before using them. The caller may mutate the
|
||||
// request after any method returns. A successful PrepareExecution retains its
|
||||
// own private execution snapshot for RunPrepared.
|
||||
type RunRequest struct {
|
||||
// PromptID is the required non-empty prompt identifier.
|
||||
PromptID string
|
||||
@@ -98,12 +99,14 @@ type RunRequest struct {
|
||||
// opaque consumer metadata, not a credential, and may be exposed in
|
||||
// prepared values, results, collaborator requests, provider requests, and
|
||||
// provider observability. Callers should use stable, non-sensitive
|
||||
// identifiers. An overlong direct value makes Prepare or Run return an
|
||||
// error matching ErrInvalidRequest.
|
||||
// identifiers. An overlong direct value makes Prepare, PrepareExecution, or
|
||||
// Run return an error matching ErrInvalidRequest.
|
||||
SessionID string
|
||||
// APIKey is a request-scoped direct credential. It takes precedence over
|
||||
// APIKeyEnv, is passed to the selected LLMClient, and is never included in
|
||||
// prepared values, results, hashes, JSON, String, or GoString output.
|
||||
// prepared values, results, hashes, JSON, String, or GoString output. A
|
||||
// successful PrepareExecution retains it only in the opaque handle until
|
||||
// RunPrepared claims the handle or Discard invalidates it.
|
||||
APIKey string `json:"-"`
|
||||
// Inputs maps prompt input names to references. A nil or empty map is valid
|
||||
// only when the selected prompt and its templates require no inputs.
|
||||
@@ -119,9 +122,10 @@ type RunRequest struct {
|
||||
Validation *OutputContract
|
||||
}
|
||||
|
||||
// PreparedRun contains prepared prompt execution state. It does not include
|
||||
// resolved API key values, model output, validation results, or internal target
|
||||
// presence metadata. PreparedRun has a stable JSON representation.
|
||||
// PreparedRun contains prepared prompt execution state returned by
|
||||
// [Engine.Prepare] or [PreparedExecution.Details]. It does not include resolved
|
||||
// API key values, model output, validation results, or internal target presence
|
||||
// metadata. PreparedRun has a stable JSON representation.
|
||||
//
|
||||
// All maps, slices, pointers, and schema values are caller-owned copies. JSON
|
||||
// timestamps use RFC 3339 and zero timing values are omitted. Hash formats are
|
||||
@@ -154,7 +158,8 @@ type PreparedRun struct {
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
// RenderedPromptHash is an opaque equality value for SessionID and Messages.
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
// Messages are the rendered messages that Run passes to the LLM client.
|
||||
// Messages are the rendered messages that Run or RunPrepared passes to the
|
||||
// LLM client.
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
// StartTime is the UTC time at which preparation began.
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
@@ -214,12 +219,15 @@ type RunResult struct {
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
// Usage is the token accounting reported by the LLM client.
|
||||
Usage TokenUsage `json:"usage"`
|
||||
// StartTime is the UTC time immediately before preparation begins.
|
||||
// StartTime is the UTC time immediately before ordinary Run preparation or
|
||||
// after RunPrepared claims its handle.
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
// EndTime is the UTC time after generation and validation complete.
|
||||
EndTime time.Time `json:"end_time,omitempty"`
|
||||
// Duration covers preparation, generation, and validation. JSON represents
|
||||
// it as integer milliseconds in duration_ms and omits a zero value.
|
||||
// Duration covers preparation, generation, and validation for Run. For
|
||||
// RunPrepared it covers only the execution attempt after claim and excludes
|
||||
// preparation and consumer-held delay. JSON represents it as integer
|
||||
// milliseconds in duration_ms and omits a zero value.
|
||||
Duration time.Duration `json:"-"`
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user