312 lines
13 KiB
Go
312 lines
13 KiB
Go
package promptexec
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"strings"
|
||
"testing"
|
||
"unicode/utf8"
|
||
)
|
||
|
||
type fakeExecutor struct{}
|
||
|
||
func (fakeExecutor) InspectPrompt(context.Context, string, string) (PromptInspection, error) {
|
||
return PromptInspection{}, nil
|
||
}
|
||
|
||
func (fakeExecutor) InspectProfile(context.Context, string) (ProfileInspection, error) {
|
||
return ProfileInspection{}, nil
|
||
}
|
||
|
||
func (fakeExecutor) Execute(context.Context, ExecuteRequest, PreparationCallback) (*Execution, error) {
|
||
return nil, nil
|
||
}
|
||
|
||
var _ Executor = fakeExecutor{}
|
||
|
||
type lifecycleExecutor struct {
|
||
providerCalled bool
|
||
operationalFailure error
|
||
validationRejected bool
|
||
}
|
||
|
||
func (executor *lifecycleExecutor) InspectPrompt(context.Context, string, string) (PromptInspection, error) {
|
||
return PromptInspection{}, nil
|
||
}
|
||
|
||
func (executor *lifecycleExecutor) InspectProfile(context.Context, string) (ProfileInspection, error) {
|
||
return ProfileInspection{}, nil
|
||
}
|
||
|
||
func (executor *lifecycleExecutor) Execute(_ context.Context, request ExecuteRequest, callback PreparationCallback) (*Execution, error) {
|
||
if executor.operationalFailure != nil {
|
||
return nil, executor.operationalFailure
|
||
}
|
||
preparation := Preparation{PromptID: request.PromptID, PromptVersion: request.PromptVersion}
|
||
var debug *PreparationDebug
|
||
if request.CaptureDebug {
|
||
debug = &PreparationDebug{RenderedMessages: []RenderedMessage{{Role: "user", Content: "sensitive rendered message"}}}
|
||
}
|
||
if callback != nil {
|
||
if err := callback(copyPreparation(preparation), copyPreparationDebug(debug)); err != nil {
|
||
return nil, err
|
||
}
|
||
}
|
||
executor.providerCalled = true
|
||
status := ValidationPassed
|
||
if executor.validationRejected {
|
||
status = ValidationFailed
|
||
}
|
||
result := Execution{PromptID: request.PromptID, PromptVersion: request.PromptVersion, Validation: Validation{Status: status}, RawOutput: []byte("generated content")}
|
||
if request.CaptureDebug {
|
||
result.Debug = &ExecutionDebug{RawOutput: []byte("generated content")}
|
||
}
|
||
return &result, nil
|
||
}
|
||
|
||
func TestExecutorLifecycleFixtures(t *testing.T) {
|
||
request := ExecuteRequest{PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0"}
|
||
t.Run("callback failure prevents provider execution", func(t *testing.T) {
|
||
executor := &lifecycleExecutor{}
|
||
callbackError := errors.New("persistence failed")
|
||
result, err := executor.Execute(context.Background(), request, func(Preparation, *PreparationDebug) error { return callbackError })
|
||
if result != nil || !errors.Is(err, callbackError) || executor.providerCalled {
|
||
t.Fatalf("result/error/provider = %#v/%v/%t", result, err, executor.providerCalled)
|
||
}
|
||
})
|
||
t.Run("validation rejection is completed result", func(t *testing.T) {
|
||
executor := &lifecycleExecutor{validationRejected: true}
|
||
result, err := executor.Execute(context.Background(), request, nil)
|
||
if err != nil || result == nil || result.Validation.Status != ValidationFailed || !executor.providerCalled {
|
||
t.Fatalf("result/error/provider = %#v/%v/%t", result, err, executor.providerCalled)
|
||
}
|
||
})
|
||
t.Run("operational failure has no completed result", func(t *testing.T) {
|
||
failure := NewError(Generation, "generation failed", nil)
|
||
executor := &lifecycleExecutor{operationalFailure: failure}
|
||
result, err := executor.Execute(context.Background(), request, nil)
|
||
if result != nil || !errors.Is(err, failure) || executor.providerCalled {
|
||
t.Fatalf("result/error/provider = %#v/%v/%t", result, err, executor.providerCalled)
|
||
}
|
||
})
|
||
t.Run("debug requires explicit request", func(t *testing.T) {
|
||
executor := &lifecycleExecutor{}
|
||
var callbackDebug *PreparationDebug
|
||
result, err := executor.Execute(context.Background(), request, func(_ Preparation, debug *PreparationDebug) error {
|
||
callbackDebug = debug
|
||
return nil
|
||
})
|
||
if err != nil || callbackDebug != nil || result.Debug != nil {
|
||
t.Fatalf("debug = %#v/%#v, error = %v", callbackDebug, result.Debug, err)
|
||
}
|
||
request.CaptureDebug = true
|
||
result, err = executor.Execute(context.Background(), request, func(_ Preparation, debug *PreparationDebug) error {
|
||
callbackDebug = debug
|
||
return nil
|
||
})
|
||
if err != nil || callbackDebug == nil || result.Debug == nil {
|
||
t.Fatalf("debug = %#v/%#v, error = %v", callbackDebug, result.Debug, err)
|
||
}
|
||
})
|
||
}
|
||
|
||
func TestErrorCategoriesAndCapacityError(t *testing.T) {
|
||
categories := []ErrorCategory{
|
||
InvalidConfiguration,
|
||
InvalidRequest,
|
||
PromptNotFound,
|
||
PromptLoad,
|
||
ProfileNotFound,
|
||
ProfileLoad,
|
||
MissingCredential,
|
||
ArtifactLoad,
|
||
PromptRender,
|
||
Capacity,
|
||
Generation,
|
||
OperationalValidation,
|
||
ValidationRejected,
|
||
Canceled,
|
||
DeadlineExceeded,
|
||
}
|
||
cause := errors.New("dependency details must not become safe error text")
|
||
for _, category := range categories {
|
||
t.Run(string(category), func(t *testing.T) {
|
||
err := NewError(category, "safe workflow failure", cause)
|
||
if err.Category() != category || CategoryOf(err) != category {
|
||
t.Fatalf("category = %q/%q, want %q", err.Category(), CategoryOf(err), category)
|
||
}
|
||
if !errors.Is(err, cause) {
|
||
t.Fatal("errors.Is() = false, want preserved cause")
|
||
}
|
||
if strings.Contains(err.Error(), cause.Error()) {
|
||
t.Fatalf("error leaks cause: %q", err)
|
||
}
|
||
})
|
||
}
|
||
|
||
capacity := NewCapacityError("local", "safe capacity failure", cause)
|
||
if capacity.Category() != Capacity || CategoryOf(capacity) != Capacity || capacity.BackendID != "local" {
|
||
t.Fatalf("capacity error = %#v", capacity)
|
||
}
|
||
if !errors.Is(capacity, cause) {
|
||
t.Fatal("capacity error does not preserve cause")
|
||
}
|
||
}
|
||
|
||
func TestBoundDiagnosticAndErrorText(t *testing.T) {
|
||
longUTF8 := strings.Repeat("é", maxDiagnosticBytes)
|
||
values := make([]string, maxValidationDiagnostics+2)
|
||
for index := range values {
|
||
values[index] = longUTF8
|
||
}
|
||
values[0] = string([]byte{'a', 0xff, 'b'})
|
||
bounded := boundDiagnostics(values)
|
||
if len(bounded) != maxValidationDiagnostics {
|
||
t.Fatalf("diagnostics length = %d, want %d", len(bounded), maxValidationDiagnostics)
|
||
}
|
||
for index, value := range bounded {
|
||
if len(value) > maxDiagnosticBytes || !utf8.ValidString(value) {
|
||
t.Fatalf("diagnostic %d = %q, want valid UTF-8 within %d bytes", index, value, maxDiagnosticBytes)
|
||
}
|
||
}
|
||
if bounded[0] != "a<>b" {
|
||
t.Fatalf("invalid UTF-8 diagnostic = %q, want replacement", bounded[0])
|
||
}
|
||
validation := NewValidation(ValidationFailed, "json_schema", "daily.schema.json", 2, values)
|
||
if validation.RepairAttempts != 2 || len(validation.Diagnostics) != maxValidationDiagnostics || validation.Diagnostics[0] != "a<>b" {
|
||
t.Fatalf("validation = %#v, want bounded diagnostics", validation)
|
||
}
|
||
|
||
err := NewError(Generation, strings.Repeat("é", maxErrorMessageBytes), nil)
|
||
if len(err.Error()) > maxErrorMessageBytes || !utf8.ValidString(err.Error()) {
|
||
t.Fatalf("error = %q, want valid UTF-8 within %d bytes", err, maxErrorMessageBytes)
|
||
}
|
||
capacity := NewCapacityError(strings.Repeat("x", 300), strings.Repeat("é", maxErrorMessageBytes), nil)
|
||
if len(capacity.Error()) > maxErrorMessageBytes || !utf8.ValidString(capacity.Error()) {
|
||
t.Fatalf("capacity error = %q, want valid UTF-8 within %d bytes", capacity, maxErrorMessageBytes)
|
||
}
|
||
}
|
||
|
||
func TestGenerationErrorKeepsProviderDetailsOutOfRoutineFormatting(t *testing.T) {
|
||
cause := errors.New("provider response must stay hidden")
|
||
error := NewGenerationError(
|
||
429,
|
||
string([]byte{'c', 0xff})+strings.Repeat("界", 300),
|
||
strings.Repeat("type", 100),
|
||
string([]byte{'m', 0xff})+strings.Repeat("界", 4_200),
|
||
cause,
|
||
)
|
||
if error.StatusCode() != 429 || error.Category() != Generation || CategoryOf(error) != Generation {
|
||
t.Fatalf("generation error identity = %#v", error)
|
||
}
|
||
if utf8.RuneCountInString(error.ProviderCode()) != 256 || utf8.RuneCountInString(error.ProviderType()) != 256 || utf8.RuneCountInString(error.ProviderMessage()) != 4096 {
|
||
t.Fatalf("provider detail bounds = %d/%d/%d", utf8.RuneCountInString(error.ProviderCode()), utf8.RuneCountInString(error.ProviderType()), utf8.RuneCountInString(error.ProviderMessage()))
|
||
}
|
||
if !utf8.ValidString(error.ProviderCode()) || !utf8.ValidString(error.ProviderType()) || !utf8.ValidString(error.ProviderMessage()) {
|
||
t.Fatalf("provider details are not valid UTF-8: %#v", error)
|
||
}
|
||
if !errors.Is(error, cause) {
|
||
t.Fatal("errors.Is() = false, want preserved dependency cause")
|
||
}
|
||
var owned *Error
|
||
if !errors.As(error, &owned) || owned.Category() != Generation {
|
||
t.Fatalf("errors.As(*Error) = %#v, want project-owned generation error", owned)
|
||
}
|
||
want := "generation: provider generation failed (HTTP 429)"
|
||
if error.Error() != want || fmt.Sprintf("%#v", error) != want {
|
||
t.Fatalf("ordinary formatting = %q / %#v, want %q", error.Error(), error, want)
|
||
}
|
||
for _, private := range []string{cause.Error(), error.ProviderCode(), error.ProviderType(), error.ProviderMessage()} {
|
||
if strings.Contains(error.Error(), private) || strings.Contains(fmt.Sprintf("%#v", error), private) {
|
||
t.Fatalf("generation error leaks provider detail %q", private)
|
||
}
|
||
}
|
||
|
||
statusOnly := NewGenerationError(503, "", "", "", nil)
|
||
if statusOnly.Error() != "generation: provider generation failed (HTTP 503)" {
|
||
t.Fatalf("status-only error = %q", statusOnly)
|
||
}
|
||
var nilError *GenerationError
|
||
if nilError.StatusCode() != 0 || nilError.ProviderCode() != "" || nilError.ProviderType() != "" || nilError.ProviderMessage() != "" || nilError.Category() != Generation || nilError.Error() != "" || nilError.GoString() != "" || nilError.Unwrap() != nil {
|
||
t.Fatalf("nil generation error = %#v", nilError)
|
||
}
|
||
}
|
||
|
||
func TestContractCopiesMutableValues(t *testing.T) {
|
||
preparation := Preparation{InputHashes: map[string]string{"data_package": "input-hash"}, Output: OutputContract{RepairAttempts: 3}}
|
||
preparationDebug := &PreparationDebug{
|
||
RenderedMessages: []RenderedMessage{{Role: "user", Content: "rendered input"}},
|
||
StructuredSchema: []byte("schema body"),
|
||
ParametersJSON: []byte(`{"temperature":0.2}`),
|
||
}
|
||
execution := Execution{
|
||
InputHashes: map[string]string{"data_package": "input-hash"},
|
||
Validation: Validation{RepairAttempts: 2, Diagnostics: []string{"validation detail"}},
|
||
RawOutput: []byte("generated output"),
|
||
Debug: &ExecutionDebug{
|
||
RawOutput: []byte("provider output"),
|
||
ValidationDiagnostics: []string{"detailed validation"},
|
||
},
|
||
}
|
||
|
||
preparationCopy := copyPreparation(preparation)
|
||
debugCopy := copyPreparationDebug(preparationDebug)
|
||
executionCopy := copyExecution(execution)
|
||
preparation.InputHashes["data_package"] = "changed"
|
||
preparationDebug.RenderedMessages[0].Content = "changed"
|
||
preparationDebug.StructuredSchema[0] = 'x'
|
||
preparationDebug.ParametersJSON[0] = 'x'
|
||
execution.InputHashes["data_package"] = "changed"
|
||
execution.Validation.Diagnostics[0] = "changed"
|
||
execution.RawOutput[0] = 'x'
|
||
execution.Debug.RawOutput[0] = 'x'
|
||
execution.Debug.ValidationDiagnostics[0] = "changed"
|
||
|
||
if preparationCopy.InputHashes["data_package"] != "input-hash" || preparationCopy.Output.RepairAttempts != 3 {
|
||
t.Fatalf("preparation copy = %#v", preparationCopy)
|
||
}
|
||
if debugCopy.RenderedMessages[0].Content != "rendered input" || string(debugCopy.StructuredSchema) != "schema body" || string(debugCopy.ParametersJSON) != `{"temperature":0.2}` {
|
||
t.Fatalf("preparation debug copy = %#v", debugCopy)
|
||
}
|
||
if executionCopy.InputHashes["data_package"] != "input-hash" || executionCopy.Validation.RepairAttempts != 2 || executionCopy.Validation.Diagnostics[0] != "validation detail" || string(executionCopy.RawOutput) != "generated output" || string(executionCopy.Debug.RawOutput) != "provider output" || executionCopy.Debug.ValidationDiagnostics[0] != "detailed validation" {
|
||
t.Fatalf("execution copy = %#v", executionCopy)
|
||
}
|
||
}
|
||
|
||
func TestSafeContractValuesExcludeSensitiveFields(t *testing.T) {
|
||
preparation := Preparation{
|
||
PromptID: "weather.daily_generated_text",
|
||
PromptVersion: "1.0.0",
|
||
PromptHash: "prompt-hash",
|
||
RenderedPromptHash: "rendered-hash",
|
||
InputHashes: map[string]string{"data_package": "input-hash"},
|
||
ProfileID: "configured-profile",
|
||
BackendID: "local",
|
||
ModelName: "model-name",
|
||
Output: OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: "daily.generated_text.schema.json"},
|
||
}
|
||
execution := Execution{
|
||
RunID: "run-id",
|
||
PromptID: preparation.PromptID,
|
||
PromptVersion: preparation.PromptVersion,
|
||
PromptHash: preparation.PromptHash,
|
||
RenderedPromptHash: preparation.RenderedPromptHash,
|
||
InputHashes: copyStringMap(preparation.InputHashes),
|
||
ProfileID: preparation.ProfileID,
|
||
BackendID: preparation.BackendID,
|
||
ModelName: preparation.ModelName,
|
||
GeneratedHash: "generated-hash",
|
||
RawOutput: []byte("generated content"),
|
||
}
|
||
text := preparation.PromptID + preparation.PromptVersion + preparation.PromptHash + preparation.RenderedPromptHash + preparation.ProfileID + preparation.BackendID + preparation.ModelName + preparation.Output.SchemaPath + execution.RunID + execution.GeneratedHash
|
||
for _, unwanted := range []string{"https://provider.example", "API_KEY_ENV", "rendered message", "schema body", "input body", "provider response body", "full parameters"} {
|
||
if strings.Contains(text, unwanted) {
|
||
t.Fatalf("safe values contain %q: %s", unwanted, text)
|
||
}
|
||
}
|
||
if string(execution.RawOutput) != "generated content" {
|
||
t.Fatalf("raw output = %q, want generated content", execution.RawOutput)
|
||
}
|
||
}
|