package promptexec import ( "context" "errors" "fmt" "strings" "testing" "unicode/utf8" ) type fakeExecutor struct{} func (fakeExecutor) InspectPrompt(context.Context, string, string) (PromptInspection, error) { return PromptInspection{}, nil } func (fakeExecutor) InspectProfile(context.Context, string) (ProfileInspection, error) { return ProfileInspection{}, nil } func (fakeExecutor) Execute(context.Context, ExecuteRequest, PreparationCallback) (*Execution, error) { return nil, nil } var _ Executor = fakeExecutor{} type lifecycleExecutor struct { providerCalled bool operationalFailure error validationRejected bool } func (executor *lifecycleExecutor) InspectPrompt(context.Context, string, string) (PromptInspection, error) { return PromptInspection{}, nil } func (executor *lifecycleExecutor) InspectProfile(context.Context, string) (ProfileInspection, error) { return ProfileInspection{}, nil } func (executor *lifecycleExecutor) Execute(_ context.Context, request ExecuteRequest, callback PreparationCallback) (*Execution, error) { if executor.operationalFailure != nil { return nil, executor.operationalFailure } preparation := Preparation{PromptID: request.PromptID, PromptVersion: request.PromptVersion} var debug *PreparationDebug if request.CaptureDebug { debug = &PreparationDebug{RenderedMessages: []RenderedMessage{{Role: "user", Content: "sensitive rendered message"}}} } if callback != nil { if err := callback(copyPreparation(preparation), copyPreparationDebug(debug)); err != nil { return nil, err } } executor.providerCalled = true status := ValidationPassed if executor.validationRejected { status = ValidationFailed } result := Execution{PromptID: request.PromptID, PromptVersion: request.PromptVersion, Validation: Validation{Status: status}, RawOutput: []byte("generated content")} if request.CaptureDebug { result.Debug = &ExecutionDebug{RawOutput: []byte("generated content")} } return &result, nil } func TestExecutorLifecycleFixtures(t *testing.T) { request := ExecuteRequest{PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0"} t.Run("callback failure prevents provider execution", func(t *testing.T) { executor := &lifecycleExecutor{} callbackError := errors.New("persistence failed") result, err := executor.Execute(context.Background(), request, func(Preparation, *PreparationDebug) error { return callbackError }) if result != nil || !errors.Is(err, callbackError) || executor.providerCalled { t.Fatalf("result/error/provider = %#v/%v/%t", result, err, executor.providerCalled) } }) t.Run("validation rejection is completed result", func(t *testing.T) { executor := &lifecycleExecutor{validationRejected: true} result, err := executor.Execute(context.Background(), request, nil) if err != nil || result == nil || result.Validation.Status != ValidationFailed || !executor.providerCalled { t.Fatalf("result/error/provider = %#v/%v/%t", result, err, executor.providerCalled) } }) t.Run("operational failure has no completed result", func(t *testing.T) { failure := NewError(Generation, "generation failed", nil) executor := &lifecycleExecutor{operationalFailure: failure} result, err := executor.Execute(context.Background(), request, nil) if result != nil || !errors.Is(err, failure) || executor.providerCalled { t.Fatalf("result/error/provider = %#v/%v/%t", result, err, executor.providerCalled) } }) t.Run("debug requires explicit request", func(t *testing.T) { executor := &lifecycleExecutor{} var callbackDebug *PreparationDebug result, err := executor.Execute(context.Background(), request, func(_ Preparation, debug *PreparationDebug) error { callbackDebug = debug return nil }) if err != nil || callbackDebug != nil || result.Debug != nil { t.Fatalf("debug = %#v/%#v, error = %v", callbackDebug, result.Debug, err) } request.CaptureDebug = true result, err = executor.Execute(context.Background(), request, func(_ Preparation, debug *PreparationDebug) error { callbackDebug = debug return nil }) if err != nil || callbackDebug == nil || result.Debug == nil { t.Fatalf("debug = %#v/%#v, error = %v", callbackDebug, result.Debug, err) } }) } func TestErrorCategoriesAndCapacityError(t *testing.T) { categories := []ErrorCategory{ InvalidConfiguration, InvalidRequest, PromptNotFound, PromptLoad, ProfileNotFound, ProfileLoad, MissingCredential, ArtifactLoad, PromptRender, Capacity, Generation, OperationalValidation, ValidationRejected, Canceled, DeadlineExceeded, } cause := errors.New("dependency details must not become safe error text") for _, category := range categories { t.Run(string(category), func(t *testing.T) { err := NewError(category, "safe workflow failure", cause) if err.Category() != category || CategoryOf(err) != category { t.Fatalf("category = %q/%q, want %q", err.Category(), CategoryOf(err), category) } if !errors.Is(err, cause) { t.Fatal("errors.Is() = false, want preserved cause") } if strings.Contains(err.Error(), cause.Error()) { t.Fatalf("error leaks cause: %q", err) } }) } capacity := NewCapacityError("local", "safe capacity failure", cause) if capacity.Category() != Capacity || CategoryOf(capacity) != Capacity || capacity.BackendID != "local" { t.Fatalf("capacity error = %#v", capacity) } if !errors.Is(capacity, cause) { t.Fatal("capacity error does not preserve cause") } } func TestBoundDiagnosticAndErrorText(t *testing.T) { longUTF8 := strings.Repeat("é", maxDiagnosticBytes) values := make([]string, maxValidationDiagnostics+2) for index := range values { values[index] = longUTF8 } values[0] = string([]byte{'a', 0xff, 'b'}) bounded := boundDiagnostics(values) if len(bounded) != maxValidationDiagnostics { t.Fatalf("diagnostics length = %d, want %d", len(bounded), maxValidationDiagnostics) } for index, value := range bounded { if len(value) > maxDiagnosticBytes || !utf8.ValidString(value) { t.Fatalf("diagnostic %d = %q, want valid UTF-8 within %d bytes", index, value, maxDiagnosticBytes) } } if bounded[0] != "a�b" { t.Fatalf("invalid UTF-8 diagnostic = %q, want replacement", bounded[0]) } validation := NewValidation(ValidationFailed, "json_schema", "daily.schema.json", 2, values) if validation.RepairAttempts != 2 || len(validation.Diagnostics) != maxValidationDiagnostics || validation.Diagnostics[0] != "a�b" { t.Fatalf("validation = %#v, want bounded diagnostics", validation) } err := NewError(Generation, strings.Repeat("é", maxErrorMessageBytes), nil) if len(err.Error()) > maxErrorMessageBytes || !utf8.ValidString(err.Error()) { t.Fatalf("error = %q, want valid UTF-8 within %d bytes", err, maxErrorMessageBytes) } capacity := NewCapacityError(strings.Repeat("x", 300), strings.Repeat("é", maxErrorMessageBytes), nil) if len(capacity.Error()) > maxErrorMessageBytes || !utf8.ValidString(capacity.Error()) { t.Fatalf("capacity error = %q, want valid UTF-8 within %d bytes", capacity, maxErrorMessageBytes) } } func TestGenerationErrorKeepsProviderDetailsOutOfRoutineFormatting(t *testing.T) { cause := errors.New("provider response must stay hidden") error := NewGenerationError( 429, string([]byte{'c', 0xff})+strings.Repeat("界", 300), strings.Repeat("type", 100), string([]byte{'m', 0xff})+strings.Repeat("界", 4_200), cause, ) if error.StatusCode() != 429 || error.Category() != Generation || CategoryOf(error) != Generation { t.Fatalf("generation error identity = %#v", error) } if utf8.RuneCountInString(error.ProviderCode()) != 256 || utf8.RuneCountInString(error.ProviderType()) != 256 || utf8.RuneCountInString(error.ProviderMessage()) != 4096 { t.Fatalf("provider detail bounds = %d/%d/%d", utf8.RuneCountInString(error.ProviderCode()), utf8.RuneCountInString(error.ProviderType()), utf8.RuneCountInString(error.ProviderMessage())) } if !utf8.ValidString(error.ProviderCode()) || !utf8.ValidString(error.ProviderType()) || !utf8.ValidString(error.ProviderMessage()) { t.Fatalf("provider details are not valid UTF-8: %#v", error) } if !errors.Is(error, cause) { t.Fatal("errors.Is() = false, want preserved dependency cause") } var owned *Error if !errors.As(error, &owned) || owned.Category() != Generation { t.Fatalf("errors.As(*Error) = %#v, want project-owned generation error", owned) } want := "generation: provider generation failed (HTTP 429)" if error.Error() != want || fmt.Sprintf("%#v", error) != want { t.Fatalf("ordinary formatting = %q / %#v, want %q", error.Error(), error, want) } for _, private := range []string{cause.Error(), error.ProviderCode(), error.ProviderType(), error.ProviderMessage()} { if strings.Contains(error.Error(), private) || strings.Contains(fmt.Sprintf("%#v", error), private) { t.Fatalf("generation error leaks provider detail %q", private) } } statusOnly := NewGenerationError(503, "", "", "", nil) if statusOnly.Error() != "generation: provider generation failed (HTTP 503)" { t.Fatalf("status-only error = %q", statusOnly) } var nilError *GenerationError if nilError.StatusCode() != 0 || nilError.ProviderCode() != "" || nilError.ProviderType() != "" || nilError.ProviderMessage() != "" || nilError.Category() != Generation || nilError.Error() != "" || nilError.GoString() != "" || nilError.Unwrap() != nil { t.Fatalf("nil generation error = %#v", nilError) } } func TestContractCopiesMutableValues(t *testing.T) { preparation := Preparation{InputHashes: map[string]string{"data_package": "input-hash"}, Output: OutputContract{RepairAttempts: 3}} preparationDebug := &PreparationDebug{ RenderedMessages: []RenderedMessage{{Role: "user", Content: "rendered input"}}, StructuredSchema: []byte("schema body"), ParametersJSON: []byte(`{"temperature":0.2}`), } execution := Execution{ InputHashes: map[string]string{"data_package": "input-hash"}, Validation: Validation{RepairAttempts: 2, Diagnostics: []string{"validation detail"}}, RawOutput: []byte("generated output"), Debug: &ExecutionDebug{ RawOutput: []byte("provider output"), ValidationDiagnostics: []string{"detailed validation"}, }, } preparationCopy := copyPreparation(preparation) debugCopy := copyPreparationDebug(preparationDebug) executionCopy := copyExecution(execution) preparation.InputHashes["data_package"] = "changed" preparationDebug.RenderedMessages[0].Content = "changed" preparationDebug.StructuredSchema[0] = 'x' preparationDebug.ParametersJSON[0] = 'x' execution.InputHashes["data_package"] = "changed" execution.Validation.Diagnostics[0] = "changed" execution.RawOutput[0] = 'x' execution.Debug.RawOutput[0] = 'x' execution.Debug.ValidationDiagnostics[0] = "changed" if preparationCopy.InputHashes["data_package"] != "input-hash" || preparationCopy.Output.RepairAttempts != 3 { t.Fatalf("preparation copy = %#v", preparationCopy) } if debugCopy.RenderedMessages[0].Content != "rendered input" || string(debugCopy.StructuredSchema) != "schema body" || string(debugCopy.ParametersJSON) != `{"temperature":0.2}` { t.Fatalf("preparation debug copy = %#v", debugCopy) } if executionCopy.InputHashes["data_package"] != "input-hash" || executionCopy.Validation.RepairAttempts != 2 || executionCopy.Validation.Diagnostics[0] != "validation detail" || string(executionCopy.RawOutput) != "generated output" || string(executionCopy.Debug.RawOutput) != "provider output" || executionCopy.Debug.ValidationDiagnostics[0] != "detailed validation" { t.Fatalf("execution copy = %#v", executionCopy) } } func TestSafeContractValuesExcludeSensitiveFields(t *testing.T) { preparation := Preparation{ PromptID: "weather.daily_generated_text", PromptVersion: "1.0.0", PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", InputHashes: map[string]string{"data_package": "input-hash"}, ProfileID: "configured-profile", BackendID: "local", ModelName: "model-name", Output: OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: "daily.generated_text.schema.json"}, } execution := Execution{ RunID: "run-id", PromptID: preparation.PromptID, PromptVersion: preparation.PromptVersion, PromptHash: preparation.PromptHash, RenderedPromptHash: preparation.RenderedPromptHash, InputHashes: copyStringMap(preparation.InputHashes), ProfileID: preparation.ProfileID, BackendID: preparation.BackendID, ModelName: preparation.ModelName, GeneratedHash: "generated-hash", RawOutput: []byte("generated content"), } text := preparation.PromptID + preparation.PromptVersion + preparation.PromptHash + preparation.RenderedPromptHash + preparation.ProfileID + preparation.BackendID + preparation.ModelName + preparation.Output.SchemaPath + execution.RunID + execution.GeneratedHash for _, unwanted := range []string{"https://provider.example", "API_KEY_ENV", "rendered message", "schema body", "input body", "provider response body", "full parameters"} { if strings.Contains(text, unwanted) { t.Fatalf("safe values contain %q: %s", unwanted, text) } } if string(execution.RawOutput) != "generated content" { t.Fatalf("raw output = %q, want generated content", execution.RawOutput) } }