607 lines
18 KiB
Go
607 lines
18 KiB
Go
package usecase
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"reflect"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
|
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
|
)
|
|
|
|
type recordingPreparedValidation struct {
|
|
contract domain.OutputContract
|
|
schemaDocument any
|
|
results []domain.ValidationResult
|
|
errs []error
|
|
artifacts []string
|
|
}
|
|
|
|
func (p *recordingPreparedValidation) Validate(
|
|
_ context.Context,
|
|
artifact *domain.Artifact,
|
|
) (domain.ValidationResult, error) {
|
|
p.artifacts = append(p.artifacts, string(artifact.Body))
|
|
index := len(p.artifacts) - 1
|
|
if index < len(p.errs) && p.errs[index] != nil {
|
|
return domain.ValidationResult{}, p.errs[index]
|
|
}
|
|
if len(p.results) == 0 {
|
|
return domain.ValidationResult{
|
|
Status: domain.ValidationPassed,
|
|
Mode: p.contract.ValidationMode,
|
|
IsValid: true,
|
|
}, nil
|
|
}
|
|
if index >= len(p.results) {
|
|
index = len(p.results) - 1
|
|
}
|
|
return p.results[index], nil
|
|
}
|
|
|
|
func (p *recordingPreparedValidation) SchemaDocument() any {
|
|
return p.schemaDocument
|
|
}
|
|
|
|
type recordingValidationPreparer struct {
|
|
plan *recordingPreparedValidation
|
|
prepareErr error
|
|
prepareCalls int
|
|
directValidateCalls int
|
|
}
|
|
|
|
type validationOnly struct{}
|
|
|
|
func (validationOnly) Validate(context.Context, *domain.Artifact, domain.OutputContract) (domain.ValidationResult, error) {
|
|
return domain.ValidationResult{}, nil
|
|
}
|
|
|
|
func (v *recordingValidationPreparer) Validate(
|
|
context.Context,
|
|
*domain.Artifact,
|
|
domain.OutputContract,
|
|
) (domain.ValidationResult, error) {
|
|
v.directValidateCalls++
|
|
return domain.ValidationResult{}, errors.New("live validation must not be used")
|
|
}
|
|
|
|
func (v *recordingValidationPreparer) PrepareValidation(
|
|
_ context.Context,
|
|
contract domain.OutputContract,
|
|
) (validate.PreparedValidation, error) {
|
|
v.prepareCalls++
|
|
if v.prepareErr != nil {
|
|
return nil, v.prepareErr
|
|
}
|
|
v.plan.contract = contract
|
|
return v.plan, nil
|
|
}
|
|
|
|
func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.T) {
|
|
schemaDocument := map[string]any{
|
|
"type": "object",
|
|
"properties": map[string]any{
|
|
"value": map[string]any{"type": "string"},
|
|
"": map[string]any{"type": "boolean"},
|
|
},
|
|
}
|
|
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
|
def.Validation.SchemaPath = "schema.json"
|
|
reader := defaultArtifactReader()
|
|
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{
|
|
SessionID: "prepared-session",
|
|
Messages: []domain.RenderedMessage{{
|
|
Role: "user",
|
|
Content: "original message",
|
|
}},
|
|
}}
|
|
llmClient := &fakeLLM{forbid: true}
|
|
validator := &recordingValidationPreparer{
|
|
plan: &recordingPreparedValidation{schemaDocument: schemaDocument},
|
|
}
|
|
admitter := &fakeRunAdmitter{}
|
|
profile := defaultExecutionProfile()
|
|
profile.ExtraParams = map[string]any{
|
|
"metadata": map[string]any{"source": "original"},
|
|
}
|
|
runner := NewRunner(
|
|
&fakePromptRepo{def: def},
|
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
|
nil,
|
|
reader,
|
|
renderer,
|
|
llmClient,
|
|
validator,
|
|
admitter,
|
|
)
|
|
|
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
|
PromptID: "p",
|
|
ProfileID: "exec",
|
|
APIKey: "direct-test-key",
|
|
Inputs: singleInputRef(),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("prepare execution: %v", err)
|
|
}
|
|
defer prepared.Discard()
|
|
|
|
if validator.prepareCalls != 1 || validator.directValidateCalls != 0 {
|
|
t.Fatalf(
|
|
"validation calls=(prepare=%d direct=%d), want (1, 0)",
|
|
validator.prepareCalls,
|
|
validator.directValidateCalls,
|
|
)
|
|
}
|
|
if reader.calls != 1 || renderer.calls != 1 {
|
|
t.Fatalf("completion calls=(artifact=%d render=%d), want (1, 1)", reader.calls, renderer.calls)
|
|
}
|
|
if len(admitter.backendIDs) != 0 || llmClient.calls != 0 {
|
|
t.Fatalf("prepare invoked execution collaborators: admission=%v generation=%d", admitter.backendIDs, llmClient.calls)
|
|
}
|
|
|
|
first := prepared.Details()
|
|
if first == nil {
|
|
t.Fatal("prepared details are nil")
|
|
}
|
|
if first.EffectiveModelParams.APIKey != "" {
|
|
t.Fatal("prepared details retained the direct API key")
|
|
}
|
|
if first.StructuredOutput == nil ||
|
|
first.StructuredOutput.JSONSchema == nil ||
|
|
!reflect.DeepEqual(first.StructuredOutput.JSONSchema.Schema, schemaDocument) {
|
|
t.Fatalf("prepared details have unexpected structured output: %#v", first.StructuredOutput)
|
|
}
|
|
|
|
first.Messages[0].Content = "caller mutation"
|
|
first.InputHashes["input"] = "caller mutation"
|
|
first.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] = "caller mutation"
|
|
first.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] = "string"
|
|
renderer.rendered.Messages[0].Content = "source mutation"
|
|
|
|
second := prepared.Details()
|
|
if second.Messages[0].Content != "original message" ||
|
|
second.InputHashes["input"] == "caller mutation" ||
|
|
second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" ||
|
|
second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" {
|
|
t.Fatalf("details did not preserve an independent snapshot: %#v", second)
|
|
}
|
|
}
|
|
|
|
func TestRunnerPreparationRejectsExcessivelyDeepPreparedSchema(t *testing.T) {
|
|
operations := []struct {
|
|
name string
|
|
run func(*Runner, domain.RunRequest) error
|
|
}{
|
|
{
|
|
name: "Prepare",
|
|
run: func(runner *Runner, request domain.RunRequest) error {
|
|
_, err := runner.Prepare(context.Background(), request)
|
|
return err
|
|
},
|
|
},
|
|
{
|
|
name: "Run",
|
|
run: func(runner *Runner, request domain.RunRequest) error {
|
|
_, err := runner.Run(context.Background(), request)
|
|
return err
|
|
},
|
|
},
|
|
{
|
|
name: "PrepareExecution",
|
|
run: func(runner *Runner, request domain.RunRequest) error {
|
|
_, err := runner.PrepareExecution(context.Background(), request)
|
|
return err
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, operation := range operations {
|
|
t.Run(operation.name, func(t *testing.T) {
|
|
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
|
def.Validation.SchemaPath = "schema.json"
|
|
llmClient := &fakeLLM{forbid: true}
|
|
validator := &recordingValidationPreparer{
|
|
plan: &recordingPreparedValidation{schemaDocument: excessivelyDeepPreparedJSONValue()},
|
|
}
|
|
runner := NewRunner(
|
|
&fakePromptRepo{def: def},
|
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
|
nil,
|
|
defaultArtifactReader(),
|
|
defaultRenderer(),
|
|
llmClient,
|
|
validator,
|
|
nil,
|
|
)
|
|
|
|
err := operation.run(runner, domain.RunRequest{
|
|
PromptID: "p",
|
|
ProfileID: "exec",
|
|
Inputs: singleInputRef(),
|
|
})
|
|
if !errors.Is(err, ErrValidation) {
|
|
t.Fatalf("expected ErrValidation, got %v", err)
|
|
}
|
|
if llmClient.calls != 0 {
|
|
t.Fatalf("invalid prepared schema reached generation: %d calls", llmClient.calls)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func excessivelyDeepPreparedJSONValue() any {
|
|
const clearlyUnsafeContainerDepth = 1_000
|
|
var value any = true
|
|
for level := 0; level < clearlyUnsafeContainerDepth; level++ {
|
|
value = map[string]any{"child": value}
|
|
}
|
|
return value
|
|
}
|
|
|
|
func TestRunnerRunPreparedRechecksEnvironmentCredentialBeforeAdmission(t *testing.T) {
|
|
const environmentName = "PROMPTKIT_PREPARED_EXECUTION_TEST_KEY"
|
|
t.Setenv(environmentName, "available-during-preparation")
|
|
|
|
profile := defaultExecutionProfile()
|
|
profile.APIKeyEnv = environmentName
|
|
validator := &recordingValidationPreparer{plan: &recordingPreparedValidation{}}
|
|
admitter := &fakeRunAdmitter{}
|
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "unexpected"}}
|
|
runner := NewRunner(
|
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
|
nil,
|
|
defaultArtifactReader(),
|
|
defaultRenderer(),
|
|
llmClient,
|
|
validator,
|
|
admitter,
|
|
)
|
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
|
PromptID: "p",
|
|
ProfileID: "exec",
|
|
Inputs: singleInputRef(),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("prepare execution: %v", err)
|
|
}
|
|
if err := os.Unsetenv(environmentName); err != nil {
|
|
t.Fatalf("unset credential environment: %v", err)
|
|
}
|
|
|
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
|
if result != nil {
|
|
t.Fatalf("credential failure returned partial result: %+v", result)
|
|
}
|
|
if !errors.Is(err, ErrInvalidRequest) || !errors.Is(err, ErrAPIKeyEnvMissing) {
|
|
t.Fatalf("credential error identities are missing: %v", err)
|
|
}
|
|
if len(admitter.backendIDs) != 0 || llmClient.calls != 0 {
|
|
t.Fatalf("credential failure reached admission or generation: admission=%v generation=%d", admitter.backendIDs, llmClient.calls)
|
|
}
|
|
if _, err := runner.RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
|
|
t.Fatalf("credential failure did not consume execution: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestRunnerRunPreparedKeepsDirectCredentialOutOfMetadata(t *testing.T) {
|
|
const directKey = "direct-prepared-test-key"
|
|
|
|
profile := defaultExecutionProfile()
|
|
profile.APIKeyRequired = true
|
|
client := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
|
runner := NewRunner(
|
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
|
nil,
|
|
defaultArtifactReader(),
|
|
defaultRenderer(),
|
|
client,
|
|
&recordingValidationPreparer{plan: &recordingPreparedValidation{}},
|
|
nil,
|
|
)
|
|
|
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
|
PromptID: "p",
|
|
ProfileID: "exec",
|
|
APIKey: directKey,
|
|
Inputs: singleInputRef(),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("prepare execution: %v", err)
|
|
}
|
|
if details := prepared.Details(); details.EffectiveModelParams.APIKey != "" {
|
|
t.Fatal("prepared details retained direct credential")
|
|
}
|
|
|
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
|
if err != nil {
|
|
t.Fatalf("run prepared: %v", err)
|
|
}
|
|
if client.lastReq.Target.APIKey != directKey {
|
|
t.Fatal("generation did not receive direct credential")
|
|
}
|
|
if result.EffectiveModelParams.APIKey != "" {
|
|
t.Fatal("run result retained direct credential")
|
|
}
|
|
}
|
|
|
|
func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *testing.T) {
|
|
validator := &recordingValidationPreparer{
|
|
plan: &recordingPreparedValidation{
|
|
results: []domain.ValidationResult{
|
|
{
|
|
Status: domain.ValidationFailed,
|
|
Mode: domain.ValidationJSON,
|
|
Errors: []string{"invalid"},
|
|
IsValid: false,
|
|
},
|
|
{
|
|
Status: domain.ValidationPassed,
|
|
Mode: domain.ValidationJSON,
|
|
IsValid: true,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
repairer := &fakeRepairer{
|
|
responses: []*domain.GenerateResponse{{
|
|
Content: `{"repaired":true}`,
|
|
Usage: domain.TokenUsage{
|
|
PromptTokens: 2, CompletionTokens: 3, TotalTokens: 5,
|
|
CachedTokens: 7, CacheWriteTokens: 11,
|
|
},
|
|
}},
|
|
}
|
|
admitter := &fakeRunAdmitter{}
|
|
reader := defaultArtifactReader()
|
|
renderer := defaultRenderer()
|
|
runner := NewRunnerWithRepairer(
|
|
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
|
nil,
|
|
reader,
|
|
renderer,
|
|
&fakeLLM{resp: &domain.GenerateResponse{
|
|
Content: `{"broken":true}`,
|
|
Usage: domain.TokenUsage{
|
|
PromptTokens: 13, CompletionTokens: 17, TotalTokens: 19,
|
|
CachedTokens: 23, CacheWriteTokens: 29,
|
|
},
|
|
}},
|
|
validator,
|
|
repairer,
|
|
admitter,
|
|
)
|
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
|
PromptID: "p",
|
|
ProfileID: "exec",
|
|
Inputs: singleInputRef(),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("prepare execution: %v", err)
|
|
}
|
|
|
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
|
if err != nil {
|
|
t.Fatalf("run prepared: %v", err)
|
|
}
|
|
if !reflect.DeepEqual(validator.plan.artifacts, []string{`{"broken":true}`, `{"repaired":true}`}) {
|
|
t.Fatalf("prepared validation artifacts=%#v", validator.plan.artifacts)
|
|
}
|
|
if validator.directValidateCalls != 0 || repairer.calls != 1 {
|
|
t.Fatalf("validation/repair calls=(direct=%d repair=%d), want (0, 1)", validator.directValidateCalls, repairer.calls)
|
|
}
|
|
if result.Validation.Status != domain.ValidationPassed || result.Validation.RepairAttempts != 1 {
|
|
t.Fatalf("unexpected repaired validation result: %+v", result.Validation)
|
|
}
|
|
wantUsage := domain.TokenUsage{
|
|
PromptTokens: 15, CompletionTokens: 20, TotalTokens: 24,
|
|
CachedTokens: 30, CacheWriteTokens: 40,
|
|
}
|
|
if result.Usage != wantUsage {
|
|
t.Fatalf("prepared cumulative usage = %+v, want %+v", result.Usage, wantUsage)
|
|
}
|
|
if admitter.releaseCalls != 1 {
|
|
t.Fatalf("admission releases=%d, want 1", admitter.releaseCalls)
|
|
}
|
|
if reader.calls != 1 || renderer.calls != 1 {
|
|
t.Fatalf("execution reopened preparation sources: artifact=%d render=%d", reader.calls, renderer.calls)
|
|
}
|
|
}
|
|
|
|
func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
|
|
generationFailure := errors.New("generation failed")
|
|
validationFailure := errors.New("validation failed")
|
|
repairFailure := errors.New("repair failed")
|
|
|
|
tests := []struct {
|
|
name string
|
|
generationErr error
|
|
validation *recordingPreparedValidation
|
|
repairer *fakeRepairer
|
|
wantError error
|
|
}{
|
|
{
|
|
name: "generation failure",
|
|
generationErr: generationFailure,
|
|
validation: &recordingPreparedValidation{},
|
|
wantError: ErrLLMGenerate,
|
|
},
|
|
{
|
|
name: "validation failure",
|
|
validation: &recordingPreparedValidation{
|
|
errs: []error{validationFailure},
|
|
},
|
|
wantError: ErrValidation,
|
|
},
|
|
{
|
|
name: "repair failure",
|
|
validation: &recordingPreparedValidation{
|
|
results: []domain.ValidationResult{{
|
|
Status: domain.ValidationFailed,
|
|
Mode: domain.ValidationJSON,
|
|
Errors: []string{"invalid"},
|
|
IsValid: false,
|
|
}},
|
|
},
|
|
repairer: &fakeRepairer{err: repairFailure},
|
|
wantError: ErrValidation,
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
def := promptDef(domain.FormatJSON, domain.ValidationJSON, 1)
|
|
if test.repairer == nil {
|
|
def.Validation.RepairAttempts = 0
|
|
}
|
|
validator := &recordingValidationPreparer{plan: test.validation}
|
|
admitter := &fakeRunAdmitter{}
|
|
runner := NewRunnerWithRepairer(
|
|
&fakePromptRepo{def: def},
|
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
|
nil,
|
|
defaultArtifactReader(),
|
|
defaultRenderer(),
|
|
&fakeLLM{
|
|
resp: &domain.GenerateResponse{Content: `{"value":true}`},
|
|
err: test.generationErr,
|
|
},
|
|
validator,
|
|
test.repairer,
|
|
admitter,
|
|
)
|
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
|
PromptID: "p",
|
|
ProfileID: "exec",
|
|
Inputs: singleInputRef(),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("prepare execution: %v", err)
|
|
}
|
|
|
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
|
if result != nil || !errors.Is(err, test.wantError) {
|
|
t.Fatalf("run prepared=(%+v, %v), want %v", result, err, test.wantError)
|
|
}
|
|
if len(admitter.backendIDs) != 1 || admitter.releaseCalls != 1 {
|
|
t.Fatalf(
|
|
"admission calls=%#v releases=%d, want one each",
|
|
admitter.backendIDs,
|
|
admitter.releaseCalls,
|
|
)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
|
|
newRunner := func() *Runner {
|
|
return NewRunner(
|
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
|
nil,
|
|
defaultArtifactReader(),
|
|
defaultRenderer(),
|
|
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
|
&recordingValidationPreparer{plan: &recordingPreparedValidation{}},
|
|
nil,
|
|
)
|
|
}
|
|
request := domain.RunRequest{
|
|
PromptID: "p",
|
|
ProfileID: "exec",
|
|
Inputs: singleInputRef(),
|
|
}
|
|
|
|
owner := newRunner()
|
|
prepared, err := owner.PrepareExecution(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatalf("prepare execution: %v", err)
|
|
}
|
|
if _, err := newRunner().RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
|
|
t.Fatalf("foreign runner error=%v, want ErrInvalidRequest", err)
|
|
}
|
|
if _, err := owner.RunPrepared(context.Background(), prepared); err != nil {
|
|
t.Fatalf("owner run prepared: %v", err)
|
|
}
|
|
if _, err := owner.RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
|
|
t.Fatalf("second owner run error=%v, want ErrInvalidRequest", err)
|
|
}
|
|
if prepared.Details() == nil {
|
|
t.Fatal("details unavailable after execution")
|
|
}
|
|
|
|
discarded, err := owner.PrepareExecution(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatalf("prepare discarded execution: %v", err)
|
|
}
|
|
discarded.Discard()
|
|
discarded.Discard()
|
|
if _, err := owner.RunPrepared(context.Background(), discarded); !errors.Is(err, ErrInvalidRequest) {
|
|
t.Fatalf("discarded execution error=%v, want ErrInvalidRequest", err)
|
|
}
|
|
if discarded.Details() == nil {
|
|
t.Fatal("details unavailable after discard")
|
|
}
|
|
}
|
|
|
|
func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
|
|
runner := NewRunner(
|
|
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)},
|
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
|
nil,
|
|
defaultArtifactReader(),
|
|
defaultRenderer(),
|
|
&fakeLLM{resp: &domain.GenerateResponse{Content: `{}`}},
|
|
nil,
|
|
nil,
|
|
)
|
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
|
PromptID: "p",
|
|
ProfileID: "exec",
|
|
Inputs: singleInputRef(),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("prepare execution: %v", err)
|
|
}
|
|
|
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
|
if err != nil {
|
|
t.Fatalf("run prepared: %v", err)
|
|
}
|
|
if result.Validation.Status != domain.ValidationSkipped || !result.Validation.IsValid {
|
|
t.Fatalf("unexpected no-validator result: %+v", result.Validation)
|
|
}
|
|
}
|
|
|
|
func TestRunnerPrepareExecutionRequiresValidationPreparer(t *testing.T) {
|
|
reader := defaultArtifactReader()
|
|
runner := NewRunner(
|
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
|
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
|
nil,
|
|
reader,
|
|
defaultRenderer(),
|
|
&fakeLLM{forbid: true},
|
|
validationOnly{},
|
|
nil,
|
|
)
|
|
|
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
|
PromptID: "p",
|
|
ProfileID: "exec",
|
|
Inputs: singleInputRef(),
|
|
})
|
|
if prepared != nil || !errors.Is(err, ErrValidation) {
|
|
t.Fatalf("prepare execution=(%+v, %v), want ErrValidation", prepared, err)
|
|
}
|
|
if reader.calls != 0 {
|
|
t.Fatalf("unsupported validator allowed completion, artifact calls=%d", reader.calls)
|
|
}
|
|
}
|