262 lines
10 KiB
Go
262 lines
10 KiB
Go
package usecase
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
|
|
"go.maximumdirect.net/promptkit/internal/domain"
|
|
"go.maximumdirect.net/promptkit/internal/validate"
|
|
)
|
|
|
|
type outputContractTestCollaborators struct {
|
|
artifacts *fakeArtifactReader
|
|
renderer *fakeRenderer
|
|
llm *fakeLLM
|
|
validator *recordingValidationPreparer
|
|
admitter *fakeRunAdmitter
|
|
}
|
|
|
|
func newOutputContractTestRunner() (*Runner, outputContractTestCollaborators) {
|
|
collaborators := outputContractTestCollaborators{
|
|
artifacts: defaultArtifactReader(),
|
|
renderer: defaultRenderer(),
|
|
llm: &fakeLLM{forbid: true},
|
|
validator: &recordingValidationPreparer{plan: &recordingPreparedValidation{}},
|
|
admitter: &fakeRunAdmitter{},
|
|
}
|
|
return NewRunner(
|
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
|
|
|
collaborators.artifacts,
|
|
collaborators.renderer,
|
|
collaborators.llm,
|
|
collaborators.validator,
|
|
collaborators.admitter,
|
|
), collaborators
|
|
}
|
|
|
|
func TestResolveInlineSchemaOwnsRuntimeValue(t *testing.T) {
|
|
value := map[string]any{"properties": map[string]any{"name": map[string]any{"type": "string"}}}
|
|
override := domain.OutputContractInput{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, Schema: value}
|
|
contract, err := resolveOutputContract(context.Background(), &domain.PromptDefinition{}, &override)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
value["properties"].(map[string]any)["name"] = false
|
|
if _, ok := contract.Schema.(map[string]any)["properties"].(map[string]any)["name"].(map[string]any); !ok {
|
|
t.Fatal("schema shares caller storage")
|
|
}
|
|
var null map[string]any
|
|
override.Schema = null
|
|
override.SchemaPath = "schema.json"
|
|
contract, err = resolveOutputContract(context.Background(), &domain.PromptDefinition{}, &override)
|
|
if err != nil || contract.Schema != nil {
|
|
t.Fatalf("typed null selection: %#v, %v", contract.Schema, err)
|
|
}
|
|
}
|
|
|
|
func TestRunnerPreparationNormalizesOutputContractConsistently(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
override domain.OutputContractInput
|
|
want domain.OutputContract
|
|
}{
|
|
{
|
|
name: "empty replacement format defaults to text",
|
|
override: domain.OutputContractInput{ValidationMode: domain.ValidationNone},
|
|
want: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone},
|
|
},
|
|
{
|
|
name: "markdown basic replacement",
|
|
override: domain.OutputContractInput{Format: domain.FormatMarkdown, ValidationMode: domain.ValidationBasic},
|
|
want: domain.OutputContract{Format: domain.FormatMarkdown, ValidationMode: domain.ValidationBasic, RepairAttempts: 1},
|
|
},
|
|
{
|
|
name: "json replacement preserves non-schema fields",
|
|
override: domain.OutputContractInput{
|
|
Format: domain.FormatJSON,
|
|
ValidationMode: domain.ValidationJSON,
|
|
SchemaPath: "ignored.json",
|
|
RepairAttempts: repairBudget(2),
|
|
},
|
|
want: domain.OutputContract{
|
|
Format: domain.FormatJSON,
|
|
ValidationMode: domain.ValidationJSON,
|
|
SchemaPath: "ignored.json",
|
|
RepairAttempts: 2,
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
runner, _ := newOutputContractTestRunner()
|
|
req := domain.RunRequest{
|
|
PromptID: "p",
|
|
ProfileID: "exec",
|
|
Inputs: singleInputRef(),
|
|
Validation: &tt.override,
|
|
}
|
|
|
|
prepared, err := runner.Prepare(context.Background(), req)
|
|
if err != nil {
|
|
t.Fatalf("prepare: %v", err)
|
|
}
|
|
preparedExecution, err := runner.PrepareExecution(context.Background(), req)
|
|
if err != nil {
|
|
t.Fatalf("prepare execution: %v", err)
|
|
}
|
|
details := preparedExecution.Details()
|
|
if details == nil {
|
|
t.Fatal("prepared execution returned nil details")
|
|
}
|
|
if prepared.OutputContract != tt.want || details.OutputContract != tt.want {
|
|
t.Fatalf("output contracts = (%+v, %+v), want %+v", prepared.OutputContract, details.OutputContract, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunnerPreparationRejectsInvalidOutputContractsBeforeCompletion(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
override domain.OutputContractInput
|
|
}{
|
|
{name: "unsupported format", override: domain.OutputContractInput{Format: "binary", ValidationMode: domain.ValidationNone}},
|
|
{name: "empty validation mode", override: domain.OutputContractInput{Format: domain.FormatText}},
|
|
{name: "negative repair attempts", override: domain.OutputContractInput{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: repairBudget(-1)}},
|
|
{name: "repair attempts above maximum", override: domain.OutputContractInput{Format: domain.FormatText, ValidationMode: domain.ValidationBasic, RepairAttempts: repairBudget(4)}},
|
|
{name: "none validation with repair attempts", override: domain.OutputContractInput{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: repairBudget(1)}},
|
|
{name: "json schema without path", override: domain.OutputContractInput{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema}},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
for _, operation := range []string{"Prepare", "PrepareExecution"} {
|
|
t.Run(tt.name+"/"+operation, func(t *testing.T) {
|
|
runner, collaborators := newOutputContractTestRunner()
|
|
req := domain.RunRequest{
|
|
PromptID: "p",
|
|
ProfileID: "exec",
|
|
Inputs: singleInputRef(),
|
|
Validation: &tt.override,
|
|
}
|
|
|
|
var err error
|
|
switch operation {
|
|
case "Prepare":
|
|
var prepared *domain.PreparedRun
|
|
prepared, err = runner.Prepare(context.Background(), req)
|
|
if prepared != nil {
|
|
t.Fatalf("expected no partial prepared run, got %+v", prepared)
|
|
}
|
|
case "PrepareExecution":
|
|
var prepared *PreparedExecution
|
|
prepared, err = runner.PrepareExecution(context.Background(), req)
|
|
if prepared != nil {
|
|
t.Fatalf("expected no partial prepared execution, got %+v", prepared)
|
|
}
|
|
default:
|
|
t.Fatalf("unknown operation %q", operation)
|
|
}
|
|
|
|
if !errors.Is(err, ErrInvalidRequest) {
|
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
|
}
|
|
assertOutputContractCompletionSkipped(t, collaborators)
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunnerRunRejectsInvalidOutputContractBeforeAdmission(t *testing.T) {
|
|
runner, collaborators := newOutputContractTestRunner()
|
|
result, err := runner.Run(context.Background(), domain.RunRequest{
|
|
PromptID: "p",
|
|
ProfileID: "exec",
|
|
Inputs: singleInputRef(),
|
|
Validation: &domain.OutputContractInput{
|
|
Format: domain.OutputFormat("binary"),
|
|
ValidationMode: domain.ValidationNone,
|
|
},
|
|
})
|
|
if result != nil {
|
|
t.Fatalf("expected no partial result, got %+v", result)
|
|
}
|
|
if !errors.Is(err, ErrInvalidRequest) {
|
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
|
}
|
|
assertOutputContractCompletionSkipped(t, collaborators)
|
|
}
|
|
|
|
func repairBudget(value int) *int { return &value }
|
|
|
|
func TestReplacementRepairExecution(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
budget *int
|
|
wantCalls int
|
|
wantStatus domain.ValidationStatus
|
|
}{
|
|
{"default", nil, 2, domain.ValidationPassed},
|
|
{"explicit zero", repairBudget(0), 1, domain.ValidationFailed},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
client := &sequenceLLM{responses: []*domain.GenerateResponse{{Content: "not json"}, {Content: `{"ok":true}`}}}
|
|
runner := NewRunnerWithRepairer(&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
|
defaultArtifactReader(), defaultRenderer(), client, validate.NewValidator(), NewDefaultOutputRepairer(client), nil)
|
|
input := &domain.OutputContractInput{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON, RepairAttempts: tc.budget}
|
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), Validation: input})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
input.ValidationMode = domain.ValidationNone
|
|
input.RepairAttempts = repairBudget(3)
|
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
|
if err != nil || result.Validation.Status != tc.wantStatus || len(client.requests) != tc.wantCalls || result.Validation.RepairAttempts != tc.wantCalls-1 {
|
|
t.Fatalf("result = %+v, calls = %d, error = %v", result, len(client.requests), err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestResolveReplacementRepairBudget(t *testing.T) {
|
|
configured := &domain.PromptDefinition{Validation: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationBasic, RepairAttempts: 3}}
|
|
for _, tc := range []struct {
|
|
name string
|
|
input domain.OutputContractInput
|
|
want int
|
|
}{
|
|
{"none", domain.OutputContractInput{ValidationMode: domain.ValidationNone}, 0},
|
|
{"basic", domain.OutputContractInput{ValidationMode: domain.ValidationBasic}, 1},
|
|
{"json", domain.OutputContractInput{ValidationMode: domain.ValidationJSON}, 1},
|
|
{"schema", domain.OutputContractInput{ValidationMode: domain.ValidationJSONSchema, Schema: true}, 1},
|
|
{"explicit zero", domain.OutputContractInput{ValidationMode: domain.ValidationBasic, RepairAttempts: repairBudget(0)}, 0},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got, err := resolveOutputContract(context.Background(), configured, &tc.input)
|
|
if err != nil || got.RepairAttempts != tc.want {
|
|
t.Fatalf("contract = %+v, %v", got, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func assertOutputContractCompletionSkipped(t *testing.T, collaborators outputContractTestCollaborators) {
|
|
t.Helper()
|
|
if collaborators.artifacts.calls != 0 || collaborators.renderer.calls != 0 ||
|
|
collaborators.validator.prepareCalls != 0 ||
|
|
len(collaborators.admitter.backendIDs) != 0 || collaborators.llm.calls != 0 {
|
|
t.Fatalf(
|
|
"invalid output contract reached downstream work: artifacts=%d renderer=%d prepare_validation=%d admissions=%d generation=%d",
|
|
collaborators.artifacts.calls,
|
|
collaborators.renderer.calls,
|
|
collaborators.validator.prepareCalls,
|
|
len(collaborators.admitter.backendIDs),
|
|
collaborators.llm.calls,
|
|
)
|
|
}
|
|
}
|