Enforce bounded output repair contracts
This commit is contained in:
@@ -22,27 +22,52 @@ func TestPreparationRejectsInvalidOutputContractWithPublicError(t *testing.T) {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
|
||||
req := promptkit.RunRequest{
|
||||
PromptID: "prompt",
|
||||
Validation: &promptkit.OutputContract{
|
||||
Format: promptkit.OutputFormat("binary"),
|
||||
ValidationMode: promptkit.ValidationNone,
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
contract promptkit.OutputContract
|
||||
}{
|
||||
{
|
||||
name: "unsupported format",
|
||||
contract: promptkit.OutputContract{
|
||||
Format: promptkit.OutputFormat("binary"),
|
||||
ValidationMode: promptkit.ValidationNone,
|
||||
},
|
||||
},
|
||||
}
|
||||
{
|
||||
name: "repair attempts above maximum",
|
||||
contract: promptkit.OutputContract{
|
||||
Format: promptkit.FormatText,
|
||||
ValidationMode: promptkit.ValidationBasic,
|
||||
RepairAttempts: 4,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "none validation with repair attempts",
|
||||
contract: promptkit.OutputContract{
|
||||
Format: promptkit.FormatText,
|
||||
ValidationMode: promptkit.ValidationNone,
|
||||
RepairAttempts: 1,
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
req := promptkit.RunRequest{PromptID: "prompt", Validation: &tc.contract}
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), req)
|
||||
if prepared != nil {
|
||||
t.Fatalf("expected no partial prepared run, got %+v", prepared)
|
||||
}
|
||||
if !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("prepare error = %v, want ErrInvalidRequest", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), req)
|
||||
if prepared != nil {
|
||||
t.Fatalf("expected no partial prepared run, got %+v", prepared)
|
||||
}
|
||||
if !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("prepare error = %v, want ErrInvalidRequest", err)
|
||||
}
|
||||
|
||||
preparedExecution, err := engine.PrepareExecution(context.Background(), req)
|
||||
if preparedExecution != nil {
|
||||
t.Fatalf("expected no partial prepared execution, got %+v", preparedExecution)
|
||||
}
|
||||
if !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("prepare execution error = %v, want ErrInvalidRequest", err)
|
||||
preparedExecution, err := engine.PrepareExecution(context.Background(), req)
|
||||
if preparedExecution != nil {
|
||||
t.Fatalf("expected no partial prepared execution, got %+v", preparedExecution)
|
||||
}
|
||||
if !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("prepare execution error = %v, want ErrInvalidRequest", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user