eric/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Run bounded output repair attempts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-25 09:51:28 +00:00
parent 00ee5893e9
commit ee99dc9478
4 changed files with 109 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
internal/usecase/runner_test.go
View File

@@ -2143,6 +2143,8 @@ func TestRunnerRepairStateMachine(t *testing.T) {
TopP: true,
TimeoutSeconds: true,
}
emptyThenValid := responses(2)
emptyThenValid[0].Content = " \t "
tests := []struct {
name string
@@ -2165,12 +2167,13 @@ func TestRunnerRepairStateMachine(t *testing.T) {
wantStatus: domain.ValidationPassed,
},
{
name: "basic failure is ineligible despite budget",
name: "empty basic output repairs successfully",
mode: domain.ValidationBasic,
budget: 3,
validationResults: []domain.ValidationResult{failed(domain.ValidationBasic, "empty output")},
responses: responses(1),
wantStatus: domain.ValidationFailed,
validationResults: []domain.ValidationResult{failed(domain.ValidationBasic, "empty output"), passed(domain.ValidationBasic)},
responses: emptyThenValid,
wantRepairs: 1,
wantStatus: domain.ValidationPassed,
},
{
name: "inherited numeric values remain absent",
@@ -2305,6 +2308,9 @@ func TestRunnerRepairStateMachine(t *testing.T) {
!reflect.DeepEqual(req.ValidationErrors, tc.validationResults[index].Errors) {
t.Fatalf("repair request %d prior state = %+v", index, req)
}
if !reflect.DeepEqual(req.OriginalMessages, initialRequest.Prompt.Messages) {
t.Fatalf("repair request %d original messages drifted: %#v", index, req.OriginalMessages)
}
if req.TargetPresence != tc.wantPresence || !reflect.DeepEqual(req.Target, initialRequest.Target) ||
req.SessionID != initialRequest.Prompt.SessionID ||
!reflect.DeepEqual(req.StructuredOutput, initialRequest.StructuredOutput) {
@@ -2318,8 +2324,19 @@ func TestRunnerRepairStateMachine(t *testing.T) {
!reflect.DeepEqual(generated.StructuredOutput, initialRequest.StructuredOutput) {
t.Fatalf("repair generation request %d common fields drifted: %+v", index, generated)
}
if reflect.DeepEqual(generated.Prompt.Messages, initialRequest.Prompt.Messages) {
t.Fatalf("repair generation request %d reused the initial prompt", index)
expectedMessages := len(initialRequest.Prompt.Messages) + 1
if strings.TrimSpace(tc.responses[index].Content) != "" {
expectedMessages++
}
if len(generated.Prompt.Messages) != expectedMessages ||
generated.Prompt.Messages[len(generated.Prompt.Messages)-1].Role != "user" {
t.Fatalf("repair generation request %d messages = %#v", index, generated.Prompt.Messages)
}
if strings.TrimSpace(tc.responses[index].Content) != "" {
assistant := generated.Prompt.Messages[len(generated.Prompt.Messages)-2]
if assistant.Role != "assistant" || assistant.Content != tc.responses[index].Content {
t.Fatalf("repair generation request %d candidate = %+v", index, assistant)
}
}
}