eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Hardened the proposal modules to skip malformed proposals rather than hard failing the entire run
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
Details

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-17 07:19:24 -05:00
parent a84941d681
commit 9c2d8338d7
17 changed files with 436 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
internal/framework/runner/runner_test.go
View File

@@ -1054,7 +1054,7 @@ func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
}
}
func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *testing.T) {
func TestRunnerValidatorCardinalityErrorSoftRejectsAndContinues(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
good := fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
@@ -1069,14 +1069,17 @@ func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *test
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m1": good, "m2": bad}})
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}})
if err == nil {
t.Fatal("expected cardinality error")
if err != nil {
t.Fatalf("unexpected run error: %v", err)
}
if out.FinalTranscript.Segments[0].Text != "the cat" {
t.Fatalf("expected partial progress preserved, got %q", out.FinalTranscript.Segments[0].Text)
t.Fatalf("expected second-module proposal to be rejected, got %q", out.FinalTranscript.Segments[0].Text)
}
if len(out.ModuleResults) != 2 || out.ModuleResults[1].Status != ModuleStatusFailed {
t.Fatalf("expected second module failed")
if len(out.ModuleResults) != 2 || out.ModuleResults[1].Status != ModuleStatusSuccess {
t.Fatalf("expected second module success with rejection, got %+v", out.ModuleResults)
}
if len(out.ModuleResults[1].ValidatorRejected) == 0 {
t.Fatalf("expected validator rejection record, got %+v", out.ModuleResults[1])
}
}
@@ -1191,7 +1194,7 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
}
}
func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing.T) {
func TestRunnerLLMValidatorMalformedResponseSoftRejectsWithProgress(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "bad index"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
@@ -1209,15 +1212,21 @@ func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing.
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}},
ValidationLLMClient: client,
})
if err == nil {
t.Fatal("expected llm validator failure")
if err != nil {
t.Fatalf("unexpected run error: %v", err)
}
if out.FinalTranscript.Segments[0].Text != "the cat" {
t.Fatalf("expected partial progress retained")
}
if len(out.ModuleResults) != 2 || out.ModuleResults[1].Status != ModuleStatusSuccess {
t.Fatalf("expected second module success, got %+v", out.ModuleResults)
}
if len(out.ModuleResults[1].ValidatorRejected) == 0 {
t.Fatalf("expected llm validator rejection record")
}
}
func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) {
func TestRunnerLLMValidatorMissingDecisionSoftRejects(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
@@ -1226,18 +1235,21 @@ func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) {
}},
}})
cfg := config.Default()
_, err := r.Run(context.Background(), RunInput{
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
})
if err == nil {
t.Fatal("expected missing decision failure")
if err != nil {
t.Fatalf("unexpected run error: %v", err)
}
if len(out.ModuleResults[0].ValidatorRejected) != 1 {
t.Fatalf("expected soft rejection, got %+v", out.ModuleResults[0].ValidatorRejected)
}
}
func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) {
func TestRunnerLLMValidatorDuplicateDecisionSoftRejects(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
@@ -1249,14 +1261,17 @@ func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) {
}},
}})
cfg := config.Default()
_, err := r.Run(context.Background(), RunInput{
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
})
if err == nil {
t.Fatal("expected duplicate decision failure")
if err != nil {
t.Fatalf("unexpected run error: %v", err)
}
if len(out.ModuleResults[0].ValidatorRejected) != 1 {
t.Fatalf("expected soft rejection, got %+v", out.ModuleResults[0].ValidatorRejected)
}
}