diff --git a/cmd/audita/main_integration_test.go b/cmd/audita/main_integration_test.go index 0ac5d11..fcb667c 100644 --- a/cmd/audita/main_integration_test.go +++ b/cmd/audita/main_integration_test.go @@ -364,7 +364,7 @@ func TestProcessSuccessLargeTranscriptSubprocess(t *testing.T) { } } -func TestProcessFailureMalformedStructuredLLMResponseViaSubprocessHook(t *testing.T) { +func TestProcessMalformedStructuredLLMResponseViaSubprocessHookSoftSkips(t *testing.T) { workDir := t.TempDir() reportPath := filepath.Join(t.TempDir(), "report.json") result := runCLISubprocessWithEnv(t, @@ -382,25 +382,34 @@ func TestProcessFailureMalformedStructuredLLMResponseViaSubprocessHook(t *testin "--work-dir-retention", "always", ) - if result.exitCode == 0 { - t.Fatalf("expected nonzero exit code") + if result.exitCode != 0 { + t.Fatalf("expected success exit code, got %d stderr=%q", result.exitCode, result.stderr) } - if result.stdout != "" { - t.Fatalf("expected empty stdout on failure, got %q", result.stdout) + if !json.Valid([]byte(result.stdout)) { + t.Fatalf("expected transcript JSON on stdout, got %q", result.stdout) } - if !strings.Contains(result.stderr, "runner_execution") { - t.Fatalf("expected runner_execution failure, got %q", result.stderr) - } - if !strings.Contains(result.stderr, "diagnostics:") { - t.Fatalf("expected diagnostics path in stderr, got %q", result.stderr) + if result.stderr != "" { + t.Fatalf("expected quiet stderr on success, got %q", result.stderr) } report := readFile(t, reportPath) if !json.Valid(report) { - t.Fatalf("expected valid failure report JSON") + t.Fatalf("expected valid report JSON") } - runDir := onlyRunDir(t, workDir) - if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil { - t.Fatalf("expected error.log, got: %v", err) + var parsed struct { + Status string `json:"status"` + ModuleResults []struct { + Status string `json:"status"` + ValidatorRejected []any `json:"validator_rejected"` + } `json:"module_results"` + } + if err := json.Unmarshal(report, &parsed); err != nil { + t.Fatalf("unmarshal report: %v", err) + } + if parsed.Status != "success" { + t.Fatalf("expected success report status, got %+v", parsed) + } + if len(parsed.ModuleResults) == 0 || len(parsed.ModuleResults[0].ValidatorRejected) == 0 { + t.Fatalf("expected soft-skip validator rejection records, got %+v", parsed.ModuleResults) } } @@ -436,7 +445,7 @@ func TestProcessFailureBackendLLMViaSubprocessHook(t *testing.T) { } } -func TestProcessFailureMidPipelinePreservesPartialReportsSubprocess(t *testing.T) { +func TestProcessMidPipelineMalformedCorrectionsSoftSkipSubprocess(t *testing.T) { workDir := t.TempDir() reportPath := filepath.Join(t.TempDir(), "report.json") result := runCLISubprocessWithEnv(t, @@ -454,11 +463,11 @@ func TestProcessFailureMidPipelinePreservesPartialReportsSubprocess(t *testing.T "--work-dir-retention", "always", ) - if result.exitCode == 0 { - t.Fatalf("expected nonzero exit code") + if result.exitCode != 0 { + t.Fatalf("expected success exit code, got %d stderr=%q", result.exitCode, result.stderr) } - if result.stdout != "" { - t.Fatalf("expected empty stdout on failure, got %q", result.stdout) + if !json.Valid([]byte(result.stdout)) { + t.Fatalf("expected transcript JSON on stdout, got %q", result.stdout) } reportRaw := readFile(t, reportPath) var report struct { @@ -472,11 +481,11 @@ func TestProcessFailureMidPipelinePreservesPartialReportsSubprocess(t *testing.T if err := json.Unmarshal(reportRaw, &report); err != nil { t.Fatalf("unmarshal report: %v", err) } - if report.Status != "failed" || report.ErrorPhase != "runner_execution" { - t.Fatalf("expected failed runner_execution report, got %+v", report) + if report.Status != "success" || report.ErrorPhase != "" { + t.Fatalf("expected successful report with no runner_execution phase, got %+v", report) } if len(report.ModuleResults) == 0 { - t.Fatalf("expected partial module results in failure report") + t.Fatalf("expected module results in report") } } diff --git a/internal/framework/contracts/contracts.go b/internal/framework/contracts/contracts.go index 9fa4114..8b21c6e 100644 --- a/internal/framework/contracts/contracts.go +++ b/internal/framework/contracts/contracts.go @@ -101,6 +101,7 @@ type ProposalRequest struct { RunSpec ModuleRunSpec `json:"run_spec"` LLMClient StructuredLLMClient `json:"-"` LLMScheduler LLMScheduler `json:"-"` + OnDroppedCandidate func(index int, targetSegmentID int, originalText string, correctedText string, confidence float64, reasonCode string, message string) `json:"-"` } // ValidationRequest is the input to validator execution. diff --git a/internal/framework/proposal_generation/generate.go b/internal/framework/proposal_generation/generate.go index a9cdacd..da194ec 100644 --- a/internal/framework/proposal_generation/generate.go +++ b/internal/framework/proposal_generation/generate.go @@ -44,6 +44,16 @@ type StructuredCorrectionSet struct { Corrections []StructuredCorrectionProposal `json:"corrections"` } +type DroppedCandidate struct { + ProposalIndex int `json:"proposal_index"` + TargetSegmentID int `json:"target_segment_id"` + OriginalText string `json:"original_text"` + CorrectedText string `json:"corrected_text"` + Confidence float64 `json:"confidence"` + ReasonCode string `json:"reason_code"` + Message string `json:"message"` +} + // Request captures reusable proposal-generation inputs for future modules. type Request struct { ModuleKey string `json:"module_key"` @@ -62,12 +72,14 @@ type Request struct { Scheduler contracts.LLMScheduler DiagnosticsDir string DiagnosticsWriter InteractionDiagnosticsWriter + OnDroppedCandidate func(dropped DroppedCandidate) } // Result contains generated candidate proposals and optional diagnostics paths. type Result struct { Corrections []proposals.CorrectionProposal `json:"corrections"` Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"` + Dropped []DroppedCandidate `json:"dropped,omitempty"` Artifacts InteractionArtifacts `json:"artifacts,omitempty"` } @@ -161,7 +173,9 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections)) enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections)) + dropped := make([]DroppedCandidate, 0) for i, raw := range response.Corrections { + proposalIndex := req.StartIndex + i candidate := proposals.CorrectionProposal{ TargetSegmentID: raw.TargetSegmentID, OriginalText: raw.OriginalText, @@ -169,14 +183,27 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { Confidence: raw.Confidence, } if err := candidate.Validate(); err != nil { - return Result{}, fmt.Errorf("invalid structured correction at index %d: %w", i, err) + record := DroppedCandidate{ + ProposalIndex: proposalIndex, + TargetSegmentID: raw.TargetSegmentID, + OriginalText: raw.OriginalText, + CorrectedText: raw.CorrectedText, + Confidence: raw.Confidence, + ReasonCode: "invalid_structured_correction", + Message: fmt.Sprintf("invalid structured correction at index %d: %v", i, err), + } + dropped = append(dropped, record) + if req.OnDroppedCandidate != nil { + req.OnDroppedCandidate(record) + } + continue } corrections = append(corrections, candidate) enrichedCandidate := proposals.EnrichedCorrectionProposal{ CorrectionProposal: candidate, ProposalMetadata: proposals.ProposalMetadata{ - ProposalIndex: req.StartIndex + i, + ProposalIndex: proposalIndex, ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, }, @@ -191,6 +218,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { return Result{ Corrections: corrections, Enriched: enriched, + Dropped: dropped, Artifacts: artifacts, }, nil } diff --git a/internal/framework/proposal_generation/generate_test.go b/internal/framework/proposal_generation/generate_test.go index 9e05fca..ea66cd3 100644 --- a/internal/framework/proposal_generation/generate_test.go +++ b/internal/framework/proposal_generation/generate_test.go @@ -229,7 +229,7 @@ func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) { responses: []StructuredCorrectionSet{ { Corrections: []StructuredCorrectionProposal{ - {TargetSegmentID: 1, OriginalText: "x", CorrectedText: "", Confidence: 0.9}, + {TargetSegmentID: 0, OriginalText: "x", CorrectedText: "", Confidence: 0.9}, }, }, }, @@ -237,9 +237,18 @@ func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) { req := defaultRequest(t) req.LLMClient = client - _, err := GenerateCandidates(context.Background(), req) - if err == nil || !strings.Contains(err.Error(), "invalid structured correction") { - t.Fatalf("expected structured response validation failure, got %v", err) + got, err := GenerateCandidates(context.Background(), req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got.Corrections) != 0 || len(got.Enriched) != 0 { + t.Fatalf("expected invalid proposal to be dropped, got %+v", got) + } + if len(got.Dropped) != 1 { + t.Fatalf("expected one dropped candidate, got %+v", got.Dropped) + } + if got.Dropped[0].ReasonCode != "invalid_structured_correction" { + t.Fatalf("unexpected dropped reason: %+v", got.Dropped[0]) } } diff --git a/internal/framework/proposals/proposal.go b/internal/framework/proposals/proposal.go index a55e1b0..8dd2a10 100644 --- a/internal/framework/proposals/proposal.go +++ b/internal/framework/proposals/proposal.go @@ -38,9 +38,6 @@ func (p CorrectionProposal) Validate() error { if strings.TrimSpace(p.OriginalText) == "" { return fmt.Errorf("proposal original_text must not be empty") } - if strings.TrimSpace(p.CorrectedText) == "" { - return fmt.Errorf("proposal corrected_text must not be empty") - } if p.Confidence < 0.0 || p.Confidence > 1.0 { return fmt.Errorf("proposal confidence must be between 0.0 and 1.0") } diff --git a/internal/framework/proposals/proposal_test.go b/internal/framework/proposals/proposal_test.go index 93e27b8..9ac38a1 100644 --- a/internal/framework/proposals/proposal_test.go +++ b/internal/framework/proposals/proposal_test.go @@ -35,7 +35,7 @@ func TestCorrectionProposalValidate_InvalidEmptyOriginalText(t *testing.T) { } } -func TestCorrectionProposalValidate_InvalidEmptyCorrectedText(t *testing.T) { +func TestCorrectionProposalValidate_AllowsEmptyCorrectedText(t *testing.T) { proposal := CorrectionProposal{ TargetSegmentID: 42, OriginalText: "gestures", @@ -44,11 +44,8 @@ func TestCorrectionProposalValidate_InvalidEmptyCorrectedText(t *testing.T) { } err := proposal.Validate() - if err == nil { - t.Fatal("expected validation error, got nil") - } - if err.Error() != "proposal corrected_text must not be empty" { - t.Fatalf("unexpected error: %v", err) + if err != nil { + t.Fatalf("expected valid proposal, got %v", err) } } diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index e33c608..aeb041a 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -238,6 +238,8 @@ type sectionProposalResult struct { sectionPos int section chunking.Section corrected []proposals.CorrectionProposal + dropped []droppedProposalCandidate + rawCount int err error } @@ -257,6 +259,22 @@ type modulePipelineResult struct { ValidatorRejected []ValidatorRejectedChange } +type droppedProposalCandidate struct { + ProposalIndex int + TargetSegmentID int + OriginalText string + CorrectedText string + ReasonCode string + Message string +} + +const ( + validatorNameProposalGeneration = "proposal_generation" + reasonValidatorExecutionError = "validator_execution_error" + reasonValidatorMalformedResponse = "validator_malformed_response" + reasonValidatorMissingDecision = "validator_missing_decision" +) + func collectSectionProposals(ctx context.Context, input collectSectionProposalsInput) (context.Context, <-chan sectionProposalResult, context.CancelFunc) { results := make(chan sectionProposalResult, len(input.Sections)) runCtx, cancel := context.WithCancel(ctx) @@ -271,6 +289,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI go func() { defer wg.Done() meta := contracts.SectionMetadataFromSection(section) + dropped := make([]droppedProposalCandidate, 0) corrected, err := input.Module.Propose(withModuleInstanceContext(runCtx, input.Spec.InstanceName), contracts.ProposalRequest{ ExecutionContext: contracts.ExecutionContext{ Config: input.Config, @@ -286,12 +305,25 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI }, LLMClient: input.ProposalClient, LLMScheduler: input.ProposalScheduler, + OnDroppedCandidate: func(index int, targetSegmentID int, originalText string, correctedText string, confidence float64, reasonCode string, message string) { + _ = confidence + dropped = append(dropped, droppedProposalCandidate{ + ProposalIndex: index, + TargetSegmentID: targetSegmentID, + OriginalText: originalText, + CorrectedText: correctedText, + ReasonCode: reasonCode, + Message: message, + }) + }, }) select { case results <- sectionProposalResult{ sectionPos: sectionPos, section: section, corrected: corrected, + dropped: dropped, + rawCount: len(corrected) + len(dropped), err: err, }: case <-runCtx.Done(): @@ -362,7 +394,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) delete(pending, nextSectionToProcess) sectionMeta := contracts.SectionMetadataFromSection(sectionResult.section) sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.corrected)) - for i, corrected := range sectionResult.corrected { + for i, corrected := range sectionResult.corrected { sectionIndex := sectionMeta.Index sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{ CorrectionProposal: corrected, @@ -373,9 +405,22 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) SectionIndex: §ionIndex, }, }) - } - nextProposalIndex += len(sectionEnriched) - out.ProposalCount += len(sectionEnriched) + } + out.ProposalCount += sectionResult.rawCount + nextProposalIndex += sectionResult.rawCount + for _, dropped := range sectionResult.dropped { + out.ValidatorRejected = append(out.ValidatorRejected, ValidatorRejectedChange{ + ValidatorName: validatorNameProposalGeneration, + ProposalIndex: dropped.ProposalIndex, + ModuleKey: input.Spec.ModuleKey, + ModuleInstance: input.Spec.InstanceName, + TargetSegmentID: dropped.TargetSegmentID, + OriginalText: dropped.OriginalText, + CorrectedText: dropped.CorrectedText, + ReasonCode: dropped.ReasonCode, + Message: dropped.Message, + }) + } validationLaunches++ vwg.Add(1) @@ -508,54 +553,33 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida ) } if err != nil { - return validateSectionCandidatesResult{ - approved: eligible, - decisions: decisions, - rejected: rejected, - }, fmt.Errorf("validator %q failed: %w", validator.Name(), err) - } - if err := validators.EnforceDecisionCardinality(eligible, vResult.Decisions); err != nil { - return validateSectionCandidatesResult{ - approved: eligible, - decisions: decisions, - rejected: rejected, - }, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), err) - } - - nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(eligible)) - byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(eligible)) - for _, p := range eligible { - byIndex[p.ProposalIndex] = p - } - - for _, d := range vResult.Decisions { - decisions = append(decisions, ValidatorDecisionRecord{ - ValidatorName: validator.Name(), - ProposalIndex: d.ProposalIndex, - Approved: d.Approved, - ReasonCode: d.ReasonCode, - Message: d.Message, - DiagnosticArtifactPath: d.DiagnosticArtifactPath, - }) - if d.Approved { - nextEligible = append(nextEligible, byIndex[d.ProposalIndex]) - continue + for _, p := range eligible { + decisions = append(decisions, ValidatorDecisionRecord{ + ValidatorName: validator.Name(), + ProposalIndex: p.ProposalIndex, + Approved: false, + ReasonCode: reasonValidatorExecutionError, + Message: fmt.Sprintf("validator execution failed: %v", err), + }) + rejected = append(rejected, ValidatorRejectedChange{ + ValidatorName: validator.Name(), + ProposalIndex: p.ProposalIndex, + ModuleKey: p.ModuleKey, + ModuleInstance: p.ModuleInstance, + TargetSegmentID: p.TargetSegmentID, + OriginalText: p.OriginalText, + CorrectedText: p.CorrectedText, + ReasonCode: reasonValidatorExecutionError, + Message: fmt.Sprintf("validator execution failed: %v", err), + }) } - p := byIndex[d.ProposalIndex] - rejected = append(rejected, ValidatorRejectedChange{ - ValidatorName: validator.Name(), - ProposalIndex: p.ProposalIndex, - ModuleKey: p.ModuleKey, - ModuleInstance: p.ModuleInstance, - TargetSegmentID: p.TargetSegmentID, - OriginalText: p.OriginalText, - CorrectedText: p.CorrectedText, - ReasonCode: d.ReasonCode, - Message: d.Message, - }) + eligible = nil + continue } - - eligible = nextEligible + reconciled := reconcileValidatorDecisions(validator.Name(), eligible, vResult.Decisions) + decisions = append(decisions, reconciled.decisions...) + rejected = append(rejected, reconciled.rejected...) + eligible = reconciled.nextEligible } return validateSectionCandidatesResult{ @@ -565,6 +589,114 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida }, nil } +type reconciledValidatorDecisions struct { + nextEligible []proposals.EnrichedCorrectionProposal + decisions []ValidatorDecisionRecord + rejected []ValidatorRejectedChange +} + +func reconcileValidatorDecisions(validatorName string, eligible []proposals.EnrichedCorrectionProposal, in []validators.Decision) reconciledValidatorDecisions { + byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(eligible)) + for _, p := range eligible { + byIndex[p.ProposalIndex] = p + } + + seen := make(map[int]struct{}, len(in)) + forcedReject := make(map[int]bool) + valid := make(map[int]validators.Decision, len(in)) + for _, d := range in { + if _, ok := byIndex[d.ProposalIndex]; !ok { + continue + } + if _, dup := seen[d.ProposalIndex]; dup { + forcedReject[d.ProposalIndex] = true + continue + } + seen[d.ProposalIndex] = struct{}{} + valid[d.ProposalIndex] = d + } + + nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(eligible)) + decisions := make([]ValidatorDecisionRecord, 0, len(eligible)) + rejected := make([]ValidatorRejectedChange, 0, len(eligible)) + + for _, p := range eligible { + idx := p.ProposalIndex + if forcedReject[idx] { + decisions = append(decisions, ValidatorDecisionRecord{ + ValidatorName: validatorName, + ProposalIndex: idx, + Approved: false, + ReasonCode: reasonValidatorMalformedResponse, + Message: "validator returned malformed response for this proposal index", + }) + rejected = append(rejected, ValidatorRejectedChange{ + ValidatorName: validatorName, + ProposalIndex: p.ProposalIndex, + ModuleKey: p.ModuleKey, + ModuleInstance: p.ModuleInstance, + TargetSegmentID: p.TargetSegmentID, + OriginalText: p.OriginalText, + CorrectedText: p.CorrectedText, + ReasonCode: reasonValidatorMalformedResponse, + Message: "validator returned malformed response for this proposal index", + }) + continue + } + d, ok := valid[idx] + if !ok { + decisions = append(decisions, ValidatorDecisionRecord{ + ValidatorName: validatorName, + ProposalIndex: idx, + Approved: false, + ReasonCode: reasonValidatorMissingDecision, + Message: "validator did not return a decision for this proposal index", + }) + rejected = append(rejected, ValidatorRejectedChange{ + ValidatorName: validatorName, + ProposalIndex: p.ProposalIndex, + ModuleKey: p.ModuleKey, + ModuleInstance: p.ModuleInstance, + TargetSegmentID: p.TargetSegmentID, + OriginalText: p.OriginalText, + CorrectedText: p.CorrectedText, + ReasonCode: reasonValidatorMissingDecision, + Message: "validator did not return a decision for this proposal index", + }) + continue + } + decisions = append(decisions, ValidatorDecisionRecord{ + ValidatorName: validatorName, + ProposalIndex: d.ProposalIndex, + Approved: d.Approved, + ReasonCode: d.ReasonCode, + Message: d.Message, + DiagnosticArtifactPath: d.DiagnosticArtifactPath, + }) + if d.Approved { + nextEligible = append(nextEligible, p) + continue + } + rejected = append(rejected, ValidatorRejectedChange{ + ValidatorName: validatorName, + ProposalIndex: p.ProposalIndex, + ModuleKey: p.ModuleKey, + ModuleInstance: p.ModuleInstance, + TargetSegmentID: p.TargetSegmentID, + OriginalText: p.OriginalText, + CorrectedText: p.CorrectedText, + ReasonCode: d.ReasonCode, + Message: d.Message, + }) + } + + return reconciledValidatorDecisions{ + nextEligible: nextEligible, + decisions: decisions, + rejected: rejected, + } +} + func reorderValidatorsForPipeline(in []contracts.Validator) ([]contracts.Validator, map[string]int) { deterministic := make([]contracts.Validator, 0, len(in)) llmBacked := make([]contracts.Validator, 0, len(in)) diff --git a/internal/framework/runner/runner_test.go b/internal/framework/runner/runner_test.go index 09eae98..c6cdb8e 100644 --- a/internal/framework/runner/runner_test.go +++ b/internal/framework/runner/runner_test.go @@ -1054,7 +1054,7 @@ func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) { } } -func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *testing.T) { +func TestRunnerValidatorCardinalityErrorSoftRejectsAndContinues(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}} good := fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil @@ -1069,14 +1069,17 @@ func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *test r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m1": good, "m2": bad}}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}}) - if err == nil { - t.Fatal("expected cardinality error") + if err != nil { + t.Fatalf("unexpected run error: %v", err) } if out.FinalTranscript.Segments[0].Text != "the cat" { - t.Fatalf("expected partial progress preserved, got %q", out.FinalTranscript.Segments[0].Text) + t.Fatalf("expected second-module proposal to be rejected, got %q", out.FinalTranscript.Segments[0].Text) } - if len(out.ModuleResults) != 2 || out.ModuleResults[1].Status != ModuleStatusFailed { - t.Fatalf("expected second module failed") + if len(out.ModuleResults) != 2 || out.ModuleResults[1].Status != ModuleStatusSuccess { + t.Fatalf("expected second module success with rejection, got %+v", out.ModuleResults) + } + if len(out.ModuleResults[1].ValidatorRejected) == 0 { + t.Fatalf("expected validator rejection record, got %+v", out.ModuleResults[1]) } } @@ -1191,7 +1194,7 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) { } } -func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing.T) { +func TestRunnerLLMValidatorMalformedResponseSoftRejectsWithProgress(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "bad index"}}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "") r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ @@ -1209,15 +1212,21 @@ func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing. ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}, ValidationLLMClient: client, }) - if err == nil { - t.Fatal("expected llm validator failure") + if err != nil { + t.Fatalf("unexpected run error: %v", err) } if out.FinalTranscript.Segments[0].Text != "the cat" { t.Fatalf("expected partial progress retained") } + if len(out.ModuleResults) != 2 || out.ModuleResults[1].Status != ModuleStatusSuccess { + t.Fatalf("expected second module success, got %+v", out.ModuleResults) + } + if len(out.ModuleResults[1].ValidatorRejected) == 0 { + t.Fatalf("expected llm validator rejection record") + } } -func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) { +func TestRunnerLLMValidatorMissingDecisionSoftRejects(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "") r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ @@ -1226,18 +1235,21 @@ func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) { }}, }}) cfg := config.Default() - _, err := r.Run(context.Background(), RunInput{ + out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client, }) - if err == nil { - t.Fatal("expected missing decision failure") + if err != nil { + t.Fatalf("unexpected run error: %v", err) + } + if len(out.ModuleResults[0].ValidatorRejected) != 1 { + t.Fatalf("expected soft rejection, got %+v", out.ModuleResults[0].ValidatorRejected) } } -func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) { +func TestRunnerLLMValidatorDuplicateDecisionSoftRejects(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{ {CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, {CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"}, @@ -1249,14 +1261,17 @@ func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) { }}, }}) cfg := config.Default() - _, err := r.Run(context.Background(), RunInput{ + out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client, }) - if err == nil { - t.Fatal("expected duplicate decision failure") + if err != nil { + t.Fatalf("unexpected run error: %v", err) + } + if len(out.ModuleResults[0].ValidatorRejected) != 1 { + t.Fatalf("expected soft rejection, got %+v", out.ModuleResults[0].ValidatorRejected) } } diff --git a/internal/framework/validators/deterministic.go b/internal/framework/validators/deterministic.go index 81fda80..6c871a5 100644 --- a/internal/framework/validators/deterministic.go +++ b/internal/framework/validators/deterministic.go @@ -4,6 +4,9 @@ import ( "context" "fmt" "strings" + + "gitea.maximumdirect.net/eric/audita/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" ) type ConfidenceThresholdValidator struct{} @@ -62,11 +65,26 @@ type NonEmptyCorrectionValidator struct{} func (v NonEmptyCorrectionValidator) Name() string { return "non_empty_corrected_text" } func (v NonEmptyCorrectionValidator) Validate(_ context.Context, req Request) (Result, error) { + segments := make(map[int]string) + if req.WorkingTranscript != nil { + for _, seg := range req.WorkingTranscript.Segments { + segments[seg.ID] = seg.Text + } + } + decisions := make([]Decision, 0, len(req.CandidateProposal)) for _, c := range req.CandidateProposal { - if strings.TrimSpace(c.CorrectedText) == "" { - decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyCorrectedText, "corrected_text must not be empty")) - continue + segmentText, ok := segments[c.TargetSegmentID] + if ok { + preview := proposals.PreviewProposalForSegment( + &schema.Segment{ID: c.TargetSegmentID, Text: segmentText}, + c.CorrectionProposal, + req.ReplacementPolicy, + ) + if preview.Applicable && strings.TrimSpace(preview.CorrectedSegmentText) == "" { + decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyCorrectedText, "corrected segment text must not be empty")) + continue + } } decisions = append(decisions, approval(c.ProposalIndex)) } diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index 634d709..3886d82 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -143,10 +143,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, return Result{}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err) } - batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response) - if err != nil { - return Result{}, fmt.Errorf("LLM validator %q response invalid: %w", v.name, err) - } + batchDecisions := mapLLMResponseToDecisions(batch.Items, response) for i := range batchDecisions { batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath } @@ -259,34 +256,48 @@ func promptBuilderForType(validatorType LLMValidatorType) (LLMPromptBuilder, err } } -func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidationResponse) ([]Decision, error) { +func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidationResponse) []Decision { expected := make(map[int]LLMValidationItem, len(items)) for _, item := range items { expected[item.CorrectionIndex] = item } - if len(response.Validations) == 0 { - return nil, fmt.Errorf("missing validations in structured response") - } - seen := make(map[int]LLMValidationDecision, len(response.Validations)) + forcedReject := make(map[int]bool) for _, d := range response.Validations { - if d.Confidence < 0.0 || d.Confidence > 1.0 { - return nil, fmt.Errorf("confidence for correction_index %d must be between 0.0 and 1.0", d.CorrectionIndex) - } if _, ok := expected[d.CorrectionIndex]; !ok { - return nil, fmt.Errorf("unknown correction_index %d", d.CorrectionIndex) + continue } if _, exists := seen[d.CorrectionIndex]; exists { - return nil, fmt.Errorf("duplicate correction_index %d", d.CorrectionIndex) + forcedReject[d.CorrectionIndex] = true + continue + } + if d.Confidence < 0.0 || d.Confidence > 1.0 { + forcedReject[d.CorrectionIndex] = true + continue } seen[d.CorrectionIndex] = d } decisions := make([]Decision, 0, len(items)) for _, item := range items { + if forcedReject[item.CorrectionIndex] { + decisions = append(decisions, Decision{ + ProposalIndex: item.CorrectionIndex, + Approved: false, + ReasonCode: ReasonValidatorMalformed, + Message: "validator returned malformed decision payload for this proposal index", + }) + continue + } d, ok := seen[item.CorrectionIndex] if !ok { - return nil, fmt.Errorf("missing correction_index %d", item.CorrectionIndex) + decisions = append(decisions, Decision{ + ProposalIndex: item.CorrectionIndex, + Approved: false, + ReasonCode: ReasonValidatorMissing, + Message: "validator did not return a decision for this proposal index", + }) + continue } reasonCode := ReasonApproved if !d.Approved { @@ -299,5 +310,5 @@ func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidation Message: strings.TrimSpace(d.Reason), }) } - return decisions, nil + return decisions } diff --git a/internal/framework/validators/llm_validators_test.go b/internal/framework/validators/llm_validators_test.go index 6830e2a..539cc38 100644 --- a/internal/framework/validators/llm_validators_test.go +++ b/internal/framework/validators/llm_validators_test.go @@ -259,18 +259,24 @@ func TestLLMBackedValidatorMalformedOutputFails(t *testing.T) { } } -func TestLLMBackedValidatorMissingDecisionFails(t *testing.T) { +func TestLLMBackedValidatorMissingDecisionSoftRejects(t *testing.T) { client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}} v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client - _, err := v.Validate(context.Background(), req) - if err == nil || !strings.Contains(err.Error(), "response invalid") { - t.Fatalf("expected missing decision error, got %v", err) + res, err := v.Validate(context.Background(), req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(res.Decisions) != 1 || res.Decisions[0].Approved { + t.Fatalf("expected one soft rejection, got %+v", res.Decisions) + } + if res.Decisions[0].ReasonCode != ReasonValidatorMissing { + t.Fatalf("expected %q, got %+v", ReasonValidatorMissing, res.Decisions[0]) } } -func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) { +func TestLLMBackedValidatorDuplicateDecisionSoftRejects(t *testing.T) { client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{ {CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, {CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"}, @@ -278,20 +284,32 @@ func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) { v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client - _, err := v.Validate(context.Background(), req) - if err == nil || !strings.Contains(err.Error(), "duplicate") { - t.Fatalf("expected duplicate decision error, got %v", err) + res, err := v.Validate(context.Background(), req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(res.Decisions) != 1 || res.Decisions[0].Approved { + t.Fatalf("expected one soft rejection, got %+v", res.Decisions) + } + if res.Decisions[0].ReasonCode != ReasonValidatorMalformed { + t.Fatalf("expected %q, got %+v", ReasonValidatorMalformed, res.Decisions[0]) } } -func TestLLMBackedValidatorUnknownProposalIndexFails(t *testing.T) { +func TestLLMBackedValidatorUnknownProposalIndexSoftRejectsMissing(t *testing.T) { client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}} v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client - _, err := v.Validate(context.Background(), req) - if err == nil || !strings.Contains(err.Error(), "unknown") { - t.Fatalf("expected unknown index error, got %v", err) + res, err := v.Validate(context.Background(), req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(res.Decisions) != 1 || res.Decisions[0].Approved { + t.Fatalf("expected one soft rejection, got %+v", res.Decisions) + } + if res.Decisions[0].ReasonCode != ReasonValidatorMissing { + t.Fatalf("expected %q, got %+v", ReasonValidatorMissing, res.Decisions[0]) } } diff --git a/internal/framework/validators/models.go b/internal/framework/validators/models.go index 0aae19f..644b493 100644 --- a/internal/framework/validators/models.go +++ b/internal/framework/validators/models.go @@ -18,6 +18,8 @@ const ( ReasonEmptyCorrectedText = "empty_corrected_text" ReasonNoEffect = "no_effect" ReasonProtectedGlossaryTerm = "protected_glossary_term" + ReasonValidatorMalformed = "validator_malformed_response" + ReasonValidatorMissing = "validator_missing_decision" ) // Request is the runtime input shared by deterministic validators. diff --git a/internal/framework/validators/validators_test.go b/internal/framework/validators/validators_test.go index ba18a05..fd41fbe 100644 --- a/internal/framework/validators/validators_test.go +++ b/internal/framework/validators/validators_test.go @@ -90,10 +90,14 @@ func TestOriginalTextPresenceValidator(t *testing.T) { } func TestNonEmptyCorrectionValidator(t *testing.T) { - req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{ + req := Request{ + WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello"}}}, + ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, + CandidateProposal: []proposals.EnrichedCorrectionProposal{ mkCandidate(0, 1, "hello", "hi", 0.9), mkCandidate(1, 1, "hello", " ", 0.9), - }} + }, + } res, err := (NonEmptyCorrectionValidator{}).Validate(context.Background(), req) if err != nil { t.Fatalf("Validate error: %v", err) @@ -154,6 +158,8 @@ func TestStableReasonCodes(t *testing.T) { ReasonEmptyCorrectedText, ReasonNoEffect, ReasonProtectedGlossaryTerm, + ReasonValidatorMalformed, + ReasonValidatorMissing, } for _, code := range codes { if strings.TrimSpace(code) == "" { diff --git a/internal/modules/glossary/module.go b/internal/modules/glossary/module.go index 2448b86..dcfd291 100644 --- a/internal/modules/glossary/module.go +++ b/internal/modules/glossary/module.go @@ -72,6 +72,20 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([] LLMClient: req.LLMClient, Scheduler: req.LLMScheduler, DiagnosticsDir: req.DiagnosticsDir, + OnDroppedCandidate: func(dropped proposal_generation.DroppedCandidate) { + if req.OnDroppedCandidate == nil { + return + } + req.OnDroppedCandidate( + dropped.ProposalIndex, + dropped.TargetSegmentID, + dropped.OriginalText, + dropped.CorrectedText, + dropped.Confidence, + dropped.ReasonCode, + dropped.Message, + ) + }, }) if err != nil { return nil, err diff --git a/internal/modules/grammar/module.go b/internal/modules/grammar/module.go index 6bdefc3..b918b16 100644 --- a/internal/modules/grammar/module.go +++ b/internal/modules/grammar/module.go @@ -72,6 +72,20 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([] LLMClient: req.LLMClient, Scheduler: req.LLMScheduler, DiagnosticsDir: req.DiagnosticsDir, + OnDroppedCandidate: func(dropped proposal_generation.DroppedCandidate) { + if req.OnDroppedCandidate == nil { + return + } + req.OnDroppedCandidate( + dropped.ProposalIndex, + dropped.TargetSegmentID, + dropped.OriginalText, + dropped.CorrectedText, + dropped.Confidence, + dropped.ReasonCode, + dropped.Message, + ) + }, }) if err != nil { return nil, err diff --git a/internal/modules/homophones/module.go b/internal/modules/homophones/module.go index e07dbdd..1faaf8e 100644 --- a/internal/modules/homophones/module.go +++ b/internal/modules/homophones/module.go @@ -72,6 +72,20 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([] LLMClient: req.LLMClient, Scheduler: req.LLMScheduler, DiagnosticsDir: req.DiagnosticsDir, + OnDroppedCandidate: func(dropped proposal_generation.DroppedCandidate) { + if req.OnDroppedCandidate == nil { + return + } + req.OnDroppedCandidate( + dropped.ProposalIndex, + dropped.TargetSegmentID, + dropped.OriginalText, + dropped.CorrectedText, + dropped.Confidence, + dropped.ReasonCode, + dropped.Message, + ) + }, }) if err != nil { return nil, err diff --git a/internal/modules/spoken_word/module.go b/internal/modules/spoken_word/module.go index 1b8bed0..5f20c46 100644 --- a/internal/modules/spoken_word/module.go +++ b/internal/modules/spoken_word/module.go @@ -72,6 +72,20 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([] LLMClient: req.LLMClient, Scheduler: req.LLMScheduler, DiagnosticsDir: req.DiagnosticsDir, + OnDroppedCandidate: func(dropped proposal_generation.DroppedCandidate) { + if req.OnDroppedCandidate == nil { + return + } + req.OnDroppedCandidate( + dropped.ProposalIndex, + dropped.TargetSegmentID, + dropped.OriginalText, + dropped.CorrectedText, + dropped.Confidence, + dropped.ReasonCode, + dropped.Message, + ) + }, }) if err != nil { return nil, err