eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Make module-stage LLM handling resilient and report warnings

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-23 10:07:06 -05:00
parent a84941d681
commit a3655f5540
43 changed files with 856 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/cli/run.go
View File

@@ -26,6 +26,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
)
type noOpStructuredLLMClient struct{}
@@ -803,6 +804,7 @@ func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummar
ReplacementPolicy: string(r.ReplacementPolicy),
Status: r.Status,
ProposalCount: r.ProposalCount,
Warnings: append([]stagewarnings.StageWarning(nil), r.Warnings...),
ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions),
ValidatorRejected: mapValidatorRejected(r.ValidatorRejected),
AppliedChanges: r.AppliedChanges,

126
internal/cli/run_test.go
View File

@@ -1793,11 +1793,12 @@ type fakeModule struct {
func (m fakeModule) Key() string { return m.key }
func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
func (m fakeModule) Validators() []contracts.Validator { return m.validators }
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
if m.proposeF == nil {
return nil, nil
return contracts.ProposalResult{}, nil
}
return m.proposeF(req)
proposalsOut, err := m.proposeF(req)
return contracts.ProposalResult{Proposals: proposalsOut}, err
}
type fakeValidator struct {
@@ -2384,7 +2385,7 @@ func TestRunProcessExplicitGrammarRejectedAndApplicationSkipAreDistinct(t *testi
}
}
func TestRunProcessExplicitGrammarMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
func TestRunProcessExplicitGrammarMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
t.Cleanup(func() { processProposalLLMClient = nil })
@@ -2403,22 +2404,32 @@ func TestRunProcessExplicitGrammarMalformedLLMOutputFailsWithErrorLog(t *testing
"--work-dir-retention", "always",
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatal("expected failure")
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
if stderr.Len() != 0 {
t.Fatalf("expected empty stderr on success, got %q", stderr.String())
}
if !strings.Contains(stderr.String(), "runner_execution") {
t.Fatalf("expected runner_execution error, got %q", stderr.String())
parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
if err != nil {
t.Fatalf("expected transcript stdout on success: %v", err)
}
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log on failed grammar run: %v", err)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
t.Fatalf("did not expect error.log on successful grammar run: %v", err)
}
report := readProcessReport(t, reportPath)
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
t.Fatalf("expected failed runner_execution report, got %+v", report)
if report.Status != "success" || report.ErrorPhase != "" {
t.Fatalf("expected successful report, got %+v", report)
}
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
}
if report.ModuleResults[0].Warnings[0].ReasonCode != "proposal_response_malformed" {
t.Fatalf("unexpected warning: %+v", report.ModuleResults[0].Warnings[0])
}
}
@@ -3041,7 +3052,7 @@ func TestRunProcessExplicitGlossaryRepeatedStagesUseDeterministicInstanceNamesAn
}
}
func TestRunProcessExplicitGlossaryMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
func TestRunProcessExplicitGlossaryMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
t.Cleanup(func() { processProposalLLMClient = nil })
@@ -3060,22 +3071,29 @@ func TestRunProcessExplicitGlossaryMalformedLLMOutputFailsWithErrorLog(t *testin
"--work-dir-retention", "always",
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatal("expected failure")
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
if stderr.Len() != 0 {
t.Fatalf("expected empty stderr on success, got %q", stderr.String())
}
if !strings.Contains(stderr.String(), "runner_execution") {
t.Fatalf("expected runner_execution error, got %q", stderr.String())
parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
if err != nil {
t.Fatalf("expected transcript stdout on success: %v", err)
}
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log on failed glossary run: %v", err)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
t.Fatalf("did not expect error.log on successful glossary run: %v", err)
}
report := readProcessReport(t, reportPath)
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
t.Fatalf("expected failed runner_execution report, got %+v", report)
if report.Status != "success" || report.ErrorPhase != "" {
t.Fatalf("expected successful report, got %+v", report)
}
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
}
}
@@ -3295,7 +3313,7 @@ func TestRunProcessExplicitHomophonesProtectedGlossaryTermRejected(t *testing.T)
}
}
func TestRunProcessExplicitHomophonesMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
func TestRunProcessExplicitHomophonesMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
t.Cleanup(func() { processProposalLLMClient = nil })
@@ -3314,22 +3332,29 @@ func TestRunProcessExplicitHomophonesMalformedLLMOutputFailsWithErrorLog(t *test
"--work-dir-retention", "always",
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatal("expected failure")
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
if stderr.Len() != 0 {
t.Fatalf("expected empty stderr on success, got %q", stderr.String())
}
if !strings.Contains(stderr.String(), "runner_execution") {
t.Fatalf("expected runner_execution error, got %q", stderr.String())
parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
if err != nil {
t.Fatalf("expected transcript stdout on success: %v", err)
}
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log on failed homophones run: %v", err)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
t.Fatalf("did not expect error.log on successful homophones run: %v", err)
}
report := readProcessReport(t, reportPath)
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
t.Fatalf("expected failed runner_execution report, got %+v", report)
if report.Status != "success" || report.ErrorPhase != "" {
t.Fatalf("expected successful report, got %+v", report)
}
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
}
}
@@ -3674,7 +3699,7 @@ func TestRunProcessExplicitSpokenWordProtectedGlossaryTermRejected(t *testing.T)
}
}
func TestRunProcessExplicitSpokenWordMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
func TestRunProcessExplicitSpokenWordMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
t.Cleanup(func() { processProposalLLMClient = nil })
@@ -3693,22 +3718,29 @@ func TestRunProcessExplicitSpokenWordMalformedLLMOutputFailsWithErrorLog(t *test
"--work-dir-retention", "always",
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatal("expected failure")
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
if stderr.Len() != 0 {
t.Fatalf("expected empty stderr on success, got %q", stderr.String())
}
if !strings.Contains(stderr.String(), "runner_execution") {
t.Fatalf("expected runner_execution error, got %q", stderr.String())
parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
if err != nil {
t.Fatalf("expected transcript stdout on success: %v", err)
}
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log on failed spoken_word run: %v", err)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
t.Fatalf("did not expect error.log on successful spoken_word run: %v", err)
}
report := readProcessReport(t, reportPath)
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
t.Fatalf("expected failed runner_execution report, got %+v", report)
if report.Status != "success" || report.ErrorPhase != "" {
t.Fatalf("expected successful report, got %+v", report)
}
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
}
}

23
internal/cli/subprocess_test_hooks.go
View File

@@ -78,25 +78,22 @@ func (c *subprocessTestLLMClient) CompleteStructured(ctx context.Context, req co
}
}
case "mid_pipeline_fail":
switch target := out.(type) {
case *proposal_generation.StructuredCorrectionSet:
if _, ok := out.(*proposal_generation.StructuredCorrectionSet); ok {
c.mu.Lock()
c.proposals++
proposalCall := c.proposals
c.mu.Unlock()
if proposalCall >= 3 {
*target = proposal_generation.StructuredCorrectionSet{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y", Confidence: 0.99},
},
}
} else {
*target = proposal_generation.StructuredCorrectionSet{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99},
},
}
return contracts.StructuredCompletionResponse{}, errors.New("synthetic mid-pipeline failure")
}
}
switch target := out.(type) {
case *proposal_generation.StructuredCorrectionSet:
*target = proposal_generation.StructuredCorrectionSet{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99},
},
}
case *validators.LLMValidationResponse:
*target = validators.LLMValidationResponse{Validations: nil}