eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Complete Phase 12 grammar module

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-12 02:57:06 +00:00
parent b360493cdc
commit fc3a7b7a67
12 changed files with 816 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

278
internal/cli/run_test.go
View File

@@ -15,6 +15,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
@@ -615,8 +616,8 @@ func TestRunProcessReportJSONIncludesChunkingSummary(t *testing.T) {
if report.Chunking.MaxSectionTokens == 0 {
t.Errorf("expected max_section_tokens in report")
}
if report.Phase != "phase11-proposal-generation-framework" {
t.Errorf("expected phase 'phase11-proposal-generation-framework', got %q", report.Phase)
if report.Phase != "phase12-grammar-module" {
t.Errorf("expected phase 'phase12-grammar-module', got %q", report.Phase)
}
}
@@ -661,8 +662,9 @@ func (v fakeValidator) Validate(ctx context.Context, req contracts.ValidationReq
}
type fakeStructuredLLMClient struct {
responses []validators.LLMValidationResponse
err error
validationResponses []validators.LLMValidationResponse
proposalResponses []proposal_generation.StructuredCorrectionSet
err error
}
func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
@@ -671,13 +673,24 @@ func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req co
if f.err != nil {
return contracts.StructuredCompletionResponse{}, f.err
}
if len(f.responses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected llm call")
switch target := out.(type) {
case *validators.LLMValidationResponse:
if len(f.validationResponses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected validation llm call")
}
*target = f.validationResponses[0]
f.validationResponses = f.validationResponses[1:]
return contracts.StructuredCompletionResponse{}, nil
case *proposal_generation.StructuredCorrectionSet:
if len(f.proposalResponses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected proposal llm call")
}
*target = f.proposalResponses[0]
f.proposalResponses = f.proposalResponses[1:]
return contracts.StructuredCompletionResponse{}, nil
default:
return contracts.StructuredCompletionResponse{}, errors.New("unexpected llm output type")
}
target := out.(*validators.LLMValidationResponse)
*target = f.responses[0]
f.responses = f.responses[1:]
return contracts.StructuredCompletionResponse{}, nil
}
func TestRunProcessInjectedFactoryExecutesRunnerAndReportsModules(t *testing.T) {
@@ -771,7 +784,7 @@ func TestRunProcessInjectedFactoryLLMValidatorResultsInReports(t *testing.T) {
t.Fatalf("NewLLMBackedValidator: %v", err)
}
processValidationLLMClient = &fakeStructuredLLMClient{
responses: []validators.LLMValidationResponse{{
validationResponses: []validators.LLMValidationResponse{{
Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: false, Confidence: 0.99, Reason: "reject"}},
}},
}
@@ -940,6 +953,249 @@ func TestRunProcessProductionRegistryUnimplementedModuleFailsCleanly(t *testing.
}
}
func TestRunProcessExplicitUnimplementedModulesFailClearly(t *testing.T) {
for _, moduleKey := range []string{"glossary", "homophones", "spoken_word"} {
t.Run(moduleKey, func(t *testing.T) {
var stdout, stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", moduleKey,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatalf("expected failure for %s", moduleKey)
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "recognized but not implemented") {
t.Fatalf("expected unimplemented message, got %q", stderr.String())
}
})
}
}
func TestRunProcessExplicitGrammarAppliesCorrectionAndReportsDiagnostics(t *testing.T) {
secret := "phase12-secret"
processProposalLLMClient = &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "hello ,world", CorrectedText: "Hello, world", Confidence: 0.95},
},
},
},
}
processValidationLLMClient = &fakeStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: secret}}},
},
}
t.Setenv("AUDITA_LLM_API_KEY", secret)
t.Setenv("AUDITA_VALIDATION_LLM_API_KEY", secret)
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
processProposalLLMScheduler = nil
processValidationLLMScheduler = nil
})
var stdout, stderr bytes.Buffer
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello ,world"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "grammar",
"--output", outputPath,
"--report-json", reportPath,
"--work-dir", workDir,
"--work-dir-retention", "always",
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout with --output, got %q", stdout.String())
}
parsed, err := schema.ParseTranscriptJSON(readFile(t, outputPath))
if err != nil {
t.Fatalf("parse output: %v", err)
}
if parsed.Segments[0].Text != "Hello, world" {
t.Fatalf("expected grammar correction applied, got %q", parsed.Segments[0].Text)
}
report := readProcessReport(t, reportPath)
if len(report.ModuleResults) != 1 || report.ModuleResults[0].ModuleKey != "grammar" {
t.Fatalf("expected one grammar module result, got %+v", report.ModuleResults)
}
if len(report.ModuleResults[0].AppliedChanges) != 1 {
t.Fatalf("expected one applied grammar change, got %+v", report.ModuleResults[0].AppliedChanges)
}
if len(report.ModuleResults[0].ValidatorDecisions) == 0 {
t.Fatalf("expected validator decisions in report")
}
runDir := onlyRunDir(t, workDir)
runReport := readProcessReport(t, filepath.Join(runDir, "report.json"))
if len(runReport.ModuleResults) != 1 {
t.Fatalf("expected module results in run-dir report")
}
diagFiles, globErr := filepath.Glob(filepath.Join(runDir, "grammar", "*response-payload.json"))
if globErr != nil {
t.Fatalf("glob diagnostics: %v", globErr)
}
if len(diagFiles) == 0 {
t.Fatalf("expected grammar diagnostics payload files in %s", filepath.Join(runDir, "grammar"))
}
for _, f := range diagFiles {
raw := string(readFile(t, f))
if strings.Contains(raw, secret) {
t.Fatalf("secret leaked in diagnostics %q: %s", f, raw)
}
}
}
func TestRunProcessExplicitGrammarRejectedAndApplicationSkipAreDistinct(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "Hello", CorrectedText: "Hi", Confidence: 0.99},
{TargetSegmentID: 1, OriginalText: "Hello world", CorrectedText: "Hi world", Confidence: 0.99},
{TargetSegmentID: 1, OriginalText: "world", CorrectedText: "earth", Confidence: 0.99},
},
},
},
}
processValidationLLMClient = &fakeStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 2, Approved: false, Confidence: 0.9, Reason: "reject"},
},
},
{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
},
},
},
}
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
var stdout, stderr bytes.Buffer
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello world"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "grammar",
"--output", outputPath,
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
report := readProcessReport(t, reportPath)
if len(report.ModuleResults) != 1 {
t.Fatalf("expected one module result")
}
module := report.ModuleResults[0]
if len(module.ValidatorRejected) != 1 {
t.Fatalf("expected one validator rejection, got %+v", module.ValidatorRejected)
}
if len(module.SkippedChanges) != 1 {
t.Fatalf("expected one application skip, got %+v", module.SkippedChanges)
}
if module.ValidatorRejected[0].ReasonCode == string(module.SkippedChanges[0].SkipReason) {
t.Fatalf("validator rejection and application skip should remain distinct")
}
}
func TestRunProcessExplicitGrammarMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
t.Cleanup(func() { processProposalLLMClient = nil })
var stdout, stderr bytes.Buffer
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "grammar",
"--work-dir", workDir,
"--work-dir-retention", "always",
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatal("expected failure")
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "runner_execution") {
t.Fatalf("expected runner_execution error, got %q", stderr.String())
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log on failed grammar run: %v", err)
}
report := readProcessReport(t, reportPath)
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
t.Fatalf("expected failed runner_execution report, got %+v", report)
}
}
func TestRunProcessDefaultBehaviorRemainsDeterministicWithoutExplicitModules(t *testing.T) {
var stdout, stderr bytes.Buffer
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello ,world"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--output", outputPath,
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
report := readProcessReport(t, reportPath)
if report.ModulesSummary != nil || len(report.ModuleResults) != 0 {
t.Fatalf("expected no module execution without explicit --modules, got summary=%+v results=%+v", report.ModulesSummary, report.ModuleResults)
}
}
func TestRunProcessChunkingSummaryArtifactWritten(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer