eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Complete Phase 15 spoken-word module

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-12 11:56:28 +00:00
parent dbf3605712
commit a9f7fa27ff
10 changed files with 945 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/cli/run.go
View File

@@ -460,7 +460,7 @@ func extractErrorPhase(err error) (phase string, message string) {
func buildProcessReport(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) reporting.ProcessReport {
report := reporting.ProcessReport{
Phase: "phase14-homophones-module",
Phase: "phase15-spoken-word-module",
Status: status,
Operation: "process",
TranscriptPath: inv.TranscriptPath,

403
internal/cli/run_test.go
View File

@@ -616,8 +616,8 @@ func TestRunProcessReportJSONIncludesChunkingSummary(t *testing.T) {
if report.Chunking.MaxSectionTokens == 0 {
t.Errorf("expected max_section_tokens in report")
}
if report.Phase != "phase14-homophones-module" {
t.Errorf("expected phase 'phase14-homophones-module', got %q", report.Phase)
if report.Phase != "phase15-spoken-word-module" {
t.Errorf("expected phase 'phase15-spoken-word-module', got %q", report.Phase)
}
}
@@ -909,7 +909,7 @@ func TestRunProcessInjectedFactoryFailureWritesFailedReport(t *testing.T) {
}
}
func TestRunProcessProductionRegistryUnimplementedModuleFailsCleanly(t *testing.T) {
func TestRunProcessProductionRegistryUnsupportedModuleFailsCleanly(t *testing.T) {
cfg := modules.Dependencies{}
processModuleFactory = modules.NewFactory(cfg)
t.Cleanup(func() { processModuleFactory = nil })
@@ -924,7 +924,7 @@ func TestRunProcessProductionRegistryUnimplementedModuleFailsCleanly(t *testing.
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "spoken_word",
"--modules", "made_up",
"--work-dir", workDir,
"--work-dir-retention", "always",
"--report-json", reportPath,
@@ -938,8 +938,8 @@ func TestRunProcessProductionRegistryUnimplementedModuleFailsCleanly(t *testing.
if !strings.Contains(stderr.String(), "runner_execution") {
t.Fatalf("expected runner_execution failure on stderr, got %q", stderr.String())
}
if !strings.Contains(stderr.String(), "recognized but not implemented") {
t.Fatalf("expected explicit unimplemented module message, got %q", stderr.String())
if !strings.Contains(stderr.String(), "unsupported module key") {
t.Fatalf("expected explicit unsupported module message, got %q", stderr.String())
}
report := readProcessReport(t, reportPath)
@@ -949,13 +949,13 @@ func TestRunProcessProductionRegistryUnimplementedModuleFailsCleanly(t *testing.
if report.ErrorPhase != "runner_execution" {
t.Fatalf("expected runner_execution phase, got %q", report.ErrorPhase)
}
if !strings.Contains(report.ErrorMessage, "recognized but not implemented") {
t.Fatalf("expected report error message to mention unimplemented module, got %q", report.ErrorMessage)
if !strings.Contains(report.ErrorMessage, "unsupported module key") {
t.Fatalf("expected report error message to mention unsupported module, got %q", report.ErrorMessage)
}
}
func TestRunProcessExplicitUnimplementedModulesFailClearly(t *testing.T) {
for _, moduleKey := range []string{"spoken_word"} {
func TestRunProcessExplicitUnsupportedModulesFailClearly(t *testing.T) {
for _, moduleKey := range []string{"made_up"} {
t.Run(moduleKey, func(t *testing.T) {
var stdout, stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
@@ -972,8 +972,8 @@ func TestRunProcessExplicitUnimplementedModulesFailClearly(t *testing.T) {
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "recognized but not implemented") {
t.Fatalf("expected unimplemented message, got %q", stderr.String())
if !strings.Contains(stderr.String(), "unsupported module key") {
t.Fatalf("expected unsupported-module message, got %q", stderr.String())
}
})
}
@@ -1790,6 +1790,385 @@ func TestRunProcessExplicitGlossaryThenHomophonesSeesWorkingTranscriptChanges(t
}
}
func TestRunProcessExplicitSpokenWordAppliesCleanupAndReportsDiagnostics(t *testing.T) {
secret := "phase15-secret"
proposalClient := &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "I I think", CorrectedText: "I think", Confidence: 0.95},
},
},
},
}
validationClient := &fakeStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "spoken cleanup acceptable"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: secret}}},
},
}
processProposalLLMClient = proposalClient
processValidationLLMClient = validationClient
t.Setenv("AUDITA_LLM_API_KEY", secret)
t.Setenv("AUDITA_VALIDATION_LLM_API_KEY", secret)
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
var stdout, stderr bytes.Buffer
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"I I think we should go."}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "spoken_word",
"--output", outputPath,
"--report-json", reportPath,
"--work-dir", workDir,
"--work-dir-retention", "always",
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout with --output, got %q", stdout.String())
}
parsed, err := schema.ParseTranscriptJSON(readFile(t, outputPath))
if err != nil {
t.Fatalf("parse output: %v", err)
}
if parsed.Segments[0].Text != "I think we should go." {
t.Fatalf("expected spoken_word cleanup applied, got %q", parsed.Segments[0].Text)
}
report := readProcessReport(t, reportPath)
if len(report.ModuleResults) != 1 || report.ModuleResults[0].ModuleKey != "spoken_word" {
t.Fatalf("expected one spoken_word module result, got %+v", report.ModuleResults)
}
if len(report.ModuleResults[0].AppliedChanges) != 1 {
t.Fatalf("expected one applied spoken_word change, got %+v", report.ModuleResults[0].AppliedChanges)
}
if len(report.ModuleResults[0].ValidatorDecisions) == 0 {
t.Fatalf("expected validator decisions in report")
}
runDir := onlyRunDir(t, workDir)
runReport := readProcessReport(t, filepath.Join(runDir, "report.json"))
if len(runReport.ModuleResults) != 1 || runReport.ModuleResults[0].ModuleKey != "spoken_word" {
t.Fatalf("expected spoken_word module results in run-dir report, got %+v", runReport.ModuleResults)
}
diagFiles, globErr := filepath.Glob(filepath.Join(runDir, "spoken_word", "*response-payload.json"))
if globErr != nil {
t.Fatalf("glob diagnostics: %v", globErr)
}
if len(diagFiles) == 0 {
t.Fatalf("expected spoken_word diagnostics payload files in %s", filepath.Join(runDir, "spoken_word"))
}
for _, f := range diagFiles {
raw := string(readFile(t, f))
if strings.Contains(raw, secret) {
t.Fatalf("secret leaked in diagnostics %q: %s", f, raw)
}
}
}
func TestRunProcessExplicitSpokenWordRejectedAndApplicationSkipAreDistinct(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "hall", CorrectedText: "temple", Confidence: 0.99},
{TargetSegmentID: 1, OriginalText: "I I think in the hall.", CorrectedText: "I think in the temple.", Confidence: 0.99},
{TargetSegmentID: 1, OriginalText: "I I think", CorrectedText: "I suppose", Confidence: 0.99},
},
},
},
}
processValidationLLMClient = &fakeStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 2, Approved: false, Confidence: 0.9, Reason: "reject"},
},
},
{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
},
},
},
}
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
var stdout, stderr bytes.Buffer
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"I I think in the hall."}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "spoken_word",
"--output", outputPath,
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
report := readProcessReport(t, reportPath)
if len(report.ModuleResults) != 1 {
t.Fatalf("expected one module result")
}
module := report.ModuleResults[0]
if len(module.ValidatorRejected) != 1 {
t.Fatalf("expected one validator rejection, got %+v", module.ValidatorRejected)
}
if len(module.SkippedChanges) != 1 {
t.Fatalf("expected one application skip, got %+v", module.SkippedChanges)
}
if module.ValidatorRejected[0].ReasonCode == string(module.SkippedChanges[0].SkipReason) {
t.Fatalf("validator rejection and application skip should remain distinct")
}
}
func TestRunProcessExplicitSpokenWordMeaningChangingCleanupRejected(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "I I think", CorrectedText: "I know", Confidence: 0.99},
},
},
},
}
processValidationLLMClient = &fakeStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: false, Confidence: 0.99, Reason: "meaning changed"}}},
},
}
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
var stdout, stderr bytes.Buffer
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"I I think we should go."}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "spoken_word",
"--output", outputPath,
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
parsed, err := schema.ParseTranscriptJSON(readFile(t, outputPath))
if err != nil {
t.Fatalf("parse output: %v", err)
}
if parsed.Segments[0].Text != "I I think we should go." {
t.Fatalf("expected meaning-changing cleanup to be rejected, got %q", parsed.Segments[0].Text)
}
report := readProcessReport(t, reportPath)
if len(report.ModuleResults) != 1 {
t.Fatalf("expected one module result, got %+v", report.ModuleResults)
}
module := report.ModuleResults[0]
if len(module.AppliedChanges) != 0 {
t.Fatalf("expected no applied changes, got %+v", module.AppliedChanges)
}
if len(module.ValidatorRejected) != 1 {
t.Fatalf("expected one semantic guardrail rejection, got %+v", module.ValidatorRejected)
}
if module.ValidatorRejected[0].ValidatorName != "spoken_word_review" {
t.Fatalf("expected spoken_word_review rejection, got %+v", module.ValidatorRejected[0])
}
}
func TestRunProcessExplicitSpokenWordProtectedGlossaryTermRejected(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "Audita", CorrectedText: "audita", Confidence: 0.99},
},
},
},
}
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
var stdout, stderr bytes.Buffer
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Audita held the line."}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "spoken_word",
"--output", outputPath,
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
parsed, err := schema.ParseTranscriptJSON(readFile(t, outputPath))
if err != nil {
t.Fatalf("parse output: %v", err)
}
if parsed.Segments[0].Text != "Audita held the line." {
t.Fatalf("expected protected glossary term to remain unchanged, got %q", parsed.Segments[0].Text)
}
report := readProcessReport(t, reportPath)
if len(report.ModuleResults) != 1 {
t.Fatalf("expected one module result, got %+v", report.ModuleResults)
}
module := report.ModuleResults[0]
if len(module.AppliedChanges) != 0 {
t.Fatalf("expected no applied changes, got %+v", module.AppliedChanges)
}
if len(module.ValidatorRejected) != 1 {
t.Fatalf("expected one validator rejection, got %+v", module.ValidatorRejected)
}
if module.ValidatorRejected[0].ReasonCode != validators.ReasonProtectedGlossaryTerm {
t.Fatalf("expected protected glossary term rejection, got %+v", module.ValidatorRejected[0])
}
}
func TestRunProcessExplicitSpokenWordMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
t.Cleanup(func() { processProposalLLMClient = nil })
var stdout, stderr bytes.Buffer
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "spoken_word",
"--work-dir", workDir,
"--work-dir-retention", "always",
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatal("expected failure")
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "runner_execution") {
t.Fatalf("expected runner_execution error, got %q", stderr.String())
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log on failed spoken_word run: %v", err)
}
report := readProcessReport(t, reportPath)
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
t.Fatalf("expected failed runner_execution report, got %+v", report)
}
}
func TestRunProcessExplicitSpokenWordThenGrammarSeesWorkingTranscriptChanges(t *testing.T) {
proposalClient := &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "i i think", CorrectedText: "i think", Confidence: 0.99},
},
},
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "i think we should go.", CorrectedText: "I think we should go.", Confidence: 0.99},
},
},
},
}
validationClient := &fakeStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
},
}
processProposalLLMClient = proposalClient
processValidationLLMClient = validationClient
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
var stdout, stderr bytes.Buffer
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"i i think we should go."}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "spoken_word,grammar",
"--output", outputPath,
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
parsed, err := schema.ParseTranscriptJSON(readFile(t, outputPath))
if err != nil {
t.Fatalf("parse output: %v", err)
}
if parsed.Segments[0].Text != "I think we should go." {
t.Fatalf("expected grammar stage to see spoken_word output, got %q", parsed.Segments[0].Text)
}
report := readProcessReport(t, reportPath)
if len(report.ModuleResults) != 2 {
t.Fatalf("expected two module results, got %+v", report.ModuleResults)
}
if report.ModuleResults[0].ModuleKey != "spoken_word" || report.ModuleResults[1].ModuleKey != "grammar" {
t.Fatalf("expected spoken_word then grammar results, got %+v", report.ModuleResults)
}
if len(report.ModuleResults[0].AppliedChanges) != 1 || len(report.ModuleResults[1].AppliedChanges) != 1 {
t.Fatalf("expected one applied change per module, got %+v", report.ModuleResults)
}
}
func TestRunProcessChunkingSummaryArtifactWritten(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer