package cli import ( "bytes" "context" "encoding/json" "errors" "os" "path/filepath" "strings" "testing" "gitea.maximumdirect.net/eric/audita/internal/core/normalization" "gitea.maximumdirect.net/eric/audita/internal/core/reporting" "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/modules" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/validators" ) func TestRunRootHelp(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer exitCode := Run([]string{"--help"}, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected exit code 0, got %d", exitCode) } if !strings.Contains(stdout.String(), "audita [options]") { t.Fatalf("expected root usage in stdout, got %q", stdout.String()) } if stderr.Len() != 0 { t.Fatalf("expected empty stderr, got %q", stderr.String()) } } func TestRunProcessHelpListsExpectedFlags(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer exitCode := Run([]string{"process", "--help"}, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected exit code 0, got %d", exitCode) } for _, expectedFlag := range []string{ "--glossary", "--output", "--report-json", "--modules", "--llm-api-key", "--validation-llm-api-key", "--model", "--validation-model", "--base-url", "--validation-base-url", "--llm-timeout-seconds", "--validation-llm-timeout-seconds", "--validation-max-prompt-tokens", "--target-sections", "--max-retries", "--validation-max-retries", "--validation-llm-concurrency", "--max-section-tokens", "--min-section-tokens", "--glossary-confidence-threshold", "--grammar-confidence-threshold", "--homophones-confidence-threshold", "--spoken-word-confidence-threshold", "--normalize-max-segment-gap", "--normalize-ellipsis-gap", "--normalize-max-segment-duration", "--normalize-max-segment-tokens", "--work-dir", "--work-dir-retention", } { if !strings.Contains(stdout.String(), expectedFlag) { t.Fatalf("expected process help to include %q", expectedFlag) } } if stderr.Len() != 0 { t.Fatalf("expected empty stderr, got %q", stderr.String()) } } func TestRunProcessMissingTranscriptPath(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer exitCode := Run([]string{"process", "--glossary", fixturePath("tiny_glossary.yaml")}, &stdout, &stderr) if exitCode == 0 { t.Fatalf("expected nonzero exit code") } if stdout.Len() != 0 { t.Fatalf("expected empty stdout, got %q", stdout.String()) } if !strings.Contains(stderr.String(), "expected exactly 1 transcript JSON path argument") { t.Fatalf("expected missing transcript error, got %q", stderr.String()) } } func TestRunProcessMissingGlossary(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer exitCode := Run([]string{"process", fixturePath("tiny_transcript.json")}, &stdout, &stderr) if exitCode == 0 { t.Fatalf("expected nonzero exit code") } if stdout.Len() != 0 { t.Fatalf("expected empty stdout, got %q", stdout.String()) } if !strings.Contains(stderr.String(), "--glossary is required") { t.Fatalf("expected missing glossary error, got %q", stderr.String()) } } func TestRunProcessAcceptsBothTranscriptTopLevelForms(t *testing.T) { testCases := []string{ schemaFixturePath("transcript_bare_array.json"), schemaFixturePath("transcript_object.json"), } for _, transcriptPath := range testCases { t.Run(filepath.Base(transcriptPath), func(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer exitCode := Run([]string{"process", transcriptPath, "--glossary", fixturePath("tiny_glossary.yaml")}, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) } if stderr.Len() != 0 { t.Fatalf("expected empty stderr on success, got %q", stderr.String()) } parsed, err := schema.ParseTranscriptJSON(stdout.Bytes()) if err != nil { t.Fatalf("expected normalized transcript JSON output, got %v", err) } if len(parsed.Segments) == 0 { t.Fatalf("expected at least one output segment") } }) } } func TestRunProcessOutputToFileLeavesStdoutEmpty(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer transcriptPath := fixturePath("tiny_transcript.json") glossaryPath := fixturePath("tiny_glossary.yaml") outputPath := filepath.Join(t.TempDir(), "normalized.json") exitCode := Run([]string{"process", transcriptPath, "--glossary", glossaryPath, "--output", outputPath}, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) } if stdout.Len() != 0 { t.Fatalf("expected empty stdout with --output, got %q", stdout.String()) } if stderr.Len() != 0 { t.Fatalf("expected empty stderr on success, got %q", stderr.String()) } outputBytes := readFile(t, outputPath) if !json.Valid(outputBytes) { t.Fatalf("expected valid JSON output file, got %q", string(outputBytes)) } } func TestRunProcessInvalidTranscriptSchema(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer exitCode := Run([]string{"process", schemaFixturePath("transcript_empty_speaker.json"), "--glossary", fixturePath("tiny_glossary.yaml")}, &stdout, &stderr) if exitCode == 0 { t.Fatalf("expected nonzero exit code for invalid transcript schema") } if stdout.Len() != 0 { t.Fatalf("expected empty stdout, got %q", stdout.String()) } if !strings.Contains(stderr.String(), "transcript_schema") { t.Fatalf("expected transcript schema phase in stderr, got %q", stderr.String()) } if !strings.Contains(stderr.String(), "speaker") { t.Fatalf("expected speaker validation details, got %q", stderr.String()) } } func TestRunProcessInvalidGlossarySchema(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer exitCode := Run([]string{"process", fixturePath("tiny_transcript.json"), "--glossary", schemaFixturePath("glossary_missing_fields.yaml")}, &stdout, &stderr) if exitCode == 0 { t.Fatalf("expected nonzero exit code for invalid glossary schema") } if stdout.Len() != 0 { t.Fatalf("expected empty stdout, got %q", stdout.String()) } if !strings.Contains(stderr.String(), "glossary_schema") { t.Fatalf("expected glossary schema phase in stderr, got %q", stderr.String()) } } func TestRunProcessCLIOverridesEnvironment(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer transcriptPath := writeFile(t, "transcript.json", `[ {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}, {"id":2,"speaker":"Alice","start":1.5,"end":2.5,"text":"world"} ]`) t.Setenv("AUDITA_NORMALIZE_MAX_SEGMENT_GAP", "0") exitCode := Run([]string{ "process", transcriptPath, "--glossary", fixturePath("tiny_glossary.yaml"), "--normalize-max-segment-gap", "2.0", }, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) } transcript, err := schema.ParseTranscriptJSON(stdout.Bytes()) if err != nil { t.Fatalf("expected valid transcript JSON output: %v", err) } if len(transcript.Segments) != 1 { t.Fatalf("expected merged output from CLI override, got %d segments", len(transcript.Segments)) } } func TestRunProcessReportJSONSuccessIncludesNormalizationSummary(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer transcriptPath := writeFile(t, "transcript.json", `[ {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}, {"id":2,"speaker":"Alice","start":1.5,"end":2.5,"text":"world"} ]`) reportPath := filepath.Join(t.TempDir(), "report.json") outputPath := filepath.Join(t.TempDir(), "normalized.json") exitCode := Run([]string{ "process", transcriptPath, "--glossary", fixturePath("tiny_glossary.yaml"), "--output", outputPath, "--report-json", reportPath, }, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) } if stdout.Len() != 0 { t.Fatalf("expected empty stdout with --output, got %q", stdout.String()) } report := readProcessReport(t, reportPath) if report.Status != "success" { t.Fatalf("expected success status, got %q", report.Status) } if report.Operation != "process" { t.Fatalf("expected operation process, got %q", report.Operation) } if report.InputSegmentCount == nil || *report.InputSegmentCount != 2 { t.Fatalf("expected input segment count 2, got %v", report.InputSegmentCount) } if report.NormalizedSegmentCount == nil || *report.NormalizedSegmentCount != 1 { t.Fatalf("expected normalized segment count 1, got %v", report.NormalizedSegmentCount) } if report.NormalizationMerges == nil || *report.NormalizationMerges != 1 { t.Fatalf("expected 1 merge, got %v", report.NormalizationMerges) } if report.Diagnostics == nil { t.Fatalf("expected diagnostics metadata in success report") } if report.Diagnostics.DirectoryPath == "" { t.Fatalf("expected diagnostics directory path in success report") } if report.Diagnostics.SourceTranscriptPath == "" || report.Diagnostics.ParsedSourceTranscriptPath == "" || report.Diagnostics.NormalizedTranscriptPath == "" || report.Diagnostics.NormalizationSummaryPath == "" || report.Diagnostics.ChunkingSummaryPath == "" || report.Diagnostics.InvocationMetadataPath == "" || report.Diagnostics.RedactedEffectiveConfigPath == "" { t.Fatalf("expected populated diagnostics artifact references in success report: %+v", report.Diagnostics) } } func TestRunProcessReportJSONBestEffortOnFailure(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer reportPath := filepath.Join(t.TempDir(), "report.json") workDir := t.TempDir() exitCode := Run([]string{ "process", fixturePath("malformed_transcript.json"), "--glossary", fixturePath("tiny_glossary.yaml"), "--report-json", reportPath, "--work-dir", workDir, "--work-dir-retention", "always", }, &stdout, &stderr) if exitCode == 0 { t.Fatalf("expected nonzero exit code") } if stdout.Len() != 0 { t.Fatalf("expected empty stdout on failure, got %q", stdout.String()) } report := readProcessReport(t, reportPath) if report.Status != "failed" { t.Fatalf("expected failed status, got %q", report.Status) } if report.ErrorPhase != "transcript_schema" { t.Fatalf("expected transcript_schema error phase, got %q", report.ErrorPhase) } if !strings.Contains(report.ErrorMessage, "not valid JSON") { t.Fatalf("expected parse error message, got %q", report.ErrorMessage) } if report.Diagnostics == nil { t.Fatalf("expected diagnostics metadata in failed report") } if report.Diagnostics.DirectoryPath == "" || report.Diagnostics.ErrorLogPath == "" { t.Fatalf("expected diagnostics directory and error log references in failed report: %+v", report.Diagnostics) } if _, err := os.Stat(report.Diagnostics.ErrorLogPath); err != nil { t.Fatalf("expected error log file at reported path: %v", err) } } func TestRunProcessReportJSONDoesNotLeakAPIKeys(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer primaryKey := "top-secret-primary" validationKey := "top-secret-validation" t.Setenv("AUDITA_LLM_API_KEY", primaryKey) t.Setenv("AUDITA_VALIDATION_LLM_API_KEY", validationKey) reportPath := filepath.Join(t.TempDir(), "report.json") exitCode := Run([]string{ "process", fixturePath("tiny_transcript.json"), "--glossary", fixturePath("tiny_glossary.yaml"), "--report-json", reportPath, }, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) } reportBytes := readFile(t, reportPath) if strings.Contains(string(reportBytes), primaryKey) || strings.Contains(string(reportBytes), validationKey) { t.Fatalf("report leaked API key material: %s", string(reportBytes)) } } func TestRunProcessReportJSONAndRunDirReportShareDiagnosticsMetadata(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer workDir := t.TempDir() reportPath := filepath.Join(t.TempDir(), "report.json") exitCode := Run([]string{ "process", fixturePath("tiny_transcript.json"), "--glossary", fixturePath("tiny_glossary.yaml"), "--report-json", reportPath, "--work-dir", workDir, "--work-dir-retention", "always", }, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) } externalReport := readProcessReport(t, reportPath) runPath := onlyRunDir(t, workDir) runDirReport := readProcessReport(t, filepath.Join(runPath, "report.json")) if externalReport.Diagnostics == nil || runDirReport.Diagnostics == nil { t.Fatalf("expected diagnostics metadata in both reports") } if *externalReport.Diagnostics != *runDirReport.Diagnostics { t.Fatalf("expected same diagnostics metadata in --report-json and run-dir report\nexternal=%+v\nrun-dir=%+v", *externalReport.Diagnostics, *runDirReport.Diagnostics) } } func TestRunProcessWritesNormalizationDiagnosticsArtifacts(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer workDir := t.TempDir() transcriptPath := writeFile(t, "transcript.json", `[ {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}, {"id":2,"speaker":"Alice","start":1.5,"end":2.5,"text":"world"} ]`) exitCode := Run([]string{ "process", transcriptPath, "--glossary", fixturePath("tiny_glossary.yaml"), "--work-dir", workDir, "--work-dir-retention", "always", }, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) } runPath := onlyRunDir(t, workDir) for _, artifact := range []string{ "source-transcript.json", "source-transcript-parsed.json", "normalized-transcript.json", "normalization-summary.json", "report.json", } { if _, err := os.Stat(filepath.Join(runPath, artifact)); err != nil { t.Fatalf("expected artifact %s: %v", artifact, err) } } summaryBytes := readFile(t, filepath.Join(runPath, "normalization-summary.json")) var summary normalization.NormalizationSummary if err := json.Unmarshal(summaryBytes, &summary); err != nil { t.Fatalf("failed to parse normalization summary: %v", err) } if summary.OutputSegmentCount != 1 { t.Fatalf("expected merged output in normalization summary, got %d", summary.OutputSegmentCount) } } func TestRunProcessFailedRunRetainedWithNeverRetention(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer workDir := t.TempDir() exitCode := Run([]string{ "process", schemaFixturePath("transcript_empty_speaker.json"), "--glossary", fixturePath("tiny_glossary.yaml"), "--work-dir", workDir, "--work-dir-retention", "never", }, &stdout, &stderr) if exitCode == 0 { t.Fatalf("expected nonzero exit code") } if stdout.Len() != 0 { t.Fatalf("expected empty stdout on failure, got %q", stdout.String()) } runPath := onlyRunDir(t, workDir) if _, err := os.Stat(filepath.Join(runPath, "error.log")); err != nil { t.Fatalf("expected error.log in retained failed run: %v", err) } } func TestRunProcessSuccessfulRunAutoRetentionRemovesRunDir(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer workDir := t.TempDir() reportPath := filepath.Join(t.TempDir(), "report.json") exitCode := Run([]string{ "process", fixturePath("tiny_transcript.json"), "--glossary", fixturePath("tiny_glossary.yaml"), "--work-dir", workDir, "--work-dir-retention", "auto", "--report-json", reportPath, }, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) } entries, err := os.ReadDir(workDir) if err != nil { t.Fatalf("failed to read work dir: %v", err) } if len(entries) != 0 { t.Fatalf("expected auto retention to remove successful clean run dir, found %d entries", len(entries)) } // Explicit report path must still exist even if run dir is removed. if _, err := os.Stat(reportPath); err != nil { t.Fatalf("expected --report-json output to survive run-dir removal: %v", err) } } func TestRunProcessSuccessfulRunAlwaysRetentionKeepsRunDir(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer workDir := t.TempDir() exitCode := Run([]string{ "process", fixturePath("tiny_transcript.json"), "--glossary", fixturePath("tiny_glossary.yaml"), "--work-dir", workDir, "--work-dir-retention", "always", }, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) } runPath := onlyRunDir(t, workDir) if _, err := os.Stat(runPath); err != nil { t.Fatalf("expected run dir retained under always: %v", err) } } func TestRunProcessFailedRunRetainedWithAutoRetention(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer workDir := t.TempDir() exitCode := Run([]string{ "process", schemaFixturePath("transcript_empty_speaker.json"), "--glossary", fixturePath("tiny_glossary.yaml"), "--work-dir", workDir, "--work-dir-retention", "auto", }, &stdout, &stderr) if exitCode == 0 { t.Fatalf("expected nonzero exit code") } if stdout.Len() != 0 { t.Fatalf("expected empty stdout on failure, got %q", stdout.String()) } runPath := onlyRunDir(t, workDir) if _, err := os.Stat(filepath.Join(runPath, "error.log")); err != nil { t.Fatalf("expected error.log in retained failed run under auto: %v", err) } } func TestRunProcessReportJSONIncludesChunkingSummary(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer transcriptPath := writeFile(t, "transcript.json", `[ {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello world this is a test"}, {"id":2,"speaker":"Bob","start":2.0,"end":3.0,"text":"Another segment here with more text"}, {"id":3,"speaker":"Alice","start":4.0,"end":5.0,"text":"Final segment text"} ]`) reportPath := filepath.Join(t.TempDir(), "report.json") outputPath := filepath.Join(t.TempDir(), "normalized.json") exitCode := Run([]string{ "process", transcriptPath, "--glossary", fixturePath("tiny_glossary.yaml"), "--output", outputPath, "--report-json", reportPath, }, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) } if stdout.Len() != 0 { t.Fatalf("expected empty stdout with --output, got %q", stdout.String()) } report := readProcessReport(t, reportPath) if report.Chunking == nil { t.Fatalf("expected chunking summary in report") } if report.Chunking.ChunkCount == 0 { t.Errorf("expected non-zero chunk count") } if report.Chunking.MaxSectionTokens == 0 { t.Errorf("expected max_section_tokens in report") } if report.Phase != "phase11-proposal-generation-framework" { t.Errorf("expected phase 'phase11-proposal-generation-framework', got %q", report.Phase) } } type fakeModuleFactory struct { modules map[string]contracts.TranscriptModule } func (f fakeModuleFactory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.TranscriptModule, error) { m, ok := f.modules[spec.InstanceName] if !ok { return nil, errors.New("module not registered") } return m, nil } type fakeModule struct { key string policy proposals.ReplacementPolicy validators []contracts.Validator proposeF func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) } func (m fakeModule) Key() string { return m.key } func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy } func (m fakeModule) Validators() []contracts.Validator { return m.validators } func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { if m.proposeF == nil { return nil, nil } return m.proposeF(req) } type fakeValidator struct { name string validateF func(req contracts.ValidationRequest) (validators.Result, error) } func (v fakeValidator) Name() string { return v.name } func (v fakeValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (validators.Result, error) { _ = ctx return v.validateF(req) } type fakeStructuredLLMClient struct { responses []validators.LLMValidationResponse err error } func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { _ = ctx _ = req if f.err != nil { return contracts.StructuredCompletionResponse{}, f.err } if len(f.responses) == 0 { return contracts.StructuredCompletionResponse{}, errors.New("unexpected llm call") } target := out.(*validators.LLMValidationResponse) *target = f.responses[0] f.responses = f.responses[1:] return contracts.StructuredCompletionResponse{}, nil } func TestRunProcessInjectedFactoryExecutesRunnerAndReportsModules(t *testing.T) { allow := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { decisions := make([]validators.Decision, len(req.CandidateProposal)) for i, p := range req.CandidateProposal { decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "approved"} } return validators.Result{ValidatorName: "allow", Decisions: decisions}, nil }} processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{ "m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{ {TargetSegmentID: 1, OriginalText: "Hello", CorrectedText: "Hi", Confidence: 1}, }, nil }}, "m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { if req.WorkingTranscript.Segments[0].Text != "Hi world" { t.Fatalf("expected module 2 to see module 1 changes, got %q", req.WorkingTranscript.Segments[0].Text) } return []proposals.CorrectionProposal{ {TargetSegmentID: 1, OriginalText: "Hi", CorrectedText: "Hey", Confidence: 1}, {TargetSegmentID: 1, OriginalText: "missing", CorrectedText: "noop", Confidence: 1}, }, nil }}, }} t.Cleanup(func() { processModuleFactory = nil }) var stdout, stderr bytes.Buffer workDir := t.TempDir() reportPath := filepath.Join(t.TempDir(), "report.json") outputPath := filepath.Join(t.TempDir(), "out.json") transcriptPath := writeFile(t, "transcript.json", `[ {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello world"} ]`) exitCode := Run([]string{ "process", transcriptPath, "--glossary", fixturePath("tiny_glossary.yaml"), "--modules", "m1,m2", "--output", outputPath, "--report-json", reportPath, "--work-dir", workDir, "--work-dir-retention", "always", }, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected success, got exit %d stderr=%q", exitCode, stderr.String()) } if stdout.Len() != 0 { t.Fatalf("expected empty stdout with --output, got %q", stdout.String()) } out := readFile(t, outputPath) parsed, err := schema.ParseTranscriptJSON(out) if err != nil { t.Fatalf("parse output transcript: %v", err) } if parsed.Segments[0].Text != "Hey world" { t.Fatalf("expected runner-applied output text, got %q", parsed.Segments[0].Text) } report := readProcessReport(t, reportPath) if report.ModulesSummary == nil || report.ModulesSummary.ModuleCount != 2 { t.Fatalf("expected module summary for 2 modules, got %+v", report.ModulesSummary) } if report.ModulesSummary.TotalAppliedChanges != 2 || report.ModulesSummary.TotalSkippedChanges != 1 { t.Fatalf("unexpected module totals: %+v", report.ModulesSummary) } if len(report.ModuleResults) != 2 { t.Fatalf("expected 2 module results, got %d", len(report.ModuleResults)) } if len(report.ModuleResults[1].SkippedChanges) != 1 { t.Fatalf("expected skipped change for module 2, got %+v", report.ModuleResults[1].SkippedChanges) } if len(report.ModuleResults[0].ValidatorDecisions) == 0 { t.Fatalf("expected validator decisions in module report") } runDirReport := readProcessReport(t, filepath.Join(onlyRunDir(t, workDir), "report.json")) if len(runDirReport.ModuleResults) != 2 { t.Fatalf("expected module results in run-dir report") } if len(runDirReport.ModuleResults[1].ValidatorDecisions) == 0 { t.Fatalf("expected validator decisions in run-dir report") } } func TestRunProcessInjectedFactoryLLMValidatorResultsInReports(t *testing.T) { llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "") if err != nil { t.Fatalf("NewLLMBackedValidator: %v", err) } processValidationLLMClient = &fakeStructuredLLMClient{ responses: []validators.LLMValidationResponse{{ Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: false, Confidence: 0.99, Reason: "reject"}}, }}, } processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "Hello", CorrectedText: "Hi", Confidence: 1}}, nil }}, }} t.Cleanup(func() { processModuleFactory = nil processValidationLLMClient = nil processValidationLLMScheduler = nil }) var stdout, stderr bytes.Buffer workDir := t.TempDir() reportPath := filepath.Join(t.TempDir(), "report.json") outputPath := filepath.Join(t.TempDir(), "out.json") transcriptPath := writeFile(t, "transcript.json", `[ {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello world"} ]`) exitCode := Run([]string{ "process", transcriptPath, "--glossary", fixturePath("tiny_glossary.yaml"), "--modules", "m", "--output", outputPath, "--report-json", reportPath, "--work-dir", workDir, "--work-dir-retention", "always", }, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String()) } report := readProcessReport(t, reportPath) if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].ValidatorDecisions) == 0 { t.Fatalf("expected llm validator decisions in external report") } runReport := readProcessReport(t, filepath.Join(onlyRunDir(t, workDir), "report.json")) if len(runReport.ModuleResults) != 1 || len(runReport.ModuleResults[0].ValidatorRejected) != 1 { t.Fatalf("expected llm validator rejection in run report") } } func TestRunProcessInjectedFactorySkippedKeepsAutoRetention(t *testing.T) { processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{ "m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{ {TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term", Confidence: 1}, }, nil }}, }} t.Cleanup(func() { processModuleFactory = nil }) var stdout, stderr bytes.Buffer workDir := t.TempDir() transcriptPath := writeFile(t, "transcript.json", `[ {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"word word"} ]`) exitCode := Run([]string{ "process", transcriptPath, "--glossary", fixturePath("tiny_glossary.yaml"), "--modules", "m1", "--work-dir", workDir, "--work-dir-retention", "auto", }, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String()) } if _, err := os.Stat(onlyRunDir(t, workDir)); err != nil { t.Fatalf("expected run dir retained for skipped corrections under auto: %v", err) } } func TestRunProcessInjectedFactoryFailureWritesFailedReport(t *testing.T) { processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{ "m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return nil, errors.New("test failure") }}, }} t.Cleanup(func() { processModuleFactory = nil }) var stdout, stderr bytes.Buffer workDir := t.TempDir() reportPath := filepath.Join(t.TempDir(), "report.json") transcriptPath := writeFile(t, "transcript.json", `[ {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"} ]`) exitCode := Run([]string{ "process", transcriptPath, "--glossary", fixturePath("tiny_glossary.yaml"), "--modules", "m1", "--work-dir", workDir, "--work-dir-retention", "always", "--report-json", reportPath, }, &stdout, &stderr) if exitCode == 0 { t.Fatal("expected failure exit code") } if stdout.Len() != 0 { t.Fatalf("expected empty stdout on failure, got %q", stdout.String()) } if !strings.Contains(stderr.String(), "runner_execution") { t.Fatalf("expected runner_execution failure on stderr, got %q", stderr.String()) } report := readProcessReport(t, reportPath) if report.Status != "failed" { t.Fatalf("expected failed report status, got %q", report.Status) } if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance == "" { t.Fatalf("expected failed module summary, got %+v", report.ModulesSummary) } if len(report.ModuleResults) != 1 || report.ModuleResults[0].Status != "failed" { t.Fatalf("expected failed module result, got %+v", report.ModuleResults) } runPath := onlyRunDir(t, workDir) if _, err := os.Stat(filepath.Join(runPath, "error.log")); err != nil { t.Fatalf("expected error.log for failed runner module: %v", err) } } func TestRunProcessProductionRegistryUnimplementedModuleFailsCleanly(t *testing.T) { cfg := modules.Dependencies{} processModuleFactory = modules.NewFactory(cfg) t.Cleanup(func() { processModuleFactory = nil }) var stdout, stderr bytes.Buffer workDir := t.TempDir() reportPath := filepath.Join(t.TempDir(), "report.json") transcriptPath := writeFile(t, "transcript.json", `[ {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"} ]`) exitCode := Run([]string{ "process", transcriptPath, "--glossary", fixturePath("tiny_glossary.yaml"), "--modules", "glossary", "--work-dir", workDir, "--work-dir-retention", "always", "--report-json", reportPath, }, &stdout, &stderr) if exitCode == 0 { t.Fatal("expected failure exit code") } if stdout.Len() != 0 { t.Fatalf("expected empty stdout on failure, got %q", stdout.String()) } if !strings.Contains(stderr.String(), "runner_execution") { t.Fatalf("expected runner_execution failure on stderr, got %q", stderr.String()) } if !strings.Contains(stderr.String(), "recognized but not implemented") { t.Fatalf("expected explicit unimplemented module message, got %q", stderr.String()) } report := readProcessReport(t, reportPath) if report.Status != "failed" { t.Fatalf("expected failed report status, got %q", report.Status) } if report.ErrorPhase != "runner_execution" { t.Fatalf("expected runner_execution phase, got %q", report.ErrorPhase) } if !strings.Contains(report.ErrorMessage, "recognized but not implemented") { t.Fatalf("expected report error message to mention unimplemented module, got %q", report.ErrorMessage) } } func TestRunProcessChunkingSummaryArtifactWritten(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer workDir := t.TempDir() transcriptPath := writeFile(t, "transcript.json", `[ {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}, {"id":2,"speaker":"Bob","start":2.0,"end":3.0,"text":"World"} ]`) exitCode := Run([]string{ "process", transcriptPath, "--glossary", fixturePath("tiny_glossary.yaml"), "--work-dir", workDir, "--work-dir-retention", "always", }, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) } runPath := onlyRunDir(t, workDir) // Verify chunking-summary.json exists chunkingSummaryPath := filepath.Join(runPath, "chunking-summary.json") if _, err := os.Stat(chunkingSummaryPath); err != nil { t.Fatalf("expected chunking-summary.json artifact: %v", err) } // Verify it contains valid JSON with expected fields content := readFile(t, chunkingSummaryPath) var summary struct { ChunkCount int `json:"chunk_count"` MinEstimatedTokens int `json:"min_estimated_chunk_tokens"` MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"` TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"` Chunks []struct { Index int `json:"index"` StartSegmentID int `json:"start_segment_id"` EndSegmentID int `json:"end_segment_id"` EstimatedTokens int `json:"estimated_tokens"` SegmentCount int `json:"segment_count"` } `json:"chunks"` } if err := json.Unmarshal(content, &summary); err != nil { t.Fatalf("failed to parse chunking summary: %v", err) } if summary.ChunkCount == 0 { t.Errorf("expected non-zero chunk count in summary") } if len(summary.Chunks) != summary.ChunkCount { t.Errorf("expected %d chunks, got %d", summary.ChunkCount, len(summary.Chunks)) } } func TestRunProcessWritesRedactedRunMetadataArtifacts(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer workDir := t.TempDir() outputPath := filepath.Join(t.TempDir(), "normalized.json") reportPath := filepath.Join(t.TempDir(), "report.json") primaryKey := "super-secret-primary-key" validationKey := "super-secret-validation-key" t.Setenv("AUDITA_LLM_API_KEY", primaryKey) t.Setenv("AUDITA_VALIDATION_LLM_API_KEY", validationKey) exitCode := Run([]string{ "process", fixturePath("tiny_transcript.json"), "--glossary", fixturePath("tiny_glossary.yaml"), "--output", outputPath, "--report-json", reportPath, "--work-dir", workDir, "--work-dir-retention", "always", }, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) } if stdout.Len() != 0 { t.Fatalf("expected empty stdout when --output is provided, got %q", stdout.String()) } // Existing report-json behavior should still work. report := readProcessReport(t, reportPath) if report.Status != "success" { t.Fatalf("expected success report status, got %q", report.Status) } runPath := onlyRunDir(t, workDir) invocationPath := filepath.Join(runPath, "invocation.json") configPath := filepath.Join(runPath, "effective-config.json") if _, err := os.Stat(invocationPath); err != nil { t.Fatalf("expected invocation metadata artifact: %v", err) } if _, err := os.Stat(configPath); err != nil { t.Fatalf("expected effective config metadata artifact: %v", err) } invocationBytes := readFile(t, invocationPath) configBytes := readFile(t, configPath) if strings.Contains(string(invocationBytes), primaryKey) || strings.Contains(string(invocationBytes), validationKey) { t.Fatalf("invocation artifact leaked API key material") } if strings.Contains(string(configBytes), primaryKey) || strings.Contains(string(configBytes), validationKey) { t.Fatalf("effective config artifact leaked API key material") } var invocation struct { Operation string `json:"operation"` TranscriptPath string `json:"transcript_path"` GlossaryPath string `json:"glossary_path"` OutputPath string `json:"output_path"` ReportJSONPath string `json:"report_json_path"` Modules []string `json:"modules"` RunID string `json:"run_id"` StartedAt string `json:"started_at"` } if err := json.Unmarshal(invocationBytes, &invocation); err != nil { t.Fatalf("failed to parse invocation metadata: %v", err) } if invocation.Operation != "process" { t.Fatalf("expected operation=process, got %q", invocation.Operation) } if invocation.TranscriptPath != fixturePath("tiny_transcript.json") { t.Fatalf("unexpected transcript_path: %q", invocation.TranscriptPath) } if invocation.GlossaryPath != fixturePath("tiny_glossary.yaml") { t.Fatalf("unexpected glossary_path: %q", invocation.GlossaryPath) } if invocation.OutputPath != outputPath { t.Fatalf("unexpected output_path: %q", invocation.OutputPath) } if invocation.ReportJSONPath != reportPath { t.Fatalf("unexpected report_json_path: %q", invocation.ReportJSONPath) } if len(invocation.Modules) == 0 { t.Fatalf("expected non-empty modules list in invocation metadata") } if invocation.RunID == "" { t.Fatalf("expected non-empty run_id in invocation metadata") } if invocation.StartedAt == "" { t.Fatalf("expected non-empty started_at in invocation metadata") } } func TestRunProcessStdoutTranscriptWhenNoOutputWithRunMetadataArtifacts(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer workDir := t.TempDir() exitCode := Run([]string{ "process", fixturePath("tiny_transcript.json"), "--glossary", fixturePath("tiny_glossary.yaml"), "--work-dir", workDir, "--work-dir-retention", "always", }, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) } if _, err := schema.ParseTranscriptJSON(stdout.Bytes()); err != nil { t.Fatalf("expected stdout to contain only transcript JSON, got parse error: %v", err) } runPath := onlyRunDir(t, workDir) if _, err := os.Stat(filepath.Join(runPath, "invocation.json")); err != nil { t.Fatalf("expected invocation metadata artifact: %v", err) } if _, err := os.Stat(filepath.Join(runPath, "effective-config.json")); err != nil { t.Fatalf("expected effective config metadata artifact: %v", err) } } func TestRunProcessTargetSectionsThroughCLI(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer transcriptPath := writeFile(t, "transcript.json", `[ {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello world"}, {"id":2,"speaker":"Bob","start":2.0,"end":3.0,"text":"Another test"}, {"id":3,"speaker":"Charlie","start":4.0,"end":5.0,"text":"Third segment"}, {"id":4,"speaker":"Alice","start":6.0,"end":7.0,"text":"Final segment"} ]`) reportPath := filepath.Join(t.TempDir(), "report.json") exitCode := Run([]string{ "process", transcriptPath, "--glossary", fixturePath("tiny_glossary.yaml"), "--target-sections", "2", "--report-json", reportPath, }, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) } report := readProcessReport(t, reportPath) if report.Chunking == nil { t.Fatalf("expected chunking summary in report") } if report.Chunking.ChunkCount != 2 { t.Errorf("expected 2 chunks with --target-sections 2, got %d", report.Chunking.ChunkCount) } if report.Chunking.TargetSections == nil || *report.Chunking.TargetSections != 2 { t.Errorf("expected target_sections=2 in report") } } func TestRunProcessImpossibleTargetSectionsFailsCleanly(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer // Create transcript with 2 segments transcriptPath := writeFile(t, "transcript.json", `[ {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}, {"id":2,"speaker":"Bob","start":2.0,"end":3.0,"text":"World"} ]`) reportPath := filepath.Join(t.TempDir(), "report.json") workDir := t.TempDir() exitCode := Run([]string{ "process", transcriptPath, "--glossary", fixturePath("tiny_glossary.yaml"), "--target-sections", "5", "--report-json", reportPath, "--work-dir", workDir, "--work-dir-retention", "always", }, &stdout, &stderr) if exitCode == 0 { t.Fatalf("expected nonzero exit code for impossible target-sections") } if stdout.Len() != 0 { t.Fatalf("expected empty stdout on failure, got %q", stdout.String()) } if !strings.Contains(stderr.String(), "target_sections") { t.Errorf("expected target_sections error in stderr, got %q", stderr.String()) } // Verify report was written with failure info report := readProcessReport(t, reportPath) if report.Status != "failed" { t.Errorf("expected failed status, got %q", report.Status) } if report.ErrorPhase != "chunking" { t.Errorf("expected error phase 'chunking', got %q", report.ErrorPhase) } // Verify run dir was retained with error.log runPath := onlyRunDir(t, workDir) if _, err := os.Stat(filepath.Join(runPath, "error.log")); err != nil { t.Errorf("expected error.log in retained failed run: %v", err) } } func TestRunProcessOutputTranscriptStillNormalizedOnly(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer transcriptPath := writeFile(t, "transcript.json", `[ {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}, {"id":2,"speaker":"Alice","start":1.5,"end":2.5,"text":"world"} ]`) outputPath := filepath.Join(t.TempDir(), "normalized.json") exitCode := Run([]string{ "process", transcriptPath, "--glossary", fixturePath("tiny_glossary.yaml"), "--output", outputPath, }, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) } if stdout.Len() != 0 { t.Fatalf("expected empty stdout with --output, got %q", stdout.String()) } // Verify output is the normalized transcript (same as before) output := readFile(t, outputPath) transcript, err := schema.ParseTranscriptJSON(output) if err != nil { t.Fatalf("expected valid transcript JSON: %v", err) } if len(transcript.Segments) != 1 { t.Errorf("expected 1 merged segment, got %d", len(transcript.Segments)) } // Text should be normalized (merged with space) if transcript.Segments[0].Text != "Hello world" { t.Errorf("expected merged text 'Hello world', got %q", transcript.Segments[0].Text) } } func TestRunProcessReportJSONNotPrintedToStdout(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer transcriptPath := writeFile(t, "transcript.json", `[ {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"} ]`) reportPath := filepath.Join(t.TempDir(), "report.json") exitCode := Run([]string{ "process", transcriptPath, "--glossary", fixturePath("tiny_glossary.yaml"), "--report-json", reportPath, }, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) } // Verify stdout contains only transcript JSON, not report JSON stdoutStr := stdout.String() if strings.Contains(stdoutStr, "chunk_count") { t.Errorf("stdout should not contain report JSON fields, got: %s", stdoutStr) } if strings.Contains(stdoutStr, "normalization_merges") { t.Errorf("stdout should not contain report JSON fields, got: %s", stdoutStr) } // Verify transcript is valid JSON if _, err := schema.ParseTranscriptJSON(stdout.Bytes()); err != nil { t.Errorf("stdout should contain valid transcript JSON: %v", err) } } func fixturePath(name string) string { return filepath.Join("testdata", name) } func schemaFixturePath(name string) string { return filepath.Join("..", "core", "schema", "testdata", name) } func writeFile(t *testing.T, name string, content string) string { t.Helper() path := filepath.Join(t.TempDir(), name) if err := os.WriteFile(path, []byte(content), 0o644); err != nil { t.Fatalf("failed to write test file %q: %v", path, err) } return path } func readFile(t *testing.T, path string) []byte { t.Helper() data, err := os.ReadFile(path) if err != nil { t.Fatalf("failed to read file %q: %v", path, err) } return data } func readProcessReport(t *testing.T, path string) reporting.ProcessReport { t.Helper() raw := readFile(t, path) var report reporting.ProcessReport if err := json.Unmarshal(raw, &report); err != nil { t.Fatalf("failed to parse process report JSON: %v (raw=%s)", err, string(raw)) } return report } func onlyRunDir(t *testing.T, workDir string) string { t.Helper() entries, err := os.ReadDir(workDir) if err != nil { t.Fatalf("failed to read work dir %q: %v", workDir, err) } if len(entries) != 1 { t.Fatalf("expected exactly one run dir in %q, got %d", workDir, len(entries)) } return filepath.Join(workDir, entries[0].Name()) }