Improvements around reporting and work directory retention options

tests/test_pipeline.py

@@ -4,7 +4,8 @@ import pytest
 
 from audita.config import AuditaConfig
 from audita.errors import AuditaError
-from audita.pipeline import process_transcript
+from audita.io import write_report
+from audita.pipeline import process_transcript, process_transcript_result
 from audita.schemas import (
     CorrectionCandidate,
     GrammarSpokenFormValidationDecision,
@@ -56,6 +57,7 @@ def _config(
     grammar_validation_enabled=False,
     grammar_validation_confidence_threshold=0.8,
     grammar_spoken_form_validation_confidence_threshold=0.8,
+    work_dir_retention="auto",
 ):
     return AuditaConfig(
         api_key="key",
@@ -69,6 +71,7 @@ def _config(
         grammar_validation_confidence_threshold=grammar_validation_confidence_threshold,
         grammar_spoken_form_validation_confidence_threshold=grammar_spoken_form_validation_confidence_threshold,
         work_dir=tmp_path / "work",
+        work_dir_retention=work_dir_retention,
     )
 
 
@@ -713,6 +716,8 @@ def test_pipeline_writes_stage_metadata_for_unresolved_retries(tmp_path):
     assert metadata["grammar_validation_enabled"] is False
     assert metadata["grammar_validation_confidence_threshold"] == 0.8
     assert metadata["grammar_spoken_form_validation_confidence_threshold"] == 0.8
+    assert metadata["work_dir_retention"] == "auto"
+    assert metadata["work_dir_retained"] is True
     assert [item["stage"] for item in metadata["stages"]] == ["glossary", "grammar"]
     assert [item["pass_number"] for item in metadata["stages"][0]["passes"]] == [1, 2]
     assert metadata["stages"][0]["passes"][0]["retry_segment_count"] == 1
@@ -1387,3 +1392,166 @@ def test_unresolved_grammar_skip_preserves_diagnostics(tmp_path):
     diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
     assert diagnostics["skipped_corrections"][0]["stage"] == "grammar"
     assert "more than once" in diagnostics["skipped_corrections"][0]["reason"]
+
+
+def test_process_transcript_result_returns_report_with_applied_changes(tmp_path):
+    correction = CorrectionCandidate(
+        id=1,
+        original_text="Chontia",
+        corrected_text="Chauntea",
+        confidence=0.95,
+    )
+    fake_client = FakeLLMClient(
+        [
+            CorrectionSet(corrections=[correction]),
+            CorrectionSet(corrections=[]),
+        ]
+    )
+
+    result = process_transcript_result(
+        _transcript(),
+        _glossary(),
+        _config(tmp_path),
+        llm_client=fake_client,
+    )
+
+    assert result.transcript[0].text == "I ask Chauntea."
+    assert result.work_dir_retained is False
+    assert result.report.status == "success"
+    assert result.report.work_dir_retained is False
+    assert result.report.work_dir is None
+    assert result.report.totals["applied_change_count"] == 1
+    assert result.report.skipped_corrections == []
+    assert result.report.applied_changes[0].stage == "glossary"
+    assert result.report.applied_changes[0].pass_number == 1
+    assert result.report.applied_changes[0].segment_text_before == "I ask Chontia."
+    assert result.report.applied_changes[0].segment_text_after == "I ask Chauntea."
+
+    report_path = tmp_path / "result-report.json"
+    write_report(report_path, result.report)
+    written = json.loads(report_path.read_text(encoding="utf-8"))
+    assert written["totals"]["applied_change_count"] == 1
+    assert written["applied_changes"][0]["stage"] == "glossary"
+
+
+def test_auto_retains_work_dir_on_success_with_skipped_corrections(tmp_path):
+    correction = CorrectionCandidate(
+        id=1,
+        original_text="Different text.",
+        corrected_text="Chauntea",
+        confidence=0.95,
+    )
+    fake_client = FakeLLMClient(
+        [
+            CorrectionSet(corrections=[correction]),
+            CorrectionSet(corrections=[]),
+        ]
+    )
+
+    result = process_transcript_result(
+        _transcript(),
+        _glossary(),
+        _config(tmp_path, glossary_max_llm_passes=1, work_dir_retention="auto"),
+        llm_client=fake_client,
+    )
+
+    assert result.work_dir_retained is True
+    assert result.run_dir.exists()
+    assert (result.run_dir / "report.json").exists()
+    report_json = json.loads((result.run_dir / "report.json").read_text(encoding="utf-8"))
+    assert report_json["work_dir_retained"] is True
+    assert report_json["skipped_corrections"][0]["stage"] == "glossary"
+
+
+def test_never_retention_removes_work_dir_after_success_even_with_skipped_corrections(tmp_path):
+    correction = CorrectionCandidate(
+        id=1,
+        original_text="Different text.",
+        corrected_text="Chauntea",
+        confidence=0.95,
+    )
+    fake_client = FakeLLMClient(
+        [
+            CorrectionSet(corrections=[correction]),
+            CorrectionSet(corrections=[]),
+        ]
+    )
+
+    result = process_transcript_result(
+        _transcript(),
+        _glossary(),
+        _config(tmp_path, glossary_max_llm_passes=1, work_dir_retention="never"),
+        llm_client=fake_client,
+    )
+
+    assert result.work_dir_retained is False
+    assert not result.run_dir.exists()
+    assert result.report.work_dir_retained is False
+    assert result.report.skipped_corrections[0].stage == "glossary"
+
+
+def test_always_preserves_work_dir_after_clean_success(tmp_path):
+    correction = CorrectionCandidate(
+        id=1,
+        original_text="Chontia",
+        corrected_text="Chauntea",
+        confidence=0.95,
+    )
+    fake_client = FakeLLMClient(
+        [
+            CorrectionSet(corrections=[correction]),
+            CorrectionSet(corrections=[]),
+        ]
+    )
+
+    result = process_transcript_result(
+        _transcript(),
+        _glossary(),
+        _config(tmp_path, work_dir_retention="always"),
+        llm_client=fake_client,
+    )
+
+    assert result.work_dir_retained is True
+    assert result.run_dir.exists()
+    assert (result.run_dir / "report.json").exists()
+    report_json = json.loads((result.run_dir / "report.json").read_text(encoding="utf-8"))
+    assert report_json["work_dir_retention"] == "always"
+    assert report_json["skipped_corrections"] == []
+
+
+def test_failure_preserves_work_dir_and_writes_failure_report(tmp_path):
+    transcript = parse_source_transcript_json(
+        """
+        [
+          {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "He became visible."}
+        ]
+        """
+    )
+    grammar_correction = CorrectionCandidate(
+        id=1,
+        original_text="visible",
+        corrected_text="invisible",
+        confidence=0.95,
+    )
+    fake_client = FakeLLMClient(
+        [
+            CorrectionSet(corrections=[]),
+            CorrectionSet(corrections=[grammar_correction]),
+        ],
+        validation_responses=[GrammarValidationSet(validations=[])],
+    )
+
+    with pytest.raises(AuditaError):
+        process_transcript_result(
+            transcript,
+            _glossary(),
+            _config(tmp_path, grammar_validation_enabled=True),
+            llm_client=fake_client,
+        )
+
+    run_dirs = list((tmp_path / "work").iterdir())
+    assert len(run_dirs) == 1
+    report_json = json.loads((run_dirs[0] / "report.json").read_text(encoding="utf-8"))
+    assert report_json["status"] == "failed"
+    assert report_json["work_dir_retained"] is True
+    assert report_json["error"] is not None