Added support for multiple LLM passes to correct the transcript based upon the provided glossary

tests/test_pipeline.py

@@ -9,18 +9,21 @@ class FakeLLMClient:
     def __init__(self, responses):
         self.responses = list(responses)
         self.calls = 0
+        self.messages = []
 
     def create_corrections(self, messages, config):
         self.calls += 1
+        self.messages.append(messages)
         return self.responses.pop(0)
 
 
-def _config(tmp_path):
+def _config(tmp_path, glossary_max_llm_passes=3):
     return AuditaConfig(
         api_key="key",
         max_section_tokens=16000,
         confidence_threshold=0.8,
         max_retries=3,
+        glossary_max_llm_passes=glossary_max_llm_passes,
         work_dir=tmp_path / "work",
     )
 
@@ -40,7 +43,8 @@ def _transcript():
     return parse_transcript_json(
         """
         [
-          {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I ask Chontia."}
+          {"speaker": "Eric", "start": 10.0, "end": 11.0, "text": "I ask Chontia."},
+          {"speaker": "Mike", "start": 0.0, "end": 1.0, "text": "Then Lyra."}
         ]
         """
     )
@@ -62,7 +66,8 @@ def test_pipeline_processes_with_fake_llm_and_cleans_work_dir(tmp_path):
         llm_client=fake_client,
     )
 
-    assert revised[0].text == "I ask Chauntea."
+    assert [segment.speaker for segment in revised] == ["Mike", "Eric"]
+    assert revised[1].text == "I ask Chauntea."
     assert fake_client.calls == 1
     assert list((tmp_path / "work").iterdir()) == []
 
@@ -80,12 +85,12 @@ def test_pipeline_skips_bad_correction_and_preserves_diagnostics(tmp_path):
     revised = process_transcript(
         _transcript(),
         _glossary(),
-        _config(tmp_path),
+        _config(tmp_path, glossary_max_llm_passes=1),
         llm_client=fake_client,
         progress=progress.append,
     )
 
-    assert revised[0].text == "I ask Chontia."
+    assert revised[1].text == "I ask Chontia."
     assert any("Skipping correction for segment 0" in message for message in progress)
     preserved = list((tmp_path / "work").iterdir())
     assert len(preserved) == 1
@@ -94,3 +99,106 @@ def test_pipeline_skips_bad_correction_and_preserves_diagnostics(tmp_path):
     diagnostics = json.loads(skipped_path.read_text(encoding="utf-8"))
     assert diagnostics["skipped_corrections"][0]["segment_id"] == 0
     assert "does not match any substring" in diagnostics["skipped_corrections"][0]["reason"]
+
+
+def test_pipeline_retries_skipped_segment_and_cleans_work_dir_when_fixed(tmp_path):
+    first_pass = CorrectionCandidate(
+        segment_id=0,
+        original_text="Contia",
+        corrected_text="Chauntea",
+        confidence=0.95,
+    )
+    second_pass = CorrectionCandidate(
+        segment_id=0,
+        original_text="Chontia",
+        corrected_text="Chauntea",
+        confidence=0.95,
+    )
+    fake_client = FakeLLMClient(
+        [
+            CorrectionSet(corrections=[first_pass]),
+            CorrectionSet(corrections=[second_pass]),
+        ]
+    )
+
+    revised = process_transcript(
+        _transcript(),
+        _glossary(),
+        _config(tmp_path, glossary_max_llm_passes=3),
+        llm_client=fake_client,
+    )
+
+    assert fake_client.calls == 2
+    assert [segment.speaker for segment in revised] == ["Mike", "Eric"]
+    assert revised[1].text == "I ask Chauntea."
+    assert list((tmp_path / "work").iterdir()) == []
+
+
+def test_pipeline_retry_prompt_contains_only_valid_deduped_segment_ids(tmp_path):
+    first_bad = CorrectionCandidate(
+        segment_id=0,
+        original_text="Contia",
+        corrected_text="Chauntea",
+        confidence=0.95,
+    )
+    second_bad_same_segment = CorrectionCandidate(
+        segment_id=0,
+        original_text="Still wrong",
+        corrected_text="Chauntea",
+        confidence=0.95,
+    )
+    invalid_segment = CorrectionCandidate(
+        segment_id=99,
+        original_text="Missing",
+        corrected_text="Chauntea",
+        confidence=0.95,
+    )
+    fake_client = FakeLLMClient(
+        [
+            CorrectionSet(corrections=[first_bad, second_bad_same_segment, invalid_segment]),
+            CorrectionSet(corrections=[]),
+        ]
+    )
+
+    process_transcript(
+        _transcript(),
+        _glossary(),
+        _config(tmp_path, glossary_max_llm_passes=2),
+        llm_client=fake_client,
+    )
+
+    assert fake_client.calls == 2
+    retry_prompt = fake_client.messages[1][1]["content"]
+    retry_payload = json.loads(retry_prompt.split("Transcript section:\n", maxsplit=1)[1])
+    assert retry_payload == [{"segment_id": 0, "original_text": "I ask Chontia."}]
+    assert "Retry guidance" in retry_prompt
+
+
+def test_pipeline_writes_pass_metadata_for_unresolved_retries(tmp_path):
+    first_pass = CorrectionCandidate(
+        segment_id=0,
+        original_text="Contia",
+        corrected_text="Chauntea",
+        confidence=0.95,
+    )
+    fake_client = FakeLLMClient(
+        [
+            CorrectionSet(corrections=[first_pass]),
+            CorrectionSet(corrections=[]),
+        ]
+    )
+
+    process_transcript(
+        _transcript(),
+        _glossary(),
+        _config(tmp_path, glossary_max_llm_passes=2),
+        llm_client=fake_client,
+    )
+
+    run_dirs = list((tmp_path / "work").iterdir())
+    assert len(run_dirs) == 1
+    metadata = json.loads((run_dirs[0] / "metadata.json").read_text(encoding="utf-8"))
+    assert metadata["glossary_max_llm_passes"] == 2
+    assert [item["pass_number"] for item in metadata["passes"]] == [1, 2]
+    assert metadata["passes"][0]["retry_segment_count"] == 1
+    assert metadata["passes"][1]["retry_pass"] is True