Raise default LLM confidence thresholds and automatically retry segments with low-confidence corrections

src/audita/config.py

@@ -10,8 +10,8 @@ from .errors import AuditaConfigError
 DEFAULT_MODEL = "openrouter/google/gemma-4-31b-it"
 DEFAULT_BASE_URL = "https://openrouter.ai/api/v1"
 DEFAULT_MAX_SECTION_TOKENS = 6144
-DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD = 0.60
+DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD = 0.80
-DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD = 0.60
+DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD = 0.80
 DEFAULT_MAX_RETRIES = 3
 DEFAULT_WORK_DIR = "/tmp/audita"
 DEFAULT_GLOSSARY_MAX_LLM_PASSES = 3

src/audita/corrections.py

@@ -29,6 +29,7 @@ class CorrectionApplicationResult:
     skipped: List[SkippedCorrection]
     applied_ids: List[int]
     ignored_ids: List[int]
+    ignored: List[SkippedCorrection]
 
 
 def apply_corrections(
@@ -48,9 +49,11 @@ def apply_corrections(
     skipped: List[SkippedCorrection] = []
     applied_ids: List[int] = []
     ignored_ids: List[int] = []
+    ignored: List[SkippedCorrection] = []
     for correction in corrections:
         if correction.confidence < confidence_threshold:
             ignored_ids.append(correction.id)
+            ignored.append(_skip(correction, "correction confidence below threshold"))
             continue
 
         reason, actual_text = _target_error(revised, id_to_position, correction, replacement_mode)
@@ -74,6 +77,7 @@ def apply_corrections(
         skipped=skipped,
         applied_ids=applied_ids,
         ignored_ids=ignored_ids,
+        ignored=ignored,
     )
 
 

src/audita/pipeline.py

@@ -216,6 +216,11 @@ def _run_correction_stage(
         working = application_result.transcript
 
         next_retry_skips: Dict[int, SkippedCorrection] = {}
+        for ignored in application_result.ignored:
+            if _is_retryable_skip(ignored, working):
+                next_retry_skips[ignored.id] = ignored
+            else:
+                final_nonretry_skips.append(ignored)
         for skipped in application_result.skipped:
             if _is_retryable_skip(skipped, working):
                 next_retry_skips[skipped.id] = skipped

tests/test_config.py

@@ -25,9 +25,9 @@ def test_config_uses_defaults_with_api_key():
     config = AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "key"})
 
     assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD
-    assert config.glossary_confidence_threshold == 0.6
+    assert config.glossary_confidence_threshold == 0.8
     assert config.grammar_confidence_threshold == DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD
-    assert config.grammar_confidence_threshold == 0.6
+    assert config.grammar_confidence_threshold == 0.8
     assert config.max_section_tokens == DEFAULT_MAX_SECTION_TOKENS
     assert config.max_retries == DEFAULT_MAX_RETRIES
     assert config.glossary_max_llm_passes == DEFAULT_GLOSSARY_MAX_LLM_PASSES

tests/test_corrections.py

@@ -50,6 +50,10 @@ def test_apply_corrections_ignores_below_threshold():
 
     assert result.transcript[0].text == "I ask Chontia for help."
     assert result.skipped == []
+    assert result.ignored_ids == [1]
+    assert len(result.ignored) == 1
+    assert result.ignored[0].id == 1
+    assert result.ignored[0].reason == "correction confidence below threshold"
 
 
 def test_apply_corrections_allows_multiple_distinct_spans_in_one_segment():

tests/test_pipeline.py

@@ -381,6 +381,133 @@ def test_pipeline_retries_skipped_segment_and_cleans_work_dir_when_fixed(tmp_pat
     assert list((tmp_path / "work").iterdir()) == []
 
 
+def test_glossary_below_threshold_correction_retries_segment(tmp_path):
+    low_confidence = CorrectionCandidate(
+        id=1,
+        original_text="Chontia",
+        corrected_text="Chauntea",
+        confidence=0.7,
+    )
+    retry_correction = CorrectionCandidate(
+        id=1,
+        original_text="Chontia",
+        corrected_text="Chauntea",
+        confidence=0.95,
+    )
+    fake_client = FakeLLMClient(
+        [
+            CorrectionSet(corrections=[low_confidence]),
+            CorrectionSet(corrections=[retry_correction]),
+            CorrectionSet(corrections=[]),
+        ]
+    )
+
+    revised = process_transcript(
+        _transcript(),
+        _glossary(),
+        _config(tmp_path, glossary_max_llm_passes=3),
+        llm_client=fake_client,
+    )
+
+    assert fake_client.calls == 3
+    retry_prompt = fake_client.messages[1][1]["content"]
+    retry_payload = json.loads(retry_prompt.split("Transcript section:\n", maxsplit=1)[1])
+    assert retry_payload == [{"id": 1, "original_text": "I ask Chontia."}]
+    assert revised[0].text == "I ask Chauntea."
+    assert list((tmp_path / "work").iterdir()) == []
+
+
+def test_empty_retry_response_after_low_confidence_glossary_correction_stops_retrying(tmp_path):
+    low_confidence = CorrectionCandidate(
+        id=1,
+        original_text="Chontia",
+        corrected_text="Chauntea",
+        confidence=0.7,
+    )
+    fake_client = FakeLLMClient(
+        [
+            CorrectionSet(corrections=[low_confidence]),
+            CorrectionSet(corrections=[]),
+            CorrectionSet(corrections=[]),
+            CorrectionSet(corrections=[]),
+        ]
+    )
+
+    revised = process_transcript(
+        _transcript(),
+        _glossary(),
+        _config(tmp_path, glossary_max_llm_passes=3),
+        llm_client=fake_client,
+    )
+
+    assert fake_client.calls == 3
+    assert "readability corrections" in fake_client.messages[2][1]["content"]
+    assert revised[0].text == "I ask Chontia."
+    assert list((tmp_path / "work").iterdir()) == []
+
+
+def test_final_below_threshold_correction_preserves_diagnostics(tmp_path):
+    low_confidence = CorrectionCandidate(
+        id=1,
+        original_text="Chontia",
+        corrected_text="Chauntea",
+        confidence=0.7,
+    )
+    fake_client = FakeLLMClient(
+        [
+            CorrectionSet(corrections=[low_confidence]),
+            CorrectionSet(corrections=[]),
+        ]
+    )
+
+    revised = process_transcript(
+        _transcript(),
+        _glossary(),
+        _config(tmp_path, glossary_max_llm_passes=1),
+        llm_client=fake_client,
+    )
+
+    assert revised[0].text == "I ask Chontia."
+    run_dirs = list((tmp_path / "work").iterdir())
+    assert len(run_dirs) == 1
+    diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
+    skipped = diagnostics["skipped_corrections"][0]
+    assert skipped["stage"] == "glossary"
+    assert skipped["id"] == 1
+    assert skipped["reason"] == "correction confidence below threshold"
+
+
+def test_below_threshold_invalid_id_is_not_retried(tmp_path):
+    low_confidence_invalid_id = CorrectionCandidate(
+        id=99,
+        original_text="Missing",
+        corrected_text="Chauntea",
+        confidence=0.7,
+    )
+    fake_client = FakeLLMClient(
+        [
+            CorrectionSet(corrections=[low_confidence_invalid_id]),
+            CorrectionSet(corrections=[]),
+        ]
+    )
+
+    process_transcript(
+        _transcript(),
+        _glossary(),
+        _config(tmp_path, glossary_max_llm_passes=3),
+        llm_client=fake_client,
+    )
+
+    assert fake_client.calls == 2
+    run_dirs = list((tmp_path / "work").iterdir())
+    assert len(run_dirs) == 1
+    diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
+    skipped = diagnostics["skipped_corrections"][0]
+    assert skipped["stage"] == "glossary"
+    assert skipped["id"] == 99
+    assert skipped["reason"] == "correction confidence below threshold"
+
+
 def test_empty_glossary_retry_response_stops_retrying_segment(tmp_path):
     first_pass = CorrectionCandidate(
         id=1,
@@ -1001,17 +1128,24 @@ def test_grammar_stage_retries_repeated_span_and_applies_unique_retry(tmp_path):
     assert retry_payload == [{"id": 1, "original_text": "there and there."}]
 
 
-def test_below_threshold_grammar_corrections_are_not_retried(tmp_path):
+def test_below_threshold_grammar_correction_retries_segment(tmp_path):
     correction = CorrectionCandidate(
         id=1,
-        original_text="I",
+        original_text="Chontia",
-        corrected_text="i",
+        corrected_text="Chauntea",
         confidence=0.7,
     )
+    retry_correction = CorrectionCandidate(
+        id=1,
+        original_text="Chontia",
+        corrected_text="Chauntea",
+        confidence=0.95,
+    )
     fake_client = FakeLLMClient(
         [
             CorrectionSet(corrections=[]),
             CorrectionSet(corrections=[correction]),
+            CorrectionSet(corrections=[retry_correction]),
         ]
     )
 
@@ -1022,8 +1156,12 @@ def test_below_threshold_grammar_corrections_are_not_retried(tmp_path):
         llm_client=fake_client,
     )
 
-    assert fake_client.calls == 2
+    assert fake_client.calls == 3
-    assert revised[0].text == "I ask Chontia."
+    retry_prompt = fake_client.messages[2][1]["content"]
+    retry_payload = json.loads(retry_prompt.split("Transcript section:\n", maxsplit=1)[1])
+    assert retry_payload == [{"id": 1, "original_text": "I ask Chontia."}]
+    assert revised[0].text == "I ask Chauntea."
+    assert list((tmp_path / "work").iterdir()) == []
 
 
 def test_unresolved_grammar_skip_preserves_diagnostics(tmp_path):