From 10bec989cef289c003ea7a051ff68dd2caf7b4ad Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Wed, 22 Apr 2026 17:50:18 -0500 Subject: [PATCH] Raise default LLM confidence thresholds and automatically retry segments with low-confidence corrections --- src/audita/config.py | 4 +- src/audita/corrections.py | 4 ++ src/audita/pipeline.py | 5 ++ tests/test_config.py | 4 +- tests/test_corrections.py | 4 ++ tests/test_pipeline.py | 148 ++++++++++++++++++++++++++++++++++++-- 6 files changed, 160 insertions(+), 9 deletions(-) diff --git a/src/audita/config.py b/src/audita/config.py index 7b81dd7..c92889f 100644 --- a/src/audita/config.py +++ b/src/audita/config.py @@ -10,8 +10,8 @@ from .errors import AuditaConfigError DEFAULT_MODEL = "openrouter/google/gemma-4-31b-it" DEFAULT_BASE_URL = "https://openrouter.ai/api/v1" DEFAULT_MAX_SECTION_TOKENS = 6144 -DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD = 0.60 -DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD = 0.60 +DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD = 0.80 +DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD = 0.80 DEFAULT_MAX_RETRIES = 3 DEFAULT_WORK_DIR = "/tmp/audita" DEFAULT_GLOSSARY_MAX_LLM_PASSES = 3 diff --git a/src/audita/corrections.py b/src/audita/corrections.py index 364a1f8..661d8ba 100644 --- a/src/audita/corrections.py +++ b/src/audita/corrections.py @@ -29,6 +29,7 @@ class CorrectionApplicationResult: skipped: List[SkippedCorrection] applied_ids: List[int] ignored_ids: List[int] + ignored: List[SkippedCorrection] def apply_corrections( @@ -48,9 +49,11 @@ def apply_corrections( skipped: List[SkippedCorrection] = [] applied_ids: List[int] = [] ignored_ids: List[int] = [] + ignored: List[SkippedCorrection] = [] for correction in corrections: if correction.confidence < confidence_threshold: ignored_ids.append(correction.id) + ignored.append(_skip(correction, "correction confidence below threshold")) continue reason, actual_text = _target_error(revised, id_to_position, correction, replacement_mode) @@ -74,6 +77,7 @@ def apply_corrections( skipped=skipped, applied_ids=applied_ids, ignored_ids=ignored_ids, + ignored=ignored, ) diff --git a/src/audita/pipeline.py b/src/audita/pipeline.py index 4254508..45365fa 100644 --- a/src/audita/pipeline.py +++ b/src/audita/pipeline.py @@ -216,6 +216,11 @@ def _run_correction_stage( working = application_result.transcript next_retry_skips: Dict[int, SkippedCorrection] = {} + for ignored in application_result.ignored: + if _is_retryable_skip(ignored, working): + next_retry_skips[ignored.id] = ignored + else: + final_nonretry_skips.append(ignored) for skipped in application_result.skipped: if _is_retryable_skip(skipped, working): next_retry_skips[skipped.id] = skipped diff --git a/tests/test_config.py b/tests/test_config.py index ca24852..bfb5921 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -25,9 +25,9 @@ def test_config_uses_defaults_with_api_key(): config = AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "key"}) assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD - assert config.glossary_confidence_threshold == 0.6 + assert config.glossary_confidence_threshold == 0.8 assert config.grammar_confidence_threshold == DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD - assert config.grammar_confidence_threshold == 0.6 + assert config.grammar_confidence_threshold == 0.8 assert config.max_section_tokens == DEFAULT_MAX_SECTION_TOKENS assert config.max_retries == DEFAULT_MAX_RETRIES assert config.glossary_max_llm_passes == DEFAULT_GLOSSARY_MAX_LLM_PASSES diff --git a/tests/test_corrections.py b/tests/test_corrections.py index 0fd8d77..8912e9d 100644 --- a/tests/test_corrections.py +++ b/tests/test_corrections.py @@ -50,6 +50,10 @@ def test_apply_corrections_ignores_below_threshold(): assert result.transcript[0].text == "I ask Chontia for help." assert result.skipped == [] + assert result.ignored_ids == [1] + assert len(result.ignored) == 1 + assert result.ignored[0].id == 1 + assert result.ignored[0].reason == "correction confidence below threshold" def test_apply_corrections_allows_multiple_distinct_spans_in_one_segment(): diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 6385fb5..a65cd76 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -381,6 +381,133 @@ def test_pipeline_retries_skipped_segment_and_cleans_work_dir_when_fixed(tmp_pat assert list((tmp_path / "work").iterdir()) == [] +def test_glossary_below_threshold_correction_retries_segment(tmp_path): + low_confidence = CorrectionCandidate( + id=1, + original_text="Chontia", + corrected_text="Chauntea", + confidence=0.7, + ) + retry_correction = CorrectionCandidate( + id=1, + original_text="Chontia", + corrected_text="Chauntea", + confidence=0.95, + ) + fake_client = FakeLLMClient( + [ + CorrectionSet(corrections=[low_confidence]), + CorrectionSet(corrections=[retry_correction]), + CorrectionSet(corrections=[]), + ] + ) + + revised = process_transcript( + _transcript(), + _glossary(), + _config(tmp_path, glossary_max_llm_passes=3), + llm_client=fake_client, + ) + + assert fake_client.calls == 3 + retry_prompt = fake_client.messages[1][1]["content"] + retry_payload = json.loads(retry_prompt.split("Transcript section:\n", maxsplit=1)[1]) + assert retry_payload == [{"id": 1, "original_text": "I ask Chontia."}] + assert revised[0].text == "I ask Chauntea." + assert list((tmp_path / "work").iterdir()) == [] + + +def test_empty_retry_response_after_low_confidence_glossary_correction_stops_retrying(tmp_path): + low_confidence = CorrectionCandidate( + id=1, + original_text="Chontia", + corrected_text="Chauntea", + confidence=0.7, + ) + fake_client = FakeLLMClient( + [ + CorrectionSet(corrections=[low_confidence]), + CorrectionSet(corrections=[]), + CorrectionSet(corrections=[]), + CorrectionSet(corrections=[]), + ] + ) + + revised = process_transcript( + _transcript(), + _glossary(), + _config(tmp_path, glossary_max_llm_passes=3), + llm_client=fake_client, + ) + + assert fake_client.calls == 3 + assert "readability corrections" in fake_client.messages[2][1]["content"] + assert revised[0].text == "I ask Chontia." + assert list((tmp_path / "work").iterdir()) == [] + + +def test_final_below_threshold_correction_preserves_diagnostics(tmp_path): + low_confidence = CorrectionCandidate( + id=1, + original_text="Chontia", + corrected_text="Chauntea", + confidence=0.7, + ) + fake_client = FakeLLMClient( + [ + CorrectionSet(corrections=[low_confidence]), + CorrectionSet(corrections=[]), + ] + ) + + revised = process_transcript( + _transcript(), + _glossary(), + _config(tmp_path, glossary_max_llm_passes=1), + llm_client=fake_client, + ) + + assert revised[0].text == "I ask Chontia." + run_dirs = list((tmp_path / "work").iterdir()) + assert len(run_dirs) == 1 + diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8")) + skipped = diagnostics["skipped_corrections"][0] + assert skipped["stage"] == "glossary" + assert skipped["id"] == 1 + assert skipped["reason"] == "correction confidence below threshold" + + +def test_below_threshold_invalid_id_is_not_retried(tmp_path): + low_confidence_invalid_id = CorrectionCandidate( + id=99, + original_text="Missing", + corrected_text="Chauntea", + confidence=0.7, + ) + fake_client = FakeLLMClient( + [ + CorrectionSet(corrections=[low_confidence_invalid_id]), + CorrectionSet(corrections=[]), + ] + ) + + process_transcript( + _transcript(), + _glossary(), + _config(tmp_path, glossary_max_llm_passes=3), + llm_client=fake_client, + ) + + assert fake_client.calls == 2 + run_dirs = list((tmp_path / "work").iterdir()) + assert len(run_dirs) == 1 + diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8")) + skipped = diagnostics["skipped_corrections"][0] + assert skipped["stage"] == "glossary" + assert skipped["id"] == 99 + assert skipped["reason"] == "correction confidence below threshold" + + def test_empty_glossary_retry_response_stops_retrying_segment(tmp_path): first_pass = CorrectionCandidate( id=1, @@ -1001,17 +1128,24 @@ def test_grammar_stage_retries_repeated_span_and_applies_unique_retry(tmp_path): assert retry_payload == [{"id": 1, "original_text": "there and there."}] -def test_below_threshold_grammar_corrections_are_not_retried(tmp_path): +def test_below_threshold_grammar_correction_retries_segment(tmp_path): correction = CorrectionCandidate( id=1, - original_text="I", - corrected_text="i", + original_text="Chontia", + corrected_text="Chauntea", confidence=0.7, ) + retry_correction = CorrectionCandidate( + id=1, + original_text="Chontia", + corrected_text="Chauntea", + confidence=0.95, + ) fake_client = FakeLLMClient( [ CorrectionSet(corrections=[]), CorrectionSet(corrections=[correction]), + CorrectionSet(corrections=[retry_correction]), ] ) @@ -1022,8 +1156,12 @@ def test_below_threshold_grammar_corrections_are_not_retried(tmp_path): llm_client=fake_client, ) - assert fake_client.calls == 2 - assert revised[0].text == "I ask Chontia." + assert fake_client.calls == 3 + retry_prompt = fake_client.messages[2][1]["content"] + retry_payload = json.loads(retry_prompt.split("Transcript section:\n", maxsplit=1)[1]) + assert retry_payload == [{"id": 1, "original_text": "I ask Chontia."}] + assert revised[0].text == "I ask Chauntea." + assert list((tmp_path / "work").iterdir()) == [] def test_unresolved_grammar_skip_preserves_diagnostics(tmp_path):