diff --git a/src/audita/pipeline.py b/src/audita/pipeline.py index 63a2565..ed44189 100644 --- a/src/audita/pipeline.py +++ b/src/audita/pipeline.py @@ -74,6 +74,7 @@ def process_transcript( max_llm_passes=config.glossary_max_llm_passes, confidence_threshold=config.glossary_confidence_threshold, replacement_mode="replace_all", + correction_guard=protected_vocabulary.violation_reason, ), StageSpec( name="grammar", diff --git a/src/audita/prompts.py b/src/audita/prompts.py index 35fd36f..0506dd0 100644 --- a/src/audita/prompts.py +++ b/src/audita/prompts.py @@ -43,6 +43,9 @@ def build_glossary_correction_messages( "- Appropriate example: correcting \"gestures\" to \"Jesters\" can be valid if \"Jesters\" appears in the glossary and nearby context supports that inference.\n" "- Inappropriate example: correcting \"Lyra\" to \"Jesters\" should be omitted because those words are not similar in spoken English, even if \"Jesters\" appears in the glossary.\n" "- Do not replace one clear glossary term, character name, location, or ordinary word with a different glossary term unless it is a plausible mishearing.\n" + "- Treat glossary names and aliases already present in the transcript as protected spellings.\n" + "- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.\n" + "- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n" "- Assign high confidence only when the correction is supported by glossary evidence, local context, and spoken-word similarity; otherwise omit the correction.\n" "- Use the exact id from the input segment.\n" "- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n" diff --git a/src/audita/protection.py b/src/audita/protection.py index 8a04cbb..8dcc5b4 100644 --- a/src/audita/protection.py +++ b/src/audita/protection.py @@ -35,12 +35,12 @@ class ProtectedVocabulary: after_folded = [term.folded for term in after_terms] for before_term in before_terms: if before_term.folded not in after_folded: - return "grammar correction changes protected glossary term usage" + return "correction changes protected glossary term usage" for after_term in after_terms: canonical = self.canonical_by_folded[after_term.folded] if after_term.text != canonical: - return "grammar correction changes protected glossary term capitalization" + return "correction changes protected glossary term capitalization" return None diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index cd46377..88f7889 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -79,6 +79,136 @@ def test_pipeline_processes_with_fake_llm_and_cleans_work_dir(tmp_path): assert list((tmp_path / "work").iterdir()) == [] +def test_glossary_stage_can_correct_toward_protected_term(tmp_path): + transcript = parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Frank moves."} + ] + """ + ) + glossary = parse_glossary_yaml( + """ + glossary: + - name: "Hrank" + category: pc + summary: "Hrank is a player character." + """ + ) + glossary_correction = CorrectionCandidate( + id=1, + original_text="Frank", + corrected_text="Hrank", + confidence=0.95, + ) + fake_client = FakeLLMClient( + [ + CorrectionSet(corrections=[glossary_correction]), + CorrectionSet(corrections=[]), + ] + ) + + revised = process_transcript( + transcript, + glossary, + _config(tmp_path), + llm_client=fake_client, + ) + + assert revised[0].text == "Hrank moves." + assert list((tmp_path / "work").iterdir()) == [] + + +def test_glossary_stage_cannot_change_away_from_protected_term(tmp_path): + transcript = parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hrank moves."} + ] + """ + ) + glossary = parse_glossary_yaml( + """ + glossary: + - name: "Hrank" + category: pc + summary: "Hrank is a player character." + """ + ) + glossary_reversal = CorrectionCandidate( + id=1, + original_text="Hrank", + corrected_text="Frank", + confidence=0.95, + ) + fake_client = FakeLLMClient( + [ + CorrectionSet(corrections=[glossary_reversal]), + CorrectionSet(corrections=[]), + ] + ) + + revised = process_transcript( + transcript, + glossary, + _config(tmp_path, glossary_max_llm_passes=1), + llm_client=fake_client, + ) + + assert revised[0].text == "Hrank moves." + run_dirs = list((tmp_path / "work").iterdir()) + assert len(run_dirs) == 1 + diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8")) + skipped = diagnostics["skipped_corrections"][0] + assert skipped["stage"] == "glossary" + assert skipped["reason"] == "correction changes protected glossary term usage" + + +def test_glossary_stage_cannot_decanonicalize_protected_term(tmp_path): + transcript = parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hrank moves."} + ] + """ + ) + glossary = parse_glossary_yaml( + """ + glossary: + - name: "Hrank" + category: pc + summary: "Hrank is a player character." + """ + ) + decapitalization = CorrectionCandidate( + id=1, + original_text="Hrank", + corrected_text="hrank", + confidence=0.95, + ) + fake_client = FakeLLMClient( + [ + CorrectionSet(corrections=[decapitalization]), + CorrectionSet(corrections=[]), + ] + ) + + revised = process_transcript( + transcript, + glossary, + _config(tmp_path, glossary_max_llm_passes=1), + llm_client=fake_client, + ) + + assert revised[0].text == "Hrank moves." + run_dirs = list((tmp_path / "work").iterdir()) + assert len(run_dirs) == 1 + diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8")) + skipped = diagnostics["skipped_corrections"][0] + assert skipped["stage"] == "glossary" + assert skipped["reason"] == "correction changes protected glossary term capitalization" + + def test_pipeline_normalizes_before_llm_prompts(tmp_path): transcript = parse_source_transcript_json( """ @@ -355,7 +485,7 @@ def test_grammar_stage_cannot_reverse_glossary_protected_term(tmp_path): assert len(run_dirs) == 1 diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8")) assert diagnostics["skipped_corrections"][0]["stage"] == "grammar" - assert "protected glossary term" in diagnostics["skipped_corrections"][0]["reason"] + assert diagnostics["skipped_corrections"][0]["reason"] == "correction changes protected glossary term usage" def test_grammar_stage_can_correct_toward_protected_term(tmp_path): @@ -439,7 +569,7 @@ def test_grammar_stage_cannot_change_away_from_protected_term(tmp_path): assert len(run_dirs) == 1 diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8")) assert diagnostics["skipped_corrections"][0]["stage"] == "grammar" - assert "protected glossary term" in diagnostics["skipped_corrections"][0]["reason"] + assert diagnostics["skipped_corrections"][0]["reason"] == "correction changes protected glossary term usage" def test_grammar_stage_retries_repeated_span_and_applies_unique_retry(tmp_path): diff --git a/tests/test_prompts.py b/tests/test_prompts.py index 06bc5c1..7679a6d 100644 --- a/tests/test_prompts.py +++ b/tests/test_prompts.py @@ -34,6 +34,9 @@ def test_prompt_requires_acoustically_plausible_transcription_errors(): assert '"gestures" to "Jesters"' in prompt_text assert '"Lyra" to "Jesters"' in prompt_text assert "should be omitted" in prompt_text + assert "glossary names and aliases already present in the transcript as protected spellings" in prompt_text + assert "Do not replace, Anglicize, normalize, lowercase" in prompt_text + assert "Preserve canonical glossary capitalization" in prompt_text assert "exact text span that needs replacement" in prompt_text assert "replacement text for that span" in prompt_text assert "Do not return corrections where original_text and corrected_text are identical" in prompt_text diff --git a/tests/test_protection.py b/tests/test_protection.py index 3824cad..73ddb03 100644 --- a/tests/test_protection.py +++ b/tests/test_protection.py @@ -22,7 +22,10 @@ def _vocabulary(): def test_protection_blocks_replacing_protected_term(): vocabulary = _vocabulary() - assert vocabulary.violation_reason("Hrank moves.", "Frank moves.") is not None + assert ( + vocabulary.violation_reason("Hrank moves.", "Frank moves.") + == "correction changes protected glossary term usage" + ) def test_protection_blocks_replacing_possessive_protected_term(): @@ -34,7 +37,10 @@ def test_protection_blocks_replacing_possessive_protected_term(): def test_protection_blocks_lowercasing_protected_term(): vocabulary = _vocabulary() - assert vocabulary.violation_reason("Hrank moves.", "hrank moves.") is not None + assert ( + vocabulary.violation_reason("Hrank moves.", "hrank moves.") + == "correction changes protected glossary term capitalization" + ) def test_protection_blocks_noncanonical_uppercase_protected_term():