From 183f8ba31d4d1baf2d203cdf3f3a4be69b5f4af7 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Wed, 22 Apr 2026 11:04:35 -0500 Subject: [PATCH] Bugfixes in the glossary protection guard: allow changes toward protected glossary terms, but deny changes away from protected terms --- src/audita/prompts.py | 5 ++- src/audita/protection.py | 11 ++++-- tests/test_pipeline.py | 84 ++++++++++++++++++++++++++++++++++++++++ tests/test_prompts.py | 2 + tests/test_protection.py | 29 +++++++++++++- 5 files changed, 124 insertions(+), 7 deletions(-) diff --git a/src/audita/prompts.py b/src/audita/prompts.py index 091edf8..35fd36f 100644 --- a/src/audita/prompts.py +++ b/src/audita/prompts.py @@ -88,8 +88,9 @@ def build_grammar_correction_messages( "- Allowed corrections are only capitalization changes, punctuation changes involving commas, periods, em dashes, and ellipses, homophone fixes, and spelling fixes.\n" "- Do not add, remove, reorder, or replace words except for clear homophone or spelling corrections that preserve the spoken content.\n" "- Do not paraphrase, summarize, clarify, smooth style, or change the speaker's intent or meaning.\n" - "- Treat glossary names and aliases as protected spellings and context; do not introduce new glossary substitutions during this grammar pass.\n" - "- Do not autocorrect, Anglicize, replace, normalize, lowercase, or otherwise alter protected glossary names or aliases.\n" + "- Treat glossary names and aliases as protected spellings and context.\n" + "- You may correct clear transcription or spelling errors toward glossary names or aliases when the correction preserves the spoken content.\n" + "- Do not autocorrect, Anglicize, replace, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.\n" "- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n" "- Use the exact id from the input segment.\n" "- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n" diff --git a/src/audita/protection.py b/src/audita/protection.py index 3d8470e..8a04cbb 100644 --- a/src/audita/protection.py +++ b/src/audita/protection.py @@ -29,12 +29,15 @@ class ProtectedVocabulary: def violation_reason(self, before: str, after: str) -> Optional[str]: before_terms = self._terms(before) after_terms = self._terms(after) - if len(before_terms) != len(after_terms): - return "grammar correction changes protected glossary term usage" + if not before_terms: + return None - for before_term, after_term in zip(before_terms, after_terms): - if before_term.folded != after_term.folded: + after_folded = [term.folded for term in after_terms] + for before_term in before_terms: + if before_term.folded not in after_folded: return "grammar correction changes protected glossary term usage" + + for after_term in after_terms: canonical = self.canonical_by_folded[after_term.folded] if after_term.text != canonical: return "grammar correction changes protected glossary term capitalization" diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 2080e3f..cd46377 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -358,6 +358,90 @@ def test_grammar_stage_cannot_reverse_glossary_protected_term(tmp_path): assert "protected glossary term" in diagnostics["skipped_corrections"][0]["reason"] +def test_grammar_stage_can_correct_toward_protected_term(tmp_path): + transcript = parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Pawpaw's just worn out."} + ] + """ + ) + glossary = parse_glossary_yaml( + """ + glossary: + - name: "Popov" + category: npc + summary: "Popov is an allied NPC." + """ + ) + grammar_correction = CorrectionCandidate( + id=1, + original_text="Pawpaw's", + corrected_text="Popov's", + confidence=0.95, + ) + fake_client = FakeLLMClient( + [ + CorrectionSet(corrections=[]), + CorrectionSet(corrections=[grammar_correction]), + ] + ) + + revised = process_transcript( + transcript, + glossary, + _config(tmp_path), + llm_client=fake_client, + ) + + assert revised[0].text == "Popov's just worn out." + assert list((tmp_path / "work").iterdir()) == [] + + +def test_grammar_stage_cannot_change_away_from_protected_term(tmp_path): + transcript = parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Popov's just worn out."} + ] + """ + ) + glossary = parse_glossary_yaml( + """ + glossary: + - name: "Popov" + category: npc + summary: "Popov is an allied NPC." + """ + ) + grammar_correction = CorrectionCandidate( + id=1, + original_text="Popov's", + corrected_text="Pawpaw's", + confidence=0.95, + ) + fake_client = FakeLLMClient( + [ + CorrectionSet(corrections=[]), + CorrectionSet(corrections=[grammar_correction]), + ] + ) + + revised = process_transcript( + transcript, + glossary, + _config(tmp_path, grammar_max_llm_passes=1), + llm_client=fake_client, + ) + + assert revised[0].text == "Popov's just worn out." + run_dirs = list((tmp_path / "work").iterdir()) + assert len(run_dirs) == 1 + diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8")) + assert diagnostics["skipped_corrections"][0]["stage"] == "grammar" + assert "protected glossary term" in diagnostics["skipped_corrections"][0]["reason"] + + def test_grammar_stage_retries_repeated_span_and_applies_unique_retry(tmp_path): transcript = parse_source_transcript_json( """ diff --git a/tests/test_prompts.py b/tests/test_prompts.py index a712d79..06bc5c1 100644 --- a/tests/test_prompts.py +++ b/tests/test_prompts.py @@ -94,7 +94,9 @@ def test_grammar_prompt_limits_readability_corrections_and_protects_glossary(): assert "spelling fixes" in prompt_text assert "Do not paraphrase" in prompt_text assert "glossary names and aliases as protected spellings" in prompt_text + assert "correct clear transcription or spelling errors toward glossary names or aliases" in prompt_text assert "Do not autocorrect, Anglicize, replace, normalize, lowercase" in prompt_text + assert "that already appear correctly in the transcript" in prompt_text assert "Preserve canonical glossary capitalization" in prompt_text assert "appears exactly once" in prompt_text assert "Do not return speaker, start, or end fields" in prompt_text diff --git a/tests/test_protection.py b/tests/test_protection.py index 74825cc..3824cad 100644 --- a/tests/test_protection.py +++ b/tests/test_protection.py @@ -11,6 +11,9 @@ def _vocabulary(): - "Greenfield" category: pc summary: "Hrank Greenfield is a player character." + - name: "Popov" + category: npc + summary: "Popov is an allied NPC." """ ) return ProtectedVocabulary.from_glossary(glossary) @@ -22,22 +25,46 @@ def test_protection_blocks_replacing_protected_term(): assert vocabulary.violation_reason("Hrank moves.", "Frank moves.") is not None +def test_protection_blocks_replacing_possessive_protected_term(): + vocabulary = _vocabulary() + + assert vocabulary.violation_reason("Popov's exhausted.", "Pawpaw's exhausted.") is not None + + def test_protection_blocks_lowercasing_protected_term(): vocabulary = _vocabulary() assert vocabulary.violation_reason("Hrank moves.", "hrank moves.") is not None +def test_protection_blocks_noncanonical_uppercase_protected_term(): + vocabulary = _vocabulary() + + assert vocabulary.violation_reason("Popov moves.", "POPOV moves.") is not None + + def test_protection_allows_canonical_capitalization(): vocabulary = _vocabulary() assert vocabulary.violation_reason("hrank moves.", "Hrank moves.") is None +def test_protection_allows_correction_toward_protected_term(): + vocabulary = _vocabulary() + + assert vocabulary.violation_reason("Pawpaw moves.", "Popov moves.") is None + + +def test_protection_allows_possessive_correction_toward_protected_term(): + vocabulary = _vocabulary() + + assert vocabulary.violation_reason("Pawpaw's exhausted.", "Popov's exhausted.") is None + + def test_protection_allows_punctuation_around_protected_term(): vocabulary = _vocabulary() - assert vocabulary.violation_reason("Hrank, moves.", "Hrank. Moves.") is None + assert vocabulary.violation_reason("Popov, moves.", "Popov. Moves.") is None def test_protection_does_not_match_terms_inside_larger_words():