Bugfixes in the glossary protection guard: allow changes toward protected glossary terms, but deny changes away from protected terms

This commit is contained in:
2026-04-22 11:04:35 -05:00
parent 9afb5c5a4e
commit 183f8ba31d
5 changed files with 124 additions and 7 deletions

View File

@@ -88,8 +88,9 @@ def build_grammar_correction_messages(
"- Allowed corrections are only capitalization changes, punctuation changes involving commas, periods, em dashes, and ellipses, homophone fixes, and spelling fixes.\n"
"- Do not add, remove, reorder, or replace words except for clear homophone or spelling corrections that preserve the spoken content.\n"
"- Do not paraphrase, summarize, clarify, smooth style, or change the speaker's intent or meaning.\n"
"- Treat glossary names and aliases as protected spellings and context; do not introduce new glossary substitutions during this grammar pass.\n"
"- Do not autocorrect, Anglicize, replace, normalize, lowercase, or otherwise alter protected glossary names or aliases.\n"
"- Treat glossary names and aliases as protected spellings and context.\n"
"- You may correct clear transcription or spelling errors toward glossary names or aliases when the correction preserves the spoken content.\n"
"- Do not autocorrect, Anglicize, replace, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.\n"
"- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n"
"- Use the exact id from the input segment.\n"
"- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n"

View File

@@ -29,12 +29,15 @@ class ProtectedVocabulary:
def violation_reason(self, before: str, after: str) -> Optional[str]:
before_terms = self._terms(before)
after_terms = self._terms(after)
if len(before_terms) != len(after_terms):
return "grammar correction changes protected glossary term usage"
if not before_terms:
return None
for before_term, after_term in zip(before_terms, after_terms):
if before_term.folded != after_term.folded:
after_folded = [term.folded for term in after_terms]
for before_term in before_terms:
if before_term.folded not in after_folded:
return "grammar correction changes protected glossary term usage"
for after_term in after_terms:
canonical = self.canonical_by_folded[after_term.folded]
if after_term.text != canonical:
return "grammar correction changes protected glossary term capitalization"

View File

@@ -358,6 +358,90 @@ def test_grammar_stage_cannot_reverse_glossary_protected_term(tmp_path):
assert "protected glossary term" in diagnostics["skipped_corrections"][0]["reason"]
def test_grammar_stage_can_correct_toward_protected_term(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Pawpaw's just worn out."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Popov"
category: npc
summary: "Popov is an allied NPC."
"""
)
grammar_correction = CorrectionCandidate(
id=1,
original_text="Pawpaw's",
corrected_text="Popov's",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[grammar_correction]),
]
)
revised = process_transcript(
transcript,
glossary,
_config(tmp_path),
llm_client=fake_client,
)
assert revised[0].text == "Popov's just worn out."
assert list((tmp_path / "work").iterdir()) == []
def test_grammar_stage_cannot_change_away_from_protected_term(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Popov's just worn out."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Popov"
category: npc
summary: "Popov is an allied NPC."
"""
)
grammar_correction = CorrectionCandidate(
id=1,
original_text="Popov's",
corrected_text="Pawpaw's",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[grammar_correction]),
]
)
revised = process_transcript(
transcript,
glossary,
_config(tmp_path, grammar_max_llm_passes=1),
llm_client=fake_client,
)
assert revised[0].text == "Popov's just worn out."
run_dirs = list((tmp_path / "work").iterdir())
assert len(run_dirs) == 1
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
assert diagnostics["skipped_corrections"][0]["stage"] == "grammar"
assert "protected glossary term" in diagnostics["skipped_corrections"][0]["reason"]
def test_grammar_stage_retries_repeated_span_and_applies_unique_retry(tmp_path):
transcript = parse_source_transcript_json(
"""

View File

@@ -94,7 +94,9 @@ def test_grammar_prompt_limits_readability_corrections_and_protects_glossary():
assert "spelling fixes" in prompt_text
assert "Do not paraphrase" in prompt_text
assert "glossary names and aliases as protected spellings" in prompt_text
assert "correct clear transcription or spelling errors toward glossary names or aliases" in prompt_text
assert "Do not autocorrect, Anglicize, replace, normalize, lowercase" in prompt_text
assert "that already appear correctly in the transcript" in prompt_text
assert "Preserve canonical glossary capitalization" in prompt_text
assert "appears exactly once" in prompt_text
assert "Do not return speaker, start, or end fields" in prompt_text

View File

@@ -11,6 +11,9 @@ def _vocabulary():
- "Greenfield"
category: pc
summary: "Hrank Greenfield is a player character."
- name: "Popov"
category: npc
summary: "Popov is an allied NPC."
"""
)
return ProtectedVocabulary.from_glossary(glossary)
@@ -22,22 +25,46 @@ def test_protection_blocks_replacing_protected_term():
assert vocabulary.violation_reason("Hrank moves.", "Frank moves.") is not None
def test_protection_blocks_replacing_possessive_protected_term():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("Popov's exhausted.", "Pawpaw's exhausted.") is not None
def test_protection_blocks_lowercasing_protected_term():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("Hrank moves.", "hrank moves.") is not None
def test_protection_blocks_noncanonical_uppercase_protected_term():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("Popov moves.", "POPOV moves.") is not None
def test_protection_allows_canonical_capitalization():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("hrank moves.", "Hrank moves.") is None
def test_protection_allows_correction_toward_protected_term():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("Pawpaw moves.", "Popov moves.") is None
def test_protection_allows_possessive_correction_toward_protected_term():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("Pawpaw's exhausted.", "Popov's exhausted.") is None
def test_protection_allows_punctuation_around_protected_term():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("Hrank, moves.", "Hrank. Moves.") is None
assert vocabulary.violation_reason("Popov, moves.", "Popov. Moves.") is None
def test_protection_does_not_match_terms_inside_larger_words():