Bugfixes in the glossary protection guard: allow changes toward protected glossary terms, but deny changes away from protected terms
This commit is contained in:
@@ -88,8 +88,9 @@ def build_grammar_correction_messages(
|
||||
"- Allowed corrections are only capitalization changes, punctuation changes involving commas, periods, em dashes, and ellipses, homophone fixes, and spelling fixes.\n"
|
||||
"- Do not add, remove, reorder, or replace words except for clear homophone or spelling corrections that preserve the spoken content.\n"
|
||||
"- Do not paraphrase, summarize, clarify, smooth style, or change the speaker's intent or meaning.\n"
|
||||
"- Treat glossary names and aliases as protected spellings and context; do not introduce new glossary substitutions during this grammar pass.\n"
|
||||
"- Do not autocorrect, Anglicize, replace, normalize, lowercase, or otherwise alter protected glossary names or aliases.\n"
|
||||
"- Treat glossary names and aliases as protected spellings and context.\n"
|
||||
"- You may correct clear transcription or spelling errors toward glossary names or aliases when the correction preserves the spoken content.\n"
|
||||
"- Do not autocorrect, Anglicize, replace, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.\n"
|
||||
"- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n"
|
||||
"- Use the exact id from the input segment.\n"
|
||||
"- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n"
|
||||
|
||||
@@ -29,12 +29,15 @@ class ProtectedVocabulary:
|
||||
def violation_reason(self, before: str, after: str) -> Optional[str]:
|
||||
before_terms = self._terms(before)
|
||||
after_terms = self._terms(after)
|
||||
if len(before_terms) != len(after_terms):
|
||||
return "grammar correction changes protected glossary term usage"
|
||||
if not before_terms:
|
||||
return None
|
||||
|
||||
for before_term, after_term in zip(before_terms, after_terms):
|
||||
if before_term.folded != after_term.folded:
|
||||
after_folded = [term.folded for term in after_terms]
|
||||
for before_term in before_terms:
|
||||
if before_term.folded not in after_folded:
|
||||
return "grammar correction changes protected glossary term usage"
|
||||
|
||||
for after_term in after_terms:
|
||||
canonical = self.canonical_by_folded[after_term.folded]
|
||||
if after_term.text != canonical:
|
||||
return "grammar correction changes protected glossary term capitalization"
|
||||
|
||||
@@ -358,6 +358,90 @@ def test_grammar_stage_cannot_reverse_glossary_protected_term(tmp_path):
|
||||
assert "protected glossary term" in diagnostics["skipped_corrections"][0]["reason"]
|
||||
|
||||
|
||||
def test_grammar_stage_can_correct_toward_protected_term(tmp_path):
|
||||
transcript = parse_source_transcript_json(
|
||||
"""
|
||||
[
|
||||
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Pawpaw's just worn out."}
|
||||
]
|
||||
"""
|
||||
)
|
||||
glossary = parse_glossary_yaml(
|
||||
"""
|
||||
glossary:
|
||||
- name: "Popov"
|
||||
category: npc
|
||||
summary: "Popov is an allied NPC."
|
||||
"""
|
||||
)
|
||||
grammar_correction = CorrectionCandidate(
|
||||
id=1,
|
||||
original_text="Pawpaw's",
|
||||
corrected_text="Popov's",
|
||||
confidence=0.95,
|
||||
)
|
||||
fake_client = FakeLLMClient(
|
||||
[
|
||||
CorrectionSet(corrections=[]),
|
||||
CorrectionSet(corrections=[grammar_correction]),
|
||||
]
|
||||
)
|
||||
|
||||
revised = process_transcript(
|
||||
transcript,
|
||||
glossary,
|
||||
_config(tmp_path),
|
||||
llm_client=fake_client,
|
||||
)
|
||||
|
||||
assert revised[0].text == "Popov's just worn out."
|
||||
assert list((tmp_path / "work").iterdir()) == []
|
||||
|
||||
|
||||
def test_grammar_stage_cannot_change_away_from_protected_term(tmp_path):
|
||||
transcript = parse_source_transcript_json(
|
||||
"""
|
||||
[
|
||||
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Popov's just worn out."}
|
||||
]
|
||||
"""
|
||||
)
|
||||
glossary = parse_glossary_yaml(
|
||||
"""
|
||||
glossary:
|
||||
- name: "Popov"
|
||||
category: npc
|
||||
summary: "Popov is an allied NPC."
|
||||
"""
|
||||
)
|
||||
grammar_correction = CorrectionCandidate(
|
||||
id=1,
|
||||
original_text="Popov's",
|
||||
corrected_text="Pawpaw's",
|
||||
confidence=0.95,
|
||||
)
|
||||
fake_client = FakeLLMClient(
|
||||
[
|
||||
CorrectionSet(corrections=[]),
|
||||
CorrectionSet(corrections=[grammar_correction]),
|
||||
]
|
||||
)
|
||||
|
||||
revised = process_transcript(
|
||||
transcript,
|
||||
glossary,
|
||||
_config(tmp_path, grammar_max_llm_passes=1),
|
||||
llm_client=fake_client,
|
||||
)
|
||||
|
||||
assert revised[0].text == "Popov's just worn out."
|
||||
run_dirs = list((tmp_path / "work").iterdir())
|
||||
assert len(run_dirs) == 1
|
||||
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
|
||||
assert diagnostics["skipped_corrections"][0]["stage"] == "grammar"
|
||||
assert "protected glossary term" in diagnostics["skipped_corrections"][0]["reason"]
|
||||
|
||||
|
||||
def test_grammar_stage_retries_repeated_span_and_applies_unique_retry(tmp_path):
|
||||
transcript = parse_source_transcript_json(
|
||||
"""
|
||||
|
||||
@@ -94,7 +94,9 @@ def test_grammar_prompt_limits_readability_corrections_and_protects_glossary():
|
||||
assert "spelling fixes" in prompt_text
|
||||
assert "Do not paraphrase" in prompt_text
|
||||
assert "glossary names and aliases as protected spellings" in prompt_text
|
||||
assert "correct clear transcription or spelling errors toward glossary names or aliases" in prompt_text
|
||||
assert "Do not autocorrect, Anglicize, replace, normalize, lowercase" in prompt_text
|
||||
assert "that already appear correctly in the transcript" in prompt_text
|
||||
assert "Preserve canonical glossary capitalization" in prompt_text
|
||||
assert "appears exactly once" in prompt_text
|
||||
assert "Do not return speaker, start, or end fields" in prompt_text
|
||||
|
||||
@@ -11,6 +11,9 @@ def _vocabulary():
|
||||
- "Greenfield"
|
||||
category: pc
|
||||
summary: "Hrank Greenfield is a player character."
|
||||
- name: "Popov"
|
||||
category: npc
|
||||
summary: "Popov is an allied NPC."
|
||||
"""
|
||||
)
|
||||
return ProtectedVocabulary.from_glossary(glossary)
|
||||
@@ -22,22 +25,46 @@ def test_protection_blocks_replacing_protected_term():
|
||||
assert vocabulary.violation_reason("Hrank moves.", "Frank moves.") is not None
|
||||
|
||||
|
||||
def test_protection_blocks_replacing_possessive_protected_term():
|
||||
vocabulary = _vocabulary()
|
||||
|
||||
assert vocabulary.violation_reason("Popov's exhausted.", "Pawpaw's exhausted.") is not None
|
||||
|
||||
|
||||
def test_protection_blocks_lowercasing_protected_term():
|
||||
vocabulary = _vocabulary()
|
||||
|
||||
assert vocabulary.violation_reason("Hrank moves.", "hrank moves.") is not None
|
||||
|
||||
|
||||
def test_protection_blocks_noncanonical_uppercase_protected_term():
|
||||
vocabulary = _vocabulary()
|
||||
|
||||
assert vocabulary.violation_reason("Popov moves.", "POPOV moves.") is not None
|
||||
|
||||
|
||||
def test_protection_allows_canonical_capitalization():
|
||||
vocabulary = _vocabulary()
|
||||
|
||||
assert vocabulary.violation_reason("hrank moves.", "Hrank moves.") is None
|
||||
|
||||
|
||||
def test_protection_allows_correction_toward_protected_term():
|
||||
vocabulary = _vocabulary()
|
||||
|
||||
assert vocabulary.violation_reason("Pawpaw moves.", "Popov moves.") is None
|
||||
|
||||
|
||||
def test_protection_allows_possessive_correction_toward_protected_term():
|
||||
vocabulary = _vocabulary()
|
||||
|
||||
assert vocabulary.violation_reason("Pawpaw's exhausted.", "Popov's exhausted.") is None
|
||||
|
||||
|
||||
def test_protection_allows_punctuation_around_protected_term():
|
||||
vocabulary = _vocabulary()
|
||||
|
||||
assert vocabulary.violation_reason("Hrank, moves.", "Hrank. Moves.") is None
|
||||
assert vocabulary.violation_reason("Popov, moves.", "Popov. Moves.") is None
|
||||
|
||||
|
||||
def test_protection_does_not_match_terms_inside_larger_words():
|
||||
|
||||
Reference in New Issue
Block a user