Bugfixes in the glossary protection guard: allow changes toward protected glossary terms, but deny changes away from protected terms
This commit is contained in:
@@ -88,8 +88,9 @@ def build_grammar_correction_messages(
|
|||||||
"- Allowed corrections are only capitalization changes, punctuation changes involving commas, periods, em dashes, and ellipses, homophone fixes, and spelling fixes.\n"
|
"- Allowed corrections are only capitalization changes, punctuation changes involving commas, periods, em dashes, and ellipses, homophone fixes, and spelling fixes.\n"
|
||||||
"- Do not add, remove, reorder, or replace words except for clear homophone or spelling corrections that preserve the spoken content.\n"
|
"- Do not add, remove, reorder, or replace words except for clear homophone or spelling corrections that preserve the spoken content.\n"
|
||||||
"- Do not paraphrase, summarize, clarify, smooth style, or change the speaker's intent or meaning.\n"
|
"- Do not paraphrase, summarize, clarify, smooth style, or change the speaker's intent or meaning.\n"
|
||||||
"- Treat glossary names and aliases as protected spellings and context; do not introduce new glossary substitutions during this grammar pass.\n"
|
"- Treat glossary names and aliases as protected spellings and context.\n"
|
||||||
"- Do not autocorrect, Anglicize, replace, normalize, lowercase, or otherwise alter protected glossary names or aliases.\n"
|
"- You may correct clear transcription or spelling errors toward glossary names or aliases when the correction preserves the spoken content.\n"
|
||||||
|
"- Do not autocorrect, Anglicize, replace, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.\n"
|
||||||
"- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n"
|
"- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n"
|
||||||
"- Use the exact id from the input segment.\n"
|
"- Use the exact id from the input segment.\n"
|
||||||
"- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n"
|
"- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n"
|
||||||
|
|||||||
@@ -29,12 +29,15 @@ class ProtectedVocabulary:
|
|||||||
def violation_reason(self, before: str, after: str) -> Optional[str]:
|
def violation_reason(self, before: str, after: str) -> Optional[str]:
|
||||||
before_terms = self._terms(before)
|
before_terms = self._terms(before)
|
||||||
after_terms = self._terms(after)
|
after_terms = self._terms(after)
|
||||||
if len(before_terms) != len(after_terms):
|
if not before_terms:
|
||||||
return "grammar correction changes protected glossary term usage"
|
return None
|
||||||
|
|
||||||
for before_term, after_term in zip(before_terms, after_terms):
|
after_folded = [term.folded for term in after_terms]
|
||||||
if before_term.folded != after_term.folded:
|
for before_term in before_terms:
|
||||||
|
if before_term.folded not in after_folded:
|
||||||
return "grammar correction changes protected glossary term usage"
|
return "grammar correction changes protected glossary term usage"
|
||||||
|
|
||||||
|
for after_term in after_terms:
|
||||||
canonical = self.canonical_by_folded[after_term.folded]
|
canonical = self.canonical_by_folded[after_term.folded]
|
||||||
if after_term.text != canonical:
|
if after_term.text != canonical:
|
||||||
return "grammar correction changes protected glossary term capitalization"
|
return "grammar correction changes protected glossary term capitalization"
|
||||||
|
|||||||
@@ -358,6 +358,90 @@ def test_grammar_stage_cannot_reverse_glossary_protected_term(tmp_path):
|
|||||||
assert "protected glossary term" in diagnostics["skipped_corrections"][0]["reason"]
|
assert "protected glossary term" in diagnostics["skipped_corrections"][0]["reason"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_grammar_stage_can_correct_toward_protected_term(tmp_path):
|
||||||
|
transcript = parse_source_transcript_json(
|
||||||
|
"""
|
||||||
|
[
|
||||||
|
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Pawpaw's just worn out."}
|
||||||
|
]
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
glossary = parse_glossary_yaml(
|
||||||
|
"""
|
||||||
|
glossary:
|
||||||
|
- name: "Popov"
|
||||||
|
category: npc
|
||||||
|
summary: "Popov is an allied NPC."
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
grammar_correction = CorrectionCandidate(
|
||||||
|
id=1,
|
||||||
|
original_text="Pawpaw's",
|
||||||
|
corrected_text="Popov's",
|
||||||
|
confidence=0.95,
|
||||||
|
)
|
||||||
|
fake_client = FakeLLMClient(
|
||||||
|
[
|
||||||
|
CorrectionSet(corrections=[]),
|
||||||
|
CorrectionSet(corrections=[grammar_correction]),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
revised = process_transcript(
|
||||||
|
transcript,
|
||||||
|
glossary,
|
||||||
|
_config(tmp_path),
|
||||||
|
llm_client=fake_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert revised[0].text == "Popov's just worn out."
|
||||||
|
assert list((tmp_path / "work").iterdir()) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_grammar_stage_cannot_change_away_from_protected_term(tmp_path):
|
||||||
|
transcript = parse_source_transcript_json(
|
||||||
|
"""
|
||||||
|
[
|
||||||
|
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Popov's just worn out."}
|
||||||
|
]
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
glossary = parse_glossary_yaml(
|
||||||
|
"""
|
||||||
|
glossary:
|
||||||
|
- name: "Popov"
|
||||||
|
category: npc
|
||||||
|
summary: "Popov is an allied NPC."
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
grammar_correction = CorrectionCandidate(
|
||||||
|
id=1,
|
||||||
|
original_text="Popov's",
|
||||||
|
corrected_text="Pawpaw's",
|
||||||
|
confidence=0.95,
|
||||||
|
)
|
||||||
|
fake_client = FakeLLMClient(
|
||||||
|
[
|
||||||
|
CorrectionSet(corrections=[]),
|
||||||
|
CorrectionSet(corrections=[grammar_correction]),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
revised = process_transcript(
|
||||||
|
transcript,
|
||||||
|
glossary,
|
||||||
|
_config(tmp_path, grammar_max_llm_passes=1),
|
||||||
|
llm_client=fake_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert revised[0].text == "Popov's just worn out."
|
||||||
|
run_dirs = list((tmp_path / "work").iterdir())
|
||||||
|
assert len(run_dirs) == 1
|
||||||
|
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
|
||||||
|
assert diagnostics["skipped_corrections"][0]["stage"] == "grammar"
|
||||||
|
assert "protected glossary term" in diagnostics["skipped_corrections"][0]["reason"]
|
||||||
|
|
||||||
|
|
||||||
def test_grammar_stage_retries_repeated_span_and_applies_unique_retry(tmp_path):
|
def test_grammar_stage_retries_repeated_span_and_applies_unique_retry(tmp_path):
|
||||||
transcript = parse_source_transcript_json(
|
transcript = parse_source_transcript_json(
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -94,7 +94,9 @@ def test_grammar_prompt_limits_readability_corrections_and_protects_glossary():
|
|||||||
assert "spelling fixes" in prompt_text
|
assert "spelling fixes" in prompt_text
|
||||||
assert "Do not paraphrase" in prompt_text
|
assert "Do not paraphrase" in prompt_text
|
||||||
assert "glossary names and aliases as protected spellings" in prompt_text
|
assert "glossary names and aliases as protected spellings" in prompt_text
|
||||||
|
assert "correct clear transcription or spelling errors toward glossary names or aliases" in prompt_text
|
||||||
assert "Do not autocorrect, Anglicize, replace, normalize, lowercase" in prompt_text
|
assert "Do not autocorrect, Anglicize, replace, normalize, lowercase" in prompt_text
|
||||||
|
assert "that already appear correctly in the transcript" in prompt_text
|
||||||
assert "Preserve canonical glossary capitalization" in prompt_text
|
assert "Preserve canonical glossary capitalization" in prompt_text
|
||||||
assert "appears exactly once" in prompt_text
|
assert "appears exactly once" in prompt_text
|
||||||
assert "Do not return speaker, start, or end fields" in prompt_text
|
assert "Do not return speaker, start, or end fields" in prompt_text
|
||||||
|
|||||||
@@ -11,6 +11,9 @@ def _vocabulary():
|
|||||||
- "Greenfield"
|
- "Greenfield"
|
||||||
category: pc
|
category: pc
|
||||||
summary: "Hrank Greenfield is a player character."
|
summary: "Hrank Greenfield is a player character."
|
||||||
|
- name: "Popov"
|
||||||
|
category: npc
|
||||||
|
summary: "Popov is an allied NPC."
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
return ProtectedVocabulary.from_glossary(glossary)
|
return ProtectedVocabulary.from_glossary(glossary)
|
||||||
@@ -22,22 +25,46 @@ def test_protection_blocks_replacing_protected_term():
|
|||||||
assert vocabulary.violation_reason("Hrank moves.", "Frank moves.") is not None
|
assert vocabulary.violation_reason("Hrank moves.", "Frank moves.") is not None
|
||||||
|
|
||||||
|
|
||||||
|
def test_protection_blocks_replacing_possessive_protected_term():
|
||||||
|
vocabulary = _vocabulary()
|
||||||
|
|
||||||
|
assert vocabulary.violation_reason("Popov's exhausted.", "Pawpaw's exhausted.") is not None
|
||||||
|
|
||||||
|
|
||||||
def test_protection_blocks_lowercasing_protected_term():
|
def test_protection_blocks_lowercasing_protected_term():
|
||||||
vocabulary = _vocabulary()
|
vocabulary = _vocabulary()
|
||||||
|
|
||||||
assert vocabulary.violation_reason("Hrank moves.", "hrank moves.") is not None
|
assert vocabulary.violation_reason("Hrank moves.", "hrank moves.") is not None
|
||||||
|
|
||||||
|
|
||||||
|
def test_protection_blocks_noncanonical_uppercase_protected_term():
|
||||||
|
vocabulary = _vocabulary()
|
||||||
|
|
||||||
|
assert vocabulary.violation_reason("Popov moves.", "POPOV moves.") is not None
|
||||||
|
|
||||||
|
|
||||||
def test_protection_allows_canonical_capitalization():
|
def test_protection_allows_canonical_capitalization():
|
||||||
vocabulary = _vocabulary()
|
vocabulary = _vocabulary()
|
||||||
|
|
||||||
assert vocabulary.violation_reason("hrank moves.", "Hrank moves.") is None
|
assert vocabulary.violation_reason("hrank moves.", "Hrank moves.") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_protection_allows_correction_toward_protected_term():
|
||||||
|
vocabulary = _vocabulary()
|
||||||
|
|
||||||
|
assert vocabulary.violation_reason("Pawpaw moves.", "Popov moves.") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_protection_allows_possessive_correction_toward_protected_term():
|
||||||
|
vocabulary = _vocabulary()
|
||||||
|
|
||||||
|
assert vocabulary.violation_reason("Pawpaw's exhausted.", "Popov's exhausted.") is None
|
||||||
|
|
||||||
|
|
||||||
def test_protection_allows_punctuation_around_protected_term():
|
def test_protection_allows_punctuation_around_protected_term():
|
||||||
vocabulary = _vocabulary()
|
vocabulary = _vocabulary()
|
||||||
|
|
||||||
assert vocabulary.violation_reason("Hrank, moves.", "Hrank. Moves.") is None
|
assert vocabulary.violation_reason("Popov, moves.", "Popov. Moves.") is None
|
||||||
|
|
||||||
|
|
||||||
def test_protection_does_not_match_terms_inside_larger_words():
|
def test_protection_does_not_match_terms_inside_larger_words():
|
||||||
|
|||||||
Reference in New Issue
Block a user