Extend glossary term protection to the initial glossary LLM stage
This commit is contained in:
@@ -74,6 +74,7 @@ def process_transcript(
|
|||||||
max_llm_passes=config.glossary_max_llm_passes,
|
max_llm_passes=config.glossary_max_llm_passes,
|
||||||
confidence_threshold=config.glossary_confidence_threshold,
|
confidence_threshold=config.glossary_confidence_threshold,
|
||||||
replacement_mode="replace_all",
|
replacement_mode="replace_all",
|
||||||
|
correction_guard=protected_vocabulary.violation_reason,
|
||||||
),
|
),
|
||||||
StageSpec(
|
StageSpec(
|
||||||
name="grammar",
|
name="grammar",
|
||||||
|
|||||||
@@ -43,6 +43,9 @@ def build_glossary_correction_messages(
|
|||||||
"- Appropriate example: correcting \"gestures\" to \"Jesters\" can be valid if \"Jesters\" appears in the glossary and nearby context supports that inference.\n"
|
"- Appropriate example: correcting \"gestures\" to \"Jesters\" can be valid if \"Jesters\" appears in the glossary and nearby context supports that inference.\n"
|
||||||
"- Inappropriate example: correcting \"Lyra\" to \"Jesters\" should be omitted because those words are not similar in spoken English, even if \"Jesters\" appears in the glossary.\n"
|
"- Inappropriate example: correcting \"Lyra\" to \"Jesters\" should be omitted because those words are not similar in spoken English, even if \"Jesters\" appears in the glossary.\n"
|
||||||
"- Do not replace one clear glossary term, character name, location, or ordinary word with a different glossary term unless it is a plausible mishearing.\n"
|
"- Do not replace one clear glossary term, character name, location, or ordinary word with a different glossary term unless it is a plausible mishearing.\n"
|
||||||
|
"- Treat glossary names and aliases already present in the transcript as protected spellings.\n"
|
||||||
|
"- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.\n"
|
||||||
|
"- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n"
|
||||||
"- Assign high confidence only when the correction is supported by glossary evidence, local context, and spoken-word similarity; otherwise omit the correction.\n"
|
"- Assign high confidence only when the correction is supported by glossary evidence, local context, and spoken-word similarity; otherwise omit the correction.\n"
|
||||||
"- Use the exact id from the input segment.\n"
|
"- Use the exact id from the input segment.\n"
|
||||||
"- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n"
|
"- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n"
|
||||||
|
|||||||
@@ -35,12 +35,12 @@ class ProtectedVocabulary:
|
|||||||
after_folded = [term.folded for term in after_terms]
|
after_folded = [term.folded for term in after_terms]
|
||||||
for before_term in before_terms:
|
for before_term in before_terms:
|
||||||
if before_term.folded not in after_folded:
|
if before_term.folded not in after_folded:
|
||||||
return "grammar correction changes protected glossary term usage"
|
return "correction changes protected glossary term usage"
|
||||||
|
|
||||||
for after_term in after_terms:
|
for after_term in after_terms:
|
||||||
canonical = self.canonical_by_folded[after_term.folded]
|
canonical = self.canonical_by_folded[after_term.folded]
|
||||||
if after_term.text != canonical:
|
if after_term.text != canonical:
|
||||||
return "grammar correction changes protected glossary term capitalization"
|
return "correction changes protected glossary term capitalization"
|
||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|||||||
@@ -79,6 +79,136 @@ def test_pipeline_processes_with_fake_llm_and_cleans_work_dir(tmp_path):
|
|||||||
assert list((tmp_path / "work").iterdir()) == []
|
assert list((tmp_path / "work").iterdir()) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_glossary_stage_can_correct_toward_protected_term(tmp_path):
|
||||||
|
transcript = parse_source_transcript_json(
|
||||||
|
"""
|
||||||
|
[
|
||||||
|
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Frank moves."}
|
||||||
|
]
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
glossary = parse_glossary_yaml(
|
||||||
|
"""
|
||||||
|
glossary:
|
||||||
|
- name: "Hrank"
|
||||||
|
category: pc
|
||||||
|
summary: "Hrank is a player character."
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
glossary_correction = CorrectionCandidate(
|
||||||
|
id=1,
|
||||||
|
original_text="Frank",
|
||||||
|
corrected_text="Hrank",
|
||||||
|
confidence=0.95,
|
||||||
|
)
|
||||||
|
fake_client = FakeLLMClient(
|
||||||
|
[
|
||||||
|
CorrectionSet(corrections=[glossary_correction]),
|
||||||
|
CorrectionSet(corrections=[]),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
revised = process_transcript(
|
||||||
|
transcript,
|
||||||
|
glossary,
|
||||||
|
_config(tmp_path),
|
||||||
|
llm_client=fake_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert revised[0].text == "Hrank moves."
|
||||||
|
assert list((tmp_path / "work").iterdir()) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_glossary_stage_cannot_change_away_from_protected_term(tmp_path):
|
||||||
|
transcript = parse_source_transcript_json(
|
||||||
|
"""
|
||||||
|
[
|
||||||
|
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hrank moves."}
|
||||||
|
]
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
glossary = parse_glossary_yaml(
|
||||||
|
"""
|
||||||
|
glossary:
|
||||||
|
- name: "Hrank"
|
||||||
|
category: pc
|
||||||
|
summary: "Hrank is a player character."
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
glossary_reversal = CorrectionCandidate(
|
||||||
|
id=1,
|
||||||
|
original_text="Hrank",
|
||||||
|
corrected_text="Frank",
|
||||||
|
confidence=0.95,
|
||||||
|
)
|
||||||
|
fake_client = FakeLLMClient(
|
||||||
|
[
|
||||||
|
CorrectionSet(corrections=[glossary_reversal]),
|
||||||
|
CorrectionSet(corrections=[]),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
revised = process_transcript(
|
||||||
|
transcript,
|
||||||
|
glossary,
|
||||||
|
_config(tmp_path, glossary_max_llm_passes=1),
|
||||||
|
llm_client=fake_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert revised[0].text == "Hrank moves."
|
||||||
|
run_dirs = list((tmp_path / "work").iterdir())
|
||||||
|
assert len(run_dirs) == 1
|
||||||
|
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
|
||||||
|
skipped = diagnostics["skipped_corrections"][0]
|
||||||
|
assert skipped["stage"] == "glossary"
|
||||||
|
assert skipped["reason"] == "correction changes protected glossary term usage"
|
||||||
|
|
||||||
|
|
||||||
|
def test_glossary_stage_cannot_decanonicalize_protected_term(tmp_path):
|
||||||
|
transcript = parse_source_transcript_json(
|
||||||
|
"""
|
||||||
|
[
|
||||||
|
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hrank moves."}
|
||||||
|
]
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
glossary = parse_glossary_yaml(
|
||||||
|
"""
|
||||||
|
glossary:
|
||||||
|
- name: "Hrank"
|
||||||
|
category: pc
|
||||||
|
summary: "Hrank is a player character."
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
decapitalization = CorrectionCandidate(
|
||||||
|
id=1,
|
||||||
|
original_text="Hrank",
|
||||||
|
corrected_text="hrank",
|
||||||
|
confidence=0.95,
|
||||||
|
)
|
||||||
|
fake_client = FakeLLMClient(
|
||||||
|
[
|
||||||
|
CorrectionSet(corrections=[decapitalization]),
|
||||||
|
CorrectionSet(corrections=[]),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
revised = process_transcript(
|
||||||
|
transcript,
|
||||||
|
glossary,
|
||||||
|
_config(tmp_path, glossary_max_llm_passes=1),
|
||||||
|
llm_client=fake_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert revised[0].text == "Hrank moves."
|
||||||
|
run_dirs = list((tmp_path / "work").iterdir())
|
||||||
|
assert len(run_dirs) == 1
|
||||||
|
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
|
||||||
|
skipped = diagnostics["skipped_corrections"][0]
|
||||||
|
assert skipped["stage"] == "glossary"
|
||||||
|
assert skipped["reason"] == "correction changes protected glossary term capitalization"
|
||||||
|
|
||||||
|
|
||||||
def test_pipeline_normalizes_before_llm_prompts(tmp_path):
|
def test_pipeline_normalizes_before_llm_prompts(tmp_path):
|
||||||
transcript = parse_source_transcript_json(
|
transcript = parse_source_transcript_json(
|
||||||
"""
|
"""
|
||||||
@@ -355,7 +485,7 @@ def test_grammar_stage_cannot_reverse_glossary_protected_term(tmp_path):
|
|||||||
assert len(run_dirs) == 1
|
assert len(run_dirs) == 1
|
||||||
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
|
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
|
||||||
assert diagnostics["skipped_corrections"][0]["stage"] == "grammar"
|
assert diagnostics["skipped_corrections"][0]["stage"] == "grammar"
|
||||||
assert "protected glossary term" in diagnostics["skipped_corrections"][0]["reason"]
|
assert diagnostics["skipped_corrections"][0]["reason"] == "correction changes protected glossary term usage"
|
||||||
|
|
||||||
|
|
||||||
def test_grammar_stage_can_correct_toward_protected_term(tmp_path):
|
def test_grammar_stage_can_correct_toward_protected_term(tmp_path):
|
||||||
@@ -439,7 +569,7 @@ def test_grammar_stage_cannot_change_away_from_protected_term(tmp_path):
|
|||||||
assert len(run_dirs) == 1
|
assert len(run_dirs) == 1
|
||||||
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
|
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
|
||||||
assert diagnostics["skipped_corrections"][0]["stage"] == "grammar"
|
assert diagnostics["skipped_corrections"][0]["stage"] == "grammar"
|
||||||
assert "protected glossary term" in diagnostics["skipped_corrections"][0]["reason"]
|
assert diagnostics["skipped_corrections"][0]["reason"] == "correction changes protected glossary term usage"
|
||||||
|
|
||||||
|
|
||||||
def test_grammar_stage_retries_repeated_span_and_applies_unique_retry(tmp_path):
|
def test_grammar_stage_retries_repeated_span_and_applies_unique_retry(tmp_path):
|
||||||
|
|||||||
@@ -34,6 +34,9 @@ def test_prompt_requires_acoustically_plausible_transcription_errors():
|
|||||||
assert '"gestures" to "Jesters"' in prompt_text
|
assert '"gestures" to "Jesters"' in prompt_text
|
||||||
assert '"Lyra" to "Jesters"' in prompt_text
|
assert '"Lyra" to "Jesters"' in prompt_text
|
||||||
assert "should be omitted" in prompt_text
|
assert "should be omitted" in prompt_text
|
||||||
|
assert "glossary names and aliases already present in the transcript as protected spellings" in prompt_text
|
||||||
|
assert "Do not replace, Anglicize, normalize, lowercase" in prompt_text
|
||||||
|
assert "Preserve canonical glossary capitalization" in prompt_text
|
||||||
assert "exact text span that needs replacement" in prompt_text
|
assert "exact text span that needs replacement" in prompt_text
|
||||||
assert "replacement text for that span" in prompt_text
|
assert "replacement text for that span" in prompt_text
|
||||||
assert "Do not return corrections where original_text and corrected_text are identical" in prompt_text
|
assert "Do not return corrections where original_text and corrected_text are identical" in prompt_text
|
||||||
|
|||||||
@@ -22,7 +22,10 @@ def _vocabulary():
|
|||||||
def test_protection_blocks_replacing_protected_term():
|
def test_protection_blocks_replacing_protected_term():
|
||||||
vocabulary = _vocabulary()
|
vocabulary = _vocabulary()
|
||||||
|
|
||||||
assert vocabulary.violation_reason("Hrank moves.", "Frank moves.") is not None
|
assert (
|
||||||
|
vocabulary.violation_reason("Hrank moves.", "Frank moves.")
|
||||||
|
== "correction changes protected glossary term usage"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_protection_blocks_replacing_possessive_protected_term():
|
def test_protection_blocks_replacing_possessive_protected_term():
|
||||||
@@ -34,7 +37,10 @@ def test_protection_blocks_replacing_possessive_protected_term():
|
|||||||
def test_protection_blocks_lowercasing_protected_term():
|
def test_protection_blocks_lowercasing_protected_term():
|
||||||
vocabulary = _vocabulary()
|
vocabulary = _vocabulary()
|
||||||
|
|
||||||
assert vocabulary.violation_reason("Hrank moves.", "hrank moves.") is not None
|
assert (
|
||||||
|
vocabulary.violation_reason("Hrank moves.", "hrank moves.")
|
||||||
|
== "correction changes protected glossary term capitalization"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_protection_blocks_noncanonical_uppercase_protected_term():
|
def test_protection_blocks_noncanonical_uppercase_protected_term():
|
||||||
|
|||||||
Reference in New Issue
Block a user