Extend glossary term protection to the initial glossary LLM stage

This commit is contained in:
2026-04-22 11:15:36 -05:00
parent 183f8ba31d
commit 13b81aab2a
6 changed files with 149 additions and 6 deletions

View File

@@ -79,6 +79,136 @@ def test_pipeline_processes_with_fake_llm_and_cleans_work_dir(tmp_path):
assert list((tmp_path / "work").iterdir()) == []
def test_glossary_stage_can_correct_toward_protected_term(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Frank moves."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Hrank"
category: pc
summary: "Hrank is a player character."
"""
)
glossary_correction = CorrectionCandidate(
id=1,
original_text="Frank",
corrected_text="Hrank",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[glossary_correction]),
CorrectionSet(corrections=[]),
]
)
revised = process_transcript(
transcript,
glossary,
_config(tmp_path),
llm_client=fake_client,
)
assert revised[0].text == "Hrank moves."
assert list((tmp_path / "work").iterdir()) == []
def test_glossary_stage_cannot_change_away_from_protected_term(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hrank moves."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Hrank"
category: pc
summary: "Hrank is a player character."
"""
)
glossary_reversal = CorrectionCandidate(
id=1,
original_text="Hrank",
corrected_text="Frank",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[glossary_reversal]),
CorrectionSet(corrections=[]),
]
)
revised = process_transcript(
transcript,
glossary,
_config(tmp_path, glossary_max_llm_passes=1),
llm_client=fake_client,
)
assert revised[0].text == "Hrank moves."
run_dirs = list((tmp_path / "work").iterdir())
assert len(run_dirs) == 1
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
skipped = diagnostics["skipped_corrections"][0]
assert skipped["stage"] == "glossary"
assert skipped["reason"] == "correction changes protected glossary term usage"
def test_glossary_stage_cannot_decanonicalize_protected_term(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hrank moves."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Hrank"
category: pc
summary: "Hrank is a player character."
"""
)
decapitalization = CorrectionCandidate(
id=1,
original_text="Hrank",
corrected_text="hrank",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[decapitalization]),
CorrectionSet(corrections=[]),
]
)
revised = process_transcript(
transcript,
glossary,
_config(tmp_path, glossary_max_llm_passes=1),
llm_client=fake_client,
)
assert revised[0].text == "Hrank moves."
run_dirs = list((tmp_path / "work").iterdir())
assert len(run_dirs) == 1
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
skipped = diagnostics["skipped_corrections"][0]
assert skipped["stage"] == "glossary"
assert skipped["reason"] == "correction changes protected glossary term capitalization"
def test_pipeline_normalizes_before_llm_prompts(tmp_path):
transcript = parse_source_transcript_json(
"""
@@ -355,7 +485,7 @@ def test_grammar_stage_cannot_reverse_glossary_protected_term(tmp_path):
assert len(run_dirs) == 1
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
assert diagnostics["skipped_corrections"][0]["stage"] == "grammar"
assert "protected glossary term" in diagnostics["skipped_corrections"][0]["reason"]
assert diagnostics["skipped_corrections"][0]["reason"] == "correction changes protected glossary term usage"
def test_grammar_stage_can_correct_toward_protected_term(tmp_path):
@@ -439,7 +569,7 @@ def test_grammar_stage_cannot_change_away_from_protected_term(tmp_path):
assert len(run_dirs) == 1
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
assert diagnostics["skipped_corrections"][0]["stage"] == "grammar"
assert "protected glossary term" in diagnostics["skipped_corrections"][0]["reason"]
assert diagnostics["skipped_corrections"][0]["reason"] == "correction changes protected glossary term usage"
def test_grammar_stage_retries_repeated_span_and_applies_unique_retry(tmp_path):