eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Extend glossary term protection to the initial glossary LLM stage

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-22 11:15:36 -05:00
parent 183f8ba31d
commit 13b81aab2a
6 changed files with 149 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/audita/pipeline.py
View File

@@ -74,6 +74,7 @@ def process_transcript(
max_llm_passes=config.glossary_max_llm_passes,
confidence_threshold=config.glossary_confidence_threshold,
replacement_mode="replace_all",
correction_guard=protected_vocabulary.violation_reason,
),
StageSpec(
name="grammar",

3
src/audita/prompts.py
View File

@@ -43,6 +43,9 @@ def build_glossary_correction_messages(
"- Appropriate example: correcting \"gestures\" to \"Jesters\" can be valid if \"Jesters\" appears in the glossary and nearby context supports that inference.\n"
"- Inappropriate example: correcting \"Lyra\" to \"Jesters\" should be omitted because those words are not similar in spoken English, even if \"Jesters\" appears in the glossary.\n"
"- Do not replace one clear glossary term, character name, location, or ordinary word with a different glossary term unless it is a plausible mishearing.\n"
"- Treat glossary names and aliases already present in the transcript as protected spellings.\n"
"- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.\n"
"- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n"
"- Assign high confidence only when the correction is supported by glossary evidence, local context, and spoken-word similarity; otherwise omit the correction.\n"
"- Use the exact id from the input segment.\n"
"- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n"

4
src/audita/protection.py
View File

@@ -35,12 +35,12 @@ class ProtectedVocabulary:
after_folded = [term.folded for term in after_terms]
for before_term in before_terms:
if before_term.folded not in after_folded:
return "grammar correction changes protected glossary term usage"
return "correction changes protected glossary term usage"
for after_term in after_terms:
canonical = self.canonical_by_folded[after_term.folded]
if after_term.text != canonical:
return "grammar correction changes protected glossary term capitalization"
return "correction changes protected glossary term capitalization"
return None

134
tests/test_pipeline.py
View File

@@ -79,6 +79,136 @@ def test_pipeline_processes_with_fake_llm_and_cleans_work_dir(tmp_path):
assert list((tmp_path / "work").iterdir()) == []
def test_glossary_stage_can_correct_toward_protected_term(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Frank moves."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Hrank"
category: pc
summary: "Hrank is a player character."
"""
)
glossary_correction = CorrectionCandidate(
id=1,
original_text="Frank",
corrected_text="Hrank",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[glossary_correction]),
CorrectionSet(corrections=[]),
]
)
revised = process_transcript(
transcript,
glossary,
_config(tmp_path),
llm_client=fake_client,
)
assert revised[0].text == "Hrank moves."
assert list((tmp_path / "work").iterdir()) == []
def test_glossary_stage_cannot_change_away_from_protected_term(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hrank moves."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Hrank"
category: pc
summary: "Hrank is a player character."
"""
)
glossary_reversal = CorrectionCandidate(
id=1,
original_text="Hrank",
corrected_text="Frank",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[glossary_reversal]),
CorrectionSet(corrections=[]),
]
)
revised = process_transcript(
transcript,
glossary,
_config(tmp_path, glossary_max_llm_passes=1),
llm_client=fake_client,
)
assert revised[0].text == "Hrank moves."
run_dirs = list((tmp_path / "work").iterdir())
assert len(run_dirs) == 1
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
skipped = diagnostics["skipped_corrections"][0]
assert skipped["stage"] == "glossary"
assert skipped["reason"] == "correction changes protected glossary term usage"
def test_glossary_stage_cannot_decanonicalize_protected_term(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hrank moves."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Hrank"
category: pc
summary: "Hrank is a player character."
"""
)
decapitalization = CorrectionCandidate(
id=1,
original_text="Hrank",
corrected_text="hrank",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[decapitalization]),
CorrectionSet(corrections=[]),
]
)
revised = process_transcript(
transcript,
glossary,
_config(tmp_path, glossary_max_llm_passes=1),
llm_client=fake_client,
)
assert revised[0].text == "Hrank moves."
run_dirs = list((tmp_path / "work").iterdir())
assert len(run_dirs) == 1
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
skipped = diagnostics["skipped_corrections"][0]
assert skipped["stage"] == "glossary"
assert skipped["reason"] == "correction changes protected glossary term capitalization"
def test_pipeline_normalizes_before_llm_prompts(tmp_path):
transcript = parse_source_transcript_json(
"""
@@ -355,7 +485,7 @@ def test_grammar_stage_cannot_reverse_glossary_protected_term(tmp_path):
assert len(run_dirs) == 1
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
assert diagnostics["skipped_corrections"][0]["stage"] == "grammar"
assert "protected glossary term" in diagnostics["skipped_corrections"][0]["reason"]
assert diagnostics["skipped_corrections"][0]["reason"] == "correction changes protected glossary term usage"
def test_grammar_stage_can_correct_toward_protected_term(tmp_path):
@@ -439,7 +569,7 @@ def test_grammar_stage_cannot_change_away_from_protected_term(tmp_path):
assert len(run_dirs) == 1
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
assert diagnostics["skipped_corrections"][0]["stage"] == "grammar"
assert "protected glossary term" in diagnostics["skipped_corrections"][0]["reason"]
assert diagnostics["skipped_corrections"][0]["reason"] == "correction changes protected glossary term usage"
def test_grammar_stage_retries_repeated_span_and_applies_unique_retry(tmp_path):

3
tests/test_prompts.py
View File

@@ -34,6 +34,9 @@ def test_prompt_requires_acoustically_plausible_transcription_errors():
assert '"gestures" to "Jesters"' in prompt_text
assert '"Lyra" to "Jesters"' in prompt_text
assert "should be omitted" in prompt_text
assert "glossary names and aliases already present in the transcript as protected spellings" in prompt_text
assert "Do not replace, Anglicize, normalize, lowercase" in prompt_text
assert "Preserve canonical glossary capitalization" in prompt_text
assert "exact text span that needs replacement" in prompt_text
assert "replacement text for that span" in prompt_text
assert "Do not return corrections where original_text and corrected_text are identical" in prompt_text

10
tests/test_protection.py
View File

@@ -22,7 +22,10 @@ def _vocabulary():
def test_protection_blocks_replacing_protected_term():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("Hrank moves.", "Frank moves.") is not None
assert (
vocabulary.violation_reason("Hrank moves.", "Frank moves.")
== "correction changes protected glossary term usage"
)
def test_protection_blocks_replacing_possessive_protected_term():
@@ -34,7 +37,10 @@ def test_protection_blocks_replacing_possessive_protected_term():
def test_protection_blocks_lowercasing_protected_term():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("Hrank moves.", "hrank moves.") is not None
assert (
vocabulary.violation_reason("Hrank moves.", "hrank moves.")
== "correction changes protected glossary term capitalization"
)
def test_protection_blocks_noncanonical_uppercase_protected_term():