More bugfixes in the glossary protection guard module
This commit is contained in:
@@ -2,7 +2,8 @@ import pytest
|
||||
|
||||
from audita.corrections import apply_corrections
|
||||
from audita.errors import AuditaValidationError
|
||||
from audita.schemas import CorrectionCandidate, parse_transcript_json
|
||||
from audita.protection import ProtectedVocabulary
|
||||
from audita.schemas import CorrectionCandidate, parse_glossary_yaml, parse_transcript_json
|
||||
|
||||
|
||||
def _transcript():
|
||||
@@ -200,6 +201,44 @@ def test_apply_corrections_skips_when_guard_rejects_replacement():
|
||||
assert result.skipped[0].actual_text == "Hrank moves."
|
||||
|
||||
|
||||
def test_apply_corrections_guards_only_replacement_span():
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
[
|
||||
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "You have to keep it bind. Svend sees the jesters."}
|
||||
]
|
||||
"""
|
||||
)
|
||||
glossary = parse_glossary_yaml(
|
||||
"""
|
||||
glossary:
|
||||
- name: "Svend"
|
||||
category: pc
|
||||
summary: "Svend is a player character."
|
||||
- name: "Jesters"
|
||||
category: faction
|
||||
summary: "The Jesters are a faction."
|
||||
"""
|
||||
)
|
||||
correction = CorrectionCandidate(
|
||||
id=1,
|
||||
original_text="keep it bind",
|
||||
corrected_text="keep in mind",
|
||||
confidence=0.8,
|
||||
)
|
||||
|
||||
result = apply_corrections(
|
||||
transcript,
|
||||
[correction],
|
||||
confidence_threshold=0.8,
|
||||
replacement_mode="require_unique",
|
||||
correction_guard=ProtectedVocabulary.from_glossary(glossary).violation_reason,
|
||||
)
|
||||
|
||||
assert result.transcript[0].text == "You have to keep in mind. Svend sees the jesters."
|
||||
assert result.skipped == []
|
||||
|
||||
|
||||
def test_apply_corrections_without_guard_remains_permissive():
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
|
||||
@@ -119,6 +119,49 @@ def test_glossary_stage_can_correct_toward_protected_term(tmp_path):
|
||||
assert list((tmp_path / "work").iterdir()) == []
|
||||
|
||||
|
||||
def test_glossary_guard_ignores_unrelated_protected_terms_elsewhere_in_segment(tmp_path):
|
||||
transcript = parse_source_transcript_json(
|
||||
"""
|
||||
[
|
||||
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "The gestures are near lyra."}
|
||||
]
|
||||
"""
|
||||
)
|
||||
glossary = parse_glossary_yaml(
|
||||
"""
|
||||
glossary:
|
||||
- name: "Jesters"
|
||||
category: faction
|
||||
summary: "The Jesters are a faction."
|
||||
- name: "Lyra"
|
||||
category: npc
|
||||
summary: "Lyra is an NPC."
|
||||
"""
|
||||
)
|
||||
glossary_correction = CorrectionCandidate(
|
||||
id=1,
|
||||
original_text="gestures",
|
||||
corrected_text="Jesters",
|
||||
confidence=0.95,
|
||||
)
|
||||
fake_client = FakeLLMClient(
|
||||
[
|
||||
CorrectionSet(corrections=[glossary_correction]),
|
||||
CorrectionSet(corrections=[]),
|
||||
]
|
||||
)
|
||||
|
||||
revised = process_transcript(
|
||||
transcript,
|
||||
glossary,
|
||||
_config(tmp_path),
|
||||
llm_client=fake_client,
|
||||
)
|
||||
|
||||
assert revised[0].text == "The Jesters are near lyra."
|
||||
assert list((tmp_path / "work").iterdir()) == []
|
||||
|
||||
|
||||
def test_glossary_stage_cannot_change_away_from_protected_term(tmp_path):
|
||||
transcript = parse_source_transcript_json(
|
||||
"""
|
||||
|
||||
@@ -70,6 +70,44 @@ def test_prompt_uses_simplified_segment_payload():
|
||||
assert "end" not in prompt_segments[0]
|
||||
|
||||
|
||||
def test_prompts_do_not_include_inferred_plurals():
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
[
|
||||
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "The gestures are nearby."}
|
||||
]
|
||||
"""
|
||||
)
|
||||
glossary = parse_glossary_yaml(
|
||||
"""
|
||||
glossary:
|
||||
- name: "Godfrey"
|
||||
aliases:
|
||||
- "Jester"
|
||||
category: npc
|
||||
summary: "Godfrey is an NPC."
|
||||
"""
|
||||
)
|
||||
section = chunk_transcript(transcript, max_section_tokens=16000)[0]
|
||||
|
||||
glossary_messages = build_glossary_correction_messages(section, glossary)
|
||||
glossary_json = glossary_messages[1]["content"].split("Glossary:\n", maxsplit=1)[1].split(
|
||||
"\n\nTranscript section:",
|
||||
maxsplit=1,
|
||||
)[0]
|
||||
grammar_messages = build_grammar_correction_messages(section, glossary)
|
||||
grammar_json = grammar_messages[1]["content"].split("Protected glossary/context:\n", maxsplit=1)[1].split(
|
||||
"\n\nTranscript section:",
|
||||
maxsplit=1,
|
||||
)[0]
|
||||
|
||||
for prompt_glossary in (json.loads(glossary_json), json.loads(grammar_json)):
|
||||
entry = prompt_glossary["glossary"][0]
|
||||
assert "plural" not in entry
|
||||
assert "Godfreys" not in json.dumps(prompt_glossary)
|
||||
assert "Jesters" not in json.dumps(prompt_glossary)
|
||||
|
||||
|
||||
def test_grammar_prompt_limits_readability_corrections_and_protects_glossary():
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
|
||||
@@ -14,6 +14,17 @@ def _vocabulary():
|
||||
- name: "Popov"
|
||||
category: npc
|
||||
summary: "Popov is an allied NPC."
|
||||
- name: "Jesters"
|
||||
aliases:
|
||||
- "Jester"
|
||||
category: faction
|
||||
summary: "The Jesters are a faction."
|
||||
- name: "Svend"
|
||||
category: pc
|
||||
summary: "Svend is a player character."
|
||||
- name: "Godfrey"
|
||||
category: npc
|
||||
summary: "Godfrey is an NPC."
|
||||
"""
|
||||
)
|
||||
return ProtectedVocabulary.from_glossary(glossary)
|
||||
@@ -59,6 +70,9 @@ def test_protection_allows_correction_toward_protected_term():
|
||||
vocabulary = _vocabulary()
|
||||
|
||||
assert vocabulary.violation_reason("Pawpaw moves.", "Popov moves.") is None
|
||||
assert vocabulary.violation_reason("gestures", "Jesters") is None
|
||||
assert vocabulary.violation_reason("rank", "Hrank") is None
|
||||
assert vocabulary.violation_reason("spend", "Svend") is None
|
||||
|
||||
|
||||
def test_protection_allows_possessive_correction_toward_protected_term():
|
||||
@@ -67,6 +81,54 @@ def test_protection_allows_possessive_correction_toward_protected_term():
|
||||
assert vocabulary.violation_reason("Pawpaw's exhausted.", "Popov's exhausted.") is None
|
||||
|
||||
|
||||
def test_protection_blocks_noncanonical_introduced_protected_term():
|
||||
vocabulary = _vocabulary()
|
||||
|
||||
assert (
|
||||
vocabulary.violation_reason("gestures", "jesters")
|
||||
== "correction changes protected glossary term capitalization"
|
||||
)
|
||||
assert (
|
||||
vocabulary.violation_reason("rank", "hrank")
|
||||
== "correction changes protected glossary term capitalization"
|
||||
)
|
||||
assert (
|
||||
vocabulary.violation_reason("spend", "svend")
|
||||
== "correction changes protected glossary term capitalization"
|
||||
)
|
||||
|
||||
|
||||
def test_protection_allows_inferred_name_plural():
|
||||
vocabulary = _vocabulary()
|
||||
|
||||
assert vocabulary.violation_reason("Godfrey's", "Godfreys") is None
|
||||
|
||||
|
||||
def test_protection_allows_inferred_alias_plural():
|
||||
vocabulary = _vocabulary()
|
||||
|
||||
assert vocabulary.violation_reason("gesture", "Jesters") is None
|
||||
|
||||
|
||||
def test_protection_allows_explicit_plural():
|
||||
glossary = parse_glossary_yaml(
|
||||
"""
|
||||
glossary:
|
||||
- name: "Mox"
|
||||
plural: "Moxen"
|
||||
category: faction
|
||||
summary: "The Mox are a faction."
|
||||
"""
|
||||
)
|
||||
vocabulary = ProtectedVocabulary.from_glossary(glossary)
|
||||
|
||||
assert vocabulary.violation_reason("Mox's", "Moxen") is None
|
||||
assert (
|
||||
vocabulary.violation_reason("Mox's", "moxen")
|
||||
== "correction changes protected glossary term capitalization"
|
||||
)
|
||||
|
||||
|
||||
def test_protection_allows_punctuation_around_protected_term():
|
||||
vocabulary = _vocabulary()
|
||||
|
||||
|
||||
@@ -177,6 +177,34 @@ def test_valid_glossary_parses():
|
||||
)
|
||||
|
||||
assert glossary.glossary[0].name == "Lyra"
|
||||
assert glossary.glossary[0].plural is None
|
||||
|
||||
|
||||
def test_glossary_accepts_optional_plural():
|
||||
glossary = parse_glossary_yaml(
|
||||
"""
|
||||
glossary:
|
||||
- name: "Godfrey"
|
||||
plural: "Godfreys"
|
||||
category: npc
|
||||
summary: "Godfrey is an NPC."
|
||||
"""
|
||||
)
|
||||
|
||||
assert glossary.glossary[0].plural == "Godfreys"
|
||||
|
||||
|
||||
def test_glossary_rejects_empty_plural():
|
||||
with pytest.raises(AuditaValidationError):
|
||||
parse_glossary_yaml(
|
||||
"""
|
||||
glossary:
|
||||
- name: "Godfrey"
|
||||
plural: ""
|
||||
category: npc
|
||||
summary: "Godfrey is an NPC."
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def test_glossary_rejects_empty_entries():
|
||||
|
||||
Reference in New Issue
Block a user