337 lines
10 KiB
Python
337 lines
10 KiB
Python
import pytest
|
|
|
|
from audita.errors import AuditaLLMError
|
|
from audita.protection import ProtectedVocabulary
|
|
from audita.schemas import (
|
|
CorrectionCandidate,
|
|
GrammarSpokenFormValidationDecision,
|
|
GrammarSpokenFormValidationSet,
|
|
GrammarValidationDecision,
|
|
GrammarValidationSet,
|
|
parse_glossary_yaml,
|
|
parse_transcript_json,
|
|
)
|
|
from audita.semantic_validation import (
|
|
filter_with_meaning_preserving_validations,
|
|
filter_with_spoken_form_validations,
|
|
keep_corrections_with_indexes,
|
|
select_grammar_validation_candidates,
|
|
)
|
|
|
|
|
|
def _transcript():
|
|
return parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "he became visible and then gestures arrived"}
|
|
]
|
|
"""
|
|
)
|
|
|
|
|
|
def _vocabulary():
|
|
glossary = parse_glossary_yaml(
|
|
"""
|
|
glossary:
|
|
- name: "Jesters"
|
|
category: faction
|
|
summary: "The Jesters are a faction."
|
|
"""
|
|
)
|
|
return ProtectedVocabulary.from_glossary(glossary)
|
|
|
|
|
|
def test_protected_vocabulary_correction_bypasses_validation():
|
|
correction = CorrectionCandidate(
|
|
id=1,
|
|
original_text="gestures",
|
|
corrected_text="Jesters",
|
|
confidence=0.95,
|
|
)
|
|
|
|
candidates, bypassed_count = select_grammar_validation_candidates(
|
|
_transcript(),
|
|
[correction],
|
|
confidence_threshold=0.8,
|
|
replacement_mode="require_unique",
|
|
protected_vocabulary=_vocabulary(),
|
|
)
|
|
|
|
assert candidates == []
|
|
assert bypassed_count == 1
|
|
|
|
|
|
def test_capitalization_only_correction_bypasses_validation():
|
|
correction = CorrectionCandidate(
|
|
id=1,
|
|
original_text="he",
|
|
corrected_text="He",
|
|
confidence=0.95,
|
|
)
|
|
|
|
candidates, bypassed_count = select_grammar_validation_candidates(
|
|
_transcript(),
|
|
[correction],
|
|
confidence_threshold=0.8,
|
|
replacement_mode="require_unique",
|
|
protected_vocabulary=_vocabulary(),
|
|
)
|
|
|
|
assert candidates == []
|
|
assert bypassed_count == 1
|
|
|
|
|
|
def test_punctuation_only_correction_bypasses_validation():
|
|
correction = CorrectionCandidate(
|
|
id=1,
|
|
original_text="visible",
|
|
corrected_text="visible.",
|
|
confidence=0.95,
|
|
)
|
|
|
|
candidates, bypassed_count = select_grammar_validation_candidates(
|
|
_transcript(),
|
|
[correction],
|
|
confidence_threshold=0.8,
|
|
replacement_mode="require_unique",
|
|
protected_vocabulary=_vocabulary(),
|
|
)
|
|
|
|
assert candidates == []
|
|
assert bypassed_count == 1
|
|
|
|
|
|
def test_meaning_sensitive_substitution_requires_validation():
|
|
correction = CorrectionCandidate(
|
|
id=1,
|
|
original_text="visible",
|
|
corrected_text="invisible",
|
|
confidence=0.95,
|
|
)
|
|
|
|
candidates, bypassed_count = select_grammar_validation_candidates(
|
|
_transcript(),
|
|
[correction],
|
|
confidence_threshold=0.8,
|
|
replacement_mode="require_unique",
|
|
protected_vocabulary=_vocabulary(),
|
|
)
|
|
|
|
assert len(candidates) == 1
|
|
assert candidates[0].correction_index == 0
|
|
assert candidates[0].original_segment_text == "he became visible and then gestures arrived"
|
|
assert candidates[0].corrected_segment_text == "he became invisible and then gestures arrived"
|
|
assert bypassed_count == 0
|
|
|
|
|
|
def test_meaning_preserving_validation_approves_without_rescue():
|
|
correction = CorrectionCandidate(
|
|
id=1,
|
|
original_text="bind",
|
|
corrected_text="mind",
|
|
confidence=0.95,
|
|
)
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Keep in bind."}
|
|
]
|
|
"""
|
|
)
|
|
candidates, _ = select_grammar_validation_candidates(
|
|
transcript,
|
|
[correction],
|
|
confidence_threshold=0.8,
|
|
replacement_mode="require_unique",
|
|
protected_vocabulary=_vocabulary(),
|
|
)
|
|
|
|
result = filter_with_meaning_preserving_validations(
|
|
candidates,
|
|
GrammarValidationSet(
|
|
validations=[
|
|
GrammarValidationDecision(
|
|
correction_index=0,
|
|
is_meaning_preserving=True,
|
|
confidence=0.95,
|
|
reason="This preserves the intended meaning.",
|
|
)
|
|
]
|
|
),
|
|
confidence_threshold=0.8,
|
|
)
|
|
|
|
assert result.approved_correction_indexes == [0]
|
|
assert result.rescue_candidates == []
|
|
assert result.approved_count == 1
|
|
assert result.rejected_count == 0
|
|
|
|
|
|
def test_spoken_form_validation_can_rescue_homophone_fix():
|
|
correction = CorrectionCandidate(
|
|
id=1,
|
|
original_text="dam",
|
|
corrected_text="damn",
|
|
confidence=0.95,
|
|
)
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."}
|
|
]
|
|
"""
|
|
)
|
|
candidates, _ = select_grammar_validation_candidates(
|
|
transcript,
|
|
[correction],
|
|
confidence_threshold=0.8,
|
|
replacement_mode="require_unique",
|
|
protected_vocabulary=_vocabulary(),
|
|
)
|
|
meaning_result = filter_with_meaning_preserving_validations(
|
|
candidates,
|
|
GrammarValidationSet(
|
|
validations=[
|
|
GrammarValidationDecision(
|
|
correction_index=0,
|
|
is_meaning_preserving=False,
|
|
confidence=0.99,
|
|
reason="Written meaning changes from a barrier to a curse word.",
|
|
)
|
|
]
|
|
),
|
|
confidence_threshold=0.8,
|
|
)
|
|
spoken_form_result = filter_with_spoken_form_validations(
|
|
meaning_result.rescue_candidates,
|
|
GrammarSpokenFormValidationSet(
|
|
validations=[
|
|
GrammarSpokenFormValidationDecision(
|
|
correction_index=0,
|
|
is_likely_spoken_form_correction=True,
|
|
confidence=0.95,
|
|
reason="The surrounding phrase strongly supports the intended spoken phrase with a curse word.",
|
|
)
|
|
]
|
|
),
|
|
confidence_threshold=0.8,
|
|
)
|
|
|
|
kept = keep_corrections_with_indexes([correction], spoken_form_result.approved_correction_indexes)
|
|
assert meaning_result.approved_correction_indexes == []
|
|
assert kept == [correction]
|
|
assert spoken_form_result.skipped == []
|
|
|
|
|
|
def test_spoken_form_validation_rejects_non_homophone_semantic_change():
|
|
correction = CorrectionCandidate(
|
|
id=1,
|
|
original_text="visible",
|
|
corrected_text="invisible",
|
|
confidence=0.95,
|
|
)
|
|
candidates, _ = select_grammar_validation_candidates(
|
|
_transcript(),
|
|
[correction],
|
|
confidence_threshold=0.8,
|
|
replacement_mode="require_unique",
|
|
protected_vocabulary=_vocabulary(),
|
|
)
|
|
meaning_result = filter_with_meaning_preserving_validations(
|
|
candidates,
|
|
GrammarValidationSet(
|
|
validations=[
|
|
GrammarValidationDecision(
|
|
correction_index=0,
|
|
is_meaning_preserving=False,
|
|
confidence=0.99,
|
|
reason="This reverses visible to invisible.",
|
|
)
|
|
]
|
|
),
|
|
confidence_threshold=0.8,
|
|
)
|
|
spoken_form_result = filter_with_spoken_form_validations(
|
|
meaning_result.rescue_candidates,
|
|
GrammarSpokenFormValidationSet(
|
|
validations=[
|
|
GrammarSpokenFormValidationDecision(
|
|
correction_index=0,
|
|
is_likely_spoken_form_correction=False,
|
|
confidence=0.99,
|
|
reason="This is a semantic reversal, not a likely spoken-form transcription error.",
|
|
)
|
|
]
|
|
),
|
|
confidence_threshold=0.8,
|
|
)
|
|
|
|
assert spoken_form_result.approved_correction_indexes == []
|
|
assert spoken_form_result.rejected_count == 1
|
|
assert spoken_form_result.skipped[0].reason == "grammar validation rejected semantic change"
|
|
assert spoken_form_result.skipped[0].validation_reason == (
|
|
"This is a semantic reversal, not a likely spoken-form transcription error."
|
|
)
|
|
|
|
|
|
def test_validation_rejects_duplicate_unknown_and_missing_decisions():
|
|
correction = CorrectionCandidate(
|
|
id=1,
|
|
original_text="visible",
|
|
corrected_text="invisible",
|
|
confidence=0.95,
|
|
)
|
|
candidates, _ = select_grammar_validation_candidates(
|
|
_transcript(),
|
|
[correction],
|
|
confidence_threshold=0.8,
|
|
replacement_mode="require_unique",
|
|
protected_vocabulary=_vocabulary(),
|
|
)
|
|
meaning_decision = GrammarValidationDecision(
|
|
correction_index=0,
|
|
is_meaning_preserving=True,
|
|
confidence=0.95,
|
|
reason="Preserves meaning.",
|
|
)
|
|
spoken_form_decision = GrammarSpokenFormValidationDecision(
|
|
correction_index=0,
|
|
is_likely_spoken_form_correction=True,
|
|
confidence=0.95,
|
|
reason="Likely spoken-form correction.",
|
|
)
|
|
|
|
with pytest.raises(AuditaLLMError):
|
|
filter_with_meaning_preserving_validations(
|
|
candidates,
|
|
GrammarValidationSet(validations=[meaning_decision, meaning_decision]),
|
|
confidence_threshold=0.8,
|
|
)
|
|
with pytest.raises(AuditaLLMError):
|
|
filter_with_meaning_preserving_validations(
|
|
candidates,
|
|
GrammarValidationSet(validations=[]),
|
|
confidence_threshold=0.8,
|
|
)
|
|
with pytest.raises(AuditaLLMError):
|
|
filter_with_spoken_form_validations(
|
|
candidates,
|
|
GrammarSpokenFormValidationSet(
|
|
validations=[
|
|
GrammarSpokenFormValidationDecision(
|
|
correction_index=99,
|
|
is_likely_spoken_form_correction=True,
|
|
confidence=0.95,
|
|
reason="Unknown.",
|
|
)
|
|
]
|
|
),
|
|
confidence_threshold=0.8,
|
|
)
|
|
with pytest.raises(AuditaLLMError):
|
|
filter_with_spoken_form_validations(
|
|
candidates,
|
|
GrammarSpokenFormValidationSet(validations=[spoken_form_decision, spoken_form_decision]),
|
|
confidence_threshold=0.8,
|
|
)
|