Implemented a second-stage validator for grammar corrections that allows homophone-related changes
This commit is contained in:
@@ -1,16 +1,22 @@
|
||||
import pytest
|
||||
|
||||
from audita.errors import AuditaLLMError
|
||||
from audita.semantic_validation import select_grammar_validation_candidates
|
||||
from audita.semantic_validation import filter_with_grammar_validations
|
||||
from audita.protection import ProtectedVocabulary
|
||||
from audita.schemas import (
|
||||
CorrectionCandidate,
|
||||
GrammarSpokenFormValidationDecision,
|
||||
GrammarSpokenFormValidationSet,
|
||||
GrammarValidationDecision,
|
||||
GrammarValidationSet,
|
||||
parse_glossary_yaml,
|
||||
parse_transcript_json,
|
||||
)
|
||||
from audita.semantic_validation import (
|
||||
filter_with_meaning_preserving_validations,
|
||||
filter_with_spoken_form_validations,
|
||||
keep_corrections_with_indexes,
|
||||
select_grammar_validation_candidates,
|
||||
)
|
||||
|
||||
|
||||
def _transcript():
|
||||
@@ -118,7 +124,106 @@ def test_meaning_sensitive_substitution_requires_validation():
|
||||
assert bypassed_count == 0
|
||||
|
||||
|
||||
def test_validation_rejects_duplicate_and_unknown_decisions():
|
||||
def test_meaning_preserving_validation_approves_without_rescue():
|
||||
correction = CorrectionCandidate(
|
||||
id=1,
|
||||
original_text="bind",
|
||||
corrected_text="mind",
|
||||
confidence=0.95,
|
||||
)
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
[
|
||||
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Keep in bind."}
|
||||
]
|
||||
"""
|
||||
)
|
||||
candidates, _ = select_grammar_validation_candidates(
|
||||
transcript,
|
||||
[correction],
|
||||
confidence_threshold=0.8,
|
||||
replacement_mode="require_unique",
|
||||
protected_vocabulary=_vocabulary(),
|
||||
)
|
||||
|
||||
result = filter_with_meaning_preserving_validations(
|
||||
candidates,
|
||||
GrammarValidationSet(
|
||||
validations=[
|
||||
GrammarValidationDecision(
|
||||
correction_index=0,
|
||||
is_meaning_preserving=True,
|
||||
confidence=0.95,
|
||||
reason="This preserves the intended meaning.",
|
||||
)
|
||||
]
|
||||
),
|
||||
confidence_threshold=0.8,
|
||||
)
|
||||
|
||||
assert result.approved_correction_indexes == [0]
|
||||
assert result.rescue_candidates == []
|
||||
assert result.approved_count == 1
|
||||
assert result.rejected_count == 0
|
||||
|
||||
|
||||
def test_spoken_form_validation_can_rescue_homophone_fix():
|
||||
correction = CorrectionCandidate(
|
||||
id=1,
|
||||
original_text="dam",
|
||||
corrected_text="damn",
|
||||
confidence=0.95,
|
||||
)
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
[
|
||||
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."}
|
||||
]
|
||||
"""
|
||||
)
|
||||
candidates, _ = select_grammar_validation_candidates(
|
||||
transcript,
|
||||
[correction],
|
||||
confidence_threshold=0.8,
|
||||
replacement_mode="require_unique",
|
||||
protected_vocabulary=_vocabulary(),
|
||||
)
|
||||
meaning_result = filter_with_meaning_preserving_validations(
|
||||
candidates,
|
||||
GrammarValidationSet(
|
||||
validations=[
|
||||
GrammarValidationDecision(
|
||||
correction_index=0,
|
||||
is_meaning_preserving=False,
|
||||
confidence=0.99,
|
||||
reason="Written meaning changes from a barrier to a curse word.",
|
||||
)
|
||||
]
|
||||
),
|
||||
confidence_threshold=0.8,
|
||||
)
|
||||
spoken_form_result = filter_with_spoken_form_validations(
|
||||
meaning_result.rescue_candidates,
|
||||
GrammarSpokenFormValidationSet(
|
||||
validations=[
|
||||
GrammarSpokenFormValidationDecision(
|
||||
correction_index=0,
|
||||
is_likely_spoken_form_correction=True,
|
||||
confidence=0.95,
|
||||
reason="The surrounding phrase strongly supports the intended spoken phrase with a curse word.",
|
||||
)
|
||||
]
|
||||
),
|
||||
confidence_threshold=0.8,
|
||||
)
|
||||
|
||||
kept = keep_corrections_with_indexes([correction], spoken_form_result.approved_correction_indexes)
|
||||
assert meaning_result.approved_correction_indexes == []
|
||||
assert kept == [correction]
|
||||
assert spoken_form_result.skipped == []
|
||||
|
||||
|
||||
def test_spoken_form_validation_rejects_non_homophone_semantic_change():
|
||||
correction = CorrectionCandidate(
|
||||
id=1,
|
||||
original_text="visible",
|
||||
@@ -132,30 +237,90 @@ def test_validation_rejects_duplicate_and_unknown_decisions():
|
||||
replacement_mode="require_unique",
|
||||
protected_vocabulary=_vocabulary(),
|
||||
)
|
||||
decision = GrammarValidationDecision(
|
||||
meaning_result = filter_with_meaning_preserving_validations(
|
||||
candidates,
|
||||
GrammarValidationSet(
|
||||
validations=[
|
||||
GrammarValidationDecision(
|
||||
correction_index=0,
|
||||
is_meaning_preserving=False,
|
||||
confidence=0.99,
|
||||
reason="This reverses visible to invisible.",
|
||||
)
|
||||
]
|
||||
),
|
||||
confidence_threshold=0.8,
|
||||
)
|
||||
spoken_form_result = filter_with_spoken_form_validations(
|
||||
meaning_result.rescue_candidates,
|
||||
GrammarSpokenFormValidationSet(
|
||||
validations=[
|
||||
GrammarSpokenFormValidationDecision(
|
||||
correction_index=0,
|
||||
is_likely_spoken_form_correction=False,
|
||||
confidence=0.99,
|
||||
reason="This is a semantic reversal, not a likely spoken-form transcription error.",
|
||||
)
|
||||
]
|
||||
),
|
||||
confidence_threshold=0.8,
|
||||
)
|
||||
|
||||
assert spoken_form_result.approved_correction_indexes == []
|
||||
assert spoken_form_result.rejected_count == 1
|
||||
assert spoken_form_result.skipped[0].reason == "grammar validation rejected semantic change"
|
||||
assert spoken_form_result.skipped[0].validation_reason == (
|
||||
"This is a semantic reversal, not a likely spoken-form transcription error."
|
||||
)
|
||||
|
||||
|
||||
def test_validation_rejects_duplicate_unknown_and_missing_decisions():
|
||||
correction = CorrectionCandidate(
|
||||
id=1,
|
||||
original_text="visible",
|
||||
corrected_text="invisible",
|
||||
confidence=0.95,
|
||||
)
|
||||
candidates, _ = select_grammar_validation_candidates(
|
||||
_transcript(),
|
||||
[correction],
|
||||
confidence_threshold=0.8,
|
||||
replacement_mode="require_unique",
|
||||
protected_vocabulary=_vocabulary(),
|
||||
)
|
||||
meaning_decision = GrammarValidationDecision(
|
||||
correction_index=0,
|
||||
is_meaning_preserving=True,
|
||||
confidence=0.95,
|
||||
reason="Preserves meaning.",
|
||||
)
|
||||
spoken_form_decision = GrammarSpokenFormValidationDecision(
|
||||
correction_index=0,
|
||||
is_likely_spoken_form_correction=True,
|
||||
confidence=0.95,
|
||||
reason="Likely spoken-form correction.",
|
||||
)
|
||||
|
||||
with pytest.raises(AuditaLLMError):
|
||||
filter_with_grammar_validations(
|
||||
[correction],
|
||||
filter_with_meaning_preserving_validations(
|
||||
candidates,
|
||||
GrammarValidationSet(validations=[decision, decision]),
|
||||
GrammarValidationSet(validations=[meaning_decision, meaning_decision]),
|
||||
confidence_threshold=0.8,
|
||||
)
|
||||
|
||||
with pytest.raises(AuditaLLMError):
|
||||
filter_with_grammar_validations(
|
||||
[correction],
|
||||
filter_with_meaning_preserving_validations(
|
||||
candidates,
|
||||
GrammarValidationSet(
|
||||
GrammarValidationSet(validations=[]),
|
||||
confidence_threshold=0.8,
|
||||
)
|
||||
with pytest.raises(AuditaLLMError):
|
||||
filter_with_spoken_form_validations(
|
||||
candidates,
|
||||
GrammarSpokenFormValidationSet(
|
||||
validations=[
|
||||
GrammarValidationDecision(
|
||||
GrammarSpokenFormValidationDecision(
|
||||
correction_index=99,
|
||||
is_meaning_preserving=True,
|
||||
is_likely_spoken_form_correction=True,
|
||||
confidence=0.95,
|
||||
reason="Unknown.",
|
||||
)
|
||||
@@ -163,3 +328,9 @@ def test_validation_rejects_duplicate_and_unknown_decisions():
|
||||
),
|
||||
confidence_threshold=0.8,
|
||||
)
|
||||
with pytest.raises(AuditaLLMError):
|
||||
filter_with_spoken_form_validations(
|
||||
candidates,
|
||||
GrammarSpokenFormValidationSet(validations=[spoken_form_decision, spoken_form_decision]),
|
||||
confidence_threshold=0.8,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user