import pytest from audita.errors import AuditaLLMError from audita.protection import ProtectedVocabulary from audita.schemas import ( CorrectionCandidate, GrammarSpokenFormValidationDecision, GrammarSpokenFormValidationSet, GrammarValidationDecision, GrammarValidationSet, parse_glossary_yaml, parse_transcript_json, ) from audita.semantic_validation import ( filter_with_meaning_preserving_validations, filter_with_spoken_form_validations, keep_corrections_with_indexes, select_grammar_validation_candidates, ) def _transcript(): return parse_transcript_json( """ [ {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "he became visible and then gestures arrived"} ] """ ) def _vocabulary(): glossary = parse_glossary_yaml( """ glossary: - name: "Jesters" category: faction summary: "The Jesters are a faction." """ ) return ProtectedVocabulary.from_glossary(glossary) def test_protected_vocabulary_correction_bypasses_validation(): correction = CorrectionCandidate( id=1, original_text="gestures", corrected_text="Jesters", confidence=0.95, ) candidates, bypassed_count = select_grammar_validation_candidates( _transcript(), [correction], confidence_threshold=0.8, replacement_mode="require_unique", protected_vocabulary=_vocabulary(), ) assert candidates == [] assert bypassed_count == 1 def test_capitalization_only_correction_bypasses_validation(): correction = CorrectionCandidate( id=1, original_text="he", corrected_text="He", confidence=0.95, ) candidates, bypassed_count = select_grammar_validation_candidates( _transcript(), [correction], confidence_threshold=0.8, replacement_mode="require_unique", protected_vocabulary=_vocabulary(), ) assert candidates == [] assert bypassed_count == 1 def test_punctuation_only_correction_bypasses_validation(): correction = CorrectionCandidate( id=1, original_text="visible", corrected_text="visible.", confidence=0.95, ) candidates, bypassed_count = select_grammar_validation_candidates( _transcript(), [correction], confidence_threshold=0.8, replacement_mode="require_unique", protected_vocabulary=_vocabulary(), ) assert candidates == [] assert bypassed_count == 1 def test_meaning_sensitive_substitution_requires_validation(): correction = CorrectionCandidate( id=1, original_text="visible", corrected_text="invisible", confidence=0.95, ) candidates, bypassed_count = select_grammar_validation_candidates( _transcript(), [correction], confidence_threshold=0.8, replacement_mode="require_unique", protected_vocabulary=_vocabulary(), ) assert len(candidates) == 1 assert candidates[0].correction_index == 0 assert candidates[0].original_segment_text == "he became visible and then gestures arrived" assert candidates[0].corrected_segment_text == "he became invisible and then gestures arrived" assert bypassed_count == 0 def test_meaning_preserving_validation_approves_without_rescue(): correction = CorrectionCandidate( id=1, original_text="bind", corrected_text="mind", confidence=0.95, ) transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Keep in bind."} ] """ ) candidates, _ = select_grammar_validation_candidates( transcript, [correction], confidence_threshold=0.8, replacement_mode="require_unique", protected_vocabulary=_vocabulary(), ) result = filter_with_meaning_preserving_validations( candidates, GrammarValidationSet( validations=[ GrammarValidationDecision( correction_index=0, is_meaning_preserving=True, confidence=0.95, reason="This preserves the intended meaning.", ) ] ), confidence_threshold=0.8, ) assert result.approved_correction_indexes == [0] assert result.rescue_candidates == [] assert result.approved_count == 1 assert result.rejected_count == 0 def test_spoken_form_validation_can_rescue_homophone_fix(): correction = CorrectionCandidate( id=1, original_text="dam", corrected_text="damn", confidence=0.95, ) transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."} ] """ ) candidates, _ = select_grammar_validation_candidates( transcript, [correction], confidence_threshold=0.8, replacement_mode="require_unique", protected_vocabulary=_vocabulary(), ) meaning_result = filter_with_meaning_preserving_validations( candidates, GrammarValidationSet( validations=[ GrammarValidationDecision( correction_index=0, is_meaning_preserving=False, confidence=0.99, reason="Written meaning changes from a barrier to a curse word.", ) ] ), confidence_threshold=0.8, ) spoken_form_result = filter_with_spoken_form_validations( meaning_result.rescue_candidates, GrammarSpokenFormValidationSet( validations=[ GrammarSpokenFormValidationDecision( correction_index=0, is_likely_spoken_form_correction=True, confidence=0.95, reason="The surrounding phrase strongly supports the intended spoken phrase with a curse word.", ) ] ), confidence_threshold=0.8, ) kept = keep_corrections_with_indexes([correction], spoken_form_result.approved_correction_indexes) assert meaning_result.approved_correction_indexes == [] assert kept == [correction] assert spoken_form_result.skipped == [] def test_spoken_form_validation_rejects_non_homophone_semantic_change(): correction = CorrectionCandidate( id=1, original_text="visible", corrected_text="invisible", confidence=0.95, ) candidates, _ = select_grammar_validation_candidates( _transcript(), [correction], confidence_threshold=0.8, replacement_mode="require_unique", protected_vocabulary=_vocabulary(), ) meaning_result = filter_with_meaning_preserving_validations( candidates, GrammarValidationSet( validations=[ GrammarValidationDecision( correction_index=0, is_meaning_preserving=False, confidence=0.99, reason="This reverses visible to invisible.", ) ] ), confidence_threshold=0.8, ) spoken_form_result = filter_with_spoken_form_validations( meaning_result.rescue_candidates, GrammarSpokenFormValidationSet( validations=[ GrammarSpokenFormValidationDecision( correction_index=0, is_likely_spoken_form_correction=False, confidence=0.99, reason="This is a semantic reversal, not a likely spoken-form transcription error.", ) ] ), confidence_threshold=0.8, ) assert spoken_form_result.approved_correction_indexes == [] assert spoken_form_result.rejected_count == 1 assert spoken_form_result.skipped[0].reason == "grammar validation rejected semantic change" assert spoken_form_result.skipped[0].validation_reason == ( "This is a semantic reversal, not a likely spoken-form transcription error." ) def test_validation_rejects_duplicate_unknown_and_missing_decisions(): correction = CorrectionCandidate( id=1, original_text="visible", corrected_text="invisible", confidence=0.95, ) candidates, _ = select_grammar_validation_candidates( _transcript(), [correction], confidence_threshold=0.8, replacement_mode="require_unique", protected_vocabulary=_vocabulary(), ) meaning_decision = GrammarValidationDecision( correction_index=0, is_meaning_preserving=True, confidence=0.95, reason="Preserves meaning.", ) spoken_form_decision = GrammarSpokenFormValidationDecision( correction_index=0, is_likely_spoken_form_correction=True, confidence=0.95, reason="Likely spoken-form correction.", ) with pytest.raises(AuditaLLMError): filter_with_meaning_preserving_validations( candidates, GrammarValidationSet(validations=[meaning_decision, meaning_decision]), confidence_threshold=0.8, ) with pytest.raises(AuditaLLMError): filter_with_meaning_preserving_validations( candidates, GrammarValidationSet(validations=[]), confidence_threshold=0.8, ) with pytest.raises(AuditaLLMError): filter_with_spoken_form_validations( candidates, GrammarSpokenFormValidationSet( validations=[ GrammarSpokenFormValidationDecision( correction_index=99, is_likely_spoken_form_correction=True, confidence=0.95, reason="Unknown.", ) ] ), confidence_threshold=0.8, ) with pytest.raises(AuditaLLMError): filter_with_spoken_form_validations( candidates, GrammarSpokenFormValidationSet(validations=[spoken_form_decision, spoken_form_decision]), confidence_threshold=0.8, )