audita/tests/test_semantic_validation.py

import pytest

from audita.errors import AuditaLLMError
from audita.semantic_validation import select_grammar_validation_candidates
from audita.semantic_validation import filter_with_grammar_validations
from audita.protection import ProtectedVocabulary
from audita.schemas import (
    CorrectionCandidate,
    GrammarValidationDecision,
    GrammarValidationSet,
    parse_glossary_yaml,
    parse_transcript_json,
)


def _transcript():
    return parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "he became visible and then gestures arrived"}
        ]
        """
    )


def _vocabulary():
    glossary = parse_glossary_yaml(
        """
        glossary:
          - name: "Jesters"
            category: faction
            summary: "The Jesters are a faction."
        """
    )
    return ProtectedVocabulary.from_glossary(glossary)


def test_protected_vocabulary_correction_bypasses_validation():
    correction = CorrectionCandidate(
        id=1,
        original_text="gestures",
        corrected_text="Jesters",
        confidence=0.95,
    )

    candidates, bypassed_count = select_grammar_validation_candidates(
        _transcript(),
        [correction],
        confidence_threshold=0.8,
        replacement_mode="require_unique",
        protected_vocabulary=_vocabulary(),
    )

    assert candidates == []
    assert bypassed_count == 1


def test_capitalization_only_correction_bypasses_validation():
    correction = CorrectionCandidate(
        id=1,
        original_text="he",
        corrected_text="He",
        confidence=0.95,
    )

    candidates, bypassed_count = select_grammar_validation_candidates(
        _transcript(),
        [correction],
        confidence_threshold=0.8,
        replacement_mode="require_unique",
        protected_vocabulary=_vocabulary(),
    )

    assert candidates == []
    assert bypassed_count == 1


def test_punctuation_only_correction_bypasses_validation():
    correction = CorrectionCandidate(
        id=1,
        original_text="visible",
        corrected_text="visible.",
        confidence=0.95,
    )

    candidates, bypassed_count = select_grammar_validation_candidates(
        _transcript(),
        [correction],
        confidence_threshold=0.8,
        replacement_mode="require_unique",
        protected_vocabulary=_vocabulary(),
    )

    assert candidates == []
    assert bypassed_count == 1


def test_meaning_sensitive_substitution_requires_validation():
    correction = CorrectionCandidate(
        id=1,
        original_text="visible",
        corrected_text="invisible",
        confidence=0.95,
    )

    candidates, bypassed_count = select_grammar_validation_candidates(
        _transcript(),
        [correction],
        confidence_threshold=0.8,
        replacement_mode="require_unique",
        protected_vocabulary=_vocabulary(),
    )

    assert len(candidates) == 1
    assert candidates[0].correction_index == 0
    assert candidates[0].original_segment_text == "he became visible and then gestures arrived"
    assert candidates[0].corrected_segment_text == "he became invisible and then gestures arrived"
    assert bypassed_count == 0


def test_validation_rejects_duplicate_and_unknown_decisions():
    correction = CorrectionCandidate(
        id=1,
        original_text="visible",
        corrected_text="invisible",
        confidence=0.95,
    )
    candidates, _ = select_grammar_validation_candidates(
        _transcript(),
        [correction],
        confidence_threshold=0.8,
        replacement_mode="require_unique",
        protected_vocabulary=_vocabulary(),
    )
    decision = GrammarValidationDecision(
        correction_index=0,
        is_meaning_preserving=True,
        confidence=0.95,
        reason="Preserves meaning.",
    )

    with pytest.raises(AuditaLLMError):
        filter_with_grammar_validations(
            [correction],
            candidates,
            GrammarValidationSet(validations=[decision, decision]),
            confidence_threshold=0.8,
        )

    with pytest.raises(AuditaLLMError):
        filter_with_grammar_validations(
            [correction],
            candidates,
            GrammarValidationSet(
                validations=[
                    GrammarValidationDecision(
                        correction_index=99,
                        is_meaning_preserving=True,
                        confidence=0.95,
                        reason="Unknown.",
                    )
                ]
            ),
            confidence_threshold=0.8,
        )