Added new deterministic validators to catch early failures

This commit is contained in:
2026-04-25 14:14:43 -05:00
parent 619c615d21
commit a255ff28b7
9 changed files with 286 additions and 8 deletions

View File

@@ -5,7 +5,12 @@ from audita.core.config import AuditaConfig
from audita.core.errors import AuditaLLMError
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
from audita.framework.models import CorrectionProposal, ModuleRunSpec
from audita.validators import GrammarOnlyValidator, NonEmptySegmentValidator
from audita.validators import (
GrammarOnlyValidator,
IdenticalTextValidator,
NonEmptySegmentValidator,
OriginalTextPresentValidator,
)
from audita.validators.base import ValidationContext
from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
from audita.validators.prompts import (
@@ -550,6 +555,96 @@ def test_non_empty_segment_validator_allows_punctuation_only_and_unpreviewable_p
]
def test_identical_text_validator_rejects_exact_noops_only(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "hello"}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="grammar",
module_key="grammar",
id=1,
original_text="hello",
corrected_text="hello",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="grammar",
module_key="grammar",
id=2,
original_text="hello",
corrected_text="Hello",
confidence=0.95,
),
]
result = IdenticalTextValidator("identical_text_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved, decision.reason) for decision in result.decisions] == [
(0, False, "proposal original_text and corrected_text are identical"),
(1, True, None),
]
def test_original_text_present_validator_rejects_missing_spans_and_allows_present_or_unknown_segments(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello hello"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "goodbye"}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="hello",
corrected_text="hi",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="homophones",
module_key="homophones",
id=2,
original_text="rank",
corrected_text="Hrank",
confidence=0.95,
),
CorrectionProposal(
proposal_index=2,
module_instance="homophones",
module_key="homophones",
id=99,
original_text="missing",
corrected_text="present",
confidence=0.95,
),
]
result = OriginalTextPresentValidator("original_text_present_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved, decision.reason) for decision in result.decisions] == [
(0, True, None),
(1, False, "proposal original_text does not match segment text"),
(2, True, None),
]
@pytest.mark.parametrize(
("validator", "payload", "message_fragment"),
[