Added new deterministic validators to catch early failures
This commit is contained in:
@@ -5,7 +5,12 @@ from audita.core.config import AuditaConfig
|
||||
from audita.core.errors import AuditaLLMError
|
||||
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
|
||||
from audita.framework.models import CorrectionProposal, ModuleRunSpec
|
||||
from audita.validators import GrammarOnlyValidator, NonEmptySegmentValidator
|
||||
from audita.validators import (
|
||||
GrammarOnlyValidator,
|
||||
IdenticalTextValidator,
|
||||
NonEmptySegmentValidator,
|
||||
OriginalTextPresentValidator,
|
||||
)
|
||||
from audita.validators.base import ValidationContext
|
||||
from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
|
||||
from audita.validators.prompts import (
|
||||
@@ -550,6 +555,96 @@ def test_non_empty_segment_validator_allows_punctuation_only_and_unpreviewable_p
|
||||
]
|
||||
|
||||
|
||||
def test_identical_text_validator_rejects_exact_noops_only(tmp_path):
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
[
|
||||
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello"},
|
||||
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "hello"}
|
||||
]
|
||||
"""
|
||||
)
|
||||
proposals = [
|
||||
CorrectionProposal(
|
||||
proposal_index=0,
|
||||
module_instance="grammar",
|
||||
module_key="grammar",
|
||||
id=1,
|
||||
original_text="hello",
|
||||
corrected_text="hello",
|
||||
confidence=0.95,
|
||||
),
|
||||
CorrectionProposal(
|
||||
proposal_index=1,
|
||||
module_instance="grammar",
|
||||
module_key="grammar",
|
||||
id=2,
|
||||
original_text="hello",
|
||||
corrected_text="Hello",
|
||||
confidence=0.95,
|
||||
),
|
||||
]
|
||||
|
||||
result = IdenticalTextValidator("identical_text_guard").validate(
|
||||
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
|
||||
)
|
||||
|
||||
assert [(decision.proposal_index, decision.approved, decision.reason) for decision in result.decisions] == [
|
||||
(0, False, "proposal original_text and corrected_text are identical"),
|
||||
(1, True, None),
|
||||
]
|
||||
|
||||
|
||||
def test_original_text_present_validator_rejects_missing_spans_and_allows_present_or_unknown_segments(tmp_path):
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
[
|
||||
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello hello"},
|
||||
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "goodbye"}
|
||||
]
|
||||
"""
|
||||
)
|
||||
proposals = [
|
||||
CorrectionProposal(
|
||||
proposal_index=0,
|
||||
module_instance="homophones",
|
||||
module_key="homophones",
|
||||
id=1,
|
||||
original_text="hello",
|
||||
corrected_text="hi",
|
||||
confidence=0.95,
|
||||
),
|
||||
CorrectionProposal(
|
||||
proposal_index=1,
|
||||
module_instance="homophones",
|
||||
module_key="homophones",
|
||||
id=2,
|
||||
original_text="rank",
|
||||
corrected_text="Hrank",
|
||||
confidence=0.95,
|
||||
),
|
||||
CorrectionProposal(
|
||||
proposal_index=2,
|
||||
module_instance="homophones",
|
||||
module_key="homophones",
|
||||
id=99,
|
||||
original_text="missing",
|
||||
corrected_text="present",
|
||||
confidence=0.95,
|
||||
),
|
||||
]
|
||||
|
||||
result = OriginalTextPresentValidator("original_text_present_guard").validate(
|
||||
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
|
||||
)
|
||||
|
||||
assert [(decision.proposal_index, decision.approved, decision.reason) for decision in result.decisions] == [
|
||||
(0, True, None),
|
||||
(1, False, "proposal original_text does not match segment text"),
|
||||
(2, True, None),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("validator", "payload", "message_fragment"),
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user