eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Added a validator to prevent changes that result in an empty or whitespace-only string

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-25 13:42:18 -05:00
parent 5944f13404
commit bf05d79fc6
9 changed files with 198 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
tests/test_llm_validators.py
View File

@@ -5,7 +5,7 @@ from audita.core.config import AuditaConfig
from audita.core.errors import AuditaLLMError
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
from audita.framework.models import CorrectionProposal, ModuleRunSpec
from audita.validators import GrammarOnlyValidator
from audita.validators import GrammarOnlyValidator, NonEmptySegmentValidator
from audita.validators.base import ValidationContext
from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
from audita.validators.prompts import (
@@ -472,6 +472,84 @@ def test_grammar_only_validator_rejects_word_level_changes(tmp_path):
)
def test_non_empty_segment_validator_rejects_empty_and_whitespace_only_segments(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "uh"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "um"}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="spoken_word",
module_key="spoken_word",
id=1,
original_text="uh",
corrected_text="",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="spoken_word",
module_key="spoken_word",
id=2,
original_text="um",
corrected_text=" ",
confidence=0.95,
),
]
result = NonEmptySegmentValidator("non_empty_segment_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
)
assert [decision.approved for decision in result.decisions] == [False, False]
assert all(decision.reason == "correction would leave the segment empty" for decision in result.decisions)
def test_non_empty_segment_validator_allows_punctuation_only_and_unpreviewable_proposals(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "uh"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "There were gestures at the temple."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="spoken_word",
module_key="spoken_word",
id=1,
original_text="uh",
corrected_text=".",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="spoken_word",
module_key="spoken_word",
id=2,
original_text="rank",
corrected_text="Hrank",
confidence=0.95,
),
]
result = NonEmptySegmentValidator("non_empty_segment_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved, decision.reason) for decision in result.decisions] == [
(0, True, None),
(1, True, None),
]
@pytest.mark.parametrize(
("validator", "payload", "message_fragment"),
[

66
tests/test_module_proposals.py
View File

@@ -500,6 +500,7 @@ def test_process_transcript_result_runs_spoken_word_module_with_full_validator_c
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
"spoken_word_review",
"meaning_reversal_review",
]
@@ -627,6 +628,7 @@ def test_process_transcript_result_runs_grammar_module_with_full_validator_chain
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
"grammar_only_guard",
"meaning_reversal_review",
]
@@ -806,3 +808,67 @@ def test_process_transcript_result_grammar_module_still_rejects_homophone_style_
assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"]
assert result.report.skipped_corrections[0].source == "validator:grammar_only_guard"
assert result.report.skipped_corrections[0].reason == "correction is not limited to punctuation, capitalization, and spacing"
def test_process_transcript_result_rejects_spoken_word_whole_segment_deletion_before_llm_validators(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "uh"}
]
"""
)
base_config = AuditaConfig.from_sources(env={})
config = AuditaConfig(
api_key=base_config.api_key,
model=base_config.model,
base_url=base_config.base_url,
max_retries=base_config.max_retries,
max_section_tokens=base_config.max_section_tokens,
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
grammar_confidence_threshold=base_config.grammar_confidence_threshold,
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold,
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
normalize_max_segment_duration=base_config.normalize_max_segment_duration,
normalize_max_segment_tokens=base_config.normalize_max_segment_tokens,
work_dir=tmp_path / "work",
work_dir_retention="always",
)
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "uh",
"corrected_text": "",
"confidence": 0.95,
}
]
}
]
)
result = process_transcript_result(
transcript,
_glossary(),
config,
module_keys=["spoken_word"],
llm_client=client,
)
assert result.transcript[0].text == "uh"
assert [call["stage_name"] for call in client.calls] == ["spoken_word:proposal"]
assert result.report.skipped_corrections[0].source == "validator:non_empty_segment_guard"
assert result.report.skipped_corrections[0].reason == "correction would leave the segment empty"
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
"spoken_word_review",
"meaning_reversal_review",
]
assert result.report.modules[0].validators[2].rejected_count == 1
assert result.report.modules[0].validators[3].candidate_count == 0

8
tests/test_new_pipeline.py
View File

@@ -131,18 +131,21 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
"spoken_form_plausibility_review",
"meaning_reversal_review",
]
assert [validator["name"] for validator in result.report.modules[3].to_dict()["validators"]] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
"spoken_word_review",
"meaning_reversal_review",
]
assert [validator["name"] for validator in result.report.modules[4].to_dict()["validators"]] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
"grammar_only_guard",
"meaning_reversal_review",
]
@@ -182,30 +185,35 @@ def test_default_module_specs_expose_final_validator_order():
assert [validator.name for validator in specs[0].module.validators()] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
"spoken_form_plausibility_review",
"meaning_reversal_review",
]
assert [validator.name for validator in specs[1].module.validators()] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
"spoken_form_plausibility_review",
"meaning_reversal_review",
]
assert [validator.name for validator in specs[2].module.validators()] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
"spoken_form_plausibility_review",
"meaning_reversal_review",
]
assert [validator.name for validator in specs[3].module.validators()] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
"spoken_word_review",
"meaning_reversal_review",
]
assert [validator.name for validator in specs[4].module.validators()] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
"grammar_only_guard",
"meaning_reversal_review",
]