diff --git a/src/audita/modules/glossary.py b/src/audita/modules/glossary.py index 815cfec..b45330d 100644 --- a/src/audita/modules/glossary.py +++ b/src/audita/modules/glossary.py @@ -6,6 +6,7 @@ from audita.framework.proposal_generation import generate_llm_correction_proposa from audita.modules.prompts import build_glossary_proposal_messages from audita.validators import ( MeaningReversalValidator, + NonEmptySegmentValidator, ProposalConfidenceValidator, ProtectedGlossaryTermsValidator, SpokenFormPlausibilityValidator, @@ -21,6 +22,7 @@ class GlossaryModule: return [ ProposalConfidenceValidator("proposal_confidence_guard", "glossary_confidence_threshold"), ProtectedGlossaryTermsValidator("protected_glossary_guard"), + NonEmptySegmentValidator("non_empty_segment_guard"), SpokenFormPlausibilityValidator("spoken_form_plausibility_review"), MeaningReversalValidator("meaning_reversal_review"), ] diff --git a/src/audita/modules/grammar.py b/src/audita/modules/grammar.py index e3f1be8..7e60eb7 100644 --- a/src/audita/modules/grammar.py +++ b/src/audita/modules/grammar.py @@ -7,6 +7,7 @@ from audita.modules.prompts import build_grammar_proposal_messages from audita.validators import ( GrammarOnlyValidator, MeaningReversalValidator, + NonEmptySegmentValidator, ProposalConfidenceValidator, ProtectedGlossaryTermsValidator, Validator, @@ -21,6 +22,7 @@ class GrammarModule: return [ ProposalConfidenceValidator("proposal_confidence_guard", "grammar_confidence_threshold"), ProtectedGlossaryTermsValidator("protected_glossary_guard"), + NonEmptySegmentValidator("non_empty_segment_guard"), GrammarOnlyValidator("grammar_only_guard"), MeaningReversalValidator("meaning_reversal_review"), ] diff --git a/src/audita/modules/homophones.py b/src/audita/modules/homophones.py index 5174f63..94fad0a 100644 --- a/src/audita/modules/homophones.py +++ b/src/audita/modules/homophones.py @@ -6,6 +6,7 @@ from audita.framework.proposal_generation import generate_llm_correction_proposa from audita.modules.prompts import build_homophones_proposal_messages from audita.validators import ( MeaningReversalValidator, + NonEmptySegmentValidator, ProposalConfidenceValidator, ProtectedGlossaryTermsValidator, SpokenFormPlausibilityValidator, @@ -21,6 +22,7 @@ class HomophonesModule: return [ ProposalConfidenceValidator("proposal_confidence_guard", "homophones_confidence_threshold"), ProtectedGlossaryTermsValidator("protected_glossary_guard"), + NonEmptySegmentValidator("non_empty_segment_guard"), SpokenFormPlausibilityValidator("spoken_form_plausibility_review"), MeaningReversalValidator("meaning_reversal_review"), ] diff --git a/src/audita/modules/spoken_word.py b/src/audita/modules/spoken_word.py index 94db328..fbd07c7 100644 --- a/src/audita/modules/spoken_word.py +++ b/src/audita/modules/spoken_word.py @@ -6,6 +6,7 @@ from audita.framework.proposal_generation import generate_llm_correction_proposa from audita.modules.prompts import build_spoken_word_proposal_messages from audita.validators import ( MeaningReversalValidator, + NonEmptySegmentValidator, ProposalConfidenceValidator, ProtectedGlossaryTermsValidator, SpokenWordValidator, @@ -21,6 +22,7 @@ class SpokenWordModule: return [ ProposalConfidenceValidator("proposal_confidence_guard", "spoken_word_confidence_threshold"), ProtectedGlossaryTermsValidator("protected_glossary_guard"), + NonEmptySegmentValidator("non_empty_segment_guard"), SpokenWordValidator("spoken_word_review"), MeaningReversalValidator("meaning_reversal_review"), ] diff --git a/src/audita/validators/__init__.py b/src/audita/validators/__init__.py index e62a3c9..66291ff 100644 --- a/src/audita/validators/__init__.py +++ b/src/audita/validators/__init__.py @@ -1,5 +1,10 @@ from .base import ValidationContext, ValidationDecision, ValidationResult, Validator -from .deterministic import GrammarOnlyValidator, ProposalConfidenceValidator, ProtectedGlossaryTermsValidator +from .deterministic import ( + GrammarOnlyValidator, + NonEmptySegmentValidator, + ProposalConfidenceValidator, + ProtectedGlossaryTermsValidator, +) from .llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator from .protection import ProtectedVocabulary @@ -10,6 +15,7 @@ __all__ = [ "Validator", "ProposalConfidenceValidator", "ProtectedGlossaryTermsValidator", + "NonEmptySegmentValidator", "GrammarOnlyValidator", "ProtectedVocabulary", "SpokenFormPlausibilityValidator", diff --git a/src/audita/validators/deterministic.py b/src/audita/validators/deterministic.py index 69ee5a6..ef2a34e 100644 --- a/src/audita/validators/deterministic.py +++ b/src/audita/validators/deterministic.py @@ -1,6 +1,8 @@ import string from dataclasses import dataclass +from audita.framework.proposals import ProposalPreviewError, preview_proposal + from .base import ValidationContext, ValidationDecision, ValidationResult from .protection import ProtectedVocabulary @@ -51,6 +53,34 @@ class ProtectedGlossaryTermsValidator: ) +@dataclass(frozen=True) +class NonEmptySegmentValidator: + name: str + execution_kind: str = "deterministic" + + def validate(self, context: ValidationContext) -> ValidationResult: + replacement_policy = context.run_spec.module.replacement_policy + decisions: list[ValidationDecision] = [] + for proposal in context.proposals: + preview = preview_proposal(context.transcript, proposal, replacement_policy) + if isinstance(preview, ProposalPreviewError): + decisions.append(ValidationDecision(proposal_index=proposal.proposal_index, approved=True)) + continue + is_non_empty = preview.corrected_segment_text.strip() != "" + decisions.append( + ValidationDecision( + proposal_index=proposal.proposal_index, + approved=is_non_empty, + reason=None if is_non_empty else "correction would leave the segment empty", + ) + ) + return ValidationResult( + validator_name=self.name, + execution_kind=self.execution_kind, + decisions=decisions, + ) + + _GRAMMAR_PUNCTUATION = set(string.punctuation) | {"—", "–", "…", "“", "”", "‘", "’"} diff --git a/tests/test_llm_validators.py b/tests/test_llm_validators.py index e7d4cae..c53b12b 100644 --- a/tests/test_llm_validators.py +++ b/tests/test_llm_validators.py @@ -5,7 +5,7 @@ from audita.core.config import AuditaConfig from audita.core.errors import AuditaLLMError from audita.core.schemas import parse_glossary_yaml, parse_transcript_json from audita.framework.models import CorrectionProposal, ModuleRunSpec -from audita.validators import GrammarOnlyValidator +from audita.validators import GrammarOnlyValidator, NonEmptySegmentValidator from audita.validators.base import ValidationContext from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator from audita.validators.prompts import ( @@ -472,6 +472,84 @@ def test_grammar_only_validator_rejects_word_level_changes(tmp_path): ) +def test_non_empty_segment_validator_rejects_empty_and_whitespace_only_segments(tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "uh"}, + {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "um"} + ] + """ + ) + proposals = [ + CorrectionProposal( + proposal_index=0, + module_instance="spoken_word", + module_key="spoken_word", + id=1, + original_text="uh", + corrected_text="", + confidence=0.95, + ), + CorrectionProposal( + proposal_index=1, + module_instance="spoken_word", + module_key="spoken_word", + id=2, + original_text="um", + corrected_text=" ", + confidence=0.95, + ), + ] + + result = NonEmptySegmentValidator("non_empty_segment_guard").validate( + _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path) + ) + + assert [decision.approved for decision in result.decisions] == [False, False] + assert all(decision.reason == "correction would leave the segment empty" for decision in result.decisions) + + +def test_non_empty_segment_validator_allows_punctuation_only_and_unpreviewable_proposals(tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "uh"}, + {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "There were gestures at the temple."} + ] + """ + ) + proposals = [ + CorrectionProposal( + proposal_index=0, + module_instance="spoken_word", + module_key="spoken_word", + id=1, + original_text="uh", + corrected_text=".", + confidence=0.95, + ), + CorrectionProposal( + proposal_index=1, + module_instance="spoken_word", + module_key="spoken_word", + id=2, + original_text="rank", + corrected_text="Hrank", + confidence=0.95, + ), + ] + + result = NonEmptySegmentValidator("non_empty_segment_guard").validate( + _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path) + ) + + assert [(decision.proposal_index, decision.approved, decision.reason) for decision in result.decisions] == [ + (0, True, None), + (1, True, None), + ] + + @pytest.mark.parametrize( ("validator", "payload", "message_fragment"), [ diff --git a/tests/test_module_proposals.py b/tests/test_module_proposals.py index 2906ee9..eeac17d 100644 --- a/tests/test_module_proposals.py +++ b/tests/test_module_proposals.py @@ -500,6 +500,7 @@ def test_process_transcript_result_runs_spoken_word_module_with_full_validator_c assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ "proposal_confidence_guard", "protected_glossary_guard", + "non_empty_segment_guard", "spoken_word_review", "meaning_reversal_review", ] @@ -627,6 +628,7 @@ def test_process_transcript_result_runs_grammar_module_with_full_validator_chain assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ "proposal_confidence_guard", "protected_glossary_guard", + "non_empty_segment_guard", "grammar_only_guard", "meaning_reversal_review", ] @@ -806,3 +808,67 @@ def test_process_transcript_result_grammar_module_still_rejects_homophone_style_ assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"] assert result.report.skipped_corrections[0].source == "validator:grammar_only_guard" assert result.report.skipped_corrections[0].reason == "correction is not limited to punctuation, capitalization, and spacing" + + +def test_process_transcript_result_rejects_spoken_word_whole_segment_deletion_before_llm_validators(tmp_path): + transcript = parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "uh"} + ] + """ + ) + base_config = AuditaConfig.from_sources(env={}) + config = AuditaConfig( + api_key=base_config.api_key, + model=base_config.model, + base_url=base_config.base_url, + max_retries=base_config.max_retries, + max_section_tokens=base_config.max_section_tokens, + glossary_confidence_threshold=base_config.glossary_confidence_threshold, + grammar_confidence_threshold=base_config.grammar_confidence_threshold, + homophones_confidence_threshold=base_config.homophones_confidence_threshold, + spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, + normalize_max_segment_gap=base_config.normalize_max_segment_gap, + normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, + normalize_max_segment_duration=base_config.normalize_max_segment_duration, + normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, + work_dir=tmp_path / "work", + work_dir_retention="always", + ) + client = FakeStructuredLLMClient( + [ + { + "corrections": [ + { + "id": 1, + "original_text": "uh", + "corrected_text": "", + "confidence": 0.95, + } + ] + } + ] + ) + + result = process_transcript_result( + transcript, + _glossary(), + config, + module_keys=["spoken_word"], + llm_client=client, + ) + + assert result.transcript[0].text == "uh" + assert [call["stage_name"] for call in client.calls] == ["spoken_word:proposal"] + assert result.report.skipped_corrections[0].source == "validator:non_empty_segment_guard" + assert result.report.skipped_corrections[0].reason == "correction would leave the segment empty" + assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ + "proposal_confidence_guard", + "protected_glossary_guard", + "non_empty_segment_guard", + "spoken_word_review", + "meaning_reversal_review", + ] + assert result.report.modules[0].validators[2].rejected_count == 1 + assert result.report.modules[0].validators[3].candidate_count == 0 diff --git a/tests/test_new_pipeline.py b/tests/test_new_pipeline.py index 57c3bdb..8116c22 100644 --- a/tests/test_new_pipeline.py +++ b/tests/test_new_pipeline.py @@ -131,18 +131,21 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy( assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ "proposal_confidence_guard", "protected_glossary_guard", + "non_empty_segment_guard", "spoken_form_plausibility_review", "meaning_reversal_review", ] assert [validator["name"] for validator in result.report.modules[3].to_dict()["validators"]] == [ "proposal_confidence_guard", "protected_glossary_guard", + "non_empty_segment_guard", "spoken_word_review", "meaning_reversal_review", ] assert [validator["name"] for validator in result.report.modules[4].to_dict()["validators"]] == [ "proposal_confidence_guard", "protected_glossary_guard", + "non_empty_segment_guard", "grammar_only_guard", "meaning_reversal_review", ] @@ -182,30 +185,35 @@ def test_default_module_specs_expose_final_validator_order(): assert [validator.name for validator in specs[0].module.validators()] == [ "proposal_confidence_guard", "protected_glossary_guard", + "non_empty_segment_guard", "spoken_form_plausibility_review", "meaning_reversal_review", ] assert [validator.name for validator in specs[1].module.validators()] == [ "proposal_confidence_guard", "protected_glossary_guard", + "non_empty_segment_guard", "spoken_form_plausibility_review", "meaning_reversal_review", ] assert [validator.name for validator in specs[2].module.validators()] == [ "proposal_confidence_guard", "protected_glossary_guard", + "non_empty_segment_guard", "spoken_form_plausibility_review", "meaning_reversal_review", ] assert [validator.name for validator in specs[3].module.validators()] == [ "proposal_confidence_guard", "protected_glossary_guard", + "non_empty_segment_guard", "spoken_word_review", "meaning_reversal_review", ] assert [validator.name for validator in specs[4].module.validators()] == [ "proposal_confidence_guard", "protected_glossary_guard", + "non_empty_segment_guard", "grammar_only_guard", "meaning_reversal_review", ]