diff --git a/src/audita/modules/glossary.py b/src/audita/modules/glossary.py index c2a106d..bcb91a9 100644 --- a/src/audita/modules/glossary.py +++ b/src/audita/modules/glossary.py @@ -6,8 +6,10 @@ from audita.framework.proposal_generation import generate_llm_correction_proposa from audita.modules.prompts import build_glossary_proposal_messages from audita.validators import ( GlossaryStageProtectedGlossaryTermsValidator, + IdenticalTextValidator, MeaningReversalValidator, NonEmptySegmentValidator, + OriginalTextPresentValidator, ProposalConfidenceValidator, SpokenFormPlausibilityValidator, Validator, @@ -20,6 +22,8 @@ class GlossaryModule: def validators(self) -> Sequence[Validator]: return [ + IdenticalTextValidator("identical_text_guard"), + OriginalTextPresentValidator("original_text_present_guard"), ProposalConfidenceValidator("proposal_confidence_guard", "glossary_confidence_threshold"), GlossaryStageProtectedGlossaryTermsValidator("glossary_stage_protected_glossary_guard"), NonEmptySegmentValidator("non_empty_segment_guard"), diff --git a/src/audita/modules/grammar.py b/src/audita/modules/grammar.py index 7e60eb7..b9e7c26 100644 --- a/src/audita/modules/grammar.py +++ b/src/audita/modules/grammar.py @@ -6,8 +6,10 @@ from audita.framework.proposal_generation import generate_llm_correction_proposa from audita.modules.prompts import build_grammar_proposal_messages from audita.validators import ( GrammarOnlyValidator, + IdenticalTextValidator, MeaningReversalValidator, NonEmptySegmentValidator, + OriginalTextPresentValidator, ProposalConfidenceValidator, ProtectedGlossaryTermsValidator, Validator, @@ -20,6 +22,8 @@ class GrammarModule: def validators(self) -> Sequence[Validator]: return [ + IdenticalTextValidator("identical_text_guard"), + OriginalTextPresentValidator("original_text_present_guard"), ProposalConfidenceValidator("proposal_confidence_guard", "grammar_confidence_threshold"), ProtectedGlossaryTermsValidator("protected_glossary_guard"), NonEmptySegmentValidator("non_empty_segment_guard"), diff --git a/src/audita/modules/homophones.py b/src/audita/modules/homophones.py index 94fad0a..e68fd46 100644 --- a/src/audita/modules/homophones.py +++ b/src/audita/modules/homophones.py @@ -5,8 +5,10 @@ from audita.framework.models import CorrectionProposal, ModuleContext from audita.framework.proposal_generation import generate_llm_correction_proposals from audita.modules.prompts import build_homophones_proposal_messages from audita.validators import ( + IdenticalTextValidator, MeaningReversalValidator, NonEmptySegmentValidator, + OriginalTextPresentValidator, ProposalConfidenceValidator, ProtectedGlossaryTermsValidator, SpokenFormPlausibilityValidator, @@ -20,6 +22,8 @@ class HomophonesModule: def validators(self) -> Sequence[Validator]: return [ + IdenticalTextValidator("identical_text_guard"), + OriginalTextPresentValidator("original_text_present_guard"), ProposalConfidenceValidator("proposal_confidence_guard", "homophones_confidence_threshold"), ProtectedGlossaryTermsValidator("protected_glossary_guard"), NonEmptySegmentValidator("non_empty_segment_guard"), diff --git a/src/audita/modules/spoken_word.py b/src/audita/modules/spoken_word.py index fbd07c7..a44db56 100644 --- a/src/audita/modules/spoken_word.py +++ b/src/audita/modules/spoken_word.py @@ -5,8 +5,10 @@ from audita.framework.models import CorrectionProposal, ModuleContext from audita.framework.proposal_generation import generate_llm_correction_proposals from audita.modules.prompts import build_spoken_word_proposal_messages from audita.validators import ( + IdenticalTextValidator, MeaningReversalValidator, NonEmptySegmentValidator, + OriginalTextPresentValidator, ProposalConfidenceValidator, ProtectedGlossaryTermsValidator, SpokenWordValidator, @@ -20,6 +22,8 @@ class SpokenWordModule: def validators(self) -> Sequence[Validator]: return [ + IdenticalTextValidator("identical_text_guard"), + OriginalTextPresentValidator("original_text_present_guard"), ProposalConfidenceValidator("proposal_confidence_guard", "spoken_word_confidence_threshold"), ProtectedGlossaryTermsValidator("protected_glossary_guard"), NonEmptySegmentValidator("non_empty_segment_guard"), diff --git a/src/audita/validators/__init__.py b/src/audita/validators/__init__.py index 796f7fb..f094e53 100644 --- a/src/audita/validators/__init__.py +++ b/src/audita/validators/__init__.py @@ -2,7 +2,9 @@ from .base import ValidationContext, ValidationDecision, ValidationResult, Valid from .deterministic import ( GlossaryStageProtectedGlossaryTermsValidator, GrammarOnlyValidator, + IdenticalTextValidator, NonEmptySegmentValidator, + OriginalTextPresentValidator, ProposalConfidenceValidator, ProtectedGlossaryTermsValidator, ) @@ -14,6 +16,8 @@ __all__ = [ "ValidationDecision", "ValidationResult", "Validator", + "IdenticalTextValidator", + "OriginalTextPresentValidator", "ProposalConfidenceValidator", "ProtectedGlossaryTermsValidator", "GlossaryStageProtectedGlossaryTermsValidator", diff --git a/src/audita/validators/deterministic.py b/src/audita/validators/deterministic.py index 8634f09..227b7f0 100644 --- a/src/audita/validators/deterministic.py +++ b/src/audita/validators/deterministic.py @@ -7,6 +7,58 @@ from .base import ValidationContext, ValidationDecision, ValidationResult from .protection import ProtectedVocabulary +@dataclass(frozen=True) +class IdenticalTextValidator: + name: str + execution_kind: str = "deterministic" + + def validate(self, context: ValidationContext) -> ValidationResult: + return ValidationResult( + validator_name=self.name, + execution_kind=self.execution_kind, + decisions=[ + ValidationDecision( + proposal_index=proposal.proposal_index, + approved=proposal.original_text != proposal.corrected_text, + reason=( + None + if proposal.original_text != proposal.corrected_text + else "proposal original_text and corrected_text are identical" + ), + ) + for proposal in context.proposals + ], + ) + + +@dataclass(frozen=True) +class OriginalTextPresentValidator: + name: str + execution_kind: str = "deterministic" + + def validate(self, context: ValidationContext) -> ValidationResult: + segment_text_by_id = {segment.id: segment.text for segment in context.transcript} + return ValidationResult( + validator_name=self.name, + execution_kind=self.execution_kind, + decisions=[ + ValidationDecision( + proposal_index=proposal.proposal_index, + approved=( + (segment_text := segment_text_by_id.get(proposal.id)) is None + or proposal.original_text in segment_text + ), + reason=( + None + if (segment_text := segment_text_by_id.get(proposal.id)) is None or proposal.original_text in segment_text + else "proposal original_text does not match segment text" + ), + ) + for proposal in context.proposals + ], + ) + + @dataclass(frozen=True) class ProposalConfidenceValidator: name: str diff --git a/tests/test_llm_validators.py b/tests/test_llm_validators.py index c53b12b..3982785 100644 --- a/tests/test_llm_validators.py +++ b/tests/test_llm_validators.py @@ -5,7 +5,12 @@ from audita.core.config import AuditaConfig from audita.core.errors import AuditaLLMError from audita.core.schemas import parse_glossary_yaml, parse_transcript_json from audita.framework.models import CorrectionProposal, ModuleRunSpec -from audita.validators import GrammarOnlyValidator, NonEmptySegmentValidator +from audita.validators import ( + GrammarOnlyValidator, + IdenticalTextValidator, + NonEmptySegmentValidator, + OriginalTextPresentValidator, +) from audita.validators.base import ValidationContext from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator from audita.validators.prompts import ( @@ -550,6 +555,96 @@ def test_non_empty_segment_validator_allows_punctuation_only_and_unpreviewable_p ] +def test_identical_text_validator_rejects_exact_noops_only(tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello"}, + {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "hello"} + ] + """ + ) + proposals = [ + CorrectionProposal( + proposal_index=0, + module_instance="grammar", + module_key="grammar", + id=1, + original_text="hello", + corrected_text="hello", + confidence=0.95, + ), + CorrectionProposal( + proposal_index=1, + module_instance="grammar", + module_key="grammar", + id=2, + original_text="hello", + corrected_text="Hello", + confidence=0.95, + ), + ] + + result = IdenticalTextValidator("identical_text_guard").validate( + _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path) + ) + + assert [(decision.proposal_index, decision.approved, decision.reason) for decision in result.decisions] == [ + (0, False, "proposal original_text and corrected_text are identical"), + (1, True, None), + ] + + +def test_original_text_present_validator_rejects_missing_spans_and_allows_present_or_unknown_segments(tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello hello"}, + {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "goodbye"} + ] + """ + ) + proposals = [ + CorrectionProposal( + proposal_index=0, + module_instance="homophones", + module_key="homophones", + id=1, + original_text="hello", + corrected_text="hi", + confidence=0.95, + ), + CorrectionProposal( + proposal_index=1, + module_instance="homophones", + module_key="homophones", + id=2, + original_text="rank", + corrected_text="Hrank", + confidence=0.95, + ), + CorrectionProposal( + proposal_index=2, + module_instance="homophones", + module_key="homophones", + id=99, + original_text="missing", + corrected_text="present", + confidence=0.95, + ), + ] + + result = OriginalTextPresentValidator("original_text_present_guard").validate( + _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path) + ) + + assert [(decision.proposal_index, decision.approved, decision.reason) for decision in result.decisions] == [ + (0, True, None), + (1, False, "proposal original_text does not match segment text"), + (2, True, None), + ] + + @pytest.mark.parametrize( ("validator", "payload", "message_fragment"), [ diff --git a/tests/test_module_proposals.py b/tests/test_module_proposals.py index eeac17d..0a73711 100644 --- a/tests/test_module_proposals.py +++ b/tests/test_module_proposals.py @@ -1,7 +1,7 @@ from pathlib import Path from audita.core.chunking import chunk_transcript -from audita.core.config import AuditaConfig +from audita.core.config import AuditaConfig, ConfigOverrides from audita.core.errors import AuditaLLMError from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_json, parse_transcript_json from audita.framework.models import ModuleContext, ModuleRunSpec @@ -418,8 +418,8 @@ def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_ ] assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold" assert result.report.skipped_corrections[0].source == "validator:proposal_confidence_guard" - assert result.report.modules[0].validators[0].rejected_count == 1 - assert result.report.modules[0].validators[1].candidate_count == 0 + assert result.report.modules[0].validators[2].rejected_count == 1 + assert result.report.modules[0].validators[3].candidate_count == 0 def test_process_transcript_result_runs_spoken_word_module_with_full_validator_chain(tmp_path): @@ -498,6 +498,8 @@ def test_process_transcript_result_runs_spoken_word_module_with_full_validator_c "spoken_word:meaning_reversal_review", ] assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ + "identical_text_guard", + "original_text_present_guard", "proposal_confidence_guard", "protected_glossary_guard", "non_empty_segment_guard", @@ -558,7 +560,7 @@ def test_process_transcript_result_rejects_spoken_word_below_threshold_before_ll assert result.transcript[0].text == "I, uh, I think we should go." assert [call["stage_name"] for call in client.calls] == ["spoken_word:proposal"] assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold" - assert result.report.modules[0].validators[1].candidate_count == 0 + assert result.report.modules[0].validators[3].candidate_count == 0 def test_process_transcript_result_runs_grammar_module_with_full_validator_chain(tmp_path): @@ -626,6 +628,8 @@ def test_process_transcript_result_runs_grammar_module_with_full_validator_chain "grammar:meaning_reversal_review", ] assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ + "identical_text_guard", + "original_text_present_guard", "proposal_confidence_guard", "protected_glossary_guard", "non_empty_segment_guard", @@ -752,7 +756,7 @@ def test_process_transcript_result_rejects_grammar_below_threshold_before_later_ assert result.transcript[0].text == "hello world" assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"] assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold" - assert result.report.modules[0].validators[1].candidate_count == 0 + assert result.report.modules[0].validators[3].candidate_count == 0 def test_process_transcript_result_grammar_module_still_rejects_homophone_style_proposals(tmp_path): @@ -864,11 +868,100 @@ def test_process_transcript_result_rejects_spoken_word_whole_segment_deletion_be assert result.report.skipped_corrections[0].source == "validator:non_empty_segment_guard" assert result.report.skipped_corrections[0].reason == "correction would leave the segment empty" assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ + "identical_text_guard", + "original_text_present_guard", "proposal_confidence_guard", "protected_glossary_guard", "non_empty_segment_guard", "spoken_word_review", "meaning_reversal_review", ] - assert result.report.modules[0].validators[2].rejected_count == 1 - assert result.report.modules[0].validators[3].candidate_count == 0 + assert result.report.modules[0].validators[4].rejected_count == 1 + assert result.report.modules[0].validators[5].candidate_count == 0 + + +def test_process_transcript_result_rejects_identical_text_before_later_validators(tmp_path): + transcript = parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} + ] + """ + ) + config = AuditaConfig.from_sources( + env={}, + overrides=ConfigOverrides(work_dir=tmp_path / "work", work_dir_retention="always"), + ) + client = FakeStructuredLLMClient( + [ + { + "corrections": [ + { + "id": 1, + "original_text": "hello", + "corrected_text": "hello", + "confidence": 0.95, + } + ] + } + ] + ) + + result = process_transcript_result( + transcript, + _glossary(), + config, + module_keys=["grammar"], + llm_client=client, + ) + + assert result.transcript[0].text == "hello world" + assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"] + assert result.report.skipped_corrections[0].source == "validator:identical_text_guard" + assert result.report.skipped_corrections[0].reason == "proposal original_text and corrected_text are identical" + assert result.report.modules[0].validators[0].rejected_count == 1 + assert result.report.modules[0].validators[1].candidate_count == 0 + + +def test_process_transcript_result_rejects_missing_original_text_before_later_validators(tmp_path): + transcript = parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} + ] + """ + ) + config = AuditaConfig.from_sources( + env={}, + overrides=ConfigOverrides(work_dir=tmp_path / "work", work_dir_retention="always"), + ) + client = FakeStructuredLLMClient( + [ + { + "corrections": [ + { + "id": 1, + "original_text": "goodbye", + "corrected_text": "farewell", + "confidence": 0.95, + } + ] + } + ] + ) + + result = process_transcript_result( + transcript, + _glossary(), + config, + module_keys=["grammar"], + llm_client=client, + ) + + assert result.transcript[0].text == "hello world" + assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"] + assert result.report.skipped_corrections[0].source == "validator:original_text_present_guard" + assert result.report.skipped_corrections[0].reason == "proposal original_text does not match segment text" + assert result.report.modules[0].validators[0].approved_count == 1 + assert result.report.modules[0].validators[1].rejected_count == 1 + assert result.report.modules[0].validators[2].candidate_count == 0 diff --git a/tests/test_new_pipeline.py b/tests/test_new_pipeline.py index ff71c4d..52b3db7 100644 --- a/tests/test_new_pipeline.py +++ b/tests/test_new_pipeline.py @@ -129,6 +129,8 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy( assert (result.run_dir / "report.json").exists() assert (result.run_dir / "normalization" / "summary.json").exists() assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ + "identical_text_guard", + "original_text_present_guard", "proposal_confidence_guard", "glossary_stage_protected_glossary_guard", "non_empty_segment_guard", @@ -136,6 +138,8 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy( "meaning_reversal_review", ] assert [validator["name"] for validator in result.report.modules[2].to_dict()["validators"]] == [ + "identical_text_guard", + "original_text_present_guard", "proposal_confidence_guard", "glossary_stage_protected_glossary_guard", "non_empty_segment_guard", @@ -143,6 +147,8 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy( "meaning_reversal_review", ] assert [validator["name"] for validator in result.report.modules[3].to_dict()["validators"]] == [ + "identical_text_guard", + "original_text_present_guard", "proposal_confidence_guard", "protected_glossary_guard", "non_empty_segment_guard", @@ -150,6 +156,8 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy( "meaning_reversal_review", ] assert [validator["name"] for validator in result.report.modules[4].to_dict()["validators"]] == [ + "identical_text_guard", + "original_text_present_guard", "proposal_confidence_guard", "protected_glossary_guard", "non_empty_segment_guard", @@ -190,6 +198,8 @@ def test_default_module_specs_expose_final_validator_order(): "grammar", ] assert [validator.name for validator in specs[0].module.validators()] == [ + "identical_text_guard", + "original_text_present_guard", "proposal_confidence_guard", "glossary_stage_protected_glossary_guard", "non_empty_segment_guard", @@ -197,6 +207,8 @@ def test_default_module_specs_expose_final_validator_order(): "meaning_reversal_review", ] assert [validator.name for validator in specs[1].module.validators()] == [ + "identical_text_guard", + "original_text_present_guard", "proposal_confidence_guard", "protected_glossary_guard", "non_empty_segment_guard", @@ -204,6 +216,8 @@ def test_default_module_specs_expose_final_validator_order(): "meaning_reversal_review", ] assert [validator.name for validator in specs[2].module.validators()] == [ + "identical_text_guard", + "original_text_present_guard", "proposal_confidence_guard", "glossary_stage_protected_glossary_guard", "non_empty_segment_guard", @@ -211,6 +225,8 @@ def test_default_module_specs_expose_final_validator_order(): "meaning_reversal_review", ] assert [validator.name for validator in specs[3].module.validators()] == [ + "identical_text_guard", + "original_text_present_guard", "proposal_confidence_guard", "protected_glossary_guard", "non_empty_segment_guard", @@ -218,6 +234,8 @@ def test_default_module_specs_expose_final_validator_order(): "meaning_reversal_review", ] assert [validator.name for validator in specs[4].module.validators()] == [ + "identical_text_guard", + "original_text_present_guard", "proposal_confidence_guard", "protected_glossary_guard", "non_empty_segment_guard",