from audita.core.config import AuditaConfig from audita.core.errors import AuditaLLMError from audita.core.schemas import parse_glossary_yaml, parse_transcript_json from audita.framework.models import CorrectionProposal, ModuleContext, ModuleRunSpec from audita.framework.runner import PipelineRunner from audita.validators import ( MeaningReversalValidator, ProposalConfidenceValidator, ProtectedGlossaryTermsValidator, SpokenFormPlausibilityValidator, ) from audita.validators.base import ValidationContext, ValidationDecision, ValidationResult class RecordingValidator: execution_kind = "deterministic" def __init__(self, name, recorder, approve=True): self.name = name self._recorder = recorder self._approve = approve def validate(self, context: ValidationContext) -> ValidationResult: self._recorder.append((self.name, [proposal.corrected_text for proposal in context.proposals])) return ValidationResult( validator_name=self.name, execution_kind=self.execution_kind, decisions=[ ValidationDecision( proposal_index=proposal.proposal_index, approved=self._approve, reason=None if self._approve else f"{self.name} rejected proposal", ) for proposal in context.proposals ], ) class RecordingLLMValidator(RecordingValidator): execution_kind = "llm" class FakeStructuredLLMClient: def __init__(self, responses): self._responses = list(responses) self.calls = [] def run_structured(self, *, stage_name, messages, response_model, config): self.calls.append( { "stage_name": stage_name, "messages": list(messages), "response_model": response_model, } ) if not self._responses: raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.") payload = self._responses.pop(0) return response_model.model_validate(payload) class RecordingModule: replacement_policy = "require_unique" def __init__(self, module_key, proposals, validators, recorder): self.module_key = module_key self._proposals = proposals self._validators = validators self._recorder = recorder def validators(self): return list(self._validators) def propose(self, transcript_section, context: ModuleContext): self._recorder.append(("propose", [item.segment.text for item in transcript_section.segments])) return list(self._proposals) def test_pipeline_runner_applies_modules_sequentially(tmp_path): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Alpha."} ] """ ) glossary = parse_glossary_yaml( """ glossary: - name: "Alpha" category: noun summary: "Alpha." """ ) seen = [] first = RecordingModule( "first", [ CorrectionProposal( proposal_index=0, module_instance="first", module_key="first", id=1, original_text="Alpha", corrected_text="Beta", confidence=0.9, ) ], [RecordingValidator("first_validator", seen)], seen, ) second = RecordingModule( "second", [ CorrectionProposal( proposal_index=0, module_instance="second", module_key="second", id=1, original_text="Beta", corrected_text="Gamma", confidence=0.9, ) ], [RecordingValidator("second_validator", seen)], seen, ) runner = PipelineRunner() result = runner.run( transcript=transcript, glossary=glossary, module_specs=[ ModuleRunSpec(instance_name="first", module_key="first", module=first), ModuleRunSpec(instance_name="second", module_key="second", module=second), ], config=AuditaConfig.from_sources(env={}), run_dir=tmp_path / "run", ) assert seen[0] == ("propose", ["Alpha."]) assert seen[1] == ("first_validator", ["Beta"]) assert seen[2] == ("propose", ["Beta."]) assert seen[3] == ("second_validator", ["Gamma"]) assert result.transcript[0].text == "Gamma." assert len(result.applied_changes) == 2 def test_pipeline_runner_validator_order_respects_survivors(tmp_path): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Hello."} ] """ ) glossary = parse_glossary_yaml( """ glossary: - name: "Hello" category: noun summary: "Hello." """ ) seen = [] module = RecordingModule( "mod", [ CorrectionProposal( proposal_index=0, module_instance="mod", module_key="mod", id=1, original_text="Hello", corrected_text="Goodbye", confidence=0.9, ) ], [ RecordingValidator("first", seen, approve=False), RecordingLLMValidator("second", seen, approve=True), ], seen, ) runner = PipelineRunner() result = runner.run( transcript=transcript, glossary=glossary, module_specs=[ModuleRunSpec(instance_name="mod", module_key="mod", module=module)], config=AuditaConfig.from_sources(env={}), run_dir=tmp_path / "run", ) assert ("first", ["Goodbye"]) in seen assert all(entry[0] != "second" for entry in seen) assert result.module_reports[0].validators[0].rejected_count == 1 assert result.module_reports[0].validators[1].candidate_count == 0 assert result.skipped_corrections[0].source == "validator:first" def test_pipeline_runner_supports_deterministic_and_llm_validators_in_one_chain(tmp_path): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Alpha."} ] """ ) glossary = parse_glossary_yaml( """ glossary: - name: "Alpha" category: noun summary: "Alpha." """ ) seen = [] module = RecordingModule( "mixed", [ CorrectionProposal( proposal_index=0, module_instance="mixed", module_key="mixed", id=1, original_text="Alpha", corrected_text="Beta", confidence=0.9, ) ], [ RecordingValidator("deterministic_guard", seen), RecordingLLMValidator("llm_review", seen), ], seen, ) runner = PipelineRunner() result = runner.run( transcript=transcript, glossary=glossary, module_specs=[ModuleRunSpec(instance_name="mixed", module_key="mixed", module=module)], config=AuditaConfig.from_sources(env={}), run_dir=tmp_path / "run", ) assert result.transcript[0].text == "Beta." assert [report.execution_kind for report in result.module_reports[0].validators] == [ "deterministic", "llm", ] def test_pipeline_runner_supports_real_llm_validators_in_one_chain(tmp_path): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the dam."} ] """ ) glossary = parse_glossary_yaml( """ glossary: - name: "Jesters" category: faction summary: "The Jesters are a faction." """ ) module = RecordingModule( "mixed_real", [ CorrectionProposal( proposal_index=0, module_instance="mixed_real", module_key="glossary", id=1, original_text="gestures", corrected_text="Jesters", confidence=0.9, ) ], [ ProposalConfidenceValidator("proposal_confidence_guard", "glossary_confidence_threshold"), ProtectedGlossaryTermsValidator("protected_glossary_guard"), SpokenFormPlausibilityValidator("spoken_form_plausibility_review"), MeaningReversalValidator("meaning_reversal_review"), ], [], ) llm_client = FakeStructuredLLMClient( [ { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.95, "reason": "Likely phonetic mistranscription in context.", } ] }, { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.98, "reason": "Does not reverse the segment meaning.", } ] }, ] ) runner = PipelineRunner() result = runner.run( transcript=transcript, glossary=glossary, module_specs=[ModuleRunSpec(instance_name="mixed_real", module_key="glossary", module=module)], config=AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}), run_dir=tmp_path / "run", llm_client=llm_client, ) assert result.transcript[0].text == "There were Jesters at the dam." assert [report.execution_kind for report in result.module_reports[0].validators] == [ "deterministic", "deterministic", "llm", "llm", ] assert [call["stage_name"] for call in llm_client.calls] == [ "mixed_real:spoken_form_plausibility_review", "mixed_real:meaning_reversal_review", ] def test_pipeline_runner_uses_real_protected_glossary_validator(tmp_path): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Hrank moves."} ] """ ) glossary = parse_glossary_yaml( """ glossary: - name: "Hrank" category: pc summary: "Hrank is a player character." """ ) module = RecordingModule( "protected", [ CorrectionProposal( proposal_index=0, module_instance="protected", module_key="protected", id=1, original_text="Hrank", corrected_text="Frank", confidence=0.9, ) ], [ProtectedGlossaryTermsValidator("protected_glossary_guard")], [], ) runner = PipelineRunner() result = runner.run( transcript=transcript, glossary=glossary, module_specs=[ModuleRunSpec(instance_name="protected", module_key="protected", module=module)], config=AuditaConfig.from_sources(env={}), run_dir=tmp_path / "run", ) assert result.transcript[0].text == "Hrank moves." assert result.skipped_corrections[0].source == "validator:protected_glossary_guard" assert result.skipped_corrections[0].reason == "correction changes protected glossary term usage"