from audita.core.config import AuditaConfig from audita.core.schemas import parse_glossary_yaml, parse_transcript_json from audita.framework.models import CorrectionProposal, ModuleContext, ModuleRunSpec from audita.framework.runner import PipelineRunner from audita.validators import ProtectedGlossaryTermsValidator from audita.validators.base import ValidationContext, ValidationDecision, ValidationResult class RecordingValidator: execution_kind = "deterministic" def __init__(self, name, recorder, approve=True): self.name = name self._recorder = recorder self._approve = approve def validate(self, context: ValidationContext) -> ValidationResult: self._recorder.append((self.name, [proposal.corrected_text for proposal in context.proposals])) return ValidationResult( validator_name=self.name, execution_kind=self.execution_kind, decisions=[ ValidationDecision( proposal_index=proposal.proposal_index, approved=self._approve, reason=None if self._approve else f"{self.name} rejected proposal", ) for proposal in context.proposals ], ) class RecordingLLMValidator(RecordingValidator): execution_kind = "llm" class RecordingModule: replacement_policy = "require_unique" def __init__(self, module_key, proposals, validators, recorder): self.module_key = module_key self._proposals = proposals self._validators = validators self._recorder = recorder def validators(self): return list(self._validators) def propose(self, transcript_section, context: ModuleContext): self._recorder.append(("propose", [segment.text for segment in transcript_section])) return list(self._proposals) def test_pipeline_runner_applies_modules_sequentially(tmp_path): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Alpha."} ] """ ) glossary = parse_glossary_yaml( """ glossary: - name: "Alpha" category: noun summary: "Alpha." """ ) seen = [] first = RecordingModule( "first", [ CorrectionProposal( proposal_index=0, module_instance="first", module_key="first", id=1, original_text="Alpha", corrected_text="Beta", confidence=0.9, ) ], [RecordingValidator("first_validator", seen)], seen, ) second = RecordingModule( "second", [ CorrectionProposal( proposal_index=0, module_instance="second", module_key="second", id=1, original_text="Beta", corrected_text="Gamma", confidence=0.9, ) ], [RecordingValidator("second_validator", seen)], seen, ) runner = PipelineRunner() result = runner.run( transcript=transcript, glossary=glossary, module_specs=[ ModuleRunSpec(instance_name="first", module_key="first", module=first), ModuleRunSpec(instance_name="second", module_key="second", module=second), ], config=AuditaConfig.from_sources(env={}), run_dir=tmp_path / "run", ) assert seen[0] == ("propose", ["Alpha."]) assert seen[1] == ("first_validator", ["Beta"]) assert seen[2] == ("propose", ["Beta."]) assert seen[3] == ("second_validator", ["Gamma"]) assert result.transcript[0].text == "Gamma." assert len(result.applied_changes) == 2 def test_pipeline_runner_validator_order_respects_survivors(tmp_path): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Hello."} ] """ ) glossary = parse_glossary_yaml( """ glossary: - name: "Hello" category: noun summary: "Hello." """ ) seen = [] module = RecordingModule( "mod", [ CorrectionProposal( proposal_index=0, module_instance="mod", module_key="mod", id=1, original_text="Hello", corrected_text="Goodbye", confidence=0.9, ) ], [ RecordingValidator("first", seen, approve=False), RecordingLLMValidator("second", seen, approve=True), ], seen, ) runner = PipelineRunner() result = runner.run( transcript=transcript, glossary=glossary, module_specs=[ModuleRunSpec(instance_name="mod", module_key="mod", module=module)], config=AuditaConfig.from_sources(env={}), run_dir=tmp_path / "run", ) assert ("first", ["Goodbye"]) in seen assert all(entry[0] != "second" for entry in seen) assert result.module_reports[0].validators[0].rejected_count == 1 assert result.module_reports[0].validators[1].candidate_count == 0 assert result.skipped_corrections[0].source == "validator:first" def test_pipeline_runner_supports_deterministic_and_llm_validators_in_one_chain(tmp_path): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Alpha."} ] """ ) glossary = parse_glossary_yaml( """ glossary: - name: "Alpha" category: noun summary: "Alpha." """ ) seen = [] module = RecordingModule( "mixed", [ CorrectionProposal( proposal_index=0, module_instance="mixed", module_key="mixed", id=1, original_text="Alpha", corrected_text="Beta", confidence=0.9, ) ], [ RecordingValidator("deterministic_guard", seen), RecordingLLMValidator("llm_review", seen), ], seen, ) runner = PipelineRunner() result = runner.run( transcript=transcript, glossary=glossary, module_specs=[ModuleRunSpec(instance_name="mixed", module_key="mixed", module=module)], config=AuditaConfig.from_sources(env={}), run_dir=tmp_path / "run", ) assert result.transcript[0].text == "Beta." assert [report.execution_kind for report in result.module_reports[0].validators] == [ "deterministic", "llm", ] def test_pipeline_runner_uses_real_protected_glossary_validator(tmp_path): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Hrank moves."} ] """ ) glossary = parse_glossary_yaml( """ glossary: - name: "Hrank" category: pc summary: "Hrank is a player character." """ ) module = RecordingModule( "protected", [ CorrectionProposal( proposal_index=0, module_instance="protected", module_key="protected", id=1, original_text="Hrank", corrected_text="Frank", confidence=0.9, ) ], [ProtectedGlossaryTermsValidator("protected_glossary_guard")], [], ) runner = PipelineRunner() result = runner.run( transcript=transcript, glossary=glossary, module_specs=[ModuleRunSpec(instance_name="protected", module_key="protected", module=module)], config=AuditaConfig.from_sources(env={}), run_dir=tmp_path / "run", ) assert result.transcript[0].text == "Hrank moves." assert result.skipped_corrections[0].source == "validator:protected_glossary_guard" assert result.skipped_corrections[0].reason == "correction changes protected glossary term usage"