audita/tests/test_framework_runner.py

from audita.core.config import AuditaConfig
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
from audita.framework.models import CorrectionProposal, ModuleContext, ModuleRunSpec
from audita.framework.runner import PipelineRunner
from audita.validators import ProtectedGlossaryTermsValidator
from audita.validators.base import ValidationContext, ValidationDecision, ValidationResult


class RecordingValidator:
    execution_kind = "deterministic"

    def __init__(self, name, recorder, approve=True):
        self.name = name
        self._recorder = recorder
        self._approve = approve

    def validate(self, context: ValidationContext) -> ValidationResult:
        self._recorder.append((self.name, [proposal.corrected_text for proposal in context.proposals]))
        return ValidationResult(
            validator_name=self.name,
            execution_kind=self.execution_kind,
            decisions=[
                ValidationDecision(
                    proposal_index=proposal.proposal_index,
                    approved=self._approve,
                    reason=None if self._approve else f"{self.name} rejected proposal",
                )
                for proposal in context.proposals
            ],
        )


class RecordingLLMValidator(RecordingValidator):
    execution_kind = "llm"


class RecordingModule:
    replacement_policy = "require_unique"

    def __init__(self, module_key, proposals, validators, recorder):
        self.module_key = module_key
        self._proposals = proposals
        self._validators = validators
        self._recorder = recorder

    def validators(self):
        return list(self._validators)

    def propose(self, transcript_section, context: ModuleContext):
        self._recorder.append(("propose", [segment.text for segment in transcript_section]))
        return list(self._proposals)


def test_pipeline_runner_applies_modules_sequentially(tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Alpha."}
        ]
        """
    )
    glossary = parse_glossary_yaml(
        """
        glossary:
          - name: "Alpha"
            category: noun
            summary: "Alpha."
        """
    )
    seen = []
    first = RecordingModule(
        "first",
        [
            CorrectionProposal(
                proposal_index=0,
                module_instance="first",
                module_key="first",
                id=1,
                original_text="Alpha",
                corrected_text="Beta",
                confidence=0.9,
            )
        ],
        [RecordingValidator("first_validator", seen)],
        seen,
    )
    second = RecordingModule(
        "second",
        [
            CorrectionProposal(
                proposal_index=0,
                module_instance="second",
                module_key="second",
                id=1,
                original_text="Beta",
                corrected_text="Gamma",
                confidence=0.9,
            )
        ],
        [RecordingValidator("second_validator", seen)],
        seen,
    )

    runner = PipelineRunner()
    result = runner.run(
        transcript=transcript,
        glossary=glossary,
        module_specs=[
            ModuleRunSpec(instance_name="first", module_key="first", module=first),
            ModuleRunSpec(instance_name="second", module_key="second", module=second),
        ],
        config=AuditaConfig.from_sources(env={}),
        run_dir=tmp_path / "run",
    )

    assert seen[0] == ("propose", ["Alpha."])
    assert seen[1] == ("first_validator", ["Beta"])
    assert seen[2] == ("propose", ["Beta."])
    assert seen[3] == ("second_validator", ["Gamma"])
    assert result.transcript[0].text == "Gamma."
    assert len(result.applied_changes) == 2


def test_pipeline_runner_validator_order_respects_survivors(tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Hello."}
        ]
        """
    )
    glossary = parse_glossary_yaml(
        """
        glossary:
          - name: "Hello"
            category: noun
            summary: "Hello."
        """
    )
    seen = []
    module = RecordingModule(
        "mod",
        [
            CorrectionProposal(
                proposal_index=0,
                module_instance="mod",
                module_key="mod",
                id=1,
                original_text="Hello",
                corrected_text="Goodbye",
                confidence=0.9,
            )
        ],
        [
            RecordingValidator("first", seen, approve=False),
            RecordingLLMValidator("second", seen, approve=True),
        ],
        seen,
    )

    runner = PipelineRunner()
    result = runner.run(
        transcript=transcript,
        glossary=glossary,
        module_specs=[ModuleRunSpec(instance_name="mod", module_key="mod", module=module)],
        config=AuditaConfig.from_sources(env={}),
        run_dir=tmp_path / "run",
    )

    assert ("first", ["Goodbye"]) in seen
    assert all(entry[0] != "second" for entry in seen)
    assert result.module_reports[0].validators[0].rejected_count == 1
    assert result.module_reports[0].validators[1].candidate_count == 0
    assert result.skipped_corrections[0].source == "validator:first"


def test_pipeline_runner_supports_deterministic_and_llm_validators_in_one_chain(tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Alpha."}
        ]
        """
    )
    glossary = parse_glossary_yaml(
        """
        glossary:
          - name: "Alpha"
            category: noun
            summary: "Alpha."
        """
    )
    seen = []
    module = RecordingModule(
        "mixed",
        [
            CorrectionProposal(
                proposal_index=0,
                module_instance="mixed",
                module_key="mixed",
                id=1,
                original_text="Alpha",
                corrected_text="Beta",
                confidence=0.9,
            )
        ],
        [
            RecordingValidator("deterministic_guard", seen),
            RecordingLLMValidator("llm_review", seen),
        ],
        seen,
    )

    runner = PipelineRunner()
    result = runner.run(
        transcript=transcript,
        glossary=glossary,
        module_specs=[ModuleRunSpec(instance_name="mixed", module_key="mixed", module=module)],
        config=AuditaConfig.from_sources(env={}),
        run_dir=tmp_path / "run",
    )

    assert result.transcript[0].text == "Beta."
    assert [report.execution_kind for report in result.module_reports[0].validators] == [
        "deterministic",
        "llm",
    ]


def test_pipeline_runner_uses_real_protected_glossary_validator(tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Hrank moves."}
        ]
        """
    )
    glossary = parse_glossary_yaml(
        """
        glossary:
          - name: "Hrank"
            category: pc
            summary: "Hrank is a player character."
        """
    )
    module = RecordingModule(
        "protected",
        [
            CorrectionProposal(
                proposal_index=0,
                module_instance="protected",
                module_key="protected",
                id=1,
                original_text="Hrank",
                corrected_text="Frank",
                confidence=0.9,
            )
        ],
        [ProtectedGlossaryTermsValidator("protected_glossary_guard")],
        [],
    )

    runner = PipelineRunner()
    result = runner.run(
        transcript=transcript,
        glossary=glossary,
        module_specs=[ModuleRunSpec(instance_name="protected", module_key="protected", module=module)],
        config=AuditaConfig.from_sources(env={}),
        run_dir=tmp_path / "run",
    )

    assert result.transcript[0].text == "Hrank moves."
    assert result.skipped_corrections[0].source == "validator:protected_glossary_guard"
    assert result.skipped_corrections[0].reason == "correction changes protected glossary term usage"