174 lines
4.7 KiB
Python
174 lines
4.7 KiB
Python
from pathlib import Path
|
|
|
|
from audita.core.config import AuditaConfig
|
|
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
|
|
from audita.framework.models import (
|
|
CorrectionProposal,
|
|
FilterDecision,
|
|
ModuleContext,
|
|
ModuleRunSpec,
|
|
ReviewDecision,
|
|
)
|
|
from audita.framework.runner import PipelineRunner
|
|
|
|
|
|
class AllowAllFilter:
|
|
name = "allow_all"
|
|
|
|
def evaluate(self, proposal, transcript, glossary, config):
|
|
return FilterDecision(approved=True)
|
|
|
|
|
|
class RejectAllFilter:
|
|
name = "reject_all"
|
|
|
|
def evaluate(self, proposal, transcript, glossary, config):
|
|
return FilterDecision(approved=False, reason="filter rejected proposal")
|
|
|
|
|
|
class AllowAllReviewStage:
|
|
name = "allow_all_review"
|
|
|
|
def review(self, proposals, transcript, glossary, config, llm_client, run_dir):
|
|
return [ReviewDecision(proposal_index=proposal.proposal_index, approved=True) for proposal in proposals]
|
|
|
|
|
|
class RecordingModule:
|
|
replacement_policy = "require_unique"
|
|
|
|
def __init__(self, module_key, proposals, recorder):
|
|
self.module_key = module_key
|
|
self._proposals = proposals
|
|
self._recorder = recorder
|
|
|
|
def deterministic_filters(self):
|
|
return [AllowAllFilter()]
|
|
|
|
def review_stages(self):
|
|
return [AllowAllReviewStage()]
|
|
|
|
def propose(self, transcript_section, context: ModuleContext):
|
|
self._recorder.append([segment.text for segment in transcript_section])
|
|
return list(self._proposals)
|
|
|
|
|
|
class RejectedModule:
|
|
module_key = "rejected"
|
|
replacement_policy = "require_unique"
|
|
|
|
def deterministic_filters(self):
|
|
return [RejectAllFilter()]
|
|
|
|
def review_stages(self):
|
|
return []
|
|
|
|
def propose(self, transcript_section, context):
|
|
return [
|
|
CorrectionProposal(
|
|
proposal_index=0,
|
|
module_instance=context.run_spec.instance_name,
|
|
module_key=context.run_spec.module_key,
|
|
id=1,
|
|
original_text="Hello",
|
|
corrected_text="Goodbye",
|
|
confidence=0.9,
|
|
)
|
|
]
|
|
|
|
|
|
def test_pipeline_runner_applies_modules_sequentially(tmp_path):
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Alpha."}
|
|
]
|
|
"""
|
|
)
|
|
glossary = parse_glossary_yaml(
|
|
"""
|
|
glossary:
|
|
- name: "Alpha"
|
|
category: noun
|
|
summary: "Alpha."
|
|
"""
|
|
)
|
|
seen = []
|
|
first = RecordingModule(
|
|
"first",
|
|
[
|
|
CorrectionProposal(
|
|
proposal_index=0,
|
|
module_instance="first",
|
|
module_key="first",
|
|
id=1,
|
|
original_text="Alpha",
|
|
corrected_text="Beta",
|
|
confidence=0.9,
|
|
)
|
|
],
|
|
seen,
|
|
)
|
|
second = RecordingModule(
|
|
"second",
|
|
[
|
|
CorrectionProposal(
|
|
proposal_index=0,
|
|
module_instance="second",
|
|
module_key="second",
|
|
id=1,
|
|
original_text="Beta",
|
|
corrected_text="Gamma",
|
|
confidence=0.9,
|
|
)
|
|
],
|
|
seen,
|
|
)
|
|
|
|
runner = PipelineRunner()
|
|
result = runner.run(
|
|
transcript=transcript,
|
|
glossary=glossary,
|
|
module_specs=[
|
|
ModuleRunSpec(instance_name="first", module_key="first", module=first),
|
|
ModuleRunSpec(instance_name="second", module_key="second", module=second),
|
|
],
|
|
config=AuditaConfig.from_sources(env={}),
|
|
run_dir=tmp_path / "run",
|
|
)
|
|
|
|
assert seen[0] == ["Alpha."]
|
|
assert seen[1] == ["Beta."]
|
|
assert result.transcript[0].text == "Gamma."
|
|
assert len(result.applied_changes) == 2
|
|
|
|
|
|
def test_pipeline_runner_reports_filter_rejections(tmp_path):
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Hello."}
|
|
]
|
|
"""
|
|
)
|
|
glossary = parse_glossary_yaml(
|
|
"""
|
|
glossary:
|
|
- name: "Hello"
|
|
category: noun
|
|
summary: "Hello."
|
|
"""
|
|
)
|
|
|
|
runner = PipelineRunner()
|
|
result = runner.run(
|
|
transcript=transcript,
|
|
glossary=glossary,
|
|
module_specs=[ModuleRunSpec(instance_name="rejected", module_key="rejected", module=RejectedModule())],
|
|
config=AuditaConfig.from_sources(env={}),
|
|
run_dir=tmp_path / "run",
|
|
)
|
|
|
|
assert result.transcript[0].text == "Hello."
|
|
assert result.module_reports[0].skipped_count == 1
|
|
assert result.skipped_corrections[0].reason == "filter rejected proposal"
|