Added an interface for validators, and implemented an initial deterministic validator to protect terms in the provided glossary
This commit is contained in:
@@ -1,81 +1,56 @@
|
||||
from pathlib import Path
|
||||
|
||||
from audita.core.config import AuditaConfig
|
||||
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
|
||||
from audita.framework.models import (
|
||||
CorrectionProposal,
|
||||
FilterDecision,
|
||||
ModuleContext,
|
||||
ModuleRunSpec,
|
||||
ReviewDecision,
|
||||
)
|
||||
from audita.framework.models import CorrectionProposal, ModuleContext, ModuleRunSpec
|
||||
from audita.framework.runner import PipelineRunner
|
||||
from audita.validators import ProtectedGlossaryTermsValidator
|
||||
from audita.validators.base import ValidationContext, ValidationDecision, ValidationResult
|
||||
|
||||
|
||||
class AllowAllFilter:
|
||||
name = "allow_all"
|
||||
class RecordingValidator:
|
||||
execution_kind = "deterministic"
|
||||
|
||||
def evaluate(self, proposal, transcript, glossary, config):
|
||||
return FilterDecision(approved=True)
|
||||
def __init__(self, name, recorder, approve=True):
|
||||
self.name = name
|
||||
self._recorder = recorder
|
||||
self._approve = approve
|
||||
|
||||
def validate(self, context: ValidationContext) -> ValidationResult:
|
||||
self._recorder.append((self.name, [proposal.corrected_text for proposal in context.proposals]))
|
||||
return ValidationResult(
|
||||
validator_name=self.name,
|
||||
execution_kind=self.execution_kind,
|
||||
decisions=[
|
||||
ValidationDecision(
|
||||
proposal_index=proposal.proposal_index,
|
||||
approved=self._approve,
|
||||
reason=None if self._approve else f"{self.name} rejected proposal",
|
||||
)
|
||||
for proposal in context.proposals
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class RejectAllFilter:
|
||||
name = "reject_all"
|
||||
|
||||
def evaluate(self, proposal, transcript, glossary, config):
|
||||
return FilterDecision(approved=False, reason="filter rejected proposal")
|
||||
|
||||
|
||||
class AllowAllReviewStage:
|
||||
name = "allow_all_review"
|
||||
|
||||
def review(self, proposals, transcript, glossary, config, llm_client, run_dir):
|
||||
return [ReviewDecision(proposal_index=proposal.proposal_index, approved=True) for proposal in proposals]
|
||||
class RecordingLLMValidator(RecordingValidator):
|
||||
execution_kind = "llm"
|
||||
|
||||
|
||||
class RecordingModule:
|
||||
replacement_policy = "require_unique"
|
||||
|
||||
def __init__(self, module_key, proposals, recorder):
|
||||
def __init__(self, module_key, proposals, validators, recorder):
|
||||
self.module_key = module_key
|
||||
self._proposals = proposals
|
||||
self._validators = validators
|
||||
self._recorder = recorder
|
||||
|
||||
def deterministic_filters(self):
|
||||
return [AllowAllFilter()]
|
||||
|
||||
def review_stages(self):
|
||||
return [AllowAllReviewStage()]
|
||||
def validators(self):
|
||||
return list(self._validators)
|
||||
|
||||
def propose(self, transcript_section, context: ModuleContext):
|
||||
self._recorder.append([segment.text for segment in transcript_section])
|
||||
self._recorder.append(("propose", [segment.text for segment in transcript_section]))
|
||||
return list(self._proposals)
|
||||
|
||||
|
||||
class RejectedModule:
|
||||
module_key = "rejected"
|
||||
replacement_policy = "require_unique"
|
||||
|
||||
def deterministic_filters(self):
|
||||
return [RejectAllFilter()]
|
||||
|
||||
def review_stages(self):
|
||||
return []
|
||||
|
||||
def propose(self, transcript_section, context):
|
||||
return [
|
||||
CorrectionProposal(
|
||||
proposal_index=0,
|
||||
module_instance=context.run_spec.instance_name,
|
||||
module_key=context.run_spec.module_key,
|
||||
id=1,
|
||||
original_text="Hello",
|
||||
corrected_text="Goodbye",
|
||||
confidence=0.9,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_pipeline_runner_applies_modules_sequentially(tmp_path):
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
@@ -106,6 +81,7 @@ def test_pipeline_runner_applies_modules_sequentially(tmp_path):
|
||||
confidence=0.9,
|
||||
)
|
||||
],
|
||||
[RecordingValidator("first_validator", seen)],
|
||||
seen,
|
||||
)
|
||||
second = RecordingModule(
|
||||
@@ -121,6 +97,7 @@ def test_pipeline_runner_applies_modules_sequentially(tmp_path):
|
||||
confidence=0.9,
|
||||
)
|
||||
],
|
||||
[RecordingValidator("second_validator", seen)],
|
||||
seen,
|
||||
)
|
||||
|
||||
@@ -136,13 +113,15 @@ def test_pipeline_runner_applies_modules_sequentially(tmp_path):
|
||||
run_dir=tmp_path / "run",
|
||||
)
|
||||
|
||||
assert seen[0] == ["Alpha."]
|
||||
assert seen[1] == ["Beta."]
|
||||
assert seen[0] == ("propose", ["Alpha."])
|
||||
assert seen[1] == ("first_validator", ["Beta"])
|
||||
assert seen[2] == ("propose", ["Beta."])
|
||||
assert seen[3] == ("second_validator", ["Gamma"])
|
||||
assert result.transcript[0].text == "Gamma."
|
||||
assert len(result.applied_changes) == 2
|
||||
|
||||
|
||||
def test_pipeline_runner_reports_filter_rejections(tmp_path):
|
||||
def test_pipeline_runner_validator_order_respects_survivors(tmp_path):
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
[
|
||||
@@ -158,16 +137,138 @@ def test_pipeline_runner_reports_filter_rejections(tmp_path):
|
||||
summary: "Hello."
|
||||
"""
|
||||
)
|
||||
seen = []
|
||||
module = RecordingModule(
|
||||
"mod",
|
||||
[
|
||||
CorrectionProposal(
|
||||
proposal_index=0,
|
||||
module_instance="mod",
|
||||
module_key="mod",
|
||||
id=1,
|
||||
original_text="Hello",
|
||||
corrected_text="Goodbye",
|
||||
confidence=0.9,
|
||||
)
|
||||
],
|
||||
[
|
||||
RecordingValidator("first", seen, approve=False),
|
||||
RecordingLLMValidator("second", seen, approve=True),
|
||||
],
|
||||
seen,
|
||||
)
|
||||
|
||||
runner = PipelineRunner()
|
||||
result = runner.run(
|
||||
transcript=transcript,
|
||||
glossary=glossary,
|
||||
module_specs=[ModuleRunSpec(instance_name="rejected", module_key="rejected", module=RejectedModule())],
|
||||
module_specs=[ModuleRunSpec(instance_name="mod", module_key="mod", module=module)],
|
||||
config=AuditaConfig.from_sources(env={}),
|
||||
run_dir=tmp_path / "run",
|
||||
)
|
||||
|
||||
assert result.transcript[0].text == "Hello."
|
||||
assert result.module_reports[0].skipped_count == 1
|
||||
assert result.skipped_corrections[0].reason == "filter rejected proposal"
|
||||
assert ("first", ["Goodbye"]) in seen
|
||||
assert all(entry[0] != "second" for entry in seen)
|
||||
assert result.module_reports[0].validators[0].rejected_count == 1
|
||||
assert result.module_reports[0].validators[1].candidate_count == 0
|
||||
assert result.skipped_corrections[0].source == "validator:first"
|
||||
|
||||
|
||||
def test_pipeline_runner_supports_deterministic_and_llm_validators_in_one_chain(tmp_path):
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
[
|
||||
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Alpha."}
|
||||
]
|
||||
"""
|
||||
)
|
||||
glossary = parse_glossary_yaml(
|
||||
"""
|
||||
glossary:
|
||||
- name: "Alpha"
|
||||
category: noun
|
||||
summary: "Alpha."
|
||||
"""
|
||||
)
|
||||
seen = []
|
||||
module = RecordingModule(
|
||||
"mixed",
|
||||
[
|
||||
CorrectionProposal(
|
||||
proposal_index=0,
|
||||
module_instance="mixed",
|
||||
module_key="mixed",
|
||||
id=1,
|
||||
original_text="Alpha",
|
||||
corrected_text="Beta",
|
||||
confidence=0.9,
|
||||
)
|
||||
],
|
||||
[
|
||||
RecordingValidator("deterministic_guard", seen),
|
||||
RecordingLLMValidator("llm_review", seen),
|
||||
],
|
||||
seen,
|
||||
)
|
||||
|
||||
runner = PipelineRunner()
|
||||
result = runner.run(
|
||||
transcript=transcript,
|
||||
glossary=glossary,
|
||||
module_specs=[ModuleRunSpec(instance_name="mixed", module_key="mixed", module=module)],
|
||||
config=AuditaConfig.from_sources(env={}),
|
||||
run_dir=tmp_path / "run",
|
||||
)
|
||||
|
||||
assert result.transcript[0].text == "Beta."
|
||||
assert [report.execution_kind for report in result.module_reports[0].validators] == [
|
||||
"deterministic",
|
||||
"llm",
|
||||
]
|
||||
|
||||
|
||||
def test_pipeline_runner_uses_real_protected_glossary_validator(tmp_path):
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
[
|
||||
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Hrank moves."}
|
||||
]
|
||||
"""
|
||||
)
|
||||
glossary = parse_glossary_yaml(
|
||||
"""
|
||||
glossary:
|
||||
- name: "Hrank"
|
||||
category: pc
|
||||
summary: "Hrank is a player character."
|
||||
"""
|
||||
)
|
||||
module = RecordingModule(
|
||||
"protected",
|
||||
[
|
||||
CorrectionProposal(
|
||||
proposal_index=0,
|
||||
module_instance="protected",
|
||||
module_key="protected",
|
||||
id=1,
|
||||
original_text="Hrank",
|
||||
corrected_text="Frank",
|
||||
confidence=0.9,
|
||||
)
|
||||
],
|
||||
[ProtectedGlossaryTermsValidator("protected_glossary_guard")],
|
||||
[],
|
||||
)
|
||||
|
||||
runner = PipelineRunner()
|
||||
result = runner.run(
|
||||
transcript=transcript,
|
||||
glossary=glossary,
|
||||
module_specs=[ModuleRunSpec(instance_name="protected", module_key="protected", module=module)],
|
||||
config=AuditaConfig.from_sources(env={}),
|
||||
run_dir=tmp_path / "run",
|
||||
)
|
||||
|
||||
assert result.transcript[0].text == "Hrank moves."
|
||||
assert result.skipped_corrections[0].source == "validator:protected_glossary_guard"
|
||||
assert result.skipped_corrections[0].reason == "correction changes protected glossary term usage"
|
||||
|
||||
Reference in New Issue
Block a user