Replaced the stub LLM-based validator class with two real LLM-based validator implementations
This commit is contained in:
@@ -1,8 +1,9 @@
|
||||
from audita.core.config import AuditaConfig
|
||||
from audita.core.errors import AuditaLLMError
|
||||
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
|
||||
from audita.framework.models import CorrectionProposal, ModuleContext, ModuleRunSpec
|
||||
from audita.framework.runner import PipelineRunner
|
||||
from audita.validators import ProtectedGlossaryTermsValidator
|
||||
from audita.validators import MeaningReversalValidator, ProtectedGlossaryTermsValidator, SpokenFormPlausibilityValidator
|
||||
from audita.validators.base import ValidationContext, ValidationDecision, ValidationResult
|
||||
|
||||
|
||||
@@ -34,6 +35,25 @@ class RecordingLLMValidator(RecordingValidator):
|
||||
execution_kind = "llm"
|
||||
|
||||
|
||||
class FakeStructuredLLMClient:
|
||||
def __init__(self, responses):
|
||||
self._responses = list(responses)
|
||||
self.calls = []
|
||||
|
||||
def run_structured(self, *, stage_name, messages, response_model, config):
|
||||
self.calls.append(
|
||||
{
|
||||
"stage_name": stage_name,
|
||||
"messages": list(messages),
|
||||
"response_model": response_model,
|
||||
}
|
||||
)
|
||||
if not self._responses:
|
||||
raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.")
|
||||
payload = self._responses.pop(0)
|
||||
return response_model.model_validate(payload)
|
||||
|
||||
|
||||
class RecordingModule:
|
||||
replacement_policy = "require_unique"
|
||||
|
||||
@@ -227,6 +247,89 @@ def test_pipeline_runner_supports_deterministic_and_llm_validators_in_one_chain(
|
||||
]
|
||||
|
||||
|
||||
def test_pipeline_runner_supports_real_llm_validators_in_one_chain(tmp_path):
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
[
|
||||
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the dam."}
|
||||
]
|
||||
"""
|
||||
)
|
||||
glossary = parse_glossary_yaml(
|
||||
"""
|
||||
glossary:
|
||||
- name: "Jesters"
|
||||
category: faction
|
||||
summary: "The Jesters are a faction."
|
||||
"""
|
||||
)
|
||||
module = RecordingModule(
|
||||
"mixed_real",
|
||||
[
|
||||
CorrectionProposal(
|
||||
proposal_index=0,
|
||||
module_instance="mixed_real",
|
||||
module_key="glossary",
|
||||
id=1,
|
||||
original_text="gestures",
|
||||
corrected_text="Jesters",
|
||||
confidence=0.9,
|
||||
)
|
||||
],
|
||||
[
|
||||
ProtectedGlossaryTermsValidator("protected_glossary_guard"),
|
||||
SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
|
||||
MeaningReversalValidator("meaning_reversal_review"),
|
||||
],
|
||||
[],
|
||||
)
|
||||
llm_client = FakeStructuredLLMClient(
|
||||
[
|
||||
{
|
||||
"validations": [
|
||||
{
|
||||
"correction_index": 0,
|
||||
"approved": True,
|
||||
"confidence": 0.95,
|
||||
"reason": "Likely phonetic mistranscription in context.",
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"validations": [
|
||||
{
|
||||
"correction_index": 0,
|
||||
"approved": True,
|
||||
"confidence": 0.98,
|
||||
"reason": "Does not reverse the segment meaning.",
|
||||
}
|
||||
]
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
runner = PipelineRunner()
|
||||
result = runner.run(
|
||||
transcript=transcript,
|
||||
glossary=glossary,
|
||||
module_specs=[ModuleRunSpec(instance_name="mixed_real", module_key="glossary", module=module)],
|
||||
config=AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}),
|
||||
run_dir=tmp_path / "run",
|
||||
llm_client=llm_client,
|
||||
)
|
||||
|
||||
assert result.transcript[0].text == "There were Jesters at the dam."
|
||||
assert [report.execution_kind for report in result.module_reports[0].validators] == [
|
||||
"deterministic",
|
||||
"llm",
|
||||
"llm",
|
||||
]
|
||||
assert [call["stage_name"] for call in llm_client.calls] == [
|
||||
"mixed_real:spoken_form_plausibility_review",
|
||||
"mixed_real:meaning_reversal_review",
|
||||
]
|
||||
|
||||
|
||||
def test_pipeline_runner_uses_real_protected_glossary_validator(tmp_path):
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
|
||||
453
tests/test_llm_validators.py
Normal file
453
tests/test_llm_validators.py
Normal file
@@ -0,0 +1,453 @@
|
||||
import pytest
|
||||
|
||||
from audita.core.chunking import TokenBatch
|
||||
from audita.core.config import AuditaConfig
|
||||
from audita.core.errors import AuditaLLMError
|
||||
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
|
||||
from audita.framework.models import CorrectionProposal, ModuleRunSpec
|
||||
from audita.validators.base import ValidationContext
|
||||
from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator
|
||||
from audita.validators.prompts import (
|
||||
build_meaning_reversal_messages,
|
||||
build_spoken_form_plausibility_messages,
|
||||
)
|
||||
import audita.validators.llm as llm_module
|
||||
|
||||
|
||||
class _Module:
|
||||
def __init__(self, replacement_policy: str) -> None:
|
||||
self.replacement_policy = replacement_policy
|
||||
|
||||
|
||||
class FakeStructuredLLMClient:
|
||||
def __init__(self, responses):
|
||||
self._responses = list(responses)
|
||||
self.calls = []
|
||||
|
||||
def run_structured(self, *, stage_name, messages, response_model, config):
|
||||
self.calls.append(
|
||||
{
|
||||
"stage_name": stage_name,
|
||||
"messages": list(messages),
|
||||
"response_model": response_model,
|
||||
}
|
||||
)
|
||||
if not self._responses:
|
||||
raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.")
|
||||
return response_model.model_validate(self._responses.pop(0))
|
||||
|
||||
|
||||
def _glossary():
|
||||
return parse_glossary_yaml(
|
||||
"""
|
||||
glossary:
|
||||
- name: "Jesters"
|
||||
category: faction
|
||||
summary: "The Jesters are a faction."
|
||||
- name: "Hrank"
|
||||
category: pc
|
||||
summary: "Hrank is a player character."
|
||||
- name: "Lyra"
|
||||
category: npc
|
||||
summary: "Lyra is an NPC."
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def _context(
|
||||
*,
|
||||
proposals,
|
||||
transcript,
|
||||
llm_client,
|
||||
tmp_path,
|
||||
replacement_policy="require_unique",
|
||||
):
|
||||
return ValidationContext(
|
||||
proposals=proposals,
|
||||
transcript=transcript,
|
||||
glossary=_glossary(),
|
||||
config=AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}),
|
||||
run_spec=ModuleRunSpec(
|
||||
instance_name="homophones",
|
||||
module_key="homophones",
|
||||
module=_Module(replacement_policy),
|
||||
),
|
||||
run_dir=tmp_path,
|
||||
llm_client=llm_client,
|
||||
)
|
||||
|
||||
|
||||
def test_spoken_form_plausibility_validator_approves_plausible_and_rejects_implausible_corrections(tmp_path):
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
[
|
||||
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."},
|
||||
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "Lyra moved first."}
|
||||
]
|
||||
"""
|
||||
)
|
||||
proposals = [
|
||||
CorrectionProposal(
|
||||
proposal_index=0,
|
||||
module_instance="homophones",
|
||||
module_key="homophones",
|
||||
id=1,
|
||||
original_text="gestures",
|
||||
corrected_text="Jesters",
|
||||
confidence=0.95,
|
||||
),
|
||||
CorrectionProposal(
|
||||
proposal_index=1,
|
||||
module_instance="homophones",
|
||||
module_key="homophones",
|
||||
id=2,
|
||||
original_text="Lyra",
|
||||
corrected_text="Jesters",
|
||||
confidence=0.95,
|
||||
),
|
||||
]
|
||||
client = FakeStructuredLLMClient(
|
||||
[
|
||||
{
|
||||
"validations": [
|
||||
{
|
||||
"correction_index": 0,
|
||||
"approved": True,
|
||||
"confidence": 0.97,
|
||||
"reason": "Likely spoken-form correction in context.",
|
||||
},
|
||||
{
|
||||
"correction_index": 1,
|
||||
"approved": False,
|
||||
"confidence": 0.99,
|
||||
"reason": "Not plausibly related by homophone or mistranscription.",
|
||||
},
|
||||
]
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
|
||||
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
|
||||
)
|
||||
|
||||
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
|
||||
(0, True),
|
||||
(1, False),
|
||||
]
|
||||
prompt_text = client.calls[0]["messages"][1]["content"]
|
||||
assert '"original_segment_text"' in prompt_text
|
||||
assert "There were Jesters at the temple." in prompt_text
|
||||
assert "Lyra moved first." in prompt_text
|
||||
|
||||
|
||||
def test_meaning_reversal_validator_rejects_reversal_and_approves_nonreversal(tmp_path):
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
[
|
||||
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam, but Claude actually can."},
|
||||
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "The figure became visible in the doorway."}
|
||||
]
|
||||
"""
|
||||
)
|
||||
proposals = [
|
||||
CorrectionProposal(
|
||||
proposal_index=0,
|
||||
module_instance="homophones",
|
||||
module_key="homophones",
|
||||
id=1,
|
||||
original_text="dam",
|
||||
corrected_text="damn",
|
||||
confidence=0.95,
|
||||
),
|
||||
CorrectionProposal(
|
||||
proposal_index=1,
|
||||
module_instance="homophones",
|
||||
module_key="homophones",
|
||||
id=2,
|
||||
original_text="visible",
|
||||
corrected_text="invisible",
|
||||
confidence=0.95,
|
||||
),
|
||||
]
|
||||
client = FakeStructuredLLMClient(
|
||||
[
|
||||
{
|
||||
"validations": [
|
||||
{
|
||||
"correction_index": 0,
|
||||
"approved": True,
|
||||
"confidence": 0.94,
|
||||
"reason": "Does not reverse the segment meaning.",
|
||||
},
|
||||
{
|
||||
"correction_index": 1,
|
||||
"approved": False,
|
||||
"confidence": 1.0,
|
||||
"reason": "Changes visible to invisible and reverses the segment meaning.",
|
||||
},
|
||||
]
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
result = MeaningReversalValidator("meaning_reversal_review").validate(
|
||||
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
|
||||
)
|
||||
|
||||
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
|
||||
(0, True),
|
||||
(1, False),
|
||||
]
|
||||
prompt_text = client.calls[0]["messages"][1]["content"]
|
||||
assert "The figure became visible in the doorway." in prompt_text
|
||||
assert "The figure became invisible in the doorway." in prompt_text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("validator", "payload", "message_fragment"),
|
||||
[
|
||||
(
|
||||
SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
|
||||
{
|
||||
"validations": [
|
||||
{
|
||||
"correction_index": 0,
|
||||
"approved": True,
|
||||
"confidence": 0.9,
|
||||
"reason": "ok",
|
||||
},
|
||||
{
|
||||
"correction_index": 0,
|
||||
"approved": True,
|
||||
"confidence": 0.9,
|
||||
"reason": "duplicate",
|
||||
},
|
||||
]
|
||||
},
|
||||
"duplicate correction_index values",
|
||||
),
|
||||
(
|
||||
MeaningReversalValidator("meaning_reversal_review"),
|
||||
{"validations": []},
|
||||
"omitted correction_index values",
|
||||
),
|
||||
(
|
||||
SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
|
||||
{
|
||||
"validations": [
|
||||
{
|
||||
"correction_index": 99,
|
||||
"approved": True,
|
||||
"confidence": 0.9,
|
||||
"reason": "unknown",
|
||||
}
|
||||
]
|
||||
},
|
||||
"unknown correction_index",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_llm_validators_reject_bad_correction_indexes(tmp_path, validator, payload, message_fragment):
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
[
|
||||
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
|
||||
]
|
||||
"""
|
||||
)
|
||||
proposals = [
|
||||
CorrectionProposal(
|
||||
proposal_index=0,
|
||||
module_instance="homophones",
|
||||
module_key="homophones",
|
||||
id=1,
|
||||
original_text="gestures",
|
||||
corrected_text="Jesters",
|
||||
confidence=0.95,
|
||||
)
|
||||
]
|
||||
client = FakeStructuredLLMClient([payload])
|
||||
|
||||
with pytest.raises(AuditaLLMError, match=message_fragment):
|
||||
validator.validate(_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path))
|
||||
|
||||
|
||||
def test_llm_validator_rejects_unpreviewable_proposals_without_calling_llm(tmp_path):
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
[
|
||||
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
|
||||
]
|
||||
"""
|
||||
)
|
||||
proposals = [
|
||||
CorrectionProposal(
|
||||
proposal_index=0,
|
||||
module_instance="homophones",
|
||||
module_key="homophones",
|
||||
id=1,
|
||||
original_text="rank",
|
||||
corrected_text="Hrank",
|
||||
confidence=0.95,
|
||||
)
|
||||
]
|
||||
client = FakeStructuredLLMClient([])
|
||||
|
||||
result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
|
||||
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
|
||||
)
|
||||
|
||||
assert result.decisions[0].approved is False
|
||||
assert result.decisions[0].reason == "proposal original_text does not match segment text"
|
||||
assert client.calls == []
|
||||
|
||||
|
||||
def test_llm_validator_skips_empty_candidate_sets_without_calling_llm(tmp_path):
|
||||
client = FakeStructuredLLMClient([])
|
||||
|
||||
result = MeaningReversalValidator("meaning_reversal_review").validate(
|
||||
_context(proposals=[], transcript=[], llm_client=client, tmp_path=tmp_path)
|
||||
)
|
||||
|
||||
assert result.decisions == []
|
||||
assert client.calls == []
|
||||
|
||||
|
||||
def test_spoken_form_prompt_emphasizes_acoustic_plausibility():
|
||||
messages = build_spoken_form_plausibility_messages(
|
||||
[
|
||||
{
|
||||
"correction_index": 0,
|
||||
"id": 1,
|
||||
"original_segment_text": "There were gestures at the temple.",
|
||||
"corrected_segment_text": "There were Jesters at the temple.",
|
||||
"original_text": "gestures",
|
||||
"corrected_text": "Jesters",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
combined = messages[0]["content"] + messages[1]["content"]
|
||||
assert "homophone" in combined
|
||||
assert "gestures" in combined
|
||||
assert "Lyra" in combined
|
||||
assert '"original_segment_text"' in messages[1]["content"]
|
||||
|
||||
|
||||
def test_meaning_reversal_prompt_emphasizes_antonyms_and_segment_context():
|
||||
messages = build_meaning_reversal_messages(
|
||||
[
|
||||
{
|
||||
"correction_index": 0,
|
||||
"id": 1,
|
||||
"original_segment_text": "The figure became visible in the doorway.",
|
||||
"corrected_segment_text": "The figure became invisible in the doorway.",
|
||||
"original_text": "visible",
|
||||
"corrected_text": "invisible",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
combined = messages[0]["content"] + messages[1]["content"]
|
||||
assert "antonym" in combined
|
||||
assert "visible" in combined
|
||||
assert "up" in combined
|
||||
assert "original_segment_text" in messages[1]["content"]
|
||||
|
||||
|
||||
def test_llm_validators_use_shared_token_batching_helper(monkeypatch, tmp_path):
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
[
|
||||
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."},
|
||||
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "We saw rank near the gate."},
|
||||
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "ChatGPT still can't do that with a dam."}
|
||||
]
|
||||
"""
|
||||
)
|
||||
proposals = [
|
||||
CorrectionProposal(
|
||||
proposal_index=0,
|
||||
module_instance="homophones",
|
||||
module_key="homophones",
|
||||
id=1,
|
||||
original_text="gestures",
|
||||
corrected_text="Jesters",
|
||||
confidence=0.95,
|
||||
),
|
||||
CorrectionProposal(
|
||||
proposal_index=1,
|
||||
module_instance="homophones",
|
||||
module_key="homophones",
|
||||
id=2,
|
||||
original_text="rank",
|
||||
corrected_text="Hrank",
|
||||
confidence=0.95,
|
||||
),
|
||||
CorrectionProposal(
|
||||
proposal_index=2,
|
||||
module_instance="homophones",
|
||||
module_key="homophones",
|
||||
id=3,
|
||||
original_text="dam",
|
||||
corrected_text="damn",
|
||||
confidence=0.95,
|
||||
),
|
||||
]
|
||||
client = FakeStructuredLLMClient(
|
||||
[
|
||||
{
|
||||
"validations": [
|
||||
{
|
||||
"correction_index": 0,
|
||||
"approved": True,
|
||||
"confidence": 0.9,
|
||||
"reason": "ok",
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"validations": [
|
||||
{
|
||||
"correction_index": 1,
|
||||
"approved": True,
|
||||
"confidence": 0.9,
|
||||
"reason": "ok",
|
||||
},
|
||||
{
|
||||
"correction_index": 2,
|
||||
"approved": True,
|
||||
"confidence": 0.9,
|
||||
"reason": "ok",
|
||||
},
|
||||
]
|
||||
},
|
||||
]
|
||||
)
|
||||
chunk_calls = []
|
||||
|
||||
def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message):
|
||||
chunk_calls.append(
|
||||
{
|
||||
"max_tokens": max_tokens,
|
||||
"payloads": [payload_fn(item) for item in items],
|
||||
"empty_error_message": empty_error_message,
|
||||
}
|
||||
)
|
||||
return [
|
||||
TokenBatch(batch_index=0, items=list(items[:1]), token_count=1),
|
||||
TokenBatch(batch_index=1, items=list(items[1:]), token_count=1),
|
||||
]
|
||||
|
||||
monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items)
|
||||
|
||||
result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
|
||||
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
|
||||
)
|
||||
|
||||
assert [decision.approved for decision in result.decisions] == [True, True, True]
|
||||
assert len(client.calls) == 2
|
||||
assert len(chunk_calls) == 1
|
||||
assert chunk_calls[0]["max_tokens"] == AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}).max_section_tokens
|
||||
assert all("corrected_segment_text" in payload for payload in chunk_calls[0]["payloads"])
|
||||
@@ -76,8 +76,8 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
|
||||
assert (result.run_dir / "normalization" / "summary.json").exists()
|
||||
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
|
||||
"protected_glossary_guard",
|
||||
"toward_glossary_term_review",
|
||||
"context_support_review",
|
||||
"spoken_form_plausibility_review",
|
||||
"meaning_reversal_review",
|
||||
]
|
||||
|
||||
|
||||
@@ -97,26 +97,18 @@ def test_default_module_specs_expose_final_validator_order():
|
||||
|
||||
assert [validator.name for validator in specs[0].module.validators()] == [
|
||||
"protected_glossary_guard",
|
||||
"toward_glossary_term_review",
|
||||
"context_support_review",
|
||||
"spoken_form_plausibility_review",
|
||||
"meaning_reversal_review",
|
||||
]
|
||||
assert [validator.name for validator in specs[1].module.validators()] == [
|
||||
"protected_glossary_guard",
|
||||
"acoustic_similarity_review",
|
||||
"contextual_plausibility_review",
|
||||
"antonym_reversal_review",
|
||||
"spoken_form_plausibility_review",
|
||||
"meaning_reversal_review",
|
||||
]
|
||||
assert [validator.name for validator in specs[2].module.validators()] == [
|
||||
"protected_glossary_guard",
|
||||
"toward_glossary_term_review",
|
||||
"context_support_review",
|
||||
]
|
||||
assert [validator.name for validator in specs[3].module.validators()] == [
|
||||
"protected_glossary_guard",
|
||||
"spoken_marker_cleanup_review",
|
||||
"meaning_preservation_review",
|
||||
]
|
||||
assert [validator.name for validator in specs[4].module.validators()] == [
|
||||
"protected_glossary_guard",
|
||||
"edited_text_readability_review",
|
||||
"spoken_form_plausibility_review",
|
||||
"meaning_reversal_review",
|
||||
]
|
||||
assert [validator.name for validator in specs[3].module.validators()] == ["protected_glossary_guard"]
|
||||
assert [validator.name for validator in specs[4].module.validators()] == ["protected_glossary_guard"]
|
||||
|
||||
Reference in New Issue
Block a user