454 lines
15 KiB
Python
454 lines
15 KiB
Python
import pytest
|
|
|
|
from audita.core.chunking import TokenBatch
|
|
from audita.core.config import AuditaConfig
|
|
from audita.core.errors import AuditaLLMError
|
|
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
|
|
from audita.framework.models import CorrectionProposal, ModuleRunSpec
|
|
from audita.validators.base import ValidationContext
|
|
from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator
|
|
from audita.validators.prompts import (
|
|
build_meaning_reversal_messages,
|
|
build_spoken_form_plausibility_messages,
|
|
)
|
|
import audita.validators.llm as llm_module
|
|
|
|
|
|
class _Module:
|
|
def __init__(self, replacement_policy: str) -> None:
|
|
self.replacement_policy = replacement_policy
|
|
|
|
|
|
class FakeStructuredLLMClient:
|
|
def __init__(self, responses):
|
|
self._responses = list(responses)
|
|
self.calls = []
|
|
|
|
def run_structured(self, *, stage_name, messages, response_model, config):
|
|
self.calls.append(
|
|
{
|
|
"stage_name": stage_name,
|
|
"messages": list(messages),
|
|
"response_model": response_model,
|
|
}
|
|
)
|
|
if not self._responses:
|
|
raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.")
|
|
return response_model.model_validate(self._responses.pop(0))
|
|
|
|
|
|
def _glossary():
|
|
return parse_glossary_yaml(
|
|
"""
|
|
glossary:
|
|
- name: "Jesters"
|
|
category: faction
|
|
summary: "The Jesters are a faction."
|
|
- name: "Hrank"
|
|
category: pc
|
|
summary: "Hrank is a player character."
|
|
- name: "Lyra"
|
|
category: npc
|
|
summary: "Lyra is an NPC."
|
|
"""
|
|
)
|
|
|
|
|
|
def _context(
|
|
*,
|
|
proposals,
|
|
transcript,
|
|
llm_client,
|
|
tmp_path,
|
|
replacement_policy="require_unique",
|
|
):
|
|
return ValidationContext(
|
|
proposals=proposals,
|
|
transcript=transcript,
|
|
glossary=_glossary(),
|
|
config=AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}),
|
|
run_spec=ModuleRunSpec(
|
|
instance_name="homophones",
|
|
module_key="homophones",
|
|
module=_Module(replacement_policy),
|
|
),
|
|
run_dir=tmp_path,
|
|
llm_client=llm_client,
|
|
)
|
|
|
|
|
|
def test_spoken_form_plausibility_validator_approves_plausible_and_rejects_implausible_corrections(tmp_path):
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."},
|
|
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "Lyra moved first."}
|
|
]
|
|
"""
|
|
)
|
|
proposals = [
|
|
CorrectionProposal(
|
|
proposal_index=0,
|
|
module_instance="homophones",
|
|
module_key="homophones",
|
|
id=1,
|
|
original_text="gestures",
|
|
corrected_text="Jesters",
|
|
confidence=0.95,
|
|
),
|
|
CorrectionProposal(
|
|
proposal_index=1,
|
|
module_instance="homophones",
|
|
module_key="homophones",
|
|
id=2,
|
|
original_text="Lyra",
|
|
corrected_text="Jesters",
|
|
confidence=0.95,
|
|
),
|
|
]
|
|
client = FakeStructuredLLMClient(
|
|
[
|
|
{
|
|
"validations": [
|
|
{
|
|
"correction_index": 0,
|
|
"approved": True,
|
|
"confidence": 0.97,
|
|
"reason": "Likely spoken-form correction in context.",
|
|
},
|
|
{
|
|
"correction_index": 1,
|
|
"approved": False,
|
|
"confidence": 0.99,
|
|
"reason": "Not plausibly related by homophone or mistranscription.",
|
|
},
|
|
]
|
|
}
|
|
]
|
|
)
|
|
|
|
result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
|
|
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
|
|
)
|
|
|
|
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
|
|
(0, True),
|
|
(1, False),
|
|
]
|
|
prompt_text = client.calls[0]["messages"][1]["content"]
|
|
assert '"original_segment_text"' in prompt_text
|
|
assert "There were Jesters at the temple." in prompt_text
|
|
assert "Lyra moved first." in prompt_text
|
|
|
|
|
|
def test_meaning_reversal_validator_rejects_reversal_and_approves_nonreversal(tmp_path):
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam, but Claude actually can."},
|
|
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "The figure became visible in the doorway."}
|
|
]
|
|
"""
|
|
)
|
|
proposals = [
|
|
CorrectionProposal(
|
|
proposal_index=0,
|
|
module_instance="homophones",
|
|
module_key="homophones",
|
|
id=1,
|
|
original_text="dam",
|
|
corrected_text="damn",
|
|
confidence=0.95,
|
|
),
|
|
CorrectionProposal(
|
|
proposal_index=1,
|
|
module_instance="homophones",
|
|
module_key="homophones",
|
|
id=2,
|
|
original_text="visible",
|
|
corrected_text="invisible",
|
|
confidence=0.95,
|
|
),
|
|
]
|
|
client = FakeStructuredLLMClient(
|
|
[
|
|
{
|
|
"validations": [
|
|
{
|
|
"correction_index": 0,
|
|
"approved": True,
|
|
"confidence": 0.94,
|
|
"reason": "Does not reverse the segment meaning.",
|
|
},
|
|
{
|
|
"correction_index": 1,
|
|
"approved": False,
|
|
"confidence": 1.0,
|
|
"reason": "Changes visible to invisible and reverses the segment meaning.",
|
|
},
|
|
]
|
|
}
|
|
]
|
|
)
|
|
|
|
result = MeaningReversalValidator("meaning_reversal_review").validate(
|
|
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
|
|
)
|
|
|
|
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
|
|
(0, True),
|
|
(1, False),
|
|
]
|
|
prompt_text = client.calls[0]["messages"][1]["content"]
|
|
assert "The figure became visible in the doorway." in prompt_text
|
|
assert "The figure became invisible in the doorway." in prompt_text
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("validator", "payload", "message_fragment"),
|
|
[
|
|
(
|
|
SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
|
|
{
|
|
"validations": [
|
|
{
|
|
"correction_index": 0,
|
|
"approved": True,
|
|
"confidence": 0.9,
|
|
"reason": "ok",
|
|
},
|
|
{
|
|
"correction_index": 0,
|
|
"approved": True,
|
|
"confidence": 0.9,
|
|
"reason": "duplicate",
|
|
},
|
|
]
|
|
},
|
|
"duplicate correction_index values",
|
|
),
|
|
(
|
|
MeaningReversalValidator("meaning_reversal_review"),
|
|
{"validations": []},
|
|
"omitted correction_index values",
|
|
),
|
|
(
|
|
SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
|
|
{
|
|
"validations": [
|
|
{
|
|
"correction_index": 99,
|
|
"approved": True,
|
|
"confidence": 0.9,
|
|
"reason": "unknown",
|
|
}
|
|
]
|
|
},
|
|
"unknown correction_index",
|
|
),
|
|
],
|
|
)
|
|
def test_llm_validators_reject_bad_correction_indexes(tmp_path, validator, payload, message_fragment):
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
|
|
]
|
|
"""
|
|
)
|
|
proposals = [
|
|
CorrectionProposal(
|
|
proposal_index=0,
|
|
module_instance="homophones",
|
|
module_key="homophones",
|
|
id=1,
|
|
original_text="gestures",
|
|
corrected_text="Jesters",
|
|
confidence=0.95,
|
|
)
|
|
]
|
|
client = FakeStructuredLLMClient([payload])
|
|
|
|
with pytest.raises(AuditaLLMError, match=message_fragment):
|
|
validator.validate(_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path))
|
|
|
|
|
|
def test_llm_validator_rejects_unpreviewable_proposals_without_calling_llm(tmp_path):
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
|
|
]
|
|
"""
|
|
)
|
|
proposals = [
|
|
CorrectionProposal(
|
|
proposal_index=0,
|
|
module_instance="homophones",
|
|
module_key="homophones",
|
|
id=1,
|
|
original_text="rank",
|
|
corrected_text="Hrank",
|
|
confidence=0.95,
|
|
)
|
|
]
|
|
client = FakeStructuredLLMClient([])
|
|
|
|
result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
|
|
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
|
|
)
|
|
|
|
assert result.decisions[0].approved is False
|
|
assert result.decisions[0].reason == "proposal original_text does not match segment text"
|
|
assert client.calls == []
|
|
|
|
|
|
def test_llm_validator_skips_empty_candidate_sets_without_calling_llm(tmp_path):
|
|
client = FakeStructuredLLMClient([])
|
|
|
|
result = MeaningReversalValidator("meaning_reversal_review").validate(
|
|
_context(proposals=[], transcript=[], llm_client=client, tmp_path=tmp_path)
|
|
)
|
|
|
|
assert result.decisions == []
|
|
assert client.calls == []
|
|
|
|
|
|
def test_spoken_form_prompt_emphasizes_acoustic_plausibility():
|
|
messages = build_spoken_form_plausibility_messages(
|
|
[
|
|
{
|
|
"correction_index": 0,
|
|
"id": 1,
|
|
"original_segment_text": "There were gestures at the temple.",
|
|
"corrected_segment_text": "There were Jesters at the temple.",
|
|
"original_text": "gestures",
|
|
"corrected_text": "Jesters",
|
|
}
|
|
]
|
|
)
|
|
|
|
combined = messages[0]["content"] + messages[1]["content"]
|
|
assert "homophone" in combined
|
|
assert "gestures" in combined
|
|
assert "Lyra" in combined
|
|
assert '"original_segment_text"' in messages[1]["content"]
|
|
|
|
|
|
def test_meaning_reversal_prompt_emphasizes_antonyms_and_segment_context():
|
|
messages = build_meaning_reversal_messages(
|
|
[
|
|
{
|
|
"correction_index": 0,
|
|
"id": 1,
|
|
"original_segment_text": "The figure became visible in the doorway.",
|
|
"corrected_segment_text": "The figure became invisible in the doorway.",
|
|
"original_text": "visible",
|
|
"corrected_text": "invisible",
|
|
}
|
|
]
|
|
)
|
|
|
|
combined = messages[0]["content"] + messages[1]["content"]
|
|
assert "antonym" in combined
|
|
assert "visible" in combined
|
|
assert "up" in combined
|
|
assert "original_segment_text" in messages[1]["content"]
|
|
|
|
|
|
def test_llm_validators_use_shared_token_batching_helper(monkeypatch, tmp_path):
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."},
|
|
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "We saw rank near the gate."},
|
|
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "ChatGPT still can't do that with a dam."}
|
|
]
|
|
"""
|
|
)
|
|
proposals = [
|
|
CorrectionProposal(
|
|
proposal_index=0,
|
|
module_instance="homophones",
|
|
module_key="homophones",
|
|
id=1,
|
|
original_text="gestures",
|
|
corrected_text="Jesters",
|
|
confidence=0.95,
|
|
),
|
|
CorrectionProposal(
|
|
proposal_index=1,
|
|
module_instance="homophones",
|
|
module_key="homophones",
|
|
id=2,
|
|
original_text="rank",
|
|
corrected_text="Hrank",
|
|
confidence=0.95,
|
|
),
|
|
CorrectionProposal(
|
|
proposal_index=2,
|
|
module_instance="homophones",
|
|
module_key="homophones",
|
|
id=3,
|
|
original_text="dam",
|
|
corrected_text="damn",
|
|
confidence=0.95,
|
|
),
|
|
]
|
|
client = FakeStructuredLLMClient(
|
|
[
|
|
{
|
|
"validations": [
|
|
{
|
|
"correction_index": 0,
|
|
"approved": True,
|
|
"confidence": 0.9,
|
|
"reason": "ok",
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"validations": [
|
|
{
|
|
"correction_index": 1,
|
|
"approved": True,
|
|
"confidence": 0.9,
|
|
"reason": "ok",
|
|
},
|
|
{
|
|
"correction_index": 2,
|
|
"approved": True,
|
|
"confidence": 0.9,
|
|
"reason": "ok",
|
|
},
|
|
]
|
|
},
|
|
]
|
|
)
|
|
chunk_calls = []
|
|
|
|
def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message):
|
|
chunk_calls.append(
|
|
{
|
|
"max_tokens": max_tokens,
|
|
"payloads": [payload_fn(item) for item in items],
|
|
"empty_error_message": empty_error_message,
|
|
}
|
|
)
|
|
return [
|
|
TokenBatch(batch_index=0, items=list(items[:1]), token_count=1),
|
|
TokenBatch(batch_index=1, items=list(items[1:]), token_count=1),
|
|
]
|
|
|
|
monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items)
|
|
|
|
result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
|
|
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
|
|
)
|
|
|
|
assert [decision.approved for decision in result.decisions] == [True, True, True]
|
|
assert len(client.calls) == 2
|
|
assert len(chunk_calls) == 1
|
|
assert chunk_calls[0]["max_tokens"] == AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}).max_section_tokens
|
|
assert all("corrected_segment_text" in payload for payload in chunk_calls[0]["payloads"])
|