Files
audita/tests/test_llm_validators.py

1425 lines
46 KiB
Python

import threading
import pytest
from audita.core.chunking import TokenBatch
from audita.core.config import AuditaConfig, ConfigOverrides
from audita.core.errors import AuditaLLMError
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
from audita.framework.models import CorrectionProposal, ModuleRunSpec
from audita.validators import (
GrammarOnlyValidator,
IdenticalTextValidator,
NonEmptySegmentValidator,
OriginalTextPresentValidator,
)
from audita.validators.base import ValidationContext
from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
from audita.validators.prompts import (
build_grammar_only_messages,
build_meaning_reversal_messages,
build_spoken_form_plausibility_messages,
build_spoken_word_messages,
)
import audita.validators.llm as llm_module
class _Module:
def __init__(self, replacement_policy: str) -> None:
self.replacement_policy = replacement_policy
class FakeStructuredLLMClient:
def __init__(self, responses):
self._responses = list(responses)
self.calls = []
def run_structured(self, *, stage_name, messages, response_model, config):
self.calls.append(
{
"stage_name": stage_name,
"messages": list(messages),
"response_model": response_model,
"config": config,
}
)
if not self._responses:
raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.")
return response_model.model_validate(self._responses.pop(0))
class CoordinatedStructuredLLMClient:
def __init__(self, responses, barrier):
self._responses = list(responses)
self._barrier = barrier
self._lock = threading.Lock()
self.calls = []
def run_structured(self, *, stage_name, messages, response_model, config):
self._barrier.wait()
with self._lock:
self.calls.append(
{
"stage_name": stage_name,
"messages": list(messages),
"response_model": response_model,
"config": config,
}
)
if not self._responses:
raise AuditaLLMError("CoordinatedStructuredLLMClient received more calls than expected.")
payload = self._responses.pop(0)
if callable(payload):
payload = payload(stage_name=stage_name, messages=messages, response_model=response_model, config=config)
return response_model.model_validate(payload)
def _glossary():
return parse_glossary_yaml(
"""
glossary:
- name: "Jesters"
category: faction
summary: "The Jesters are a faction."
- name: "Hrank"
category: pc
summary: "Hrank is a player character."
- name: "Lyra"
category: npc
summary: "Lyra is an NPC."
"""
)
def _context(
*,
proposals,
transcript,
llm_client,
tmp_path,
replacement_policy="require_unique",
llm_concurrency=1,
validation_llm_concurrency=None,
):
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "test-key"},
overrides=ConfigOverrides(
llm_concurrency=llm_concurrency,
validation_llm_concurrency=validation_llm_concurrency,
),
)
return ValidationContext(
proposals=proposals,
transcript=transcript,
glossary=_glossary(),
config=config.validation_llm_config(),
run_spec=ModuleRunSpec(
instance_name="homophones",
module_key="homophones",
module=_Module(replacement_policy),
),
run_dir=tmp_path,
llm_client=llm_client,
)
def test_spoken_form_plausibility_validator_approves_plausible_and_rejects_implausible_corrections(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple.", "categories": ["narration"]},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "Lyra moved first.", "categories": ["combat"]}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="homophones",
module_key="homophones",
id=2,
original_text="Lyra",
corrected_text="Jesters",
confidence=0.95,
),
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.97,
"reason": "Likely spoken-form correction in context.",
},
{
"correction_index": 1,
"approved": False,
"confidence": 0.99,
"reason": "Not plausibly related by homophone or mistranscription.",
},
]
}
]
)
result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
(0, True),
(1, False),
]
prompt_text = client.calls[0]["messages"][1]["content"]
assert '"original_segment_text"' in prompt_text
assert '"categories"' in prompt_text
assert "There were Jesters at the temple." in prompt_text
assert "Lyra moved first." in prompt_text
def test_meaning_reversal_validator_rejects_reversal_and_approves_nonreversal(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam, but Claude actually can."},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "The figure became visible in the doorway."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="dam",
corrected_text="damn",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="homophones",
module_key="homophones",
id=2,
original_text="visible",
corrected_text="invisible",
confidence=0.95,
),
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.94,
"reason": "Does not reverse the segment meaning.",
},
{
"correction_index": 1,
"approved": False,
"confidence": 1.0,
"reason": "Changes visible to invisible and reverses the segment meaning.",
},
]
}
]
)
result = MeaningReversalValidator("meaning_reversal_review").validate(
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
(0, True),
(1, False),
]
prompt_text = client.calls[0]["messages"][1]["content"]
assert "The figure became visible in the doorway." in prompt_text
assert "The figure became invisible in the doorway." in prompt_text
def test_spoken_word_validator_approves_cleanup_and_rejects_rewrite(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "We should maybe proceed carefully."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="spoken_word",
module_key="spoken_word",
id=1,
original_text="I, uh, I think",
corrected_text="I think",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="spoken_word",
module_key="spoken_word",
id=2,
original_text="maybe proceed carefully",
corrected_text="go now",
confidence=0.95,
),
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.97,
"reason": "Reasonable dysfluency cleanup that preserves meaning.",
},
{
"correction_index": 1,
"approved": False,
"confidence": 0.99,
"reason": "This changes the substance of the segment rather than cleaning a dysfluency.",
},
]
}
]
)
result = SpokenWordValidator("spoken_word_review").validate(
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
(0, True),
(1, False),
]
prompt_text = client.calls[0]["messages"][1]["content"]
assert "I think we should go." in prompt_text
assert "go now" in prompt_text
def test_spoken_word_validator_allows_punctuation_cleanup_tied_to_dysfluency(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Well ... I think we should go."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="spoken_word",
module_key="spoken_word",
id=1,
original_text="Well ... ",
corrected_text="",
confidence=0.95,
)
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.95,
"reason": "Removes a hesitation artifact without changing substantive meaning.",
}
]
}
]
)
result = SpokenWordValidator("spoken_word_review").validate(
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)]
def test_grammar_only_validator_approves_conservative_grammar_cleanup(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello there"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "cant we go"},
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "Hello,world"}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="grammar",
module_key="grammar",
id=1,
original_text="hello there",
corrected_text="Hello there.",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="grammar",
module_key="grammar",
id=2,
original_text="cant",
corrected_text="can't",
confidence=0.95,
),
CorrectionProposal(
proposal_index=2,
module_instance="grammar",
module_key="grammar",
id=3,
original_text="Hello,world",
corrected_text="Hello, world",
confidence=0.95,
),
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.98,
"reason": "Conservative punctuation and capitalization cleanup.",
},
{
"correction_index": 1,
"approved": True,
"confidence": 0.95,
"reason": "Conservative apostrophe insertion within grammar cleanup.",
},
{
"correction_index": 2,
"approved": True,
"confidence": 0.97,
"reason": "Conservative spacing cleanup.",
},
]
}
]
)
result = GrammarOnlyValidator("grammar_only_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
(0, True),
(1, True),
(2, True),
]
prompt_text = client.calls[0]["messages"][1]["content"]
assert "whole-word article cleanup" in prompt_text
assert "embedded within an otherwise grammatical revision" in prompt_text
assert "filler cleanup" in prompt_text
def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "a intelligence saving throw"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "an owl"}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="grammar",
module_key="grammar",
id=1,
original_text="a intelligence saving throw",
corrected_text="an intelligence saving throw",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="grammar",
module_key="grammar",
id=2,
original_text="an owl",
corrected_text="a owl",
confidence=0.95,
),
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.96,
"reason": "Allowed whole-word article cleanup.",
},
{
"correction_index": 1,
"approved": True,
"confidence": 0.93,
"reason": "Allowed whole-word article cleanup.",
},
]
}
]
)
result = GrammarOnlyValidator("grammar_only_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
(0, True),
(1, True),
]
def test_grammar_only_validator_rejects_out_of_scope_rewrites(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "their plan"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "dam"},
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "uh"},
{"id": 4, "speaker": "A", "start": 3.0, "end": 4.0, "text": "I I agree"},
{"id": 5, "speaker": "A", "start": 4.0, "end": 5.0, "text": "Eric"}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="grammar",
module_key="grammar",
id=1,
original_text="their",
corrected_text="there",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="grammar",
module_key="grammar",
id=2,
original_text="dam",
corrected_text="damn",
confidence=0.95,
),
CorrectionProposal(
proposal_index=2,
module_instance="grammar",
module_key="grammar",
id=3,
original_text="uh",
corrected_text="",
confidence=0.95,
),
CorrectionProposal(
proposal_index=3,
module_instance="grammar",
module_key="grammar",
id=4,
original_text="I I",
corrected_text="I",
confidence=0.95,
),
CorrectionProposal(
proposal_index=4,
module_instance="grammar",
module_key="grammar",
id=5,
original_text="Eric",
corrected_text="Eric's",
confidence=0.95,
),
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": False,
"confidence": 0.99,
"reason": "Unrelated word substitution rather than conservative grammar cleanup.",
},
{
"correction_index": 1,
"approved": False,
"confidence": 0.99,
"reason": "Free-standing homophone rewrite rather than conservative grammar cleanup.",
},
{
"correction_index": 2,
"approved": False,
"confidence": 1.0,
"reason": "Spoken-word filler cleanup is out of scope for the grammar module.",
},
{
"correction_index": 3,
"approved": False,
"confidence": 1.0,
"reason": "Spoken-word repetition cleanup is out of scope for the grammar module.",
},
{
"correction_index": 4,
"approved": False,
"confidence": 0.98,
"reason": "Possessive rewrite goes beyond conservative grammar cleanup.",
},
]
}
]
)
result = GrammarOnlyValidator("grammar_only_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert [decision.approved for decision in result.decisions] == [False, False, False, False, False]
def test_grammar_only_validator_allows_embedded_homophone_fix_within_grammar_revision(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't"}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="grammar",
module_key="grammar",
id=1,
original_text="question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't",
corrected_text="question: If he dies, does he stay there? The way that, yeah, the way that it's written, it's like, so if it stops, then he goes down; but what if it doesn't?",
confidence=0.95,
)
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.94,
"reason": "Primarily a grammatical revision with an embedded likely mistranscription recovery.",
}
]
}
]
)
result = GrammarOnlyValidator("grammar_only_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)]
def test_non_empty_segment_validator_rejects_empty_and_whitespace_only_segments(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "uh"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "um"}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="spoken_word",
module_key="spoken_word",
id=1,
original_text="uh",
corrected_text="",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="spoken_word",
module_key="spoken_word",
id=2,
original_text="um",
corrected_text=" ",
confidence=0.95,
),
]
result = NonEmptySegmentValidator("non_empty_segment_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
)
assert [decision.approved for decision in result.decisions] == [False, False]
assert all(decision.reason == "correction would leave the segment empty" for decision in result.decisions)
def test_non_empty_segment_validator_allows_punctuation_only_and_unpreviewable_proposals(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "uh"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "There were gestures at the temple."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="spoken_word",
module_key="spoken_word",
id=1,
original_text="uh",
corrected_text=".",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="spoken_word",
module_key="spoken_word",
id=2,
original_text="rank",
corrected_text="Hrank",
confidence=0.95,
),
]
result = NonEmptySegmentValidator("non_empty_segment_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved, decision.reason) for decision in result.decisions] == [
(0, True, None),
(1, True, None),
]
def test_identical_text_validator_rejects_exact_noops_only(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "hello"}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="grammar",
module_key="grammar",
id=1,
original_text="hello",
corrected_text="hello",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="grammar",
module_key="grammar",
id=2,
original_text="hello",
corrected_text="Hello",
confidence=0.95,
),
]
result = IdenticalTextValidator("identical_text_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved, decision.reason) for decision in result.decisions] == [
(0, False, "proposal original_text and corrected_text are identical"),
(1, True, None),
]
def test_original_text_present_validator_rejects_missing_spans_and_allows_present_or_unknown_segments(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello hello"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "goodbye"}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="hello",
corrected_text="hi",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="homophones",
module_key="homophones",
id=2,
original_text="rank",
corrected_text="Hrank",
confidence=0.95,
),
CorrectionProposal(
proposal_index=2,
module_instance="homophones",
module_key="homophones",
id=99,
original_text="missing",
corrected_text="present",
confidence=0.95,
),
]
result = OriginalTextPresentValidator("original_text_present_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved, decision.reason) for decision in result.decisions] == [
(0, True, None),
(1, False, "proposal original_text does not match segment text"),
(2, True, None),
]
@pytest.mark.parametrize(
("validator", "payload", "message_fragment"),
[
(
SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.9,
"reason": "ok",
},
{
"correction_index": 0,
"approved": True,
"confidence": 0.9,
"reason": "duplicate",
},
]
},
"duplicate correction_index values",
),
(
MeaningReversalValidator("meaning_reversal_review"),
{"validations": []},
"omitted correction_index values",
),
(
SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
{
"validations": [
{
"correction_index": 99,
"approved": True,
"confidence": 0.9,
"reason": "unknown",
}
]
},
"unknown correction_index",
),
(
SpokenWordValidator("spoken_word_review"),
{
"validations": [
{
"correction_index": 99,
"approved": True,
"confidence": 0.9,
"reason": "unknown",
}
]
},
"unknown correction_index",
),
],
)
def test_llm_validators_reject_bad_correction_indexes(tmp_path, validator, payload, message_fragment):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
)
]
client = FakeStructuredLLMClient([payload])
with pytest.raises(AuditaLLMError, match=message_fragment):
validator.validate(_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path))
def test_llm_validator_rejects_unpreviewable_proposals_without_calling_llm(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="rank",
corrected_text="Hrank",
confidence=0.95,
)
]
client = FakeStructuredLLMClient([])
result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert result.decisions[0].approved is False
assert result.decisions[0].reason == "proposal original_text does not match segment text"
assert client.calls == []
def test_llm_validator_skips_empty_candidate_sets_without_calling_llm(tmp_path):
client = FakeStructuredLLMClient([])
result = MeaningReversalValidator("meaning_reversal_review").validate(
_context(proposals=[], transcript=[], llm_client=client, tmp_path=tmp_path)
)
assert result.decisions == []
assert client.calls == []
def test_spoken_form_prompt_emphasizes_acoustic_plausibility():
messages = build_spoken_form_plausibility_messages(
[
{
"correction_index": 0,
"id": 1,
"original_segment_text": "There were gestures at the temple.",
"corrected_segment_text": "There were Jesters at the temple.",
"original_text": "gestures",
"corrected_text": "Jesters",
}
]
)
combined = messages[0]["content"] + messages[1]["content"]
assert "homophone" in combined
assert "gestures" in combined
assert "Lyra" in combined
assert '"original_segment_text"' in messages[1]["content"]
def test_meaning_reversal_prompt_emphasizes_antonyms_and_segment_context():
messages = build_meaning_reversal_messages(
[
{
"correction_index": 0,
"id": 1,
"original_segment_text": "The figure became visible in the doorway.",
"corrected_segment_text": "The figure became invisible in the doorway.",
"original_text": "visible",
"corrected_text": "invisible",
}
]
)
combined = messages[0]["content"] + messages[1]["content"]
assert "antonym" in combined
assert "visible" in combined
assert "up" in combined
assert "original_segment_text" in messages[1]["content"]
def test_spoken_word_prompt_emphasizes_dysfluency_cleanup():
messages = build_spoken_word_messages(
[
{
"correction_index": 0,
"id": 1,
"original_segment_text": "I, uh, I think we should go.",
"corrected_segment_text": "I think we should go.",
"original_text": "I, uh, I think",
"corrected_text": "I think",
}
]
)
combined = messages[0]["content"] + messages[1]["content"]
assert "dysfluencies" in combined
assert "intentional emphasis" in combined
assert "Stop! Stop! Stop!" in combined
assert "punctuation" in combined
assert "substantive meaning" in combined
assert "original_segment_text" in messages[1]["content"]
def test_llm_validators_use_shared_token_batching_helper(monkeypatch, tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "We saw rank near the gate."},
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "ChatGPT still can't do that with a dam."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="homophones",
module_key="homophones",
id=2,
original_text="rank",
corrected_text="Hrank",
confidence=0.95,
),
CorrectionProposal(
proposal_index=2,
module_instance="homophones",
module_key="homophones",
id=3,
original_text="dam",
corrected_text="damn",
confidence=0.95,
),
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.9,
"reason": "ok",
}
]
},
{
"validations": [
{
"correction_index": 1,
"approved": True,
"confidence": 0.9,
"reason": "ok",
},
{
"correction_index": 2,
"approved": True,
"confidence": 0.9,
"reason": "ok",
},
]
},
]
)
chunk_calls = []
def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message):
chunk_calls.append(
{
"max_tokens": max_tokens,
"payloads": [payload_fn(item) for item in items],
"empty_error_message": empty_error_message,
}
)
return [
TokenBatch(batch_index=0, items=list(items[:1]), token_count=1),
TokenBatch(batch_index=1, items=list(items[1:]), token_count=1),
]
monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items)
result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert [decision.approved for decision in result.decisions] == [True, True, True]
assert len(client.calls) == 2
assert len(chunk_calls) == 1
assert (
chunk_calls[0]["max_tokens"]
== AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}).validation_max_prompt_tokens
)
assert all("corrected_segment_text" in payload for payload in chunk_calls[0]["payloads"])
def test_grammar_validation_prompt_is_scoped_to_conservative_cleanup():
messages = build_grammar_only_messages(
[
{
"correction_index": 0,
"id": 1,
"original_text": "dam",
"corrected_text": "damn",
"confidence": 0.95,
"original_segment_text": "ChatGPT still can't do that with a dam.",
"corrected_segment_text": "ChatGPT still can't do that with a damn.",
}
]
)
combined = messages[0]["content"] + messages[1]["content"]
assert "whole-word article cleanup" in combined
assert "embedded within an otherwise grammatical revision" in combined
assert "free-standing homophone" in combined
assert "spoken-word dysfluency cleanup" in combined
assert "original_segment_text" in messages[1]["content"]
def test_llm_validators_process_batches_concurrently_and_preserve_proposal_order(monkeypatch, tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "ChatGPT still can't do that with a dam."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="homophones",
module_key="homophones",
id=2,
original_text="dam",
corrected_text="damn",
confidence=0.95,
),
]
client = CoordinatedStructuredLLMClient(
[
lambda **kwargs: {
"validations": [
{
"correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1,
"approved": True,
"confidence": 0.97,
"reason": "ok",
}
]
},
lambda **kwargs: {
"validations": [
{
"correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1,
"approved": True,
"confidence": 0.94,
"reason": "ok",
}
]
},
],
threading.Barrier(2, timeout=1.0),
)
def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message):
return [
TokenBatch(batch_index=0, items=list(items[:1]), token_count=1),
TokenBatch(batch_index=1, items=list(items[1:]), token_count=1),
]
monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items)
result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
_context(
proposals=proposals,
transcript=transcript,
llm_client=client,
tmp_path=tmp_path,
llm_concurrency=2,
)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
(0, True),
(1, True),
]
assert len(client.calls) == 2
def test_validator_uses_validation_llm_config(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
)
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.97,
"reason": "ok",
}
]
}
]
)
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "primary-key"},
overrides=ConfigOverrides(
model="primary-model",
base_url="http://localhost:8000/v1",
max_retries=7,
llm_timeout_seconds=120,
validation_llm_api_key="validation-key",
validation_model="validation-model",
validation_base_url="http://localhost:9000/v1",
validation_max_retries=2,
validation_llm_timeout_seconds=240,
validation_llm_concurrency=3,
),
)
context = ValidationContext(
proposals=proposals,
transcript=transcript,
glossary=_glossary(),
config=config.validation_llm_config(),
run_spec=ModuleRunSpec(
instance_name="homophones",
module_key="homophones",
module=_Module("require_unique"),
),
run_dir=tmp_path,
llm_client=client,
)
SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(context)
validation_config = client.calls[0]["config"]
assert validation_config.api_key == "validation-key"
assert validation_config.model == "validation-model"
assert validation_config.base_url == "http://localhost:9000/v1"
assert validation_config.max_retries == 2
assert validation_config.llm_timeout_seconds == 240
assert validation_config.llm_concurrency == 3
def test_validator_batches_use_validation_max_prompt_tokens(tmp_path, monkeypatch):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
)
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.97,
"reason": "ok",
}
]
}
]
)
chunk_calls = []
def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message):
chunk_calls.append(
{
"count": len(items),
"max_tokens": max_tokens,
}
)
return [TokenBatch(batch_index=0, items=list(items), token_count=1)]
monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items)
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "test-key"},
overrides=ConfigOverrides(validation_max_prompt_tokens=1024),
)
context = ValidationContext(
proposals=proposals,
transcript=transcript,
glossary=_glossary(),
config=config.validation_llm_config(),
run_spec=ModuleRunSpec(
instance_name="homophones",
module_key="homophones",
module=_Module("require_unique"),
),
run_dir=tmp_path,
llm_client=client,
)
SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(context)
assert chunk_calls[0]["max_tokens"] == 1024
def test_validator_uses_validation_llm_concurrency_override(tmp_path, monkeypatch):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "ChatGPT still can't do that with a dam."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="homophones",
module_key="homophones",
id=2,
original_text="dam",
corrected_text="damn",
confidence=0.95,
),
]
client = CoordinatedStructuredLLMClient(
[
lambda **kwargs: {
"validations": [
{
"correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1,
"approved": True,
"confidence": 0.97,
"reason": "ok",
}
]
},
lambda **kwargs: {
"validations": [
{
"correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1,
"approved": True,
"confidence": 0.94,
"reason": "ok",
}
]
},
],
threading.Barrier(2, timeout=1.0),
)
def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message):
return [
TokenBatch(batch_index=0, items=list(items[:1]), token_count=1),
TokenBatch(batch_index=1, items=list(items[1:]), token_count=1),
]
monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items)
result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
_context(
proposals=proposals,
transcript=transcript,
llm_client=client,
tmp_path=tmp_path,
llm_concurrency=1,
validation_llm_concurrency=2,
)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
(0, True),
(1, True),
]
assert len(client.calls) == 2