eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Replaced the stub LLM-based validator class with two real LLM-based validator implementations

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-24 11:52:22 -05:00
parent bca2152971
commit 34b3c09e43
14 changed files with 918 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
src/audita/framework/llm.py
View File

@@ -1,11 +1,15 @@
from typing import Any, Sequence
from __future__ import annotations
from typing import Any, Optional, Sequence, Tuple
from audita.core.config import AuditaConfig
from audita.core.errors import AuditaLLMError
class OpenRouterStructuredLLMClient:
"""Placeholder for future structured LLM integration."""
def __init__(self) -> None:
self._client = None
self._client_identity: Optional[Tuple[str, str]] = None
def run_structured(
self,
@@ -15,6 +19,48 @@ class OpenRouterStructuredLLMClient:
response_model: Any,
config: AuditaConfig,
) -> Any:
raise AuditaLLMError(
f"Structured LLM stage '{stage_name}' is not implemented in the framework skeleton."
)
if not config.api_key:
raise AuditaLLMError(
f"Structured LLM stage '{stage_name}' requires OPENROUTER_API_KEY to be configured."
)
client = self._get_client(config)
model = _normalize_openrouter_model(config.model)
try:
return client.chat.completions.create(
model=model,
messages=list(messages),
response_model=response_model,
max_retries=config.max_retries,
extra_body={"provider": {"require_parameters": True}},
)
except Exception as exc:
raise AuditaLLMError(
f"Structured LLM stage '{stage_name}' failed. Confirm the configured OpenRouter model "
"supports tool calling or structured outputs."
) from exc
def _get_client(self, config: AuditaConfig) -> Any:
identity = (config.api_key or "", config.base_url)
if self._client is not None and self._client_identity == identity:
return self._client
try:
import instructor
from openai import OpenAI
except ImportError as exc:
raise AuditaLLMError(
"The LLM dependencies are not installed. Run `uv sync` before using Audita validators."
) from exc
openai_client = OpenAI(api_key=config.api_key, base_url=config.base_url)
self._client = instructor.patch(openai_client, mode=instructor.Mode.TOOLS)
self._client_identity = identity
return self._client
def _normalize_openrouter_model(model: str) -> str:
prefix = "openrouter/"
if model.startswith(prefix):
return model[len(prefix) :]
return model

79
src/audita/framework/proposals.py Normal file
View File

@@ -0,0 +1,79 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Optional, Sequence, Union
from audita.core.schemas import TranscriptSegment
if TYPE_CHECKING:
from .models import CorrectionProposal
ReplacementPolicy = str
@dataclass(frozen=True)
class ProposalPreview:
proposal: CorrectionProposal
segment_index: int
segment: TranscriptSegment
corrected_segment_text: str
@property
def original_segment_text(self) -> str:
return self.segment.text
def to_prompt_payload(self) -> dict:
return {
"correction_index": self.proposal.proposal_index,
"id": self.proposal.id,
"original_segment_text": self.original_segment_text,
"corrected_segment_text": self.corrected_segment_text,
"original_text": self.proposal.original_text,
"corrected_text": self.proposal.corrected_text,
}
@dataclass(frozen=True)
class ProposalPreviewError:
reason: str
actual_text: Optional[str] = None
def preview_proposal(
transcript: Sequence[TranscriptSegment],
proposal: CorrectionProposal,
replacement_policy: ReplacementPolicy,
) -> Union[ProposalPreview, ProposalPreviewError]:
index_by_id = {segment.id: index for index, segment in enumerate(transcript)}
segment_index = index_by_id.get(proposal.id)
if segment_index is None:
return ProposalPreviewError(reason="proposal references unknown segment id")
segment = transcript[segment_index]
if proposal.original_text == "":
return ProposalPreviewError(
reason="proposal original_text must not be empty",
actual_text=segment.text,
)
match_count = segment.text.count(proposal.original_text)
if match_count == 0:
return ProposalPreviewError(
reason="proposal original_text does not match segment text",
actual_text=segment.text,
)
if replacement_policy == "require_unique" and match_count != 1:
return ProposalPreviewError(
reason="proposal original_text must match exactly once",
actual_text=segment.text,
)
corrected_segment_text = segment.text.replace(proposal.original_text, proposal.corrected_text)
return ProposalPreview(
proposal=proposal,
segment_index=segment_index,
segment=segment,
corrected_segment_text=corrected_segment_text,
)

57
src/audita/framework/runner.py
View File

@@ -9,6 +9,7 @@ from audita.core.schemas import Glossary, TranscriptSegment
from audita.validators.base import ValidationContext, ValidationDecision, ValidationResult
from .models import CorrectionProposal, ModuleContext, ModuleRunSpec, StructuredLLMClient
from .proposals import ProposalPreviewError, preview_proposal
ProgressCallback = Callable[[str], None]
@@ -216,65 +217,23 @@ def _apply_proposal(
proposal: CorrectionProposal,
replacement_policy: str,
) -> Union[Tuple[List[TranscriptSegment], AppliedChange], ReportedSkip]:
index_by_id = {segment.id: index for index, segment in enumerate(transcript)}
segment_index = index_by_id.get(proposal.id)
if segment_index is None:
preview = preview_proposal(transcript, proposal, replacement_policy)
if isinstance(preview, ProposalPreviewError):
return ReportedSkip(
module_instance=proposal.module_instance,
module_key=proposal.module_key,
proposal_index=proposal.proposal_index,
id=proposal.id,
reason="proposal references unknown segment id",
reason=preview.reason,
original_text=proposal.original_text,
corrected_text=proposal.corrected_text,
confidence=proposal.confidence,
source="application",
)
segment = transcript[segment_index]
match_count = segment.text.count(proposal.original_text)
if proposal.original_text == "":
return ReportedSkip(
module_instance=proposal.module_instance,
module_key=proposal.module_key,
proposal_index=proposal.proposal_index,
id=proposal.id,
reason="proposal original_text must not be empty",
original_text=proposal.original_text,
corrected_text=proposal.corrected_text,
confidence=proposal.confidence,
actual_text=segment.text,
source="application",
)
if match_count == 0:
return ReportedSkip(
module_instance=proposal.module_instance,
module_key=proposal.module_key,
proposal_index=proposal.proposal_index,
id=proposal.id,
reason="proposal original_text does not match segment text",
original_text=proposal.original_text,
corrected_text=proposal.corrected_text,
confidence=proposal.confidence,
actual_text=segment.text,
source="application",
)
if replacement_policy == "require_unique" and match_count != 1:
return ReportedSkip(
module_instance=proposal.module_instance,
module_key=proposal.module_key,
proposal_index=proposal.proposal_index,
id=proposal.id,
reason="proposal original_text must match exactly once",
original_text=proposal.original_text,
corrected_text=proposal.corrected_text,
confidence=proposal.confidence,
actual_text=segment.text,
actual_text=preview.actual_text,
source="application",
)
updated = list(transcript)
updated_text = segment.text.replace(proposal.original_text, proposal.corrected_text)
updated[segment_index] = segment.model_copy(update={"text": updated_text})
updated[preview.segment_index] = preview.segment.model_copy(update={"text": preview.corrected_segment_text})
return (
updated,
AppliedChange(
@@ -285,8 +244,8 @@ def _apply_proposal(
original_text=proposal.original_text,
corrected_text=proposal.corrected_text,
confidence=proposal.confidence,
segment_text_before=segment.text,
segment_text_after=updated_text,
segment_text_before=preview.original_segment_text,
segment_text_after=preview.corrected_segment_text,
),
)

11
src/audita/modules/glossary.py
View File

@@ -2,7 +2,12 @@ from typing import Sequence
from audita.core.schemas import TranscriptSegment
from audita.framework.models import CorrectionProposal, ModuleContext
from audita.validators import ProtectedGlossaryTermsValidator, StubLLMValidator, Validator
from audita.validators import (
MeaningReversalValidator,
ProtectedGlossaryTermsValidator,
SpokenFormPlausibilityValidator,
Validator,
)
class GlossaryModule:
@@ -12,8 +17,8 @@ class GlossaryModule:
def validators(self) -> Sequence[Validator]:
return [
ProtectedGlossaryTermsValidator("protected_glossary_guard"),
StubLLMValidator("toward_glossary_term_review"),
StubLLMValidator("context_support_review"),
SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
MeaningReversalValidator("meaning_reversal_review"),
]
def propose(

7
src/audita/modules/grammar.py
View File

@@ -2,7 +2,7 @@ from typing import Sequence
from audita.core.schemas import TranscriptSegment
from audita.framework.models import CorrectionProposal, ModuleContext
from audita.validators import ProtectedGlossaryTermsValidator, StubLLMValidator, Validator
from audita.validators import ProtectedGlossaryTermsValidator, Validator
class GrammarModule:
@@ -10,10 +10,7 @@ class GrammarModule:
replacement_policy = "require_unique"
def validators(self) -> Sequence[Validator]:
return [
ProtectedGlossaryTermsValidator("protected_glossary_guard"),
StubLLMValidator("edited_text_readability_review"),
]
return [ProtectedGlossaryTermsValidator("protected_glossary_guard")]
def propose(
self,

12
src/audita/modules/homophones.py
View File

@@ -2,7 +2,12 @@ from typing import Sequence
from audita.core.schemas import TranscriptSegment
from audita.framework.models import CorrectionProposal, ModuleContext
from audita.validators import ProtectedGlossaryTermsValidator, StubLLMValidator, Validator
from audita.validators import (
MeaningReversalValidator,
ProtectedGlossaryTermsValidator,
SpokenFormPlausibilityValidator,
Validator,
)
class HomophonesModule:
@@ -12,9 +17,8 @@ class HomophonesModule:
def validators(self) -> Sequence[Validator]:
return [
ProtectedGlossaryTermsValidator("protected_glossary_guard"),
StubLLMValidator("acoustic_similarity_review"),
StubLLMValidator("contextual_plausibility_review"),
StubLLMValidator("antonym_reversal_review"),
SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
MeaningReversalValidator("meaning_reversal_review"),
]
def propose(

8
src/audita/modules/spoken_word.py
View File

@@ -2,7 +2,7 @@ from typing import Sequence
from audita.core.schemas import TranscriptSegment
from audita.framework.models import CorrectionProposal, ModuleContext
from audita.validators import ProtectedGlossaryTermsValidator, StubLLMValidator, Validator
from audita.validators import ProtectedGlossaryTermsValidator, Validator
class SpokenWordModule:
@@ -10,11 +10,7 @@ class SpokenWordModule:
replacement_policy = "replace_all"
def validators(self) -> Sequence[Validator]:
return [
ProtectedGlossaryTermsValidator("protected_glossary_guard"),
StubLLMValidator("spoken_marker_cleanup_review"),
StubLLMValidator("meaning_preservation_review"),
]
return [ProtectedGlossaryTermsValidator("protected_glossary_guard")]
def propose(
self,

4
src/audita/pipeline.py
View File

@@ -10,6 +10,7 @@ from .core.errors import AuditaError
from .core.normalization import normalize_transcript
from .core.reporting import ProcessResult, RunReport
from .core.schemas import Glossary, SourceTranscriptSegment, TranscriptSegment, source_transcript_to_json, transcript_to_json
from .framework.llm import OpenRouterStructuredLLMClient
from .framework.runner import PipelineRunner
from .modules import default_module_specs
@@ -55,13 +56,14 @@ def process_transcript_result(
module_specs = default_module_specs()
pipeline_runner = PipelineRunner()
llm_client = OpenRouterStructuredLLMClient()
pipeline_result = pipeline_runner.run(
transcript=normalized_transcript,
glossary=glossary,
module_specs=module_specs,
config=config,
run_dir=run_dir,
llm_client=None,
llm_client=llm_client,
progress=progress,
)
revised = _sort_transcript_chronologically(pipeline_result.transcript)

5
src/audita/validators/__init__.py
View File

@@ -1,6 +1,6 @@
from .base import ValidationContext, ValidationDecision, ValidationResult, Validator
from .deterministic import ProtectedGlossaryTermsValidator
from .llm import StubLLMValidator
from .llm import MeaningReversalValidator, SpokenFormPlausibilityValidator
from .protection import ProtectedVocabulary
__all__ = [
@@ -10,5 +10,6 @@ __all__ = [
"Validator",
"ProtectedGlossaryTermsValidator",
"ProtectedVocabulary",
"StubLLMValidator",
"SpokenFormPlausibilityValidator",
"MeaningReversalValidator",
]

136
src/audita/validators/llm.py
View File

@@ -1,19 +1,145 @@
from __future__ import annotations
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Callable, Dict, List, Sequence
from pydantic import BaseModel, ConfigDict, Field, StrictStr
from audita.core.chunking import chunk_payload_items
from audita.core.errors import AuditaLLMError
from audita.framework.proposals import ProposalPreview, ProposalPreviewError, preview_proposal
from .base import ValidationContext, ValidationDecision, ValidationResult
from .prompts import build_meaning_reversal_messages, build_spoken_form_plausibility_messages
class _LLMValidationDecisionModel(BaseModel):
model_config = ConfigDict(extra="forbid")
correction_index: int
approved: bool
confidence: float = Field(ge=0.0, le=1.0)
reason: StrictStr
class _LLMValidationSetModel(BaseModel):
model_config = ConfigDict(extra="forbid")
validations: List[_LLMValidationDecisionModel]
PromptBuilder = Callable[[List[dict]], List[dict]]
@dataclass(frozen=True)
class StubLLMValidator:
class _BaseLLMValidator:
name: str
prompt_builder: PromptBuilder
execution_kind: str = "llm"
def validate(self, context: ValidationContext) -> ValidationResult:
if not context.proposals:
return ValidationResult(
validator_name=self.name,
execution_kind=self.execution_kind,
decisions=[],
)
previewable: List[ProposalPreview] = []
decisions: List[ValidationDecision] = []
replacement_policy = context.run_spec.module.replacement_policy
for proposal in context.proposals:
preview = preview_proposal(context.transcript, proposal, replacement_policy)
if isinstance(preview, ProposalPreviewError):
decisions.append(
ValidationDecision(
proposal_index=proposal.proposal_index,
approved=False,
reason=preview.reason,
)
)
continue
previewable.append(preview)
if previewable:
llm_client = context.llm_client
if llm_client is None:
raise AuditaLLMError(f"Validator '{self.name}' requires a structured LLM client.")
batches = chunk_payload_items(
previewable,
context.config.max_section_tokens,
payload_fn=lambda item: item.to_prompt_payload(),
empty_error_message="Validation input must contain at least one proposal.",
)
for batch in batches:
payload = [item.to_prompt_payload() for item in batch.items]
messages = self.prompt_builder(payload)
prompt_path = context.run_dir / f"{self.name}-prompt-{batch.batch_index:04d}.json"
response_path = context.run_dir / f"{self.name}-response-{batch.batch_index:04d}.json"
_write_json(prompt_path, {"messages": messages})
response = llm_client.run_structured(
stage_name=f"{context.run_spec.instance_name}:{self.name}",
messages=messages,
response_model=_LLMValidationSetModel,
config=context.config,
)
_write_json(response_path, response.model_dump(mode="json"))
decisions.extend(self._validate_batch_response(response, batch.items))
return ValidationResult(
validator_name=self.name,
execution_kind=self.execution_kind,
decisions=[
ValidationDecision(proposal_index=proposal.proposal_index, approved=True)
for proposal in context.proposals
],
decisions=sorted(decisions, key=lambda decision: decision.proposal_index),
)
def _validate_batch_response(
self,
response: _LLMValidationSetModel,
proposals: Sequence[ProposalPreview],
) -> List[ValidationDecision]:
expected_indexes = {proposal.proposal.proposal_index for proposal in proposals}
indexed: Dict[int, _LLMValidationDecisionModel] = {}
for decision in response.validations:
if decision.correction_index in indexed:
raise AuditaLLMError(
f"Validator '{self.name}' returned duplicate correction_index values."
)
if decision.correction_index not in expected_indexes:
raise AuditaLLMError(
f"Validator '{self.name}' returned an unknown correction_index."
)
indexed[decision.correction_index] = decision
missing_indexes = sorted(expected_indexes - set(indexed))
if missing_indexes:
raise AuditaLLMError(
f"Validator '{self.name}' omitted correction_index values: {missing_indexes}"
)
return [
ValidationDecision(
proposal_index=proposal.proposal.proposal_index,
approved=indexed[proposal.proposal.proposal_index].approved,
confidence=indexed[proposal.proposal.proposal_index].confidence,
reason=indexed[proposal.proposal.proposal_index].reason,
)
for proposal in proposals
]
@dataclass(frozen=True)
class SpokenFormPlausibilityValidator(_BaseLLMValidator):
name: str = "spoken_form_plausibility_review"
prompt_builder: PromptBuilder = build_spoken_form_plausibility_messages
@dataclass(frozen=True)
class MeaningReversalValidator(_BaseLLMValidator):
name: str = "meaning_reversal_review"
prompt_builder: PromptBuilder = build_meaning_reversal_messages
def _write_json(path: Path, payload: dict) -> None:
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")

56
src/audita/validators/prompts.py Normal file
View File

@@ -0,0 +1,56 @@
import json
from typing import Dict, List
Message = Dict[str, str]
def build_spoken_form_plausibility_messages(validation_payload: List[dict]) -> List[Message]:
payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2)
system = (
"You are Audita, a conservative spoken-form validation assistant. "
"Evaluate whether each proposed correction is plausibly explained by a homophone, phonetic similarity, "
"or a common mistranscription of spoken English. "
"Your job is not to improve style or readability. "
"Approve only when the corrected text is a plausible recovery of the words that were likely spoken."
)
user = (
"Review these proposed transcript corrections and decide whether each one is a plausible spoken-form correction.\n\n"
"Rules:\n"
"- Return one validation decision for every correction_index in the input.\n"
"- Approve when the original text and corrected text are plausibly related by homophone confusion, "
"phonetic similarity, or a common spoken-word mistranscription, and the surrounding segment context supports the correction.\n"
"- Examples that may be approved when context supports them: changing \"gestures\" to \"Jesters\", "
"\"rank\" to \"Hrank\", or \"dam\" to \"damn\".\n"
"- Reject unrelated substitutions like changing \"Lyra\" to \"Jesters\".\n"
"- Judge spoken-form plausibility, not whether the correction is cleaner, more formal, or more grammatical.\n"
"- Do not approve paraphrases, stylistic rewrites, or arbitrary semantic substitutions.\n"
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n"
"- confidence must be between 0.0 and 1.0.\n\n"
f"Corrections to validate:\n{payload_json}"
)
return [{"role": "system", "content": system}, {"role": "user", "content": user}]
def build_meaning_reversal_messages(validation_payload: List[dict]) -> List[Message]:
payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2)
system = (
"You are Audita, a narrow semantic-reversal validation assistant. "
"Evaluate whether each proposed correction changes a word to its antonym or otherwise reverses the meaning "
"of the full segment. "
"Do not treat every word substitution as a problem; focus specifically on antonyms and meaning reversals."
)
user = (
"Review these proposed transcript corrections and decide whether each one avoids reversing the segment meaning.\n\n"
"Rules:\n"
"- Return one validation decision for every correction_index in the input.\n"
"- Reject corrections that introduce antonyms or otherwise reverse the meaning of the original segment.\n"
"- Reject examples like changing \"visible\" to \"invisible\" or \"up\" to \"down\" when that reverses the segment meaning.\n"
"- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.\n"
"- Do not reject a correction merely because the literal written word changes.\n"
"- Do not act as a general semantic-style reviewer; this validator is only a guard against antonyms and meaning reversals.\n"
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n"
"- confidence must be between 0.0 and 1.0.\n\n"
f"Corrections to validate:\n{payload_json}"
)
return [{"role": "system", "content": system}, {"role": "user", "content": user}]

105
tests/test_framework_runner.py
View File

@@ -1,8 +1,9 @@
from audita.core.config import AuditaConfig
from audita.core.errors import AuditaLLMError
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
from audita.framework.models import CorrectionProposal, ModuleContext, ModuleRunSpec
from audita.framework.runner import PipelineRunner
from audita.validators import ProtectedGlossaryTermsValidator
from audita.validators import MeaningReversalValidator, ProtectedGlossaryTermsValidator, SpokenFormPlausibilityValidator
from audita.validators.base import ValidationContext, ValidationDecision, ValidationResult
@@ -34,6 +35,25 @@ class RecordingLLMValidator(RecordingValidator):
execution_kind = "llm"
class FakeStructuredLLMClient:
def __init__(self, responses):
self._responses = list(responses)
self.calls = []
def run_structured(self, *, stage_name, messages, response_model, config):
self.calls.append(
{
"stage_name": stage_name,
"messages": list(messages),
"response_model": response_model,
}
)
if not self._responses:
raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.")
payload = self._responses.pop(0)
return response_model.model_validate(payload)
class RecordingModule:
replacement_policy = "require_unique"
@@ -227,6 +247,89 @@ def test_pipeline_runner_supports_deterministic_and_llm_validators_in_one_chain(
]
def test_pipeline_runner_supports_real_llm_validators_in_one_chain(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the dam."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Jesters"
category: faction
summary: "The Jesters are a faction."
"""
)
module = RecordingModule(
"mixed_real",
[
CorrectionProposal(
proposal_index=0,
module_instance="mixed_real",
module_key="glossary",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.9,
)
],
[
ProtectedGlossaryTermsValidator("protected_glossary_guard"),
SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
MeaningReversalValidator("meaning_reversal_review"),
],
[],
)
llm_client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.95,
"reason": "Likely phonetic mistranscription in context.",
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.98,
"reason": "Does not reverse the segment meaning.",
}
]
},
]
)
runner = PipelineRunner()
result = runner.run(
transcript=transcript,
glossary=glossary,
module_specs=[ModuleRunSpec(instance_name="mixed_real", module_key="glossary", module=module)],
config=AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}),
run_dir=tmp_path / "run",
llm_client=llm_client,
)
assert result.transcript[0].text == "There were Jesters at the dam."
assert [report.execution_kind for report in result.module_reports[0].validators] == [
"deterministic",
"llm",
"llm",
]
assert [call["stage_name"] for call in llm_client.calls] == [
"mixed_real:spoken_form_plausibility_review",
"mixed_real:meaning_reversal_review",
]
def test_pipeline_runner_uses_real_protected_glossary_validator(tmp_path):
transcript = parse_transcript_json(
"""

453
tests/test_llm_validators.py Normal file
View File

@@ -0,0 +1,453 @@
import pytest
from audita.core.chunking import TokenBatch
from audita.core.config import AuditaConfig
from audita.core.errors import AuditaLLMError
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
from audita.framework.models import CorrectionProposal, ModuleRunSpec
from audita.validators.base import ValidationContext
from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator
from audita.validators.prompts import (
build_meaning_reversal_messages,
build_spoken_form_plausibility_messages,
)
import audita.validators.llm as llm_module
class _Module:
def __init__(self, replacement_policy: str) -> None:
self.replacement_policy = replacement_policy
class FakeStructuredLLMClient:
def __init__(self, responses):
self._responses = list(responses)
self.calls = []
def run_structured(self, *, stage_name, messages, response_model, config):
self.calls.append(
{
"stage_name": stage_name,
"messages": list(messages),
"response_model": response_model,
}
)
if not self._responses:
raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.")
return response_model.model_validate(self._responses.pop(0))
def _glossary():
return parse_glossary_yaml(
"""
glossary:
- name: "Jesters"
category: faction
summary: "The Jesters are a faction."
- name: "Hrank"
category: pc
summary: "Hrank is a player character."
- name: "Lyra"
category: npc
summary: "Lyra is an NPC."
"""
)
def _context(
*,
proposals,
transcript,
llm_client,
tmp_path,
replacement_policy="require_unique",
):
return ValidationContext(
proposals=proposals,
transcript=transcript,
glossary=_glossary(),
config=AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}),
run_spec=ModuleRunSpec(
instance_name="homophones",
module_key="homophones",
module=_Module(replacement_policy),
),
run_dir=tmp_path,
llm_client=llm_client,
)
def test_spoken_form_plausibility_validator_approves_plausible_and_rejects_implausible_corrections(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "Lyra moved first."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="homophones",
module_key="homophones",
id=2,
original_text="Lyra",
corrected_text="Jesters",
confidence=0.95,
),
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.97,
"reason": "Likely spoken-form correction in context.",
},
{
"correction_index": 1,
"approved": False,
"confidence": 0.99,
"reason": "Not plausibly related by homophone or mistranscription.",
},
]
}
]
)
result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
(0, True),
(1, False),
]
prompt_text = client.calls[0]["messages"][1]["content"]
assert '"original_segment_text"' in prompt_text
assert "There were Jesters at the temple." in prompt_text
assert "Lyra moved first." in prompt_text
def test_meaning_reversal_validator_rejects_reversal_and_approves_nonreversal(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam, but Claude actually can."},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "The figure became visible in the doorway."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="dam",
corrected_text="damn",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="homophones",
module_key="homophones",
id=2,
original_text="visible",
corrected_text="invisible",
confidence=0.95,
),
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.94,
"reason": "Does not reverse the segment meaning.",
},
{
"correction_index": 1,
"approved": False,
"confidence": 1.0,
"reason": "Changes visible to invisible and reverses the segment meaning.",
},
]
}
]
)
result = MeaningReversalValidator("meaning_reversal_review").validate(
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
(0, True),
(1, False),
]
prompt_text = client.calls[0]["messages"][1]["content"]
assert "The figure became visible in the doorway." in prompt_text
assert "The figure became invisible in the doorway." in prompt_text
@pytest.mark.parametrize(
("validator", "payload", "message_fragment"),
[
(
SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.9,
"reason": "ok",
},
{
"correction_index": 0,
"approved": True,
"confidence": 0.9,
"reason": "duplicate",
},
]
},
"duplicate correction_index values",
),
(
MeaningReversalValidator("meaning_reversal_review"),
{"validations": []},
"omitted correction_index values",
),
(
SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
{
"validations": [
{
"correction_index": 99,
"approved": True,
"confidence": 0.9,
"reason": "unknown",
}
]
},
"unknown correction_index",
),
],
)
def test_llm_validators_reject_bad_correction_indexes(tmp_path, validator, payload, message_fragment):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
)
]
client = FakeStructuredLLMClient([payload])
with pytest.raises(AuditaLLMError, match=message_fragment):
validator.validate(_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path))
def test_llm_validator_rejects_unpreviewable_proposals_without_calling_llm(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="rank",
corrected_text="Hrank",
confidence=0.95,
)
]
client = FakeStructuredLLMClient([])
result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert result.decisions[0].approved is False
assert result.decisions[0].reason == "proposal original_text does not match segment text"
assert client.calls == []
def test_llm_validator_skips_empty_candidate_sets_without_calling_llm(tmp_path):
client = FakeStructuredLLMClient([])
result = MeaningReversalValidator("meaning_reversal_review").validate(
_context(proposals=[], transcript=[], llm_client=client, tmp_path=tmp_path)
)
assert result.decisions == []
assert client.calls == []
def test_spoken_form_prompt_emphasizes_acoustic_plausibility():
messages = build_spoken_form_plausibility_messages(
[
{
"correction_index": 0,
"id": 1,
"original_segment_text": "There were gestures at the temple.",
"corrected_segment_text": "There were Jesters at the temple.",
"original_text": "gestures",
"corrected_text": "Jesters",
}
]
)
combined = messages[0]["content"] + messages[1]["content"]
assert "homophone" in combined
assert "gestures" in combined
assert "Lyra" in combined
assert '"original_segment_text"' in messages[1]["content"]
def test_meaning_reversal_prompt_emphasizes_antonyms_and_segment_context():
messages = build_meaning_reversal_messages(
[
{
"correction_index": 0,
"id": 1,
"original_segment_text": "The figure became visible in the doorway.",
"corrected_segment_text": "The figure became invisible in the doorway.",
"original_text": "visible",
"corrected_text": "invisible",
}
]
)
combined = messages[0]["content"] + messages[1]["content"]
assert "antonym" in combined
assert "visible" in combined
assert "up" in combined
assert "original_segment_text" in messages[1]["content"]
def test_llm_validators_use_shared_token_batching_helper(monkeypatch, tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "We saw rank near the gate."},
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "ChatGPT still can't do that with a dam."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="homophones",
module_key="homophones",
id=2,
original_text="rank",
corrected_text="Hrank",
confidence=0.95,
),
CorrectionProposal(
proposal_index=2,
module_instance="homophones",
module_key="homophones",
id=3,
original_text="dam",
corrected_text="damn",
confidence=0.95,
),
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.9,
"reason": "ok",
}
]
},
{
"validations": [
{
"correction_index": 1,
"approved": True,
"confidence": 0.9,
"reason": "ok",
},
{
"correction_index": 2,
"approved": True,
"confidence": 0.9,
"reason": "ok",
},
]
},
]
)
chunk_calls = []
def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message):
chunk_calls.append(
{
"max_tokens": max_tokens,
"payloads": [payload_fn(item) for item in items],
"empty_error_message": empty_error_message,
}
)
return [
TokenBatch(batch_index=0, items=list(items[:1]), token_count=1),
TokenBatch(batch_index=1, items=list(items[1:]), token_count=1),
]
monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items)
result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert [decision.approved for decision in result.decisions] == [True, True, True]
assert len(client.calls) == 2
assert len(chunk_calls) == 1
assert chunk_calls[0]["max_tokens"] == AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}).max_section_tokens
assert all("corrected_segment_text" in payload for payload in chunk_calls[0]["payloads"])

28
tests/test_new_pipeline.py
View File

@@ -76,8 +76,8 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
assert (result.run_dir / "normalization" / "summary.json").exists()
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
"protected_glossary_guard",
"toward_glossary_term_review",
"context_support_review",
"spoken_form_plausibility_review",
"meaning_reversal_review",
]
@@ -97,26 +97,18 @@ def test_default_module_specs_expose_final_validator_order():
assert [validator.name for validator in specs[0].module.validators()] == [
"protected_glossary_guard",
"toward_glossary_term_review",
"context_support_review",
"spoken_form_plausibility_review",
"meaning_reversal_review",
]
assert [validator.name for validator in specs[1].module.validators()] == [
"protected_glossary_guard",
"acoustic_similarity_review",
"contextual_plausibility_review",
"antonym_reversal_review",
"spoken_form_plausibility_review",
"meaning_reversal_review",
]
assert [validator.name for validator in specs[2].module.validators()] == [
"protected_glossary_guard",
"toward_glossary_term_review",
"context_support_review",
]
assert [validator.name for validator in specs[3].module.validators()] == [
"protected_glossary_guard",
"spoken_marker_cleanup_review",
"meaning_preservation_review",
]
assert [validator.name for validator in specs[4].module.validators()] == [
"protected_glossary_guard",
"edited_text_readability_review",
"spoken_form_plausibility_review",
"meaning_reversal_review",
]
assert [validator.name for validator in specs[3].module.validators()] == ["protected_glossary_guard"]
assert [validator.name for validator in specs[4].module.validators()] == ["protected_glossary_guard"]