diff --git a/src/audita/framework/llm.py b/src/audita/framework/llm.py index c0ba1ca..0f1b0bc 100644 --- a/src/audita/framework/llm.py +++ b/src/audita/framework/llm.py @@ -1,11 +1,15 @@ -from typing import Any, Sequence +from __future__ import annotations + +from typing import Any, Optional, Sequence, Tuple from audita.core.config import AuditaConfig from audita.core.errors import AuditaLLMError class OpenRouterStructuredLLMClient: - """Placeholder for future structured LLM integration.""" + def __init__(self) -> None: + self._client = None + self._client_identity: Optional[Tuple[str, str]] = None def run_structured( self, @@ -15,6 +19,48 @@ class OpenRouterStructuredLLMClient: response_model: Any, config: AuditaConfig, ) -> Any: - raise AuditaLLMError( - f"Structured LLM stage '{stage_name}' is not implemented in the framework skeleton." - ) + if not config.api_key: + raise AuditaLLMError( + f"Structured LLM stage '{stage_name}' requires OPENROUTER_API_KEY to be configured." + ) + + client = self._get_client(config) + model = _normalize_openrouter_model(config.model) + try: + return client.chat.completions.create( + model=model, + messages=list(messages), + response_model=response_model, + max_retries=config.max_retries, + extra_body={"provider": {"require_parameters": True}}, + ) + except Exception as exc: + raise AuditaLLMError( + f"Structured LLM stage '{stage_name}' failed. Confirm the configured OpenRouter model " + "supports tool calling or structured outputs." + ) from exc + + def _get_client(self, config: AuditaConfig) -> Any: + identity = (config.api_key or "", config.base_url) + if self._client is not None and self._client_identity == identity: + return self._client + + try: + import instructor + from openai import OpenAI + except ImportError as exc: + raise AuditaLLMError( + "The LLM dependencies are not installed. Run `uv sync` before using Audita validators." + ) from exc + + openai_client = OpenAI(api_key=config.api_key, base_url=config.base_url) + self._client = instructor.patch(openai_client, mode=instructor.Mode.TOOLS) + self._client_identity = identity + return self._client + + +def _normalize_openrouter_model(model: str) -> str: + prefix = "openrouter/" + if model.startswith(prefix): + return model[len(prefix) :] + return model diff --git a/src/audita/framework/proposals.py b/src/audita/framework/proposals.py new file mode 100644 index 0000000..42bb66c --- /dev/null +++ b/src/audita/framework/proposals.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional, Sequence, Union + +from audita.core.schemas import TranscriptSegment + +if TYPE_CHECKING: + from .models import CorrectionProposal + + +ReplacementPolicy = str + + +@dataclass(frozen=True) +class ProposalPreview: + proposal: CorrectionProposal + segment_index: int + segment: TranscriptSegment + corrected_segment_text: str + + @property + def original_segment_text(self) -> str: + return self.segment.text + + def to_prompt_payload(self) -> dict: + return { + "correction_index": self.proposal.proposal_index, + "id": self.proposal.id, + "original_segment_text": self.original_segment_text, + "corrected_segment_text": self.corrected_segment_text, + "original_text": self.proposal.original_text, + "corrected_text": self.proposal.corrected_text, + } + + +@dataclass(frozen=True) +class ProposalPreviewError: + reason: str + actual_text: Optional[str] = None + + +def preview_proposal( + transcript: Sequence[TranscriptSegment], + proposal: CorrectionProposal, + replacement_policy: ReplacementPolicy, +) -> Union[ProposalPreview, ProposalPreviewError]: + index_by_id = {segment.id: index for index, segment in enumerate(transcript)} + segment_index = index_by_id.get(proposal.id) + if segment_index is None: + return ProposalPreviewError(reason="proposal references unknown segment id") + + segment = transcript[segment_index] + if proposal.original_text == "": + return ProposalPreviewError( + reason="proposal original_text must not be empty", + actual_text=segment.text, + ) + + match_count = segment.text.count(proposal.original_text) + if match_count == 0: + return ProposalPreviewError( + reason="proposal original_text does not match segment text", + actual_text=segment.text, + ) + + if replacement_policy == "require_unique" and match_count != 1: + return ProposalPreviewError( + reason="proposal original_text must match exactly once", + actual_text=segment.text, + ) + + corrected_segment_text = segment.text.replace(proposal.original_text, proposal.corrected_text) + return ProposalPreview( + proposal=proposal, + segment_index=segment_index, + segment=segment, + corrected_segment_text=corrected_segment_text, + ) diff --git a/src/audita/framework/runner.py b/src/audita/framework/runner.py index 9f54c3e..c067f93 100644 --- a/src/audita/framework/runner.py +++ b/src/audita/framework/runner.py @@ -9,6 +9,7 @@ from audita.core.schemas import Glossary, TranscriptSegment from audita.validators.base import ValidationContext, ValidationDecision, ValidationResult from .models import CorrectionProposal, ModuleContext, ModuleRunSpec, StructuredLLMClient +from .proposals import ProposalPreviewError, preview_proposal ProgressCallback = Callable[[str], None] @@ -216,65 +217,23 @@ def _apply_proposal( proposal: CorrectionProposal, replacement_policy: str, ) -> Union[Tuple[List[TranscriptSegment], AppliedChange], ReportedSkip]: - index_by_id = {segment.id: index for index, segment in enumerate(transcript)} - segment_index = index_by_id.get(proposal.id) - if segment_index is None: + preview = preview_proposal(transcript, proposal, replacement_policy) + if isinstance(preview, ProposalPreviewError): return ReportedSkip( module_instance=proposal.module_instance, module_key=proposal.module_key, proposal_index=proposal.proposal_index, id=proposal.id, - reason="proposal references unknown segment id", + reason=preview.reason, original_text=proposal.original_text, corrected_text=proposal.corrected_text, confidence=proposal.confidence, - source="application", - ) - segment = transcript[segment_index] - match_count = segment.text.count(proposal.original_text) - if proposal.original_text == "": - return ReportedSkip( - module_instance=proposal.module_instance, - module_key=proposal.module_key, - proposal_index=proposal.proposal_index, - id=proposal.id, - reason="proposal original_text must not be empty", - original_text=proposal.original_text, - corrected_text=proposal.corrected_text, - confidence=proposal.confidence, - actual_text=segment.text, - source="application", - ) - if match_count == 0: - return ReportedSkip( - module_instance=proposal.module_instance, - module_key=proposal.module_key, - proposal_index=proposal.proposal_index, - id=proposal.id, - reason="proposal original_text does not match segment text", - original_text=proposal.original_text, - corrected_text=proposal.corrected_text, - confidence=proposal.confidence, - actual_text=segment.text, - source="application", - ) - if replacement_policy == "require_unique" and match_count != 1: - return ReportedSkip( - module_instance=proposal.module_instance, - module_key=proposal.module_key, - proposal_index=proposal.proposal_index, - id=proposal.id, - reason="proposal original_text must match exactly once", - original_text=proposal.original_text, - corrected_text=proposal.corrected_text, - confidence=proposal.confidence, - actual_text=segment.text, + actual_text=preview.actual_text, source="application", ) updated = list(transcript) - updated_text = segment.text.replace(proposal.original_text, proposal.corrected_text) - updated[segment_index] = segment.model_copy(update={"text": updated_text}) + updated[preview.segment_index] = preview.segment.model_copy(update={"text": preview.corrected_segment_text}) return ( updated, AppliedChange( @@ -285,8 +244,8 @@ def _apply_proposal( original_text=proposal.original_text, corrected_text=proposal.corrected_text, confidence=proposal.confidence, - segment_text_before=segment.text, - segment_text_after=updated_text, + segment_text_before=preview.original_segment_text, + segment_text_after=preview.corrected_segment_text, ), ) diff --git a/src/audita/modules/glossary.py b/src/audita/modules/glossary.py index 9e27aa8..fdc8657 100644 --- a/src/audita/modules/glossary.py +++ b/src/audita/modules/glossary.py @@ -2,7 +2,12 @@ from typing import Sequence from audita.core.schemas import TranscriptSegment from audita.framework.models import CorrectionProposal, ModuleContext -from audita.validators import ProtectedGlossaryTermsValidator, StubLLMValidator, Validator +from audita.validators import ( + MeaningReversalValidator, + ProtectedGlossaryTermsValidator, + SpokenFormPlausibilityValidator, + Validator, +) class GlossaryModule: @@ -12,8 +17,8 @@ class GlossaryModule: def validators(self) -> Sequence[Validator]: return [ ProtectedGlossaryTermsValidator("protected_glossary_guard"), - StubLLMValidator("toward_glossary_term_review"), - StubLLMValidator("context_support_review"), + SpokenFormPlausibilityValidator("spoken_form_plausibility_review"), + MeaningReversalValidator("meaning_reversal_review"), ] def propose( diff --git a/src/audita/modules/grammar.py b/src/audita/modules/grammar.py index 42299e9..968f38c 100644 --- a/src/audita/modules/grammar.py +++ b/src/audita/modules/grammar.py @@ -2,7 +2,7 @@ from typing import Sequence from audita.core.schemas import TranscriptSegment from audita.framework.models import CorrectionProposal, ModuleContext -from audita.validators import ProtectedGlossaryTermsValidator, StubLLMValidator, Validator +from audita.validators import ProtectedGlossaryTermsValidator, Validator class GrammarModule: @@ -10,10 +10,7 @@ class GrammarModule: replacement_policy = "require_unique" def validators(self) -> Sequence[Validator]: - return [ - ProtectedGlossaryTermsValidator("protected_glossary_guard"), - StubLLMValidator("edited_text_readability_review"), - ] + return [ProtectedGlossaryTermsValidator("protected_glossary_guard")] def propose( self, diff --git a/src/audita/modules/homophones.py b/src/audita/modules/homophones.py index 312a59e..cfb4cf6 100644 --- a/src/audita/modules/homophones.py +++ b/src/audita/modules/homophones.py @@ -2,7 +2,12 @@ from typing import Sequence from audita.core.schemas import TranscriptSegment from audita.framework.models import CorrectionProposal, ModuleContext -from audita.validators import ProtectedGlossaryTermsValidator, StubLLMValidator, Validator +from audita.validators import ( + MeaningReversalValidator, + ProtectedGlossaryTermsValidator, + SpokenFormPlausibilityValidator, + Validator, +) class HomophonesModule: @@ -12,9 +17,8 @@ class HomophonesModule: def validators(self) -> Sequence[Validator]: return [ ProtectedGlossaryTermsValidator("protected_glossary_guard"), - StubLLMValidator("acoustic_similarity_review"), - StubLLMValidator("contextual_plausibility_review"), - StubLLMValidator("antonym_reversal_review"), + SpokenFormPlausibilityValidator("spoken_form_plausibility_review"), + MeaningReversalValidator("meaning_reversal_review"), ] def propose( diff --git a/src/audita/modules/spoken_word.py b/src/audita/modules/spoken_word.py index aee1205..295af2d 100644 --- a/src/audita/modules/spoken_word.py +++ b/src/audita/modules/spoken_word.py @@ -2,7 +2,7 @@ from typing import Sequence from audita.core.schemas import TranscriptSegment from audita.framework.models import CorrectionProposal, ModuleContext -from audita.validators import ProtectedGlossaryTermsValidator, StubLLMValidator, Validator +from audita.validators import ProtectedGlossaryTermsValidator, Validator class SpokenWordModule: @@ -10,11 +10,7 @@ class SpokenWordModule: replacement_policy = "replace_all" def validators(self) -> Sequence[Validator]: - return [ - ProtectedGlossaryTermsValidator("protected_glossary_guard"), - StubLLMValidator("spoken_marker_cleanup_review"), - StubLLMValidator("meaning_preservation_review"), - ] + return [ProtectedGlossaryTermsValidator("protected_glossary_guard")] def propose( self, diff --git a/src/audita/pipeline.py b/src/audita/pipeline.py index a80e147..f983fbd 100644 --- a/src/audita/pipeline.py +++ b/src/audita/pipeline.py @@ -10,6 +10,7 @@ from .core.errors import AuditaError from .core.normalization import normalize_transcript from .core.reporting import ProcessResult, RunReport from .core.schemas import Glossary, SourceTranscriptSegment, TranscriptSegment, source_transcript_to_json, transcript_to_json +from .framework.llm import OpenRouterStructuredLLMClient from .framework.runner import PipelineRunner from .modules import default_module_specs @@ -55,13 +56,14 @@ def process_transcript_result( module_specs = default_module_specs() pipeline_runner = PipelineRunner() + llm_client = OpenRouterStructuredLLMClient() pipeline_result = pipeline_runner.run( transcript=normalized_transcript, glossary=glossary, module_specs=module_specs, config=config, run_dir=run_dir, - llm_client=None, + llm_client=llm_client, progress=progress, ) revised = _sort_transcript_chronologically(pipeline_result.transcript) diff --git a/src/audita/validators/__init__.py b/src/audita/validators/__init__.py index d5318fa..6fe98aa 100644 --- a/src/audita/validators/__init__.py +++ b/src/audita/validators/__init__.py @@ -1,6 +1,6 @@ from .base import ValidationContext, ValidationDecision, ValidationResult, Validator from .deterministic import ProtectedGlossaryTermsValidator -from .llm import StubLLMValidator +from .llm import MeaningReversalValidator, SpokenFormPlausibilityValidator from .protection import ProtectedVocabulary __all__ = [ @@ -10,5 +10,6 @@ __all__ = [ "Validator", "ProtectedGlossaryTermsValidator", "ProtectedVocabulary", - "StubLLMValidator", + "SpokenFormPlausibilityValidator", + "MeaningReversalValidator", ] diff --git a/src/audita/validators/llm.py b/src/audita/validators/llm.py index 2873773..62ef88d 100644 --- a/src/audita/validators/llm.py +++ b/src/audita/validators/llm.py @@ -1,19 +1,145 @@ +from __future__ import annotations + +import json from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Dict, List, Sequence + +from pydantic import BaseModel, ConfigDict, Field, StrictStr + +from audita.core.chunking import chunk_payload_items +from audita.core.errors import AuditaLLMError +from audita.framework.proposals import ProposalPreview, ProposalPreviewError, preview_proposal from .base import ValidationContext, ValidationDecision, ValidationResult +from .prompts import build_meaning_reversal_messages, build_spoken_form_plausibility_messages + + +class _LLMValidationDecisionModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + correction_index: int + approved: bool + confidence: float = Field(ge=0.0, le=1.0) + reason: StrictStr + + +class _LLMValidationSetModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + validations: List[_LLMValidationDecisionModel] + + +PromptBuilder = Callable[[List[dict]], List[dict]] @dataclass(frozen=True) -class StubLLMValidator: +class _BaseLLMValidator: name: str + prompt_builder: PromptBuilder execution_kind: str = "llm" def validate(self, context: ValidationContext) -> ValidationResult: + if not context.proposals: + return ValidationResult( + validator_name=self.name, + execution_kind=self.execution_kind, + decisions=[], + ) + + previewable: List[ProposalPreview] = [] + decisions: List[ValidationDecision] = [] + replacement_policy = context.run_spec.module.replacement_policy + for proposal in context.proposals: + preview = preview_proposal(context.transcript, proposal, replacement_policy) + if isinstance(preview, ProposalPreviewError): + decisions.append( + ValidationDecision( + proposal_index=proposal.proposal_index, + approved=False, + reason=preview.reason, + ) + ) + continue + previewable.append(preview) + + if previewable: + llm_client = context.llm_client + if llm_client is None: + raise AuditaLLMError(f"Validator '{self.name}' requires a structured LLM client.") + batches = chunk_payload_items( + previewable, + context.config.max_section_tokens, + payload_fn=lambda item: item.to_prompt_payload(), + empty_error_message="Validation input must contain at least one proposal.", + ) + for batch in batches: + payload = [item.to_prompt_payload() for item in batch.items] + messages = self.prompt_builder(payload) + prompt_path = context.run_dir / f"{self.name}-prompt-{batch.batch_index:04d}.json" + response_path = context.run_dir / f"{self.name}-response-{batch.batch_index:04d}.json" + _write_json(prompt_path, {"messages": messages}) + response = llm_client.run_structured( + stage_name=f"{context.run_spec.instance_name}:{self.name}", + messages=messages, + response_model=_LLMValidationSetModel, + config=context.config, + ) + _write_json(response_path, response.model_dump(mode="json")) + decisions.extend(self._validate_batch_response(response, batch.items)) + return ValidationResult( validator_name=self.name, execution_kind=self.execution_kind, - decisions=[ - ValidationDecision(proposal_index=proposal.proposal_index, approved=True) - for proposal in context.proposals - ], + decisions=sorted(decisions, key=lambda decision: decision.proposal_index), ) + + def _validate_batch_response( + self, + response: _LLMValidationSetModel, + proposals: Sequence[ProposalPreview], + ) -> List[ValidationDecision]: + expected_indexes = {proposal.proposal.proposal_index for proposal in proposals} + indexed: Dict[int, _LLMValidationDecisionModel] = {} + for decision in response.validations: + if decision.correction_index in indexed: + raise AuditaLLMError( + f"Validator '{self.name}' returned duplicate correction_index values." + ) + if decision.correction_index not in expected_indexes: + raise AuditaLLMError( + f"Validator '{self.name}' returned an unknown correction_index." + ) + indexed[decision.correction_index] = decision + + missing_indexes = sorted(expected_indexes - set(indexed)) + if missing_indexes: + raise AuditaLLMError( + f"Validator '{self.name}' omitted correction_index values: {missing_indexes}" + ) + + return [ + ValidationDecision( + proposal_index=proposal.proposal.proposal_index, + approved=indexed[proposal.proposal.proposal_index].approved, + confidence=indexed[proposal.proposal.proposal_index].confidence, + reason=indexed[proposal.proposal.proposal_index].reason, + ) + for proposal in proposals + ] + + +@dataclass(frozen=True) +class SpokenFormPlausibilityValidator(_BaseLLMValidator): + name: str = "spoken_form_plausibility_review" + prompt_builder: PromptBuilder = build_spoken_form_plausibility_messages + + +@dataclass(frozen=True) +class MeaningReversalValidator(_BaseLLMValidator): + name: str = "meaning_reversal_review" + prompt_builder: PromptBuilder = build_meaning_reversal_messages + + +def _write_json(path: Path, payload: dict) -> None: + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") diff --git a/src/audita/validators/prompts.py b/src/audita/validators/prompts.py new file mode 100644 index 0000000..5480b0f --- /dev/null +++ b/src/audita/validators/prompts.py @@ -0,0 +1,56 @@ +import json +from typing import Dict, List + + +Message = Dict[str, str] + + +def build_spoken_form_plausibility_messages(validation_payload: List[dict]) -> List[Message]: + payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2) + system = ( + "You are Audita, a conservative spoken-form validation assistant. " + "Evaluate whether each proposed correction is plausibly explained by a homophone, phonetic similarity, " + "or a common mistranscription of spoken English. " + "Your job is not to improve style or readability. " + "Approve only when the corrected text is a plausible recovery of the words that were likely spoken." + ) + user = ( + "Review these proposed transcript corrections and decide whether each one is a plausible spoken-form correction.\n\n" + "Rules:\n" + "- Return one validation decision for every correction_index in the input.\n" + "- Approve when the original text and corrected text are plausibly related by homophone confusion, " + "phonetic similarity, or a common spoken-word mistranscription, and the surrounding segment context supports the correction.\n" + "- Examples that may be approved when context supports them: changing \"gestures\" to \"Jesters\", " + "\"rank\" to \"Hrank\", or \"dam\" to \"damn\".\n" + "- Reject unrelated substitutions like changing \"Lyra\" to \"Jesters\".\n" + "- Judge spoken-form plausibility, not whether the correction is cleaner, more formal, or more grammatical.\n" + "- Do not approve paraphrases, stylistic rewrites, or arbitrary semantic substitutions.\n" + "- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" + "- confidence must be between 0.0 and 1.0.\n\n" + f"Corrections to validate:\n{payload_json}" + ) + return [{"role": "system", "content": system}, {"role": "user", "content": user}] + + +def build_meaning_reversal_messages(validation_payload: List[dict]) -> List[Message]: + payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2) + system = ( + "You are Audita, a narrow semantic-reversal validation assistant. " + "Evaluate whether each proposed correction changes a word to its antonym or otherwise reverses the meaning " + "of the full segment. " + "Do not treat every word substitution as a problem; focus specifically on antonyms and meaning reversals." + ) + user = ( + "Review these proposed transcript corrections and decide whether each one avoids reversing the segment meaning.\n\n" + "Rules:\n" + "- Return one validation decision for every correction_index in the input.\n" + "- Reject corrections that introduce antonyms or otherwise reverse the meaning of the original segment.\n" + "- Reject examples like changing \"visible\" to \"invisible\" or \"up\" to \"down\" when that reverses the segment meaning.\n" + "- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.\n" + "- Do not reject a correction merely because the literal written word changes.\n" + "- Do not act as a general semantic-style reviewer; this validator is only a guard against antonyms and meaning reversals.\n" + "- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" + "- confidence must be between 0.0 and 1.0.\n\n" + f"Corrections to validate:\n{payload_json}" + ) + return [{"role": "system", "content": system}, {"role": "user", "content": user}] diff --git a/tests/test_framework_runner.py b/tests/test_framework_runner.py index 3aea211..d2356ad 100644 --- a/tests/test_framework_runner.py +++ b/tests/test_framework_runner.py @@ -1,8 +1,9 @@ from audita.core.config import AuditaConfig +from audita.core.errors import AuditaLLMError from audita.core.schemas import parse_glossary_yaml, parse_transcript_json from audita.framework.models import CorrectionProposal, ModuleContext, ModuleRunSpec from audita.framework.runner import PipelineRunner -from audita.validators import ProtectedGlossaryTermsValidator +from audita.validators import MeaningReversalValidator, ProtectedGlossaryTermsValidator, SpokenFormPlausibilityValidator from audita.validators.base import ValidationContext, ValidationDecision, ValidationResult @@ -34,6 +35,25 @@ class RecordingLLMValidator(RecordingValidator): execution_kind = "llm" +class FakeStructuredLLMClient: + def __init__(self, responses): + self._responses = list(responses) + self.calls = [] + + def run_structured(self, *, stage_name, messages, response_model, config): + self.calls.append( + { + "stage_name": stage_name, + "messages": list(messages), + "response_model": response_model, + } + ) + if not self._responses: + raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.") + payload = self._responses.pop(0) + return response_model.model_validate(payload) + + class RecordingModule: replacement_policy = "require_unique" @@ -227,6 +247,89 @@ def test_pipeline_runner_supports_deterministic_and_llm_validators_in_one_chain( ] +def test_pipeline_runner_supports_real_llm_validators_in_one_chain(tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the dam."} + ] + """ + ) + glossary = parse_glossary_yaml( + """ + glossary: + - name: "Jesters" + category: faction + summary: "The Jesters are a faction." + """ + ) + module = RecordingModule( + "mixed_real", + [ + CorrectionProposal( + proposal_index=0, + module_instance="mixed_real", + module_key="glossary", + id=1, + original_text="gestures", + corrected_text="Jesters", + confidence=0.9, + ) + ], + [ + ProtectedGlossaryTermsValidator("protected_glossary_guard"), + SpokenFormPlausibilityValidator("spoken_form_plausibility_review"), + MeaningReversalValidator("meaning_reversal_review"), + ], + [], + ) + llm_client = FakeStructuredLLMClient( + [ + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.95, + "reason": "Likely phonetic mistranscription in context.", + } + ] + }, + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.98, + "reason": "Does not reverse the segment meaning.", + } + ] + }, + ] + ) + + runner = PipelineRunner() + result = runner.run( + transcript=transcript, + glossary=glossary, + module_specs=[ModuleRunSpec(instance_name="mixed_real", module_key="glossary", module=module)], + config=AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}), + run_dir=tmp_path / "run", + llm_client=llm_client, + ) + + assert result.transcript[0].text == "There were Jesters at the dam." + assert [report.execution_kind for report in result.module_reports[0].validators] == [ + "deterministic", + "llm", + "llm", + ] + assert [call["stage_name"] for call in llm_client.calls] == [ + "mixed_real:spoken_form_plausibility_review", + "mixed_real:meaning_reversal_review", + ] + + def test_pipeline_runner_uses_real_protected_glossary_validator(tmp_path): transcript = parse_transcript_json( """ diff --git a/tests/test_llm_validators.py b/tests/test_llm_validators.py new file mode 100644 index 0000000..c702bbf --- /dev/null +++ b/tests/test_llm_validators.py @@ -0,0 +1,453 @@ +import pytest + +from audita.core.chunking import TokenBatch +from audita.core.config import AuditaConfig +from audita.core.errors import AuditaLLMError +from audita.core.schemas import parse_glossary_yaml, parse_transcript_json +from audita.framework.models import CorrectionProposal, ModuleRunSpec +from audita.validators.base import ValidationContext +from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator +from audita.validators.prompts import ( + build_meaning_reversal_messages, + build_spoken_form_plausibility_messages, +) +import audita.validators.llm as llm_module + + +class _Module: + def __init__(self, replacement_policy: str) -> None: + self.replacement_policy = replacement_policy + + +class FakeStructuredLLMClient: + def __init__(self, responses): + self._responses = list(responses) + self.calls = [] + + def run_structured(self, *, stage_name, messages, response_model, config): + self.calls.append( + { + "stage_name": stage_name, + "messages": list(messages), + "response_model": response_model, + } + ) + if not self._responses: + raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.") + return response_model.model_validate(self._responses.pop(0)) + + +def _glossary(): + return parse_glossary_yaml( + """ + glossary: + - name: "Jesters" + category: faction + summary: "The Jesters are a faction." + - name: "Hrank" + category: pc + summary: "Hrank is a player character." + - name: "Lyra" + category: npc + summary: "Lyra is an NPC." + """ + ) + + +def _context( + *, + proposals, + transcript, + llm_client, + tmp_path, + replacement_policy="require_unique", +): + return ValidationContext( + proposals=proposals, + transcript=transcript, + glossary=_glossary(), + config=AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}), + run_spec=ModuleRunSpec( + instance_name="homophones", + module_key="homophones", + module=_Module(replacement_policy), + ), + run_dir=tmp_path, + llm_client=llm_client, + ) + + +def test_spoken_form_plausibility_validator_approves_plausible_and_rejects_implausible_corrections(tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}, + {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "Lyra moved first."} + ] + """ + ) + proposals = [ + CorrectionProposal( + proposal_index=0, + module_instance="homophones", + module_key="homophones", + id=1, + original_text="gestures", + corrected_text="Jesters", + confidence=0.95, + ), + CorrectionProposal( + proposal_index=1, + module_instance="homophones", + module_key="homophones", + id=2, + original_text="Lyra", + corrected_text="Jesters", + confidence=0.95, + ), + ] + client = FakeStructuredLLMClient( + [ + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.97, + "reason": "Likely spoken-form correction in context.", + }, + { + "correction_index": 1, + "approved": False, + "confidence": 0.99, + "reason": "Not plausibly related by homophone or mistranscription.", + }, + ] + } + ] + ) + + result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate( + _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) + ) + + assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [ + (0, True), + (1, False), + ] + prompt_text = client.calls[0]["messages"][1]["content"] + assert '"original_segment_text"' in prompt_text + assert "There were Jesters at the temple." in prompt_text + assert "Lyra moved first." in prompt_text + + +def test_meaning_reversal_validator_rejects_reversal_and_approves_nonreversal(tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam, but Claude actually can."}, + {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "The figure became visible in the doorway."} + ] + """ + ) + proposals = [ + CorrectionProposal( + proposal_index=0, + module_instance="homophones", + module_key="homophones", + id=1, + original_text="dam", + corrected_text="damn", + confidence=0.95, + ), + CorrectionProposal( + proposal_index=1, + module_instance="homophones", + module_key="homophones", + id=2, + original_text="visible", + corrected_text="invisible", + confidence=0.95, + ), + ] + client = FakeStructuredLLMClient( + [ + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.94, + "reason": "Does not reverse the segment meaning.", + }, + { + "correction_index": 1, + "approved": False, + "confidence": 1.0, + "reason": "Changes visible to invisible and reverses the segment meaning.", + }, + ] + } + ] + ) + + result = MeaningReversalValidator("meaning_reversal_review").validate( + _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) + ) + + assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [ + (0, True), + (1, False), + ] + prompt_text = client.calls[0]["messages"][1]["content"] + assert "The figure became visible in the doorway." in prompt_text + assert "The figure became invisible in the doorway." in prompt_text + + +@pytest.mark.parametrize( + ("validator", "payload", "message_fragment"), + [ + ( + SpokenFormPlausibilityValidator("spoken_form_plausibility_review"), + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.9, + "reason": "ok", + }, + { + "correction_index": 0, + "approved": True, + "confidence": 0.9, + "reason": "duplicate", + }, + ] + }, + "duplicate correction_index values", + ), + ( + MeaningReversalValidator("meaning_reversal_review"), + {"validations": []}, + "omitted correction_index values", + ), + ( + SpokenFormPlausibilityValidator("spoken_form_plausibility_review"), + { + "validations": [ + { + "correction_index": 99, + "approved": True, + "confidence": 0.9, + "reason": "unknown", + } + ] + }, + "unknown correction_index", + ), + ], +) +def test_llm_validators_reject_bad_correction_indexes(tmp_path, validator, payload, message_fragment): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."} + ] + """ + ) + proposals = [ + CorrectionProposal( + proposal_index=0, + module_instance="homophones", + module_key="homophones", + id=1, + original_text="gestures", + corrected_text="Jesters", + confidence=0.95, + ) + ] + client = FakeStructuredLLMClient([payload]) + + with pytest.raises(AuditaLLMError, match=message_fragment): + validator.validate(_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)) + + +def test_llm_validator_rejects_unpreviewable_proposals_without_calling_llm(tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."} + ] + """ + ) + proposals = [ + CorrectionProposal( + proposal_index=0, + module_instance="homophones", + module_key="homophones", + id=1, + original_text="rank", + corrected_text="Hrank", + confidence=0.95, + ) + ] + client = FakeStructuredLLMClient([]) + + result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate( + _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) + ) + + assert result.decisions[0].approved is False + assert result.decisions[0].reason == "proposal original_text does not match segment text" + assert client.calls == [] + + +def test_llm_validator_skips_empty_candidate_sets_without_calling_llm(tmp_path): + client = FakeStructuredLLMClient([]) + + result = MeaningReversalValidator("meaning_reversal_review").validate( + _context(proposals=[], transcript=[], llm_client=client, tmp_path=tmp_path) + ) + + assert result.decisions == [] + assert client.calls == [] + + +def test_spoken_form_prompt_emphasizes_acoustic_plausibility(): + messages = build_spoken_form_plausibility_messages( + [ + { + "correction_index": 0, + "id": 1, + "original_segment_text": "There were gestures at the temple.", + "corrected_segment_text": "There were Jesters at the temple.", + "original_text": "gestures", + "corrected_text": "Jesters", + } + ] + ) + + combined = messages[0]["content"] + messages[1]["content"] + assert "homophone" in combined + assert "gestures" in combined + assert "Lyra" in combined + assert '"original_segment_text"' in messages[1]["content"] + + +def test_meaning_reversal_prompt_emphasizes_antonyms_and_segment_context(): + messages = build_meaning_reversal_messages( + [ + { + "correction_index": 0, + "id": 1, + "original_segment_text": "The figure became visible in the doorway.", + "corrected_segment_text": "The figure became invisible in the doorway.", + "original_text": "visible", + "corrected_text": "invisible", + } + ] + ) + + combined = messages[0]["content"] + messages[1]["content"] + assert "antonym" in combined + assert "visible" in combined + assert "up" in combined + assert "original_segment_text" in messages[1]["content"] + + +def test_llm_validators_use_shared_token_batching_helper(monkeypatch, tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}, + {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "We saw rank near the gate."}, + {"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "ChatGPT still can't do that with a dam."} + ] + """ + ) + proposals = [ + CorrectionProposal( + proposal_index=0, + module_instance="homophones", + module_key="homophones", + id=1, + original_text="gestures", + corrected_text="Jesters", + confidence=0.95, + ), + CorrectionProposal( + proposal_index=1, + module_instance="homophones", + module_key="homophones", + id=2, + original_text="rank", + corrected_text="Hrank", + confidence=0.95, + ), + CorrectionProposal( + proposal_index=2, + module_instance="homophones", + module_key="homophones", + id=3, + original_text="dam", + corrected_text="damn", + confidence=0.95, + ), + ] + client = FakeStructuredLLMClient( + [ + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.9, + "reason": "ok", + } + ] + }, + { + "validations": [ + { + "correction_index": 1, + "approved": True, + "confidence": 0.9, + "reason": "ok", + }, + { + "correction_index": 2, + "approved": True, + "confidence": 0.9, + "reason": "ok", + }, + ] + }, + ] + ) + chunk_calls = [] + + def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message): + chunk_calls.append( + { + "max_tokens": max_tokens, + "payloads": [payload_fn(item) for item in items], + "empty_error_message": empty_error_message, + } + ) + return [ + TokenBatch(batch_index=0, items=list(items[:1]), token_count=1), + TokenBatch(batch_index=1, items=list(items[1:]), token_count=1), + ] + + monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items) + + result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate( + _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) + ) + + assert [decision.approved for decision in result.decisions] == [True, True, True] + assert len(client.calls) == 2 + assert len(chunk_calls) == 1 + assert chunk_calls[0]["max_tokens"] == AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}).max_section_tokens + assert all("corrected_segment_text" in payload for payload in chunk_calls[0]["payloads"]) diff --git a/tests/test_new_pipeline.py b/tests/test_new_pipeline.py index 4b55279..37436a8 100644 --- a/tests/test_new_pipeline.py +++ b/tests/test_new_pipeline.py @@ -76,8 +76,8 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy( assert (result.run_dir / "normalization" / "summary.json").exists() assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ "protected_glossary_guard", - "toward_glossary_term_review", - "context_support_review", + "spoken_form_plausibility_review", + "meaning_reversal_review", ] @@ -97,26 +97,18 @@ def test_default_module_specs_expose_final_validator_order(): assert [validator.name for validator in specs[0].module.validators()] == [ "protected_glossary_guard", - "toward_glossary_term_review", - "context_support_review", + "spoken_form_plausibility_review", + "meaning_reversal_review", ] assert [validator.name for validator in specs[1].module.validators()] == [ "protected_glossary_guard", - "acoustic_similarity_review", - "contextual_plausibility_review", - "antonym_reversal_review", + "spoken_form_plausibility_review", + "meaning_reversal_review", ] assert [validator.name for validator in specs[2].module.validators()] == [ "protected_glossary_guard", - "toward_glossary_term_review", - "context_support_review", - ] - assert [validator.name for validator in specs[3].module.validators()] == [ - "protected_glossary_guard", - "spoken_marker_cleanup_review", - "meaning_preservation_review", - ] - assert [validator.name for validator in specs[4].module.validators()] == [ - "protected_glossary_guard", - "edited_text_readability_review", + "spoken_form_plausibility_review", + "meaning_reversal_review", ] + assert [validator.name for validator in specs[3].module.validators()] == ["protected_glossary_guard"] + assert [validator.name for validator in specs[4].module.validators()] == ["protected_glossary_guard"]