From 2d1d21d314053b961b0bb85bd9575bbfdd943d7e Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 24 Apr 2026 13:17:36 -0500 Subject: [PATCH] Completed the MVP for the new Audita refactor --- README.md | 22 +- src/audita/cli.py | 18 +- src/audita/core/config.py | 24 ++ src/audita/framework/llm.py | 2 +- src/audita/framework/models.py | 10 +- src/audita/framework/proposal_generation.py | 86 ++++++ src/audita/framework/runner.py | 279 ++++++++++++-------- src/audita/modules/glossary.py | 14 +- src/audita/modules/grammar.py | 4 +- src/audita/modules/homophones.py | 14 +- src/audita/modules/prompts.py | 87 ++++++ src/audita/modules/spoken_word.py | 4 +- src/audita/pipeline.py | 117 ++++++-- src/audita/validators/__init__.py | 3 +- src/audita/validators/deterministic.py | 22 ++ tests/test_framework_runner.py | 11 +- tests/test_module_proposals.py | 273 +++++++++++++++++++ tests/test_new_cli.py | 3 +- tests/test_new_config.py | 32 +++ tests/test_new_pipeline.py | 253 +++++++++++++++++- 20 files changed, 1112 insertions(+), 166 deletions(-) create mode 100644 src/audita/framework/proposal_generation.py create mode 100644 src/audita/modules/prompts.py create mode 100644 tests/test_module_proposals.py diff --git a/README.md b/README.md index fcf0ba7..569b204 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # Audita -Audita is now a framework-first transcript correction application. The public `audita` package provides: +Audita is a framework-first transcript correction application. The public `audita` package provides: - deterministic transcript normalization - token-batched module orchestration -- reusable module / filter / review-stage contracts +- concrete `glossary` and `homophones` modules built on reusable proposal / validator contracts - structured run reporting and work-dir diagnostics The previous working implementation has been preserved as `audita_prototype` inside this repository. Its full regression suite lives under `tests/audita_prototype`. @@ -20,7 +20,7 @@ uv run pytest ## Usage -Process a transcript with the new framework skeleton: +Process a transcript with the current framework implementation: ```sh uv run audita process transcript.json --glossary glossary.yaml --output corrected.json @@ -34,7 +34,10 @@ The framework currently runs this default module sequence: 4. `spoken_word` 5. `grammar` -At this stage the module implementations are stubs. The framework is executable end to end, performs deterministic normalization, runs the full module lifecycle, and emits structured reports, but does not yet apply substantive LLM-driven corrections. +The default module sequence is partially implemented today: + +- `glossary_primary`, `homophones`, and `glossary_secondary` run real LLM-backed proposal and validation stages +- `spoken_word` and `grammar` remain stubs and currently propose no corrections To also write a structured JSON report: @@ -60,14 +63,16 @@ audita process transcript.json --glossary glossary.yaml --output corrected.json Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr. `--report-json` writes a separate machine-readable run report and never mixes report data into stdout. -Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. The framework does not currently require `OPENROUTER_API_KEY`, but the config fields remain available for future module implementations. +Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Normal runs now require `OPENROUTER_API_KEY`, because the `glossary` and `homophones` modules make real LLM calls. | Environment variable | CLI flag | Default | Purpose | | --- | --- | --- | --- | -| `AUDITA_MODEL` | `--model` | `openrouter/google/gemma-4-31b-it` | Reserved LLM model setting for future module implementations | -| `AUDITA_BASE_URL` | `--base-url` | `https://openrouter.ai/api/v1` | Reserved OpenAI-compatible API base URL | +| `AUDITA_MODEL` | `--model` | `openrouter/google/gemma-4-31b-it` | LLM model used by glossary and homophones proposal/validation stages | +| `AUDITA_BASE_URL` | `--base-url` | `https://openrouter.ai/api/v1` | OpenAI-compatible API base URL | | `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses | | `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `6144` | Maximum estimated tokens per transcript batch | +| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation | +| `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | `--homophones-confidence-threshold` | `0.8` | Minimum confidence required for homophone proposals to survive validation | | `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | `--normalize-max-segment-gap` | `4.0` | Same-speaker gaps eligible for deterministic merging | | `AUDITA_NORMALIZE_ELLIPSIS_GAP` | `--normalize-ellipsis-gap` | `3.5` | Same-speaker gaps above this value are joined with ` ... ` | | `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | `--normalize-max-segment-duration` | `60.0` | Maximum merged segment duration | @@ -76,7 +81,8 @@ Useful configuration can be supplied by CLI flag or environment variable. CLI fl | `AUDITA_WORK_DIR_RETENTION` | `--work-dir-retention` | `auto` | Whether to retain the per-run work directory: `auto`, `always`, or `never` | `AUDITA_WORK_DIR` stores per-run diagnostics while processing. Under the default `AUDITA_WORK_DIR_RETENTION=auto`, clean successful runs are removed, while failed runs and successful runs with final skipped corrections are preserved. Use `always` to keep every run directory and `never` to remove successful run directories even when skips remain. +Failed runs always preserve the run directory and include an authoritative `report.json` alongside normalization and prompt/response diagnostics. ## Prototype Archive -The archived prototype remains importable as `audita_prototype` and is still covered by its original regression suite. This is intentional: the new `audita` package is a fresh framework skeleton, not a thin wrapper around the old code. +The archived prototype remains importable as `audita_prototype` and is still covered by its original regression suite. This is intentional: the new `audita` package is a framework-oriented rewrite, not a thin wrapper around the old code. diff --git a/src/audita/cli.py b/src/audita/cli.py index 5860b8f..10c19ac 100644 --- a/src/audita/cli.py +++ b/src/audita/cli.py @@ -27,10 +27,20 @@ def _build_parser() -> argparse.ArgumentParser: process.add_argument("--glossary", type=Path, required=True, help="path to the glossary YAML") process.add_argument("--output", type=Path, help="write corrected transcript JSON to this path") process.add_argument("--report-json", type=Path, help="write structured run report JSON to this path") - process.add_argument("--model", help="OpenRouter model to use when module LLM stages are implemented") - process.add_argument("--base-url", help="OpenAI-compatible API base URL for future LLM stages") - process.add_argument("--max-retries", type=int, help="maximum structured-output retries for future LLM stages") + process.add_argument("--model", help="OpenRouter model for glossary and homophones LLM stages") + process.add_argument("--base-url", help="OpenAI-compatible API base URL for glossary and homophones LLM stages") + process.add_argument("--max-retries", type=int, help="maximum structured-output retries for LLM stages") process.add_argument("--max-section-tokens", type=int, help="maximum estimated tokens per transcript batch") + process.add_argument( + "--glossary-confidence-threshold", + type=float, + help="minimum confidence required for glossary proposals to survive validation", + ) + process.add_argument( + "--homophones-confidence-threshold", + type=float, + help="minimum confidence required for homophone proposals to survive validation", + ) process.add_argument( "--normalize-max-segment-gap", type=float, @@ -68,6 +78,8 @@ def _process(args: argparse.Namespace) -> int: base_url=args.base_url, max_retries=args.max_retries, max_section_tokens=args.max_section_tokens, + glossary_confidence_threshold=args.glossary_confidence_threshold, + homophones_confidence_threshold=args.homophones_confidence_threshold, normalize_max_segment_gap=args.normalize_max_segment_gap, normalize_ellipsis_gap=args.normalize_ellipsis_gap, normalize_max_segment_duration=args.normalize_max_segment_duration, diff --git a/src/audita/core/config.py b/src/audita/core/config.py index 3749e14..e642503 100644 --- a/src/audita/core/config.py +++ b/src/audita/core/config.py @@ -11,6 +11,8 @@ DEFAULT_MODEL = "openrouter/google/gemma-4-31b-it" DEFAULT_BASE_URL = "https://openrouter.ai/api/v1" DEFAULT_MAX_RETRIES = 3 DEFAULT_MAX_SECTION_TOKENS = 6144 +DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD = 0.80 +DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD = 0.80 DEFAULT_WORK_DIR = "/tmp/audita" DEFAULT_WORK_DIR_RETENTION = "auto" DEFAULT_NORMALIZE_MAX_SEGMENT_GAP = 4.0 @@ -25,6 +27,8 @@ class ConfigOverrides: base_url: Optional[str] = None max_retries: Optional[int] = None max_section_tokens: Optional[int] = None + glossary_confidence_threshold: Optional[float] = None + homophones_confidence_threshold: Optional[float] = None normalize_max_segment_gap: Optional[float] = None normalize_ellipsis_gap: Optional[float] = None normalize_max_segment_duration: Optional[float] = None @@ -40,6 +44,8 @@ class AuditaConfig: base_url: str = DEFAULT_BASE_URL max_retries: int = DEFAULT_MAX_RETRIES max_section_tokens: int = DEFAULT_MAX_SECTION_TOKENS + glossary_confidence_threshold: float = DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD + homophones_confidence_threshold: float = DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD normalize_max_segment_gap: float = DEFAULT_NORMALIZE_MAX_SEGMENT_GAP normalize_ellipsis_gap: float = DEFAULT_NORMALIZE_ELLIPSIS_GAP normalize_max_segment_duration: float = DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION @@ -71,6 +77,18 @@ class AuditaConfig: DEFAULT_MAX_SECTION_TOKENS, "AUDITA_MAX_SECTION_TOKENS", ), + glossary_confidence_threshold=_select_float( + selected.glossary_confidence_threshold, + source.get("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD"), + DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD, + "AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD", + ), + homophones_confidence_threshold=_select_float( + selected.homophones_confidence_threshold, + source.get("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD"), + DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD, + "AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD", + ), normalize_max_segment_gap=_select_float( selected.normalize_max_segment_gap, source.get("AUDITA_NORMALIZE_MAX_SEGMENT_GAP"), @@ -116,6 +134,10 @@ class AuditaConfig: raise AuditaConfigError("AUDITA_MAX_RETRIES must be greater than or equal to zero.") if self.max_section_tokens <= 0: raise AuditaConfigError("AUDITA_MAX_SECTION_TOKENS must be greater than zero.") + if not 0.0 <= self.glossary_confidence_threshold <= 1.0: + raise AuditaConfigError("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.") + if not 0.0 <= self.homophones_confidence_threshold <= 1.0: + raise AuditaConfigError("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.") if not math.isfinite(self.normalize_max_segment_gap): raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_GAP must be finite.") if not math.isfinite(self.normalize_ellipsis_gap): @@ -144,6 +166,8 @@ class AuditaConfig: "base_url": self.base_url, "max_retries": self.max_retries, "max_section_tokens": self.max_section_tokens, + "glossary_confidence_threshold": self.glossary_confidence_threshold, + "homophones_confidence_threshold": self.homophones_confidence_threshold, "normalize_max_segment_gap": self.normalize_max_segment_gap, "normalize_ellipsis_gap": self.normalize_ellipsis_gap, "normalize_max_segment_duration": self.normalize_max_segment_duration, diff --git a/src/audita/framework/llm.py b/src/audita/framework/llm.py index 0f1b0bc..4144830 100644 --- a/src/audita/framework/llm.py +++ b/src/audita/framework/llm.py @@ -50,7 +50,7 @@ class OpenRouterStructuredLLMClient: from openai import OpenAI except ImportError as exc: raise AuditaLLMError( - "The LLM dependencies are not installed. Run `uv sync` before using Audita validators." + "The LLM dependencies are not installed. Run `uv sync` before using Audita LLM stages." ) from exc openai_client = OpenAI(api_key=config.api_key, base_url=config.base_url) diff --git a/src/audita/framework/models.py b/src/audita/framework/models.py index b920e6f..3ba9a23 100644 --- a/src/audita/framework/models.py +++ b/src/audita/framework/models.py @@ -1,9 +1,12 @@ +from __future__ import annotations + from dataclasses import dataclass from pathlib import Path -from typing import Any, Protocol, Sequence +from typing import Any, Optional, Protocol, Sequence +from audita.core.chunking import TranscriptSection from audita.core.config import AuditaConfig -from audita.core.schemas import Glossary, TranscriptSegment +from audita.core.schemas import Glossary from audita.validators.base import Validator @@ -43,6 +46,7 @@ class ModuleContext: glossary: Glossary config: AuditaConfig run_dir: Path + llm_client: Optional["StructuredLLMClient"] = None class StructuredLLMClient(Protocol): @@ -66,7 +70,7 @@ class TranscriptModule(Protocol): def propose( self, - transcript_section: Sequence[TranscriptSegment], + transcript_section: TranscriptSection, context: ModuleContext, ) -> Sequence[CorrectionProposal]: ... diff --git a/src/audita/framework/proposal_generation.py b/src/audita/framework/proposal_generation.py new file mode 100644 index 0000000..09fbc8b --- /dev/null +++ b/src/audita/framework/proposal_generation.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import json +import math +from typing import Any, Callable, Dict, List + +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator + +from audita.core.chunking import TranscriptSection +from audita.core.errors import AuditaLLMError +from audita.core.schemas import Glossary + +from .models import CorrectionProposal, ModuleContext + + +Message = Dict[str, str] +ProposalPromptBuilder = Callable[[TranscriptSection, Glossary], List[Message]] + + +class StructuredCorrectionCandidate(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: int = Field(ge=1) + original_text: StrictStr + corrected_text: StrictStr + confidence: float = Field(ge=0.0, le=1.0) + + @field_validator("id", mode="before") + @classmethod + def require_integer_id(cls, value: Any) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError("must be an integer") + return value + + @field_validator("confidence", mode="before") + @classmethod + def require_number(cls, value: Any) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError("must be a JSON number") + number = float(value) + if not math.isfinite(number): + raise ValueError("must be finite") + return number + + +class StructuredCorrectionSet(BaseModel): + model_config = ConfigDict(extra="forbid") + + corrections: List[StructuredCorrectionCandidate] = Field(default_factory=list) + + +def generate_llm_correction_proposals( + *, + section: TranscriptSection, + context: ModuleContext, + prompt_builder: ProposalPromptBuilder, +) -> List[CorrectionProposal]: + llm_client = context.llm_client + if llm_client is None: + raise AuditaLLMError(f"Module '{context.run_spec.instance_name}' requires a structured LLM client.") + + messages = prompt_builder(section, context.glossary) + prompt_path = context.run_dir / f"prompt-{section.section_index:04d}.json" + prompt_path.write_text(json.dumps(messages, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + response = llm_client.run_structured( + stage_name=f"{context.run_spec.instance_name}:proposal", + messages=messages, + response_model=StructuredCorrectionSet, + config=context.config, + ) + response_path = context.run_dir / f"corrections-{section.section_index:04d}.json" + response_path.write_text(response.model_dump_json(indent=2) + "\n", encoding="utf-8") + + return [ + CorrectionProposal( + proposal_index=index, + module_instance=context.run_spec.instance_name, + module_key=context.run_spec.module_key, + id=correction.id, + original_text=correction.original_text, + corrected_text=correction.corrected_text, + confidence=correction.confidence, + ) + for index, correction in enumerate(response.corrections) + ] diff --git a/src/audita/framework/runner.py b/src/audita/framework/runner.py index c067f93..0f4ae0c 100644 --- a/src/audita/framework/runner.py +++ b/src/audita/framework/runner.py @@ -23,6 +23,40 @@ class PipelineRunResult: skipped_corrections: List[ReportedSkip] +class ModuleExecutionError(Exception): + def __init__( + self, + *, + transcript: List[TranscriptSegment], + applied_changes: List[AppliedChange], + skipped_corrections: List[ReportedSkip], + cause: Exception, + ) -> None: + super().__init__(str(cause)) + self.transcript = transcript + self.applied_changes = applied_changes + self.skipped_corrections = skipped_corrections + self.cause = cause + + +class PipelineRunError(Exception): + def __init__( + self, + *, + transcript: List[TranscriptSegment], + module_reports: List[ModuleRunReport], + applied_changes: List[AppliedChange], + skipped_corrections: List[ReportedSkip], + cause: Exception, + ) -> None: + super().__init__(str(cause)) + self.transcript = transcript + self.module_reports = module_reports + self.applied_changes = applied_changes + self.skipped_corrections = skipped_corrections + self.cause = cause + + class PipelineRunner: def run( self, @@ -41,21 +75,44 @@ class PipelineRunner: skipped_corrections: List[ReportedSkip] = [] for run_spec in module_specs: - module_dir = run_dir / run_spec.instance_name - module_dir.mkdir(parents=True, exist_ok=True) - context = ModuleContext(run_spec=run_spec, glossary=glossary, config=config, run_dir=module_dir) - sections = chunk_transcript(working, config.max_section_tokens) - if progress is not None: - progress( - f"Running module {run_spec.instance_name} " - f"({len(sections)} sections, {len(working)} segments)" + try: + module_dir = run_dir / run_spec.instance_name + module_dir.mkdir(parents=True, exist_ok=True) + context = ModuleContext( + run_spec=run_spec, + glossary=glossary, + config=config, + run_dir=module_dir, + llm_client=llm_client, ) - result = _run_module( - working=working, - sections=sections, - context=context, - llm_client=llm_client, - ) + sections = chunk_transcript(working, config.max_section_tokens) + if progress is not None: + progress( + f"Running module {run_spec.instance_name} " + f"({len(sections)} sections, {len(working)} segments)" + ) + result = _run_module( + working=working, + sections=sections, + context=context, + llm_client=llm_client, + ) + except ModuleExecutionError as exc: + raise PipelineRunError( + transcript=exc.transcript, + module_reports=list(module_reports), + applied_changes=[*applied_changes, *exc.applied_changes], + skipped_corrections=[*skipped_corrections, *exc.skipped_corrections], + cause=exc.cause, + ) from exc + except Exception as exc: + raise PipelineRunError( + transcript=list(working), + module_reports=list(module_reports), + applied_changes=list(applied_changes), + skipped_corrections=list(skipped_corrections), + cause=exc, + ) from exc working = result.transcript module_reports.append(result.module_report) applied_changes.extend(result.applied_changes) @@ -86,111 +143,119 @@ def _run_module( ) -> _ModuleExecutionResult: module = context.run_spec.module raw_proposals: List[CorrectionProposal] = [] - for section in sections: - proposed = list(module.propose([item.segment for item in section.segments], context)) - raw_proposals.extend(proposed) - - proposals = [ - CorrectionProposal( - proposal_index=index, - module_instance=context.run_spec.instance_name, - module_key=context.run_spec.module_key, - id=proposal.id, - original_text=proposal.original_text, - corrected_text=proposal.corrected_text, - confidence=proposal.confidence, - ) - for index, proposal in enumerate(raw_proposals) - ] - - surviving = proposals validator_reports: List[ValidatorReport] = [] skipped: List[ReportedSkip] = [] - for validator in module.validators(): - candidate_count = len(surviving) - if not surviving: + applied_changes: List[AppliedChange] = [] + updated_transcript = list(working) + try: + for section in sections: + proposed = list(module.propose(section, context)) + raw_proposals.extend(proposed) + + proposals = [ + CorrectionProposal( + proposal_index=index, + module_instance=context.run_spec.instance_name, + module_key=context.run_spec.module_key, + id=proposal.id, + original_text=proposal.original_text, + corrected_text=proposal.corrected_text, + confidence=proposal.confidence, + ) + for index, proposal in enumerate(raw_proposals) + ] + + surviving = proposals + for validator in module.validators(): + candidate_count = len(surviving) + if not surviving: + validator_reports.append( + ValidatorReport( + name=validator.name, + execution_kind=validator.execution_kind, + candidate_count=0, + approved_count=0, + rejected_count=0, + ) + ) + continue + + validation_context = ValidationContext( + proposals=surviving, + transcript=working, + glossary=context.glossary, + config=context.config, + run_spec=context.run_spec, + run_dir=context.run_dir, + llm_client=llm_client, + ) + result = validator.validate(validation_context) + decisions_by_index = _index_validation_decisions(result, surviving, validator.name) + approved: List[CorrectionProposal] = [] + rejected_count = 0 + for proposal in surviving: + decision = decisions_by_index[proposal.proposal_index] + if decision.approved: + approved.append(proposal) + continue + rejected_count += 1 + skipped.append( + ReportedSkip( + module_instance=proposal.module_instance, + module_key=proposal.module_key, + proposal_index=proposal.proposal_index, + id=proposal.id, + reason=decision.reason or f"{validator.name} rejected proposal", + original_text=proposal.original_text, + corrected_text=proposal.corrected_text, + confidence=proposal.confidence, + actual_text=_segment_text_by_id(working).get(proposal.id), + source=f"validator:{validator.name}", + ) + ) validator_reports.append( ValidatorReport( name=validator.name, execution_kind=validator.execution_kind, - candidate_count=0, - approved_count=0, - rejected_count=0, + candidate_count=candidate_count, + approved_count=len(approved), + rejected_count=rejected_count, ) ) - continue + surviving = approved - validation_context = ValidationContext( - proposals=surviving, - transcript=working, - glossary=context.glossary, - config=context.config, - run_spec=context.run_spec, - run_dir=context.run_dir, - llm_client=llm_client, - ) - result = validator.validate(validation_context) - decisions_by_index = _index_validation_decisions(result, surviving, validator.name) - approved: List[CorrectionProposal] = [] - rejected_count = 0 for proposal in surviving: - decision = decisions_by_index[proposal.proposal_index] - if decision.approved: - approved.append(proposal) + apply_result = _apply_proposal(updated_transcript, proposal, module.replacement_policy) + if isinstance(apply_result, ReportedSkip): + skipped.append(apply_result) continue - rejected_count += 1 - skipped.append( - ReportedSkip( - module_instance=proposal.module_instance, - module_key=proposal.module_key, - proposal_index=proposal.proposal_index, - id=proposal.id, - reason=decision.reason or f"{validator.name} rejected proposal", - original_text=proposal.original_text, - corrected_text=proposal.corrected_text, - confidence=proposal.confidence, - actual_text=_segment_text_by_id(working).get(proposal.id), - source=f"validator:{validator.name}", - ) - ) - validator_reports.append( - ValidatorReport( - name=validator.name, - execution_kind=validator.execution_kind, - candidate_count=candidate_count, - approved_count=len(approved), - rejected_count=rejected_count, - ) + updated_transcript, applied_change = apply_result + applied_changes.append(applied_change) + + module_report = ModuleRunReport( + instance_name=context.run_spec.instance_name, + module_key=context.run_spec.module_key, + replacement_policy=module.replacement_policy, + section_count=len(sections), + proposal_count=len(proposals), + validators=validator_reports, + approved_count=len(surviving), + applied_count=len(applied_changes), + skipped_count=len(skipped), ) - surviving = approved - - updated_transcript = list(working) - applied_changes: List[AppliedChange] = [] - for proposal in surviving: - apply_result = _apply_proposal(updated_transcript, proposal, module.replacement_policy) - if isinstance(apply_result, ReportedSkip): - skipped.append(apply_result) - continue - updated_transcript, applied_change = apply_result - applied_changes.append(applied_change) - - module_report = ModuleRunReport( - instance_name=context.run_spec.instance_name, - module_key=context.run_spec.module_key, - replacement_policy=module.replacement_policy, - section_count=len(sections), - proposal_count=len(proposals), - validators=validator_reports, - approved_count=len(surviving), - applied_count=len(applied_changes), - skipped_count=len(skipped), - ) - return _ModuleExecutionResult( - transcript=updated_transcript, - module_report=module_report, - applied_changes=applied_changes, - skipped_corrections=skipped, - ) + return _ModuleExecutionResult( + transcript=updated_transcript, + module_report=module_report, + applied_changes=applied_changes, + skipped_corrections=skipped, + ) + except Exception as exc: + raise ModuleExecutionError( + transcript=updated_transcript, + applied_changes=applied_changes, + skipped_corrections=skipped, + cause=exc, + ) from exc def _index_validation_decisions( diff --git a/src/audita/modules/glossary.py b/src/audita/modules/glossary.py index fdc8657..815cfec 100644 --- a/src/audita/modules/glossary.py +++ b/src/audita/modules/glossary.py @@ -1,9 +1,12 @@ from typing import Sequence -from audita.core.schemas import TranscriptSegment +from audita.core.chunking import TranscriptSection from audita.framework.models import CorrectionProposal, ModuleContext +from audita.framework.proposal_generation import generate_llm_correction_proposals +from audita.modules.prompts import build_glossary_proposal_messages from audita.validators import ( MeaningReversalValidator, + ProposalConfidenceValidator, ProtectedGlossaryTermsValidator, SpokenFormPlausibilityValidator, Validator, @@ -16,6 +19,7 @@ class GlossaryModule: def validators(self) -> Sequence[Validator]: return [ + ProposalConfidenceValidator("proposal_confidence_guard", "glossary_confidence_threshold"), ProtectedGlossaryTermsValidator("protected_glossary_guard"), SpokenFormPlausibilityValidator("spoken_form_plausibility_review"), MeaningReversalValidator("meaning_reversal_review"), @@ -23,7 +27,11 @@ class GlossaryModule: def propose( self, - transcript_section: Sequence[TranscriptSegment], + transcript_section: TranscriptSection, context: ModuleContext, ) -> Sequence[CorrectionProposal]: - return [] + return generate_llm_correction_proposals( + section=transcript_section, + context=context, + prompt_builder=build_glossary_proposal_messages, + ) diff --git a/src/audita/modules/grammar.py b/src/audita/modules/grammar.py index 968f38c..8cf896e 100644 --- a/src/audita/modules/grammar.py +++ b/src/audita/modules/grammar.py @@ -1,6 +1,6 @@ from typing import Sequence -from audita.core.schemas import TranscriptSegment +from audita.core.chunking import TranscriptSection from audita.framework.models import CorrectionProposal, ModuleContext from audita.validators import ProtectedGlossaryTermsValidator, Validator @@ -14,7 +14,7 @@ class GrammarModule: def propose( self, - transcript_section: Sequence[TranscriptSegment], + transcript_section: TranscriptSection, context: ModuleContext, ) -> Sequence[CorrectionProposal]: return [] diff --git a/src/audita/modules/homophones.py b/src/audita/modules/homophones.py index cfb4cf6..5174f63 100644 --- a/src/audita/modules/homophones.py +++ b/src/audita/modules/homophones.py @@ -1,9 +1,12 @@ from typing import Sequence -from audita.core.schemas import TranscriptSegment +from audita.core.chunking import TranscriptSection from audita.framework.models import CorrectionProposal, ModuleContext +from audita.framework.proposal_generation import generate_llm_correction_proposals +from audita.modules.prompts import build_homophones_proposal_messages from audita.validators import ( MeaningReversalValidator, + ProposalConfidenceValidator, ProtectedGlossaryTermsValidator, SpokenFormPlausibilityValidator, Validator, @@ -16,6 +19,7 @@ class HomophonesModule: def validators(self) -> Sequence[Validator]: return [ + ProposalConfidenceValidator("proposal_confidence_guard", "homophones_confidence_threshold"), ProtectedGlossaryTermsValidator("protected_glossary_guard"), SpokenFormPlausibilityValidator("spoken_form_plausibility_review"), MeaningReversalValidator("meaning_reversal_review"), @@ -23,7 +27,11 @@ class HomophonesModule: def propose( self, - transcript_section: Sequence[TranscriptSegment], + transcript_section: TranscriptSection, context: ModuleContext, ) -> Sequence[CorrectionProposal]: - return [] + return generate_llm_correction_proposals( + section=transcript_section, + context=context, + prompt_builder=build_homophones_proposal_messages, + ) diff --git a/src/audita/modules/prompts.py b/src/audita/modules/prompts.py new file mode 100644 index 0000000..88ef908 --- /dev/null +++ b/src/audita/modules/prompts.py @@ -0,0 +1,87 @@ +import json +from typing import Dict, List + +from audita.core.chunking import TranscriptSection +from audita.core.schemas import Glossary + + +Message = Dict[str, str] + + +def build_glossary_proposal_messages(section: TranscriptSection, glossary: Glossary) -> List[Message]: + glossary_json = json.dumps(glossary.model_dump(mode="json", exclude_none=True), ensure_ascii=False, indent=2) + section_json = json.dumps(section.prompt_payload(), ensure_ascii=False, indent=2) + + system = ( + "You are Audita, a careful transcript correction assistant. " + "Identify only transcription errors that are strongly supported by the glossary. " + "A valid correction must be acoustically plausible: the original transcript text " + "should sound similar to the proposed correction when spoken aloud. " + "Do not make generic grammar, spelling, capitalization, style, or filler-word edits. " + "Do not substitute an unrelated glossary term just because it could fit the topic. " + "Preserve speaker names, timestamps, and meaning." + ) + user = ( + "Review this transcript section and return only glossary-supported corrections that should be applied.\n\n" + "Rules:\n" + "- Correct domain-specific names, aliases, jargon, deities, locations, NPCs, players, factions, and similar terms only when both the glossary and surrounding transcript context support the correction.\n" + "- The correction must plausibly fix a transcription error: the original words should be phonetically or acoustically similar to the corrected words in spoken English.\n" + "- Appropriate example: correcting \"gestures\" to \"Jesters\" can be valid if \"Jesters\" appears in the glossary and nearby context supports that inference.\n" + "- Inappropriate example: correcting \"Lyra\" to \"Jesters\" should be omitted because those words are not similar in spoken English, even if \"Jesters\" appears in the glossary.\n" + "- Do not replace one clear glossary term, character name, location, or ordinary word with a different glossary term unless it is a plausible mishearing.\n" + "- Treat glossary names and aliases already present in the transcript as protected spellings.\n" + "- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.\n" + "- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n" + "- Plural forms of glossary names and aliases are allowed targets when spoken similarity and context support them, even if the plural is not explicitly listed in the glossary.\n" + "- Use the exact id from the input segment.\n" + "- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n" + "- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n" + "- Each returned correction must contain only id, original_text, corrected_text, and confidence.\n" + "- Do not return corrections where original_text and corrected_text are identical.\n" + "- Do not return speaker, start, or end fields.\n" + "- Return only changed segments; do not return entries for unchanged segments.\n" + "- confidence must be between 0.0 and 1.0.\n" + "- If no corrections are needed, return an empty corrections list.\n\n" + f"Glossary:\n{glossary_json}\n\n" + f"Transcript section:\n{section_json}" + ) + return [{"role": "system", "content": system}, {"role": "user", "content": user}] + + +def build_homophones_proposal_messages(section: TranscriptSection, glossary: Glossary) -> List[Message]: + glossary_json = json.dumps(glossary.model_dump(mode="json", exclude_none=True), ensure_ascii=False, indent=2) + section_json = json.dumps(section.prompt_payload(), ensure_ascii=False, indent=2) + + system = ( + "You are Audita, a conservative homophone correction assistant. " + "Identify only transcript changes that plausibly reflect homophones, phonetic similarity, " + "or common mistranscriptions of spoken English. " + "Do not make punctuation, capitalization, spacing, filler-word, repetition, style, or grammar edits. " + "Do not paraphrase, summarize, or rewrite content." + ) + user = ( + "Review this transcript section and return only homophone or spoken-form corrections that should be applied.\n\n" + "Rules:\n" + "- Approve only corrections where the original text is plausibly a mistaken homophone, phonetic rendering, or mistranscription of what was likely spoken.\n" + "- Allow examples such as changing \"dam\" to \"damn\", \"rank\" to \"Hrank\", or \"gestures\" to \"Jesters\" when local context supports the correction.\n" + "- Reject unrelated substitutions like changing \"Lyra\" to \"Jesters\".\n" + "- Reject antonyms or reversals such as changing \"visible\" to \"invisible\".\n" + "- Do not add or remove punctuation, alter capitalization only, normalize spacing, remove filler words, collapse repetitions, or make general readability edits.\n" + "- Treat glossary names and aliases as protected spellings and context.\n" + "- You may correct toward glossary names, aliases, or their plural forms when the correction is acoustically plausible and supported by local context.\n" + "- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.\n" + "- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n" + "- Use the exact id from the input segment.\n" + "- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n" + "- Choose an original_text span that appears exactly once in the current segment text.\n" + "- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n" + "- Each returned correction must contain only id, original_text, corrected_text, and confidence.\n" + "- Do not return corrections where original_text and corrected_text are identical.\n" + "- Do not return speaker, start, or end fields.\n" + "- Return only changed segments; do not return entries for unchanged segments.\n" + "- confidence must be between 0.0 and 1.0.\n" + "- If no corrections are needed, return an empty corrections list.\n\n" + f"Protected glossary/context:\n{glossary_json}\n\n" + f"Transcript section:\n{section_json}" + ) + return [{"role": "system", "content": system}, {"role": "user", "content": user}] diff --git a/src/audita/modules/spoken_word.py b/src/audita/modules/spoken_word.py index 295af2d..a771794 100644 --- a/src/audita/modules/spoken_word.py +++ b/src/audita/modules/spoken_word.py @@ -1,6 +1,6 @@ from typing import Sequence -from audita.core.schemas import TranscriptSegment +from audita.core.chunking import TranscriptSection from audita.framework.models import CorrectionProposal, ModuleContext from audita.validators import ProtectedGlossaryTermsValidator, Validator @@ -14,7 +14,7 @@ class SpokenWordModule: def propose( self, - transcript_section: Sequence[TranscriptSegment], + transcript_section: TranscriptSection, context: ModuleContext, ) -> Sequence[CorrectionProposal]: return [] diff --git a/src/audita/pipeline.py b/src/audita/pipeline.py index f983fbd..3bef871 100644 --- a/src/audita/pipeline.py +++ b/src/audita/pipeline.py @@ -8,10 +8,11 @@ from uuid import uuid4 from .core.config import AuditaConfig from .core.errors import AuditaError from .core.normalization import normalize_transcript -from .core.reporting import ProcessResult, RunReport +from .core.reporting import AppliedChange, ModuleRunReport, ProcessResult, ReportedSkip, RunReport from .core.schemas import Glossary, SourceTranscriptSegment, TranscriptSegment, source_transcript_to_json, transcript_to_json from .framework.llm import OpenRouterStructuredLLMClient -from .framework.runner import PipelineRunner +from .framework.models import StructuredLLMClient +from .framework.runner import PipelineRunError, PipelineRunner from .modules import default_module_specs @@ -22,19 +23,29 @@ def process_transcript( transcript: List[SourceTranscriptSegment], glossary: Glossary, config: AuditaConfig, + llm_client: Optional[StructuredLLMClient] = None, progress: Optional[ProgressCallback] = None, ) -> List[TranscriptSegment]: - return process_transcript_result(transcript, glossary, config, progress=progress).transcript + return process_transcript_result( + transcript, + glossary, + config, + llm_client=llm_client, + progress=progress, + ).transcript def process_transcript_result( transcript: List[SourceTranscriptSegment], glossary: Glossary, config: AuditaConfig, + llm_client: Optional[StructuredLLMClient] = None, progress: Optional[ProgressCallback] = None, ) -> ProcessResult: run_dir = _create_run_dir(config.work_dir) + module_specs = default_module_specs() normalized_transcript: List[TranscriptSegment] = [] + normalization_summary: Optional[dict] = None report: Optional[RunReport] = None try: _log(progress, f"Created work directory {run_dir}") @@ -46,6 +57,7 @@ def process_transcript_result( max_segment_tokens=config.normalize_max_segment_tokens, ) normalized_transcript = list(normalization_result.transcript) + normalization_summary = normalization_result.summary.to_dict() _write_normalization_diagnostics(run_dir, transcript, normalization_result) _log( progress, @@ -54,33 +66,28 @@ def process_transcript_result( f"{normalization_result.summary.normalized_segment_count} segments", ) - module_specs = default_module_specs() pipeline_runner = PipelineRunner() - llm_client = OpenRouterStructuredLLMClient() + effective_llm_client = OpenRouterStructuredLLMClient() if llm_client is None else llm_client pipeline_result = pipeline_runner.run( transcript=normalized_transcript, glossary=glossary, module_specs=module_specs, config=config, run_dir=run_dir, - llm_client=llm_client, + llm_client=effective_llm_client, progress=progress, ) revised = _sort_transcript_chronologically(pipeline_result.transcript) work_dir_retained = _should_retain_run_dir(config.work_dir_retention, bool(pipeline_result.skipped_corrections)) - report = RunReport( + report = _build_run_report( status="success", config=config.to_report_dict(), - normalization=normalization_result.summary.to_dict(), + normalization=normalization_summary, pipeline=[spec.instance_name for spec in module_specs], modules=pipeline_result.module_reports, applied_changes=pipeline_result.applied_changes, skipped_corrections=pipeline_result.skipped_corrections, - totals={ - "output_segment_count": len(revised), - "applied_change_count": len(pipeline_result.applied_changes), - "skipped_correction_count": len(pipeline_result.skipped_corrections), - }, + transcript=revised, work_dir_retention=config.work_dir_retention, work_dir_retained=work_dir_retained, work_dir=str(run_dir) if work_dir_retained else None, @@ -99,28 +106,26 @@ def process_transcript_result( work_dir_retained=work_dir_retained, ) except Exception as exc: - report = RunReport( + failed_report_data = _failure_report_data(exc, normalized_transcript) + report = _build_run_report( status="failed", config=config.to_report_dict(), - normalization=None, - pipeline=[spec.instance_name for spec in default_module_specs()], - modules=[], - applied_changes=[], - skipped_corrections=[], - totals={ - "output_segment_count": len(normalized_transcript), - "applied_change_count": 0, - "skipped_correction_count": 0, - }, + normalization=normalization_summary, + pipeline=[spec.instance_name for spec in module_specs], + modules=failed_report_data["modules"], + applied_changes=failed_report_data["applied_changes"], + skipped_corrections=failed_report_data["skipped_corrections"], + transcript=failed_report_data["transcript"], work_dir_retention=config.work_dir_retention, work_dir_retained=True, work_dir=str(run_dir), - error=str(exc), + error=str(failed_report_data["error"]), ) _write_run_report(run_dir / "report.json", report) - if isinstance(exc, AuditaError): - raise type(exc)(f"{exc} Diagnostics preserved at {run_dir}") from exc - raise AuditaError(f"{exc} Diagnostics preserved at {run_dir}") from exc + error = failed_report_data["error"] + if isinstance(error, AuditaError): + raise type(error)(f"{error} Diagnostics preserved at {run_dir}") from error + raise AuditaError(f"{error} Diagnostics preserved at {run_dir}") from error def _create_run_dir(root: Path) -> Path: @@ -164,3 +169,59 @@ def _write_run_report(path: Path, report: RunReport) -> None: def _log(progress: Optional[ProgressCallback], message: str) -> None: if progress is not None: progress(message) + + +def _build_run_report( + *, + status: str, + config: dict, + normalization: Optional[dict], + pipeline: List[str], + modules: List[ModuleRunReport], + applied_changes: List[AppliedChange], + skipped_corrections: List[ReportedSkip], + transcript: List[TranscriptSegment], + work_dir_retention: str, + work_dir_retained: bool, + work_dir: Optional[str], + error: Optional[str], +) -> RunReport: + return RunReport( + status=status, + config=config, + normalization=normalization, + pipeline=pipeline, + modules=modules, + applied_changes=applied_changes, + skipped_corrections=skipped_corrections, + totals={ + "output_segment_count": len(transcript), + "applied_change_count": len(applied_changes), + "skipped_correction_count": len(skipped_corrections), + }, + work_dir_retention=work_dir_retention, + work_dir_retained=work_dir_retained, + work_dir=work_dir, + error=error, + ) + + +def _failure_report_data( + exc: Exception, + normalized_transcript: List[TranscriptSegment], +) -> dict: + if isinstance(exc, PipelineRunError): + return { + "modules": exc.module_reports, + "applied_changes": exc.applied_changes, + "skipped_corrections": exc.skipped_corrections, + "transcript": _sort_transcript_chronologically(exc.transcript), + "error": exc.cause, + } + return { + "modules": [], + "applied_changes": [], + "skipped_corrections": [], + "transcript": _sort_transcript_chronologically(normalized_transcript), + "error": exc, + } diff --git a/src/audita/validators/__init__.py b/src/audita/validators/__init__.py index 6fe98aa..ef316b1 100644 --- a/src/audita/validators/__init__.py +++ b/src/audita/validators/__init__.py @@ -1,5 +1,5 @@ from .base import ValidationContext, ValidationDecision, ValidationResult, Validator -from .deterministic import ProtectedGlossaryTermsValidator +from .deterministic import ProposalConfidenceValidator, ProtectedGlossaryTermsValidator from .llm import MeaningReversalValidator, SpokenFormPlausibilityValidator from .protection import ProtectedVocabulary @@ -8,6 +8,7 @@ __all__ = [ "ValidationDecision", "ValidationResult", "Validator", + "ProposalConfidenceValidator", "ProtectedGlossaryTermsValidator", "ProtectedVocabulary", "SpokenFormPlausibilityValidator", diff --git a/src/audita/validators/deterministic.py b/src/audita/validators/deterministic.py index b355313..07a9fc2 100644 --- a/src/audita/validators/deterministic.py +++ b/src/audita/validators/deterministic.py @@ -4,6 +4,28 @@ from .base import ValidationContext, ValidationDecision, ValidationResult from .protection import ProtectedVocabulary +@dataclass(frozen=True) +class ProposalConfidenceValidator: + name: str + threshold_attr: str + execution_kind: str = "deterministic" + + def validate(self, context: ValidationContext) -> ValidationResult: + threshold = getattr(context.config, self.threshold_attr) + return ValidationResult( + validator_name=self.name, + execution_kind=self.execution_kind, + decisions=[ + ValidationDecision( + proposal_index=proposal.proposal_index, + approved=proposal.confidence >= threshold, + reason=None if proposal.confidence >= threshold else "proposal confidence below threshold", + ) + for proposal in context.proposals + ], + ) + + @dataclass(frozen=True) class ProtectedGlossaryTermsValidator: name: str diff --git a/tests/test_framework_runner.py b/tests/test_framework_runner.py index d2356ad..49b39f8 100644 --- a/tests/test_framework_runner.py +++ b/tests/test_framework_runner.py @@ -3,7 +3,12 @@ from audita.core.errors import AuditaLLMError from audita.core.schemas import parse_glossary_yaml, parse_transcript_json from audita.framework.models import CorrectionProposal, ModuleContext, ModuleRunSpec from audita.framework.runner import PipelineRunner -from audita.validators import MeaningReversalValidator, ProtectedGlossaryTermsValidator, SpokenFormPlausibilityValidator +from audita.validators import ( + MeaningReversalValidator, + ProposalConfidenceValidator, + ProtectedGlossaryTermsValidator, + SpokenFormPlausibilityValidator, +) from audita.validators.base import ValidationContext, ValidationDecision, ValidationResult @@ -67,7 +72,7 @@ class RecordingModule: return list(self._validators) def propose(self, transcript_section, context: ModuleContext): - self._recorder.append(("propose", [segment.text for segment in transcript_section])) + self._recorder.append(("propose", [item.segment.text for item in transcript_section.segments])) return list(self._proposals) @@ -277,6 +282,7 @@ def test_pipeline_runner_supports_real_llm_validators_in_one_chain(tmp_path): ) ], [ + ProposalConfidenceValidator("proposal_confidence_guard", "glossary_confidence_threshold"), ProtectedGlossaryTermsValidator("protected_glossary_guard"), SpokenFormPlausibilityValidator("spoken_form_plausibility_review"), MeaningReversalValidator("meaning_reversal_review"), @@ -320,6 +326,7 @@ def test_pipeline_runner_supports_real_llm_validators_in_one_chain(tmp_path): assert result.transcript[0].text == "There were Jesters at the dam." assert [report.execution_kind for report in result.module_reports[0].validators] == [ + "deterministic", "deterministic", "llm", "llm", diff --git a/tests/test_module_proposals.py b/tests/test_module_proposals.py new file mode 100644 index 0000000..7eade0d --- /dev/null +++ b/tests/test_module_proposals.py @@ -0,0 +1,273 @@ +from pathlib import Path + +from audita.core.chunking import chunk_transcript +from audita.core.config import AuditaConfig +from audita.core.errors import AuditaLLMError +from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_json, parse_transcript_json +from audita.framework.models import ModuleContext, ModuleRunSpec +from audita.modules.glossary import GlossaryModule +from audita.modules.homophones import HomophonesModule +from audita.modules.prompts import build_homophones_proposal_messages +from audita.pipeline import process_transcript_result + + +class FakeStructuredLLMClient: + def __init__(self, responses): + self._responses = list(responses) + self.calls = [] + + def run_structured(self, *, stage_name, messages, response_model, config): + self.calls.append( + { + "stage_name": stage_name, + "messages": list(messages), + "response_model": response_model, + } + ) + if not self._responses: + raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.") + return response_model.model_validate(self._responses.pop(0)) + + +def _glossary(): + return parse_glossary_yaml( + """ + glossary: + - name: "Jesters" + aliases: + - "Jester" + category: faction + summary: "A faction." + - name: "Hrank" + category: pc + summary: "A player character." + """ + ) + + +def test_glossary_module_propose_writes_diagnostics_and_returns_proposals_without_api_key(tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."} + ] + """ + ) + section = chunk_transcript(transcript, max_section_tokens=1000)[0] + module = GlossaryModule() + client = FakeStructuredLLMClient( + [ + { + "corrections": [ + { + "id": 1, + "original_text": "gestures", + "corrected_text": "Jesters", + "confidence": 0.95, + } + ] + } + ] + ) + context = ModuleContext( + run_spec=ModuleRunSpec(instance_name="glossary_primary", module_key="glossary", module=module), + glossary=_glossary(), + config=AuditaConfig.from_sources(env={}), + run_dir=tmp_path, + llm_client=client, + ) + + proposals = list(module.propose(section, context)) + + assert [(proposal.id, proposal.original_text, proposal.corrected_text, proposal.confidence) for proposal in proposals] == [ + (1, "gestures", "Jesters", 0.95) + ] + assert (tmp_path / "prompt-0000.json").exists() + assert (tmp_path / "corrections-0000.json").exists() + prompt_text = client.calls[0]["messages"][1]["content"] + assert "Glossary:" in prompt_text + assert "exact text span" in prompt_text + assert "gestures" in prompt_text + + +def test_homophones_prompt_is_explicitly_scoped_to_spoken_form_corrections(): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't do that with a dam."} + ] + """ + ) + section = chunk_transcript(transcript, max_section_tokens=1000)[0] + + messages = build_homophones_proposal_messages(section, _glossary()) + combined = messages[0]["content"] + messages[1]["content"] + + assert "homophone" in combined + assert "mistranscription" in combined + assert "Do not add or remove punctuation" in combined + assert "visible" in combined + assert '"id": 1' in messages[1]["content"] + + +def test_process_transcript_result_uses_injected_fake_client_and_applies_sequential_module_updates(tmp_path): + transcript = parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the dam."} + ] + """ + ) + config = AuditaConfig.from_sources( + env={}, + overrides=None, + ) + config = AuditaConfig( + api_key=config.api_key, + model=config.model, + base_url=config.base_url, + max_retries=config.max_retries, + max_section_tokens=config.max_section_tokens, + glossary_confidence_threshold=config.glossary_confidence_threshold, + homophones_confidence_threshold=config.homophones_confidence_threshold, + normalize_max_segment_gap=config.normalize_max_segment_gap, + normalize_ellipsis_gap=config.normalize_ellipsis_gap, + normalize_max_segment_duration=config.normalize_max_segment_duration, + normalize_max_segment_tokens=config.normalize_max_segment_tokens, + work_dir=tmp_path / "work", + work_dir_retention="always", + ) + client = FakeStructuredLLMClient( + [ + { + "corrections": [ + { + "id": 1, + "original_text": "gestures", + "corrected_text": "Jesters", + "confidence": 0.95, + } + ] + }, + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.97, + "reason": "Likely spoken-form correction in context.", + } + ] + }, + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.99, + "reason": "Does not reverse the segment meaning.", + } + ] + }, + { + "corrections": [ + { + "id": 1, + "original_text": "dam", + "corrected_text": "damn", + "confidence": 0.92, + } + ] + }, + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.98, + "reason": "Likely spoken-form correction in context.", + } + ] + }, + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.99, + "reason": "Does not reverse the segment meaning.", + } + ] + }, + {"corrections": []}, + ] + ) + + result = process_transcript_result(transcript, _glossary(), config, llm_client=client) + + assert result.transcript[0].text == "There were Jesters at the damn." + assert [call["stage_name"] for call in client.calls] == [ + "glossary_primary:proposal", + "glossary_primary:spoken_form_plausibility_review", + "glossary_primary:meaning_reversal_review", + "homophones:proposal", + "homophones:spoken_form_plausibility_review", + "homophones:meaning_reversal_review", + "glossary_secondary:proposal", + ] + assert "There were Jesters at the dam." in client.calls[3]["messages"][1]["content"] + + +def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_validators(tmp_path): + transcript = parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."} + ] + """ + ) + base_config = AuditaConfig.from_sources(env={}) + config = AuditaConfig( + api_key=base_config.api_key, + model=base_config.model, + base_url=base_config.base_url, + max_retries=base_config.max_retries, + max_section_tokens=base_config.max_section_tokens, + glossary_confidence_threshold=0.96, + homophones_confidence_threshold=base_config.homophones_confidence_threshold, + normalize_max_segment_gap=base_config.normalize_max_segment_gap, + normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, + normalize_max_segment_duration=base_config.normalize_max_segment_duration, + normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, + work_dir=tmp_path / "work", + work_dir_retention="always", + ) + client = FakeStructuredLLMClient( + [ + { + "corrections": [ + { + "id": 1, + "original_text": "gestures", + "corrected_text": "Jesters", + "confidence": 0.95, + } + ] + }, + {"corrections": []}, + {"corrections": []}, + ] + ) + + result = process_transcript_result(transcript, _glossary(), config, llm_client=client) + + assert result.transcript[0].text == "There were gestures at the temple." + assert [call["stage_name"] for call in client.calls] == [ + "glossary_primary:proposal", + "homophones:proposal", + "glossary_secondary:proposal", + ] + assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold" + assert result.report.skipped_corrections[0].source == "validator:proposal_confidence_guard" + assert result.report.modules[0].validators[0].rejected_count == 1 + assert result.report.modules[0].validators[1].candidate_count == 0 diff --git a/tests/test_new_cli.py b/tests/test_new_cli.py index a9ec34b..8b08665 100644 --- a/tests/test_new_cli.py +++ b/tests/test_new_cli.py @@ -24,9 +24,10 @@ def test_process_help_exposes_framework_flags(capsys): assert "--base-url" in output assert "--max-retries" in output assert "--max-section-tokens" in output + assert "--glossary-confidence-threshold" in output + assert "--homophones-confidence-threshold" in output assert "--work-dir-retention" in output assert "--normalize-max-segment-gap" in output - assert "--glossary-confidence-threshold" not in output assert "--grammar-validation-enabled" not in output diff --git a/tests/test_new_config.py b/tests/test_new_config.py index e9ee00c..40e82bb 100644 --- a/tests/test_new_config.py +++ b/tests/test_new_config.py @@ -3,6 +3,8 @@ import pytest from audita.core.config import ( AuditaConfig, ConfigOverrides, + DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD, + DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD, DEFAULT_NORMALIZE_MAX_SEGMENT_GAP, DEFAULT_WORK_DIR_RETENTION, ) @@ -13,6 +15,8 @@ def test_default_config_allows_missing_api_key(): config = AuditaConfig.from_sources(env={}) assert config.api_key is None + assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD + assert config.homophones_confidence_threshold == DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD assert config.normalize_max_segment_gap == DEFAULT_NORMALIZE_MAX_SEGMENT_GAP assert config.work_dir_retention == DEFAULT_WORK_DIR_RETENTION @@ -29,3 +33,31 @@ def test_cli_overrides_take_precedence(): def test_invalid_work_dir_retention_is_rejected(): with pytest.raises(AuditaConfigError): AuditaConfig.from_sources(env={"AUDITA_WORK_DIR_RETENTION": "sometimes"}) + + +def test_threshold_overrides_take_precedence(): + config = AuditaConfig.from_sources( + env={ + "AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.6", + "AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.7", + }, + overrides=ConfigOverrides( + glossary_confidence_threshold=0.85, + homophones_confidence_threshold=0.9, + ), + ) + + assert config.glossary_confidence_threshold == 0.85 + assert config.homophones_confidence_threshold == 0.9 + + +@pytest.mark.parametrize( + "env_name", + [ + "AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD", + "AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD", + ], +) +def test_invalid_thresholds_are_rejected(env_name): + with pytest.raises(AuditaConfigError): + AuditaConfig.from_sources(env={env_name: "1.5"}) diff --git a/tests/test_new_pipeline.py b/tests/test_new_pipeline.py index 37436a8..ed6bf0f 100644 --- a/tests/test_new_pipeline.py +++ b/tests/test_new_pipeline.py @@ -1,12 +1,36 @@ import json +import pytest + from audita.core.config import AuditaConfig +from audita.core.errors import AuditaLLMError from audita.core.io import write_report from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_json from audita.modules import default_module_specs from audita.pipeline import process_transcript, process_transcript_result +class FakeStructuredLLMClient: + def __init__(self, responses): + self._responses = list(responses) + self.calls = [] + + def run_structured(self, *, stage_name, messages, response_model, config): + self.calls.append( + { + "stage_name": stage_name, + "messages": list(messages), + "response_model": response_model, + } + ) + if not self._responses: + raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.") + response = self._responses.pop(0) + if isinstance(response, Exception): + raise response + return response_model.model_validate(response) + + def _glossary(): return parse_glossary_yaml( """ @@ -31,15 +55,28 @@ def _transcript(): def test_process_transcript_runs_noop_framework(tmp_path): + llm_client = FakeStructuredLLMClient( + [ + {"corrections": []}, + {"corrections": []}, + {"corrections": []}, + ] + ) revised = process_transcript( _transcript(), _glossary(), AuditaConfig.from_sources(env={}, overrides=None), + llm_client=llm_client, ) assert [segment.id for segment in revised] == [1, 2] assert revised[0].text == "Hello. Again." assert revised[1].text == "Done." + assert [call["stage_name"] for call in llm_client.calls] == [ + "glossary_primary:proposal", + "homophones:proposal", + "glossary_secondary:proposal", + ] def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(tmp_path): @@ -53,6 +90,8 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy( base_url=config.base_url, max_retries=config.max_retries, max_section_tokens=config.max_section_tokens, + glossary_confidence_threshold=config.glossary_confidence_threshold, + homophones_confidence_threshold=config.homophones_confidence_threshold, normalize_max_segment_gap=config.normalize_max_segment_gap, normalize_ellipsis_gap=config.normalize_ellipsis_gap, normalize_max_segment_duration=config.normalize_max_segment_duration, @@ -61,7 +100,14 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy( work_dir_retention="always", ) - result = process_transcript_result(_transcript(), _glossary(), config) + llm_client = FakeStructuredLLMClient( + [ + {"corrections": []}, + {"corrections": []}, + {"corrections": []}, + ] + ) + result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client) assert result.work_dir_retained is True assert result.report.pipeline == [ @@ -75,6 +121,7 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy( assert (result.run_dir / "report.json").exists() assert (result.run_dir / "normalization" / "summary.json").exists() assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ + "proposal_confidence_guard", "protected_glossary_guard", "spoken_form_plausibility_review", "meaning_reversal_review", @@ -83,7 +130,14 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy( def test_external_report_can_be_written(tmp_path): config = AuditaConfig.from_sources(env={}) - result = process_transcript_result(_transcript(), _glossary(), config) + llm_client = FakeStructuredLLMClient( + [ + {"corrections": []}, + {"corrections": []}, + {"corrections": []}, + ] + ) + result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client) report_path = tmp_path / "report.json" write_report(report_path, result.report) @@ -96,19 +150,214 @@ def test_default_module_specs_expose_final_validator_order(): specs = default_module_specs() assert [validator.name for validator in specs[0].module.validators()] == [ + "proposal_confidence_guard", "protected_glossary_guard", "spoken_form_plausibility_review", "meaning_reversal_review", ] assert [validator.name for validator in specs[1].module.validators()] == [ + "proposal_confidence_guard", "protected_glossary_guard", "spoken_form_plausibility_review", "meaning_reversal_review", ] assert [validator.name for validator in specs[2].module.validators()] == [ + "proposal_confidence_guard", "protected_glossary_guard", "spoken_form_plausibility_review", "meaning_reversal_review", ] assert [validator.name for validator in specs[3].module.validators()] == ["protected_glossary_guard"] assert [validator.name for validator in specs[4].module.validators()] == ["protected_glossary_guard"] + + +def test_process_transcript_result_missing_api_key_writes_failed_report(tmp_path): + config = AuditaConfig.from_sources( + env={}, + overrides=None, + ) + config = AuditaConfig( + api_key=None, + model=config.model, + base_url=config.base_url, + max_retries=config.max_retries, + max_section_tokens=config.max_section_tokens, + glossary_confidence_threshold=config.glossary_confidence_threshold, + homophones_confidence_threshold=config.homophones_confidence_threshold, + normalize_max_segment_gap=config.normalize_max_segment_gap, + normalize_ellipsis_gap=config.normalize_ellipsis_gap, + normalize_max_segment_duration=config.normalize_max_segment_duration, + normalize_max_segment_tokens=config.normalize_max_segment_tokens, + work_dir=tmp_path / "work", + work_dir_retention="never", + ) + + with pytest.raises(AuditaLLMError, match="OPENROUTER_API_KEY"): + process_transcript_result(_transcript(), _glossary(), config) + + run_dir = next((tmp_path / "work").iterdir()) + report = json.loads((run_dir / "report.json").read_text(encoding="utf-8")) + + assert report["status"] == "failed" + assert report["normalization"]["normalized_segment_count"] == 2 + assert report["pipeline"] == [ + "glossary_primary", + "homophones", + "glossary_secondary", + "spoken_word", + "grammar", + ] + assert report["modules"] == [] + assert report["applied_changes"] == [] + assert report["skipped_corrections"] == [] + assert report["work_dir_retained"] is True + assert report["work_dir"] == str(run_dir) + assert "OPENROUTER_API_KEY" in report["error"] + + +def test_process_transcript_result_preserves_partial_progress_when_later_module_fails(tmp_path): + transcript = parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the dam."} + ] + """ + ) + config = AuditaConfig.from_sources( + env={}, + overrides=None, + ) + config = AuditaConfig( + api_key=config.api_key, + model=config.model, + base_url=config.base_url, + max_retries=config.max_retries, + max_section_tokens=config.max_section_tokens, + glossary_confidence_threshold=config.glossary_confidence_threshold, + homophones_confidence_threshold=config.homophones_confidence_threshold, + normalize_max_segment_gap=config.normalize_max_segment_gap, + normalize_ellipsis_gap=config.normalize_ellipsis_gap, + normalize_max_segment_duration=config.normalize_max_segment_duration, + normalize_max_segment_tokens=config.normalize_max_segment_tokens, + work_dir=tmp_path / "work", + work_dir_retention="never", + ) + llm_client = FakeStructuredLLMClient( + [ + { + "corrections": [ + { + "id": 1, + "original_text": "gestures", + "corrected_text": "Jesters", + "confidence": 0.95, + } + ] + }, + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.97, + "reason": "Likely spoken-form correction in context.", + } + ] + }, + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.99, + "reason": "Does not reverse the segment meaning.", + } + ] + }, + AuditaLLMError("Simulated homophones proposal failure."), + ] + ) + + with pytest.raises(AuditaLLMError, match="Simulated homophones proposal failure"): + process_transcript_result(transcript, _glossary(), config, llm_client=llm_client) + + run_dir = next((tmp_path / "work").iterdir()) + report = json.loads((run_dir / "report.json").read_text(encoding="utf-8")) + + assert report["status"] == "failed" + assert report["normalization"]["normalized_segment_count"] == 1 + assert [module["instance_name"] for module in report["modules"]] == ["glossary_primary"] + assert report["applied_changes"][0]["corrected_text"] == "Jesters" + assert report["applied_changes"][0]["segment_text_after"] == "There were Jesters at the dam." + assert report["totals"]["applied_change_count"] == 1 + assert report["skipped_corrections"] == [] + assert report["pipeline"][1] == "homophones" + assert "Simulated homophones proposal failure." in report["error"] + + +def test_process_transcript_result_preserves_partial_skips_and_validator_diagnostics_on_failure(tmp_path): + config = AuditaConfig.from_sources( + env={}, + overrides=None, + ) + config = AuditaConfig( + api_key=config.api_key, + model=config.model, + base_url=config.base_url, + max_retries=config.max_retries, + max_section_tokens=config.max_section_tokens, + glossary_confidence_threshold=0.8, + homophones_confidence_threshold=config.homophones_confidence_threshold, + normalize_max_segment_gap=config.normalize_max_segment_gap, + normalize_ellipsis_gap=config.normalize_ellipsis_gap, + normalize_max_segment_duration=config.normalize_max_segment_duration, + normalize_max_segment_tokens=config.normalize_max_segment_tokens, + work_dir=tmp_path / "work", + work_dir_retention="never", + ) + llm_client = FakeStructuredLLMClient( + [ + { + "corrections": [ + { + "id": 1, + "original_text": "Hello", + "corrected_text": "Jesters", + "confidence": 0.40, + }, + { + "id": 1, + "original_text": "Hello", + "corrected_text": "Jesters", + "confidence": 0.95, + }, + ] + }, + { + "validations": [ + { + "correction_index": 99, + "approved": True, + "confidence": 0.98, + "reason": "Malformed response for testing.", + } + ] + }, + ] + ) + + with pytest.raises(AuditaLLMError, match="unknown correction_index"): + process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client) + + run_dir = next((tmp_path / "work").iterdir()) + report = json.loads((run_dir / "report.json").read_text(encoding="utf-8")) + validator_dir = run_dir / "glossary_primary" + + assert report["status"] == "failed" + assert report["modules"] == [] + assert len(report["skipped_corrections"]) == 1 + assert report["skipped_corrections"][0]["reason"] == "proposal confidence below threshold" + assert report["skipped_corrections"][0]["source"] == "validator:proposal_confidence_guard" + assert "unknown correction_index" in report["error"] + assert (validator_dir / "spoken_form_plausibility_review-prompt-0000.json").exists() + assert (validator_dir / "spoken_form_plausibility_review-response-0000.json").exists()