Move python implementation under python/ in preparation for the upcoming Go rewrite

This commit is contained in:
2026-05-10 22:37:38 +00:00
parent e797e3d9ff
commit 2e47c8a1b6
47 changed files with 0 additions and 0 deletions

147
python/audita Executable file
View File

@@ -0,0 +1,147 @@
#!/usr/bin/env python3
import json
import os
from datetime import datetime
import shutil
import subprocess
import sys
from pathlib import Path
from typing import Optional
from uuid import uuid4
DEFAULT_WORK_DIR = "/tmp/audita"
_SECRET_FLAGS = {"--llm-api-key", "--validation-llm-api-key"}
def _redact_argv(argv: list[str]) -> list[str]:
redacted: list[str] = []
index = 0
while index < len(argv):
arg = argv[index]
matched_flag = next((flag for flag in _SECRET_FLAGS if arg == flag or arg.startswith(flag + "=")), None)
if matched_flag is None:
redacted.append(arg)
index += 1
continue
if arg == matched_flag:
redacted.append(arg)
if index + 1 < len(argv):
redacted.append("[REDACTED]")
index += 2
else:
index += 1
continue
redacted.append(f"{matched_flag}=[REDACTED]")
index += 1
return redacted
def _resolve_work_root(argv: list[str]) -> Path:
for index, arg in enumerate(argv):
if arg == "--work-dir" and index + 1 < len(argv):
return Path(argv[index + 1])
if arg.startswith("--work-dir="):
return Path(arg.split("=", 1)[1])
return Path(os.environ.get("AUDITA_WORK_DIR") or DEFAULT_WORK_DIR)
def _create_run_dir(root: Path) -> Path:
root.mkdir(parents=True, exist_ok=True)
timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
run_dir = root / f"run-{timestamp}-{uuid4().hex[:8]}"
run_dir.mkdir(parents=False, exist_ok=False)
return run_dir
def _capture_run_dirs(root: Path) -> set[str]:
if not root.exists():
return set()
return {path.name for path in root.iterdir() if path.is_dir() and path.name.startswith("run-")}
def _find_new_run_dir(root: Path, before: set[str]) -> Optional[Path]:
if not root.exists():
return None
candidates = [
path for path in root.iterdir() if path.is_dir() and path.name.startswith("run-") and path.name not in before
]
if not candidates:
return None
return max(candidates, key=lambda path: path.name)
def _write_launcher_error_log(
path: Path,
*,
message: str,
exit_code: int,
argv: list[str],
command: Optional[list[str]],
) -> None:
payload = {
"timestamp": datetime.utcnow().isoformat() + "Z",
"message": message,
"exit_code": exit_code,
"argv": argv,
"cwd": os.getcwd(),
"command": command,
}
path.write_text(
"Audita Launcher Diagnostics\n" + json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
def _emit_console_line(message: str) -> None:
for stream in (sys.stderr, sys.stdout):
if stream is None:
continue
try:
stream.write(f"{message}\n")
stream.flush()
return
except (OSError, ValueError):
continue
def main() -> int:
argv = list(sys.argv[1:])
work_root = _resolve_work_root(argv)
redacted_argv = _redact_argv(argv)
uv = shutil.which("uv")
if uv is None:
run_dir = _create_run_dir(work_root)
error_log = run_dir / "error.log"
message = "uv is required to run this launcher. Install uv and run `uv sync` in the Audita project."
_write_launcher_error_log(error_log, message=message, exit_code=1, argv=redacted_argv, command=None)
_emit_console_line(f"audita: error: {message}")
_emit_console_line("audita: exit code: 1")
_emit_console_line(f"audita: run directory: {run_dir}")
_emit_console_line(f"audita: error log: {error_log}")
return 1
project_root = Path(__file__).resolve().parent
command = [uv, "run", "--project", str(project_root), "python", "-m", "audita", *sys.argv[1:]]
before = _capture_run_dirs(work_root)
result = subprocess.run(command, cwd=project_root, check=False)
if result.returncode == 0:
return 0
if _find_new_run_dir(work_root, before) is None:
run_dir = _create_run_dir(work_root)
error_log = run_dir / "error.log"
_write_launcher_error_log(
error_log,
message=f"Audita subprocess exited with status {result.returncode}.",
exit_code=result.returncode,
argv=redacted_argv,
command=_redact_argv(command),
)
_emit_console_line(f"audita: subprocess exited with status {result.returncode}")
_emit_console_line(f"audita: run directory: {run_dir}")
_emit_console_line(f"audita: error log: {error_log}")
return result.returncode
if __name__ == "__main__":
raise SystemExit(main())

35
python/pyproject.toml Normal file
View File

@@ -0,0 +1,35 @@
[project]
name = "audita"
version = "0.2.0"
description = "Framework-first audio transcript correction pipeline with an archived prototype."
readme = "README.md"
requires-python = ">=3.9"
license = { text = "BSD-3-Clause" }
dependencies = [
"instructor>=1.7",
"openai>=1.55",
"pydantic>=2.8",
"PyYAML>=6.0",
"tiktoken>=0.8",
"eval-type-backport>=0.2.0; python_version < '3.10'",
]
[project.optional-dependencies]
dev = [
"pytest>=8.0",
]
[project.scripts]
audita = "audita.cli:main"
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/audita", "src/audita_prototype"]
[tool.pytest.ini_options]
pythonpath = ["src"]
testpaths = ["tests"]
addopts = "-ra"

View File

@@ -0,0 +1,5 @@
"""Audita modular transcript correction framework."""
__all__ = ["__version__"]
__version__ = "0.2.0"

View File

@@ -0,0 +1,5 @@
from .cli import main
if __name__ == "__main__":
raise SystemExit(main())

253
python/src/audita/cli.py Normal file
View File

@@ -0,0 +1,253 @@
import argparse
import os
import sys
from pathlib import Path
from typing import Optional, Sequence
from .core.config import AuditaConfig, ConfigOverrides
from .core.diagnostics import build_error_details, create_run_dir, format_stderr_summary, format_traceback_text, redact_argv, write_error_log
from .core.errors import AuditaError
from .core.io import load_glossary, load_transcript, write_report, write_transcript
from .core.reporting import RunReport
from .core.schemas import transcript_to_json
from .pipeline import process_transcript_result
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = _build_parser()
args = parser.parse_args(argv)
if args.command == "process":
return _process(args, raw_argv=argv)
parser.print_help(sys.stderr)
return 2
def _build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="audita")
subparsers = parser.add_subparsers(dest="command", required=True)
process = subparsers.add_parser("process", help="run the modular transcript correction framework")
process.add_argument("transcript", type=Path, help="path to the input transcript JSON")
process.add_argument("--glossary", type=Path, required=True, help="path to the glossary YAML")
process.add_argument("--output", type=Path, help="write corrected transcript JSON to this path")
process.add_argument("--report-json", type=Path, help="write structured run report JSON to this path")
process.add_argument("--llm-api-key", help="LLM API key for the configured OpenAI-compatible endpoint")
process.add_argument("--llm-concurrency", type=int, help="maximum concurrent LLM calls within a module stage")
process.add_argument(
"--llm-timeout-seconds",
type=float,
help="per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint",
)
process.add_argument(
"--validation-llm-api-key",
help="LLM API key for validation phases; defaults to the primary configured OpenAI-compatible endpoint key",
)
process.add_argument(
"--validation-llm-concurrency",
type=int,
help="maximum concurrent LLM calls within validation phases; defaults to --llm-concurrency",
)
process.add_argument(
"--validation-llm-timeout-seconds",
type=float,
help="per-request timeout in seconds for validation-phase LLM calls; defaults to --llm-timeout-seconds",
)
process.add_argument("--validation-model", help="LLM model name for validation phases; defaults to --model")
process.add_argument(
"--validation-base-url",
help="OpenAI-compatible API base URL for validation phases; defaults to --base-url",
)
process.add_argument(
"--validation-max-retries",
type=int,
help="maximum structured-output retries for validation phases; defaults to --max-retries",
)
process.add_argument(
"--validation-max-prompt-tokens",
type=int,
help="maximum estimated tokens per validation-phase LLM prompt batch",
)
process.add_argument(
"--target-sections",
type=int,
help="exact number of contiguous proposal-stage transcript sections to create",
)
process.add_argument(
"--modules",
help="comma-separated module keys to run, for example: grammar or glossary,homophones,glossary",
)
process.add_argument("--model", help="LLM model name for the configured OpenAI-compatible endpoint")
process.add_argument("--base-url", help="OpenAI-compatible API base URL for Audita LLM stages")
process.add_argument("--max-retries", type=int, help="maximum structured-output retries for LLM stages")
process.add_argument("--max-section-tokens", type=int, help="maximum estimated tokens per transcript batch")
process.add_argument(
"--min-section-tokens",
type=int,
help="minimum estimated tokens per transcript batch when balancing proposal-stage sections",
)
process.add_argument(
"--glossary-confidence-threshold",
type=float,
help="minimum confidence required for glossary proposals to survive validation",
)
process.add_argument(
"--grammar-confidence-threshold",
type=float,
help="minimum confidence required for grammar proposals to survive validation",
)
process.add_argument(
"--homophones-confidence-threshold",
type=float,
help="minimum confidence required for homophone proposals to survive validation",
)
process.add_argument(
"--spoken-word-confidence-threshold",
type=float,
help="minimum confidence required for spoken-word proposals to survive validation",
)
process.add_argument(
"--normalize-max-segment-gap",
type=float,
help="maximum same-speaker gap in seconds eligible for deterministic merging",
)
process.add_argument(
"--normalize-ellipsis-gap",
type=float,
help="minimum same-speaker gap in seconds that uses an ellipsis joiner",
)
process.add_argument(
"--normalize-max-segment-duration",
type=float,
help="maximum merged segment duration in seconds",
)
process.add_argument(
"--normalize-max-segment-tokens",
type=int,
help="maximum estimated tokens for a merged segment prompt payload",
)
process.add_argument("--work-dir", type=Path, help="directory for per-run scratch diagnostics")
process.add_argument(
"--work-dir-retention",
choices=("auto", "always", "never"),
help="whether to retain the per-run work directory",
)
return parser
def _process(args: argparse.Namespace, raw_argv: Optional[Sequence[str]] = None) -> int:
argv = list(raw_argv) if raw_argv is not None else list(sys.argv[1:])
redacted_argv = redact_argv(argv)
run_dir = create_run_dir(_resolve_work_root(args))
report_path = run_dir / "report.json"
error_log_path = run_dir / "error.log"
config: Optional[AuditaConfig] = None
phase = "config"
try:
config = AuditaConfig.from_sources(
overrides=ConfigOverrides(
llm_api_key=args.llm_api_key,
llm_concurrency=args.llm_concurrency,
llm_timeout_seconds=args.llm_timeout_seconds,
validation_llm_api_key=args.validation_llm_api_key,
validation_llm_concurrency=args.validation_llm_concurrency,
validation_llm_timeout_seconds=args.validation_llm_timeout_seconds,
validation_model=args.validation_model,
validation_base_url=args.validation_base_url,
validation_max_retries=args.validation_max_retries,
validation_max_prompt_tokens=args.validation_max_prompt_tokens,
target_sections=args.target_sections,
module_keys=args.modules,
model=args.model,
base_url=args.base_url,
max_retries=args.max_retries,
max_section_tokens=args.max_section_tokens,
min_section_tokens=args.min_section_tokens,
glossary_confidence_threshold=args.glossary_confidence_threshold,
grammar_confidence_threshold=args.grammar_confidence_threshold,
homophones_confidence_threshold=args.homophones_confidence_threshold,
spoken_word_confidence_threshold=args.spoken_word_confidence_threshold,
normalize_max_segment_gap=args.normalize_max_segment_gap,
normalize_ellipsis_gap=args.normalize_ellipsis_gap,
normalize_max_segment_duration=args.normalize_max_segment_duration,
normalize_max_segment_tokens=args.normalize_max_segment_tokens,
work_dir=args.work_dir,
work_dir_retention=args.work_dir_retention,
)
)
phase = "transcript_load"
transcript = load_transcript(args.transcript)
phase = "glossary_load"
glossary = load_glossary(args.glossary)
phase = "pipeline"
result = process_transcript_result(
transcript,
glossary,
config,
progress=_emit_console_line,
run_dir=run_dir,
invocation_details={"argv": redacted_argv, "cwd": os.getcwd()},
)
if args.output is not None:
write_transcript(args.output, result.transcript)
else:
sys.stdout.write(transcript_to_json(result.transcript))
if args.report_json is not None:
write_report(args.report_json, result.report)
return 0
except Exception as exc:
error_details = getattr(exc, "audita_error_details", None)
if error_details is None:
traceback_text = format_traceback_text(exc)
error_details = build_error_details(
exc=exc,
exit_code=1,
run_dir=run_dir,
report_path=report_path,
error_log_path=error_log_path,
phase=phase,
module_instance=None,
argv=redacted_argv,
cwd=os.getcwd(),
traceback_text=traceback_text,
)
write_error_log(error_log_path, error_details=error_details, traceback_text=traceback_text)
report_config = config.to_report_dict() if isinstance(config, AuditaConfig) else {}
pipeline = [] if not isinstance(config, AuditaConfig) else list(config.module_keys)
retention = "always" if not isinstance(config, AuditaConfig) else config.work_dir_retention
report = RunReport(
status="failed",
config=report_config,
normalization=None,
pipeline=pipeline,
modules=[],
applied_changes=[],
skipped_corrections=[],
totals={"output_segment_count": 0, "applied_change_count": 0, "skipped_correction_count": 0},
work_dir_retention=retention,
work_dir_retained=True,
work_dir=str(run_dir),
error=str(exc),
error_details=error_details,
)
write_report(report_path, report)
if args.report_json is not None and report_path.exists():
args.report_json.write_text(report_path.read_text(encoding="utf-8"), encoding="utf-8")
_emit_console_line(format_stderr_summary(error_details=error_details))
return 1
def _resolve_work_root(args: argparse.Namespace) -> Path:
if args.work_dir is not None:
return args.work_dir
return Path(os.environ.get("AUDITA_WORK_DIR") or "/tmp/audita")
def _emit_console_line(message: str) -> None:
for stream in (sys.stderr, sys.stdout):
if stream is None:
continue
try:
stream.write(f"{message}\n")
stream.flush()
return
except (OSError, ValueError):
continue

View File

@@ -0,0 +1 @@
"""Core data models and utilities for the Audita framework."""

View File

@@ -0,0 +1,356 @@
import json
from dataclasses import dataclass
from math import ceil
from typing import Any, Callable, Generic, List, Optional, Protocol, Sequence, TypeVar
from .errors import AuditaValidationError
from .schemas import TranscriptSegment, parse_transcript_json
class TokenEstimatorProtocol(Protocol):
def estimate_json(self, value: Any) -> int:
...
class TokenEstimator:
def __init__(self, fallback_chars_per_token: int = 4) -> None:
self._fallback_chars_per_token = fallback_chars_per_token
self._encoding = None
try:
import tiktoken
self._encoding = tiktoken.get_encoding("cl100k_base")
except Exception:
self._encoding = None
def estimate_text(self, text: str) -> int:
if self._encoding is not None:
return len(self._encoding.encode(text))
return max(1, ceil(len(text) / self._fallback_chars_per_token))
def estimate_json(self, value: Any) -> int:
return self.estimate_text(json.dumps(value, ensure_ascii=False, separators=(",", ":")))
T = TypeVar("T")
@dataclass(frozen=True)
class TokenBatch(Generic[T]):
batch_index: int
items: List[T]
token_count: int
def chunk_payload_items(
items: Sequence[T],
max_tokens: int,
payload_fn: Callable[[T], Any],
estimator: Optional[TokenEstimatorProtocol] = None,
empty_error_message: str = "Batch input must contain at least one item.",
) -> List[TokenBatch[T]]:
if max_tokens <= 0:
raise AuditaValidationError("Maximum section token count must be greater than zero.")
if not items:
raise AuditaValidationError(empty_error_message)
token_estimator = TokenEstimator() if estimator is None else estimator
batches: List[TokenBatch[T]] = []
current: List[T] = []
current_tokens = 0
for item in items:
single_payload = [payload_fn(item)]
single_tokens = token_estimator.estimate_json(single_payload)
if single_tokens > max_tokens:
raise AuditaValidationError(
"A single transcript segment exceeds the maximum section token limit. "
"Raise the limit or pre-split the transcript."
)
candidate = current + [item]
candidate_tokens = token_estimator.estimate_json([payload_fn(candidate_item) for candidate_item in candidate])
if current and candidate_tokens > max_tokens:
batches.append(TokenBatch(batch_index=len(batches), items=list(current), token_count=current_tokens))
current = [item]
current_tokens = single_tokens
else:
current = candidate
current_tokens = candidate_tokens
if current:
batches.append(TokenBatch(batch_index=len(batches), items=list(current), token_count=current_tokens))
return batches
@dataclass(frozen=True)
class IndexedSegment:
index: int
segment: TranscriptSegment
def transcript_payload(self) -> dict:
return self.segment.model_dump(mode="json")
def prompt_payload(self) -> dict:
payload = {"id": self.segment.id, "original_text": self.segment.text}
if self.segment.categories is not None:
payload["categories"] = list(self.segment.categories)
return payload
@dataclass(frozen=True)
class TranscriptSection:
section_index: int
start_index: int
segments: List[IndexedSegment]
token_count: int
def transcript_payload(self) -> List[dict]:
return [item.transcript_payload() for item in self.segments]
def prompt_payload(self) -> List[dict]:
return [item.prompt_payload() for item in self.segments]
def transcript_json(self) -> str:
return json.dumps(self.transcript_payload(), ensure_ascii=False, indent=2) + "\n"
def chunk_transcript(
segments: List[TranscriptSegment],
max_section_tokens: int,
min_section_tokens: int = 1,
target_section_count: Optional[int] = None,
exact_target_section_count: Optional[int] = None,
estimator: Optional[TokenEstimatorProtocol] = None,
) -> List[TranscriptSection]:
indexed = [IndexedSegment(index=index, segment=segment) for index, segment in enumerate(segments)]
return chunk_indexed_segments(
indexed,
max_section_tokens,
min_section_tokens=min_section_tokens,
target_section_count=target_section_count,
exact_target_section_count=exact_target_section_count,
estimator=estimator,
)
def chunk_indexed_segments(
indexed_segments: List[IndexedSegment],
max_section_tokens: int,
min_section_tokens: int = 1,
target_section_count: Optional[int] = None,
exact_target_section_count: Optional[int] = None,
estimator: Optional[TokenEstimatorProtocol] = None,
) -> List[TranscriptSection]:
if exact_target_section_count is not None:
batches = _chunk_indexed_segments_for_exact_target_count(
indexed_segments,
max_section_tokens=max_section_tokens,
min_section_tokens=min_section_tokens,
target_section_count=exact_target_section_count,
estimator=estimator,
)
elif target_section_count is None:
batches = chunk_payload_items(
indexed_segments,
max_section_tokens,
payload_fn=lambda item: item.prompt_payload(),
estimator=estimator,
empty_error_message="Transcript must contain at least one segment.",
)
else:
batches = _chunk_indexed_segments_for_target_count(
indexed_segments,
max_section_tokens=max_section_tokens,
min_section_tokens=min_section_tokens,
target_section_count=target_section_count,
estimator=estimator,
)
sections = [
TranscriptSection(
section_index=batch.batch_index,
start_index=batch.items[0].index,
segments=list(batch.items),
token_count=batch.token_count,
)
for batch in batches
]
for section in sections:
parse_transcript_json(section.transcript_json(), require_sequential_ids=False)
return sections
def _chunk_indexed_segments_for_target_count(
indexed_segments: List[IndexedSegment],
*,
max_section_tokens: int,
min_section_tokens: int,
target_section_count: int,
estimator: Optional[TokenEstimatorProtocol],
) -> List[TokenBatch[IndexedSegment]]:
if max_section_tokens <= 0:
raise AuditaValidationError("Maximum section token count must be greater than zero.")
if min_section_tokens <= 0:
raise AuditaValidationError("Minimum section token count must be greater than zero.")
if min_section_tokens > max_section_tokens:
raise AuditaValidationError(
"Minimum section token count must be less than or equal to maximum section token count."
)
if not indexed_segments:
raise AuditaValidationError("Transcript must contain at least one segment.")
if target_section_count <= 0:
raise AuditaValidationError("Target section count must be greater than zero.")
token_estimator = TokenEstimator() if estimator is None else estimator
single_tokens = [token_estimator.estimate_json([item.prompt_payload()]) for item in indexed_segments]
if any(tokens > max_section_tokens for tokens in single_tokens):
raise AuditaValidationError(
"A single transcript segment exceeds the maximum section token limit. "
"Raise the limit or pre-split the transcript."
)
total_tokens = sum(single_tokens)
if total_tokens < min_section_tokens:
return [_build_token_batch(indexed_segments, batch_index=0, estimator=token_estimator)]
desired_count = min(target_section_count, len(indexed_segments))
section_count = _resolve_section_count(
indexed_segments=indexed_segments,
single_tokens=single_tokens,
total_tokens=total_tokens,
desired_count=desired_count,
min_section_tokens=min_section_tokens,
max_section_tokens=max_section_tokens,
estimator=token_estimator,
)
return _build_balanced_batches(indexed_segments, single_tokens, section_count, token_estimator)
def _chunk_indexed_segments_for_exact_target_count(
indexed_segments: List[IndexedSegment],
*,
max_section_tokens: int,
min_section_tokens: int,
target_section_count: int,
estimator: Optional[TokenEstimatorProtocol],
) -> List[TokenBatch[IndexedSegment]]:
if max_section_tokens <= 0:
raise AuditaValidationError("Maximum section token count must be greater than zero.")
if min_section_tokens <= 0:
raise AuditaValidationError("Minimum section token count must be greater than zero.")
if min_section_tokens > max_section_tokens:
raise AuditaValidationError(
"Minimum section token count must be less than or equal to maximum section token count."
)
if not indexed_segments:
raise AuditaValidationError("Transcript must contain at least one segment.")
if target_section_count <= 0:
raise AuditaValidationError("Target section count must be greater than zero.")
if target_section_count > len(indexed_segments):
raise AuditaValidationError(
"Target section count exceeds the number of transcript segments. "
"Lower AUDITA_TARGET_SECTIONS or pre-merge the transcript."
)
token_estimator = TokenEstimator() if estimator is None else estimator
single_tokens = [token_estimator.estimate_json([item.prompt_payload()]) for item in indexed_segments]
if any(tokens > max_section_tokens for tokens in single_tokens):
raise AuditaValidationError(
"A single transcript segment exceeds the maximum section token limit. "
"Raise the limit or pre-split the transcript."
)
batches = _build_balanced_batches(indexed_segments, single_tokens, target_section_count, token_estimator)
if not _batches_within_bounds(batches, min_section_tokens=min_section_tokens, max_section_tokens=max_section_tokens):
raise AuditaValidationError(
"AUDITA_TARGET_SECTIONS cannot produce contiguous transcript sections within "
"AUDITA_MIN_SECTION_TOKENS and AUDITA_MAX_SECTION_TOKENS."
)
return batches
def _resolve_section_count(
*,
indexed_segments: List[IndexedSegment],
single_tokens: List[int],
total_tokens: int,
desired_count: int,
min_section_tokens: int,
max_section_tokens: int,
estimator: TokenEstimatorProtocol,
) -> int:
batches = _build_balanced_batches(indexed_segments, single_tokens, desired_count, estimator)
if _batches_within_bounds(batches, min_section_tokens=min_section_tokens, max_section_tokens=max_section_tokens):
return desired_count
average_tokens = total_tokens / desired_count
if average_tokens > max_section_tokens:
candidates = range(desired_count + 1, len(indexed_segments) + 1)
elif average_tokens < min_section_tokens:
candidates = range(desired_count - 1, 0, -1)
else:
candidates = list(range(desired_count + 1, len(indexed_segments) + 1)) + list(
range(desired_count - 1, 0, -1)
)
for count in candidates:
batches = _build_balanced_batches(indexed_segments, single_tokens, count, estimator)
if _batches_within_bounds(batches, min_section_tokens=min_section_tokens, max_section_tokens=max_section_tokens):
return count
return 1
def _build_balanced_batches(
indexed_segments: List[IndexedSegment],
single_tokens: List[int],
section_count: int,
estimator: TokenEstimatorProtocol,
) -> List[TokenBatch[IndexedSegment]]:
if section_count == 1:
return [_build_token_batch(indexed_segments, batch_index=0, estimator=estimator)]
prefix_tokens = [0]
for tokens in single_tokens:
prefix_tokens.append(prefix_tokens[-1] + tokens)
cuts = [0]
total_tokens = prefix_tokens[-1]
for section_index in range(1, section_count):
target = total_tokens * section_index / section_count
min_cut = cuts[-1] + 1
max_cut = len(indexed_segments) - (section_count - section_index)
best_cut = min_cut
best_distance = None
for cut in range(min_cut, max_cut + 1):
distance = abs(prefix_tokens[cut] - target)
if best_distance is None or distance < best_distance:
best_cut = cut
best_distance = distance
cuts.append(best_cut)
cuts.append(len(indexed_segments))
return [
_build_token_batch(indexed_segments[cuts[index] : cuts[index + 1]], batch_index=index, estimator=estimator)
for index in range(section_count)
]
def _build_token_batch(
items: Sequence[IndexedSegment],
*,
batch_index: int,
estimator: TokenEstimatorProtocol,
) -> TokenBatch[IndexedSegment]:
return TokenBatch(
batch_index=batch_index,
items=list(items),
token_count=estimator.estimate_json([item.prompt_payload() for item in items]),
)
def _batches_within_bounds(
batches: Sequence[TokenBatch[IndexedSegment]],
*,
min_section_tokens: int,
max_section_tokens: int,
) -> bool:
return all(min_section_tokens <= batch.token_count <= max_section_tokens for batch in batches)

View File

@@ -0,0 +1,506 @@
import math
import os
from dataclasses import dataclass, replace
from pathlib import Path
from typing import Mapping, Optional, Sequence, Tuple, Union
from audita.modules import DEFAULT_MODULE_KEYS, normalize_module_keys
from .errors import AuditaConfigError
DEFAULT_MODEL = "openrouter/google/gemma-4-31b-it"
DEFAULT_BASE_URL = "https://openrouter.ai/api/v1"
DEFAULT_LLM_CONCURRENCY = 1
DEFAULT_LLM_TIMEOUT_SECONDS = 600
DEFAULT_MAX_RETRIES = 3
DEFAULT_VALIDATION_MAX_PROMPT_TOKENS = 2048
DEFAULT_MAX_SECTION_TOKENS = 8192
DEFAULT_MIN_SECTION_TOKENS = 2048
DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD = 0.80
DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD = 0.80
DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD = 0.80
DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD = 0.80
DEFAULT_WORK_DIR = "/tmp/audita"
DEFAULT_WORK_DIR_RETENTION = "auto"
DEFAULT_NORMALIZE_MAX_SEGMENT_GAP = 4.0
DEFAULT_NORMALIZE_ELLIPSIS_GAP = 3.5
DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION = 60.0
DEFAULT_NORMALIZE_MAX_SEGMENT_TOKENS = 2048
@dataclass(frozen=True)
class ConfigOverrides:
llm_api_key: Optional[str] = None
llm_concurrency: Optional[int] = None
llm_timeout_seconds: Optional[float] = None
validation_llm_api_key: Optional[str] = None
validation_llm_concurrency: Optional[int] = None
validation_llm_timeout_seconds: Optional[float] = None
validation_model: Optional[str] = None
validation_base_url: Optional[str] = None
validation_max_retries: Optional[int] = None
validation_max_prompt_tokens: Optional[int] = None
target_sections: Optional[int] = None
module_keys: Optional[Union[str, Sequence[str]]] = None
model: Optional[str] = None
base_url: Optional[str] = None
max_retries: Optional[int] = None
max_section_tokens: Optional[int] = None
min_section_tokens: Optional[int] = None
glossary_confidence_threshold: Optional[float] = None
grammar_confidence_threshold: Optional[float] = None
homophones_confidence_threshold: Optional[float] = None
spoken_word_confidence_threshold: Optional[float] = None
normalize_max_segment_gap: Optional[float] = None
normalize_ellipsis_gap: Optional[float] = None
normalize_max_segment_duration: Optional[float] = None
normalize_max_segment_tokens: Optional[int] = None
work_dir: Optional[Path] = None
work_dir_retention: Optional[str] = None
@dataclass(frozen=True)
class AuditaConfig:
api_key: Optional[str] = None
llm_concurrency: int = DEFAULT_LLM_CONCURRENCY
llm_timeout_seconds: float = DEFAULT_LLM_TIMEOUT_SECONDS
validation_llm_api_key: Optional[str] = None
validation_llm_concurrency: Optional[int] = None
validation_llm_timeout_seconds: Optional[float] = None
validation_model: Optional[str] = None
validation_base_url: Optional[str] = None
validation_max_retries: Optional[int] = None
validation_max_prompt_tokens: int = DEFAULT_VALIDATION_MAX_PROMPT_TOKENS
target_sections: Optional[int] = None
module_keys: Tuple[str, ...] = DEFAULT_MODULE_KEYS
model: str = DEFAULT_MODEL
base_url: str = DEFAULT_BASE_URL
max_retries: int = DEFAULT_MAX_RETRIES
max_section_tokens: int = DEFAULT_MAX_SECTION_TOKENS
min_section_tokens: int = DEFAULT_MIN_SECTION_TOKENS
glossary_confidence_threshold: float = DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD
grammar_confidence_threshold: float = DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD
homophones_confidence_threshold: float = DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD
spoken_word_confidence_threshold: float = DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD
normalize_max_segment_gap: float = DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
normalize_ellipsis_gap: float = DEFAULT_NORMALIZE_ELLIPSIS_GAP
normalize_max_segment_duration: float = DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION
normalize_max_segment_tokens: int = DEFAULT_NORMALIZE_MAX_SEGMENT_TOKENS
work_dir: Path = Path(DEFAULT_WORK_DIR)
work_dir_retention: str = DEFAULT_WORK_DIR_RETENTION
@classmethod
def from_sources(
cls,
env: Optional[Mapping[str, str]] = None,
overrides: Optional[ConfigOverrides] = None,
) -> "AuditaConfig":
source = os.environ if env is None else env
selected = ConfigOverrides() if overrides is None else overrides
config = cls(
api_key=_select_api_key(
selected.llm_api_key,
source.get("AUDITA_LLM_API_KEY"),
source.get("OPENROUTER_API_KEY"),
),
llm_concurrency=_select_int(
selected.llm_concurrency,
source.get("AUDITA_LLM_CONCURRENCY"),
DEFAULT_LLM_CONCURRENCY,
"AUDITA_LLM_CONCURRENCY",
),
llm_timeout_seconds=_select_float(
selected.llm_timeout_seconds,
source.get("AUDITA_LLM_TIMEOUT_SECONDS"),
DEFAULT_LLM_TIMEOUT_SECONDS,
"AUDITA_LLM_TIMEOUT_SECONDS",
),
validation_llm_api_key=_select_optional_api_key_override(
selected.validation_llm_api_key,
source.get("AUDITA_VALIDATION_LLM_API_KEY"),
),
validation_llm_concurrency=_select_optional_int(
selected.validation_llm_concurrency,
source.get("AUDITA_VALIDATION_LLM_CONCURRENCY"),
"AUDITA_VALIDATION_LLM_CONCURRENCY",
),
validation_llm_timeout_seconds=_select_optional_float(
selected.validation_llm_timeout_seconds,
source.get("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"),
"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS",
),
validation_model=_select_optional_string_override(
selected.validation_model,
source.get("AUDITA_VALIDATION_MODEL"),
),
validation_base_url=_select_optional_string_override(
selected.validation_base_url,
source.get("AUDITA_VALIDATION_BASE_URL"),
),
validation_max_retries=_select_optional_int(
selected.validation_max_retries,
source.get("AUDITA_VALIDATION_MAX_RETRIES"),
"AUDITA_VALIDATION_MAX_RETRIES",
),
validation_max_prompt_tokens=_select_int(
selected.validation_max_prompt_tokens,
source.get("AUDITA_VALIDATION_MAX_PROMPT_TOKENS"),
DEFAULT_VALIDATION_MAX_PROMPT_TOKENS,
"AUDITA_VALIDATION_MAX_PROMPT_TOKENS",
),
target_sections=_select_optional_int(
selected.target_sections,
source.get("AUDITA_TARGET_SECTIONS"),
"AUDITA_TARGET_SECTIONS",
),
module_keys=_select_module_keys(
selected.module_keys,
source.get("AUDITA_MODULES"),
DEFAULT_MODULE_KEYS,
"AUDITA_MODULES",
),
model=selected.model or source.get("AUDITA_MODEL") or DEFAULT_MODEL,
base_url=selected.base_url or source.get("AUDITA_BASE_URL") or DEFAULT_BASE_URL,
max_retries=_select_int(
selected.max_retries,
source.get("AUDITA_MAX_RETRIES"),
DEFAULT_MAX_RETRIES,
"AUDITA_MAX_RETRIES",
),
max_section_tokens=_select_int(
selected.max_section_tokens,
source.get("AUDITA_MAX_SECTION_TOKENS"),
DEFAULT_MAX_SECTION_TOKENS,
"AUDITA_MAX_SECTION_TOKENS",
),
min_section_tokens=_select_int(
selected.min_section_tokens,
source.get("AUDITA_MIN_SECTION_TOKENS"),
DEFAULT_MIN_SECTION_TOKENS,
"AUDITA_MIN_SECTION_TOKENS",
),
glossary_confidence_threshold=_select_float(
selected.glossary_confidence_threshold,
source.get("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD"),
DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD,
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD",
),
grammar_confidence_threshold=_select_float(
selected.grammar_confidence_threshold,
source.get("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD"),
DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD,
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD",
),
homophones_confidence_threshold=_select_float(
selected.homophones_confidence_threshold,
source.get("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD"),
DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD,
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD",
),
spoken_word_confidence_threshold=_select_float(
selected.spoken_word_confidence_threshold,
source.get("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD"),
DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD,
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD",
),
normalize_max_segment_gap=_select_float(
selected.normalize_max_segment_gap,
source.get("AUDITA_NORMALIZE_MAX_SEGMENT_GAP"),
DEFAULT_NORMALIZE_MAX_SEGMENT_GAP,
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP",
),
normalize_ellipsis_gap=_select_float(
selected.normalize_ellipsis_gap,
source.get("AUDITA_NORMALIZE_ELLIPSIS_GAP"),
DEFAULT_NORMALIZE_ELLIPSIS_GAP,
"AUDITA_NORMALIZE_ELLIPSIS_GAP",
),
normalize_max_segment_duration=_select_float(
selected.normalize_max_segment_duration,
source.get("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION"),
DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION,
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION",
),
normalize_max_segment_tokens=_select_int(
selected.normalize_max_segment_tokens,
source.get("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS"),
DEFAULT_NORMALIZE_MAX_SEGMENT_TOKENS,
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS",
),
work_dir=selected.work_dir or Path(source.get("AUDITA_WORK_DIR") or DEFAULT_WORK_DIR),
work_dir_retention=_select_choice(
selected.work_dir_retention,
source.get("AUDITA_WORK_DIR_RETENTION"),
DEFAULT_WORK_DIR_RETENTION,
"AUDITA_WORK_DIR_RETENTION",
("auto", "always", "never"),
),
)
config.validate()
return config
def validate(self) -> None:
normalize_module_keys(self.module_keys)
if self.llm_concurrency <= 0:
raise AuditaConfigError("AUDITA_LLM_CONCURRENCY must be greater than zero.")
if not math.isfinite(self.llm_timeout_seconds):
raise AuditaConfigError("AUDITA_LLM_TIMEOUT_SECONDS must be finite.")
if self.llm_timeout_seconds <= 0:
raise AuditaConfigError("AUDITA_LLM_TIMEOUT_SECONDS must be greater than zero.")
if self.validation_llm_concurrency is not None and self.validation_llm_concurrency <= 0:
raise AuditaConfigError("AUDITA_VALIDATION_LLM_CONCURRENCY must be greater than zero.")
if self.validation_llm_timeout_seconds is not None and not math.isfinite(self.validation_llm_timeout_seconds):
raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be finite.")
if self.validation_llm_timeout_seconds is not None and self.validation_llm_timeout_seconds <= 0:
raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be greater than zero.")
if self.validation_max_retries is not None and self.validation_max_retries < 0:
raise AuditaConfigError("AUDITA_VALIDATION_MAX_RETRIES must be greater than or equal to zero.")
if self.validation_max_prompt_tokens <= 0:
raise AuditaConfigError("AUDITA_VALIDATION_MAX_PROMPT_TOKENS must be greater than zero.")
if self.target_sections is not None and self.target_sections <= 0:
raise AuditaConfigError("AUDITA_TARGET_SECTIONS must be greater than zero.")
if not self.model.strip():
raise AuditaConfigError("AUDITA_MODEL must not be empty.")
if not self.base_url.strip():
raise AuditaConfigError("AUDITA_BASE_URL must not be empty.")
if self.validation_model is not None and not self.validation_model.strip():
raise AuditaConfigError("AUDITA_VALIDATION_MODEL must not be empty.")
if self.validation_base_url is not None and not self.validation_base_url.strip():
raise AuditaConfigError("AUDITA_VALIDATION_BASE_URL must not be empty.")
if self.max_retries < 0:
raise AuditaConfigError("AUDITA_MAX_RETRIES must be greater than or equal to zero.")
if self.max_section_tokens <= 0:
raise AuditaConfigError("AUDITA_MAX_SECTION_TOKENS must be greater than zero.")
if self.min_section_tokens <= 0:
raise AuditaConfigError("AUDITA_MIN_SECTION_TOKENS must be greater than zero.")
if self.min_section_tokens > self.max_section_tokens:
raise AuditaConfigError(
"AUDITA_MIN_SECTION_TOKENS must be less than or equal to AUDITA_MAX_SECTION_TOKENS."
)
if not 0.0 <= self.glossary_confidence_threshold <= 1.0:
raise AuditaConfigError("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.")
if not 0.0 <= self.grammar_confidence_threshold <= 1.0:
raise AuditaConfigError("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.")
if not 0.0 <= self.homophones_confidence_threshold <= 1.0:
raise AuditaConfigError("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.")
if not 0.0 <= self.spoken_word_confidence_threshold <= 1.0:
raise AuditaConfigError("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.")
if not math.isfinite(self.normalize_max_segment_gap):
raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_GAP must be finite.")
if not math.isfinite(self.normalize_ellipsis_gap):
raise AuditaConfigError("AUDITA_NORMALIZE_ELLIPSIS_GAP must be finite.")
if not math.isfinite(self.normalize_max_segment_duration):
raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION must be finite.")
if self.normalize_max_segment_gap < 0:
raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_GAP must be greater than or equal to zero.")
if self.normalize_ellipsis_gap < 0:
raise AuditaConfigError("AUDITA_NORMALIZE_ELLIPSIS_GAP must be greater than or equal to zero.")
if self.normalize_ellipsis_gap > self.normalize_max_segment_gap:
raise AuditaConfigError(
"AUDITA_NORMALIZE_ELLIPSIS_GAP must be less than or equal to AUDITA_NORMALIZE_MAX_SEGMENT_GAP."
)
if self.normalize_max_segment_duration <= 0:
raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION must be greater than zero.")
if self.normalize_max_segment_tokens <= 0:
raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS must be greater than zero.")
if self.work_dir_retention not in ("auto", "always", "never"):
raise AuditaConfigError("AUDITA_WORK_DIR_RETENTION must be one of auto, always, or never.")
validation_config = self.validation_llm_config()
if validation_config.llm_concurrency <= 0:
raise AuditaConfigError("AUDITA_VALIDATION_LLM_CONCURRENCY must be greater than zero.")
if not math.isfinite(validation_config.llm_timeout_seconds):
raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be finite.")
if validation_config.llm_timeout_seconds <= 0:
raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be greater than zero.")
if not validation_config.model.strip():
raise AuditaConfigError("AUDITA_VALIDATION_MODEL must not be empty.")
if not validation_config.base_url.strip():
raise AuditaConfigError("AUDITA_VALIDATION_BASE_URL must not be empty.")
if validation_config.max_retries < 0:
raise AuditaConfigError("AUDITA_VALIDATION_MAX_RETRIES must be greater than or equal to zero.")
def proposal_llm_config(self) -> "AuditaConfig":
return self
def effective_validation_llm_api_key(self) -> Optional[str]:
return self.validation_llm_api_key if self.validation_llm_api_key is not None else self.api_key
def effective_validation_llm_concurrency(self) -> int:
return self.validation_llm_concurrency if self.validation_llm_concurrency is not None else self.llm_concurrency
def effective_validation_llm_timeout_seconds(self) -> float:
if self.validation_llm_timeout_seconds is not None:
return self.validation_llm_timeout_seconds
return self.llm_timeout_seconds
def effective_validation_model(self) -> str:
return self.validation_model if self.validation_model is not None else self.model
def effective_validation_base_url(self) -> str:
return self.validation_base_url if self.validation_base_url is not None else self.base_url
def effective_validation_max_retries(self) -> int:
return self.validation_max_retries if self.validation_max_retries is not None else self.max_retries
def validation_llm_config(self) -> "AuditaConfig":
return replace(
self,
api_key=self.effective_validation_llm_api_key(),
llm_concurrency=self.effective_validation_llm_concurrency(),
llm_timeout_seconds=self.effective_validation_llm_timeout_seconds(),
model=self.effective_validation_model(),
base_url=self.effective_validation_base_url(),
max_retries=self.effective_validation_max_retries(),
)
def to_report_dict(self) -> dict:
effective_validation_config = self.validation_llm_config()
return {
"api_key_configured": bool(self.api_key),
"llm_concurrency": self.llm_concurrency,
"llm_timeout_seconds": self.llm_timeout_seconds,
"validation_llm_api_key_configured": bool(self.validation_llm_api_key),
"validation_llm_concurrency": self.validation_llm_concurrency,
"validation_llm_timeout_seconds": self.validation_llm_timeout_seconds,
"validation_model": self.validation_model,
"validation_base_url": self.validation_base_url,
"validation_max_retries": self.validation_max_retries,
"validation_max_prompt_tokens": self.validation_max_prompt_tokens,
"target_sections": self.target_sections,
"module_keys": list(self.module_keys),
"model": self.model,
"base_url": self.base_url,
"max_retries": self.max_retries,
"effective_validation_llm": {
"api_key_configured": bool(effective_validation_config.api_key),
"llm_concurrency": effective_validation_config.llm_concurrency,
"llm_timeout_seconds": effective_validation_config.llm_timeout_seconds,
"model": effective_validation_config.model,
"base_url": effective_validation_config.base_url,
"max_retries": effective_validation_config.max_retries,
},
"max_section_tokens": self.max_section_tokens,
"min_section_tokens": self.min_section_tokens,
"glossary_confidence_threshold": self.glossary_confidence_threshold,
"grammar_confidence_threshold": self.grammar_confidence_threshold,
"homophones_confidence_threshold": self.homophones_confidence_threshold,
"spoken_word_confidence_threshold": self.spoken_word_confidence_threshold,
"normalize_max_segment_gap": self.normalize_max_segment_gap,
"normalize_ellipsis_gap": self.normalize_ellipsis_gap,
"normalize_max_segment_duration": self.normalize_max_segment_duration,
"normalize_max_segment_tokens": self.normalize_max_segment_tokens,
"work_dir_retention": self.work_dir_retention,
}
def _select_optional_string(value: Optional[str]) -> Optional[str]:
if value is None:
return None
stripped = value.strip()
return stripped or None
def _select_optional_string_override(cli_value: Optional[str], env_value: Optional[str]) -> Optional[str]:
if cli_value is not None:
return _select_optional_string(cli_value)
return _select_optional_string(env_value)
def _select_optional_api_key_override(cli_value: Optional[str], env_value: Optional[str]) -> Optional[str]:
if cli_value is not None:
return cli_value.strip()
if env_value is not None:
return env_value.strip()
return None
def _select_api_key(
cli_value: Optional[str],
generic_env_value: Optional[str],
legacy_env_value: Optional[str],
) -> Optional[str]:
if cli_value is not None:
return _select_optional_string(cli_value)
generic = _select_optional_string(generic_env_value)
if generic is not None:
return generic
return _select_optional_string(legacy_env_value)
def _select_int(cli_value: Optional[int], env_value: Optional[str], default: int, name: str) -> int:
if cli_value is not None:
return cli_value
if env_value is None:
return default
try:
return int(env_value)
except ValueError as exc:
raise AuditaConfigError(f"{name} must be an integer.") from exc
def _select_optional_int(cli_value: Optional[int], env_value: Optional[str], name: str) -> Optional[int]:
if cli_value is not None:
return cli_value
if env_value is None:
return None
try:
return int(env_value)
except ValueError as exc:
raise AuditaConfigError(f"{name} must be an integer.") from exc
def _select_optional_float(cli_value: Optional[float], env_value: Optional[str], name: str) -> Optional[float]:
if cli_value is not None:
return cli_value
if env_value is None:
return None
try:
return float(env_value)
except ValueError as exc:
raise AuditaConfigError(f"{name} must be a number.") from exc
def _select_float(cli_value: Optional[float], env_value: Optional[str], default: float, name: str) -> float:
if cli_value is not None:
return cli_value
if env_value is None:
return default
try:
return float(env_value)
except ValueError as exc:
raise AuditaConfigError(f"{name} must be a number.") from exc
def _select_choice(
cli_value: Optional[str],
env_value: Optional[str],
default: str,
name: str,
valid_choices: tuple[str, ...],
) -> str:
value = cli_value or env_value or default
if value not in valid_choices:
raise AuditaConfigError(f"{name} must be one of {', '.join(valid_choices)}.")
return value
def _select_module_keys(
cli_value: Optional[Union[str, Sequence[str]]],
env_value: Optional[str],
default: Tuple[str, ...],
name: str,
) -> Tuple[str, ...]:
if cli_value is not None:
return _coerce_module_keys(cli_value, name)
if env_value is not None:
return _coerce_module_keys(env_value, name)
return default
def _coerce_module_keys(value: Union[str, Sequence[str]], name: str) -> Tuple[str, ...]:
try:
raw_items = value.split(",") if isinstance(value, str) else list(value)
return normalize_module_keys(raw_items)
except AuditaConfigError as exc:
raise AuditaConfigError(f"{name} is invalid: {exc}") from exc

View File

@@ -0,0 +1,148 @@
from __future__ import annotations
import json
import traceback
from datetime import datetime
from pathlib import Path
import re
from typing import Any, Mapping, Optional, Sequence
from uuid import uuid4
_SECRET_FLAGS = {"--llm-api-key", "--validation-llm-api-key"}
_STAGE_PATTERN = re.compile(r"stage '([^']+)'")
def create_run_dir(root: Path) -> Path:
root.mkdir(parents=True, exist_ok=True)
timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
run_dir = root / f"run-{timestamp}-{uuid4().hex[:8]}"
run_dir.mkdir(parents=False, exist_ok=False)
return run_dir
def redact_argv(argv: Sequence[str]) -> list[str]:
redacted: list[str] = []
index = 0
while index < len(argv):
arg = argv[index]
matched_flag = next((flag for flag in _SECRET_FLAGS if arg == flag or arg.startswith(flag + "=")), None)
if matched_flag is None:
redacted.append(arg)
index += 1
continue
if arg == matched_flag:
redacted.append(arg)
if index + 1 < len(argv):
redacted.append("[REDACTED]")
index += 2
else:
index += 1
continue
redacted.append(f"{matched_flag}=[REDACTED]")
index += 1
return redacted
def format_traceback_text(exc: BaseException) -> str:
return "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))
def extract_stage_name(message: str) -> Optional[str]:
match = _STAGE_PATTERN.search(message)
return match.group(1) if match is not None else None
def build_error_details(
*,
exc: BaseException,
exit_code: int,
run_dir: Path,
report_path: Path,
error_log_path: Path,
phase: str,
module_instance: Optional[str],
argv: Optional[Sequence[str]] = None,
cwd: Optional[str] = None,
traceback_text: Optional[str] = None,
) -> dict[str, Any]:
message = str(exc)
stage_name = extract_stage_name(message)
excerpt_source = traceback_text or format_traceback_text(exc)
excerpt_lines = excerpt_source.strip().splitlines()
traceback_excerpt = "\n".join(excerpt_lines[-12:]) if excerpt_lines else ""
details: dict[str, Any] = {
"type": type(exc).__name__,
"message": message,
"exit_code": exit_code,
"phase": phase,
"stage": stage_name,
"module_instance": module_instance,
"run_dir": str(run_dir),
"report_path": str(report_path),
"error_log": str(error_log_path),
"traceback_excerpt": traceback_excerpt,
}
if argv is not None:
details["argv"] = list(argv)
if cwd is not None:
details["cwd"] = cwd
return details
def write_error_log(
path: Path,
*,
error_details: Mapping[str, Any],
traceback_text: str,
) -> None:
lines = [
"Audita Error Diagnostics",
f"timestamp: {datetime.utcnow().isoformat()}Z",
f"type: {error_details.get('type')}",
f"message: {error_details.get('message')}",
f"exit_code: {error_details.get('exit_code')}",
f"phase: {error_details.get('phase')}",
f"stage: {error_details.get('stage') or ''}",
f"module_instance: {error_details.get('module_instance') or ''}",
f"run_dir: {error_details.get('run_dir')}",
f"report_path: {error_details.get('report_path')}",
f"error_log: {error_details.get('error_log')}",
]
argv = error_details.get("argv")
if argv is not None:
lines.append(f"argv: {json.dumps(argv, ensure_ascii=False)}")
cwd = error_details.get("cwd")
if cwd is not None:
lines.append(f"cwd: {cwd}")
lines.extend(
[
"",
"Traceback:",
traceback_text.rstrip(),
"",
]
)
path.write_text("\n".join(lines), encoding="utf-8")
def format_stderr_summary(*, error_details: Mapping[str, Any]) -> str:
lines = [
f"audita: error: {error_details.get('message')}",
f"audita: exit code: {error_details.get('exit_code')}",
f"audita: phase: {error_details.get('phase')}",
]
stage = error_details.get("stage")
if stage:
lines.append(f"audita: stage: {stage}")
module_instance = error_details.get("module_instance")
if module_instance:
lines.append(f"audita: module: {module_instance}")
lines.extend(
[
f"audita: run directory: {error_details.get('run_dir')}",
f"audita: error log: {error_details.get('error_log')}",
f"audita: report: {error_details.get('report_path')}",
]
)
return "\n".join(lines)

View File

@@ -0,0 +1,15 @@
class AuditaError(Exception):
"""Base exception for user-facing Audita failures."""
class AuditaValidationError(AuditaError):
"""Raised when input data does not match Audita's expected schema."""
class AuditaConfigError(AuditaError):
"""Raised when runtime configuration is invalid or incomplete."""
class AuditaLLMError(AuditaError):
"""Raised when an LLM request or structured response fails."""

View File

@@ -0,0 +1,22 @@
from pathlib import Path
from typing import List
from .reporting import RunReport
from .schemas import Glossary, SourceTranscriptSegment, TranscriptSegment
from .schemas import parse_glossary_yaml, parse_source_transcript_json, transcript_to_json
def load_transcript(path: Path) -> List[SourceTranscriptSegment]:
return parse_source_transcript_json(path.read_text(encoding="utf-8"))
def load_glossary(path: Path) -> Glossary:
return parse_glossary_yaml(path.read_text(encoding="utf-8"))
def write_transcript(path: Path, segments: List[TranscriptSegment]) -> None:
path.write_text(transcript_to_json(segments), encoding="utf-8")
def write_report(path: Path, report: RunReport) -> None:
path.write_text(report.to_json(), encoding="utf-8")

View File

@@ -0,0 +1,184 @@
from dataclasses import asdict, dataclass
from typing import List, Optional
from .chunking import TokenEstimator, TokenEstimatorProtocol
from .schemas import SourceTranscriptSegment, TranscriptSegment
@dataclass(frozen=True)
class NormalizationSummary:
source_segment_count: int
normalized_segment_count: int
merge_count: int
max_segment_gap: float
ellipsis_gap: float
max_segment_duration: float
max_segment_tokens: int
def to_dict(self) -> dict:
return asdict(self)
@dataclass(frozen=True)
class NormalizationResult:
transcript: List[TranscriptSegment]
summary: NormalizationSummary
@dataclass(frozen=True)
class _WorkingSegment:
speaker: str
start: float
end: float
text: str
categories: Optional[List[str]]
order: int
def normalize_transcript(
segments: List[SourceTranscriptSegment],
max_segment_gap: float,
ellipsis_gap: float,
max_segment_duration: float,
max_segment_tokens: int,
estimator: Optional[TokenEstimatorProtocol] = None,
) -> NormalizationResult:
token_estimator = TokenEstimator() if estimator is None else estimator
working = [
_WorkingSegment(
speaker=segment.speaker,
start=segment.start,
end=segment.end,
text=segment.text,
categories=None if segment.categories is None else list(segment.categories),
order=index,
)
for index, segment in enumerate(segments)
]
working.sort(key=lambda segment: (segment.start, segment.end, segment.order))
merge_count = 0
while True:
candidate_index = _shortest_mergeable_gap_index(
working,
max_segment_gap,
ellipsis_gap,
max_segment_duration,
max_segment_tokens,
token_estimator,
)
if candidate_index is None:
break
left = working[candidate_index]
right = working[candidate_index + 1]
working[candidate_index : candidate_index + 2] = [_merge_segments(left, right, ellipsis_gap)]
merge_count += 1
normalized = _assign_ids(working)
return NormalizationResult(
transcript=normalized,
summary=NormalizationSummary(
source_segment_count=len(segments),
normalized_segment_count=len(normalized),
merge_count=merge_count,
max_segment_gap=max_segment_gap,
ellipsis_gap=ellipsis_gap,
max_segment_duration=max_segment_duration,
max_segment_tokens=max_segment_tokens,
),
)
def _shortest_mergeable_gap_index(
segments: List[_WorkingSegment],
max_segment_gap: float,
ellipsis_gap: float,
max_segment_duration: float,
max_segment_tokens: int,
estimator: TokenEstimatorProtocol,
) -> Optional[int]:
best_index = None
best_gap = None
for index in range(len(segments) - 1):
left = segments[index]
right = segments[index + 1]
gap = right.start - left.end
if not _can_merge(
left,
right,
gap,
max_segment_gap,
ellipsis_gap,
max_segment_duration,
max_segment_tokens,
estimator,
):
continue
if best_gap is None or gap < best_gap:
best_index = index
best_gap = gap
return best_index
def _can_merge(
left: _WorkingSegment,
right: _WorkingSegment,
gap: float,
max_segment_gap: float,
ellipsis_gap: float,
max_segment_duration: float,
max_segment_tokens: int,
estimator: TokenEstimatorProtocol,
) -> bool:
if left.speaker != right.speaker:
return False
if gap < 0 or gap > max_segment_gap:
return False
if right.end - left.start > max_segment_duration:
return False
merged_text = _joined_text(left.text, right.text, gap, ellipsis_gap)
return _estimate_prompt_tokens(merged_text, estimator) <= max_segment_tokens
def _merge_segments(left: _WorkingSegment, right: _WorkingSegment, ellipsis_gap: float) -> _WorkingSegment:
gap = right.start - left.end
return _WorkingSegment(
speaker=left.speaker,
start=left.start,
end=right.end,
text=_joined_text(left.text, right.text, gap, ellipsis_gap),
categories=_merged_categories(left.categories, right.categories),
order=left.order,
)
def _joined_text(left_text: str, right_text: str, gap: float, ellipsis_gap: float) -> str:
joiner = " " if gap <= ellipsis_gap else " ... "
return f"{left_text.rstrip()}{joiner}{right_text.lstrip()}"
def _estimate_prompt_tokens(text: str, estimator: TokenEstimatorProtocol) -> int:
return estimator.estimate_json([{"id": 1, "original_text": text}])
def _merged_categories(left: Optional[List[str]], right: Optional[List[str]]) -> Optional[List[str]]:
merged: List[str] = []
for category in (left or []) + (right or []):
if category not in merged:
merged.append(category)
return merged or None
def _assign_ids(segments: List[_WorkingSegment]) -> List[TranscriptSegment]:
ordered = sorted(segments, key=lambda segment: (segment.start, segment.end, segment.order))
return [
TranscriptSegment(
id=index + 1,
speaker=segment.speaker,
start=segment.start,
end=segment.end,
text=segment.text,
categories=None if segment.categories is None else list(segment.categories),
)
for index, segment in enumerate(ordered)
]

View File

@@ -0,0 +1,122 @@
import json
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import List, Optional
from .schemas import TranscriptSegment
@dataclass(frozen=True)
class AppliedChange:
module_instance: str
module_key: str
proposal_index: int
id: int
original_text: str
corrected_text: str
confidence: float
segment_text_before: str
segment_text_after: str
def to_dict(self) -> dict:
return asdict(self)
@dataclass(frozen=True)
class ReportedSkip:
module_instance: str
module_key: str
proposal_index: int
id: int
reason: str
original_text: str
corrected_text: str
confidence: float
actual_text: Optional[str] = None
source: Optional[str] = None
def to_dict(self) -> dict:
return asdict(self)
@dataclass(frozen=True)
class ValidatorReport:
name: str
execution_kind: str
candidate_count: int
approved_count: int
rejected_count: int
def to_dict(self) -> dict:
return asdict(self)
@dataclass(frozen=True)
class ModuleRunReport:
instance_name: str
module_key: str
replacement_policy: str
section_count: int
proposal_count: int
validators: List[ValidatorReport]
approved_count: int
applied_count: int
skipped_count: int
def to_dict(self) -> dict:
return {
"instance_name": self.instance_name,
"module_key": self.module_key,
"replacement_policy": self.replacement_policy,
"section_count": self.section_count,
"proposal_count": self.proposal_count,
"validators": [item.to_dict() for item in self.validators],
"approved_count": self.approved_count,
"applied_count": self.applied_count,
"skipped_count": self.skipped_count,
}
@dataclass(frozen=True)
class RunReport:
status: str
config: dict
normalization: Optional[dict]
pipeline: List[str]
modules: List[ModuleRunReport]
applied_changes: List[AppliedChange]
skipped_corrections: List[ReportedSkip]
totals: dict
work_dir_retention: str
work_dir_retained: bool
work_dir: Optional[str]
error: Optional[str] = None
error_details: Optional[dict] = None
def to_dict(self) -> dict:
return {
"status": self.status,
"config": self.config,
"normalization": self.normalization,
"pipeline": self.pipeline,
"modules": [item.to_dict() for item in self.modules],
"applied_changes": [item.to_dict() for item in self.applied_changes],
"skipped_corrections": [item.to_dict() for item in self.skipped_corrections],
"totals": self.totals,
"work_dir_retention": self.work_dir_retention,
"work_dir_retained": self.work_dir_retained,
"work_dir": self.work_dir,
"error": self.error,
"error_details": self.error_details,
}
def to_json(self) -> str:
return json.dumps(self.to_dict(), ensure_ascii=False, indent=2) + "\n"
@dataclass(frozen=True)
class ProcessResult:
transcript: List[TranscriptSegment]
report: RunReport
run_dir: Path
work_dir_retained: bool

View File

@@ -0,0 +1,252 @@
import json
import math
from typing import Any, List, Optional
from pydantic import BaseModel, ConfigDict, Field, StrictStr, TypeAdapter, ValidationError, field_validator, model_validator
from .errors import AuditaValidationError
class TranscriptSegment(BaseModel):
model_config = ConfigDict(extra="ignore")
id: int = Field(ge=1)
speaker: StrictStr
start: float
end: float
text: StrictStr
categories: Optional[List[StrictStr]] = None
@field_validator("id", mode="before")
@classmethod
def require_integer_id(cls, value: Any) -> int:
if isinstance(value, bool) or not isinstance(value, int):
raise ValueError("must be an integer")
return value
@field_validator("speaker", "text")
@classmethod
def require_non_empty_text(cls, value: str) -> str:
if not value.strip():
raise ValueError("must not be empty")
return value
@field_validator("categories")
@classmethod
def require_non_empty_categories(cls, categories: Optional[List[str]]) -> Optional[List[str]]:
if categories is None:
return None
for category in categories:
if not category.strip():
raise ValueError("categories must not contain empty strings")
return categories
@field_validator("start", "end", mode="before")
@classmethod
def require_number(cls, value: Any) -> float:
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise ValueError("must be a JSON number")
number = float(value)
if not math.isfinite(number):
raise ValueError("must be finite")
if number < 0:
raise ValueError("must be non-negative")
return number
@model_validator(mode="after")
def require_ordered_times(self) -> "TranscriptSegment":
if self.end < self.start:
raise ValueError("end must be greater than or equal to start")
return self
class SourceTranscriptSegment(BaseModel):
model_config = ConfigDict(extra="ignore")
id: Optional[int] = Field(default=None, ge=1)
speaker: StrictStr
start: float
end: float
text: StrictStr
categories: Optional[List[StrictStr]] = None
@field_validator("id", mode="before")
@classmethod
def require_optional_integer_id(cls, value: Any) -> Optional[int]:
if value is None:
return None
if isinstance(value, bool) or not isinstance(value, int):
raise ValueError("must be an integer")
return value
@field_validator("speaker", "text")
@classmethod
def require_non_empty_text(cls, value: str) -> str:
if not value.strip():
raise ValueError("must not be empty")
return value
@field_validator("categories")
@classmethod
def require_non_empty_categories(cls, categories: Optional[List[str]]) -> Optional[List[str]]:
if categories is None:
return None
for category in categories:
if not category.strip():
raise ValueError("categories must not contain empty strings")
return categories
@field_validator("start", "end", mode="before")
@classmethod
def require_number(cls, value: Any) -> float:
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise ValueError("must be a JSON number")
number = float(value)
if not math.isfinite(number):
raise ValueError("must be finite")
if number < 0:
raise ValueError("must be non-negative")
return number
@model_validator(mode="after")
def require_ordered_times(self) -> "SourceTranscriptSegment":
if self.end < self.start:
raise ValueError("end must be greater than or equal to start")
return self
class GlossaryEntry(BaseModel):
model_config = ConfigDict(extra="forbid")
name: StrictStr
aliases: List[StrictStr] = Field(default_factory=list)
plural: Optional[StrictStr] = None
category: StrictStr
summary: StrictStr
@field_validator("name", "category", "summary")
@classmethod
def require_non_empty_text(cls, value: str) -> str:
if not value.strip():
raise ValueError("must not be empty")
return value
@field_validator("aliases")
@classmethod
def require_non_empty_aliases(cls, aliases: List[str]) -> List[str]:
for alias in aliases:
if not alias.strip():
raise ValueError("aliases must not contain empty strings")
return aliases
@field_validator("plural")
@classmethod
def require_non_empty_plural(cls, plural: Optional[str]) -> Optional[str]:
if plural is not None and not plural.strip():
raise ValueError("plural must not be empty")
return plural
class Glossary(BaseModel):
model_config = ConfigDict(extra="forbid")
glossary: List[GlossaryEntry]
@field_validator("glossary")
@classmethod
def require_entries(cls, entries: List[GlossaryEntry]) -> List[GlossaryEntry]:
if not entries:
raise ValueError("glossary must contain at least one entry")
return entries
_TRANSCRIPT_ADAPTER = TypeAdapter(List[TranscriptSegment])
_SOURCE_TRANSCRIPT_ADAPTER = TypeAdapter(List[SourceTranscriptSegment])
def validate_transcript_data(
data: Any,
require_sequential_ids: bool = True,
) -> List[TranscriptSegment]:
data = _extract_transcript_segments(data)
if not isinstance(data, list):
raise AuditaValidationError("Transcript must be a JSON array or an object with a segments array.")
if not data:
raise AuditaValidationError("Transcript must contain at least one segment.")
try:
transcript = _TRANSCRIPT_ADAPTER.validate_python(data)
except ValidationError as exc:
raise AuditaValidationError(f"Transcript schema validation failed: {exc}") from exc
if require_sequential_ids:
_validate_sequential_ids(transcript)
return transcript
def validate_source_transcript_data(data: Any) -> List[SourceTranscriptSegment]:
data = _extract_transcript_segments(data)
if not isinstance(data, list):
raise AuditaValidationError("Transcript must be a JSON array or an object with a segments array.")
if not data:
raise AuditaValidationError("Transcript must contain at least one segment.")
try:
return _SOURCE_TRANSCRIPT_ADAPTER.validate_python(data)
except ValidationError as exc:
raise AuditaValidationError(f"Transcript schema validation failed: {exc}") from exc
def parse_transcript_json(raw: str, require_sequential_ids: bool = True) -> List[TranscriptSegment]:
try:
data = json.loads(raw)
except json.JSONDecodeError as exc:
raise AuditaValidationError(f"Transcript is not valid JSON: {exc}") from exc
return validate_transcript_data(data, require_sequential_ids=require_sequential_ids)
def parse_source_transcript_json(raw: str) -> List[SourceTranscriptSegment]:
try:
data = json.loads(raw)
except json.JSONDecodeError as exc:
raise AuditaValidationError(f"Transcript is not valid JSON: {exc}") from exc
return validate_source_transcript_data(data)
def parse_glossary_yaml(raw: str) -> Glossary:
try:
import yaml
except ImportError as exc:
raise AuditaValidationError("PyYAML is required to read glossary files.") from exc
try:
data = yaml.safe_load(raw)
except yaml.YAMLError as exc:
raise AuditaValidationError(f"Glossary is not valid YAML: {exc}") from exc
if data is None:
data = {}
try:
return Glossary.model_validate(data)
except ValidationError as exc:
raise AuditaValidationError(f"Glossary schema validation failed: {exc}") from exc
def transcript_to_json(segments: List[TranscriptSegment]) -> str:
payload = [segment.model_dump(mode="json", exclude_none=True) for segment in segments]
return json.dumps(payload, ensure_ascii=False, indent=2) + "\n"
def source_transcript_to_json(segments: List[SourceTranscriptSegment]) -> str:
payload = [segment.model_dump(mode="json", exclude_none=True) for segment in segments]
return json.dumps(payload, ensure_ascii=False, indent=2) + "\n"
def _extract_transcript_segments(data: Any) -> Any:
if isinstance(data, dict) and "segments" in data:
return data["segments"]
return data
def _validate_sequential_ids(transcript: List[TranscriptSegment]) -> None:
ids = [segment.id for segment in transcript]
expected = list(range(1, len(transcript) + 1))
if ids != expected:
raise AuditaValidationError("Transcript segment ids must be sequential starting at 1.")

View File

@@ -0,0 +1 @@
"""Reusable module, filter, review, and runner contracts."""

View File

@@ -0,0 +1,108 @@
from __future__ import annotations
import threading
from typing import Any, Optional, Sequence, Tuple
from audita.core.config import AuditaConfig, DEFAULT_BASE_URL
from audita.core.errors import AuditaLLMError
_NO_KEY_PLACEHOLDER = "audita-no-key-required"
class OpenAICompatibleStructuredLLMClient:
def __init__(self) -> None:
self._client = None
self._client_identity: Optional[Tuple[str, str, float]] = None
self._client_lock = threading.Lock()
def run_structured(
self,
*,
stage_name: str,
messages: Sequence[dict],
response_model: Any,
config: AuditaConfig,
) -> Any:
if _requires_api_key(config) and not config.api_key:
raise AuditaLLMError(
f"Structured LLM stage '{stage_name}' requires LLM API credentials for the configured OpenRouter endpoint "
"via --llm-api-key, --validation-llm-api-key, AUDITA_LLM_API_KEY, "
"AUDITA_VALIDATION_LLM_API_KEY, or OPENROUTER_API_KEY."
)
client = self._get_client(config)
request = {
"model": _request_model_name(config),
"messages": list(messages),
"response_model": response_model,
"max_retries": config.max_retries,
}
if _uses_openrouter_shape(config):
request["extra_body"] = {"provider": {"require_parameters": True}}
try:
return client.chat.completions.create(**request)
except Exception as exc:
raise AuditaLLMError(
f"Structured LLM stage '{stage_name}' failed. Confirm the configured model and "
"OpenAI-compatible endpoint support tool calling or structured outputs."
) from exc
def _get_client(self, config: AuditaConfig) -> Any:
identity = (config.api_key or "", config.base_url, config.llm_timeout_seconds)
with self._client_lock:
if self._client is not None and self._client_identity == identity:
return self._client
try:
import instructor
from openai import OpenAI
except ImportError as exc:
raise AuditaLLMError(
"The LLM dependencies are not installed. Run `uv sync` before using Audita LLM stages."
) from exc
openai_client = OpenAI(
api_key=_client_api_key(config),
base_url=config.base_url,
timeout=config.llm_timeout_seconds,
)
self._client = instructor.patch(openai_client, mode=instructor.Mode.TOOLS)
self._client_identity = identity
return self._client
def _normalize_openrouter_model(model: str) -> str:
prefix = "openrouter/"
if model.startswith(prefix):
return model[len(prefix) :]
return model
def _request_model_name(config: AuditaConfig) -> str:
if _uses_openrouter_shape(config):
return _normalize_openrouter_model(config.model)
return config.model
def _uses_openrouter_shape(config: AuditaConfig) -> bool:
return _normalized_base_url(config.base_url) == _normalized_base_url(DEFAULT_BASE_URL) or config.model.startswith(
"openrouter/"
)
def _requires_api_key(config: AuditaConfig) -> bool:
return _normalized_base_url(config.base_url) == _normalized_base_url(DEFAULT_BASE_URL)
def _client_api_key(config: AuditaConfig) -> str:
if config.api_key:
return config.api_key
if _requires_api_key(config):
raise AuditaLLMError(
"OpenRouter credentials are required but missing for the configured endpoint."
)
return _NO_KEY_PLACEHOLDER
def _normalized_base_url(base_url: str) -> str:
return base_url.rstrip("/")

View File

@@ -0,0 +1,17 @@
from __future__ import annotations
import threading
from typing import Callable, TypeVar
T = TypeVar("T")
class ModuleLLMScheduler:
def __init__(self, max_concurrency: int) -> None:
self.max_concurrency = max_concurrency
self._semaphore = threading.BoundedSemaphore(max_concurrency)
def run_backend_call(self, fn: Callable[[], T]) -> T:
with self._semaphore:
return fn()

View File

@@ -0,0 +1,80 @@
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Optional, Protocol, Sequence
from audita.core.chunking import TranscriptSection
from audita.core.config import AuditaConfig
from audita.core.schemas import Glossary
from audita.validators.base import Validator
from .llm_scheduler import ModuleLLMScheduler
ReplacementPolicy = str
@dataclass(frozen=True)
class ModuleRunSpec:
instance_name: str
module_key: str
module: "TranscriptModule"
@dataclass(frozen=True)
class CorrectionProposal:
proposal_index: int
module_instance: str
module_key: str
id: int
original_text: str
corrected_text: str
confidence: float
def to_prompt_payload(self) -> dict:
return {
"proposal_index": self.proposal_index,
"id": self.id,
"original_text": self.original_text,
"corrected_text": self.corrected_text,
"confidence": self.confidence,
}
@dataclass(frozen=True)
class ModuleContext:
run_spec: ModuleRunSpec
glossary: Glossary
config: AuditaConfig
run_dir: Path
llm_client: Optional["StructuredLLMClient"] = None
llm_scheduler: Optional[ModuleLLMScheduler] = None
validation_llm_scheduler: Optional[ModuleLLMScheduler] = None
class StructuredLLMClient(Protocol):
def run_structured(
self,
*,
stage_name: str,
messages: Sequence[dict],
response_model: Any,
config: AuditaConfig,
) -> Any:
...
class TranscriptModule(Protocol):
module_key: str
replacement_policy: ReplacementPolicy
def validators(self) -> Sequence[Validator]:
...
def propose(
self,
transcript_section: TranscriptSection,
context: ModuleContext,
) -> Sequence[CorrectionProposal]:
...

View File

@@ -0,0 +1,96 @@
from __future__ import annotations
import json
import math
from typing import Any, Callable, Dict, List
from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator
from audita.core.chunking import TranscriptSection
from audita.core.errors import AuditaLLMError
from audita.core.schemas import Glossary
from .models import CorrectionProposal, ModuleContext
Message = Dict[str, str]
ProposalPromptBuilder = Callable[[TranscriptSection, Glossary], List[Message]]
class StructuredCorrectionCandidate(BaseModel):
model_config = ConfigDict(extra="forbid")
id: int = Field(ge=1)
original_text: StrictStr
corrected_text: StrictStr
confidence: float = Field(ge=0.0, le=1.0)
@field_validator("id", mode="before")
@classmethod
def require_integer_id(cls, value: Any) -> int:
if isinstance(value, bool) or not isinstance(value, int):
raise ValueError("must be an integer")
return value
@field_validator("confidence", mode="before")
@classmethod
def require_number(cls, value: Any) -> float:
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise ValueError("must be a JSON number")
number = float(value)
if not math.isfinite(number):
raise ValueError("must be finite")
return number
class StructuredCorrectionSet(BaseModel):
model_config = ConfigDict(extra="forbid")
corrections: List[StructuredCorrectionCandidate] = Field(default_factory=list)
def generate_llm_correction_proposals(
*,
section: TranscriptSection,
context: ModuleContext,
prompt_builder: ProposalPromptBuilder,
) -> List[CorrectionProposal]:
llm_client = context.llm_client
if llm_client is None:
raise AuditaLLMError(f"Module '{context.run_spec.instance_name}' requires a structured LLM client.")
messages = prompt_builder(section, context.glossary)
prompt_path = context.run_dir / f"prompt-{section.section_index:04d}.json"
prompt_path.write_text(json.dumps(messages, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
if context.llm_scheduler is not None:
response = context.llm_scheduler.run_backend_call(
lambda: llm_client.run_structured(
stage_name=f"{context.run_spec.instance_name}:proposal",
messages=messages,
response_model=StructuredCorrectionSet,
config=context.config.proposal_llm_config(),
)
)
else:
response = llm_client.run_structured(
stage_name=f"{context.run_spec.instance_name}:proposal",
messages=messages,
response_model=StructuredCorrectionSet,
config=context.config.proposal_llm_config(),
)
response_path = context.run_dir / f"corrections-{section.section_index:04d}.json"
response_path.write_text(response.model_dump_json(indent=2) + "\n", encoding="utf-8")
return [
CorrectionProposal(
proposal_index=index,
module_instance=context.run_spec.instance_name,
module_key=context.run_spec.module_key,
id=correction.id,
original_text=correction.original_text,
corrected_text=correction.corrected_text,
confidence=correction.confidence,
)
for index, correction in enumerate(response.corrections)
]

View File

@@ -0,0 +1,82 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Optional, Sequence, Union
from audita.core.schemas import TranscriptSegment
if TYPE_CHECKING:
from .models import CorrectionProposal
ReplacementPolicy = str
@dataclass(frozen=True)
class ProposalPreview:
proposal: CorrectionProposal
segment_index: int
segment: TranscriptSegment
corrected_segment_text: str
@property
def original_segment_text(self) -> str:
return self.segment.text
def to_prompt_payload(self) -> dict:
payload = {
"correction_index": self.proposal.proposal_index,
"id": self.proposal.id,
"original_segment_text": self.original_segment_text,
"corrected_segment_text": self.corrected_segment_text,
"original_text": self.proposal.original_text,
"corrected_text": self.proposal.corrected_text,
}
if self.segment.categories is not None:
payload["categories"] = list(self.segment.categories)
return payload
@dataclass(frozen=True)
class ProposalPreviewError:
reason: str
actual_text: Optional[str] = None
def preview_proposal(
transcript: Sequence[TranscriptSegment],
proposal: CorrectionProposal,
replacement_policy: ReplacementPolicy,
) -> Union[ProposalPreview, ProposalPreviewError]:
index_by_id = {segment.id: index for index, segment in enumerate(transcript)}
segment_index = index_by_id.get(proposal.id)
if segment_index is None:
return ProposalPreviewError(reason="proposal references unknown segment id")
segment = transcript[segment_index]
if proposal.original_text == "":
return ProposalPreviewError(
reason="proposal original_text must not be empty",
actual_text=segment.text,
)
match_count = segment.text.count(proposal.original_text)
if match_count == 0:
return ProposalPreviewError(
reason="proposal original_text does not match segment text",
actual_text=segment.text,
)
if replacement_policy == "require_unique" and match_count != 1:
return ProposalPreviewError(
reason="proposal original_text must match exactly once",
actual_text=segment.text,
)
corrected_segment_text = segment.text.replace(proposal.original_text, proposal.corrected_text)
return ProposalPreview(
proposal=proposal,
segment_index=segment_index,
segment=segment,
corrected_segment_text=corrected_segment_text,
)

View File

@@ -0,0 +1,485 @@
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from pathlib import Path
from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
from audita.core.chunking import TranscriptSection, chunk_transcript
from audita.core.config import AuditaConfig
from audita.core.reporting import AppliedChange, ModuleRunReport, ReportedSkip, ValidatorReport
from audita.core.schemas import Glossary, TranscriptSegment
from audita.validators.base import ValidationContext, ValidationDecision, ValidationResult
from .llm_scheduler import ModuleLLMScheduler
from .models import CorrectionProposal, ModuleContext, ModuleRunSpec, StructuredLLMClient
from .proposals import ProposalPreviewError, preview_proposal
ProgressCallback = Callable[[str], None]
@dataclass(frozen=True)
class PipelineRunResult:
transcript: List[TranscriptSegment]
module_reports: List[ModuleRunReport]
applied_changes: List[AppliedChange]
skipped_corrections: List[ReportedSkip]
class ModuleExecutionError(Exception):
def __init__(
self,
*,
transcript: List[TranscriptSegment],
applied_changes: List[AppliedChange],
skipped_corrections: List[ReportedSkip],
cause: Exception,
) -> None:
super().__init__(str(cause))
self.transcript = transcript
self.applied_changes = applied_changes
self.skipped_corrections = skipped_corrections
self.cause = cause
class PipelineRunError(Exception):
def __init__(
self,
*,
transcript: List[TranscriptSegment],
module_reports: List[ModuleRunReport],
applied_changes: List[AppliedChange],
skipped_corrections: List[ReportedSkip],
cause: Exception,
module_instance: Optional[str],
) -> None:
super().__init__(str(cause))
self.transcript = transcript
self.module_reports = module_reports
self.applied_changes = applied_changes
self.skipped_corrections = skipped_corrections
self.cause = cause
self.module_instance = module_instance
class PipelineRunner:
def run(
self,
*,
transcript: List[TranscriptSegment],
glossary: Glossary,
module_specs: Sequence[ModuleRunSpec],
config: AuditaConfig,
run_dir: Path,
llm_client: Optional[StructuredLLMClient] = None,
progress: Optional[ProgressCallback] = None,
) -> PipelineRunResult:
working = list(transcript)
module_reports: List[ModuleRunReport] = []
applied_changes: List[AppliedChange] = []
skipped_corrections: List[ReportedSkip] = []
for run_spec in module_specs:
try:
module_dir = run_dir / run_spec.instance_name
module_dir.mkdir(parents=True, exist_ok=True)
context = ModuleContext(
run_spec=run_spec,
glossary=glossary,
config=config,
run_dir=module_dir,
llm_client=llm_client,
llm_scheduler=ModuleLLMScheduler(config.llm_concurrency),
validation_llm_scheduler=ModuleLLMScheduler(config.effective_validation_llm_concurrency()),
)
sections = chunk_transcript(
working,
config.max_section_tokens,
min_section_tokens=config.min_section_tokens,
target_section_count=config.llm_concurrency if config.target_sections is None else None,
exact_target_section_count=config.target_sections,
)
if progress is not None:
progress(
f"Running module {run_spec.instance_name} "
f"({len(sections)} sections, {len(working)} segments)"
)
result = _run_module(
working=working,
sections=sections,
context=context,
llm_client=llm_client,
)
except ModuleExecutionError as exc:
raise PipelineRunError(
transcript=exc.transcript,
module_reports=list(module_reports),
applied_changes=[*applied_changes, *exc.applied_changes],
skipped_corrections=[*skipped_corrections, *exc.skipped_corrections],
cause=exc.cause,
module_instance=run_spec.instance_name,
) from exc
except Exception as exc:
raise PipelineRunError(
transcript=list(working),
module_reports=list(module_reports),
applied_changes=list(applied_changes),
skipped_corrections=list(skipped_corrections),
cause=exc,
module_instance=run_spec.instance_name,
) from exc
working = result.transcript
module_reports.append(result.module_report)
applied_changes.extend(result.applied_changes)
skipped_corrections.extend(result.skipped_corrections)
return PipelineRunResult(
transcript=working,
module_reports=module_reports,
applied_changes=applied_changes,
skipped_corrections=skipped_corrections,
)
@dataclass(frozen=True)
class _ModuleExecutionResult:
transcript: List[TranscriptSegment]
module_report: ModuleRunReport
applied_changes: List[AppliedChange]
skipped_corrections: List[ReportedSkip]
def _run_module(
*,
working: List[TranscriptSegment],
sections: Sequence[TranscriptSection],
context: ModuleContext,
llm_client: Optional[StructuredLLMClient],
) -> _ModuleExecutionResult:
module = context.run_spec.module
validators = list(module.validators())
raw_proposals: List[CorrectionProposal] = []
validator_reports: List[ValidatorReport] = []
skipped: List[ReportedSkip] = []
applied_changes: List[AppliedChange] = []
updated_transcript = list(working)
try:
section_proposals = _collect_module_proposals(sections=sections, context=context)
for proposed in section_proposals:
raw_proposals.extend(proposed)
proposals = [
CorrectionProposal(
proposal_index=index,
module_instance=context.run_spec.instance_name,
module_key=context.run_spec.module_key,
id=proposal.id,
original_text=proposal.original_text,
corrected_text=proposal.corrected_text,
confidence=proposal.confidence,
)
for index, proposal in enumerate(raw_proposals)
]
surviving = proposals
validator_index = 0
while validator_index < len(validators):
validator = validators[validator_index]
candidate_count = len(surviving)
if not surviving:
validator_reports.append(
ValidatorReport(
name=validator.name,
execution_kind=validator.execution_kind,
candidate_count=0,
approved_count=0,
rejected_count=0,
)
)
validator_index += 1
continue
if validator.execution_kind != "llm":
result = _run_single_validator(
validator=validator,
proposals=surviving,
working=working,
context=context,
llm_client=llm_client,
)
validator_reports.append(result.report)
skipped.extend(result.skipped)
surviving = result.approved
validator_index += 1
continue
group_start = validator_index
llm_group = []
while validator_index < len(validators) and validators[validator_index].execution_kind == "llm":
llm_group.append(validators[validator_index])
validator_index += 1
group_result = _run_parallel_llm_validator_group(
validators=llm_group,
proposals=surviving,
working=working,
context=context,
llm_client=llm_client,
)
validator_reports.extend(group_result.reports)
skipped.extend(group_result.skipped)
surviving = group_result.approved
for proposal in surviving:
apply_result = _apply_proposal(updated_transcript, proposal, module.replacement_policy)
if isinstance(apply_result, ReportedSkip):
skipped.append(apply_result)
continue
updated_transcript, applied_change = apply_result
applied_changes.append(applied_change)
module_report = ModuleRunReport(
instance_name=context.run_spec.instance_name,
module_key=context.run_spec.module_key,
replacement_policy=module.replacement_policy,
section_count=len(sections),
proposal_count=len(proposals),
validators=validator_reports,
approved_count=len(surviving),
applied_count=len(applied_changes),
skipped_count=len(skipped),
)
return _ModuleExecutionResult(
transcript=updated_transcript,
module_report=module_report,
applied_changes=applied_changes,
skipped_corrections=skipped,
)
except Exception as exc:
raise ModuleExecutionError(
transcript=updated_transcript,
applied_changes=applied_changes,
skipped_corrections=skipped,
cause=exc,
) from exc
def _collect_module_proposals(
*,
sections: Sequence[TranscriptSection],
context: ModuleContext,
) -> List[List[CorrectionProposal]]:
module = context.run_spec.module
max_workers = context.llm_scheduler.max_concurrency if context.llm_scheduler is not None else context.config.llm_concurrency
if max_workers == 1 or len(sections) <= 1:
return [list(module.propose(section, context)) for section in sections]
with ThreadPoolExecutor(max_workers=max_workers) as executor:
return list(executor.map(lambda section: list(module.propose(section, context)), sections))
@dataclass(frozen=True)
class _SingleValidatorResult:
approved: List[CorrectionProposal]
report: ValidatorReport
skipped: List[ReportedSkip]
@dataclass(frozen=True)
class _ParallelValidatorGroupResult:
approved: List[CorrectionProposal]
reports: List[ValidatorReport]
skipped: List[ReportedSkip]
def _build_validation_context(
*,
proposals: Sequence[CorrectionProposal],
working: Sequence[TranscriptSegment],
context: ModuleContext,
llm_client: Optional[StructuredLLMClient],
) -> ValidationContext:
return ValidationContext(
proposals=proposals,
transcript=working,
glossary=context.glossary,
config=context.config.validation_llm_config(),
run_spec=context.run_spec,
run_dir=context.run_dir,
llm_client=llm_client,
llm_scheduler=context.validation_llm_scheduler,
)
def _run_single_validator(
*,
validator,
proposals: Sequence[CorrectionProposal],
working: Sequence[TranscriptSegment],
context: ModuleContext,
llm_client: Optional[StructuredLLMClient],
) -> _SingleValidatorResult:
candidate_count = len(proposals)
validation_context = _build_validation_context(
proposals=proposals,
working=working,
context=context,
llm_client=llm_client,
)
result = validator.validate(validation_context)
decisions_by_index = _index_validation_decisions(result, proposals, validator.name)
approved: List[CorrectionProposal] = []
skipped: List[ReportedSkip] = []
rejected_count = 0
for proposal in proposals:
decision = decisions_by_index[proposal.proposal_index]
if decision.approved:
approved.append(proposal)
continue
rejected_count += 1
skipped.append(_reported_skip_from_decision(proposal, working, validator.name, decision))
return _SingleValidatorResult(
approved=approved,
report=ValidatorReport(
name=validator.name,
execution_kind=validator.execution_kind,
candidate_count=candidate_count,
approved_count=len(approved),
rejected_count=rejected_count,
),
skipped=skipped,
)
def _run_parallel_llm_validator_group(
*,
validators: Sequence,
proposals: Sequence[CorrectionProposal],
working: Sequence[TranscriptSegment],
context: ModuleContext,
llm_client: Optional[StructuredLLMClient],
) -> _ParallelValidatorGroupResult:
validation_context = _build_validation_context(
proposals=proposals,
working=working,
context=context,
llm_client=llm_client,
)
scheduler = context.validation_llm_scheduler
max_workers = scheduler.max_concurrency if scheduler is not None else context.config.effective_validation_llm_concurrency()
if max_workers == 1 or len(validators) <= 1:
results = [validator.validate(validation_context) for validator in validators]
else:
with ThreadPoolExecutor(max_workers=max_workers) as executor:
results = list(executor.map(lambda validator: validator.validate(validation_context), validators))
indexed_results = [
_index_validation_decisions(result, proposals, validator.name)
for validator, result in zip(validators, results)
]
reports = [
ValidatorReport(
name=validator.name,
execution_kind=validator.execution_kind,
candidate_count=len(proposals),
approved_count=sum(1 for decision in decisions.values() if decision.approved),
rejected_count=sum(1 for decision in decisions.values() if not decision.approved),
)
for validator, decisions in zip(validators, indexed_results)
]
approved: List[CorrectionProposal] = []
skipped: List[ReportedSkip] = []
for proposal in proposals:
rejection = None
for validator, decisions in zip(validators, indexed_results):
decision = decisions[proposal.proposal_index]
if not decision.approved:
rejection = (validator.name, decision)
break
if rejection is None:
approved.append(proposal)
continue
validator_name, decision = rejection
skipped.append(_reported_skip_from_decision(proposal, working, validator_name, decision))
return _ParallelValidatorGroupResult(approved=approved, reports=reports, skipped=skipped)
def _reported_skip_from_decision(
proposal: CorrectionProposal,
working: Sequence[TranscriptSegment],
validator_name: str,
decision: ValidationDecision,
) -> ReportedSkip:
return ReportedSkip(
module_instance=proposal.module_instance,
module_key=proposal.module_key,
proposal_index=proposal.proposal_index,
id=proposal.id,
reason=decision.reason or f"{validator_name} rejected proposal",
original_text=proposal.original_text,
corrected_text=proposal.corrected_text,
confidence=proposal.confidence,
actual_text=_segment_text_by_id(working).get(proposal.id),
source=f"validator:{validator_name}",
)
def _index_validation_decisions(
result: ValidationResult,
proposals: Sequence[CorrectionProposal],
validator_name: str,
) -> Dict[int, ValidationDecision]:
expected_indexes = {proposal.proposal_index for proposal in proposals}
indexed: Dict[int, ValidationDecision] = {}
for decision in result.decisions:
if decision.proposal_index in indexed:
raise ValueError(f"Validator '{validator_name}' returned duplicate proposal indexes.")
if decision.proposal_index not in expected_indexes:
raise ValueError(f"Validator '{validator_name}' returned an unknown proposal index.")
indexed[decision.proposal_index] = decision
missing = expected_indexes - set(indexed)
if missing:
raise ValueError(f"Validator '{validator_name}' omitted proposal indexes: {sorted(missing)}")
return indexed
def _apply_proposal(
transcript: List[TranscriptSegment],
proposal: CorrectionProposal,
replacement_policy: str,
) -> Union[Tuple[List[TranscriptSegment], AppliedChange], ReportedSkip]:
preview = preview_proposal(transcript, proposal, replacement_policy)
if isinstance(preview, ProposalPreviewError):
return ReportedSkip(
module_instance=proposal.module_instance,
module_key=proposal.module_key,
proposal_index=proposal.proposal_index,
id=proposal.id,
reason=preview.reason,
original_text=proposal.original_text,
corrected_text=proposal.corrected_text,
confidence=proposal.confidence,
actual_text=preview.actual_text,
source="application",
)
updated = list(transcript)
updated[preview.segment_index] = preview.segment.model_copy(update={"text": preview.corrected_segment_text})
return (
updated,
AppliedChange(
module_instance=proposal.module_instance,
module_key=proposal.module_key,
proposal_index=proposal.proposal_index,
id=proposal.id,
original_text=proposal.original_text,
corrected_text=proposal.corrected_text,
confidence=proposal.confidence,
segment_text_before=preview.original_segment_text,
segment_text_after=preview.corrected_segment_text,
),
)
def _segment_text_by_id(transcript: Sequence[TranscriptSegment]) -> Dict[int, str]:
return {segment.id: segment.text for segment in transcript}

View File

@@ -0,0 +1,80 @@
from __future__ import annotations
from collections import Counter, defaultdict
from typing import Sequence
from audita.core.errors import AuditaConfigError
SUPPORTED_MODULE_KEYS = ("glossary", "homophones", "spoken_word", "grammar")
DEFAULT_MODULE_KEYS = ("glossary", "homophones", "glossary", "spoken_word", "grammar")
def normalize_module_keys(module_keys: Sequence[str]) -> tuple[str, ...]:
normalized = tuple(_normalize_module_key(item) for item in module_keys)
if not normalized:
raise AuditaConfigError("Module sequence must contain at least one module key.")
invalid = tuple(item for item in normalized if item not in SUPPORTED_MODULE_KEYS)
if invalid:
valid = ", ".join(SUPPORTED_MODULE_KEYS)
unknown = ", ".join(invalid)
raise AuditaConfigError(f"Unknown module key(s): {unknown}. Valid module keys: {valid}.")
return normalized
def resolve_module_specs(module_keys: Sequence[str]) -> list["ModuleRunSpec"]:
from audita.framework.models import ModuleRunSpec
normalized = normalize_module_keys(module_keys)
counts = Counter(normalized)
seen: defaultdict[str, int] = defaultdict(int)
return [
ModuleRunSpec(
instance_name=_instance_name(module_key, counts, seen),
module_key=module_key,
module=_instantiate_module(module_key),
)
for module_key in normalized
]
def default_module_specs() -> list["ModuleRunSpec"]:
return resolve_module_specs(DEFAULT_MODULE_KEYS)
def _instance_name(module_key: str, counts: Counter[str], seen: defaultdict[str, int]) -> str:
seen[module_key] += 1
if counts[module_key] == 1:
return module_key
return f"{module_key}_{seen[module_key]}"
def _instantiate_module(module_key: str):
if module_key == "glossary":
from .glossary import GlossaryModule
return GlossaryModule()
if module_key == "homophones":
from .homophones import HomophonesModule
return HomophonesModule()
if module_key == "spoken_word":
from .spoken_word import SpokenWordModule
return SpokenWordModule()
if module_key == "grammar":
from .grammar import GrammarModule
return GrammarModule()
raise AuditaConfigError(f"Unknown module key '{module_key}'.")
def _normalize_module_key(value: str) -> str:
if not isinstance(value, str):
raise AuditaConfigError("Module keys must be strings.")
normalized = value.strip()
if not normalized:
raise AuditaConfigError("Module keys must not contain empty values.")
return normalized

View File

@@ -0,0 +1,43 @@
from typing import Sequence
from audita.core.chunking import TranscriptSection
from audita.framework.models import CorrectionProposal, ModuleContext
from audita.framework.proposal_generation import generate_llm_correction_proposals
from audita.modules.prompts import build_glossary_proposal_messages
from audita.validators import (
GlossaryStageProtectedGlossaryTermsValidator,
IdenticalTextValidator,
MeaningReversalValidator,
NonEmptySegmentValidator,
OriginalTextPresentValidator,
ProposalConfidenceValidator,
SpokenFormPlausibilityValidator,
Validator,
)
class GlossaryModule:
module_key = "glossary"
replacement_policy = "replace_all"
def validators(self) -> Sequence[Validator]:
return [
IdenticalTextValidator("identical_text_guard"),
OriginalTextPresentValidator("original_text_present_guard"),
ProposalConfidenceValidator("proposal_confidence_guard", "glossary_confidence_threshold"),
GlossaryStageProtectedGlossaryTermsValidator("glossary_stage_protected_glossary_guard"),
NonEmptySegmentValidator("non_empty_segment_guard"),
SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
MeaningReversalValidator("meaning_reversal_review"),
]
def propose(
self,
transcript_section: TranscriptSection,
context: ModuleContext,
) -> Sequence[CorrectionProposal]:
return generate_llm_correction_proposals(
section=transcript_section,
context=context,
prompt_builder=build_glossary_proposal_messages,
)

View File

@@ -0,0 +1,43 @@
from typing import Sequence
from audita.core.chunking import TranscriptSection
from audita.framework.models import CorrectionProposal, ModuleContext
from audita.framework.proposal_generation import generate_llm_correction_proposals
from audita.modules.prompts import build_grammar_proposal_messages
from audita.validators import (
GrammarOnlyValidator,
IdenticalTextValidator,
MeaningReversalValidator,
NonEmptySegmentValidator,
OriginalTextPresentValidator,
ProposalConfidenceValidator,
ProtectedGlossaryTermsValidator,
Validator,
)
class GrammarModule:
module_key = "grammar"
replacement_policy = "require_unique"
def validators(self) -> Sequence[Validator]:
return [
IdenticalTextValidator("identical_text_guard"),
OriginalTextPresentValidator("original_text_present_guard"),
ProposalConfidenceValidator("proposal_confidence_guard", "grammar_confidence_threshold"),
ProtectedGlossaryTermsValidator("protected_glossary_guard"),
NonEmptySegmentValidator("non_empty_segment_guard"),
GrammarOnlyValidator("grammar_only_guard"),
MeaningReversalValidator("meaning_reversal_review"),
]
def propose(
self,
transcript_section: TranscriptSection,
context: ModuleContext,
) -> Sequence[CorrectionProposal]:
return generate_llm_correction_proposals(
section=transcript_section,
context=context,
prompt_builder=build_grammar_proposal_messages,
)

View File

@@ -0,0 +1,43 @@
from typing import Sequence
from audita.core.chunking import TranscriptSection
from audita.framework.models import CorrectionProposal, ModuleContext
from audita.framework.proposal_generation import generate_llm_correction_proposals
from audita.modules.prompts import build_homophones_proposal_messages
from audita.validators import (
IdenticalTextValidator,
MeaningReversalValidator,
NonEmptySegmentValidator,
OriginalTextPresentValidator,
ProposalConfidenceValidator,
ProtectedGlossaryTermsValidator,
SpokenFormPlausibilityValidator,
Validator,
)
class HomophonesModule:
module_key = "homophones"
replacement_policy = "require_unique"
def validators(self) -> Sequence[Validator]:
return [
IdenticalTextValidator("identical_text_guard"),
OriginalTextPresentValidator("original_text_present_guard"),
ProposalConfidenceValidator("proposal_confidence_guard", "homophones_confidence_threshold"),
ProtectedGlossaryTermsValidator("protected_glossary_guard"),
NonEmptySegmentValidator("non_empty_segment_guard"),
SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
MeaningReversalValidator("meaning_reversal_review"),
]
def propose(
self,
transcript_section: TranscriptSection,
context: ModuleContext,
) -> Sequence[CorrectionProposal]:
return generate_llm_correction_proposals(
section=transcript_section,
context=context,
prompt_builder=build_homophones_proposal_messages,
)

View File

@@ -0,0 +1,169 @@
import json
from typing import Dict, List
from audita.core.chunking import TranscriptSection
from audita.core.schemas import Glossary
Message = Dict[str, str]
def build_glossary_proposal_messages(section: TranscriptSection, glossary: Glossary) -> List[Message]:
glossary_json = json.dumps(glossary.model_dump(mode="json", exclude_none=True), ensure_ascii=False, indent=2)
section_json = json.dumps(section.prompt_payload(), ensure_ascii=False, indent=2)
system = (
"You are Audita, a careful transcript correction assistant. "
"Identify only transcription errors that are strongly supported by the glossary. "
"A valid correction must be acoustically plausible: the original transcript text "
"should sound similar to the proposed correction when spoken aloud. "
"Do not make generic grammar, spelling, capitalization, style, or filler-word edits. "
"Do not substitute an unrelated glossary term just because it could fit the topic. "
"Preserve speaker names, timestamps, and meaning."
)
user = (
"Review this transcript section and return only glossary-supported corrections that should be applied.\n\n"
"Rules:\n"
"- Correct domain-specific names, aliases, jargon, deities, locations, NPCs, players, factions, and similar terms only when both the glossary and surrounding transcript context support the correction.\n"
"- The correction must plausibly fix a transcription error: the original words should be phonetically or acoustically similar to the corrected words in spoken English.\n"
"- Appropriate example: correcting \"gestures\" to \"Jesters\" can be valid if \"Jesters\" appears in the glossary and nearby context supports that inference.\n"
"- Inappropriate example: correcting \"Lyra\" to \"Jesters\" should be omitted because those words are not similar in spoken English, even if \"Jesters\" appears in the glossary.\n"
"- Do not replace one clear glossary term, character name, location, or ordinary word with a different glossary term unless it is a plausible mishearing.\n"
"- Treat glossary names and aliases already present in the transcript as protected spellings.\n"
"- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.\n"
"- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n"
"- If a segment includes categories, treat them as additional transcript context.\n"
"- Plural forms of glossary names and aliases are allowed targets when spoken similarity and context support them, even if the plural is not explicitly listed in the glossary.\n"
"- Use the exact id from the input segment.\n"
"- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n"
"- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n"
"- Each returned correction must contain only id, original_text, corrected_text, and confidence.\n"
"- Do not return corrections where original_text and corrected_text are identical.\n"
"- Do not return speaker, start, or end fields.\n"
"- Return only changed segments; do not return entries for unchanged segments.\n"
"- confidence must be between 0.0 and 1.0.\n"
"- If no corrections are needed, return an empty corrections list.\n\n"
f"Glossary:\n{glossary_json}\n\n"
f"Transcript section:\n{section_json}"
)
return [{"role": "system", "content": system}, {"role": "user", "content": user}]
def build_homophones_proposal_messages(section: TranscriptSection, glossary: Glossary) -> List[Message]:
glossary_json = json.dumps(glossary.model_dump(mode="json", exclude_none=True), ensure_ascii=False, indent=2)
section_json = json.dumps(section.prompt_payload(), ensure_ascii=False, indent=2)
system = (
"You are Audita, a conservative homophone correction assistant. "
"Identify only transcript changes that plausibly reflect homophones, phonetic similarity, "
"or common mistranscriptions of spoken English. "
"Do not make punctuation, capitalization, spacing, filler-word, repetition, style, or grammar edits. "
"Do not paraphrase, summarize, or rewrite content."
)
user = (
"Review this transcript section and return only homophone or spoken-form corrections that should be applied.\n\n"
"Rules:\n"
"- Approve only corrections where the original text is plausibly a mistaken homophone, phonetic rendering, or mistranscription of what was likely spoken.\n"
"- Allow examples such as changing \"dam\" to \"damn\", \"rank\" to \"Hrank\", or \"gestures\" to \"Jesters\" when local context supports the correction.\n"
"- Reject unrelated substitutions like changing \"Lyra\" to \"Jesters\".\n"
"- Reject antonyms or reversals such as changing \"visible\" to \"invisible\".\n"
"- Do not add or remove punctuation, alter capitalization only, normalize spacing, remove filler words, collapse repetitions, or make general readability edits.\n"
"- Treat glossary names and aliases as protected spellings and context.\n"
"- You may correct toward glossary names, aliases, or their plural forms when the correction is acoustically plausible and supported by local context.\n"
"- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.\n"
"- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n"
"- If a segment includes categories, treat them as additional transcript context.\n"
"- Use the exact id from the input segment.\n"
"- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n"
"- Choose an original_text span that appears exactly once in the current segment text.\n"
"- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n"
"- Each returned correction must contain only id, original_text, corrected_text, and confidence.\n"
"- Do not return corrections where original_text and corrected_text are identical.\n"
"- Do not return speaker, start, or end fields.\n"
"- Return only changed segments; do not return entries for unchanged segments.\n"
"- confidence must be between 0.0 and 1.0.\n"
"- If no corrections are needed, return an empty corrections list.\n\n"
f"Protected glossary/context:\n{glossary_json}\n\n"
f"Transcript section:\n{section_json}"
)
return [{"role": "system", "content": system}, {"role": "user", "content": user}]
def build_spoken_word_proposal_messages(section: TranscriptSection, glossary: Glossary) -> List[Message]:
glossary_json = json.dumps(glossary.model_dump(mode="json", exclude_none=True), ensure_ascii=False, indent=2)
section_json = json.dumps(section.prompt_payload(), ensure_ascii=False, indent=2)
system = (
"You are Audita, a conservative spoken-word cleanup assistant. "
"Identify only low-risk cleanup of repeated words or short phrases, filler words, hesitation artifacts, "
"and similar dysfluencies that commonly appear in spoken English transcripts. "
"Preserve substantive meaning, named entities, and transcript content."
)
user = (
"Review this transcript section and return only spoken-word cleanup corrections that should be applied.\n\n"
"Rules:\n"
"- Approve only conservative cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar spoken dysfluencies.\n"
"- You may collapse adjacent repetition such as \"I I think\" to \"I think\" or remove filler spans such as \"you know\" or \"uh\" when local context supports that cleanup.\n"
"- Do not collapse repeated words or short phrases when the repetition plausibly expresses urgency, excitement, insistence, or deliberate rhetorical emphasis rather than dysfluency.\n"
"- Phrases such as \"Help! Help! Help!\", \"Stop! Stop! Stop!\", \"No! No! No!\", \"Yes! Yes! Yes!\", and \"Go! Go! Go!\" are often intentional emphasis and should usually be preserved.\n"
"- Only collapse repetition when local context supports it as accidental spoken repetition, hesitation, or verbal restart.\n"
"- You may include low-risk punctuation, spacing, or capitalization cleanup when it is part of removing a dysfluency, such as removing ellipses or hesitation punctuation that no longer belongs after the cleanup.\n"
"- Do not paraphrase, summarize, reorder ideas, replace content with different wording, or make substantive semantic edits.\n"
"- Do not change clear content words just because a different phrasing reads better.\n"
"- Treat glossary names and aliases as protected spellings and context.\n"
"- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.\n"
"- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n"
"- If a segment includes categories, treat them as additional transcript context.\n"
"- Use the exact id from the input segment.\n"
"- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n"
"- Choose an original_text span that appears exactly once in the current segment text.\n"
"- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n"
"- Each returned correction must contain only id, original_text, corrected_text, and confidence.\n"
"- Do not return corrections where original_text and corrected_text are identical.\n"
"- Do not return speaker, start, or end fields.\n"
"- Return only changed segments; do not return entries for unchanged segments.\n"
"- confidence must be between 0.0 and 1.0.\n"
"- If no corrections are needed, return an empty corrections list.\n\n"
f"Protected glossary/context:\n{glossary_json}\n\n"
f"Transcript section:\n{section_json}"
)
return [{"role": "system", "content": system}, {"role": "user", "content": user}]
def build_grammar_proposal_messages(section: TranscriptSection, glossary: Glossary) -> List[Message]:
glossary_json = json.dumps(glossary.model_dump(mode="json", exclude_none=True), ensure_ascii=False, indent=2)
section_json = json.dumps(section.prompt_payload(), ensure_ascii=False, indent=2)
system = (
"You are Audita, a conservative grammar cleanup assistant. "
"Identify only punctuation, capitalization, and spacing cleanup that preserves the same underlying words. "
"Do not change content, substitute words, or rewrite the speaker's phrasing."
)
user = (
"Review this transcript section and return only grammar cleanup corrections that should be applied.\n\n"
"Rules:\n"
"- Allowed changes are punctuation, capitalization, spacing, and article cleanup only.\n"
"- You may add, remove, or adjust commas, periods, quotation marks, apostrophes, dashes, ellipses, spacing, and capitalization when the underlying words stay the same.\n"
"- You may change the whole-word article \"a\" to \"an\" or \"an\" to \"a\" when the surrounding text otherwise stays the same.\n"
"- Homophone, spoken-form, and mistranscription corrections are handled during a later review stage; do not propose them here.\n"
"- Do not make word substitutions, spelling fixes, homophone fixes, filler cleanup, repetition cleanup, paraphrases, or other semantic rewrites.\n"
"- Do not change one written word into a different written word, except for capitalization changes to the same letters.\n"
"- If a possible correction depends on changing a content word into a different word, omit it here rather than bundling it together with formatting cleanup.\n"
"- Treat glossary names and aliases as protected spellings and context.\n"
"- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.\n"
"- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n"
"- If a segment includes categories, treat them as additional transcript context.\n"
"- Use the exact id from the input segment.\n"
"- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n"
"- Choose an original_text span that appears exactly once in the current segment text.\n"
"- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n"
"- Each returned correction must contain only id, original_text, corrected_text, and confidence.\n"
"- Do not return corrections where original_text and corrected_text are identical.\n"
"- Do not return speaker, start, or end fields.\n"
"- Return only changed segments; do not return entries for unchanged segments.\n"
"- confidence must be between 0.0 and 1.0.\n"
"- If no corrections are needed, return an empty corrections list.\n\n"
f"Protected glossary/context:\n{glossary_json}\n\n"
f"Transcript section:\n{section_json}"
)
return [{"role": "system", "content": system}, {"role": "user", "content": user}]

View File

@@ -0,0 +1,43 @@
from typing import Sequence
from audita.core.chunking import TranscriptSection
from audita.framework.models import CorrectionProposal, ModuleContext
from audita.framework.proposal_generation import generate_llm_correction_proposals
from audita.modules.prompts import build_spoken_word_proposal_messages
from audita.validators import (
IdenticalTextValidator,
MeaningReversalValidator,
NonEmptySegmentValidator,
OriginalTextPresentValidator,
ProposalConfidenceValidator,
ProtectedGlossaryTermsValidator,
SpokenWordValidator,
Validator,
)
class SpokenWordModule:
module_key = "spoken_word"
replacement_policy = "require_unique"
def validators(self) -> Sequence[Validator]:
return [
IdenticalTextValidator("identical_text_guard"),
OriginalTextPresentValidator("original_text_present_guard"),
ProposalConfidenceValidator("proposal_confidence_guard", "spoken_word_confidence_threshold"),
ProtectedGlossaryTermsValidator("protected_glossary_guard"),
NonEmptySegmentValidator("non_empty_segment_guard"),
SpokenWordValidator("spoken_word_review"),
MeaningReversalValidator("meaning_reversal_review"),
]
def propose(
self,
transcript_section: TranscriptSection,
context: ModuleContext,
) -> Sequence[CorrectionProposal]:
return generate_llm_correction_proposals(
section=transcript_section,
context=context,
prompt_builder=build_spoken_word_proposal_messages,
)

View File

@@ -0,0 +1,258 @@
import json
import shutil
from pathlib import Path
from typing import Any, Callable, List, Optional, Sequence
from .core.config import AuditaConfig
from .core.diagnostics import build_error_details, create_run_dir, format_traceback_text, write_error_log
from .core.errors import AuditaError
from .core.normalization import normalize_transcript
from .core.reporting import AppliedChange, ModuleRunReport, ProcessResult, ReportedSkip, RunReport
from .core.schemas import Glossary, SourceTranscriptSegment, TranscriptSegment, source_transcript_to_json, transcript_to_json
from .framework.llm import OpenAICompatibleStructuredLLMClient
from .framework.models import StructuredLLMClient
from .framework.runner import PipelineRunError, PipelineRunner
from .modules import resolve_module_specs
ProgressCallback = Callable[[str], None]
def process_transcript(
transcript: List[SourceTranscriptSegment],
glossary: Glossary,
config: AuditaConfig,
module_keys: Optional[Sequence[str]] = None,
llm_client: Optional[StructuredLLMClient] = None,
progress: Optional[ProgressCallback] = None,
) -> List[TranscriptSegment]:
return process_transcript_result(
transcript,
glossary,
config,
module_keys=module_keys,
llm_client=llm_client,
progress=progress,
).transcript
def process_transcript_result(
transcript: List[SourceTranscriptSegment],
glossary: Glossary,
config: AuditaConfig,
module_keys: Optional[Sequence[str]] = None,
llm_client: Optional[StructuredLLMClient] = None,
progress: Optional[ProgressCallback] = None,
run_dir: Optional[Path] = None,
invocation_details: Optional[dict[str, Any]] = None,
) -> ProcessResult:
run_dir = create_run_dir(config.work_dir) if run_dir is None else run_dir
module_specs = resolve_module_specs(config.module_keys if module_keys is None else module_keys)
normalized_transcript: List[TranscriptSegment] = []
normalization_summary: Optional[dict] = None
report: Optional[RunReport] = None
report_path = run_dir / "report.json"
error_log_path = run_dir / "error.log"
try:
_log(progress, f"Created work directory {run_dir}")
normalization_result = normalize_transcript(
transcript,
max_segment_gap=config.normalize_max_segment_gap,
ellipsis_gap=config.normalize_ellipsis_gap,
max_segment_duration=config.normalize_max_segment_duration,
max_segment_tokens=config.normalize_max_segment_tokens,
)
normalized_transcript = list(normalization_result.transcript)
normalization_summary = normalization_result.summary.to_dict()
_write_normalization_diagnostics(run_dir, transcript, normalization_result)
_log(
progress,
"Normalized transcript from "
f"{normalization_result.summary.source_segment_count} to "
f"{normalization_result.summary.normalized_segment_count} segments",
)
pipeline_runner = PipelineRunner()
effective_llm_client = OpenAICompatibleStructuredLLMClient() if llm_client is None else llm_client
pipeline_result = pipeline_runner.run(
transcript=normalized_transcript,
glossary=glossary,
module_specs=module_specs,
config=config,
run_dir=run_dir,
llm_client=effective_llm_client,
progress=progress,
)
revised = _sort_transcript_chronologically(pipeline_result.transcript)
work_dir_retained = _should_retain_run_dir(config.work_dir_retention, bool(pipeline_result.skipped_corrections))
report = _build_run_report(
status="success",
config=config.to_report_dict(),
normalization=normalization_summary,
pipeline=[spec.instance_name for spec in module_specs],
modules=pipeline_result.module_reports,
applied_changes=pipeline_result.applied_changes,
skipped_corrections=pipeline_result.skipped_corrections,
transcript=revised,
work_dir_retention=config.work_dir_retention,
work_dir_retained=work_dir_retained,
work_dir=str(run_dir) if work_dir_retained else None,
error=None,
error_details=None,
)
_write_run_report(report_path, report)
if work_dir_retained:
_log(progress, f"Work directory preserved at {run_dir}")
else:
shutil.rmtree(run_dir)
_log(progress, "Removed work directory after successful run")
return ProcessResult(
transcript=revised,
report=report,
run_dir=run_dir,
work_dir_retained=work_dir_retained,
)
except Exception as exc:
failed_report_data = _failure_report_data(exc, normalized_transcript)
root_error = failed_report_data["error"]
traceback_text = format_traceback_text(root_error)
error_details = build_error_details(
exc=root_error,
exit_code=1,
run_dir=run_dir,
report_path=report_path,
error_log_path=error_log_path,
phase=failed_report_data["phase"],
module_instance=failed_report_data["module_instance"],
argv=None if invocation_details is None else invocation_details.get("argv"),
cwd=None if invocation_details is None else invocation_details.get("cwd"),
traceback_text=traceback_text,
)
write_error_log(error_log_path, error_details=error_details, traceback_text=traceback_text)
report = _build_run_report(
status="failed",
config=config.to_report_dict(),
normalization=normalization_summary,
pipeline=[spec.instance_name for spec in module_specs],
modules=failed_report_data["modules"],
applied_changes=failed_report_data["applied_changes"],
skipped_corrections=failed_report_data["skipped_corrections"],
transcript=failed_report_data["transcript"],
work_dir_retention=config.work_dir_retention,
work_dir_retained=True,
work_dir=str(run_dir),
error=str(root_error),
error_details=error_details,
)
_write_run_report(report_path, report)
error = root_error
if isinstance(error, AuditaError):
reraised = type(error)(f"{error} Diagnostics preserved at {run_dir}")
setattr(reraised, "audita_run_dir", run_dir)
setattr(reraised, "audita_report_path", report_path)
setattr(reraised, "audita_error_log_path", error_log_path)
setattr(reraised, "audita_error_details", error_details)
raise reraised from error
reraised = AuditaError(f"{error} Diagnostics preserved at {run_dir}")
setattr(reraised, "audita_run_dir", run_dir)
setattr(reraised, "audita_report_path", report_path)
setattr(reraised, "audita_error_log_path", error_log_path)
setattr(reraised, "audita_error_details", error_details)
raise reraised from error
def _should_retain_run_dir(retention: str, has_skips: bool) -> bool:
if retention == "always":
return True
if retention == "never":
return False
return has_skips
def _sort_transcript_chronologically(transcript: List[TranscriptSegment]) -> List[TranscriptSegment]:
return list(sorted(transcript, key=lambda segment: (segment.start, segment.end, segment.id)))
def _write_normalization_diagnostics(run_dir: Path, transcript, normalization_result) -> None:
normalization_dir = run_dir / "normalization"
normalization_dir.mkdir(parents=True, exist_ok=True)
(normalization_dir / "source-transcript.json").write_text(source_transcript_to_json(transcript), encoding="utf-8")
(normalization_dir / "normalized-transcript.json").write_text(
transcript_to_json(normalization_result.transcript),
encoding="utf-8",
)
(normalization_dir / "summary.json").write_text(
json.dumps(normalization_result.summary.to_dict(), indent=2) + "\n",
encoding="utf-8",
)
def _write_run_report(path: Path, report: RunReport) -> None:
path.write_text(report.to_json(), encoding="utf-8")
def _log(progress: Optional[ProgressCallback], message: str) -> None:
if progress is not None:
progress(message)
def _build_run_report(
*,
status: str,
config: dict,
normalization: Optional[dict],
pipeline: List[str],
modules: List[ModuleRunReport],
applied_changes: List[AppliedChange],
skipped_corrections: List[ReportedSkip],
transcript: List[TranscriptSegment],
work_dir_retention: str,
work_dir_retained: bool,
work_dir: Optional[str],
error: Optional[str],
error_details: Optional[dict],
) -> RunReport:
return RunReport(
status=status,
config=config,
normalization=normalization,
pipeline=pipeline,
modules=modules,
applied_changes=applied_changes,
skipped_corrections=skipped_corrections,
totals={
"output_segment_count": len(transcript),
"applied_change_count": len(applied_changes),
"skipped_correction_count": len(skipped_corrections),
},
work_dir_retention=work_dir_retention,
work_dir_retained=work_dir_retained,
work_dir=work_dir,
error=error,
error_details=error_details,
)
def _failure_report_data(
exc: Exception,
normalized_transcript: List[TranscriptSegment],
) -> dict:
if isinstance(exc, PipelineRunError):
return {
"modules": exc.module_reports,
"applied_changes": exc.applied_changes,
"skipped_corrections": exc.skipped_corrections,
"transcript": _sort_transcript_chronologically(exc.transcript),
"error": exc.cause,
"phase": "pipeline",
"module_instance": exc.module_instance,
}
return {
"modules": [],
"applied_changes": [],
"skipped_corrections": [],
"transcript": _sort_transcript_chronologically(normalized_transcript),
"error": exc,
"phase": "normalization",
"module_instance": None,
}

View File

@@ -0,0 +1 @@

View File

@@ -0,0 +1,30 @@
from .base import ValidationContext, ValidationDecision, ValidationResult, Validator
from .deterministic import (
GlossaryStageProtectedGlossaryTermsValidator,
IdenticalTextValidator,
NonEmptySegmentValidator,
OriginalTextPresentValidator,
ProposalConfidenceValidator,
ProtectedGlossaryTermsValidator,
)
from .llm import EditorialValidator, GrammarOnlyValidator, MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
from .protection import ProtectedVocabulary
__all__ = [
"ValidationContext",
"ValidationDecision",
"ValidationResult",
"Validator",
"IdenticalTextValidator",
"OriginalTextPresentValidator",
"ProposalConfidenceValidator",
"ProtectedGlossaryTermsValidator",
"GlossaryStageProtectedGlossaryTermsValidator",
"NonEmptySegmentValidator",
"EditorialValidator",
"GrammarOnlyValidator",
"ProtectedVocabulary",
"SpokenFormPlausibilityValidator",
"SpokenWordValidator",
"MeaningReversalValidator",
]

View File

@@ -0,0 +1,47 @@
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, List, Optional, Protocol, Sequence
from audita.core.config import AuditaConfig
from audita.core.schemas import Glossary, TranscriptSegment
if TYPE_CHECKING:
from audita.framework.models import CorrectionProposal, ModuleRunSpec, StructuredLLMClient
from audita.framework.llm_scheduler import ModuleLLMScheduler
@dataclass(frozen=True)
class ValidationContext:
proposals: Sequence["CorrectionProposal"]
transcript: Sequence[TranscriptSegment]
glossary: Glossary
config: AuditaConfig
run_spec: "ModuleRunSpec"
run_dir: Path
llm_client: Optional["StructuredLLMClient"] = None
llm_scheduler: Optional["ModuleLLMScheduler"] = None
@dataclass(frozen=True)
class ValidationDecision:
proposal_index: int
approved: bool
confidence: Optional[float] = None
reason: Optional[str] = None
@dataclass(frozen=True)
class ValidationResult:
validator_name: str
execution_kind: str
decisions: List[ValidationDecision]
class Validator(Protocol):
name: str
execution_kind: str
def validate(self, context: ValidationContext) -> ValidationResult:
...

View File

@@ -0,0 +1,156 @@
from dataclasses import dataclass
from audita.framework.proposals import ProposalPreviewError, preview_proposal
from .base import ValidationContext, ValidationDecision, ValidationResult
from .protection import ProtectedVocabulary
@dataclass(frozen=True)
class IdenticalTextValidator:
name: str
execution_kind: str = "deterministic"
def validate(self, context: ValidationContext) -> ValidationResult:
return ValidationResult(
validator_name=self.name,
execution_kind=self.execution_kind,
decisions=[
ValidationDecision(
proposal_index=proposal.proposal_index,
approved=proposal.original_text != proposal.corrected_text,
reason=(
None
if proposal.original_text != proposal.corrected_text
else "proposal original_text and corrected_text are identical"
),
)
for proposal in context.proposals
],
)
@dataclass(frozen=True)
class OriginalTextPresentValidator:
name: str
execution_kind: str = "deterministic"
def validate(self, context: ValidationContext) -> ValidationResult:
segment_text_by_id = {segment.id: segment.text for segment in context.transcript}
return ValidationResult(
validator_name=self.name,
execution_kind=self.execution_kind,
decisions=[
ValidationDecision(
proposal_index=proposal.proposal_index,
approved=(
(segment_text := segment_text_by_id.get(proposal.id)) is None
or proposal.original_text in segment_text
),
reason=(
None
if (segment_text := segment_text_by_id.get(proposal.id)) is None or proposal.original_text in segment_text
else "proposal original_text does not match segment text"
),
)
for proposal in context.proposals
],
)
@dataclass(frozen=True)
class ProposalConfidenceValidator:
name: str
threshold_attr: str
execution_kind: str = "deterministic"
def validate(self, context: ValidationContext) -> ValidationResult:
threshold = getattr(context.config, self.threshold_attr)
return ValidationResult(
validator_name=self.name,
execution_kind=self.execution_kind,
decisions=[
ValidationDecision(
proposal_index=proposal.proposal_index,
approved=proposal.confidence >= threshold,
reason=None if proposal.confidence >= threshold else "proposal confidence below threshold",
)
for proposal in context.proposals
],
)
@dataclass(frozen=True)
class ProtectedGlossaryTermsValidator:
name: str
execution_kind: str = "deterministic"
def validate(self, context: ValidationContext) -> ValidationResult:
vocabulary = ProtectedVocabulary.from_glossary(context.glossary)
return ValidationResult(
validator_name=self.name,
execution_kind=self.execution_kind,
decisions=[
ValidationDecision(
proposal_index=proposal.proposal_index,
approved=(reason := vocabulary.violation_reason(
proposal.original_text,
proposal.corrected_text,
)) is None,
reason=reason,
)
for proposal in context.proposals
],
)
@dataclass(frozen=True)
class GlossaryStageProtectedGlossaryTermsValidator:
name: str
execution_kind: str = "deterministic"
def validate(self, context: ValidationContext) -> ValidationResult:
vocabulary = ProtectedVocabulary.from_glossary(context.glossary)
return ValidationResult(
validator_name=self.name,
execution_kind=self.execution_kind,
decisions=[
ValidationDecision(
proposal_index=proposal.proposal_index,
approved=(reason := vocabulary.glossary_stage_violation_reason(
proposal.original_text,
proposal.corrected_text,
)) is None,
reason=reason,
)
for proposal in context.proposals
],
)
@dataclass(frozen=True)
class NonEmptySegmentValidator:
name: str
execution_kind: str = "deterministic"
def validate(self, context: ValidationContext) -> ValidationResult:
replacement_policy = context.run_spec.module.replacement_policy
decisions: list[ValidationDecision] = []
for proposal in context.proposals:
preview = preview_proposal(context.transcript, proposal, replacement_policy)
if isinstance(preview, ProposalPreviewError):
decisions.append(ValidationDecision(proposal_index=proposal.proposal_index, approved=True))
continue
is_non_empty = preview.corrected_segment_text.strip() != ""
decisions.append(
ValidationDecision(
proposal_index=proposal.proposal_index,
approved=is_non_empty,
reason=None if is_non_empty else "correction would leave the segment empty",
)
)
return ValidationResult(
validator_name=self.name,
execution_kind=self.execution_kind,
decisions=decisions,
)

View File

@@ -0,0 +1,195 @@
from __future__ import annotations
from concurrent.futures import ThreadPoolExecutor
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Callable, Dict, List, Sequence
from pydantic import BaseModel, ConfigDict, Field, StrictStr
from audita.core.chunking import chunk_payload_items
from audita.core.errors import AuditaLLMError
from audita.framework.proposals import ProposalPreview, ProposalPreviewError, preview_proposal
from .base import ValidationContext, ValidationDecision, ValidationResult
from .prompts import (
build_editorial_messages,
build_meaning_reversal_messages,
build_spoken_form_plausibility_messages,
)
class _LLMValidationDecisionModel(BaseModel):
model_config = ConfigDict(extra="forbid")
correction_index: int
approved: bool
confidence: float = Field(ge=0.0, le=1.0)
reason: StrictStr
class _LLMValidationSetModel(BaseModel):
model_config = ConfigDict(extra="forbid")
validations: List[_LLMValidationDecisionModel]
PromptBuilder = Callable[[List[dict]], List[dict]]
@dataclass(frozen=True)
class _BaseLLMValidator:
name: str
prompt_builder: PromptBuilder
execution_kind: str = "llm"
def validate(self, context: ValidationContext) -> ValidationResult:
if not context.proposals:
return ValidationResult(
validator_name=self.name,
execution_kind=self.execution_kind,
decisions=[],
)
previewable: List[ProposalPreview] = []
decisions: List[ValidationDecision] = []
replacement_policy = context.run_spec.module.replacement_policy
for proposal in context.proposals:
preview = preview_proposal(context.transcript, proposal, replacement_policy)
if isinstance(preview, ProposalPreviewError):
decisions.append(
ValidationDecision(
proposal_index=proposal.proposal_index,
approved=False,
reason=preview.reason,
)
)
continue
previewable.append(preview)
if previewable:
llm_client = context.llm_client
if llm_client is None:
raise AuditaLLMError(f"Validator '{self.name}' requires a structured LLM client.")
batches = chunk_payload_items(
previewable,
context.config.validation_max_prompt_tokens,
payload_fn=lambda item: item.to_prompt_payload(),
empty_error_message="Validation input must contain at least one proposal.",
)
max_workers = context.llm_scheduler.max_concurrency if context.llm_scheduler is not None else context.config.llm_concurrency
if max_workers == 1 or len(batches) <= 1:
for batch in batches:
decisions.extend(self._run_batch(context, llm_client, batch))
else:
with ThreadPoolExecutor(max_workers=max_workers) as executor:
for batch_decisions in executor.map(
lambda batch: self._run_batch(context, llm_client, batch),
batches,
):
decisions.extend(batch_decisions)
return ValidationResult(
validator_name=self.name,
execution_kind=self.execution_kind,
decisions=sorted(decisions, key=lambda decision: decision.proposal_index),
)
def _validate_batch_response(
self,
response: _LLMValidationSetModel,
proposals: Sequence[ProposalPreview],
) -> List[ValidationDecision]:
expected_indexes = {proposal.proposal.proposal_index for proposal in proposals}
indexed: Dict[int, _LLMValidationDecisionModel] = {}
for decision in response.validations:
if decision.correction_index in indexed:
raise AuditaLLMError(
f"Validator '{self.name}' returned duplicate correction_index values."
)
if decision.correction_index not in expected_indexes:
raise AuditaLLMError(
f"Validator '{self.name}' returned an unknown correction_index."
)
indexed[decision.correction_index] = decision
missing_indexes = sorted(expected_indexes - set(indexed))
if missing_indexes:
raise AuditaLLMError(
f"Validator '{self.name}' omitted correction_index values: {missing_indexes}"
)
return [
ValidationDecision(
proposal_index=proposal.proposal.proposal_index,
approved=indexed[proposal.proposal.proposal_index].approved,
confidence=indexed[proposal.proposal.proposal_index].confidence,
reason=indexed[proposal.proposal.proposal_index].reason,
)
for proposal in proposals
]
def _run_batch(
self,
context: ValidationContext,
llm_client,
batch,
) -> List[ValidationDecision]:
payload = [item.to_prompt_payload() for item in batch.items]
messages = self.prompt_builder(payload)
prompt_path = context.run_dir / f"{self.name}-prompt-{batch.batch_index:04d}.json"
response_path = context.run_dir / f"{self.name}-response-{batch.batch_index:04d}.json"
_write_json(prompt_path, {"messages": messages})
if context.llm_scheduler is not None:
response = context.llm_scheduler.run_backend_call(
lambda: llm_client.run_structured(
stage_name=f"{context.run_spec.instance_name}:{self.name}",
messages=messages,
response_model=_LLMValidationSetModel,
config=context.config,
)
)
else:
response = llm_client.run_structured(
stage_name=f"{context.run_spec.instance_name}:{self.name}",
messages=messages,
response_model=_LLMValidationSetModel,
config=context.config,
)
_write_json(response_path, response.model_dump(mode="json"))
return self._validate_batch_response(response, batch.items)
@dataclass(frozen=True)
class SpokenFormPlausibilityValidator(_BaseLLMValidator):
name: str = "spoken_form_plausibility_review"
prompt_builder: PromptBuilder = build_spoken_form_plausibility_messages
@dataclass(frozen=True)
class MeaningReversalValidator(_BaseLLMValidator):
name: str = "meaning_reversal_review"
prompt_builder: PromptBuilder = build_meaning_reversal_messages
@dataclass(frozen=True)
class EditorialValidator(_BaseLLMValidator):
name: str = "editorial_review"
prompt_builder: PromptBuilder = build_editorial_messages
@dataclass(frozen=True)
class SpokenWordValidator(_BaseLLMValidator):
name: str = "spoken_word_review"
prompt_builder: PromptBuilder = build_editorial_messages
@dataclass(frozen=True)
class GrammarOnlyValidator(_BaseLLMValidator):
name: str = "grammar_only_guard"
prompt_builder: PromptBuilder = build_editorial_messages
def _write_json(path: Path, payload: dict) -> None:
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")

View File

@@ -0,0 +1,96 @@
import json
from typing import Dict, List
Message = Dict[str, str]
def build_spoken_form_plausibility_messages(validation_payload: List[dict]) -> List[Message]:
payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2)
system = (
"You are Audita, a conservative spoken-form validation assistant. "
"Evaluate whether each proposed correction is plausibly explained by a homophone, phonetic similarity, "
"or a common mistranscription of spoken English. "
"Your job is not to improve style or readability. "
"Approve only when the corrected text is a plausible recovery of the words that were likely spoken."
)
user = (
"Review these proposed transcript corrections and decide whether each one is a plausible spoken-form correction.\n\n"
"Rules:\n"
"- Return one validation decision for every correction_index in the input.\n"
"- Approve when the original text and corrected text are plausibly related by homophone confusion, "
"phonetic similarity, or a common spoken-word mistranscription, and the surrounding segment context supports the correction.\n"
"- Examples that may be approved when context supports them: changing \"gestures\" to \"Jesters\", "
"\"rank\" to \"Hrank\", or \"dam\" to \"damn\".\n"
"- Reject unrelated substitutions like changing \"Lyra\" to \"Jesters\".\n"
"- Judge spoken-form plausibility, not whether the correction is cleaner, more formal, or more grammatical.\n"
"- Do not approve paraphrases, stylistic rewrites, or arbitrary semantic substitutions.\n"
"- If a correction includes categories, treat them as additional segment context.\n"
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n"
"- confidence must be between 0.0 and 1.0.\n\n"
f"Corrections to validate:\n{payload_json}"
)
return [{"role": "system", "content": system}, {"role": "user", "content": user}]
def build_meaning_reversal_messages(validation_payload: List[dict]) -> List[Message]:
payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2)
system = (
"You are Audita, a narrow semantic-reversal validation assistant. "
"Evaluate whether each proposed correction changes a word to its antonym or otherwise reverses the meaning "
"of the full segment. "
"Do not treat every word substitution as a problem; focus specifically on antonyms and meaning reversals."
)
user = (
"Review these proposed transcript corrections and decide whether each one avoids reversing the segment meaning.\n\n"
"Rules:\n"
"- Return one validation decision for every correction_index in the input.\n"
"- Reject corrections that introduce antonyms or otherwise reverse the meaning of the original segment.\n"
"- Reject examples like changing \"visible\" to \"invisible\" or \"up\" to \"down\" when that reverses the segment meaning.\n"
"- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.\n"
"- Do not reject a correction merely because the literal written word changes.\n"
"- Do not act as a general semantic-style reviewer; this validator is only a guard against antonyms and meaning reversals.\n"
"- If a correction includes categories, treat them as additional segment context.\n"
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n"
"- confidence must be between 0.0 and 1.0.\n\n"
f"Corrections to validate:\n{payload_json}"
)
return [{"role": "system", "content": system}, {"role": "user", "content": user}]
def build_editorial_messages(validation_payload: List[dict]) -> List[Message]:
payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2)
system = (
"You are Audita, a conservative editorial validation assistant. "
"Evaluate whether each proposed correction is editorial in nature and preserves the segment's underlying meaning. "
"Editorial revisions may include dysfluency cleanup, punctuation changes, capitalization changes, "
"homophone or mistranscription corrections, and similar low-risk editorial cleanup."
)
user = (
"Review these proposed transcript corrections and decide whether each one is an acceptable editorial revision.\n\n"
"Rules:\n"
"- Return one validation decision for every correction_index in the input.\n"
"- Approve editorial revisions that preserve the underlying meaning of the segment.\n"
"- Approve dysfluency cleanup, including cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies.\n"
"- Approve punctuation, spacing, capitalization, and article cleanup when they preserve meaning.\n"
"- Approve low-risk homophone or mistranscription corrections when they are contextually well supported.\n"
"- Approve conservative combinations of these editorial changes when the overall revision remains meaning-preserving.\n"
"- Reject repetition cleanup when the repetition plausibly serves urgency, excitement, insistence, or deliberate rhetorical emphasis rather than dysfluency.\n"
"- Phrases such as \"Help! Help! Help!\", \"Stop! Stop! Stop!\", \"No! No! No!\", \"Yes! Yes! Yes!\", and \"Go! Go! Go!\" are often intentional emphasis and should usually be preserved.\n"
"- Approve repetition cleanup only when local context supports it as accidental repetition, hesitation, or verbal restart.\n"
"- Reject broad paraphrase, substantive semantic changes, substantive meaning changes, and revisions that materially change, obscure, distort, or reverse the segment's meaning.\n"
"- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.\n"
"- If a correction includes categories, treat them as additional segment context.\n"
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n"
"- confidence must be between 0.0 and 1.0.\n\n"
f"Corrections to validate:\n{payload_json}"
)
return [{"role": "system", "content": system}, {"role": "user", "content": user}]
def build_grammar_only_messages(validation_payload: List[dict]) -> List[Message]:
return build_editorial_messages(validation_payload)
def build_spoken_word_messages(validation_payload: List[dict]) -> List[Message]:
return build_editorial_messages(validation_payload)

View File

@@ -0,0 +1,136 @@
import re
from dataclasses import dataclass
from typing import Dict, List, Optional, Pattern
from audita.core.schemas import Glossary
@dataclass(frozen=True)
class ProtectedVocabulary:
terms_by_folded: Dict[str, "_ProtectedTermDefinition"]
pattern: Optional[Pattern[str]]
@classmethod
def from_glossary(cls, glossary: Glossary) -> "ProtectedVocabulary":
terms_by_folded: Dict[str, _ProtectedTermDefinition] = {}
for identity, entry in enumerate(glossary.glossary):
entry_terms = [entry.name, *entry.aliases]
for term in entry_terms:
_add_term(terms_by_folded, term, identity)
_add_term(terms_by_folded, f"{term}s", identity)
if entry.plural is not None:
_add_term(terms_by_folded, entry.plural, identity)
terms = [definition.canonical for definition in terms_by_folded.values()]
if not terms:
return cls(terms_by_folded=terms_by_folded, pattern=None)
alternatives = sorted((re.escape(term) for term in terms), key=len, reverse=True)
pattern = re.compile(r"(?<!\w)(" + "|".join(alternatives) + r")(?!\w)", flags=re.IGNORECASE)
return cls(terms_by_folded=terms_by_folded, pattern=pattern)
def violation_reason(self, before: str, after: str) -> Optional[str]:
before_occurrences = self._occurrences_by_identity(before)
after_occurrences = self._occurrences_by_identity(after)
reason = self._validate_identity_preservation(before_occurrences, after_occurrences)
if reason is not None:
return reason
return self._validate_capitalization_transitions(before_occurrences, after_occurrences)
def glossary_stage_violation_reason(self, before: str, after: str) -> Optional[str]:
before_occurrences = self._occurrences_by_identity(before)
after_occurrences = self._occurrences_by_identity(after)
reason = self._validate_glossary_stage_identity_preservation(before_occurrences, after_occurrences)
if reason is not None:
return reason
return self._validate_capitalization_transitions(before_occurrences, after_occurrences)
def _occurrences(self, text: str) -> List["_ProtectedOccurrence"]:
if self.pattern is None:
return []
occurrences = []
for match in self.pattern.finditer(text):
matched_text = match.group(0)
definition = self.terms_by_folded[matched_text.casefold()]
occurrences.append(
_ProtectedOccurrence(
text=matched_text,
identity=definition.identity,
canonical=definition.canonical,
)
)
return occurrences
def _occurrences_by_identity(self, text: str) -> Dict[int, List["_ProtectedOccurrence"]]:
occurrences_by_identity: Dict[int, List["_ProtectedOccurrence"]] = {}
for occurrence in self._occurrences(text):
occurrences_by_identity.setdefault(occurrence.identity, []).append(occurrence)
return occurrences_by_identity
def _validate_identity_preservation(
self,
before_occurrences: Dict[int, List["_ProtectedOccurrence"]],
after_occurrences: Dict[int, List["_ProtectedOccurrence"]],
) -> Optional[str]:
for identity, before_items in before_occurrences.items():
if len(after_occurrences.get(identity, [])) < len(before_items):
return "correction changes protected glossary term usage"
return None
def _validate_glossary_stage_identity_preservation(
self,
before_occurrences: Dict[int, List["_ProtectedOccurrence"]],
after_occurrences: Dict[int, List["_ProtectedOccurrence"]],
) -> Optional[str]:
before_total = sum(len(items) for items in before_occurrences.values())
after_total = sum(len(items) for items in after_occurrences.values())
if after_total < before_total:
return "correction changes protected glossary term usage"
return None
def _validate_capitalization_transitions(
self,
before_occurrences: Dict[int, List["_ProtectedOccurrence"]],
after_occurrences: Dict[int, List["_ProtectedOccurrence"]],
) -> Optional[str]:
for identity, after_items in after_occurrences.items():
before_items = before_occurrences.get(identity, [])
before_count = len(before_items)
for index, after_item in enumerate(after_items):
if index < before_count:
before_item = before_items[index]
if after_item.text == before_item.text:
continue
if after_item.text == after_item.canonical:
continue
return "correction changes protected glossary term capitalization"
return None
@dataclass(frozen=True)
class _ProtectedOccurrence:
text: str
identity: int
canonical: str
@dataclass(frozen=True)
class _ProtectedTermDefinition:
identity: int
canonical: str
def _add_term(
terms_by_folded: Dict[str, _ProtectedTermDefinition],
term: str,
identity: int,
) -> None:
stripped = term.strip()
if not stripped:
return
terms_by_folded.setdefault(
stripped.casefold(),
_ProtectedTermDefinition(identity=identity, canonical=stripped),
)

1
python/tests/__init__.py Normal file
View File

@@ -0,0 +1 @@
"""Test package root."""

View File

@@ -0,0 +1,206 @@
import pytest
from audita.core.chunking import TokenEstimatorProtocol, chunk_transcript
from audita.core.errors import AuditaValidationError
from audita.core.schemas import parse_transcript_json
class FakeEstimator(TokenEstimatorProtocol):
def estimate_json(self, value):
if len(value) == 1:
return 4
return len(value) * 4
def test_chunk_transcript_batches_sections_by_token_limit():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"},
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "three"}
]
"""
)
sections = chunk_transcript(transcript, max_section_tokens=8, estimator=FakeEstimator())
assert len(sections) == 2
assert [segment.segment.id for segment in sections[0].segments] == [1, 2]
assert [segment.segment.id for segment in sections[1].segments] == [3]
def test_chunk_transcript_targets_llm_concurrency_when_feasible():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"},
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "three"},
{"id": 4, "speaker": "A", "start": 3.0, "end": 4.0, "text": "four"}
]
"""
)
sections = chunk_transcript(
transcript,
max_section_tokens=8,
min_section_tokens=4,
target_section_count=2,
estimator=FakeEstimator(),
)
assert len(sections) == 2
assert [segment.segment.id for segment in sections[0].segments] == [1, 2]
assert [segment.segment.id for segment in sections[1].segments] == [3, 4]
def test_chunk_transcript_increases_section_count_when_target_sections_exceed_max_tokens():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"},
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "three"},
{"id": 4, "speaker": "A", "start": 3.0, "end": 4.0, "text": "four"},
{"id": 5, "speaker": "A", "start": 4.0, "end": 5.0, "text": "five"}
]
"""
)
sections = chunk_transcript(
transcript,
max_section_tokens=8,
min_section_tokens=4,
target_section_count=1,
estimator=FakeEstimator(),
)
assert len(sections) > 1
assert all(section.token_count <= 8 for section in sections)
def test_chunk_transcript_reduces_section_count_when_target_sections_fall_below_min_tokens():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"},
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "three"}
]
"""
)
sections = chunk_transcript(
transcript,
max_section_tokens=12,
min_section_tokens=8,
target_section_count=3,
estimator=FakeEstimator(),
)
assert len(sections) == 1
assert sections[0].token_count >= 8
def test_chunk_transcript_exact_target_sections_returns_exact_count_when_feasible():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"},
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "three"},
{"id": 4, "speaker": "A", "start": 3.0, "end": 4.0, "text": "four"}
]
"""
)
sections = chunk_transcript(
transcript,
max_section_tokens=8,
min_section_tokens=4,
exact_target_section_count=2,
estimator=FakeEstimator(),
)
assert len(sections) == 2
assert [segment.segment.id for segment in sections[0].segments] == [1, 2]
assert [segment.segment.id for segment in sections[1].segments] == [3, 4]
def test_chunk_transcript_exact_target_sections_errors_when_target_exceeds_segment_count():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"}
]
"""
)
with pytest.raises(AuditaValidationError, match="Target section count exceeds the number of transcript segments"):
chunk_transcript(
transcript,
max_section_tokens=8,
min_section_tokens=4,
exact_target_section_count=3,
estimator=FakeEstimator(),
)
def test_chunk_transcript_exact_target_sections_errors_when_section_would_exceed_max():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"},
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "three"}
]
"""
)
with pytest.raises(AuditaValidationError, match="AUDITA_TARGET_SECTIONS cannot produce contiguous transcript sections"):
chunk_transcript(
transcript,
max_section_tokens=8,
min_section_tokens=4,
exact_target_section_count=1,
estimator=FakeEstimator(),
)
def test_chunk_transcript_exact_target_sections_errors_when_section_would_fall_below_min():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"},
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "three"}
]
"""
)
with pytest.raises(AuditaValidationError, match="AUDITA_TARGET_SECTIONS cannot produce contiguous transcript sections"):
chunk_transcript(
transcript,
max_section_tokens=12,
min_section_tokens=8,
exact_target_section_count=3,
estimator=FakeEstimator(),
)
def test_chunk_transcript_prompt_payload_includes_categories_when_present():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one", "categories": ["intro", "aside"]}
]
"""
)
sections = chunk_transcript(transcript, max_section_tokens=8, estimator=FakeEstimator())
assert sections[0].prompt_payload() == [
{"id": 1, "original_text": "one", "categories": ["intro", "aside"]}
]

View File

@@ -0,0 +1,255 @@
from concurrent.futures import ThreadPoolExecutor
import sys
import types
import pytest
from audita.core.config import AuditaConfig
from audita.core.errors import AuditaLLMError
from audita.framework.llm import OpenAICompatibleStructuredLLMClient
class DummyResponseModel:
pass
def _config(**overrides):
base = AuditaConfig.from_sources(env={})
data = {
"api_key": "test-key",
"llm_concurrency": base.llm_concurrency,
"llm_timeout_seconds": base.llm_timeout_seconds,
"validation_llm_api_key": base.validation_llm_api_key,
"validation_llm_concurrency": base.validation_llm_concurrency,
"validation_llm_timeout_seconds": base.validation_llm_timeout_seconds,
"validation_model": base.validation_model,
"validation_base_url": base.validation_base_url,
"validation_max_retries": base.validation_max_retries,
"module_keys": base.module_keys,
"model": base.model,
"base_url": base.base_url,
"max_retries": base.max_retries,
"max_section_tokens": base.max_section_tokens,
"glossary_confidence_threshold": base.glossary_confidence_threshold,
"grammar_confidence_threshold": base.grammar_confidence_threshold,
"homophones_confidence_threshold": base.homophones_confidence_threshold,
"spoken_word_confidence_threshold": base.spoken_word_confidence_threshold,
"normalize_max_segment_gap": base.normalize_max_segment_gap,
"normalize_ellipsis_gap": base.normalize_ellipsis_gap,
"normalize_max_segment_duration": base.normalize_max_segment_duration,
"normalize_max_segment_tokens": base.normalize_max_segment_tokens,
"work_dir": base.work_dir,
"work_dir_retention": base.work_dir_retention,
}
data.update(overrides)
return AuditaConfig(**data)
def _install_fake_llm_modules(monkeypatch):
create_calls = []
openai_inits = []
class FakePatchedClient:
def __init__(self):
self.chat = types.SimpleNamespace(completions=types.SimpleNamespace(create=self._create))
def _create(self, **kwargs):
create_calls.append(kwargs)
return {"ok": True}
class FakeOpenAI:
def __init__(self, *, api_key, base_url, timeout):
openai_inits.append({"api_key": api_key, "base_url": base_url, "timeout": timeout})
fake_instructor = types.SimpleNamespace(
Mode=types.SimpleNamespace(TOOLS="TOOLS"),
patch=lambda client, mode: FakePatchedClient(),
)
fake_openai = types.SimpleNamespace(OpenAI=FakeOpenAI)
monkeypatch.setitem(sys.modules, "instructor", fake_instructor)
monkeypatch.setitem(sys.modules, "openai", fake_openai)
return create_calls, openai_inits
def test_openrouter_requests_strip_prefix_and_include_extra_body(monkeypatch):
create_calls, _ = _install_fake_llm_modules(monkeypatch)
client = OpenAICompatibleStructuredLLMClient()
client.run_structured(
stage_name="test-stage",
messages=[{"role": "user", "content": "Hello"}],
response_model=DummyResponseModel,
config=_config(model="openrouter/google/gemma-4-31b-it"),
)
assert create_calls == [
{
"model": "google/gemma-4-31b-it",
"messages": [{"role": "user", "content": "Hello"}],
"response_model": DummyResponseModel,
"max_retries": 3,
"extra_body": {"provider": {"require_parameters": True}},
}
]
def test_openrouter_default_base_url_uses_openrouter_request_shape(monkeypatch):
create_calls, _ = _install_fake_llm_modules(monkeypatch)
client = OpenAICompatibleStructuredLLMClient()
client.run_structured(
stage_name="test-stage",
messages=[{"role": "user", "content": "Hello"}],
response_model=DummyResponseModel,
config=_config(model="google/gemma-4-31b-it"),
)
assert create_calls == [
{
"model": "google/gemma-4-31b-it",
"messages": [{"role": "user", "content": "Hello"}],
"response_model": DummyResponseModel,
"max_retries": 3,
"extra_body": {"provider": {"require_parameters": True}},
}
]
def test_generic_endpoint_requests_keep_model_and_omit_extra_body(monkeypatch):
create_calls, _ = _install_fake_llm_modules(monkeypatch)
client = OpenAICompatibleStructuredLLMClient()
client.run_structured(
stage_name="test-stage",
messages=[{"role": "user", "content": "Hello"}],
response_model=DummyResponseModel,
config=_config(
model="meta-llama/Llama-3.1-8B-Instruct",
base_url="http://localhost:8000/v1",
),
)
assert create_calls == [
{
"model": "meta-llama/Llama-3.1-8B-Instruct",
"messages": [{"role": "user", "content": "Hello"}],
"response_model": DummyResponseModel,
"max_retries": 3,
}
]
def test_client_cache_identity_uses_api_key_and_base_url(monkeypatch):
_, openai_inits = _install_fake_llm_modules(monkeypatch)
client = OpenAICompatibleStructuredLLMClient()
first = _config(api_key="key-1", base_url="http://localhost:8000/v1")
second = _config(api_key="key-1", base_url="http://localhost:8000/v1")
third = _config(api_key="key-1", base_url="https://api.openai.com/v1")
client.run_structured(
stage_name="one",
messages=[{"role": "user", "content": "Hello"}],
response_model=DummyResponseModel,
config=first,
)
client.run_structured(
stage_name="two",
messages=[{"role": "user", "content": "Hello"}],
response_model=DummyResponseModel,
config=second,
)
client.run_structured(
stage_name="three",
messages=[{"role": "user", "content": "Hello"}],
response_model=DummyResponseModel,
config=third,
)
assert openai_inits == [
{"api_key": "key-1", "base_url": "http://localhost:8000/v1", "timeout": 600},
{"api_key": "key-1", "base_url": "https://api.openai.com/v1", "timeout": 600},
]
def test_client_cache_identity_uses_timeout(monkeypatch):
_, openai_inits = _install_fake_llm_modules(monkeypatch)
client = OpenAICompatibleStructuredLLMClient()
first = _config(api_key="key-1", base_url="http://localhost:8000/v1", llm_timeout_seconds=600)
second = _config(api_key="key-1", base_url="http://localhost:8000/v1", llm_timeout_seconds=1200)
client.run_structured(
stage_name="one",
messages=[{"role": "user", "content": "Hello"}],
response_model=DummyResponseModel,
config=first,
)
client.run_structured(
stage_name="two",
messages=[{"role": "user", "content": "Hello"}],
response_model=DummyResponseModel,
config=second,
)
assert openai_inits == [
{"api_key": "key-1", "base_url": "http://localhost:8000/v1", "timeout": 600},
{"api_key": "key-1", "base_url": "http://localhost:8000/v1", "timeout": 1200},
]
def test_missing_api_key_error_is_provider_neutral():
client = OpenAICompatibleStructuredLLMClient()
with pytest.raises(AuditaLLMError, match="OpenRouter endpoint"):
client.run_structured(
stage_name="test-stage",
messages=[{"role": "user", "content": "Hello"}],
response_model=DummyResponseModel,
config=_config(api_key=None),
)
def test_missing_api_key_is_allowed_for_nondefault_endpoint(monkeypatch):
create_calls, openai_inits = _install_fake_llm_modules(monkeypatch)
client = OpenAICompatibleStructuredLLMClient()
client.run_structured(
stage_name="test-stage",
messages=[{"role": "user", "content": "Hello"}],
response_model=DummyResponseModel,
config=_config(
api_key=None,
model="meta-llama/Llama-3.1-8B-Instruct",
base_url="http://localhost:8000/v1",
),
)
assert openai_inits == [{"api_key": "audita-no-key-required", "base_url": "http://localhost:8000/v1", "timeout": 600}]
assert create_calls == [
{
"model": "meta-llama/Llama-3.1-8B-Instruct",
"messages": [{"role": "user", "content": "Hello"}],
"response_model": DummyResponseModel,
"max_retries": 3,
}
]
def test_client_initialization_is_safe_under_concurrent_calls(monkeypatch):
_, openai_inits = _install_fake_llm_modules(monkeypatch)
client = OpenAICompatibleStructuredLLMClient()
config = _config(api_key="key-1", base_url="http://localhost:8000/v1")
with ThreadPoolExecutor(max_workers=4) as executor:
list(
executor.map(
lambda _: client.run_structured(
stage_name="test-stage",
messages=[{"role": "user", "content": "Hello"}],
response_model=DummyResponseModel,
config=config,
),
range(4),
)
)
assert openai_inits == [{"api_key": "key-1", "base_url": "http://localhost:8000/v1", "timeout": 600}]

View File

@@ -0,0 +1,642 @@
import threading
from audita.core.chunking import IndexedSegment, TranscriptSection
from audita.core.config import AuditaConfig, ConfigOverrides
from audita.core.errors import AuditaLLMError
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
from audita.framework.models import CorrectionProposal, ModuleContext, ModuleRunSpec
from audita.framework.runner import PipelineRunner
from audita.validators import (
MeaningReversalValidator,
ProposalConfidenceValidator,
ProtectedGlossaryTermsValidator,
SpokenFormPlausibilityValidator,
)
from audita.validators.base import ValidationContext, ValidationDecision, ValidationResult
class RecordingValidator:
execution_kind = "deterministic"
def __init__(self, name, recorder, approve=True):
self.name = name
self._recorder = recorder
self._approve = approve
def validate(self, context: ValidationContext) -> ValidationResult:
self._recorder.append((self.name, [proposal.corrected_text for proposal in context.proposals]))
return ValidationResult(
validator_name=self.name,
execution_kind=self.execution_kind,
decisions=[
ValidationDecision(
proposal_index=proposal.proposal_index,
approved=self._approve,
reason=None if self._approve else f"{self.name} rejected proposal",
)
for proposal in context.proposals
],
)
class RecordingLLMValidator(RecordingValidator):
execution_kind = "llm"
class FakeStructuredLLMClient:
def __init__(self, responses):
self._responses = responses
self._lock = threading.Lock()
self.calls = []
def run_structured(self, *, stage_name, messages, response_model, config):
with self._lock:
self.calls.append(
{
"stage_name": stage_name,
"messages": list(messages),
"response_model": response_model,
}
)
payload = _pop_llm_response(self._responses, stage_name)
return response_model.model_validate(payload)
def _pop_llm_response(responses, stage_name):
if isinstance(responses, dict):
if stage_name not in responses:
raise AuditaLLMError(f"FakeStructuredLLMClient received unexpected stage_name: {stage_name}")
payloads = responses[stage_name]
if isinstance(payloads, list):
if not payloads:
raise AuditaLLMError(f"FakeStructuredLLMClient received too many calls for stage_name: {stage_name}")
return payloads.pop(0)
payload = payloads
del responses[stage_name]
return payload
if not responses:
raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.")
return responses.pop(0)
class TrackingStructuredLLMClient:
def __init__(self, responses, barrier=None):
self._responses = responses
self._barrier = barrier
self._lock = threading.Lock()
self.calls = []
self.in_flight = 0
self.max_in_flight = 0
def run_structured(self, *, stage_name, messages, response_model, config):
if self._barrier is not None:
self._barrier.wait()
with self._lock:
self.in_flight += 1
self.max_in_flight = max(self.max_in_flight, self.in_flight)
self.calls.append(
{
"stage_name": stage_name,
"messages": list(messages),
"response_model": response_model,
}
)
payload = _pop_llm_response(self._responses, stage_name)
try:
return response_model.model_validate(payload)
finally:
with self._lock:
self.in_flight -= 1
class RecordingModule:
replacement_policy = "require_unique"
def __init__(self, module_key, proposals, validators, recorder):
self.module_key = module_key
self._proposals = proposals
self._validators = validators
self._recorder = recorder
def validators(self):
return list(self._validators)
def propose(self, transcript_section, context: ModuleContext):
self._recorder.append(("propose", [item.segment.text for item in transcript_section.segments]))
return list(self._proposals)
class ConcurrentRecordingModule:
replacement_policy = "require_unique"
def __init__(self, recorder, barrier):
self.module_key = "concurrent"
self._recorder = recorder
self._barrier = barrier
def validators(self):
return []
def propose(self, transcript_section, context: ModuleContext):
texts = [item.segment.text for item in transcript_section.segments]
self._recorder.append(("start", transcript_section.section_index, texts))
self._barrier.wait()
segment = transcript_section.segments[0].segment
return [
CorrectionProposal(
proposal_index=0,
module_instance=context.run_spec.instance_name,
module_key=context.run_spec.module_key,
id=segment.id,
original_text=segment.text.rstrip("."),
corrected_text=f"{segment.text.rstrip('.')} revised",
confidence=0.9,
)
]
def test_pipeline_runner_applies_modules_sequentially(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Alpha."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Alpha"
category: noun
summary: "Alpha."
"""
)
seen = []
first = RecordingModule(
"first",
[
CorrectionProposal(
proposal_index=0,
module_instance="first",
module_key="first",
id=1,
original_text="Alpha",
corrected_text="Beta",
confidence=0.9,
)
],
[RecordingValidator("first_validator", seen)],
seen,
)
second = RecordingModule(
"second",
[
CorrectionProposal(
proposal_index=0,
module_instance="second",
module_key="second",
id=1,
original_text="Beta",
corrected_text="Gamma",
confidence=0.9,
)
],
[RecordingValidator("second_validator", seen)],
seen,
)
runner = PipelineRunner()
result = runner.run(
transcript=transcript,
glossary=glossary,
module_specs=[
ModuleRunSpec(instance_name="first", module_key="first", module=first),
ModuleRunSpec(instance_name="second", module_key="second", module=second),
],
config=AuditaConfig.from_sources(env={}),
run_dir=tmp_path / "run",
)
assert seen[0] == ("propose", ["Alpha."])
assert seen[1] == ("first_validator", ["Beta"])
assert seen[2] == ("propose", ["Beta."])
assert seen[3] == ("second_validator", ["Gamma"])
assert result.transcript[0].text == "Gamma."
assert len(result.applied_changes) == 2
def test_pipeline_runner_validator_order_respects_survivors(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Hello."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Hello"
category: noun
summary: "Hello."
"""
)
seen = []
module = RecordingModule(
"mod",
[
CorrectionProposal(
proposal_index=0,
module_instance="mod",
module_key="mod",
id=1,
original_text="Hello",
corrected_text="Goodbye",
confidence=0.9,
)
],
[
RecordingValidator("first", seen, approve=False),
RecordingLLMValidator("second", seen, approve=True),
],
seen,
)
runner = PipelineRunner()
result = runner.run(
transcript=transcript,
glossary=glossary,
module_specs=[ModuleRunSpec(instance_name="mod", module_key="mod", module=module)],
config=AuditaConfig.from_sources(env={}),
run_dir=tmp_path / "run",
)
assert ("first", ["Goodbye"]) in seen
assert all(entry[0] != "second" for entry in seen)
assert result.module_reports[0].validators[0].rejected_count == 1
assert result.module_reports[0].validators[1].candidate_count == 0
assert result.skipped_corrections[0].source == "validator:first"
def test_pipeline_runner_supports_deterministic_and_llm_validators_in_one_chain(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Alpha."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Alpha"
category: noun
summary: "Alpha."
"""
)
seen = []
module = RecordingModule(
"mixed",
[
CorrectionProposal(
proposal_index=0,
module_instance="mixed",
module_key="mixed",
id=1,
original_text="Alpha",
corrected_text="Beta",
confidence=0.9,
)
],
[
RecordingValidator("deterministic_guard", seen),
RecordingLLMValidator("llm_review", seen),
],
seen,
)
runner = PipelineRunner()
result = runner.run(
transcript=transcript,
glossary=glossary,
module_specs=[ModuleRunSpec(instance_name="mixed", module_key="mixed", module=module)],
config=AuditaConfig.from_sources(env={}),
run_dir=tmp_path / "run",
)
assert result.transcript[0].text == "Beta."
assert [report.execution_kind for report in result.module_reports[0].validators] == [
"deterministic",
"llm",
]
def test_pipeline_runner_supports_real_llm_validators_in_one_chain(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the dam."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Jesters"
category: faction
summary: "The Jesters are a faction."
"""
)
module = RecordingModule(
"mixed_real",
[
CorrectionProposal(
proposal_index=0,
module_instance="mixed_real",
module_key="glossary",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.9,
)
],
[
ProposalConfidenceValidator("proposal_confidence_guard", "glossary_confidence_threshold"),
ProtectedGlossaryTermsValidator("protected_glossary_guard"),
SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
MeaningReversalValidator("meaning_reversal_review"),
],
[],
)
llm_client = FakeStructuredLLMClient(
{
"mixed_real:spoken_form_plausibility_review": {
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.95,
"reason": "Likely phonetic mistranscription in context.",
}
]
},
"mixed_real:meaning_reversal_review": {
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.98,
"reason": "Does not reverse the segment meaning.",
}
]
},
}
)
runner = PipelineRunner()
result = runner.run(
transcript=transcript,
glossary=glossary,
module_specs=[ModuleRunSpec(instance_name="mixed_real", module_key="glossary", module=module)],
config=AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}),
run_dir=tmp_path / "run",
llm_client=llm_client,
)
assert result.transcript[0].text == "There were Jesters at the dam."
assert [report.execution_kind for report in result.module_reports[0].validators] == [
"deterministic",
"deterministic",
"llm",
"llm",
]
assert {call["stage_name"] for call in llm_client.calls} == {
"mixed_real:spoken_form_plausibility_review",
"mixed_real:meaning_reversal_review",
}
def test_pipeline_runner_uses_real_protected_glossary_validator(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Hrank moves."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Hrank"
category: pc
summary: "Hrank is a player character."
"""
)
module = RecordingModule(
"protected",
[
CorrectionProposal(
proposal_index=0,
module_instance="protected",
module_key="protected",
id=1,
original_text="Hrank",
corrected_text="Frank",
confidence=0.9,
)
],
[ProtectedGlossaryTermsValidator("protected_glossary_guard")],
[],
)
runner = PipelineRunner()
result = runner.run(
transcript=transcript,
glossary=glossary,
module_specs=[ModuleRunSpec(instance_name="protected", module_key="protected", module=module)],
config=AuditaConfig.from_sources(env={}),
run_dir=tmp_path / "run",
)
assert result.transcript[0].text == "Hrank moves."
assert result.skipped_corrections[0].source == "validator:protected_glossary_guard"
assert result.skipped_corrections[0].reason == "correction changes protected glossary term usage"
def test_pipeline_runner_collects_section_proposals_concurrently_and_preserves_section_order(tmp_path, monkeypatch):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Alpha."},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "Beta."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Alpha"
category: noun
summary: "Alpha."
"""
)
sections = [
TranscriptSection(
section_index=0,
start_index=0,
segments=[IndexedSegment(index=0, segment=transcript[0])],
token_count=1,
),
TranscriptSection(
section_index=1,
start_index=1,
segments=[IndexedSegment(index=1, segment=transcript[1])],
token_count=1,
),
]
seen = []
module = ConcurrentRecordingModule(seen, threading.Barrier(2, timeout=1.0))
monkeypatch.setattr(
"audita.framework.runner.chunk_transcript",
lambda working, max_tokens, min_section_tokens=1, target_section_count=None, exact_target_section_count=None: sections,
)
runner = PipelineRunner()
result = runner.run(
transcript=transcript,
glossary=glossary,
module_specs=[ModuleRunSpec(instance_name="concurrent", module_key="concurrent", module=module)],
config=AuditaConfig.from_sources(env={}, overrides=ConfigOverrides(llm_concurrency=2)),
run_dir=tmp_path / "run",
)
assert [change.id for change in result.applied_changes] == [1, 2]
assert result.applied_changes[0].corrected_text == "Alpha revised"
assert result.applied_changes[1].corrected_text == "Beta revised"
assert result.transcript[0].text == "Alpha revised."
assert result.transcript[1].text == "Beta revised."
def test_pipeline_runner_passes_exact_target_sections_to_chunker(tmp_path, monkeypatch):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Alpha."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Alpha"
category: noun
summary: "Alpha."
"""
)
seen_args = {}
sections = [
TranscriptSection(
section_index=0,
start_index=0,
segments=[IndexedSegment(index=0, segment=transcript[0])],
token_count=1,
)
]
module = RecordingModule("noop", [], [], [])
def _fake_chunk_transcript(working, max_tokens, min_section_tokens=1, target_section_count=None, exact_target_section_count=None):
seen_args["target_section_count"] = target_section_count
seen_args["exact_target_section_count"] = exact_target_section_count
return sections
monkeypatch.setattr("audita.framework.runner.chunk_transcript", _fake_chunk_transcript)
runner = PipelineRunner()
runner.run(
transcript=transcript,
glossary=glossary,
module_specs=[ModuleRunSpec(instance_name="noop", module_key="noop", module=module)],
config=AuditaConfig.from_sources(env={}, overrides=ConfigOverrides(target_sections=3)),
run_dir=tmp_path / "run",
)
assert seen_args == {
"target_section_count": None,
"exact_target_section_count": 3,
}
def test_pipeline_runner_reports_first_llm_validator_rejection_in_chain_order(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the dam."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Jesters"
category: faction
summary: "The Jesters are a faction."
"""
)
module = RecordingModule(
"ordered_real",
[
CorrectionProposal(
proposal_index=0,
module_instance="ordered_real",
module_key="glossary",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.9,
)
],
[
ProposalConfidenceValidator("proposal_confidence_guard", "glossary_confidence_threshold"),
ProtectedGlossaryTermsValidator("protected_glossary_guard"),
SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
MeaningReversalValidator("meaning_reversal_review"),
],
[],
)
llm_client = FakeStructuredLLMClient(
{
"ordered_real:spoken_form_plausibility_review": {
"validations": [
{
"correction_index": 0,
"approved": False,
"confidence": 0.95,
"reason": "Not plausibly supported by spoken-form context.",
}
]
},
"ordered_real:meaning_reversal_review": {
"validations": [
{
"correction_index": 0,
"approved": False,
"confidence": 0.98,
"reason": "Changes meaning too much.",
}
]
},
}
)
runner = PipelineRunner()
result = runner.run(
transcript=transcript,
glossary=glossary,
module_specs=[ModuleRunSpec(instance_name="ordered_real", module_key="glossary", module=module)],
config=AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}),
run_dir=tmp_path / "run",
llm_client=llm_client,
)
assert result.skipped_corrections[0].source == "validator:spoken_form_plausibility_review"
assert result.skipped_corrections[0].reason == "Not plausibly supported by spoken-form context."

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,659 @@
import json
import io
import pytest
from audita.cli import main
from audita.core.errors import AuditaConfigError
from audita.core.reporting import ProcessResult, RunReport
from audita.core.schemas import parse_transcript_json
def test_cli_help_uses_audita_program_name(capsys):
with pytest.raises(SystemExit) as exc:
main(["--help"])
assert exc.value.code == 0
assert capsys.readouterr().out.startswith("usage: audita ")
def test_process_help_exposes_framework_flags(capsys):
with pytest.raises(SystemExit) as exc:
main(["process", "--help"])
assert exc.value.code == 0
output = capsys.readouterr().out
assert "--report-json" in output
assert "--llm-api-key" in output
assert "--llm-concurrency" in output
assert "--llm-timeout-seconds" in output
assert "--validation-llm-api-key" in output
assert "--validation-llm-concurrency" in output
assert "--validation-llm-timeout-seconds" in output
assert "--validation-model" in output
assert "--validation-base-url" in output
assert "--validation-max-retries" in output
assert "--validation-max-prompt-tokens" in output
assert "--target-sections" in output
assert "--modules" in output
assert "--model" in output
assert "--base-url" in output
assert "--max-retries" in output
assert "--max-section-tokens" in output
assert "--min-section-tokens" in output
assert "--glossary-confidence-threshold" in output
assert "--grammar-confidence-threshold" in output
assert "--homophones-confidence-threshold" in output
assert "--spoken-word-confidence-threshold" in output
assert "--work-dir-retention" in output
assert "--normalize-max-segment-gap" in output
assert "--grammar-validation-enabled" not in output
def test_cli_process_writes_report_json(monkeypatch, tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
]
"""
)
report = RunReport(
status="success",
config={"model": "m", "base_url": "b"},
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
pipeline=["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
modules=[],
applied_changes=[],
skipped_corrections=[],
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
work_dir_retention="auto",
work_dir_retained=False,
work_dir=None,
error=None,
)
result = ProcessResult(
transcript=transcript,
report=report,
run_dir=tmp_path / "run",
work_dir_retained=False,
)
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", lambda overrides=None: object())
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
output_path = tmp_path / "out.json"
report_path = tmp_path / "report.json"
exit_code = main(
[
"process",
"transcript.json",
"--glossary",
"glossary.yaml",
"--output",
str(output_path),
"--report-json",
str(report_path),
]
)
assert exit_code == 0
assert report_path.exists()
def test_cli_process_passes_modules_override_to_config(monkeypatch, tmp_path):
captured = {}
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
]
"""
)
report = RunReport(
status="success",
config={"model": "m", "base_url": "b"},
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
pipeline=["grammar"],
modules=[],
applied_changes=[],
skipped_corrections=[],
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
work_dir_retention="auto",
work_dir_retained=False,
work_dir=None,
error=None,
)
result = ProcessResult(
transcript=transcript,
report=report,
run_dir=tmp_path / "run",
work_dir_retained=False,
)
def _fake_from_sources(*, overrides=None):
captured["module_keys"] = overrides.module_keys
return object()
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
exit_code = main(
[
"process",
"transcript.json",
"--glossary",
"glossary.yaml",
"--modules",
"grammar",
]
)
assert exit_code == 0
assert captured["module_keys"] == "grammar"
def test_cli_process_passes_llm_api_key_override_to_config(monkeypatch, tmp_path):
captured = {}
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
]
"""
)
report = RunReport(
status="success",
config={"model": "m", "base_url": "b"},
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
pipeline=["grammar"],
modules=[],
applied_changes=[],
skipped_corrections=[],
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
work_dir_retention="auto",
work_dir_retained=False,
work_dir=None,
error=None,
)
result = ProcessResult(
transcript=transcript,
report=report,
run_dir=tmp_path / "run",
work_dir_retained=False,
)
def _fake_from_sources(*, overrides=None):
captured["llm_api_key"] = overrides.llm_api_key
return object()
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
exit_code = main(
[
"process",
"transcript.json",
"--glossary",
"glossary.yaml",
"--llm-api-key",
"cli-key",
]
)
assert exit_code == 0
assert captured["llm_api_key"] == "cli-key"
def test_cli_process_passes_llm_concurrency_override_to_config(monkeypatch, tmp_path):
captured = {}
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
]
"""
)
report = RunReport(
status="success",
config={"model": "m", "base_url": "b"},
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
pipeline=["grammar"],
modules=[],
applied_changes=[],
skipped_corrections=[],
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
work_dir_retention="auto",
work_dir_retained=False,
work_dir=None,
error=None,
)
result = ProcessResult(
transcript=transcript,
report=report,
run_dir=tmp_path / "run",
work_dir_retained=False,
)
def _fake_from_sources(*, overrides=None):
captured["llm_concurrency"] = overrides.llm_concurrency
return object()
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
exit_code = main(
[
"process",
"transcript.json",
"--glossary",
"glossary.yaml",
"--llm-concurrency",
"3",
]
)
assert exit_code == 0
assert captured["llm_concurrency"] == 3
def test_cli_process_passes_llm_timeout_seconds_override_to_config(monkeypatch, tmp_path):
captured = {}
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
]
"""
)
report = RunReport(
status="success",
config={"model": "m", "base_url": "b"},
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
pipeline=["grammar"],
modules=[],
applied_changes=[],
skipped_corrections=[],
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
work_dir_retention="auto",
work_dir_retained=False,
work_dir=None,
error=None,
)
result = ProcessResult(
transcript=transcript,
report=report,
run_dir=tmp_path / "run",
work_dir_retained=False,
)
def _fake_from_sources(*, overrides=None):
captured["llm_timeout_seconds"] = overrides.llm_timeout_seconds
return object()
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
exit_code = main(
[
"process",
"transcript.json",
"--glossary",
"glossary.yaml",
"--llm-timeout-seconds",
"900",
]
)
assert exit_code == 0
assert captured["llm_timeout_seconds"] == 900.0
def test_cli_process_passes_validation_llm_overrides_to_config(monkeypatch, tmp_path):
captured = {}
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
]
"""
)
report = RunReport(
status="success",
config={"model": "m", "base_url": "b"},
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
pipeline=["grammar"],
modules=[],
applied_changes=[],
skipped_corrections=[],
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
work_dir_retention="auto",
work_dir_retained=False,
work_dir=None,
error=None,
)
result = ProcessResult(
transcript=transcript,
report=report,
run_dir=tmp_path / "run",
work_dir_retained=False,
)
def _fake_from_sources(*, overrides=None):
captured["validation_llm_api_key"] = overrides.validation_llm_api_key
captured["validation_llm_concurrency"] = overrides.validation_llm_concurrency
captured["validation_llm_timeout_seconds"] = overrides.validation_llm_timeout_seconds
captured["validation_model"] = overrides.validation_model
captured["validation_base_url"] = overrides.validation_base_url
captured["validation_max_retries"] = overrides.validation_max_retries
captured["validation_max_prompt_tokens"] = overrides.validation_max_prompt_tokens
return object()
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
exit_code = main(
[
"process",
"transcript.json",
"--glossary",
"glossary.yaml",
"--validation-llm-api-key",
"validator-key",
"--validation-llm-concurrency",
"4",
"--validation-llm-timeout-seconds",
"180",
"--validation-model",
"validator-model",
"--validation-base-url",
"http://localhost:9000/v1",
"--validation-max-retries",
"2",
"--validation-max-prompt-tokens",
"1024",
]
)
assert exit_code == 0
assert captured == {
"validation_llm_api_key": "validator-key",
"validation_llm_concurrency": 4,
"validation_llm_timeout_seconds": 180.0,
"validation_model": "validator-model",
"validation_base_url": "http://localhost:9000/v1",
"validation_max_retries": 2,
"validation_max_prompt_tokens": 1024,
}
def test_cli_process_passes_target_sections_override_to_config(monkeypatch, tmp_path):
captured = {}
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
]
"""
)
report = RunReport(
status="success",
config={"model": "m", "base_url": "b"},
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
pipeline=["grammar"],
modules=[],
applied_changes=[],
skipped_corrections=[],
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
work_dir_retention="auto",
work_dir_retained=False,
work_dir=None,
error=None,
)
result = ProcessResult(
transcript=transcript,
report=report,
run_dir=tmp_path / "run",
work_dir_retained=False,
)
def _fake_from_sources(*, overrides=None):
captured["target_sections"] = overrides.target_sections
return object()
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
exit_code = main(
[
"process",
"transcript.json",
"--glossary",
"glossary.yaml",
"--target-sections",
"4",
]
)
assert exit_code == 0
assert captured["target_sections"] == 4
def test_cli_process_passes_min_section_tokens_override_to_config(monkeypatch, tmp_path):
captured = {}
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
]
"""
)
report = RunReport(
status="success",
config={"model": "m", "base_url": "b"},
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
pipeline=["grammar"],
modules=[],
applied_changes=[],
skipped_corrections=[],
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
work_dir_retention="auto",
work_dir_retained=False,
work_dir=None,
error=None,
)
result = ProcessResult(
transcript=transcript,
report=report,
run_dir=tmp_path / "run",
work_dir_retained=False,
)
def _fake_from_sources(*, overrides=None):
captured["min_section_tokens"] = overrides.min_section_tokens
return object()
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
exit_code = main(
[
"process",
"transcript.json",
"--glossary",
"glossary.yaml",
"--min-section-tokens",
"5000",
]
)
assert exit_code == 0
assert captured["min_section_tokens"] == 5000
def test_cli_process_writes_failure_diagnostics_and_keeps_stdout_empty(monkeypatch, tmp_path, capsys):
monkeypatch.setattr(
"audita.cli.AuditaConfig.from_sources",
lambda overrides=None: (_ for _ in ()).throw(AuditaConfigError("bad config")),
)
report_path = tmp_path / "external-report.json"
exit_code = main(
[
"process",
"transcript.json",
"--glossary",
"glossary.yaml",
"--work-dir",
str(tmp_path / "work"),
"--report-json",
str(report_path),
]
)
captured = capsys.readouterr()
assert exit_code == 1
assert captured.out == ""
assert "audita: error: bad config" in captured.err
assert "audita: exit code: 1" in captured.err
run_dir = next((tmp_path / "work").iterdir())
assert f"audita: run directory: {run_dir}" in captured.err
assert f"audita: error log: {run_dir / 'error.log'}" in captured.err
assert f"audita: report: {run_dir / 'report.json'}" in captured.err
assert (run_dir / "error.log").exists()
report = json.loads((run_dir / "report.json").read_text(encoding="utf-8"))
assert report["status"] == "failed"
assert report["error"] == "bad config"
assert report["error_details"]["phase"] == "config"
assert report["error_details"]["type"] == "AuditaConfigError"
external = json.loads(report_path.read_text(encoding="utf-8"))
assert external["error"] == "bad config"
def test_cli_process_writes_failure_diagnostics_for_unexpected_exceptions(monkeypatch, tmp_path, capsys):
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", lambda overrides=None: object())
monkeypatch.setattr("audita.cli.load_transcript", lambda path: (_ for _ in ()).throw(RuntimeError("boom")))
exit_code = main(
[
"process",
"transcript.json",
"--glossary",
"glossary.yaml",
"--work-dir",
str(tmp_path / "work"),
]
)
captured = capsys.readouterr()
assert exit_code == 1
assert captured.out == ""
assert "audita: error: boom" in captured.err
run_dir = next((tmp_path / "work").iterdir())
report = json.loads((run_dir / "report.json").read_text(encoding="utf-8"))
assert report["status"] == "failed"
assert report["error_details"]["phase"] == "transcript_load"
assert report["error_details"]["type"] == "RuntimeError"
assert "RuntimeError: boom" in (run_dir / "error.log").read_text(encoding="utf-8")
class _BrokenWriter:
def write(self, _message):
raise OSError(9, "Bad file descriptor")
def flush(self):
raise OSError(9, "Bad file descriptor")
def test_cli_process_progress_falls_back_to_stdout_when_stderr_is_invalid(monkeypatch, tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
]
"""
)
report = RunReport(
status="success",
config={"model": "m", "base_url": "b"},
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
pipeline=["grammar"],
modules=[],
applied_changes=[],
skipped_corrections=[],
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
work_dir_retention="auto",
work_dir_retained=False,
work_dir=None,
error=None,
)
result = ProcessResult(
transcript=transcript,
report=report,
run_dir=tmp_path / "run",
work_dir_retained=False,
)
def _fake_pipeline(*args, **kwargs):
kwargs["progress"]("progress-line")
return result
stdout_capture = io.StringIO()
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", lambda overrides=None: object())
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
monkeypatch.setattr("audita.cli.process_transcript_result", _fake_pipeline)
monkeypatch.setattr("audita.cli.sys.stderr", _BrokenWriter())
monkeypatch.setattr("audita.cli.sys.stdout", stdout_capture)
output_path = tmp_path / "out.json"
exit_code = main(
[
"process",
"transcript.json",
"--glossary",
"glossary.yaml",
"--output",
str(output_path),
]
)
assert exit_code == 0
assert "progress-line" in stdout_capture.getvalue()
def test_cli_process_failure_summary_falls_back_to_stdout_when_stderr_is_invalid(monkeypatch, tmp_path):
stdout_capture = io.StringIO()
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", lambda overrides=None: object())
monkeypatch.setattr("audita.cli.load_transcript", lambda path: (_ for _ in ()).throw(RuntimeError("boom")))
monkeypatch.setattr("audita.cli.sys.stderr", _BrokenWriter())
monkeypatch.setattr("audita.cli.sys.stdout", stdout_capture)
exit_code = main(
[
"process",
"transcript.json",
"--glossary",
"glossary.yaml",
"--work-dir",
str(tmp_path / "work"),
]
)
assert exit_code == 1
assert "audita: error: boom" in stdout_capture.getvalue()

View File

@@ -0,0 +1,435 @@
import pytest
from audita.core.config import (
AuditaConfig,
ConfigOverrides,
DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD,
DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD,
DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD,
DEFAULT_LLM_CONCURRENCY,
DEFAULT_LLM_TIMEOUT_SECONDS,
DEFAULT_MAX_SECTION_TOKENS,
DEFAULT_MIN_SECTION_TOKENS,
DEFAULT_NORMALIZE_MAX_SEGMENT_GAP,
DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD,
DEFAULT_VALIDATION_MAX_PROMPT_TOKENS,
DEFAULT_WORK_DIR_RETENTION,
)
from audita.core.errors import AuditaConfigError
from audita.modules import DEFAULT_MODULE_KEYS
def test_default_config_allows_missing_api_key():
config = AuditaConfig.from_sources(env={})
assert config.api_key is None
assert config.llm_concurrency == DEFAULT_LLM_CONCURRENCY
assert config.llm_timeout_seconds == DEFAULT_LLM_TIMEOUT_SECONDS
assert config.validation_llm_api_key is None
assert config.validation_llm_concurrency is None
assert config.validation_llm_timeout_seconds is None
assert config.validation_model is None
assert config.validation_base_url is None
assert config.validation_max_retries is None
assert config.validation_max_prompt_tokens == DEFAULT_VALIDATION_MAX_PROMPT_TOKENS
assert config.target_sections is None
assert config.max_section_tokens == DEFAULT_MAX_SECTION_TOKENS
assert config.min_section_tokens == DEFAULT_MIN_SECTION_TOKENS
assert config.module_keys == DEFAULT_MODULE_KEYS
assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD
assert config.grammar_confidence_threshold == DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD
assert config.homophones_confidence_threshold == DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD
assert config.spoken_word_confidence_threshold == DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD
assert config.normalize_max_segment_gap == DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
assert config.work_dir_retention == DEFAULT_WORK_DIR_RETENTION
def test_cli_overrides_take_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_MAX_SECTION_TOKENS": "1000"},
overrides=ConfigOverrides(max_section_tokens=2000, min_section_tokens=1000),
)
assert config.max_section_tokens == 2000
def test_min_section_tokens_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_MIN_SECTION_TOKENS": "2000"},
overrides=ConfigOverrides(min_section_tokens=6000),
)
assert config.min_section_tokens == 6000
def test_llm_concurrency_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_LLM_CONCURRENCY": "2"},
overrides=ConfigOverrides(llm_concurrency=4),
)
assert config.llm_concurrency == 4
def test_llm_concurrency_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_LLM_CONCURRENCY": "3"})
assert config.llm_concurrency == 3
def test_llm_timeout_seconds_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_LLM_TIMEOUT_SECONDS": "120"},
overrides=ConfigOverrides(llm_timeout_seconds=900.0),
)
assert config.llm_timeout_seconds == 900.0
def test_llm_timeout_seconds_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_LLM_TIMEOUT_SECONDS": "120.5"})
assert config.llm_timeout_seconds == 120.5
def test_validation_llm_concurrency_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_VALIDATION_LLM_CONCURRENCY": "2"},
overrides=ConfigOverrides(validation_llm_concurrency=4),
)
assert config.validation_llm_concurrency == 4
def test_validation_llm_concurrency_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_CONCURRENCY": "3"})
assert config.validation_llm_concurrency == 3
def test_validation_llm_timeout_seconds_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "120"},
overrides=ConfigOverrides(validation_llm_timeout_seconds=900.0),
)
assert config.validation_llm_timeout_seconds == 900.0
def test_validation_llm_timeout_seconds_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "120.5"})
assert config.validation_llm_timeout_seconds == 120.5
def test_validation_model_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MODEL": "validator-model"})
assert config.validation_model == "validator-model"
def test_validation_base_url_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_BASE_URL": "http://localhost:9000/v1"})
assert config.validation_base_url == "http://localhost:9000/v1"
def test_validation_max_retries_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_RETRIES": "7"})
assert config.validation_max_retries == 7
def test_validation_max_prompt_tokens_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_VALIDATION_MAX_PROMPT_TOKENS": "1024"},
overrides=ConfigOverrides(validation_max_prompt_tokens=4096),
)
assert config.validation_max_prompt_tokens == 4096
def test_validation_max_prompt_tokens_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_PROMPT_TOKENS": "1024"})
assert config.validation_max_prompt_tokens == 1024
def test_target_sections_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_TARGET_SECTIONS": "2"},
overrides=ConfigOverrides(target_sections=5),
)
assert config.target_sections == 5
def test_target_sections_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_TARGET_SECTIONS": "3"})
assert config.target_sections == 3
def test_min_section_tokens_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_MIN_SECTION_TOKENS": "3000"})
assert config.min_section_tokens == 3000
def test_generic_llm_api_key_env_is_read():
config = AuditaConfig.from_sources(env={"AUDITA_LLM_API_KEY": "generic-key"})
assert config.api_key == "generic-key"
def test_generic_llm_api_key_takes_precedence_over_openrouter_env():
config = AuditaConfig.from_sources(
env={
"AUDITA_LLM_API_KEY": "generic-key",
"OPENROUTER_API_KEY": "legacy-key",
}
)
assert config.api_key == "generic-key"
def test_llm_api_key_cli_override_takes_precedence_over_env():
config = AuditaConfig.from_sources(
env={
"AUDITA_LLM_API_KEY": "generic-key",
"OPENROUTER_API_KEY": "legacy-key",
},
overrides=ConfigOverrides(llm_api_key="cli-key"),
)
assert config.api_key == "cli-key"
def test_blank_llm_api_key_override_resolves_to_none():
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "legacy-key"},
overrides=ConfigOverrides(llm_api_key=" "),
)
assert config.api_key is None
def test_validation_llm_api_key_env_is_read():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_API_KEY": "validation-key"})
assert config.validation_llm_api_key == "validation-key"
def test_blank_validation_llm_api_key_override_disables_primary_fallback():
config = AuditaConfig.from_sources(
env={"AUDITA_LLM_API_KEY": "primary-key"},
overrides=ConfigOverrides(validation_llm_api_key=" "),
)
assert config.validation_llm_api_key == ""
assert config.validation_llm_config().api_key == ""
def test_effective_validation_fields_fall_back_to_primary_settings():
config = AuditaConfig.from_sources(
env={
"AUDITA_LLM_API_KEY": "primary-key",
"AUDITA_MODEL": "primary-model",
"AUDITA_BASE_URL": "http://localhost:8000/v1",
"AUDITA_LLM_TIMEOUT_SECONDS": "120",
"AUDITA_LLM_CONCURRENCY": "5",
"AUDITA_MAX_RETRIES": "9",
}
)
validation = config.validation_llm_config()
assert validation.api_key == "primary-key"
assert validation.model == "primary-model"
assert validation.base_url == "http://localhost:8000/v1"
assert validation.llm_timeout_seconds == 120.0
assert validation.llm_concurrency == 5
assert validation.max_retries == 9
def test_effective_validation_fields_use_overrides_when_set():
config = AuditaConfig.from_sources(
env={
"AUDITA_LLM_API_KEY": "primary-key",
"AUDITA_MODEL": "primary-model",
"AUDITA_BASE_URL": "http://localhost:8000/v1",
"AUDITA_LLM_TIMEOUT_SECONDS": "120",
"AUDITA_LLM_CONCURRENCY": "5",
"AUDITA_MAX_RETRIES": "9",
"AUDITA_VALIDATION_LLM_API_KEY": "validation-key",
"AUDITA_VALIDATION_MODEL": "validation-model",
"AUDITA_VALIDATION_BASE_URL": "http://localhost:9000/v1",
"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "240",
"AUDITA_VALIDATION_LLM_CONCURRENCY": "3",
"AUDITA_VALIDATION_MAX_RETRIES": "2",
}
)
validation = config.validation_llm_config()
assert validation.api_key == "validation-key"
assert validation.model == "validation-model"
assert validation.base_url == "http://localhost:9000/v1"
assert validation.llm_timeout_seconds == 240.0
assert validation.llm_concurrency == 3
assert validation.max_retries == 2
def test_module_key_overrides_take_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_MODULES": "grammar"},
overrides=ConfigOverrides(module_keys="homophones,grammar"),
)
assert config.module_keys == ("homophones", "grammar")
def test_module_key_env_is_parsed_and_trimmed():
config = AuditaConfig.from_sources(env={"AUDITA_MODULES": " glossary , grammar "})
assert config.module_keys == ("glossary", "grammar")
def test_invalid_work_dir_retention_is_rejected():
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(env={"AUDITA_WORK_DIR_RETENTION": "sometimes"})
def test_report_dict_includes_llm_timeout_seconds():
config = AuditaConfig.from_sources(env={"AUDITA_LLM_TIMEOUT_SECONDS": "321"})
assert config.to_report_dict()["llm_timeout_seconds"] == 321.0
def test_report_dict_includes_target_sections():
config = AuditaConfig.from_sources(env={"AUDITA_TARGET_SECTIONS": "7"})
assert config.to_report_dict()["target_sections"] == 7
def test_report_dict_includes_effective_validation_llm_config():
config = AuditaConfig.from_sources(
env={
"AUDITA_LLM_API_KEY": "primary-key",
"AUDITA_VALIDATION_MODEL": "validation-model",
}
)
report = config.to_report_dict()
assert report["validation_model"] == "validation-model"
assert report["validation_max_prompt_tokens"] == DEFAULT_VALIDATION_MAX_PROMPT_TOKENS
assert report["effective_validation_llm"]["api_key_configured"] is True
assert report["effective_validation_llm"]["model"] == "validation-model"
def test_threshold_overrides_take_precedence():
config = AuditaConfig.from_sources(
env={
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.6",
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.65",
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.7",
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.75",
},
overrides=ConfigOverrides(
glossary_confidence_threshold=0.85,
grammar_confidence_threshold=0.88,
homophones_confidence_threshold=0.9,
spoken_word_confidence_threshold=0.95,
),
)
assert config.glossary_confidence_threshold == 0.85
assert config.grammar_confidence_threshold == 0.88
assert config.homophones_confidence_threshold == 0.9
assert config.spoken_word_confidence_threshold == 0.95
@pytest.mark.parametrize(
"value",
[
"",
"grammar,,homophones",
"bogus",
],
)
def test_invalid_module_sequences_are_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_MODULES"):
AuditaConfig.from_sources(env={"AUDITA_MODULES": value})
@pytest.mark.parametrize(
"env_name",
[
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD",
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD",
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD",
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD",
],
)
def test_invalid_thresholds_are_rejected(env_name):
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(env={env_name: "1.5"})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_llm_concurrency_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_LLM_CONCURRENCY"):
AuditaConfig.from_sources(env={"AUDITA_LLM_CONCURRENCY": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_llm_timeout_seconds_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_LLM_TIMEOUT_SECONDS"):
AuditaConfig.from_sources(env={"AUDITA_LLM_TIMEOUT_SECONDS": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_validation_llm_concurrency_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_LLM_CONCURRENCY"):
AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_CONCURRENCY": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_validation_llm_timeout_seconds_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"):
AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": value})
@pytest.mark.parametrize("value", ["-1", "many"])
def test_invalid_validation_max_retries_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_MAX_RETRIES"):
AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_RETRIES": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_validation_max_prompt_tokens_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_MAX_PROMPT_TOKENS"):
AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_PROMPT_TOKENS": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_target_sections_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_TARGET_SECTIONS"):
AuditaConfig.from_sources(env={"AUDITA_TARGET_SECTIONS": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_min_section_tokens_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_MIN_SECTION_TOKENS"):
AuditaConfig.from_sources(env={"AUDITA_MIN_SECTION_TOKENS": value})
def test_min_section_tokens_must_not_exceed_max_section_tokens():
with pytest.raises(AuditaConfigError, match="AUDITA_MIN_SECTION_TOKENS"):
AuditaConfig.from_sources(
env={
"AUDITA_MIN_SECTION_TOKENS": "9000",
"AUDITA_MAX_SECTION_TOKENS": "8000",
}
)

View File

@@ -0,0 +1,117 @@
import importlib.machinery
import importlib.util
import io
import os
import shutil
import subprocess
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
LAUNCHER = ROOT / "audita"
def _load_launcher_module():
loader = importlib.machinery.SourceFileLoader("audita_launcher", str(LAUNCHER))
spec = importlib.util.spec_from_file_location("audita_launcher", LAUNCHER, loader=loader)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_root_launcher_exists_and_is_executable():
assert LAUNCHER.is_file()
assert os.access(LAUNCHER, os.X_OK)
def test_root_launcher_help_smoke():
if shutil.which("uv") is None:
pytest.skip("uv is not installed")
result = subprocess.run(
[str(LAUNCHER), "--help"],
cwd=ROOT,
text=True,
capture_output=True,
check=False,
)
assert result.returncode == 0
assert result.stdout.startswith("usage: audita ")
def test_root_launcher_writes_error_log_when_uv_is_missing(tmp_path):
work_dir = tmp_path / "work"
result = subprocess.run(
[sys.executable, str(LAUNCHER), "process", "transcript.json", "--glossary", "glossary.yaml", "--work-dir", str(work_dir)],
cwd=ROOT,
text=True,
capture_output=True,
env={**os.environ, "PATH": ""},
check=False,
)
assert result.returncode == 1
assert result.stdout == ""
assert "audita: error: uv is required to run this launcher" in result.stderr
run_dir = next(work_dir.iterdir())
assert f"audita: run directory: {run_dir}" in result.stderr
assert (run_dir / "error.log").exists()
def test_root_launcher_preserves_child_exit_code_and_writes_fallback_error_log(tmp_path):
work_dir = tmp_path / "work"
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
fake_uv = fake_bin / "uv"
fake_uv.write_text("#!/bin/sh\nexit 120\n", encoding="utf-8")
fake_uv.chmod(0o755)
result = subprocess.run(
[sys.executable, str(LAUNCHER), "process", "transcript.json", "--glossary", "glossary.yaml", "--work-dir", str(work_dir)],
cwd=ROOT,
text=True,
capture_output=True,
env={**os.environ, "PATH": str(fake_bin)},
check=False,
)
assert result.returncode == 120
assert result.stdout == ""
assert "audita: subprocess exited with status 120" in result.stderr
run_dir = next(work_dir.iterdir())
assert (run_dir / "error.log").exists()
class _BrokenWriter:
def write(self, _message):
raise OSError(9, "Bad file descriptor")
def flush(self):
raise OSError(9, "Bad file descriptor")
def test_root_launcher_falls_back_to_stdout_when_stderr_is_invalid(monkeypatch, tmp_path):
launcher = _load_launcher_module()
work_dir = tmp_path / "work"
stdout_capture = io.StringIO()
monkeypatch.setattr(launcher.shutil, "which", lambda _name: None)
monkeypatch.setattr(
launcher.sys,
"argv",
["audita", "process", "transcript.json", "--glossary", "glossary.yaml", "--work-dir", str(work_dir)],
)
monkeypatch.setattr(launcher.sys, "stderr", _BrokenWriter())
monkeypatch.setattr(launcher.sys, "stdout", stdout_capture)
exit_code = launcher.main()
assert exit_code == 1
assert "audita: error: uv is required to run this launcher" in stdout_capture.getvalue()
run_dir = next(work_dir.iterdir())
assert (run_dir / "error.log").exists()

View File

@@ -0,0 +1,698 @@
import json
import threading
import pytest
from audita.core.config import AuditaConfig, ConfigOverrides
from audita.core.errors import AuditaLLMError
from audita.core.io import write_report
from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_json
from audita.modules import DEFAULT_MODULE_KEYS, default_module_specs, resolve_module_specs
from audita.pipeline import process_transcript, process_transcript_result
class FakeStructuredLLMClient:
def __init__(self, responses):
self._responses = responses
self._lock = threading.Lock()
self.calls = []
def run_structured(self, *, stage_name, messages, response_model, config):
with self._lock:
self.calls.append(
{
"stage_name": stage_name,
"messages": list(messages),
"response_model": response_model,
"config": config,
}
)
response = _pop_llm_response(self._responses, stage_name)
if isinstance(response, Exception):
raise response
return response_model.model_validate(response)
def _pop_llm_response(responses, stage_name):
if isinstance(responses, dict):
if stage_name not in responses:
raise AuditaLLMError(f"FakeStructuredLLMClient received unexpected stage_name: {stage_name}")
payloads = responses[stage_name]
if isinstance(payloads, list):
if not payloads:
raise AuditaLLMError(f"FakeStructuredLLMClient received too many calls for stage_name: {stage_name}")
return payloads.pop(0)
payload = payloads
del responses[stage_name]
return payload
if not responses:
raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.")
return responses.pop(0)
def _glossary():
return parse_glossary_yaml(
"""
glossary:
- name: "Jesters"
category: faction
summary: "A faction."
"""
)
def _transcript():
return parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hello.", "categories": ["intro"]},
{"speaker": "Eric", "start": 1.5, "end": 2.0, "text": "Again.", "categories": ["intro", "aside"]},
{"speaker": "Mike", "start": 10.0, "end": 11.0, "text": "Done.", "categories": ["response"]}
]
"""
)
def test_process_transcript_runs_noop_framework(tmp_path):
llm_client = FakeStructuredLLMClient(
[
{"corrections": []},
{"corrections": []},
{"corrections": []},
{"corrections": []},
{"corrections": []},
]
)
revised = process_transcript(
_transcript(),
_glossary(),
AuditaConfig.from_sources(env={}, overrides=None),
llm_client=llm_client,
)
assert [segment.id for segment in revised] == [1, 2]
assert revised[0].text == "Hello. Again."
assert revised[1].text == "Done."
assert revised[0].categories == ["intro", "aside"]
assert revised[1].categories == ["response"]
assert [call["stage_name"] for call in llm_client.calls] == [
"glossary_1:proposal",
"homophones:proposal",
"glossary_2:proposal",
"spoken_word:proposal",
"grammar:proposal",
]
def test_process_transcript_result_can_use_different_validation_llm_settings(tmp_path):
llm_client = FakeStructuredLLMClient(
{
"grammar:proposal": {
"corrections": [
{
"id": 1,
"original_text": "hello world",
"corrected_text": "Hello world.",
"confidence": 0.95,
}
]
},
"grammar:grammar_only_guard": {
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.98,
"reason": "ok",
}
]
},
"grammar:meaning_reversal_review": {
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.99,
"reason": "ok",
}
]
},
}
)
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "primary-key"},
overrides=ConfigOverrides(
model="primary-model",
base_url="http://localhost:8000/v1",
max_retries=7,
llm_timeout_seconds=120,
validation_llm_api_key="validation-key",
validation_model="validation-model",
validation_base_url="http://localhost:9000/v1",
validation_max_retries=2,
validation_llm_timeout_seconds=240,
validation_llm_concurrency=3,
work_dir=tmp_path / "work",
work_dir_retention="always",
),
)
result = process_transcript_result(
parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"}
]
"""
),
_glossary(),
config,
module_keys=["grammar"],
llm_client=llm_client,
)
assert result.transcript[0].text == "Hello world."
calls_by_stage = {call["stage_name"]: call["config"] for call in llm_client.calls}
assert calls_by_stage["grammar:proposal"].model == "primary-model"
assert calls_by_stage["grammar:proposal"].base_url == "http://localhost:8000/v1"
assert calls_by_stage["grammar:proposal"].api_key == "primary-key"
assert calls_by_stage["grammar:grammar_only_guard"].model == "validation-model"
assert calls_by_stage["grammar:grammar_only_guard"].base_url == "http://localhost:9000/v1"
assert calls_by_stage["grammar:grammar_only_guard"].api_key == "validation-key"
assert calls_by_stage["grammar:grammar_only_guard"].max_retries == 2
assert calls_by_stage["grammar:grammar_only_guard"].llm_timeout_seconds == 240
def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(tmp_path):
config = AuditaConfig.from_sources(
env={},
overrides=None,
)
config = AuditaConfig(
api_key=config.api_key,
model=config.model,
base_url=config.base_url,
max_retries=config.max_retries,
max_section_tokens=config.max_section_tokens,
glossary_confidence_threshold=config.glossary_confidence_threshold,
grammar_confidence_threshold=config.grammar_confidence_threshold,
homophones_confidence_threshold=config.homophones_confidence_threshold,
spoken_word_confidence_threshold=config.spoken_word_confidence_threshold,
normalize_max_segment_gap=config.normalize_max_segment_gap,
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
normalize_max_segment_duration=config.normalize_max_segment_duration,
normalize_max_segment_tokens=config.normalize_max_segment_tokens,
work_dir=tmp_path / "work",
work_dir_retention="always",
)
llm_client = FakeStructuredLLMClient(
[
{"corrections": []},
{"corrections": []},
{"corrections": []},
{"corrections": []},
{"corrections": []},
]
)
result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client)
assert result.work_dir_retained is True
assert result.report.pipeline == [
"glossary_1",
"homophones",
"glossary_2",
"spoken_word",
"grammar",
]
assert result.report.totals["applied_change_count"] == 0
assert (result.run_dir / "report.json").exists()
assert (result.run_dir / "normalization" / "summary.json").exists()
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"glossary_stage_protected_glossary_guard",
"non_empty_segment_guard",
"spoken_form_plausibility_review",
"meaning_reversal_review",
]
assert [validator["name"] for validator in result.report.modules[2].to_dict()["validators"]] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"glossary_stage_protected_glossary_guard",
"non_empty_segment_guard",
"spoken_form_plausibility_review",
"meaning_reversal_review",
]
assert [validator["name"] for validator in result.report.modules[3].to_dict()["validators"]] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
"spoken_word_review",
"meaning_reversal_review",
]
assert [validator["name"] for validator in result.report.modules[4].to_dict()["validators"]] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
"grammar_only_guard",
"meaning_reversal_review",
]
def test_external_report_can_be_written(tmp_path):
config = AuditaConfig.from_sources(env={})
llm_client = FakeStructuredLLMClient(
[
{"corrections": []},
{"corrections": []},
{"corrections": []},
{"corrections": []},
{"corrections": []},
]
)
result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client)
report_path = tmp_path / "report.json"
write_report(report_path, result.report)
payload = json.loads(report_path.read_text(encoding="utf-8"))
assert payload["pipeline"][0] == "glossary_1"
assert payload["totals"]["applied_change_count"] == 0
def test_process_transcript_preserves_categories_in_llm_prompt_payloads(tmp_path):
llm_client = FakeStructuredLLMClient(
[
{"corrections": []},
{"corrections": []},
{"corrections": []},
{"corrections": []},
{"corrections": []},
]
)
process_transcript(
_transcript(),
_glossary(),
AuditaConfig.from_sources(env={}, overrides=None),
llm_client=llm_client,
)
proposal_prompt = llm_client.calls[0]["messages"][1]["content"]
assert '"categories": [' in proposal_prompt
assert '"intro"' in proposal_prompt
assert '"aside"' in proposal_prompt
def test_default_module_specs_expose_final_validator_order():
specs = default_module_specs()
assert DEFAULT_MODULE_KEYS == ("glossary", "homophones", "glossary", "spoken_word", "grammar")
assert [spec.instance_name for spec in specs] == [
"glossary_1",
"homophones",
"glossary_2",
"spoken_word",
"grammar",
]
assert [validator.name for validator in specs[0].module.validators()] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"glossary_stage_protected_glossary_guard",
"non_empty_segment_guard",
"spoken_form_plausibility_review",
"meaning_reversal_review",
]
assert [validator.name for validator in specs[1].module.validators()] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
"spoken_form_plausibility_review",
"meaning_reversal_review",
]
assert [validator.name for validator in specs[2].module.validators()] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"glossary_stage_protected_glossary_guard",
"non_empty_segment_guard",
"spoken_form_plausibility_review",
"meaning_reversal_review",
]
assert [validator.name for validator in specs[3].module.validators()] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
"spoken_word_review",
"meaning_reversal_review",
]
assert [validator.name for validator in specs[4].module.validators()] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
"grammar_only_guard",
"meaning_reversal_review",
]
def test_process_transcript_result_missing_api_key_writes_failed_report(tmp_path):
config = AuditaConfig.from_sources(
env={},
overrides=None,
)
config = AuditaConfig(
api_key=None,
model=config.model,
base_url=config.base_url,
max_retries=config.max_retries,
max_section_tokens=config.max_section_tokens,
glossary_confidence_threshold=config.glossary_confidence_threshold,
grammar_confidence_threshold=config.grammar_confidence_threshold,
homophones_confidence_threshold=config.homophones_confidence_threshold,
spoken_word_confidence_threshold=config.spoken_word_confidence_threshold,
normalize_max_segment_gap=config.normalize_max_segment_gap,
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
normalize_max_segment_duration=config.normalize_max_segment_duration,
normalize_max_segment_tokens=config.normalize_max_segment_tokens,
work_dir=tmp_path / "work",
work_dir_retention="never",
)
with pytest.raises(AuditaLLMError, match="OpenRouter endpoint"):
process_transcript_result(_transcript(), _glossary(), config)
run_dir = next((tmp_path / "work").iterdir())
report = json.loads((run_dir / "report.json").read_text(encoding="utf-8"))
assert report["status"] == "failed"
assert report["normalization"]["normalized_segment_count"] == 2
assert report["pipeline"] == [
"glossary_1",
"homophones",
"glossary_2",
"spoken_word",
"grammar",
]
assert report["modules"] == []
assert report["applied_changes"] == []
assert report["skipped_corrections"] == []
assert report["work_dir_retained"] is True
assert report["work_dir"] == str(run_dir)
assert "AUDITA_LLM_API_KEY" in report["error"]
assert "OPENROUTER_API_KEY" in report["error"]
assert "OpenRouter endpoint" in report["error"]
assert report["error_details"]["type"] == "AuditaLLMError"
assert report["error_details"]["phase"] == "pipeline"
assert report["error_details"]["error_log"] == str(run_dir / "error.log")
assert (run_dir / "error.log").exists()
def test_process_transcript_result_allows_missing_api_key_for_nondefault_proposal_endpoint(tmp_path):
llm_client = FakeStructuredLLMClient(
[
{"corrections": []},
{"corrections": []},
{"corrections": []},
{"corrections": []},
{"corrections": []},
]
)
config = AuditaConfig.from_sources(
env={},
overrides=ConfigOverrides(
base_url="http://localhost:8000/v1",
model="meta-llama/Llama-3.1-8B-Instruct",
work_dir=tmp_path / "work",
work_dir_retention="always",
),
)
result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client)
assert result.report.status == "success"
assert llm_client.calls[0]["config"].api_key is None
assert llm_client.calls[0]["config"].base_url == "http://localhost:8000/v1"
def test_process_transcript_result_allows_missing_validation_api_key_for_nondefault_validation_endpoint(tmp_path):
llm_client = FakeStructuredLLMClient(
{
"grammar:proposal": {
"corrections": [
{
"id": 1,
"original_text": "hello world",
"corrected_text": "Hello world.",
"confidence": 0.95,
}
]
},
"grammar:grammar_only_guard": {
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.98,
"reason": "ok",
}
]
},
"grammar:meaning_reversal_review": {
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.99,
"reason": "ok",
}
]
},
}
)
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "primary-key"},
overrides=ConfigOverrides(
work_dir=tmp_path / "work",
work_dir_retention="always",
validation_llm_api_key=" ",
validation_base_url="http://localhost:9000/v1",
validation_model="meta-llama/Llama-3.1-8B-Instruct",
),
)
result = process_transcript_result(
parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"}
]
"""
),
_glossary(),
config,
module_keys=["grammar"],
llm_client=llm_client,
)
assert result.transcript[0].text == "Hello world."
calls_by_stage = {call["stage_name"]: call["config"] for call in llm_client.calls}
assert calls_by_stage["grammar:proposal"].api_key == "primary-key"
assert calls_by_stage["grammar:grammar_only_guard"].api_key == ""
assert calls_by_stage["grammar:grammar_only_guard"].base_url == "http://localhost:9000/v1"
def test_process_transcript_result_preserves_partial_progress_when_later_module_fails(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the dam."}
]
"""
)
config = AuditaConfig.from_sources(
env={},
overrides=None,
)
config = AuditaConfig(
api_key=config.api_key,
model=config.model,
base_url=config.base_url,
max_retries=config.max_retries,
max_section_tokens=config.max_section_tokens,
glossary_confidence_threshold=config.glossary_confidence_threshold,
grammar_confidence_threshold=config.grammar_confidence_threshold,
homophones_confidence_threshold=config.homophones_confidence_threshold,
spoken_word_confidence_threshold=config.spoken_word_confidence_threshold,
normalize_max_segment_gap=config.normalize_max_segment_gap,
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
normalize_max_segment_duration=config.normalize_max_segment_duration,
normalize_max_segment_tokens=config.normalize_max_segment_tokens,
work_dir=tmp_path / "work",
work_dir_retention="never",
)
llm_client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "gestures",
"corrected_text": "Jesters",
"confidence": 0.95,
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.97,
"reason": "Likely spoken-form correction in context.",
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.99,
"reason": "Does not reverse the segment meaning.",
}
]
},
AuditaLLMError("Simulated homophones proposal failure."),
]
)
with pytest.raises(AuditaLLMError, match="Simulated homophones proposal failure"):
process_transcript_result(transcript, _glossary(), config, llm_client=llm_client)
run_dir = next((tmp_path / "work").iterdir())
report = json.loads((run_dir / "report.json").read_text(encoding="utf-8"))
assert report["status"] == "failed"
assert report["normalization"]["normalized_segment_count"] == 1
assert [module["instance_name"] for module in report["modules"]] == ["glossary_1"]
assert report["applied_changes"][0]["corrected_text"] == "Jesters"
assert report["applied_changes"][0]["segment_text_after"] == "There were Jesters at the dam."
assert report["totals"]["applied_change_count"] == 1
assert report["skipped_corrections"] == []
assert report["pipeline"][1] == "homophones"
assert "Simulated homophones proposal failure." in report["error"]
assert report["error_details"]["module_instance"] == "homophones"
assert report["error_details"]["phase"] == "pipeline"
assert (run_dir / "error.log").exists()
def test_process_transcript_result_preserves_partial_skips_and_validator_diagnostics_on_failure(tmp_path):
config = AuditaConfig.from_sources(
env={},
overrides=None,
)
config = AuditaConfig(
api_key=config.api_key,
model=config.model,
base_url=config.base_url,
max_retries=config.max_retries,
max_section_tokens=config.max_section_tokens,
glossary_confidence_threshold=0.8,
grammar_confidence_threshold=config.grammar_confidence_threshold,
homophones_confidence_threshold=config.homophones_confidence_threshold,
spoken_word_confidence_threshold=config.spoken_word_confidence_threshold,
normalize_max_segment_gap=config.normalize_max_segment_gap,
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
normalize_max_segment_duration=config.normalize_max_segment_duration,
normalize_max_segment_tokens=config.normalize_max_segment_tokens,
work_dir=tmp_path / "work",
work_dir_retention="never",
)
llm_client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "Hello",
"corrected_text": "Jesters",
"confidence": 0.40,
},
{
"id": 1,
"original_text": "Hello",
"corrected_text": "Jesters",
"confidence": 0.95,
},
]
},
{
"validations": [
{
"correction_index": 99,
"approved": True,
"confidence": 0.98,
"reason": "Malformed response for testing.",
}
]
},
]
)
with pytest.raises(AuditaLLMError, match="unknown correction_index"):
process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client)
run_dir = next((tmp_path / "work").iterdir())
report = json.loads((run_dir / "report.json").read_text(encoding="utf-8"))
validator_dir = run_dir / "glossary_1"
assert report["status"] == "failed"
assert report["modules"] == []
assert len(report["skipped_corrections"]) == 1
assert report["skipped_corrections"][0]["reason"] == "proposal confidence below threshold"
assert report["skipped_corrections"][0]["source"] == "validator:proposal_confidence_guard"
assert "unknown correction_index" in report["error"]
assert report["error_details"]["module_instance"] == "glossary_1"
assert report["error_details"]["phase"] == "pipeline"
assert (run_dir / "error.log").exists()
assert (validator_dir / "spoken_form_plausibility_review-prompt-0000.json").exists()
assert (validator_dir / "spoken_form_plausibility_review-response-0000.json").exists()
def test_resolve_module_specs_numbers_repeated_keys():
specs = resolve_module_specs(["glossary", "homophones", "glossary"])
assert [spec.instance_name for spec in specs] == ["glossary_1", "homophones", "glossary_2"]
assert [spec.module_key for spec in specs] == ["glossary", "homophones", "glossary"]
def test_process_transcript_result_supports_grammar_only_module_override(tmp_path):
config = AuditaConfig.from_sources(
env={},
overrides=ConfigOverrides(work_dir=tmp_path / "work", work_dir_retention="always"),
)
result = process_transcript_result(
_transcript(),
_glossary(),
config,
module_keys=["grammar"],
llm_client=FakeStructuredLLMClient([{"corrections": []}]),
)
assert [segment.id for segment in result.transcript] == [1, 2]
assert result.report.pipeline == ["grammar"]
assert result.report.totals["applied_change_count"] == 0

View File

@@ -0,0 +1,84 @@
import json
from audita.core.schemas import parse_source_transcript_json, parse_transcript_json, transcript_to_json
SERIATIM_TRANSCRIPT = """
{
"metadata": {
"application": "seriatim",
"version": "dev",
"input_reader": "json-files",
"input_files": ["eric.json", "mike.json"],
"preprocessing_modules": ["validate-raw", "normalize-speakers", "trim-text"],
"postprocessing_modules": ["detect-overlaps", "resolve-overlaps", "backchannel"],
"output_modules": ["json"]
},
"segments": [
{
"id": 1,
"source": "eric.json",
"source_segment_index": 0,
"speaker": "Eric Rakestraw",
"start": 1.25,
"end": 3.5,
"text": "Hello there.",
"overlap_group_id": 1
},
{
"id": 2,
"source": "eric.json",
"source_ref": "word-run:1:1:1",
"derived_from": ["eric.json#0"],
"speaker": "Eric Rakestraw",
"start": 4.0,
"end": 4.5,
"text": "Resolved word run",
"categories": ["backchannel"]
}
],
"overlap_groups": [
{
"id": 1,
"start": 1.25,
"end": 4.0,
"segments": ["eric.json#0", "mike.json#0"],
"speakers": ["Eric Rakestraw", "Mike Brown"],
"class": "unknown",
"resolution": "unresolved"
}
]
}
"""
def test_parse_source_transcript_json_accepts_seriatim_transcript_object():
segments = parse_source_transcript_json(SERIATIM_TRANSCRIPT)
assert len(segments) == 2
assert segments[0].id == 1
assert segments[0].speaker == "Eric Rakestraw"
assert segments[0].start == 1.25
assert segments[0].end == 3.5
assert segments[0].text == "Hello there."
assert segments[1].text == "Resolved word run"
assert segments[0].categories is None
assert segments[1].categories == ["backchannel"]
def test_parse_transcript_json_accepts_seriatim_transcript_object_and_ignores_unused_fields():
segments = parse_transcript_json(SERIATIM_TRANSCRIPT)
assert [segment.id for segment in segments] == [1, 2]
assert [segment.text for segment in segments] == ["Hello there.", "Resolved word run"]
assert segments[0].categories is None
assert segments[1].categories == ["backchannel"]
def test_transcript_to_json_emits_categories_only_when_present():
segments = parse_transcript_json(SERIATIM_TRANSCRIPT)
payload = json.loads(transcript_to_json(segments))
assert "categories" not in payload[0]
assert payload[1]["categories"] == ["backchannel"]

View File

@@ -0,0 +1,320 @@
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
from audita.framework.models import CorrectionProposal, ModuleRunSpec
from audita.validators import (
GlossaryStageProtectedGlossaryTermsValidator,
ProtectedGlossaryTermsValidator,
ProtectedVocabulary,
)
from audita.validators.base import ValidationContext
def _glossary():
return parse_glossary_yaml(
"""
glossary:
- name: "Hrank"
aliases:
- "Greenfield"
category: pc
summary: "Hrank Greenfield is a player character."
- name: "Popov"
category: npc
summary: "Popov is an allied NPC."
- name: "Jesters"
aliases:
- "Jester"
category: faction
summary: "The Jesters are a faction."
- name: "Svend"
category: pc
summary: "Svend is a player character."
- name: "Godfrey"
category: npc
summary: "Godfrey is an NPC."
- name: "Lyra"
category: npc
summary: "Lyra is an NPC."
- name: "Loviator"
category: deity
summary: "Loviator is a deity."
"""
)
def test_protected_vocabulary_blocks_replacing_protected_term():
vocabulary = ProtectedVocabulary.from_glossary(_glossary())
assert (
vocabulary.violation_reason("Hrank moves.", "Frank moves.")
== "correction changes protected glossary term usage"
)
def test_protected_vocabulary_glossary_stage_allows_glossary_to_glossary_changes():
vocabulary = ProtectedVocabulary.from_glossary(_glossary())
assert vocabulary.violation_reason("Hrank moves.", "Popov moves.") == "correction changes protected glossary term usage"
assert vocabulary.glossary_stage_violation_reason("Hrank moves.", "Popov moves.") is None
def test_protected_vocabulary_glossary_stage_still_blocks_glossary_to_nonglossary_changes():
vocabulary = ProtectedVocabulary.from_glossary(_glossary())
assert (
vocabulary.glossary_stage_violation_reason("Hrank moves.", "Frank moves.")
== "correction changes protected glossary term usage"
)
def test_protected_vocabulary_blocks_noncanonical_capitalization():
vocabulary = ProtectedVocabulary.from_glossary(_glossary())
assert (
vocabulary.violation_reason("Popov moves.", "POPOV moves.")
== "correction changes protected glossary term capitalization"
)
assert (
vocabulary.violation_reason("Jesters", "jesters")
== "correction changes protected glossary term capitalization"
)
assert (
vocabulary.glossary_stage_violation_reason("Popov moves.", "POPOV moves.")
== "correction changes protected glossary term capitalization"
)
def test_protected_vocabulary_allows_corrections_toward_protected_terms():
vocabulary = ProtectedVocabulary.from_glossary(_glossary())
assert vocabulary.violation_reason("Pawpaw moves.", "Popov moves.") is None
assert vocabulary.violation_reason("gestures", "Jesters") is None
assert vocabulary.violation_reason("gestures", "jesters") is None
assert vocabulary.violation_reason("rank", "Hrank") is None
assert vocabulary.violation_reason("rank", "hrank") is None
assert vocabulary.violation_reason("spend", "Svend") is None
def test_protected_vocabulary_allows_unchanged_noncanonical_terms_and_quote_wrapping():
vocabulary = ProtectedVocabulary.from_glossary(_glossary())
assert vocabulary.violation_reason("jesters advance.", "jesters advance.") is None
before = (
"When you say that, Popov will say, when I was in that room with the jesters, "
"I just knew that Godfrey and Lyra came directly from Loviator herself."
)
after = (
'When you say that, Popov will say, "When I was in that room with the jesters, '
'I just knew that Godfrey and Lyra came directly from Loviator herself."'
)
assert vocabulary.violation_reason(before, after) is None
def test_protected_vocabulary_allows_inferred_and_explicit_plurals():
vocabulary = ProtectedVocabulary.from_glossary(_glossary())
explicit = ProtectedVocabulary.from_glossary(
parse_glossary_yaml(
"""
glossary:
- name: "Mox"
plural: "Moxen"
category: faction
summary: "The Mox are a faction."
"""
)
)
assert vocabulary.violation_reason("Godfrey's", "Godfreys") is None
assert vocabulary.violation_reason("gesture", "Jesters") is None
assert explicit.violation_reason("Mox's", "Moxen") is None
def test_protected_vocabulary_does_not_match_embedded_substrings():
vocabulary = ProtectedVocabulary.from_glossary(_glossary())
assert vocabulary.violation_reason("The shrank spell worked.", "The shrank spell works.") is None
def test_protected_glossary_terms_validator_returns_proposal_indexed_decisions():
validator = ProtectedGlossaryTermsValidator("protected_glossary_guard")
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Frank moves."},
{"id": 2, "speaker": "Eric", "start": 1.0, "end": 2.0, "text": "Pawpaw waits."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="glossary_primary",
module_key="glossary",
id=1,
original_text="Hrank",
corrected_text="Frank",
confidence=0.9,
),
CorrectionProposal(
proposal_index=1,
module_instance="glossary_primary",
module_key="glossary",
id=2,
original_text="Pawpaw",
corrected_text="Popov",
confidence=0.9,
),
]
result = validator.validate(
ValidationContext(
proposals=proposals,
transcript=transcript,
glossary=_glossary(),
config=None, # type: ignore[arg-type]
run_spec=ModuleRunSpec(instance_name="glossary_primary", module_key="glossary", module=None), # type: ignore[arg-type]
run_dir=transcript[0].__class__.__module__ and __import__("pathlib").Path("."),
)
)
assert [decision.proposal_index for decision in result.decisions] == [0, 1]
assert result.decisions[0].approved is False
assert result.decisions[0].reason == "correction changes protected glossary term usage"
assert result.decisions[1].approved is True
def test_protected_glossary_terms_validator_allows_nonglossary_to_lowercase_glossary_replacement():
validator = ProtectedGlossaryTermsValidator("protected_glossary_guard")
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "gestures advance."},
{"id": 2, "speaker": "Eric", "start": 1.0, "end": 2.0, "text": "rank moves."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="gestures",
corrected_text="jesters",
confidence=0.9,
),
CorrectionProposal(
proposal_index=1,
module_instance="homophones",
module_key="homophones",
id=2,
original_text="rank",
corrected_text="hrank",
confidence=0.9,
),
]
result = validator.validate(
ValidationContext(
proposals=proposals,
transcript=transcript,
glossary=_glossary(),
config=None, # type: ignore[arg-type]
run_spec=ModuleRunSpec(instance_name="homophones", module_key="homophones", module=None), # type: ignore[arg-type]
run_dir=__import__("pathlib").Path("."),
)
)
assert [decision.approved for decision in result.decisions] == [True, True]
def test_glossary_stage_protected_glossary_terms_validator_allows_glossary_to_glossary_replacement():
validator = GlossaryStageProtectedGlossaryTermsValidator("glossary_stage_protected_glossary_guard")
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hrank moves."},
{"id": 2, "speaker": "Eric", "start": 1.0, "end": 2.0, "text": "Hrank moves."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="glossary_1",
module_key="glossary",
id=1,
original_text="Hrank",
corrected_text="Popov",
confidence=0.9,
),
CorrectionProposal(
proposal_index=1,
module_instance="glossary_1",
module_key="glossary",
id=2,
original_text="Hrank",
corrected_text="POPOV",
confidence=0.9,
),
]
result = validator.validate(
ValidationContext(
proposals=proposals,
transcript=transcript,
glossary=_glossary(),
config=None, # type: ignore[arg-type]
run_spec=ModuleRunSpec(instance_name="glossary_1", module_key="glossary", module=None), # type: ignore[arg-type]
run_dir=__import__("pathlib").Path("."),
)
)
assert [decision.proposal_index for decision in result.decisions] == [0, 1]
assert result.decisions[0].approved is True
assert result.decisions[1].approved is True
assert result.decisions[1].reason is None
def test_protected_glossary_terms_validator_uses_proposal_span_only():
validator = ProtectedGlossaryTermsValidator("protected_glossary_guard")
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Svend"
category: pc
summary: "Svend is a player character."
- name: "Jesters"
category: faction
summary: "The Jesters are a faction."
"""
)
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "You have to keep it bind. Svend sees the jesters."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="spoken_word",
module_key="spoken_word",
id=1,
original_text="keep it bind",
corrected_text="keep in mind",
confidence=0.9,
)
]
result = validator.validate(
ValidationContext(
proposals=proposals,
transcript=transcript,
glossary=glossary,
config=None, # type: ignore[arg-type]
run_spec=ModuleRunSpec(instance_name="spoken_word", module_key="spoken_word", module=None), # type: ignore[arg-type]
run_dir=__import__("pathlib").Path("."),
)
)
assert result.decisions[0].approved is True