eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Improvements around reporting and work directory retention options

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-23 12:08:44 -05:00
parent c4e2db75f1
commit 3277587e3a
11 changed files with 620 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
README.md
View File

@@ -20,6 +20,12 @@ export OPENROUTER_API_KEY=...
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json
```
To also write a structured JSON report describing what Audita applied or skipped:
```sh
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json --report-json report.json
```
From a checked-out repository, you can also use the root launcher:
```sh
@@ -36,6 +42,7 @@ audita process transcript.json --glossary glossary.yaml --output corrected.json
```
Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr.
`--report-json` writes a separate machine-readable run report and never mixes report data into stdout.
Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables.
@@ -48,6 +55,7 @@ Useful configuration can be supplied by CLI flag or environment variable. CLI fl
| `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | `--grammar-confidence-threshold` | `0.80` | Minimum confidence required to apply a grammar correction |
| `AUDITA_GRAMMAR_VALIDATION_ENABLED` | `--grammar-validation-enabled` / `--no-grammar-validation-enabled` | `true` | Whether grammar corrections are checked by the semantic validator |
| `AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD` | `--grammar-validation-confidence-threshold` | `0.80` | Minimum validator confidence required for validated grammar corrections |
| `AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD` | `--grammar-spoken-form-validation-confidence-threshold` | `0.80` | Minimum validator confidence required for spoken-form rescue corrections |
| `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses |
| `AUDITA_GLOSSARY_MAX_LLM_PASSES` | `--glossary-max-llm-passes` | `3` | Total glossary correction passes |
| `AUDITA_GRAMMAR_MAX_LLM_PASSES` | `--grammar-max-llm-passes` | `3` | Total grammar/readability correction passes |
@@ -56,7 +64,8 @@ Useful configuration can be supplied by CLI flag or environment variable. CLI fl
| `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | `--normalize-max-segment-duration` | `60.0` | Maximum merged segment duration |
| `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | `--normalize-max-segment-tokens` | `2048` | Maximum merged segment prompt payload size |
| `AUDITA_WORK_DIR` | `--work-dir` | `/tmp/audita` | Per-run scratch diagnostics directory |
| `AUDITA_WORK_DIR_RETENTION` | `--work-dir-retention` | `auto` | Whether to retain the per-run work directory: `auto`, `always`, or `never` |
`OPENROUTER_API_KEY` is required and is read from the environment.
`AUDITA_WORK_DIR` stores per-run diagnostics while processing. Successful runs clean up their run directory unless corrections are skipped; failed runs and skipped-correction runs preserve diagnostics for debugging.
`AUDITA_WORK_DIR` stores per-run diagnostics while processing. Under the default `AUDITA_WORK_DIR_RETENTION=auto`, clean successful runs are removed, while failed runs and successful runs with final skipped corrections are preserved. Use `always` to keep every run directory and `never` to remove successful run directories even when skips remain.

19
src/audita/cli.py
View File

@@ -5,8 +5,8 @@ from typing import Optional, Sequence
from .config import AuditaConfig, ConfigOverrides
from .errors import AuditaError
from .io import load_glossary, load_transcript, write_transcript
from .pipeline import process_transcript
from .io import load_glossary, load_transcript, write_report, write_transcript
from .pipeline import process_transcript_result
from .schemas import transcript_to_json
@@ -29,6 +29,7 @@ def _build_parser() -> argparse.ArgumentParser:
process.add_argument("transcript", type=Path, help="path to the input transcript JSON")
process.add_argument("--glossary", type=Path, required=True, help="path to the glossary YAML")
process.add_argument("--output", type=Path, help="write corrected transcript JSON to this path")
process.add_argument("--report-json", type=Path, help="write structured run report JSON to this path")
process.add_argument("--model", help="OpenRouter model to use")
process.add_argument("--base-url", help="OpenAI-compatible API base URL")
process.add_argument("--max-section-tokens", type=int, help="maximum estimated tokens per transcript section")
@@ -82,6 +83,11 @@ def _build_parser() -> argparse.ArgumentParser:
help="maximum estimated tokens for a merged segment prompt payload",
)
process.add_argument("--work-dir", type=Path, help="directory for per-run scratch diagnostics")
process.add_argument(
"--work-dir-retention",
choices=("auto", "always", "never"),
help="whether to retain the per-run work directory",
)
return parser
@@ -107,11 +113,12 @@ def _process(args: argparse.Namespace) -> int:
normalize_max_segment_duration=args.normalize_max_segment_duration,
normalize_max_segment_tokens=args.normalize_max_segment_tokens,
work_dir=args.work_dir,
work_dir_retention=args.work_dir_retention,
)
)
transcript = load_transcript(args.transcript)
glossary = load_glossary(args.glossary)
revised = process_transcript(
result = process_transcript_result(
transcript,
glossary,
config,
@@ -119,9 +126,11 @@ def _process(args: argparse.Namespace) -> int:
)
if args.output is not None:
write_transcript(args.output, revised)
write_transcript(args.output, result.transcript)
else:
sys.stdout.write(transcript_to_json(revised))
sys.stdout.write(transcript_to_json(result.transcript))
if args.report_json is not None:
write_report(args.report_json, result.report)
return 0
except AuditaError as exc:
print(f"audita: error: {exc}", file=sys.stderr)

33
src/audita/config.py
View File

@@ -14,6 +14,7 @@ DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD = 0.80
DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD = 0.80
DEFAULT_MAX_RETRIES = 3
DEFAULT_WORK_DIR = "/tmp/audita"
DEFAULT_WORK_DIR_RETENTION = "auto"
DEFAULT_GLOSSARY_MAX_LLM_PASSES = 3
DEFAULT_GRAMMAR_MAX_LLM_PASSES = 3
DEFAULT_GRAMMAR_VALIDATION_ENABLED = True
@@ -43,6 +44,7 @@ class ConfigOverrides:
normalize_max_segment_duration: Optional[float] = None
normalize_max_segment_tokens: Optional[int] = None
work_dir: Optional[Path] = None
work_dir_retention: Optional[str] = None
@dataclass(frozen=True)
@@ -66,6 +68,7 @@ class AuditaConfig:
normalize_max_segment_duration: float = DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION
normalize_max_segment_tokens: int = DEFAULT_NORMALIZE_MAX_SEGMENT_TOKENS
work_dir: Path = Path(DEFAULT_WORK_DIR)
work_dir_retention: str = DEFAULT_WORK_DIR_RETENTION
@classmethod
def from_sources(
@@ -158,6 +161,13 @@ class AuditaConfig:
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS",
)
work_dir_value = selected.work_dir or Path(source.get("AUDITA_WORK_DIR") or DEFAULT_WORK_DIR)
work_dir_retention = _select_choice(
selected.work_dir_retention,
source.get("AUDITA_WORK_DIR_RETENTION"),
DEFAULT_WORK_DIR_RETENTION,
"AUDITA_WORK_DIR_RETENTION",
("auto", "always", "never"),
)
config = cls(
api_key=api_key,
@@ -177,6 +187,7 @@ class AuditaConfig:
normalize_max_segment_duration=normalize_max_segment_duration,
normalize_max_segment_tokens=normalize_max_segment_tokens,
work_dir=Path(work_dir_value),
work_dir_retention=work_dir_retention,
)
config.validate()
return config
@@ -226,6 +237,8 @@ class AuditaConfig:
raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION must be greater than zero.")
if self.normalize_max_segment_tokens <= 0:
raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS must be greater than zero.")
if self.work_dir_retention not in ("auto", "always", "never"):
raise AuditaConfigError("AUDITA_WORK_DIR_RETENTION must be one of auto, always, or never.")
def _get_required_env(env: Mapping[str, str], name: str) -> str:
@@ -278,3 +291,23 @@ def _select_float(
return float(env_value)
except ValueError as exc:
raise AuditaConfigError(f"{name} must be a number.") from exc
def _select_choice(
cli_value: Optional[str],
env_value: Optional[str],
default: str,
name: str,
choices: tuple[str, ...],
) -> str:
if cli_value is not None:
value = cli_value
elif env_value is not None:
value = env_value
else:
return default
normalized = value.strip().casefold()
if normalized not in choices:
allowed = ", ".join(choices)
raise AuditaConfigError(f"{name} must be one of {allowed}.")
return normalized

26
src/audita/corrections.py
View File

@@ -23,10 +23,24 @@ class SkippedCorrection:
return asdict(self)
@dataclass(frozen=True)
class AppliedCorrection:
id: int
original_text: str
corrected_text: str
confidence: float
segment_text_before: str
segment_text_after: str
def to_dict(self) -> dict:
return asdict(self)
@dataclass(frozen=True)
class CorrectionApplicationResult:
transcript: List[TranscriptSegment]
skipped: List[SkippedCorrection]
applied_corrections: List[AppliedCorrection]
applied_ids: List[int]
ignored_ids: List[int]
ignored: List[SkippedCorrection]
@@ -47,6 +61,7 @@ def apply_corrections(
revised = list(transcript)
id_to_position = _id_to_position(revised)
skipped: List[SkippedCorrection] = []
applied_corrections: List[AppliedCorrection] = []
applied_ids: List[int] = []
ignored_ids: List[int] = []
ignored: List[SkippedCorrection] = []
@@ -70,11 +85,22 @@ def apply_corrections(
skipped.append(_skip(correction, reason, actual_text=segment.text))
continue
revised[position] = segment.model_copy(update={"text": revised_text})
applied_corrections.append(
AppliedCorrection(
id=correction.id,
original_text=correction.original_text,
corrected_text=correction.corrected_text,
confidence=correction.confidence,
segment_text_before=segment.text,
segment_text_after=revised_text,
)
)
applied_ids.append(correction.id)
return CorrectionApplicationResult(
transcript=revised,
skipped=skipped,
applied_corrections=applied_corrections,
applied_ids=applied_ids,
ignored_ids=ignored_ids,
ignored=ignored,

5
src/audita/io.py
View File

@@ -1,6 +1,7 @@
from pathlib import Path
from typing import List
from .reporting import RunReport
from .schemas import Glossary, SourceTranscriptSegment, TranscriptSegment
from .schemas import parse_glossary_yaml, parse_source_transcript_json, transcript_to_json
@@ -15,3 +16,7 @@ def load_glossary(path: Path) -> Glossary:
def write_transcript(path: Path, segments: List[TranscriptSegment]) -> None:
path.write_text(transcript_to_json(segments), encoding="utf-8")
def write_report(path: Path, report: RunReport) -> None:
path.write_text(report.to_json(), encoding="utf-8")

255
src/audita/pipeline.py
View File

@@ -14,6 +14,7 @@ from .normalization import NormalizationResult, normalize_transcript
from .passes import CorrectionPass, GlossaryCorrectionPass, GrammarCorrectionPass, LLMClient
from .prompts import build_grammar_spoken_form_validation_messages, build_grammar_validation_messages
from .protection import ProtectedVocabulary
from .reporting import AppliedChange, ProcessResult, ReportedSkippedCorrection, RunReport
from .semantic_validation import (
filter_with_meaning_preserving_validations,
filter_with_spoken_form_validations,
@@ -37,6 +38,13 @@ class StageSpec:
protected_vocabulary: Optional[ProtectedVocabulary] = None
@dataclass(frozen=True)
class StageRunResult:
transcript: List[TranscriptSegment]
applied_changes: List[AppliedChange]
skipped_corrections: List[ReportedSkippedCorrection]
def process_transcript(
transcript: List[SourceTranscriptSegment],
glossary: Glossary,
@@ -44,12 +52,31 @@ def process_transcript(
llm_client: Optional[LLMClient] = None,
progress: Optional[ProgressCallback] = None,
) -> List[TranscriptSegment]:
return process_transcript_result(
transcript,
glossary,
config,
llm_client=llm_client,
progress=progress,
).transcript
def process_transcript_result(
transcript: List[SourceTranscriptSegment],
glossary: Glossary,
config: AuditaConfig,
llm_client: Optional[LLMClient] = None,
progress: Optional[ProgressCallback] = None,
) -> ProcessResult:
run_dir = _create_run_dir(config.work_dir)
stage_summaries: List[dict] = []
normalization_summary: Optional[dict] = None
applied_changes: List[AppliedChange] = []
final_skipped: List[ReportedSkippedCorrection] = []
working: List[TranscriptSegment] = []
try:
_log(progress, f"Created work directory {run_dir}")
stage_summaries: List[dict] = []
normalization_summary: Optional[dict] = None
_write_run_metadata(run_dir, config, normalization_summary, stage_summaries)
_write_run_metadata(run_dir, config, normalization_summary, stage_summaries, work_dir_retained=True)
normalization_result = normalize_transcript(
transcript,
@@ -66,7 +93,7 @@ def process_transcript(
f"{normalization_result.summary.source_segment_count} to "
f"{normalization_result.summary.normalized_segment_count} segments",
)
_write_run_metadata(run_dir, config, normalization_summary, stage_summaries)
_write_run_metadata(run_dir, config, normalization_summary, stage_summaries, work_dir_retained=True)
if llm_client is None:
from .llm import InstructorLLMClient
@@ -95,7 +122,6 @@ def process_transcript(
protected_vocabulary=protected_vocabulary,
),
]
final_skipped: List[Tuple[str, SkippedCorrection]] = []
for stage in stages:
stage_dir = run_dir / stage.name
@@ -108,9 +134,9 @@ def process_transcript(
"passes": [],
}
stage_summaries.append(stage_summary)
_write_run_metadata(run_dir, config, normalization_summary, stage_summaries)
_write_run_metadata(run_dir, config, normalization_summary, stage_summaries, work_dir_retained=True)
working, stage_skipped = _run_correction_stage(
stage_result = _run_correction_stage(
working,
glossary,
config,
@@ -123,28 +149,69 @@ def process_transcript(
llm_client,
progress,
)
final_skipped.extend((stage.name, skipped) for skipped in stage_skipped)
_write_run_metadata(run_dir, config, normalization_summary, stage_summaries)
working = stage_result.transcript
applied_changes.extend(stage_result.applied_changes)
final_skipped.extend(stage_result.skipped_corrections)
_write_run_metadata(run_dir, config, normalization_summary, stage_summaries, work_dir_retained=True)
_write_skipped_corrections(run_dir, final_skipped)
for stage_name, skipped in final_skipped:
for skipped in final_skipped:
_log(
progress,
f"Skipping {stage_name} correction for id {skipped.id}: {skipped.reason}",
f"Skipping {skipped.stage} correction for id {skipped.id}: {skipped.reason}",
)
revised = _sort_transcript_chronologically(working)
except Exception as exc:
_write_skipped_corrections(run_dir, final_skipped)
report = _build_run_report(
config=config,
normalization_summary=normalization_summary,
stage_summaries=stage_summaries,
applied_changes=applied_changes,
skipped_corrections=final_skipped,
work_dir_retention=config.work_dir_retention,
work_dir_retained=True,
run_dir=run_dir,
transcript=working,
status="failed",
error=str(exc),
)
_write_run_report(run_dir / "report.json", report)
message = f"{exc} Diagnostics preserved at {run_dir}"
if isinstance(exc, AuditaError):
raise type(exc)(message) from exc
raise AuditaError(message) from exc
if final_skipped:
_log(progress, f"Skipped correction diagnostics preserved at {run_dir}")
work_dir_retained = _should_retain_run_dir(config.work_dir_retention, bool(final_skipped))
report = _build_run_report(
config=config,
normalization_summary=normalization_summary,
stage_summaries=stage_summaries,
applied_changes=applied_changes,
skipped_corrections=final_skipped,
work_dir_retention=config.work_dir_retention,
work_dir_retained=work_dir_retained,
run_dir=run_dir,
transcript=revised,
status="success",
error=None,
)
_write_run_metadata(run_dir, config, normalization_summary, stage_summaries, work_dir_retained=work_dir_retained)
if work_dir_retained:
_write_run_report(run_dir / "report.json", report)
if final_skipped:
_log(progress, f"Skipped correction diagnostics preserved at {run_dir}")
else:
_log(progress, f"Work directory preserved at {run_dir}")
else:
shutil.rmtree(run_dir)
_log(progress, "Removed work directory after successful run")
return revised
return ProcessResult(
transcript=revised,
report=report,
run_dir=run_dir,
work_dir_retained=work_dir_retained,
)
def _run_correction_stage(
@@ -159,10 +226,11 @@ def _run_correction_stage(
pass_summaries: List[dict],
llm_client: LLMClient,
progress: Optional[ProgressCallback],
) -> Tuple[List[TranscriptSegment], List[SkippedCorrection]]:
) -> StageRunResult:
working = list(transcript)
unresolved_retry_skips: Dict[int, SkippedCorrection] = {}
final_nonretry_skips: List[SkippedCorrection] = []
stage_applied_changes: List[AppliedChange] = []
unresolved_retry_skips: Dict[int, ReportedSkippedCorrection] = {}
final_nonretry_skips: List[ReportedSkippedCorrection] = []
for pass_number in range(1, stage.max_llm_passes + 1):
if pass_number == 1:
@@ -206,7 +274,9 @@ def _run_correction_stage(
pass_dir,
llm_client,
)
final_nonretry_skips.extend(validation_skips)
final_nonretry_skips.extend(
_reported_skip(stage.name, pass_number, skipped) for skipped in validation_skips
)
application_result = apply_corrections(
working,
@@ -216,18 +286,33 @@ def _run_correction_stage(
correction_guard=stage.correction_guard,
)
working = application_result.transcript
stage_applied_changes.extend(
AppliedChange(
stage=stage.name,
pass_number=pass_number,
id=applied.id,
original_text=applied.original_text,
corrected_text=applied.corrected_text,
confidence=applied.confidence,
segment_text_before=applied.segment_text_before,
segment_text_after=applied.segment_text_after,
)
for applied in application_result.applied_corrections
)
next_retry_skips: Dict[int, SkippedCorrection] = {}
next_retry_skips: Dict[int, ReportedSkippedCorrection] = {}
for ignored in application_result.ignored:
if _is_retryable_skip(ignored, working):
next_retry_skips[ignored.id] = ignored
reported_ignored = _reported_skip(stage.name, pass_number, ignored)
if _is_retryable_skip(reported_ignored, working):
next_retry_skips[reported_ignored.id] = reported_ignored
else:
final_nonretry_skips.append(ignored)
final_nonretry_skips.append(reported_ignored)
for skipped in application_result.skipped:
if _is_retryable_skip(skipped, working):
next_retry_skips[skipped.id] = skipped
reported_skip = _reported_skip(stage.name, pass_number, skipped)
if _is_retryable_skip(reported_skip, working):
next_retry_skips[reported_skip.id] = reported_skip
else:
final_nonretry_skips.append(skipped)
final_nonretry_skips.append(reported_skip)
unresolved_retry_skips = next_retry_skips
pass_summaries.append(
@@ -237,14 +322,14 @@ def _run_correction_stage(
"section_count": len(sections),
"segment_count": len(indexed_segments),
"corrections_returned": len(corrections),
"applied_count": len(application_result.applied_ids),
"applied_count": len(application_result.applied_corrections),
"ignored_below_threshold_count": len(application_result.ignored_ids),
"skipped_count": len(application_result.skipped),
"retry_segment_count": len(unresolved_retry_skips),
**validation_summary,
}
)
_write_run_metadata(run_dir, config, normalization_summary, stage_summaries)
_write_run_metadata(run_dir, config, normalization_summary, stage_summaries, work_dir_retained=True)
if not unresolved_retry_skips:
break
@@ -252,7 +337,11 @@ def _run_correction_stage(
final_skipped = final_nonretry_skips + [
unresolved_retry_skips[correction_id] for correction_id in sorted(unresolved_retry_skips)
]
return working, final_skipped
return StageRunResult(
transcript=working,
applied_changes=stage_applied_changes,
skipped_corrections=final_skipped,
)
def _validate_grammar_corrections(
@@ -358,23 +447,13 @@ def _write_run_metadata(
config: AuditaConfig,
normalization_summary: Optional[dict],
stage_summaries: List[dict],
work_dir_retained: bool,
) -> None:
metadata = {
"model": config.model,
"base_url": config.base_url,
"max_section_tokens": config.max_section_tokens,
"glossary_confidence_threshold": config.glossary_confidence_threshold,
"grammar_confidence_threshold": config.grammar_confidence_threshold,
"grammar_validation_enabled": config.grammar_validation_enabled,
"grammar_validation_confidence_threshold": config.grammar_validation_confidence_threshold,
"grammar_spoken_form_validation_confidence_threshold": (
config.grammar_spoken_form_validation_confidence_threshold
),
"max_retries": config.max_retries,
"glossary_max_llm_passes": config.glossary_max_llm_passes,
"grammar_max_llm_passes": config.grammar_max_llm_passes,
**_config_summary(config),
"normalization": normalization_summary,
"stages": stage_summaries,
"work_dir_retained": work_dir_retained,
}
(run_dir / "metadata.json").write_text(
json.dumps(metadata, ensure_ascii=False, indent=2) + "\n",
@@ -415,14 +494,12 @@ def _write_and_validate_section(run_dir: Path, section: TranscriptSection) -> No
parse_transcript_json(section_json, require_sequential_ids=False)
def _write_skipped_corrections(run_dir: Path, skipped: List[Tuple[str, SkippedCorrection]]) -> None:
def _write_skipped_corrections(run_dir: Path, skipped: List[ReportedSkippedCorrection]) -> None:
skipped_path = run_dir / "skipped-corrections.json"
skipped_path.write_text(
json.dumps(
{
"skipped_corrections": [
{"stage": stage_name, **item.to_dict()} for stage_name, item in skipped
]
"skipped_corrections": [item.to_dict() for item in skipped]
},
ensure_ascii=False,
indent=2,
@@ -444,7 +521,7 @@ def _indexed_segments_for_ids(
]
def _is_retryable_skip(skipped: SkippedCorrection, transcript: List[TranscriptSegment]) -> bool:
def _is_retryable_skip(skipped: ReportedSkippedCorrection, transcript: List[TranscriptSegment]) -> bool:
return any(segment.id == skipped.id for segment in transcript)
@@ -459,3 +536,89 @@ def _sort_transcript_chronologically(
def _log(progress: Optional[ProgressCallback], message: str) -> None:
if progress is not None:
progress(message)
def _reported_skip(stage: str, pass_number: int, skipped: SkippedCorrection) -> ReportedSkippedCorrection:
return ReportedSkippedCorrection(
stage=stage,
pass_number=pass_number,
id=skipped.id,
reason=skipped.reason,
original_text=skipped.original_text,
corrected_text=skipped.corrected_text,
confidence=skipped.confidence,
actual_text=skipped.actual_text,
validation_confidence=skipped.validation_confidence,
validation_reason=skipped.validation_reason,
)
def _should_retain_run_dir(work_dir_retention: str, has_final_skipped: bool) -> bool:
if work_dir_retention == "always":
return True
if work_dir_retention == "never":
return False
return has_final_skipped
def _config_summary(config: AuditaConfig) -> dict:
return {
"model": config.model,
"base_url": config.base_url,
"max_section_tokens": config.max_section_tokens,
"glossary_confidence_threshold": config.glossary_confidence_threshold,
"grammar_confidence_threshold": config.grammar_confidence_threshold,
"grammar_validation_enabled": config.grammar_validation_enabled,
"grammar_validation_confidence_threshold": config.grammar_validation_confidence_threshold,
"grammar_spoken_form_validation_confidence_threshold": (
config.grammar_spoken_form_validation_confidence_threshold
),
"max_retries": config.max_retries,
"glossary_max_llm_passes": config.glossary_max_llm_passes,
"grammar_max_llm_passes": config.grammar_max_llm_passes,
"normalize_max_segment_gap": config.normalize_max_segment_gap,
"normalize_ellipsis_gap": config.normalize_ellipsis_gap,
"normalize_max_segment_duration": config.normalize_max_segment_duration,
"normalize_max_segment_tokens": config.normalize_max_segment_tokens,
"work_dir_retention": config.work_dir_retention,
}
def _build_run_report(
config: AuditaConfig,
normalization_summary: Optional[dict],
stage_summaries: List[dict],
applied_changes: List[AppliedChange],
skipped_corrections: List[ReportedSkippedCorrection],
work_dir_retention: str,
work_dir_retained: bool,
run_dir: Path,
transcript: List[TranscriptSegment],
status: str,
error: Optional[str],
) -> RunReport:
totals = {
"output_segment_count": len(transcript),
"applied_change_count": len(applied_changes),
"skipped_correction_count": len(skipped_corrections),
}
if normalization_summary is not None:
totals["source_segment_count"] = normalization_summary["source_segment_count"]
totals["normalized_segment_count"] = normalization_summary["normalized_segment_count"]
return RunReport(
status=status,
config=_config_summary(config),
normalization=normalization_summary,
stages=stage_summaries,
applied_changes=applied_changes,
skipped_corrections=skipped_corrections,
totals=totals,
work_dir_retention=work_dir_retention,
work_dir_retained=work_dir_retained,
work_dir=str(run_dir) if work_dir_retained else None,
error=error,
)
def _write_run_report(path: Path, report: RunReport) -> None:
path.write_text(report.to_json(), encoding="utf-8")

79
src/audita/reporting.py Normal file
View File

@@ -0,0 +1,79 @@
import json
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import List, Optional
from .schemas import TranscriptSegment
@dataclass(frozen=True)
class AppliedChange:
stage: str
pass_number: int
id: int
original_text: str
corrected_text: str
confidence: float
segment_text_before: str
segment_text_after: str
def to_dict(self) -> dict:
return asdict(self)
@dataclass(frozen=True)
class ReportedSkippedCorrection:
stage: str
pass_number: int
id: int
reason: str
original_text: str
corrected_text: str
confidence: float
actual_text: Optional[str] = None
validation_confidence: Optional[float] = None
validation_reason: Optional[str] = None
def to_dict(self) -> dict:
return asdict(self)
@dataclass(frozen=True)
class RunReport:
status: str
config: dict
normalization: Optional[dict]
stages: List[dict]
applied_changes: List[AppliedChange]
skipped_corrections: List[ReportedSkippedCorrection]
totals: dict
work_dir_retention: str
work_dir_retained: bool
work_dir: Optional[str]
error: Optional[str] = None
def to_dict(self) -> dict:
return {
"status": self.status,
"config": self.config,
"normalization": self.normalization,
"stages": self.stages,
"applied_changes": [item.to_dict() for item in self.applied_changes],
"skipped_corrections": [item.to_dict() for item in self.skipped_corrections],
"totals": self.totals,
"work_dir_retention": self.work_dir_retention,
"work_dir_retained": self.work_dir_retained,
"work_dir": self.work_dir,
"error": self.error,
}
def to_json(self) -> str:
return json.dumps(self.to_dict(), ensure_ascii=False, indent=2) + "\n"
@dataclass(frozen=True)
class ProcessResult:
transcript: List[TranscriptSegment]
report: RunReport
run_dir: Path
work_dir_retained: bool

56
tests/test_cli.py
View File

@@ -1,6 +1,8 @@
import pytest
from audita.cli import main
from audita.reporting import ProcessResult, RunReport
from audita.schemas import parse_transcript_json
def test_cli_help_uses_audita_program_name(capsys):
@@ -17,6 +19,7 @@ def test_process_help_includes_glossary_pass_flag(capsys):
assert exc.value.code == 0
output = capsys.readouterr().out
assert "--report-json" in output
assert "--glossary-max-llm-passes" in output
assert "--grammar-max-llm-passes" in output
assert "--glossary-confidence-threshold" in output
@@ -24,8 +27,61 @@ def test_process_help_includes_glossary_pass_flag(capsys):
assert "--grammar-validation-enabled" in output
assert "--grammar-validation-confidence-threshold" in output
assert "--grammar-spoken-form-validation-confidence-threshold" in output
assert "--work-dir-retention" in output
assert "--normalize-max-segment-gap" in output
assert "--normalize-ellipsis-gap" in output
assert "--normalize-max-segment-duration" in output
assert "--normalize-max-segment-tokens" in output
assert "--confidence-threshold" not in output
def test_cli_process_writes_report_json(monkeypatch, tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
]
"""
)
report = RunReport(
status="success",
config={"model": "m", "base_url": "b"},
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
stages=[],
applied_changes=[],
skipped_corrections=[],
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
work_dir_retention="auto",
work_dir_retained=False,
work_dir=None,
error=None,
)
result = ProcessResult(
transcript=transcript,
report=report,
run_dir=tmp_path / "run",
work_dir_retained=False,
)
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", lambda overrides=None: object())
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
output_path = tmp_path / "out.json"
report_path = tmp_path / "report.json"
exit_code = main(
[
"process",
"transcript.json",
"--glossary",
"glossary.yaml",
"--output",
str(output_path),
"--report-json",
str(report_path),
]
)
assert exit_code == 0
assert report_path.exists()

15
tests/test_config.py
View File

@@ -18,6 +18,7 @@ from audita.config import (
DEFAULT_NORMALIZE_MAX_SEGMENT_GAP,
DEFAULT_NORMALIZE_MAX_SEGMENT_TOKENS,
DEFAULT_WORK_DIR,
DEFAULT_WORK_DIR_RETENTION,
)
from audita.errors import AuditaConfigError
@@ -50,6 +51,8 @@ def test_config_uses_defaults_with_api_key():
assert config.normalize_max_segment_tokens == DEFAULT_NORMALIZE_MAX_SEGMENT_TOKENS
assert config.normalize_max_segment_tokens == 2048
assert config.work_dir == Path(DEFAULT_WORK_DIR)
assert config.work_dir_retention == DEFAULT_WORK_DIR_RETENTION
assert config.work_dir_retention == "auto"
def test_config_env_overrides_defaults():
@@ -70,6 +73,7 @@ def test_config_env_overrides_defaults():
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0",
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "512",
"AUDITA_WORK_DIR": "/tmp/custom-audita",
"AUDITA_WORK_DIR_RETENTION": "always",
}
)
@@ -87,6 +91,7 @@ def test_config_env_overrides_defaults():
assert config.normalize_max_segment_duration == 45.0
assert config.normalize_max_segment_tokens == 512
assert config.work_dir == Path("/tmp/custom-audita")
assert config.work_dir_retention == "always"
def test_config_cli_overrides_env():
@@ -105,6 +110,7 @@ def test_config_cli_overrides_env():
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0",
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "512",
"AUDITA_WORK_DIR": "/tmp/env-audita",
"AUDITA_WORK_DIR_RETENTION": "always",
},
overrides=ConfigOverrides(
max_section_tokens=100,
@@ -121,6 +127,7 @@ def test_config_cli_overrides_env():
normalize_max_segment_duration=30.0,
normalize_max_segment_tokens=256,
work_dir=Path("/tmp/cli-audita"),
work_dir_retention="never",
),
)
@@ -138,6 +145,7 @@ def test_config_cli_overrides_env():
assert config.normalize_max_segment_duration == 30.0
assert config.normalize_max_segment_tokens == 256
assert config.work_dir == Path("/tmp/cli-audita")
assert config.work_dir_retention == "never"
def test_config_requires_api_key():
@@ -195,6 +203,13 @@ def test_config_rejects_invalid_grammar_validation_enabled():
)
def test_config_rejects_invalid_work_dir_retention():
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_WORK_DIR_RETENTION": "sometimes"}
)
def test_legacy_confidence_threshold_env_is_ignored():
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_CONFIDENCE_THRESHOLD": "0.9"}

4
tests/test_corrections.py
View File

@@ -33,6 +33,9 @@ def test_apply_corrections_uses_threshold_and_preserves_id_order():
assert [segment.speaker for segment in result.transcript] == ["Eric", "Mike"]
assert result.transcript[0].text == "I ask Chauntea for help."
assert result.skipped == []
assert len(result.applied_corrections) == 1
assert result.applied_corrections[0].segment_text_before == "I ask Chontia for help."
assert result.applied_corrections[0].segment_text_after == "I ask Chauntea for help."
def test_apply_corrections_ignores_below_threshold():
@@ -75,6 +78,7 @@ def test_apply_corrections_allows_multiple_distinct_spans_in_one_segment():
assert result.transcript[0].text == "I ask Chauntea for guidance."
assert result.skipped == []
assert len(result.applied_corrections) == 2
def test_apply_corrections_skips_missing_substring():

170
tests/test_pipeline.py
View File

@@ -4,7 +4,8 @@ import pytest
from audita.config import AuditaConfig
from audita.errors import AuditaError
from audita.pipeline import process_transcript
from audita.io import write_report
from audita.pipeline import process_transcript, process_transcript_result
from audita.schemas import (
CorrectionCandidate,
GrammarSpokenFormValidationDecision,
@@ -56,6 +57,7 @@ def _config(
grammar_validation_enabled=False,
grammar_validation_confidence_threshold=0.8,
grammar_spoken_form_validation_confidence_threshold=0.8,
work_dir_retention="auto",
):
return AuditaConfig(
api_key="key",
@@ -69,6 +71,7 @@ def _config(
grammar_validation_confidence_threshold=grammar_validation_confidence_threshold,
grammar_spoken_form_validation_confidence_threshold=grammar_spoken_form_validation_confidence_threshold,
work_dir=tmp_path / "work",
work_dir_retention=work_dir_retention,
)
@@ -713,6 +716,8 @@ def test_pipeline_writes_stage_metadata_for_unresolved_retries(tmp_path):
assert metadata["grammar_validation_enabled"] is False
assert metadata["grammar_validation_confidence_threshold"] == 0.8
assert metadata["grammar_spoken_form_validation_confidence_threshold"] == 0.8
assert metadata["work_dir_retention"] == "auto"
assert metadata["work_dir_retained"] is True
assert [item["stage"] for item in metadata["stages"]] == ["glossary", "grammar"]
assert [item["pass_number"] for item in metadata["stages"][0]["passes"]] == [1, 2]
assert metadata["stages"][0]["passes"][0]["retry_segment_count"] == 1
@@ -1387,3 +1392,166 @@ def test_unresolved_grammar_skip_preserves_diagnostics(tmp_path):
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
assert diagnostics["skipped_corrections"][0]["stage"] == "grammar"
assert "more than once" in diagnostics["skipped_corrections"][0]["reason"]
def test_process_transcript_result_returns_report_with_applied_changes(tmp_path):
correction = CorrectionCandidate(
id=1,
original_text="Chontia",
corrected_text="Chauntea",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[correction]),
CorrectionSet(corrections=[]),
]
)
result = process_transcript_result(
_transcript(),
_glossary(),
_config(tmp_path),
llm_client=fake_client,
)
assert result.transcript[0].text == "I ask Chauntea."
assert result.work_dir_retained is False
assert result.report.status == "success"
assert result.report.work_dir_retained is False
assert result.report.work_dir is None
assert result.report.totals["applied_change_count"] == 1
assert result.report.skipped_corrections == []
assert result.report.applied_changes[0].stage == "glossary"
assert result.report.applied_changes[0].pass_number == 1
assert result.report.applied_changes[0].segment_text_before == "I ask Chontia."
assert result.report.applied_changes[0].segment_text_after == "I ask Chauntea."
report_path = tmp_path / "result-report.json"
write_report(report_path, result.report)
written = json.loads(report_path.read_text(encoding="utf-8"))
assert written["totals"]["applied_change_count"] == 1
assert written["applied_changes"][0]["stage"] == "glossary"
def test_auto_retains_work_dir_on_success_with_skipped_corrections(tmp_path):
correction = CorrectionCandidate(
id=1,
original_text="Different text.",
corrected_text="Chauntea",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[correction]),
CorrectionSet(corrections=[]),
]
)
result = process_transcript_result(
_transcript(),
_glossary(),
_config(tmp_path, glossary_max_llm_passes=1, work_dir_retention="auto"),
llm_client=fake_client,
)
assert result.work_dir_retained is True
assert result.run_dir.exists()
assert (result.run_dir / "report.json").exists()
report_json = json.loads((result.run_dir / "report.json").read_text(encoding="utf-8"))
assert report_json["work_dir_retained"] is True
assert report_json["skipped_corrections"][0]["stage"] == "glossary"
def test_never_retention_removes_work_dir_after_success_even_with_skipped_corrections(tmp_path):
correction = CorrectionCandidate(
id=1,
original_text="Different text.",
corrected_text="Chauntea",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[correction]),
CorrectionSet(corrections=[]),
]
)
result = process_transcript_result(
_transcript(),
_glossary(),
_config(tmp_path, glossary_max_llm_passes=1, work_dir_retention="never"),
llm_client=fake_client,
)
assert result.work_dir_retained is False
assert not result.run_dir.exists()
assert result.report.work_dir_retained is False
assert result.report.skipped_corrections[0].stage == "glossary"
def test_always_preserves_work_dir_after_clean_success(tmp_path):
correction = CorrectionCandidate(
id=1,
original_text="Chontia",
corrected_text="Chauntea",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[correction]),
CorrectionSet(corrections=[]),
]
)
result = process_transcript_result(
_transcript(),
_glossary(),
_config(tmp_path, work_dir_retention="always"),
llm_client=fake_client,
)
assert result.work_dir_retained is True
assert result.run_dir.exists()
assert (result.run_dir / "report.json").exists()
report_json = json.loads((result.run_dir / "report.json").read_text(encoding="utf-8"))
assert report_json["work_dir_retention"] == "always"
assert report_json["skipped_corrections"] == []
def test_failure_preserves_work_dir_and_writes_failure_report(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "He became visible."}
]
"""
)
grammar_correction = CorrectionCandidate(
id=1,
original_text="visible",
corrected_text="invisible",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[grammar_correction]),
],
validation_responses=[GrammarValidationSet(validations=[])],
)
with pytest.raises(AuditaError):
process_transcript_result(
transcript,
_glossary(),
_config(tmp_path, grammar_validation_enabled=True),
llm_client=fake_client,
)
run_dirs = list((tmp_path / "work").iterdir())
assert len(run_dirs) == 1
report_json = json.loads((run_dirs[0] / "report.json").read_text(encoding="utf-8"))
assert report_json["status"] == "failed"
assert report_json["work_dir_retained"] is True
assert report_json["error"] is not None