From 3277587e3a07ea42331356bcf256476b6c5a3bfc Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 23 Apr 2026 12:08:44 -0500 Subject: [PATCH] Improvements around reporting and work directory retention options --- README.md | 11 +- src/audita/cli.py | 19 ++- src/audita/config.py | 33 +++++ src/audita/corrections.py | 26 ++++ src/audita/io.py | 5 + src/audita/pipeline.py | 255 +++++++++++++++++++++++++++++++------- src/audita/reporting.py | 79 ++++++++++++ tests/test_cli.py | 56 +++++++++ tests/test_config.py | 15 +++ tests/test_corrections.py | 4 + tests/test_pipeline.py | 170 ++++++++++++++++++++++++- 11 files changed, 620 insertions(+), 53 deletions(-) create mode 100644 src/audita/reporting.py diff --git a/README.md b/README.md index 982f772..d72a72d 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,12 @@ export OPENROUTER_API_KEY=... uv run audita process transcript.json --glossary glossary.yaml --output corrected.json ``` +To also write a structured JSON report describing what Audita applied or skipped: + +```sh +uv run audita process transcript.json --glossary glossary.yaml --output corrected.json --report-json report.json +``` + From a checked-out repository, you can also use the root launcher: ```sh @@ -36,6 +42,7 @@ audita process transcript.json --glossary glossary.yaml --output corrected.json ``` Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr. +`--report-json` writes a separate machine-readable run report and never mixes report data into stdout. Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. @@ -48,6 +55,7 @@ Useful configuration can be supplied by CLI flag or environment variable. CLI fl | `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | `--grammar-confidence-threshold` | `0.80` | Minimum confidence required to apply a grammar correction | | `AUDITA_GRAMMAR_VALIDATION_ENABLED` | `--grammar-validation-enabled` / `--no-grammar-validation-enabled` | `true` | Whether grammar corrections are checked by the semantic validator | | `AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD` | `--grammar-validation-confidence-threshold` | `0.80` | Minimum validator confidence required for validated grammar corrections | +| `AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD` | `--grammar-spoken-form-validation-confidence-threshold` | `0.80` | Minimum validator confidence required for spoken-form rescue corrections | | `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses | | `AUDITA_GLOSSARY_MAX_LLM_PASSES` | `--glossary-max-llm-passes` | `3` | Total glossary correction passes | | `AUDITA_GRAMMAR_MAX_LLM_PASSES` | `--grammar-max-llm-passes` | `3` | Total grammar/readability correction passes | @@ -56,7 +64,8 @@ Useful configuration can be supplied by CLI flag or environment variable. CLI fl | `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | `--normalize-max-segment-duration` | `60.0` | Maximum merged segment duration | | `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | `--normalize-max-segment-tokens` | `2048` | Maximum merged segment prompt payload size | | `AUDITA_WORK_DIR` | `--work-dir` | `/tmp/audita` | Per-run scratch diagnostics directory | +| `AUDITA_WORK_DIR_RETENTION` | `--work-dir-retention` | `auto` | Whether to retain the per-run work directory: `auto`, `always`, or `never` | `OPENROUTER_API_KEY` is required and is read from the environment. -`AUDITA_WORK_DIR` stores per-run diagnostics while processing. Successful runs clean up their run directory unless corrections are skipped; failed runs and skipped-correction runs preserve diagnostics for debugging. +`AUDITA_WORK_DIR` stores per-run diagnostics while processing. Under the default `AUDITA_WORK_DIR_RETENTION=auto`, clean successful runs are removed, while failed runs and successful runs with final skipped corrections are preserved. Use `always` to keep every run directory and `never` to remove successful run directories even when skips remain. diff --git a/src/audita/cli.py b/src/audita/cli.py index 469c0e2..23c25dc 100644 --- a/src/audita/cli.py +++ b/src/audita/cli.py @@ -5,8 +5,8 @@ from typing import Optional, Sequence from .config import AuditaConfig, ConfigOverrides from .errors import AuditaError -from .io import load_glossary, load_transcript, write_transcript -from .pipeline import process_transcript +from .io import load_glossary, load_transcript, write_report, write_transcript +from .pipeline import process_transcript_result from .schemas import transcript_to_json @@ -29,6 +29,7 @@ def _build_parser() -> argparse.ArgumentParser: process.add_argument("transcript", type=Path, help="path to the input transcript JSON") process.add_argument("--glossary", type=Path, required=True, help="path to the glossary YAML") process.add_argument("--output", type=Path, help="write corrected transcript JSON to this path") + process.add_argument("--report-json", type=Path, help="write structured run report JSON to this path") process.add_argument("--model", help="OpenRouter model to use") process.add_argument("--base-url", help="OpenAI-compatible API base URL") process.add_argument("--max-section-tokens", type=int, help="maximum estimated tokens per transcript section") @@ -82,6 +83,11 @@ def _build_parser() -> argparse.ArgumentParser: help="maximum estimated tokens for a merged segment prompt payload", ) process.add_argument("--work-dir", type=Path, help="directory for per-run scratch diagnostics") + process.add_argument( + "--work-dir-retention", + choices=("auto", "always", "never"), + help="whether to retain the per-run work directory", + ) return parser @@ -107,11 +113,12 @@ def _process(args: argparse.Namespace) -> int: normalize_max_segment_duration=args.normalize_max_segment_duration, normalize_max_segment_tokens=args.normalize_max_segment_tokens, work_dir=args.work_dir, + work_dir_retention=args.work_dir_retention, ) ) transcript = load_transcript(args.transcript) glossary = load_glossary(args.glossary) - revised = process_transcript( + result = process_transcript_result( transcript, glossary, config, @@ -119,9 +126,11 @@ def _process(args: argparse.Namespace) -> int: ) if args.output is not None: - write_transcript(args.output, revised) + write_transcript(args.output, result.transcript) else: - sys.stdout.write(transcript_to_json(revised)) + sys.stdout.write(transcript_to_json(result.transcript)) + if args.report_json is not None: + write_report(args.report_json, result.report) return 0 except AuditaError as exc: print(f"audita: error: {exc}", file=sys.stderr) diff --git a/src/audita/config.py b/src/audita/config.py index 9e9ba34..abbc7b7 100644 --- a/src/audita/config.py +++ b/src/audita/config.py @@ -14,6 +14,7 @@ DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD = 0.80 DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD = 0.80 DEFAULT_MAX_RETRIES = 3 DEFAULT_WORK_DIR = "/tmp/audita" +DEFAULT_WORK_DIR_RETENTION = "auto" DEFAULT_GLOSSARY_MAX_LLM_PASSES = 3 DEFAULT_GRAMMAR_MAX_LLM_PASSES = 3 DEFAULT_GRAMMAR_VALIDATION_ENABLED = True @@ -43,6 +44,7 @@ class ConfigOverrides: normalize_max_segment_duration: Optional[float] = None normalize_max_segment_tokens: Optional[int] = None work_dir: Optional[Path] = None + work_dir_retention: Optional[str] = None @dataclass(frozen=True) @@ -66,6 +68,7 @@ class AuditaConfig: normalize_max_segment_duration: float = DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION normalize_max_segment_tokens: int = DEFAULT_NORMALIZE_MAX_SEGMENT_TOKENS work_dir: Path = Path(DEFAULT_WORK_DIR) + work_dir_retention: str = DEFAULT_WORK_DIR_RETENTION @classmethod def from_sources( @@ -158,6 +161,13 @@ class AuditaConfig: "AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS", ) work_dir_value = selected.work_dir or Path(source.get("AUDITA_WORK_DIR") or DEFAULT_WORK_DIR) + work_dir_retention = _select_choice( + selected.work_dir_retention, + source.get("AUDITA_WORK_DIR_RETENTION"), + DEFAULT_WORK_DIR_RETENTION, + "AUDITA_WORK_DIR_RETENTION", + ("auto", "always", "never"), + ) config = cls( api_key=api_key, @@ -177,6 +187,7 @@ class AuditaConfig: normalize_max_segment_duration=normalize_max_segment_duration, normalize_max_segment_tokens=normalize_max_segment_tokens, work_dir=Path(work_dir_value), + work_dir_retention=work_dir_retention, ) config.validate() return config @@ -226,6 +237,8 @@ class AuditaConfig: raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION must be greater than zero.") if self.normalize_max_segment_tokens <= 0: raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS must be greater than zero.") + if self.work_dir_retention not in ("auto", "always", "never"): + raise AuditaConfigError("AUDITA_WORK_DIR_RETENTION must be one of auto, always, or never.") def _get_required_env(env: Mapping[str, str], name: str) -> str: @@ -278,3 +291,23 @@ def _select_float( return float(env_value) except ValueError as exc: raise AuditaConfigError(f"{name} must be a number.") from exc + + +def _select_choice( + cli_value: Optional[str], + env_value: Optional[str], + default: str, + name: str, + choices: tuple[str, ...], +) -> str: + if cli_value is not None: + value = cli_value + elif env_value is not None: + value = env_value + else: + return default + normalized = value.strip().casefold() + if normalized not in choices: + allowed = ", ".join(choices) + raise AuditaConfigError(f"{name} must be one of {allowed}.") + return normalized diff --git a/src/audita/corrections.py b/src/audita/corrections.py index 661d8ba..fac5985 100644 --- a/src/audita/corrections.py +++ b/src/audita/corrections.py @@ -23,10 +23,24 @@ class SkippedCorrection: return asdict(self) +@dataclass(frozen=True) +class AppliedCorrection: + id: int + original_text: str + corrected_text: str + confidence: float + segment_text_before: str + segment_text_after: str + + def to_dict(self) -> dict: + return asdict(self) + + @dataclass(frozen=True) class CorrectionApplicationResult: transcript: List[TranscriptSegment] skipped: List[SkippedCorrection] + applied_corrections: List[AppliedCorrection] applied_ids: List[int] ignored_ids: List[int] ignored: List[SkippedCorrection] @@ -47,6 +61,7 @@ def apply_corrections( revised = list(transcript) id_to_position = _id_to_position(revised) skipped: List[SkippedCorrection] = [] + applied_corrections: List[AppliedCorrection] = [] applied_ids: List[int] = [] ignored_ids: List[int] = [] ignored: List[SkippedCorrection] = [] @@ -70,11 +85,22 @@ def apply_corrections( skipped.append(_skip(correction, reason, actual_text=segment.text)) continue revised[position] = segment.model_copy(update={"text": revised_text}) + applied_corrections.append( + AppliedCorrection( + id=correction.id, + original_text=correction.original_text, + corrected_text=correction.corrected_text, + confidence=correction.confidence, + segment_text_before=segment.text, + segment_text_after=revised_text, + ) + ) applied_ids.append(correction.id) return CorrectionApplicationResult( transcript=revised, skipped=skipped, + applied_corrections=applied_corrections, applied_ids=applied_ids, ignored_ids=ignored_ids, ignored=ignored, diff --git a/src/audita/io.py b/src/audita/io.py index a740a37..44c658e 100644 --- a/src/audita/io.py +++ b/src/audita/io.py @@ -1,6 +1,7 @@ from pathlib import Path from typing import List +from .reporting import RunReport from .schemas import Glossary, SourceTranscriptSegment, TranscriptSegment from .schemas import parse_glossary_yaml, parse_source_transcript_json, transcript_to_json @@ -15,3 +16,7 @@ def load_glossary(path: Path) -> Glossary: def write_transcript(path: Path, segments: List[TranscriptSegment]) -> None: path.write_text(transcript_to_json(segments), encoding="utf-8") + + +def write_report(path: Path, report: RunReport) -> None: + path.write_text(report.to_json(), encoding="utf-8") diff --git a/src/audita/pipeline.py b/src/audita/pipeline.py index 88e9ac9..7d730de 100644 --- a/src/audita/pipeline.py +++ b/src/audita/pipeline.py @@ -14,6 +14,7 @@ from .normalization import NormalizationResult, normalize_transcript from .passes import CorrectionPass, GlossaryCorrectionPass, GrammarCorrectionPass, LLMClient from .prompts import build_grammar_spoken_form_validation_messages, build_grammar_validation_messages from .protection import ProtectedVocabulary +from .reporting import AppliedChange, ProcessResult, ReportedSkippedCorrection, RunReport from .semantic_validation import ( filter_with_meaning_preserving_validations, filter_with_spoken_form_validations, @@ -37,6 +38,13 @@ class StageSpec: protected_vocabulary: Optional[ProtectedVocabulary] = None +@dataclass(frozen=True) +class StageRunResult: + transcript: List[TranscriptSegment] + applied_changes: List[AppliedChange] + skipped_corrections: List[ReportedSkippedCorrection] + + def process_transcript( transcript: List[SourceTranscriptSegment], glossary: Glossary, @@ -44,12 +52,31 @@ def process_transcript( llm_client: Optional[LLMClient] = None, progress: Optional[ProgressCallback] = None, ) -> List[TranscriptSegment]: + return process_transcript_result( + transcript, + glossary, + config, + llm_client=llm_client, + progress=progress, + ).transcript + + +def process_transcript_result( + transcript: List[SourceTranscriptSegment], + glossary: Glossary, + config: AuditaConfig, + llm_client: Optional[LLMClient] = None, + progress: Optional[ProgressCallback] = None, +) -> ProcessResult: run_dir = _create_run_dir(config.work_dir) + stage_summaries: List[dict] = [] + normalization_summary: Optional[dict] = None + applied_changes: List[AppliedChange] = [] + final_skipped: List[ReportedSkippedCorrection] = [] + working: List[TranscriptSegment] = [] try: _log(progress, f"Created work directory {run_dir}") - stage_summaries: List[dict] = [] - normalization_summary: Optional[dict] = None - _write_run_metadata(run_dir, config, normalization_summary, stage_summaries) + _write_run_metadata(run_dir, config, normalization_summary, stage_summaries, work_dir_retained=True) normalization_result = normalize_transcript( transcript, @@ -66,7 +93,7 @@ def process_transcript( f"{normalization_result.summary.source_segment_count} to " f"{normalization_result.summary.normalized_segment_count} segments", ) - _write_run_metadata(run_dir, config, normalization_summary, stage_summaries) + _write_run_metadata(run_dir, config, normalization_summary, stage_summaries, work_dir_retained=True) if llm_client is None: from .llm import InstructorLLMClient @@ -95,7 +122,6 @@ def process_transcript( protected_vocabulary=protected_vocabulary, ), ] - final_skipped: List[Tuple[str, SkippedCorrection]] = [] for stage in stages: stage_dir = run_dir / stage.name @@ -108,9 +134,9 @@ def process_transcript( "passes": [], } stage_summaries.append(stage_summary) - _write_run_metadata(run_dir, config, normalization_summary, stage_summaries) + _write_run_metadata(run_dir, config, normalization_summary, stage_summaries, work_dir_retained=True) - working, stage_skipped = _run_correction_stage( + stage_result = _run_correction_stage( working, glossary, config, @@ -123,28 +149,69 @@ def process_transcript( llm_client, progress, ) - final_skipped.extend((stage.name, skipped) for skipped in stage_skipped) - _write_run_metadata(run_dir, config, normalization_summary, stage_summaries) + working = stage_result.transcript + applied_changes.extend(stage_result.applied_changes) + final_skipped.extend(stage_result.skipped_corrections) + _write_run_metadata(run_dir, config, normalization_summary, stage_summaries, work_dir_retained=True) _write_skipped_corrections(run_dir, final_skipped) - for stage_name, skipped in final_skipped: + for skipped in final_skipped: _log( progress, - f"Skipping {stage_name} correction for id {skipped.id}: {skipped.reason}", + f"Skipping {skipped.stage} correction for id {skipped.id}: {skipped.reason}", ) revised = _sort_transcript_chronologically(working) except Exception as exc: + _write_skipped_corrections(run_dir, final_skipped) + report = _build_run_report( + config=config, + normalization_summary=normalization_summary, + stage_summaries=stage_summaries, + applied_changes=applied_changes, + skipped_corrections=final_skipped, + work_dir_retention=config.work_dir_retention, + work_dir_retained=True, + run_dir=run_dir, + transcript=working, + status="failed", + error=str(exc), + ) + _write_run_report(run_dir / "report.json", report) message = f"{exc} Diagnostics preserved at {run_dir}" if isinstance(exc, AuditaError): raise type(exc)(message) from exc raise AuditaError(message) from exc - if final_skipped: - _log(progress, f"Skipped correction diagnostics preserved at {run_dir}") + work_dir_retained = _should_retain_run_dir(config.work_dir_retention, bool(final_skipped)) + report = _build_run_report( + config=config, + normalization_summary=normalization_summary, + stage_summaries=stage_summaries, + applied_changes=applied_changes, + skipped_corrections=final_skipped, + work_dir_retention=config.work_dir_retention, + work_dir_retained=work_dir_retained, + run_dir=run_dir, + transcript=revised, + status="success", + error=None, + ) + _write_run_metadata(run_dir, config, normalization_summary, stage_summaries, work_dir_retained=work_dir_retained) + if work_dir_retained: + _write_run_report(run_dir / "report.json", report) + if final_skipped: + _log(progress, f"Skipped correction diagnostics preserved at {run_dir}") + else: + _log(progress, f"Work directory preserved at {run_dir}") else: shutil.rmtree(run_dir) _log(progress, "Removed work directory after successful run") - return revised + return ProcessResult( + transcript=revised, + report=report, + run_dir=run_dir, + work_dir_retained=work_dir_retained, + ) def _run_correction_stage( @@ -159,10 +226,11 @@ def _run_correction_stage( pass_summaries: List[dict], llm_client: LLMClient, progress: Optional[ProgressCallback], -) -> Tuple[List[TranscriptSegment], List[SkippedCorrection]]: +) -> StageRunResult: working = list(transcript) - unresolved_retry_skips: Dict[int, SkippedCorrection] = {} - final_nonretry_skips: List[SkippedCorrection] = [] + stage_applied_changes: List[AppliedChange] = [] + unresolved_retry_skips: Dict[int, ReportedSkippedCorrection] = {} + final_nonretry_skips: List[ReportedSkippedCorrection] = [] for pass_number in range(1, stage.max_llm_passes + 1): if pass_number == 1: @@ -206,7 +274,9 @@ def _run_correction_stage( pass_dir, llm_client, ) - final_nonretry_skips.extend(validation_skips) + final_nonretry_skips.extend( + _reported_skip(stage.name, pass_number, skipped) for skipped in validation_skips + ) application_result = apply_corrections( working, @@ -216,18 +286,33 @@ def _run_correction_stage( correction_guard=stage.correction_guard, ) working = application_result.transcript + stage_applied_changes.extend( + AppliedChange( + stage=stage.name, + pass_number=pass_number, + id=applied.id, + original_text=applied.original_text, + corrected_text=applied.corrected_text, + confidence=applied.confidence, + segment_text_before=applied.segment_text_before, + segment_text_after=applied.segment_text_after, + ) + for applied in application_result.applied_corrections + ) - next_retry_skips: Dict[int, SkippedCorrection] = {} + next_retry_skips: Dict[int, ReportedSkippedCorrection] = {} for ignored in application_result.ignored: - if _is_retryable_skip(ignored, working): - next_retry_skips[ignored.id] = ignored + reported_ignored = _reported_skip(stage.name, pass_number, ignored) + if _is_retryable_skip(reported_ignored, working): + next_retry_skips[reported_ignored.id] = reported_ignored else: - final_nonretry_skips.append(ignored) + final_nonretry_skips.append(reported_ignored) for skipped in application_result.skipped: - if _is_retryable_skip(skipped, working): - next_retry_skips[skipped.id] = skipped + reported_skip = _reported_skip(stage.name, pass_number, skipped) + if _is_retryable_skip(reported_skip, working): + next_retry_skips[reported_skip.id] = reported_skip else: - final_nonretry_skips.append(skipped) + final_nonretry_skips.append(reported_skip) unresolved_retry_skips = next_retry_skips pass_summaries.append( @@ -237,14 +322,14 @@ def _run_correction_stage( "section_count": len(sections), "segment_count": len(indexed_segments), "corrections_returned": len(corrections), - "applied_count": len(application_result.applied_ids), + "applied_count": len(application_result.applied_corrections), "ignored_below_threshold_count": len(application_result.ignored_ids), "skipped_count": len(application_result.skipped), "retry_segment_count": len(unresolved_retry_skips), **validation_summary, } ) - _write_run_metadata(run_dir, config, normalization_summary, stage_summaries) + _write_run_metadata(run_dir, config, normalization_summary, stage_summaries, work_dir_retained=True) if not unresolved_retry_skips: break @@ -252,7 +337,11 @@ def _run_correction_stage( final_skipped = final_nonretry_skips + [ unresolved_retry_skips[correction_id] for correction_id in sorted(unresolved_retry_skips) ] - return working, final_skipped + return StageRunResult( + transcript=working, + applied_changes=stage_applied_changes, + skipped_corrections=final_skipped, + ) def _validate_grammar_corrections( @@ -358,23 +447,13 @@ def _write_run_metadata( config: AuditaConfig, normalization_summary: Optional[dict], stage_summaries: List[dict], + work_dir_retained: bool, ) -> None: metadata = { - "model": config.model, - "base_url": config.base_url, - "max_section_tokens": config.max_section_tokens, - "glossary_confidence_threshold": config.glossary_confidence_threshold, - "grammar_confidence_threshold": config.grammar_confidence_threshold, - "grammar_validation_enabled": config.grammar_validation_enabled, - "grammar_validation_confidence_threshold": config.grammar_validation_confidence_threshold, - "grammar_spoken_form_validation_confidence_threshold": ( - config.grammar_spoken_form_validation_confidence_threshold - ), - "max_retries": config.max_retries, - "glossary_max_llm_passes": config.glossary_max_llm_passes, - "grammar_max_llm_passes": config.grammar_max_llm_passes, + **_config_summary(config), "normalization": normalization_summary, "stages": stage_summaries, + "work_dir_retained": work_dir_retained, } (run_dir / "metadata.json").write_text( json.dumps(metadata, ensure_ascii=False, indent=2) + "\n", @@ -415,14 +494,12 @@ def _write_and_validate_section(run_dir: Path, section: TranscriptSection) -> No parse_transcript_json(section_json, require_sequential_ids=False) -def _write_skipped_corrections(run_dir: Path, skipped: List[Tuple[str, SkippedCorrection]]) -> None: +def _write_skipped_corrections(run_dir: Path, skipped: List[ReportedSkippedCorrection]) -> None: skipped_path = run_dir / "skipped-corrections.json" skipped_path.write_text( json.dumps( { - "skipped_corrections": [ - {"stage": stage_name, **item.to_dict()} for stage_name, item in skipped - ] + "skipped_corrections": [item.to_dict() for item in skipped] }, ensure_ascii=False, indent=2, @@ -444,7 +521,7 @@ def _indexed_segments_for_ids( ] -def _is_retryable_skip(skipped: SkippedCorrection, transcript: List[TranscriptSegment]) -> bool: +def _is_retryable_skip(skipped: ReportedSkippedCorrection, transcript: List[TranscriptSegment]) -> bool: return any(segment.id == skipped.id for segment in transcript) @@ -459,3 +536,89 @@ def _sort_transcript_chronologically( def _log(progress: Optional[ProgressCallback], message: str) -> None: if progress is not None: progress(message) + + +def _reported_skip(stage: str, pass_number: int, skipped: SkippedCorrection) -> ReportedSkippedCorrection: + return ReportedSkippedCorrection( + stage=stage, + pass_number=pass_number, + id=skipped.id, + reason=skipped.reason, + original_text=skipped.original_text, + corrected_text=skipped.corrected_text, + confidence=skipped.confidence, + actual_text=skipped.actual_text, + validation_confidence=skipped.validation_confidence, + validation_reason=skipped.validation_reason, + ) + + +def _should_retain_run_dir(work_dir_retention: str, has_final_skipped: bool) -> bool: + if work_dir_retention == "always": + return True + if work_dir_retention == "never": + return False + return has_final_skipped + + +def _config_summary(config: AuditaConfig) -> dict: + return { + "model": config.model, + "base_url": config.base_url, + "max_section_tokens": config.max_section_tokens, + "glossary_confidence_threshold": config.glossary_confidence_threshold, + "grammar_confidence_threshold": config.grammar_confidence_threshold, + "grammar_validation_enabled": config.grammar_validation_enabled, + "grammar_validation_confidence_threshold": config.grammar_validation_confidence_threshold, + "grammar_spoken_form_validation_confidence_threshold": ( + config.grammar_spoken_form_validation_confidence_threshold + ), + "max_retries": config.max_retries, + "glossary_max_llm_passes": config.glossary_max_llm_passes, + "grammar_max_llm_passes": config.grammar_max_llm_passes, + "normalize_max_segment_gap": config.normalize_max_segment_gap, + "normalize_ellipsis_gap": config.normalize_ellipsis_gap, + "normalize_max_segment_duration": config.normalize_max_segment_duration, + "normalize_max_segment_tokens": config.normalize_max_segment_tokens, + "work_dir_retention": config.work_dir_retention, + } + + +def _build_run_report( + config: AuditaConfig, + normalization_summary: Optional[dict], + stage_summaries: List[dict], + applied_changes: List[AppliedChange], + skipped_corrections: List[ReportedSkippedCorrection], + work_dir_retention: str, + work_dir_retained: bool, + run_dir: Path, + transcript: List[TranscriptSegment], + status: str, + error: Optional[str], +) -> RunReport: + totals = { + "output_segment_count": len(transcript), + "applied_change_count": len(applied_changes), + "skipped_correction_count": len(skipped_corrections), + } + if normalization_summary is not None: + totals["source_segment_count"] = normalization_summary["source_segment_count"] + totals["normalized_segment_count"] = normalization_summary["normalized_segment_count"] + return RunReport( + status=status, + config=_config_summary(config), + normalization=normalization_summary, + stages=stage_summaries, + applied_changes=applied_changes, + skipped_corrections=skipped_corrections, + totals=totals, + work_dir_retention=work_dir_retention, + work_dir_retained=work_dir_retained, + work_dir=str(run_dir) if work_dir_retained else None, + error=error, + ) + + +def _write_run_report(path: Path, report: RunReport) -> None: + path.write_text(report.to_json(), encoding="utf-8") diff --git a/src/audita/reporting.py b/src/audita/reporting.py new file mode 100644 index 0000000..57c1c37 --- /dev/null +++ b/src/audita/reporting.py @@ -0,0 +1,79 @@ +import json +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import List, Optional + +from .schemas import TranscriptSegment + + +@dataclass(frozen=True) +class AppliedChange: + stage: str + pass_number: int + id: int + original_text: str + corrected_text: str + confidence: float + segment_text_before: str + segment_text_after: str + + def to_dict(self) -> dict: + return asdict(self) + + +@dataclass(frozen=True) +class ReportedSkippedCorrection: + stage: str + pass_number: int + id: int + reason: str + original_text: str + corrected_text: str + confidence: float + actual_text: Optional[str] = None + validation_confidence: Optional[float] = None + validation_reason: Optional[str] = None + + def to_dict(self) -> dict: + return asdict(self) + + +@dataclass(frozen=True) +class RunReport: + status: str + config: dict + normalization: Optional[dict] + stages: List[dict] + applied_changes: List[AppliedChange] + skipped_corrections: List[ReportedSkippedCorrection] + totals: dict + work_dir_retention: str + work_dir_retained: bool + work_dir: Optional[str] + error: Optional[str] = None + + def to_dict(self) -> dict: + return { + "status": self.status, + "config": self.config, + "normalization": self.normalization, + "stages": self.stages, + "applied_changes": [item.to_dict() for item in self.applied_changes], + "skipped_corrections": [item.to_dict() for item in self.skipped_corrections], + "totals": self.totals, + "work_dir_retention": self.work_dir_retention, + "work_dir_retained": self.work_dir_retained, + "work_dir": self.work_dir, + "error": self.error, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), ensure_ascii=False, indent=2) + "\n" + + +@dataclass(frozen=True) +class ProcessResult: + transcript: List[TranscriptSegment] + report: RunReport + run_dir: Path + work_dir_retained: bool diff --git a/tests/test_cli.py b/tests/test_cli.py index ab53515..b48ad14 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,8 @@ import pytest from audita.cli import main +from audita.reporting import ProcessResult, RunReport +from audita.schemas import parse_transcript_json def test_cli_help_uses_audita_program_name(capsys): @@ -17,6 +19,7 @@ def test_process_help_includes_glossary_pass_flag(capsys): assert exc.value.code == 0 output = capsys.readouterr().out + assert "--report-json" in output assert "--glossary-max-llm-passes" in output assert "--grammar-max-llm-passes" in output assert "--glossary-confidence-threshold" in output @@ -24,8 +27,61 @@ def test_process_help_includes_glossary_pass_flag(capsys): assert "--grammar-validation-enabled" in output assert "--grammar-validation-confidence-threshold" in output assert "--grammar-spoken-form-validation-confidence-threshold" in output + assert "--work-dir-retention" in output assert "--normalize-max-segment-gap" in output assert "--normalize-ellipsis-gap" in output assert "--normalize-max-segment-duration" in output assert "--normalize-max-segment-tokens" in output assert "--confidence-threshold" not in output + + +def test_cli_process_writes_report_json(monkeypatch, tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."} + ] + """ + ) + report = RunReport( + status="success", + config={"model": "m", "base_url": "b"}, + normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0}, + stages=[], + applied_changes=[], + skipped_corrections=[], + totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0}, + work_dir_retention="auto", + work_dir_retained=False, + work_dir=None, + error=None, + ) + result = ProcessResult( + transcript=transcript, + report=report, + run_dir=tmp_path / "run", + work_dir_retained=False, + ) + + monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", lambda overrides=None: object()) + monkeypatch.setattr("audita.cli.load_transcript", lambda path: []) + monkeypatch.setattr("audita.cli.load_glossary", lambda path: object()) + monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result) + + output_path = tmp_path / "out.json" + report_path = tmp_path / "report.json" + exit_code = main( + [ + "process", + "transcript.json", + "--glossary", + "glossary.yaml", + "--output", + str(output_path), + "--report-json", + str(report_path), + ] + ) + + assert exit_code == 0 + assert report_path.exists() diff --git a/tests/test_config.py b/tests/test_config.py index 988739b..cae6933 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -18,6 +18,7 @@ from audita.config import ( DEFAULT_NORMALIZE_MAX_SEGMENT_GAP, DEFAULT_NORMALIZE_MAX_SEGMENT_TOKENS, DEFAULT_WORK_DIR, + DEFAULT_WORK_DIR_RETENTION, ) from audita.errors import AuditaConfigError @@ -50,6 +51,8 @@ def test_config_uses_defaults_with_api_key(): assert config.normalize_max_segment_tokens == DEFAULT_NORMALIZE_MAX_SEGMENT_TOKENS assert config.normalize_max_segment_tokens == 2048 assert config.work_dir == Path(DEFAULT_WORK_DIR) + assert config.work_dir_retention == DEFAULT_WORK_DIR_RETENTION + assert config.work_dir_retention == "auto" def test_config_env_overrides_defaults(): @@ -70,6 +73,7 @@ def test_config_env_overrides_defaults(): "AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0", "AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "512", "AUDITA_WORK_DIR": "/tmp/custom-audita", + "AUDITA_WORK_DIR_RETENTION": "always", } ) @@ -87,6 +91,7 @@ def test_config_env_overrides_defaults(): assert config.normalize_max_segment_duration == 45.0 assert config.normalize_max_segment_tokens == 512 assert config.work_dir == Path("/tmp/custom-audita") + assert config.work_dir_retention == "always" def test_config_cli_overrides_env(): @@ -105,6 +110,7 @@ def test_config_cli_overrides_env(): "AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0", "AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "512", "AUDITA_WORK_DIR": "/tmp/env-audita", + "AUDITA_WORK_DIR_RETENTION": "always", }, overrides=ConfigOverrides( max_section_tokens=100, @@ -121,6 +127,7 @@ def test_config_cli_overrides_env(): normalize_max_segment_duration=30.0, normalize_max_segment_tokens=256, work_dir=Path("/tmp/cli-audita"), + work_dir_retention="never", ), ) @@ -138,6 +145,7 @@ def test_config_cli_overrides_env(): assert config.normalize_max_segment_duration == 30.0 assert config.normalize_max_segment_tokens == 256 assert config.work_dir == Path("/tmp/cli-audita") + assert config.work_dir_retention == "never" def test_config_requires_api_key(): @@ -195,6 +203,13 @@ def test_config_rejects_invalid_grammar_validation_enabled(): ) +def test_config_rejects_invalid_work_dir_retention(): + with pytest.raises(AuditaConfigError): + AuditaConfig.from_sources( + env={"OPENROUTER_API_KEY": "key", "AUDITA_WORK_DIR_RETENTION": "sometimes"} + ) + + def test_legacy_confidence_threshold_env_is_ignored(): config = AuditaConfig.from_sources( env={"OPENROUTER_API_KEY": "key", "AUDITA_CONFIDENCE_THRESHOLD": "0.9"} diff --git a/tests/test_corrections.py b/tests/test_corrections.py index 8912e9d..f43a17e 100644 --- a/tests/test_corrections.py +++ b/tests/test_corrections.py @@ -33,6 +33,9 @@ def test_apply_corrections_uses_threshold_and_preserves_id_order(): assert [segment.speaker for segment in result.transcript] == ["Eric", "Mike"] assert result.transcript[0].text == "I ask Chauntea for help." assert result.skipped == [] + assert len(result.applied_corrections) == 1 + assert result.applied_corrections[0].segment_text_before == "I ask Chontia for help." + assert result.applied_corrections[0].segment_text_after == "I ask Chauntea for help." def test_apply_corrections_ignores_below_threshold(): @@ -75,6 +78,7 @@ def test_apply_corrections_allows_multiple_distinct_spans_in_one_segment(): assert result.transcript[0].text == "I ask Chauntea for guidance." assert result.skipped == [] + assert len(result.applied_corrections) == 2 def test_apply_corrections_skips_missing_substring(): diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index acddacd..e16c64a 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -4,7 +4,8 @@ import pytest from audita.config import AuditaConfig from audita.errors import AuditaError -from audita.pipeline import process_transcript +from audita.io import write_report +from audita.pipeline import process_transcript, process_transcript_result from audita.schemas import ( CorrectionCandidate, GrammarSpokenFormValidationDecision, @@ -56,6 +57,7 @@ def _config( grammar_validation_enabled=False, grammar_validation_confidence_threshold=0.8, grammar_spoken_form_validation_confidence_threshold=0.8, + work_dir_retention="auto", ): return AuditaConfig( api_key="key", @@ -69,6 +71,7 @@ def _config( grammar_validation_confidence_threshold=grammar_validation_confidence_threshold, grammar_spoken_form_validation_confidence_threshold=grammar_spoken_form_validation_confidence_threshold, work_dir=tmp_path / "work", + work_dir_retention=work_dir_retention, ) @@ -713,6 +716,8 @@ def test_pipeline_writes_stage_metadata_for_unresolved_retries(tmp_path): assert metadata["grammar_validation_enabled"] is False assert metadata["grammar_validation_confidence_threshold"] == 0.8 assert metadata["grammar_spoken_form_validation_confidence_threshold"] == 0.8 + assert metadata["work_dir_retention"] == "auto" + assert metadata["work_dir_retained"] is True assert [item["stage"] for item in metadata["stages"]] == ["glossary", "grammar"] assert [item["pass_number"] for item in metadata["stages"][0]["passes"]] == [1, 2] assert metadata["stages"][0]["passes"][0]["retry_segment_count"] == 1 @@ -1387,3 +1392,166 @@ def test_unresolved_grammar_skip_preserves_diagnostics(tmp_path): diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8")) assert diagnostics["skipped_corrections"][0]["stage"] == "grammar" assert "more than once" in diagnostics["skipped_corrections"][0]["reason"] + + +def test_process_transcript_result_returns_report_with_applied_changes(tmp_path): + correction = CorrectionCandidate( + id=1, + original_text="Chontia", + corrected_text="Chauntea", + confidence=0.95, + ) + fake_client = FakeLLMClient( + [ + CorrectionSet(corrections=[correction]), + CorrectionSet(corrections=[]), + ] + ) + + result = process_transcript_result( + _transcript(), + _glossary(), + _config(tmp_path), + llm_client=fake_client, + ) + + assert result.transcript[0].text == "I ask Chauntea." + assert result.work_dir_retained is False + assert result.report.status == "success" + assert result.report.work_dir_retained is False + assert result.report.work_dir is None + assert result.report.totals["applied_change_count"] == 1 + assert result.report.skipped_corrections == [] + assert result.report.applied_changes[0].stage == "glossary" + assert result.report.applied_changes[0].pass_number == 1 + assert result.report.applied_changes[0].segment_text_before == "I ask Chontia." + assert result.report.applied_changes[0].segment_text_after == "I ask Chauntea." + + report_path = tmp_path / "result-report.json" + write_report(report_path, result.report) + written = json.loads(report_path.read_text(encoding="utf-8")) + assert written["totals"]["applied_change_count"] == 1 + assert written["applied_changes"][0]["stage"] == "glossary" + + +def test_auto_retains_work_dir_on_success_with_skipped_corrections(tmp_path): + correction = CorrectionCandidate( + id=1, + original_text="Different text.", + corrected_text="Chauntea", + confidence=0.95, + ) + fake_client = FakeLLMClient( + [ + CorrectionSet(corrections=[correction]), + CorrectionSet(corrections=[]), + ] + ) + + result = process_transcript_result( + _transcript(), + _glossary(), + _config(tmp_path, glossary_max_llm_passes=1, work_dir_retention="auto"), + llm_client=fake_client, + ) + + assert result.work_dir_retained is True + assert result.run_dir.exists() + assert (result.run_dir / "report.json").exists() + report_json = json.loads((result.run_dir / "report.json").read_text(encoding="utf-8")) + assert report_json["work_dir_retained"] is True + assert report_json["skipped_corrections"][0]["stage"] == "glossary" + + +def test_never_retention_removes_work_dir_after_success_even_with_skipped_corrections(tmp_path): + correction = CorrectionCandidate( + id=1, + original_text="Different text.", + corrected_text="Chauntea", + confidence=0.95, + ) + fake_client = FakeLLMClient( + [ + CorrectionSet(corrections=[correction]), + CorrectionSet(corrections=[]), + ] + ) + + result = process_transcript_result( + _transcript(), + _glossary(), + _config(tmp_path, glossary_max_llm_passes=1, work_dir_retention="never"), + llm_client=fake_client, + ) + + assert result.work_dir_retained is False + assert not result.run_dir.exists() + assert result.report.work_dir_retained is False + assert result.report.skipped_corrections[0].stage == "glossary" + + +def test_always_preserves_work_dir_after_clean_success(tmp_path): + correction = CorrectionCandidate( + id=1, + original_text="Chontia", + corrected_text="Chauntea", + confidence=0.95, + ) + fake_client = FakeLLMClient( + [ + CorrectionSet(corrections=[correction]), + CorrectionSet(corrections=[]), + ] + ) + + result = process_transcript_result( + _transcript(), + _glossary(), + _config(tmp_path, work_dir_retention="always"), + llm_client=fake_client, + ) + + assert result.work_dir_retained is True + assert result.run_dir.exists() + assert (result.run_dir / "report.json").exists() + report_json = json.loads((result.run_dir / "report.json").read_text(encoding="utf-8")) + assert report_json["work_dir_retention"] == "always" + assert report_json["skipped_corrections"] == [] + + +def test_failure_preserves_work_dir_and_writes_failure_report(tmp_path): + transcript = parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "He became visible."} + ] + """ + ) + grammar_correction = CorrectionCandidate( + id=1, + original_text="visible", + corrected_text="invisible", + confidence=0.95, + ) + fake_client = FakeLLMClient( + [ + CorrectionSet(corrections=[]), + CorrectionSet(corrections=[grammar_correction]), + ], + validation_responses=[GrammarValidationSet(validations=[])], + ) + + with pytest.raises(AuditaError): + process_transcript_result( + transcript, + _glossary(), + _config(tmp_path, grammar_validation_enabled=True), + llm_client=fake_client, + ) + + run_dirs = list((tmp_path / "work").iterdir()) + assert len(run_dirs) == 1 + report_json = json.loads((run_dirs[0] / "report.json").read_text(encoding="utf-8")) + assert report_json["status"] == "failed" + assert report_json["work_dir_retained"] is True + assert report_json["error"] is not None