From 2b2a3fc0245be408e1ff4ef9d8d5564599b9ba40 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Wed, 13 May 2026 20:13:33 -0500 Subject: [PATCH] Removed the old python code and cleaned up outdated documentation --- docs/python/migration-from-python.md | 84 - docs/python/python-parity.md | 78 - docs/python/rewrite-notes.md | 770 --------- docs/roadmap.md | 1059 ------------ docs/validator-refactor.md | 688 -------- python/README.md | 157 -- python/audita | 147 -- python/pyproject.toml | 35 - python/src/audita/__init__.py | 5 - python/src/audita/__main__.py | 5 - python/src/audita/cli.py | 253 --- python/src/audita/core/__init__.py | 1 - python/src/audita/core/chunking.py | 356 ---- python/src/audita/core/config.py | 506 ------ python/src/audita/core/diagnostics.py | 148 -- python/src/audita/core/errors.py | 15 - python/src/audita/core/io.py | 22 - python/src/audita/core/normalization.py | 184 --- python/src/audita/core/reporting.py | 122 -- python/src/audita/core/schemas.py | 252 --- python/src/audita/framework/__init__.py | 1 - python/src/audita/framework/llm.py | 108 -- python/src/audita/framework/llm_scheduler.py | 17 - python/src/audita/framework/models.py | 80 - .../audita/framework/proposal_generation.py | 96 -- python/src/audita/framework/proposals.py | 82 - python/src/audita/framework/runner.py | 485 ------ python/src/audita/modules/__init__.py | 80 - python/src/audita/modules/glossary.py | 43 - python/src/audita/modules/grammar.py | 43 - python/src/audita/modules/homophones.py | 43 - python/src/audita/modules/prompts.py | 169 -- python/src/audita/modules/spoken_word.py | 43 - python/src/audita/pipeline.py | 258 --- python/src/audita/py.typed | 1 - python/src/audita/validators/__init__.py | 30 - python/src/audita/validators/base.py | 47 - python/src/audita/validators/deterministic.py | 156 -- python/src/audita/validators/llm.py | 195 --- python/src/audita/validators/prompts.py | 96 -- python/src/audita/validators/protection.py | 136 -- python/tests/__init__.py | 1 - python/tests/test_framework_chunking.py | 206 --- python/tests/test_framework_llm.py | 255 --- python/tests/test_framework_runner.py | 642 -------- python/tests/test_llm_validators.py | 1441 ----------------- python/tests/test_module_proposals.py | 1173 -------------- python/tests/test_new_cli.py | 659 -------- python/tests/test_new_config.py | 435 ----- python/tests/test_new_launcher.py | 117 -- python/tests/test_new_pipeline.py | 698 -------- python/tests/test_new_transcript_schema.py | 84 - python/tests/test_protected_validator.py | 320 ---- 53 files changed, 13127 deletions(-) delete mode 100644 docs/python/migration-from-python.md delete mode 100644 docs/python/python-parity.md delete mode 100644 docs/python/rewrite-notes.md delete mode 100644 docs/roadmap.md delete mode 100644 docs/validator-refactor.md delete mode 100644 python/README.md delete mode 100755 python/audita delete mode 100644 python/pyproject.toml delete mode 100644 python/src/audita/__init__.py delete mode 100644 python/src/audita/__main__.py delete mode 100644 python/src/audita/cli.py delete mode 100644 python/src/audita/core/__init__.py delete mode 100644 python/src/audita/core/chunking.py delete mode 100644 python/src/audita/core/config.py delete mode 100644 python/src/audita/core/diagnostics.py delete mode 100644 python/src/audita/core/errors.py delete mode 100644 python/src/audita/core/io.py delete mode 100644 python/src/audita/core/normalization.py delete mode 100644 python/src/audita/core/reporting.py delete mode 100644 python/src/audita/core/schemas.py delete mode 100644 python/src/audita/framework/__init__.py delete mode 100644 python/src/audita/framework/llm.py delete mode 100644 python/src/audita/framework/llm_scheduler.py delete mode 100644 python/src/audita/framework/models.py delete mode 100644 python/src/audita/framework/proposal_generation.py delete mode 100644 python/src/audita/framework/proposals.py delete mode 100644 python/src/audita/framework/runner.py delete mode 100644 python/src/audita/modules/__init__.py delete mode 100644 python/src/audita/modules/glossary.py delete mode 100644 python/src/audita/modules/grammar.py delete mode 100644 python/src/audita/modules/homophones.py delete mode 100644 python/src/audita/modules/prompts.py delete mode 100644 python/src/audita/modules/spoken_word.py delete mode 100644 python/src/audita/pipeline.py delete mode 100644 python/src/audita/py.typed delete mode 100644 python/src/audita/validators/__init__.py delete mode 100644 python/src/audita/validators/base.py delete mode 100644 python/src/audita/validators/deterministic.py delete mode 100644 python/src/audita/validators/llm.py delete mode 100644 python/src/audita/validators/prompts.py delete mode 100644 python/src/audita/validators/protection.py delete mode 100644 python/tests/__init__.py delete mode 100644 python/tests/test_framework_chunking.py delete mode 100644 python/tests/test_framework_llm.py delete mode 100644 python/tests/test_framework_runner.py delete mode 100644 python/tests/test_llm_validators.py delete mode 100644 python/tests/test_module_proposals.py delete mode 100644 python/tests/test_new_cli.py delete mode 100644 python/tests/test_new_config.py delete mode 100644 python/tests/test_new_launcher.py delete mode 100644 python/tests/test_new_pipeline.py delete mode 100644 python/tests/test_new_transcript_schema.py delete mode 100644 python/tests/test_protected_validator.py diff --git a/docs/python/migration-from-python.md b/docs/python/migration-from-python.md deleted file mode 100644 index 63719df..0000000 --- a/docs/python/migration-from-python.md +++ /dev/null @@ -1,84 +0,0 @@ -# Migrating from Python Audita to Go Audita - -This guide describes the operational migration from the legacy Python Audita implementation to the Go Audita implementation in this repository. - -## Status summary - -- The Go CLI is now the primary Audita implementation. -- The Go `audita process` command is intended to replace the Python CLI for normal operation. -- The Python implementation under [`python/`](../python/) is preserved as a legacy/reference implementation for parity history and troubleshooting context. - -## What changes for operators - -Use the Go binary as the integration target in orchestrators and parent processes. - -Default Go runtime behavior executes the full module sequence: -1. `glossary` -2. `homophones` -3. `glossary` -4. `spoken_word` -5. `grammar` - -Repeated stages are resolved deterministically in reports (for example `glossary_1`, `glossary_2`). - -## Recommended invocation pattern - -For orchestrated runs, use explicit output files: - -```sh -audita process \ - --glossary \ - --output \ - --report-json -``` - -Why: -- `--output` keeps stdout empty on success, simplifying subprocess integration. -- `--report-json` provides machine-readable run metadata independent of stderr. - -Without `--output`, stdout contains transcript JSON only on successful runs. - -## Diagnostics and retention behavior - -- Each run creates a diagnostics run directory when initialization succeeds. -- Failed runs retain diagnostics and include `error.log`. -- Retention mode is controlled by `AUDITA_WORK_DIR_RETENTION` / `--work-dir-retention`: - - `always`: keep all run directories. - - `never`: keep successful run directories. - - `auto`: keep failed runs and successful runs with skipped/rejected corrections. - -For subprocess behavior and pipe-handling guidance, see: -- [`docs/subprocess-operations.md`](subprocess-operations.md) - -## Parity notes and known differences - -Python-vs-Go parity fixtures and intentional differences are documented in: -- [`docs/python-parity.md`](python-parity.md) - -Known open parity gaps are tracked there and should be treated as real gaps, not intentional differences. Current documented gaps include: -- broader direct import/use of Python fixture corpus; -- a repository-standard Python+Go side-by-side runner command; -- wider transcript/glossary corpus coverage. - -## Testing expectations - -- Normal `go test ./...` does not require real LLM credentials. -- Normal `go test ./...` does not require Python dependencies. -- Deterministic fake-LLM fixtures are used for routine CI-friendly testing. - -## Legacy Python status - -The Python implementation remains in-repo as a legacy/reference baseline. It is not the primary operational path. - -Do not route new production orchestration to Python unless you are doing explicit parity/debug work. - -## Rollout checklist - -Use this checklist when switching an environment from Python invocation to Go invocation: - -1. Run `go test ./...`. -2. Build the Go binary (`go build -o ./bin/audita ./cmd/audita`). -3. Run one representative fixture through `audita process`. -4. Verify `--report-json` output is written and machine-readable. -5. Verify failure runs print diagnostics path to stderr and retain diagnostics with `error.log`. -6. Update orchestrator configuration to call the Go binary and pass `--output` and `--report-json`. diff --git a/docs/python/python-parity.md b/docs/python/python-parity.md deleted file mode 100644 index f501677..0000000 --- a/docs/python/python-parity.md +++ /dev/null @@ -1,78 +0,0 @@ -# Python vs Go Parity Notes - -This document tracks parity-fixture coverage and intentional differences between historical Python behavior and the current Audita runtime. - -## Scope - -- Uses deterministic fixture-driven tests in `internal/cli/testdata/parity`. -- Uses fake structured LLM responses for proposal generation and LLM validators. -- Verifies functional contract fields (module order, instance naming, applied/skipped counts, report status, diagnostics presence). -- Does not require real LLM credentials or Python dependencies during `go test ./...`. - -## Intentional Differences - -The following differences are expected and treated as intentional unless they break contract behavior: - -1. JSON formatting and field ordering -- Serialized JSON whitespace and object key order may differ. -- Parity tests compare JSON semantically, not byte-for-byte. - -2. Diagnostics path values -- Absolute run-directory paths, run IDs, and temp directory roots differ by runtime and platform. -- Parity checks assert artifact presence/shape, not exact absolute paths. - -3. Time-variant metadata -- Timestamps (`started_at`, `completed_at`) and generated run IDs are runtime-specific. -- Parity checks ignore exact timestamp/run-id values. - -4. Provider metadata/token accounting -- Provider/token usage metadata may vary by adapter behavior and is not asserted as strict parity fields. - -5. Internal adapter implementation details -- Go uses its own structured LLM adapter implementation details while preserving the same high-level contract semantics. - -## Current Fixture Coverage - -Current parity fixtures cover: - -- Transcript schema handling failure path. -- Glossary schema handling failure path. -- Default full module sequence shape: - - `glossary_1`, `homophones`, `glossary_2`, `spoken_word`, `grammar`. -- Mutable transcript handoff across default stages. -- Module-specific behavior inside the default sequence: - - glossary correction - - homophone-style correction - - spoken-word cleanup - - grammar cleanup -- Protected glossary-term guardrail behavior. -- Deterministic validator rejection behavior. -- LLM validator decision/rejection behavior. -- Application-level skip behavior (`ambiguous_original_text`). -- Mid-pipeline failure with partial progress preserved in reports. -- Diagnostics artifact presence and secret-redaction checks. - -## Open Parity Gaps (Not Intentional) - -These are known parity expansion opportunities and should not be labeled as intentional compatibility differences: - -1. Broader Python fixture import -- The current Go parity fixtures are native fixture cases; they do not yet ingest all existing Python test fixtures directly. - -2. Side-by-side runner command -- No repository-standard Python+Go side-by-side parity command is required or enforced yet. - -3. Wider transcript corpus -- Current fixtures are representative but not exhaustive across all transcript/glossary edge combinations. - -## How To Extend - -1. Add a new `*.case.json` file under `internal/cli/testdata/parity`. -2. Add referenced transcript/glossary/fake-LLM response files. -3. Encode deterministic expectations in the case: -- module order and instance names -- transcript output -- applied/skipped/rejected counts -- report status and failure metadata -- diagnostics artifact presence/redaction markers -4. Run `go test ./...`. diff --git a/docs/python/rewrite-notes.md b/docs/python/rewrite-notes.md deleted file mode 100644 index 64fe2b8..0000000 --- a/docs/python/rewrite-notes.md +++ /dev/null @@ -1,770 +0,0 @@ -# Audita Rewrite Notes (Historical) - -This document is a historical record of the Python-to-Go rewrite project. -It is retained for engineering context, not as primary runtime guidance. - -## Definition of done for the Go rewrite - -The Go rewrite is complete when both of the following are true: - -1. Feature parity with the initial Python implementation: - - `audita process` performs end-to-end transcript polishing, not only deterministic preprocessing. - - The default module sequence is implemented and active in the runtime path: - - `glossary` - - `homophones` - - `glossary` - - `spoken_word` - - `grammar` - - Repeated module instances are resolved deterministically, for example `glossary_1` and `glossary_2`. - - Real LLM-backed proposal generation is implemented through an OpenAI-compatible structured-output client. - - Deterministic validators and LLM-backed validators are implemented and enforced in the runtime path. - - Proposal application preserves the safety-first semantics of the Python implementation. - - Structured run reports and diagnostics are sufficient for debugging successful runs, skipped corrections, and failures. - - The Go CLI can replace the Python CLI in the surrounding orchestration pipeline without downstream contract surprises. - -2. Adherence to the intended application architecture in `docs/architecture.md`: - - Sequential module pipeline over a mutable working transcript. - - Bounded intra-module concurrency only. - - Provider-neutral LLM abstraction. - - Separate proposal and validation LLM settings. - - Prompt/response diagnostics for LLM stages. - - Module-level run reports with applied and skipped changes. - - Strict stdout/stderr discipline for subprocess callers. - - No hidden service dependency; the CLI remains the primary integration surface. - -## Final status (Phase 19 complete) - -The Go rewrite definition of done is met, with documented parity caveats tracked in `docs/python-parity.md`. - -- Phase 19 is complete. -- Go Audita is the active implementation. -- The default full module pipeline is implemented (`glossary,homophones,glossary,spoken_word,grammar`). -- Parity fixtures and operational hardening coverage exist in normal `go test ./...`. -- Python is retained as a legacy/reference implementation and is not the primary operational path. - -## Current implementation status - -The Go rewrite is complete and feature-complete for the intended CLI runtime architecture. - -Implemented: -- CLI command surface for `audita process`. -- Config/env/flag loading and validation. -- Transcript/glossary schema parsing and validation. -- Deterministic normalization with summary stats. -- Deterministic chunking with summary stats. -- Per-run diagnostics directory plus source/normalization/chunking artifacts. -- Redacted invocation/effective-config diagnostics metadata artifacts. -- Process report output through `--report-json` and run-dir `report.json`. -- Report-level diagnostics artifact references. -- Framework foundation packages for contracts and proposal preview/apply semantics. -- Production runner orchestration over a mutable working transcript. -- Module-level report structures and run-level module summaries. -- CLI runner integration point via injectable module factory/registry (used by deterministic tests). -- Runtime validator models and deterministic validator implementations. -- Validator cardinality enforcement (missing/duplicate/unknown proposal index errors). -- Deterministic validator-chain execution in the production runner. -- Module reports including validator decisions and validator rejections. -- LLM-backed validator request/response models and prompt builders. -- LLM validator batching by validation prompt-token budget. -- LLM validator runtime integration through structured LLM client abstraction and scheduler hooks. -- LLM validator prompt/response diagnostics artifact wiring with secret redaction. -- Shared LLM proposal-generation helper with structured correction-set parsing. -- Deterministic proposal-index assignment for shared proposal generation. -- Proposal-generation prompt/response diagnostics artifact wiring with secret redaction. -- Production module-registry scaffolding with known key recognition and explicit unsupported/unimplemented errors. -- Production grammar module package with Python-aligned prompt intent and guardrails. -- Explicit `--modules grammar` runtime path through runner, shared proposal generation, validators, application, reporting, and diagnostics. -- Production glossary module package with Python-aligned prompt intent and guardrails. -- Glossary-derived deterministic protected-term extraction and validator integration. -- Explicit `--modules glossary` runtime path through runner, shared proposal generation, validators, application, reporting, and diagnostics. -- Repeated glossary stage support with deterministic instance names (`glossary_1`, `glossary_2`), including mutable working-transcript handoff. -- Production homophones module package with Python-aligned prompt intent and guardrails. -- Explicit `--modules homophones` runtime path through runner, shared proposal generation, validators, application, reporting, and diagnostics. -- Focused multi-module runtime tests for already-implemented interoperability (for example `glossary,homophones`) without claiming full default-pipeline completion. -- Production spoken_word module package with Python-aligned prompt intent and guardrails. -- Explicit `--modules spoken_word` runtime path through runner, shared proposal generation, validators, application, reporting, and diagnostics. -- Focused multi-module runtime tests for already-implemented interoperability (for example `spoken_word,grammar`) without claiming full default-pipeline completion. -- All production modules now exist (`glossary`, `homophones`, `spoken_word`, `grammar`) and are integrated into the default full-sequence runtime path. -- Default runtime sequence is now active and ordered as: - - `glossary` - - `homophones` - - `glossary` - - `spoken_word` - - `grammar` -- Repeated glossary stages resolve and report deterministically as `glossary_1` and `glossary_2`. -- Full-pipeline module reports and run-level summaries aggregate applied/skipped/failed metadata across all module instances. -- Mid-pipeline failure reporting preserves partial progress and failed-module metadata. -- Full-pipeline diagnostics include proposal/validator prompt-response artifacts with redaction. -- Skip-aware retention uses actual module skipped/rejected correction data. -- Broad deterministic and CLI/subprocess test coverage for implemented phases through `go test ./...`. -- Internal typed structured LLM contract (`StructuredLLMClient.CompleteStructured(ctx, req, out)`). -- `internal/framework/llm` instructor-go-backed adapter with: - - configurable base URL/model/retries/mode/timeout - - optional API key support for local-compatible endpoints - - API-key redaction in returned errors - - typed structured decode into caller-provided outputs. -- LLM scheduler/semaphore infrastructure for bounded concurrency with context-aware acquisition and reliable release. -- LLM effective-config resolution helpers: - - primary config resolution - - validation config inheritance from primary when validation fields are unset - - validation override behavior when validation fields are set. -- Generic JSON diagnostics primitives for LLM interactions (request metadata, request payload, response payload, optional error payload) with secret redaction. - -Phase 19 documentation and rollout items are implemented: -- Go-first README/build/test/usage guidance. -- subprocess/orchestrator operational guidance. -- Python-to-Go migration guidance. -- explicit legacy/reference Python status documentation. - -## Completed phases - -### Phase 1: Go CLI skeleton - -Completed. - -Implemented: -- Go module and `cmd/audita` entrypoint. -- `audita process` command surface. -- Core flags and config wiring. -- Subprocess-safe command behavior foundation. - -### Phase 2: Schemas and file I/O - -Completed. - -Implemented: -- Transcript parsing for bare arrays and `{ "segments": [...] }` input. -- Glossary YAML parsing. -- Transcript and glossary validation. -- Canonical transcript output serialization. - -### Phase 3: Deterministic normalization - -Completed. - -Implemented: -- Chronological sorting. -- Same-speaker segment merging. -- Gap-sensitive join behavior. -- Duration and token-budget merge constraints. -- Sequential normalized segment IDs. -- Normalization summary stats. - -### Phase 4: Chunking and token estimation - -Completed. - -Implemented: -- Deterministic heuristic token estimation. -- Contiguous transcript sectioning. -- Min/max section token behavior. -- Optional target section handling. -- Chunking summaries and diagnostics artifacts. - -### Phase 5: Proposal model and application semantics - -Completed. - -Implemented: -- Correction proposal and enriched proposal models. -- Replacement policies: - - `require_unique` - - `replace_all` -- Safe preview logic. -- Deterministic proposal application. -- Applied/skipped change records. -- Stable skip reasons. - -### Phase 6: Reports and diagnostics - -Completed for the current deterministic runtime scope. - -Implemented: -- Per-run diagnostics directory creation. -- Source transcript artifacts. -- Parsed source transcript artifacts. -- Normalized transcript artifact. -- Normalization summary artifact. -- Chunking summary artifact. -- Redacted invocation metadata artifact. -- Redacted effective-config artifact. -- Run-dir `report.json`. -- Optional external `--report-json`. -- Failure `error.log`. -- Report-level diagnostics artifact references. -- Retention decision model with future skipped-correction hook. - -Intentionally deferred: -- Module prompt/response diagnostics artifacts are not produced yet because module execution and LLM calls are not implemented in the runtime path. - -### Phase 7: Pipeline runner with deterministic test modules - -Completed. - -Implemented: -- `internal/framework/runner` production package with sequential module orchestration. -- Deterministic module run-spec resolution and repeated instance naming (`glossary_1`, `glossary_2`, etc.). -- Mutable working transcript handoff across module instances. -- Proposal application through `internal/framework/proposals`. -- Per-module applied/skipped change capture and module status/timing metadata. -- Partial-progress return on module failure, with pipeline stop on first failure. -- Process report support for module-level results and run-level module summaries. -- CLI runtime integration point via injectable module factory/registry, exercised by deterministic fake-module tests. - -Not implemented in Phase 7 (by design): -- Real `glossary`, `homophones`, `spoken_word`, `grammar` production modules. -- Validators (Phase 8). -- Structured LLM calls or scheduler behavior. -- Prompt/response diagnostics. -- End-to-end transcript polishing. - -Current runtime behavior note: -- Default user-facing CLI behavior now executes the full production module sequence unless `--modules` explicitly overrides it. - -## Phase 8: Runtime validator framework and deterministic validators -Completed. - -Implemented: -- Runtime validator request/result models in `internal/framework/validators`. -- Deterministic validator reason codes for stable reporting. -- Validator cardinality enforcement: - - one decision per candidate proposal index - - missing indexes are errors - - duplicate indexes are errors - - unknown indexes are errors -- Deterministic validators: - - confidence threshold - - original-text presence against working transcript - - non-empty correction - - identical/no-effect rejection - - conservative protected glossary-term guard -- Ordered validator-chain execution in the production runner. -- Runner behavior where only validator-approved proposals proceed to proposal application. -- Module-level reporting of validator decisions and validator rejections, distinct from application-level skips. -- Deterministic fake-module tests covering approvals, rejections, validator order/filtering, and cardinality failure pipeline-stop behavior. - -Not implemented in Phase 8 (by design): -- LLM-backed validators (Phase 10). -- Structured LLM runtime wiring (Phase 9 follow-up). -- Real correction modules. -- Prompt/response diagnostics runtime wiring. -- End-to-end transcript polishing. - -## Phase 9: Structured LLM client and scheduler infrastructure - -### Status - -Completed for Phase 9 infrastructure scope. - -Implemented in this phase so far: -- Added internal structured LLM contract support for caller-provided typed outputs. -- Added `internal/framework/llm` adapter backed by `github.com/jxnl/instructor-go`. -- Confirmed OpenAI-compatible base URL support through the adapter path. -- Added adapter unit tests for model/base URL handling, retries, context cancellation, optional API key behavior, and error redaction. - -Explicitly deferred from Phase 9 into later phases: -- Runtime wiring in runner/module infrastructure (without introducing real modules yet). -- Wiring prompt/response diagnostics primitives into future module/validator call sites. -- Wiring effective primary/validation LLM config resolution into runtime LLM call sites. - -### Purpose - -Implement the provider-neutral LLM infrastructure needed by both proposal generation and LLM-backed validators, without yet implementing real modules. - -### Scope completed in this phase - -Implemented: -- Provider-neutral internal structured LLM contract with caller-provided typed output decoding. -- OpenAI-compatible structured-output adapter (`instructor-go`) with: - - configurable base URL and model - - optional API key behavior - - retry budget - - timeout-aware HTTP client handling - - context cancellation propagation - - structured response decoding into caller-provided typed targets - - redacted error surfaces. -- Scheduler/semaphore infrastructure for bounded backend concurrency with context-aware acquisition and reliable permit release. -- Effective LLM config resolution helpers for: - - primary LLM settings - - validation inheritance from primary when unset - - validation overrides when set. -- Generic JSON diagnostics primitives for request metadata, request payload, response payload, and optional error payload, with secret redaction. -- Unit tests covering adapter behavior, scheduler behavior, effective config resolution, and diagnostics redaction/JSON validity. - -Do not implement: -- Real correction modules. -- LLM-backed validators. -- Prompt text for domain modules. -- End-to-end transcript polishing. - -### Expected behavior at end of phase - -At the end of Phase 9, the codebase had tested LLM infrastructure primitives, while default CLI runtime behavior remained deterministic preprocessing/reporting because real modules were not implemented yet. - -### Definition of done status - -Met: -- Structured LLM client infrastructure is implemented and tested. -- OpenAI-compatible structured-output client exists and is tested. -- Scheduler enforces configured concurrency and is tested. -- Primary and validation effective config resolution exists and is tested. -- Prompt/response diagnostics primitives exist and are tested. -- API keys are redacted in LLM adapter errors and diagnostics artifacts; config/report diagnostics redaction remains in place. -- No real module behavior was introduced. -- `go test ./...` passes. - -## Phase 10: LLM-backed validators - -Completed. - -Implemented: -- LLM-backed validator request/response models in `internal/framework/validators`. -- Prompt builders for: - - spoken-form plausibility - - meaning reversal detection - - editorial review - - grammar review - - spoken-word review -- Deterministic batching by `validation_max_prompt_tokens` with stable ordering and no drop/dup behavior. -- LLM validator execution through the internal structured client abstraction (no direct provider calls in validator code). -- Scheduler/concurrency hooks for LLM validator calls. -- Prompt/response diagnostics artifact writing for LLM validator batches using Phase 9 diagnostics primitives. -- Secret redaction in validator LLM diagnostics artifacts. -- Strict structured-response safety and cardinality checks (missing/duplicate/unknown indexes fail closed). -- Runner/report integration so LLM validator decisions and rejections appear in module reports. -- Fake-module and fake-client tests for approval/rejection, malformed output, cardinality errors, batching, scheduler usage, and diagnostics redaction. - -Not implemented in Phase 10 (by design): -- Real correction modules (`glossary`, `homophones`, `spoken_word`, `grammar`). -- Real module implementation and full runtime wiring (Phase 12+). -- Domain proposal prompts. -- Default CLI end-to-end transcript polishing behavior. - -## Phase 11: Shared LLM proposal generation framework and module registry - -Completed. - -Implemented: -- Shared proposal-generation package `internal/framework/proposal_generation`. -- Reusable request model for proposal generation including: - - module key/instance - - replacement policy - - working transcript context - - optional section metadata - - glossary/config context - - diagnostics context - - injected structured LLM client/scheduler dependencies. -- Structured correction-set response model and parsing into existing proposal models: - - `proposals.CorrectionProposal` - - `proposals.EnrichedCorrectionProposal`. -- Deterministic proposal-index assignment via caller-provided start index. -- Proposal-generation diagnostics artifact writing using generic LLM diagnostics primitives with secret redaction. -- Scheduler-aware proposal generation through the internal LLM scheduler interface. -- Production module-registry scaffolding in `internal/framework/modules` with: - - known module-key recognition for `glossary`, `homophones`, `spoken_word`, `grammar` - - constructor registration and dependency-injection path - - explicit unsupported and recognized-but-unimplemented module errors. -- Runner/CLI injection-path tests showing shared proposal generation can flow through runner validation/application semantics using fake modules/clients. - -Not implemented in Phase 11 (by design): -- Real production `glossary`, `homophones`, and `spoken_word` modules. -- Domain proposal prompts for production modules. -- Default CLI end-to-end transcript polishing behavior. - -## Phase 12: Grammar module - -Completed. - -Implemented: -- Production grammar module package in `internal/modules/grammar`. -- Grammar prompt builder aligned to Python intent and constrained to punctuation/capitalization/spacing/article cleanup. -- Grammar proposal generation through shared `internal/framework/proposal_generation` using `contracts.StructuredLLMClient`. -- Scheduler-aware grammar proposal generation through existing scheduler hooks. -- Grammar replacement policy `require_unique` (matching Python behavior). -- Grammar validator chain using existing deterministic and LLM-backed validator infrastructure. -- Grammar confidence threshold enforcement through existing config + confidence-threshold validator behavior. -- Explicit runtime support for `--modules grammar` through normalization, chunking, runner, proposal generation, validation, application, and reporting. -- Prompt/response diagnostics artifacts for grammar proposal + validator interactions with secret redaction. -- Module-level reports for grammar including validator decisions/rejections, applied changes, and skipped changes. -- CLI/runtime fake-client tests for approved proposals, validator rejection, application skips, diagnostics, failure/error.log behavior, and report outputs (`--report-json` and run-dir `report.json`). - -Not implemented in Phase 12 (by design): -- Production `glossary`, `homophones`, and `spoken_word` modules. -- Full default module sequence execution as a feature-complete claim. - -## Phase 13: Glossary module and protected-term behavior - -Completed. - -Implemented: -- Production glossary module package in `internal/modules/glossary`. -- Glossary prompt builder aligned to Python intent and constrained to glossary-supported domain/acoustic corrections. -- Prompt context using glossary names, aliases, categories, summaries, and plural forms where available. -- Glossary proposal generation through shared `internal/framework/proposal_generation` using `contracts.StructuredLLMClient`. -- Scheduler-aware glossary proposal generation through existing scheduler hooks. -- Glossary replacement policy `replace_all` (matching Python behavior). -- Glossary validator chain using existing deterministic and LLM-backed validators. -- Glossary confidence threshold enforcement through existing config + confidence-threshold validator behavior. -- Deterministic glossary-derived protected-term extraction (`internal/framework/validators/protected_terms.go`) from names, aliases, and plural forms, with stable deduplicated ordering. -- Protected-term validator behavior remaining available to non-glossary modules via existing deterministic validators. -- Explicit runtime support for `--modules glossary` through normalization, chunking, runner, proposal generation, validation, application, and reporting. -- Repeated glossary-stage support (`--modules glossary,glossary`) with deterministic instance naming and mutable working-transcript handoff across stages. -- Prompt/response diagnostics artifacts for glossary proposal + validator interactions with secret redaction. -- Module-level reports for glossary including generated proposals, validator decisions/rejections, applied changes, and application skips. -- CLI/runtime fake-client tests for approved proposals, validator rejection, application skips, repeated stages, diagnostics, failure/error.log behavior, and report outputs (`--report-json` and run-dir `report.json`). - -Not implemented in Phase 13 (by design): -- Production `homophones` and `spoken_word` modules. -- Full default module sequence execution as a feature-complete claim. - -## Phase 14: Homophones module - -Completed. - -Implemented: -- Production homophones module package in `internal/modules/homophones`. -- Homophones prompt builder aligned to Python intent and constrained to conservative homophone/near-homophone/mistranscription corrections. -- Prompt context using glossary/protected-term information (names, aliases, plurals where present) to avoid damaging known domain terms. -- Homophones proposal generation through shared `internal/framework/proposal_generation` using `contracts.StructuredLLMClient`. -- Scheduler-aware homophones proposal generation through existing scheduler hooks. -- Homophones replacement policy `require_unique` (matching Python behavior). -- Homophones validator chain using existing deterministic and LLM-backed validators. -- Homophones confidence threshold enforcement through existing config + confidence-threshold validator behavior. -- Protected-term guardrails remaining active for homophones via existing deterministic validators. -- Explicit runtime support for `--modules homophones` through normalization, chunking, runner, proposal generation, validation, application, and reporting. -- Prompt/response diagnostics artifacts for homophones proposal + validator interactions with secret redaction. -- Module-level reports for homophones including generated proposals, validator decisions/rejections, applied changes, and application skips. -- CLI/runtime fake-client tests for approved proposals, validator rejection, application skips, diagnostics, protected-term rejection behavior, failure/error.log behavior, and report outputs (`--report-json` and run-dir `report.json`). -- Focused interoperability tests for already-implemented module combinations (for example `glossary,homophones`) to verify working-transcript handoff and guardrails without claiming full default-sequence parity. - -Not implemented in Phase 14 (by design): -- Production `spoken_word` module. -- Full default module sequence execution as a feature-complete claim. - -## Phase 15: Spoken-word module - -Completed. - -Implemented: -- Production spoken_word module package in `internal/modules/spoken_word`. -- Spoken_word prompt builder aligned to Python intent and constrained to conservative dysfluency cleanup. -- Prompt context using glossary/protected-term information (names, aliases, plurals where present) to avoid damaging known domain terms. -- Strong prompt guardrails preserving meaning, intent, speaker voice, named entities, game/domain terms, and substantive content. -- Explicit prompt guardrails against summarization, style rewriting, grammar-only cleanup, punctuation-only cleanup, invention, event reordering, and certainty inflation. -- Spoken_word proposal generation through shared `internal/framework/proposal_generation` using `contracts.StructuredLLMClient`. -- Scheduler-aware spoken_word proposal generation through existing scheduler hooks. -- Spoken_word replacement policy `require_unique` (matching Python behavior). -- Spoken_word validator chain using existing deterministic and LLM-backed validators. -- Strong semantic guardrails in runtime validator chain through existing LLM-backed validators (`spoken_word_review`, `meaning_reversal_review`). -- Spoken-word confidence threshold enforcement through existing config + confidence-threshold validator behavior. -- Protected-term guardrails remaining active for spoken_word via existing deterministic validators. -- Explicit runtime support for `--modules spoken_word` through normalization, chunking, runner, proposal generation, validation, application, and reporting. -- Prompt/response diagnostics artifacts for spoken_word proposal + validator interactions with secret redaction. -- Module-level reports for spoken_word including generated proposals, validator decisions/rejections, applied changes, and application skips. -- CLI/runtime fake-client tests for approved cleanup, validator rejection, meaning-changing rejection, application skips, diagnostics, protected-term rejection behavior, failure/error.log behavior, and report outputs (`--report-json` and run-dir `report.json`). -- Focused interoperability tests for already-implemented module combinations (for example `spoken_word,grammar`) to verify working-transcript handoff and guardrails without claiming full default-sequence parity. - -Not implemented in Phase 15 (by design): -- Full default module sequence execution as a feature-complete claim. - -## Phase 16: Default full pipeline integration - -Completed. - -### Purpose - -Enable and harden the full default module sequence in the Go runtime path. - -### Scope - -Implement: -- Default runtime sequence: - - `glossary` - - `homophones` - - `glossary` - - `spoken_word` - - `grammar` -- End-to-end execution through all real modules. -- Accurate resolved module instance names. -- Module-level report aggregation. -- Full run-level applied/skipped summaries. -- Skip-aware retention behavior using actual skipped correction data. -- Failure behavior with partial module progress. -- PipelineRunError or equivalent partial-progress error type if not already implemented. -- Diagnostics for each module instance. -- Tests for successful full pipeline using fake LLM. -- Tests for mid-pipeline failure preserving partial report and diagnostics. - -Do not implement: -- Python archive removal. -- Side-by-side rollout tooling beyond what is needed for tests. - -### Expected behavior at end of phase - -Running `audita process transcript.json --glossary glossary.yaml --output corrected.json` should execute the full Go module pipeline and produce a corrected transcript. - -### Definition of done - -- Default module sequence runs end-to-end. -- All real modules participate in runtime path. -- Reports include all module instances. -- Applied/skipped changes are aggregated at run level. -- Failed runs retain useful partial reports and diagnostics. -- `auto` retention keeps successful runs with skipped corrections. -- CLI stdout/stderr behavior remains subprocess-safe. -- `go test ./...` passes without requiring external LLM credentials. - -### Phase 16 completion status - -Implemented: -- Normal `audita process` runs without `--modules` now execute the full sequence: - - `glossary` - - `homophones` - - `glossary` - - `spoken_word` - - `grammar` -- Explicit `--modules` still overrides the default sequence. -- Repeated glossary stages are deterministic (`glossary_1`, `glossary_2`) and reported distinctly. -- Full-pipeline module reports aggregate applied changes, application skips, validator rejections, and failed-module metadata. -- Full-pipeline diagnostics include prompt/response artifacts for module proposal generation and validator LLM interactions with secret redaction. -- Mid-pipeline failure preserves partial module progress in reports and retains diagnostics + `error.log`. -- Auto-retention keeps successful runs with actual skipped/rejected corrections and always retains failed runs. - -Intentionally deferred: -- Phase 17 parity fixture suite. -- Phase 18 operational hardening. -- Phase 19 rollout/Python retirement work. - -## Phase 17: Python parity fixture suite - -Completed. - -### Purpose - -Establish confidence that the Go implementation matches the behavior and safety posture of the initial Python implementation. - -### Scope - -Implement: -- Parity fixtures based on representative Python-era inputs and expected behaviors. -- Fake LLM response fixtures where exact deterministic behavior is required. -- Golden tests for: - - transcript schema handling - - glossary schema handling - - normalization - - chunking - - proposal application - - validator behavior - - module reports - - diagnostics artifacts - - default pipeline shape -- Comparison tests or scripts that can run Python and Go side by side where practical. -- Documentation of intentional differences between Python and Go. - -Do not require: -- Real LLM credentials for normal automated tests. -- Exact nondeterministic natural-language output equivalence across Python and Go. - -### Expected behavior at end of phase - -The repository has a durable test suite demonstrating that the Go implementation preserves the functional contract of the Python implementation. - -### Definition of done - -- Representative parity fixtures exist. -- Golden tests cover deterministic behavior. -- Fake LLM tests cover full pipeline behavior. -- Intentional differences from Python are documented. -- Unintentional compatibility breaks are fixed. -- `go test ./...` passes. - -### Phase 17 completion status - -Implemented: -- Durable parity fixture harness in Go test path (`internal/cli/parity_test.go`). -- Representative parity fixture corpus under `internal/cli/testdata/parity`. -- Fake LLM proposal/validator fixtures driving deterministic parity tests. -- Default full-pipeline parity coverage including sequence/order and repeated glossary naming: - - `glossary_1` - - `homophones` - - `glossary_2` - - `spoken_word` - - `grammar` -- Parity checks that are strict for deterministic contract fields: - - transcript content - - module order and instance names - - applied/skipped/rejected counts - - report status and failed-module metadata -- Parity checks that ignore nondeterministic metadata fields: - - timestamps - - run IDs - - temp/absolute paths - - provider token usage details -- Documentation of intentional Python-vs-Go differences and honest open parity gaps in `docs/python-parity.md`. -- Normal `go test ./...` path remains independent of real LLM credentials and Python dependencies. - -Intentionally deferred: -- Phase 18 operational hardening and subprocess integration expansion. -- Phase 19 rollout and Python retirement work. - -## Phase 18: Operational hardening and subprocess integration - -### Purpose - -Harden the Go binary for use as the production Audita implementation in the surrounding application suite. - -### Scope - -Implement: -- Additional subprocess tests for large transcripts. -- Timeout/cancellation tests. -- Failure-mode tests for unreadable inputs, unwritable outputs, malformed LLM responses, and backend failures. -- Clear stderr summaries pointing to diagnostics. -- Review of all output paths and file-close behavior. -- Review of all secret redaction paths. -- Review of LLM retry and timeout behavior. -- Documentation for production use from orchestrators such as Narratio. - -### Expected behavior at end of phase - -The Go binary should be safe to call from other Go applications and should not reproduce the Python subprocess/stdio integration problems. - -### Definition of done - -- Subprocess tests cover success, failure, large inputs, and cancellation. -- stdout contains machine output only. -- stderr contains human-readable logs/errors only. -- API keys do not appear in reports, diagnostics, logs, or tests. -- Failed runs always preserve diagnostics. -- Operational docs are accurate. -- `go test ./...` passes. - -### Phase 18 completion status - -Completed. - -Implemented: -- Expanded subprocess integration coverage in `cmd/audita/main_integration_test.go` for: - - successful default full-pipeline runs with `--output` and without `--output`; - - successful `--report-json` writes; - - large-transcript subprocess behavior; - - missing/unreadable transcript and missing/malformed glossary failures (portable handling where required); - - unwritable output and unwritable report-json failure behavior (portable handling where required); - - malformed structured LLM response failure behavior; - - synthetic backend LLM failure behavior; - - timeout/cancellation behavior with deterministic context cancellation hooks; - - mid-pipeline failure with partial module progress preserved in reports. -- Hardened subprocess failure stderr output to include diagnostics path when available. -- Added deterministic test-only subprocess LLM/runtime hooks used only in helper-process tests: - - backend failure mode; - - malformed structured-output mode; - - block-until-cancel mode; - - mid-pipeline fail mode. -- Added regression coverage for secret redaction across subprocess stdout/stderr/report/diagnostics artifacts. -- Confirmed existing LLM adapter and scheduler tests continue to cover: - - timeout and context cancellation propagation; - - retry behavior; - - malformed output safety; - - permit release on error/cancellation. -- Added focused subprocess-caller operational guidance in `docs/subprocess-operations.md`. - -Historical note: -- Phase 19 documentation/rollout/Python-retirement transitions were deferred at the end of Phase 18 and completed in Phase 19. - -## Phase 19: Documentation, rollout, and Python retirement - -### Purpose - -Make the Go implementation the documented active implementation and preserve the Python implementation only as historical reference if desired. - -### Scope - -Implement: -- README updates. -- Architecture updates. -- Rewrite notes updates. -- Installation/build instructions for Go binary. -- Migration notes from Python to Go. -- Documentation of any intentionally changed behavior. -- Removal or archival of Python-specific operational instructions from the primary path. -- Clear statement that the Go implementation is now feature-complete. - -### Expected behavior at end of phase - -The repository clearly presents the Go implementation as the active Audita implementation. - -### Definition of done - -- Documentation no longer describes Go as only a deterministic foundation. -- Documentation accurately describes full transcript polishing behavior. -- Python implementation is marked archived/prototype/reference, or removed if that is the chosen repository policy. -- Users can build, test, and run the Go implementation from docs alone. -- `go test ./...` passes. - -### Phase 19 completion status - -Completed. - -Implemented: -- README is Go-first and documents active full-pipeline runtime behavior. -- Build/install/test usage guidance is Go-primary and operationally oriented. -- Subprocess/orchestrator behavior is documented in `docs/subprocess-operations.md`. -- Migration guidance from Python to Go is documented in `docs/migration-from-python.md`. -- Python parity notes and open parity gaps remain documented in `docs/python-parity.md`. -- Python implementation status is explicitly legacy/reference in primary docs and `python/README.md`. -- Documentation no longer routes normal operations to Python. -- Normal `go test ./...` remains independent of real LLM credentials and Python dependencies. - -## Cross-phase compatibility requirements - -These constraints were used across implementation phases and remain useful maintenance checks: - -- Preserve subprocess-safe stdout/stderr behavior. -- Preserve CLI-over-env precedence. -- Preserve existing accepted transcript input forms. -- Preserve glossary YAML compatibility unless a deliberate migration is documented. -- Preserve deterministic safety-first proposal application semantics. -- Preserve run-dir diagnostics and report writing. -- Preserve API key redaction. -- Do not require real LLM credentials for normal `go test ./...`. -- Do not claim a feature is implemented until it is in the runtime path. -- Keep default behavior honest in docs and CLI help. -- Prefer fake LLMs and golden fixtures for automated tests. -- Keep real LLM smoke tests opt-in. - -## Guidance for Codex-style prompts - -When requesting implementation work, use one phase at a time. - -Good prompt shape: -- Name the exact phase. -- State what is in scope. -- State what is explicitly out of scope. -- Require tests. -- Require `go test ./...`. -- Require documentation updates only when the phase changes user-visible or architectural status. -- Require that no later-phase features be implemented opportunistically. - -Avoid broad prompts such as: -- “finish the rewrite” -- “make Audita work” -- “port the Python app” -- “implement all modules” - -Those prompts blur phase boundaries and make review difficult. - -## Suggested review checklist for every phase - -Before accepting a phase implementation, verify: - -- Does the implementation match the phase scope? -- Did it avoid implementing unrelated later-phase behavior? -- Does `go test ./...` pass? -- Are stdout and stderr still clean for subprocess callers? -- Are API keys redacted everywhere? -- Are reports machine-readable? -- Are diagnostics sufficient for debugging? -- Are fake LLM tests used instead of requiring real credentials? -- Are new public behaviors documented? -- Does the code follow the architecture in `docs/architecture.md`? -- Does the implementation move Audita closer to Python feature parity? diff --git a/docs/roadmap.md b/docs/roadmap.md deleted file mode 100644 index 576c08f..0000000 --- a/docs/roadmap.md +++ /dev/null @@ -1,1059 +0,0 @@ -# Audita Pre-1.0 Roadmap - -## Purpose - -This roadmap consolidates the remaining architectural, operational, and documentation work planned before the Audita 1.0 release. - -Audita already has the core transcript-polishing pipeline in place: production modules are wired, LLM-backed proposal generation and validation are working, validator chains are enforced, diagnostics are emitted, and subprocess behavior has been hardened. The remaining pre-1.0 work should therefore focus on stabilizing the public contract, reducing avoidable dependency and operational risk, improving maintainability around prompts and validators, and making runtime behavior easier to understand after the fact. - -This roadmap is intentionally scoped. The goal is not to turn Audita into a fully user-programmable LLM framework before 1.0. The goal is to make the built-in pipeline stable, explainable, testable, and maintainable. - -## Release principles - -The following principles should guide all pre-1.0 work: - -1. Preserve Audita's conservative correction posture. New features should not make the LLM more free-form or less accountable. -2. Keep the Go binary self-contained by default. Built-in prompts, schemas, validators, and output encoders should work without external runtime assets. -3. Prefer registered, versioned extension points over arbitrary user-supplied behavior. -4. Keep subprocess behavior stable: stdout, stderr, exit codes, output files, diagnostics, and reports should be predictable. -5. Treat prompts, validators, schemas, config, and report fields as public or semi-public contracts once 1.0 is released. -6. Make diagnostics good enough that a failed, slow, or surprising run can be understood after the fact. -7. Keep secrets out of config files, logs, reports, and diagnostics. -8. Reduce avoidable dependency weight before 1.0, especially around core LLM infrastructure. - -## Target pre-1.0 scope - -The agreed pre-1.0 scope consists of the following workstreams: - -- Replace the current `instructor-go` structured-output helper with an Audita-owned OpenAI-compatible structured LLM adapter. -- Add versioned configuration file support with a reduced and stabilized CLI surface. -- Document the stable public contract for CLI, subprocess, config, output, reports, diagnostics, and compatibility. -- Add a small registry of supported output schemas. -- Refactor validators into first-class composable components with stable validator keys and built-in chain definitions. -- Move prompt text into embedded Markdown assets with prompt IDs, versions, hashes, and shared prompt-injection hardening. -- Add transcript context support through an explicit description option. -- Add scheduler/module utilization diagnostics. -- Add a correction ledger / review artifact. -- Complete release-hardening tests and documentation updates. - -The following should generally be treated as post-1.0 unless they become trivial after the refactors above: - -- Arbitrary filesystem prompt overrides. -- User-configurable validator chains. -- Arbitrary user-supplied output schemas. -- Resume/start-at/stop-after execution. -- `--diff`, `--check`, or `--propose-only` review modes. -- Generated transcript descriptions enabled by default. -- Interactive correction review UI. -- Provider-specific model benchmarking harness. - -## Recommended implementation order - -The work is best handled in seven phases. The ordering is intentional: - -1. Replace the structured-output dependency first, because it is core LLM infrastructure and affects schema metadata, diagnostics, and binary/dependency posture. -2. Stabilize configuration and public surfaces before adding more knobs. -3. Define output schema and report contracts before expanding diagnostics. -4. Refactor validators before moving prompt assets, because prompt ownership and validator ownership are closely linked. -5. Move prompts into embedded Markdown files after validator/module identities are stable. -6. Add utilization and correction-ledger diagnostics after module, validator, prompt, and schema IDs are available. -7. Finish with release-hardening documentation and evaluation fixtures. - ---- - -# Phase 0: Replace `instructor-go` with an Audita-owned structured LLM adapter - -## Goal - -Remove the heavy `instructor-go` dependency and replace it with a small Audita-owned OpenAI-compatible structured-output adapter. - -## Implementation status (2026-05-13) - -This workstream is now implemented in the repository: -- production runtime uses an Audita-owned OpenAI-compatible structured LLM adapter (`internal/framework/llm/openai_compatible_client.go`); -- `StructuredLLMClient` remains the stable internal boundary used by proposal generation and validators; -- structured response schemas are registered with stable IDs, versions, names, and SHA-256 hashes (`internal/framework/responseschema`); -- schema metadata is attached to structured completion requests and included in diagnostics metadata; -- `instructor-go` has been removed from runtime code and module dependencies. - -This status update applies only to the structured LLM dependency replacement workstream. Other roadmap workstreams remain planned unless explicitly marked otherwise. - -This should happen before 1.0 because structured LLM calls are core runtime infrastructure. Replacing this layer after 1.0 would risk subtle compatibility changes in request construction, schema strictness, retry behavior, error reporting, diagnostics, and provider compatibility. - -## Design direction - -Audita should retain its own internal structured LLM contract and replace only the implementation behind it. - -The rest of the application should continue depending on an Audita-owned interface such as: - -- `StructuredLLMClient` -- `CompleteStructured(ctx, req, out)` - -The new adapter should be responsible for: - -- building OpenAI-compatible chat completion requests; -- attaching `response_format.type = json_schema`; -- attaching a strict JSON Schema response contract; -- sending requests to a configurable OpenAI-compatible endpoint; -- decoding the returned JSON into caller-provided Go structs; -- preserving timeout, retry, cancellation, scheduler, diagnostics, and redaction behavior; -- surfacing provider/model/token metadata where available. - -## SDK versus direct HTTP - -Two implementation options are reasonable: - -1. Use the official OpenAI Go SDK. -2. Use a small direct `net/http` adapter for the OpenAI-compatible request shape Audita needs. - -Given Audita's goals, a direct HTTP adapter is attractive because: - -- the request shape is small and stable; -- Audita already owns its internal LLM contract; -- several target providers are OpenAI-compatible rather than necessarily OpenAI itself; -- dependency weight and binary size are part of the motivation for the change; -- direct request/response diagnostics are easier to reason about. - -The implementation should not expose SDK types beyond the adapter boundary if an SDK is used. - -## JSON Schema handling - -Do not replace `instructor-go` with another broad schema-generation framework unless there is a clear need. - -Audita likely has only a small number of structured response shapes: - -- proposal correction set; -- validator decision set; -- optional transcript description summary, if implemented; -- possibly future small metadata responses. - -These schemas should be stable Audita contracts. Prefer hand-authored or explicitly defined schemas with IDs, versions, and hashes. - -Suggested schema assets: - -- `correction_set_v1` -- `validator_decision_set_v1` -- `transcript_description_v1` - -Each schema should have: - -- schema ID; -- schema version; -- schema hash; -- strict `additionalProperties: false` behavior where appropriate; -- tests showing accepted and rejected example payloads. - -## Runtime behavior - -The adapter should: - -1. Build the OpenAI-compatible request. -2. Include `response_format` with `type: json_schema`. -3. Set `strict: true` where the backend supports it. -4. Receive the response. -5. Extract the assistant message content or equivalent provider response field. -6. Decode JSON into the caller-provided output struct. -7. Run existing Audita-side validation and cardinality checks. -8. Emit diagnostics with secrets redacted. - -Provider-level structured output should reduce malformed responses, but Audita should continue treating all model output as untrusted until locally decoded and validated. - -## Deliverables - -- New Audita-owned OpenAI-compatible structured LLM adapter. -- Removal of `instructor-go` from `go.mod`. -- Preservation of the existing internal `StructuredLLMClient` contract where practical. -- Stable schema registry or schema definitions for Audita structured responses. -- Schema ID/version/hash metadata added to LLM diagnostics. -- Provider/API error redaction. -- Retry, timeout, and cancellation behavior preserved. -- Scheduler integration preserved. -- Binary size and dependency tree comparison before and after the change. - -## Tests - -- Request body includes `response_format` with `type: json_schema`. -- Request body includes the expected schema name and strictness setting. -- Valid proposal responses decode into existing proposal models. -- Valid validator responses decode into existing validator decision models. -- Malformed JSON fails safely. -- Missing required fields fail safely. -- Unknown extra fields fail according to schema/decoder policy. -- Provider errors are redacted. -- API keys do not appear in errors, reports, or diagnostics. -- Retry behavior remains correct. -- Timeout and context cancellation remain correct. -- Existing fake LLM tests continue passing. -- Normal `go test ./...` does not require live LLM credentials. - -## Documentation - -Update architecture documentation to explain: - -- Audita's internal structured LLM contract; -- the OpenAI-compatible adapter; -- the supported structured-output request shape; -- schema IDs and versions; -- provider compatibility expectations; -- local validation after provider-level structured output. - ---- - -# Phase 1: Stabilize public surfaces and configuration - -## Goal - -Introduce a versioned configuration file and clarify which settings are stable CLI flags, which settings belong in config, and which settings should remain environment-only. - -## Implementation status (2026-05-13) - -This workstream is now implemented for runtime loading and basic command surface: -- versioned YAML file config with strict unknown-field rejection and `version: 1` validation; -- runtime config-source behavior for `audita process`: - - defaults; - - config file (`--config`, then `AUDITA_CONFIG`, then `/etc/audita/config.yml` if present); - - environment overrides; - - CLI overrides; -- explicit missing-file errors for `--config` and `AUDITA_CONFIG`, with non-fatal missing default-path behavior; -- `api_key_env` support in file config for proposal and validation LLM credentials; -- config command surface: - - `audita config validate --config ` - - `audita config print-effective [--config ]` -- redacted effective-config behavior preserved across diagnostics and config printing; -- compatibility environment variables and lower-level process flags remain available. - -This status update applies only to versioned config support and config commands. Output schema registries, broader public-contract work, validator refactors, prompt-asset registries, utilization diagnostics, and correction ledgers remain planned. - -This phase should happen early because later workstreams need clean config locations for prompt registry settings, output schema selection, validator settings, diagnostics settings, and concurrency tuning. - -## Configuration precedence - -Use the following precedence order: - -1. Built-in defaults. -2. Configuration file. -3. Environment variables for secrets and selected deployment overrides. -4. CLI flags for high-value per-run overrides. - -This replaces the current broad “defaults < environment < CLI” model with a cleaner 1.0 contract. - -## Default config path - -Support a default config path: - -- `/etc/audita/config.yml` - -Also support: - -- `--config ` -- optionally `AUDITA_CONFIG` - -## Config versioning - -Every config file should include: - -- `version: 1` - -Unknown versions should produce a clear validation error. - -## Secrets policy - -Do not encourage raw API keys in config files. - -Preferred pattern: - -- config contains `api_key_env`; -- the actual key is resolved from the environment at runtime; -- diagnostics and effective config output redact secret-bearing values. - -## Reduced CLI surface - -Keep CLI flags for frequently changed per-run values and subprocess integration: - -- `--config` -- transcript positional argument -- `--glossary` -- `--output` -- `--report-json` -- `--modules` -- `--work-dir` -- `--work-dir-retention` -- `--transcript-description` -- `--output-schema` -- `--total-llm-concurrency` -- `--target-sections` - -Move lower-level tuning into config-only or config-primary settings: - -- normalization knobs; -- min/max section tokens; -- confidence thresholds; -- LLM timeouts and retry counts; -- proposal/validation split concurrency; -- validator batching limits; -- diagnostics retention details; -- default module sequence. - -Existing environment variables and CLI flags may remain temporarily as compatibility aliases, but the 1.0 documentation should clearly identify the preferred surface. - -## Suggested config shape - -Example: - -```yaml -version: 1 - -pipeline: - modules: - - glossary - - homophones - - glossary - - spoken_word - - grammar - -llm: - proposal: - base_url: http://localhost:8000/v1 - model: nvidia/Nemotron-3-Nano-30B-A3B - api_key_env: AUDITA_LLM_API_KEY - timeout: 120s - max_retries: 2 - - validation: - base_url: http://localhost:8001/v1 - model: nvidia/Gemma-4-31B - api_key_env: AUDITA_VALIDATION_LLM_API_KEY - timeout: 180s - max_retries: 2 - -concurrency: - total_llm: 4 - proposal_llm: 4 - validation_llm: 4 - -chunking: - target_sections: 8 - max_section_tokens: 3000 - min_section_tokens: 800 - -normalization: - max_segment_gap: 1.25s - ellipsis_gap: 2s - max_segment_duration: 30s - max_segment_tokens: 120 - -thresholds: - glossary: 0.70 - homophones: 0.75 - spoken_word: 0.80 - grammar: 0.70 - -context: - description: "" - -output: - schema: bare-segments - -diagnostics: - work_dir: /tmp/audita - retention: auto -``` - -## Suggested commands - -Add: - -- `audita config validate --config ` -- `audita config print-effective --config ` - -The second command should use the same redaction behavior as run diagnostics. - -## Deliverables - -- `internal/core/config` support for loading YAML config files. -- Config schema with `version: 1`. -- `--config` flag. -- Optional `AUDITA_CONFIG` environment variable. -- Config validation errors that identify the exact invalid field where practical. -- Redacted effective config output updated to include config-derived values. -- Documentation for precedence and supported fields. -- Compatibility/deprecation notes for legacy env vars and flags. - -## Tests - -- Defaults-only config behavior. -- Config file loading. -- CLI overrides config. -- Environment secrets resolve through `api_key_env`. -- Unknown config version fails. -- Unknown config fields either fail or warn according to an explicit policy. -- Effective config redacts secrets. -- Existing process tests continue passing. - ---- - -# Phase 2: Define the stable public contract and output schema registry - -## Goal - -Document and implement the stable boundaries that external callers can rely on for 1.0. - -## Implementation status (2026-05-13) - -This workstream is now partially implemented in the repository: -- output schema registry is implemented with supported schemas: - - `bare-segments` (default); - - `audita-v1`; -- `audita process --output-schema ` is implemented; -- versioned file config supports `output.schema`; -- process reports include explicit report metadata with: - - report schema name/version; - - selected output schema; - - config file version when a file config is used; -- public contract documentation and output schema documentation are now present in: - - `docs/public-contract.md` - - `docs/output-schemas.md` - -Current intentional gap: -- `seriatim-intermediate` remains deferred because no concrete implemented contract exists yet. - -This status update only applies to output schema/report metadata/public contract documentation. Validator refactors, prompt-asset registries, scheduler utilization diagnostics, correction ledgers, and generated summaries remain planned. - -This phase should happen early because Audita is both a user-facing CLI and a subprocess dependency. The public contract should guide the remaining implementation decisions rather than merely documenting them after the fact. - -## Public contract document - -Add or expand a document such as: - -- `docs/public-contract.md` - -It should cover: - -- CLI stability guarantees; -- config file stability guarantees; -- supported input transcript forms; -- supported glossary form; -- output schema options; -- report schema versioning; -- diagnostics directory behavior; -- stdout/stderr contract; -- exit-code behavior; -- secret redaction guarantees; -- compatibility and deprecation policy; -- what counts as a breaking change after 1.0. - -This can complement the current subprocess operations document. - -## Output schema registry - -Implement a small registry of supported output encoders. Do not support arbitrary user-supplied schemas at runtime. - -Suggested initial schemas: - -1. `bare-segments` - - Current behavior. - - Top-level JSON array. - - Best for backward compatibility. - -2. `audita-v1` - - Object format with schema/version metadata and a `segments` array. - - Preferred stable Audita-native format going forward. - -3. `seriatim-intermediate` - - Compatibility format for downstream Seriatim/Narratio workflows, if meaningfully distinct from `audita-v1`. - -The command surface should be: - -- `--output-schema ` - -Config should also support: - -- `output.schema: ` - -For 1.0, it is reasonable to keep `bare-segments` as the default to avoid breaking existing consumers, while documenting `audita-v1` as the preferred stable format. - -## Report schema versioning - -Add explicit report schema metadata, for example: - -- report schema name; -- report schema version; -- Audita binary version; -- config version; -- output schema name; -- structured response schema IDs and versions; -- pipeline/module sequence; -- prompt and validator metadata once those phases are complete. - -## Deliverables - -- Output encoder package or extension to `internal/core/schema`. -- Output schema registry with stable names. -- `--output-schema` and config support. -- `docs/public-contract.md`. -- `docs/output-schemas.md`, if output schema detail becomes too large for the public contract document. -- Tests for each supported output schema. -- Backward-compatibility test for current bare-array output. - -## Tests - -- Each output schema produces valid JSON. -- Each output schema preserves segment fields correctly. -- Unknown output schema fails clearly. -- Report includes selected output schema. -- Existing stdout/file-output behavior remains unchanged except for selected schema shape. - ---- - -# Phase 3: Refactor validators into first-class composable components - -## Goal - -Make validators as easy to reason about as modules. - -## Implementation status (2026-05-13) - -This workstream is now implemented for built-in validator composition: -- first-class built-in validator registry exists in `internal/validators`; -- stable validator keys are defined and used for built-in chains and runtime validator names; -- explicit built-in module validator-chain definitions are implemented and resolved through registry-backed chain wiring; -- production module constructors use built-in chain resolution instead of ad hoc manual validator construction; -- runner execution preserves deterministic ordering (deterministic validators before LLM-backed validators); -- report validator decision/rejection records now carry stable validator keys. - -Current boundary: -- validator chains are built-in and not user-configurable. -- prompt-asset registries, prompt metadata, scheduler utilization diagnostics, correction ledgers, and generated summaries remain planned. - -Validators are now central runtime components. They are reused across modules, have deterministic and LLM-backed implementations, produce diagnostics, and affect final correction acceptance. They should therefore have stable identities, registry metadata, and composable chain definitions. - -## Package structure - -Move from a monolithic `internal/framework/validators` package toward a first-class validator package layout. - -One possible structure: - -- `internal/validators` - - shared contracts; - - registry; - - chain definitions; - - common models. -- `internal/validators/deterministic` - - confidence threshold; - - original text presence; - - non-empty corrected text; - - no-effect rejection; - - protected terms. -- `internal/validators/llm` - - spoken form plausibility; - - meaning reversal; - - editorial review; - - grammar review; - - spoken-word review. - -Alternatively, each validator can live in its own package. The key requirement is that validators have stable keys and are registered in one place. - -## Validator keys - -Every validator should have a stable key, such as: - -- `confidence_threshold` -- `original_text_presence` -- `non_empty_corrected_text` -- `no_effect` -- `protected_terms` -- `spoken_form_plausibility` -- `meaning_reversal_review` -- `editorial_review` -- `grammar_review` -- `spoken_word_review` - -These keys should appear consistently in: - -- reports; -- diagnostics paths; -- chain definitions; -- tests; -- documentation. - -## Validator chains - -Each module should define its validator chain compositionally, using validator keys rather than ad hoc wiring. - -Example chains: - -- `glossary` - - confidence threshold; - - original text presence; - - non-empty corrected text; - - no-effect rejection; - - spoken form plausibility; - - editorial review. - -- `homophones` - - confidence threshold; - - original text presence; - - non-empty corrected text; - - no-effect rejection; - - protected terms; - - spoken form plausibility; - - meaning reversal review; - - editorial review. - -- `spoken_word` - - confidence threshold; - - original text presence; - - non-empty corrected text; - - no-effect rejection; - - protected terms; - - spoken-word review; - - meaning reversal review. - -- `grammar` - - confidence threshold; - - original text presence; - - non-empty corrected text; - - no-effect rejection; - - protected terms; - - grammar review. - -These exact chains can be adjusted during implementation, but the chain definition should become explicit and auditable. - -## Config boundary - -For 1.0, do not expose arbitrary user-configurable validator chains. - -Recommended 1.0 boundary: - -- built-in chains are fixed and documented; -- thresholds and batching knobs are configurable; -- validator keys and chain membership appear in reports; -- future user-configurable chains remain possible because the registry exists. - -## Deliverables - -- First-class validator registry. -- Stable validator keys. -- Explicit module validator-chain definitions. -- Updated runner wiring to use chains. -- Updated report fields using stable validator IDs. -- Documentation of built-in validators and validator chains. -- Migration of existing tests to the new package structure. - -## Tests - -- Every built-in validator is registered. -- Every module references only registered validators. -- Missing/unknown validator keys fail deterministically. -- Existing validator behavior remains unchanged. -- Reports preserve or improve existing validator rejection detail. -- Repeated glossary stages continue to report deterministically. - ---- - -# Phase 4: Move prompts into embedded Markdown assets and harden prompt boundaries - -## Goal - -Treat prompts as first-class behavioral assets rather than multiline strings embedded in Go source. - -## Implementation status (2026-05-13) - -This workstream is now implemented for built-in prompt assets and diagnostics metadata: -- module and LLM-validator prompt text has been moved to embedded Markdown assets under `internal/prompts/assets`; -- a built-in prompt registry exists in `internal/prompts` with stable prompt IDs and initial versions (`v1`); -- prompt metadata is registered with: - - prompt ID; - - prompt version; - - prompt source (`builtin`); - - embedded source path; - - deterministic SHA-256 source hash; -- prompt rendering uses typed template data and `text/template` missing-key errors; -- a shared prompt-injection hardening fragment is centrally embedded and included in module proposal and LLM-validator prompts; -- transcript description context remains background-only and non-authoritative, and empty descriptions do not render blank sections; -- prompt metadata is propagated into proposal-generation and LLM-validator diagnostics request metadata. - -Current boundary: -- detailed prompt metadata is currently diagnostics-scoped; reports are not broadly redesigned for prompt metadata yet. -- filesystem prompt overrides, user-selectable prompt sources, scheduler utilization diagnostics, correction ledgers, and generated summaries remain planned/deferred. - -This phase pairs naturally with the validator refactor because both modules and LLM-backed validators need prompt assets, stable prompt IDs, prompt versions, and prompt diagnostics. - -## Built-in embedded prompts - -Use Go's `embed` package to compile built-in Markdown prompts into the Audita binary. - -A possible layout: - -- `internal/prompts` - - registry; - - loader; - - renderer; - - metadata. -- `internal/prompts/embedded/modules/glossary` -- `internal/prompts/embedded/modules/homophones` -- `internal/prompts/embedded/modules/spoken_word` -- `internal/prompts/embedded/modules/grammar` -- `internal/prompts/embedded/validators/spoken_form_plausibility` -- `internal/prompts/embedded/validators/meaning_reversal` -- `internal/prompts/embedded/validators/editorial_review` -- `internal/prompts/embedded/validators/grammar_review` -- `internal/prompts/embedded/validators/spoken_word_review` - -For 1.0, built-in embedded prompts should be the only supported prompt source. Filesystem prompt overrides can be considered later. - -## Prompt rendering - -Use Go `text/template` with typed template data structs. - -Rendering should use missing-key errors so prompt changes do not silently omit required data. - -The Markdown files should contain the natural-language instruction text. Go code should still own: - -- typed request and response structs; -- output schema enforcement; -- transcript section formatting; -- glossary formatting; -- module and validator selection; -- diagnostics; -- template data construction. - -## Prompt metadata - -Every prompt should have: - -- stable prompt ID; -- prompt version; -- source type, initially `builtin`; -- file path within the embedded prompt registry; -- SHA-256 hash of the source or rendered prompt text. - -Reports and diagnostics should include enough prompt metadata to reproduce or debug behavior later. - -Suggested prompt metadata fields: - -- `prompt_id` -- `prompt_version` -- `prompt_source` -- `prompt_sha256` - -## Prompt-injection hardening - -Add a shared hardening fragment to every module and LLM-validator prompt. - -Core policy: - -- transcript text is untrusted data; -- glossary entries are reference data, not instructions; -- the model must not obey instructions contained in transcript text; -- the model must perform only the requested correction or validation task; -- the model must not invent facts, names, events, motivations, or speaker intent; -- the transcript remains the source of truth for what was said. - -This hardening language should be centralized so it is consistently applied. - -## Transcript context support - -Add support for a brief transcript description, supplied by the user. - -Recommended CLI/config surface: - -- `--transcript-description ` -- `context.description: ` - -The description should be included in every proposal and validator prompt as background context only. - -The prompt should clearly state that the description may help interpret ambiguous terms but must not override the transcript. - -Implementation status (2026-05-13): -- implemented via `audita process --transcript-description `; -- stored in runtime config as transcript description context and propagated through proposal-generation and LLM-validator prompt builders; -- prompt text explicitly marks this context as background-only and non-authoritative; -- prompt text explicitly forbids inventing corrections, facts, names, events, motivations, or speaker intent from the description; -- empty descriptions do not add blank context sections. - -Generated transcript descriptions should remain opt-in or deferred. If implemented before 1.0, they should be: - -- explicitly requested; -- one sentence; -- cached in diagnostics; -- clearly labeled as generated; -- never treated as authoritative. - -## Deliverables - -- Embedded Markdown prompt files for all modules and LLM-backed validators. -- Prompt registry with stable IDs and versions. -- Typed prompt rendering layer. -- Shared prompt hardening fragment. -- Prompt metadata in diagnostics and reports. -- Transcript description plumbed through proposal and validator prompts. -- Documentation for built-in prompts and prompt metadata. - -## Tests - -- Every registered prompt renders successfully. -- Missing template data fails. -- Rendered prompts include prompt-injection hardening language. -- Rendered module prompts include transcript context when supplied. -- Rendered glossary prompts include glossary context. -- Prompt metadata appears in diagnostics. -- Prompt hashes are deterministic. -- Existing fake-LLM tests continue passing. - ---- - -# Phase 5: Add utilization diagnostics and correction review artifacts - -## Goal - -Improve operational observability so Audita runs can be debugged and performance-tuned after the fact. - -This phase should follow the registry/prompt work so diagnostics can include stable module, validator, prompt, and schema identifiers. - -## Implementation status (2026-05-13) - -This workstream is now implemented: -- run diagnostics include `utilization-diagnostics.json` with run-level and module-level timing/utilization data; -- utilization diagnostics include per-validator timing summaries keyed by stable validator keys; -- process reports include diagnostics references for utilization and correction-ledger artifacts; -- run diagnostics include `correction-ledger.json` with flattened correction disposition records (`applied`, `rejected`, `skipped`, `failed`) keyed by stable module instance and proposal index; -- successful runs emit both artifacts; failure paths emit partial artifacts when runner state exists. - -This status update applies only to utilization diagnostics and correction-ledger artifacts. Release-hardening evaluation fixtures and later roadmap items remain planned. - -## Scheduler utilization diagnostics - -Add module-level and run-level metrics for LLM scheduling and execution. - -Suggested fields: - -- total proposal LLM calls; -- total validation LLM calls; -- scheduler queue wait time; -- LLM execution time; -- deterministic validation time; -- total module wall time; -- max observed in-flight LLM calls; -- average observed in-flight LLM calls, if easy to compute; -- total/proposal/validation concurrency limits in effect; -- per-module timing summary; -- per-validator timing summary. - -This should be emitted as machine-readable diagnostics and summarized in the process report. - -A diagnostics artifact such as `scheduler-utilization.json` or `timing-summary.json` is sufficient. - -## Correction ledger - -Add a normalized machine-readable correction ledger that records each proposed correction and its final disposition. - -This may be a JSON array or JSONL file. - -Each record should identify: - -- run ID; -- module key; -- module instance; -- section ID or section range; -- proposal index; -- segment ID; -- speaker; -- original text; -- proposed corrected text; -- replacement policy; -- confidence; -- deterministic validator decisions; -- LLM validator decisions; -- final disposition: applied, rejected, skipped, or failed; -- reason codes; -- diagnostics artifact references where available. - -This ledger should not replace existing reports. It should provide a flattened review-friendly artifact across all modules. - -## Deliverables - -- Scheduler/timing metrics collection. -- Run-level timing diagnostics artifact. -- Module-level utilization summaries. -- Correction ledger artifact. -- Report references to the new artifacts. -- Documentation for interpreting utilization and correction ledger fields. - -## Tests - -- Metrics are emitted on successful runs. -- Metrics are emitted or partially emitted on failed runs where possible. -- Scheduler permit release behavior remains correct. -- Correction ledger includes applied, rejected, and skipped examples. -- Secret redaction still applies. -- Clean successful runs still respect work-dir retention behavior. - ---- - -# Phase 6: Release hardening and 1.0 documentation pass - -## Goal - -Finish 1.0 by turning the new architecture into a documented, tested, stable release candidate. - -## Implementation status (2026-05-13) - -This workstream is now implemented: -- architecture, contract, configuration, output-schema, structured-LLM, validator, prompt, diagnostics, and subprocess docs are aligned to the current runtime; -- a practical release checklist document exists (`docs/release-checklist.md`); -- curated release fixtures exist under `internal/cli/testdata/release` and run under `go test ./...` using fake structured LLM responses only; -- release fixture coverage includes: - - must-apply and must-not-apply expectations; - - protected-term survival; - - default module sequence and stable instance naming; - - report schema metadata; - - diagnostics artifact references; - - output schema checks (`bare-segments`, `audita-v1`); - - prompt/structured-schema diagnostics metadata presence; - - stable validator keys; - - utilization diagnostics and correction-ledger artifacts; - - idempotence-oriented second-pass no-op expectations with deterministic fake responses. - -Deferred work remains deferred (no filesystem prompt overrides, no user-configurable validator chains, no arbitrary user-supplied output schemas, no resume/start-at/stop-after execution, no diff/check/propose-only modes, no generated transcript descriptions enabled by default, no interactive review UI, no UI/server wrapper, and no provider-specific benchmarking harness). - -## Documentation updates - -Update or add: - -- `README.md` -- `docs/architecture.md` -- `docs/public-contract.md` -- `docs/configuration.md` -- `docs/output-schemas.md` -- `docs/structured-llm.md` -- `docs/validators.md` -- `docs/prompts.md` -- `docs/subprocess-operations.md` -- `docs/release-checklist.md` - -The README should stay concise. Detailed contract and architecture material should live in the docs directory. - -## Evaluation fixtures - -Add a small curated evaluation set for release confidence. - -Each fixture should track: - -- expected must-apply corrections; -- expected must-not-apply corrections; -- protected terms that must survive; -- expected module sequence; -- expected broad applied/rejected/skipped counts; -- idempotence expectations for a second pass where practical. - -The purpose is not to perfectly score LLM behavior across all providers. The purpose is to catch prompt, validator, schema, and pipeline regressions before 1.0. - -## Idempotence check - -Add at least one test or documented manual release check where Audita is run twice on the same transcript. - -The second run should ideally be close to a no-op. If it is not, the result should be explainable. - -## Release checklist - -Create a checklist covering: - -- structured LLM adapter behavior; -- structured response schema IDs and versions; -- dependency tree and binary size review; -- config validation; -- output schemas; -- subprocess contract; -- report schema; -- diagnostics redaction; -- prompt metadata; -- validator chain metadata; -- scheduler utilization diagnostics; -- correction ledger; -- default full-pipeline run; -- explicit module runs; -- failure diagnostics; -- cancellation/timeout behavior; -- clean stdout/stderr behavior. - -## Deliverables - -- Updated docs. -- Release checklist. -- Curated evaluation fixtures. -- Final architecture update reflecting the new package layout. -- Final public contract review. -- Optional migration notes from pre-1.0 CLI/env behavior. - -## Tests - -- `go test ./...` -- CLI integration tests for config and output schemas. -- Subprocess behavior tests. -- Structured LLM adapter tests. -- Structured response schema tests. -- Prompt rendering tests. -- Validator registry tests. -- Report/diagnostics shape tests. -- Redaction tests. -- Fixture/evaluation tests that can run without live LLM credentials, using fake structured responses. - ---- - -# Suggested commit grouping - -The phases above can map cleanly to a series of implementation prompts or pull-request commits. - -Recommended grouping: - -1. Replace `instructor-go` with an Audita-owned structured LLM adapter. -2. Add structured response schema registry and schema metadata diagnostics. -3. Add versioned config file support and config commands. -4. Stabilize CLI/config precedence and update documentation. -5. Add output schema registry and public contract docs. -6. Refactor validators into registry-backed composable chains. -7. Move prompts into embedded Markdown assets with prompt registry metadata. -8. Add shared prompt-injection hardening and transcript description support. -9. Add scheduler utilization diagnostics. -10. Add correction ledger artifact. -11. Complete docs, evaluation fixtures, and release checklist. - -The structured LLM adapter and structured response schema registry should be completed before the prompt registry work. The validator and prompt work can be developed in parallel if the interfaces are agreed first, but they should be merged carefully because both affect module construction, reports, diagnostics, and tests. - -# Proposed 1.0 completion criteria - -Audita is ready for 1.0 when the following are true: - -- `instructor-go` has been removed. -- Audita owns its structured LLM adapter behind an internal interface. -- Structured response schemas have stable IDs, versions, and hashes. -- A user can run Audita with a concise command and a versioned config file. -- Secrets are not stored in config files or surfaced in diagnostics. -- The public subprocess contract is documented and tested. -- The output schema is selected from a small supported registry. -- Validators have stable keys and explicit built-in chains. -- Prompts are embedded Markdown assets with stable IDs, versions, and hashes. -- All proposal and validator prompts include prompt-injection hardening. -- Transcript description context can be supplied explicitly. -- Reports include module, validator, prompt, structured response schema, output schema, and timing metadata. -- Scheduler utilization diagnostics make concurrency behavior observable. -- A correction ledger exists for review and debugging. -- Existing full-pipeline behavior remains compatible with current use cases. -- The default run path and explicit module paths are covered by tests. -- Failure diagnostics are retained and useful. -- Clean successful runs remain quiet on stdout/stderr according to the documented contract. - -# Deferred post-1.0 opportunities - -The following ideas remain attractive but should not block 1.0: - -- Filesystem prompt overrides. -- User-configurable validator chains. -- Arbitrary user-supplied output schemas. -- Resume/start-at/stop-after execution. -- `--diff`, `--check`, or `--propose-only` modes. -- Generated transcript descriptions enabled by default. -- Interactive correction review. -- UI or web service wrapper. -- Provider-specific prompt/model benchmarking harness. diff --git a/docs/validator-refactor.md b/docs/validator-refactor.md deleted file mode 100644 index cd4319b..0000000 --- a/docs/validator-refactor.md +++ /dev/null @@ -1,688 +0,0 @@ -# Validator Refactor Plan - -## Purpose - -This document describes a concrete package-ownership refactor for Audita's built-in validators. - -Target outcome: -- each built-in validator has its own package under `internal/validators//`; -- the rest of the application depends on the existing common validator runtime contract; -- validator-specific construction and configuration are owned by validator-specific packages; -- shared validator runtime machinery remains centralized where reuse is high; -- runner and module code do not need to know concrete validator implementation types. - -This is not a runtime redesign. Audita already has the important validator abstraction: the runner receives validators through a common interface, calls `Validate`, and consumes validator results without needing to understand validator internals. This plan preserves that model and improves package ownership around it. - -## Current state - -The current codebase already has the core runtime pieces: - -- `contracts.Validator` in `internal/framework/contracts`; -- `Validate(ctx, req)` semantics; -- validator request/result/decision models; -- runner orchestration that treats validators as decision producers; -- stable built-in validator keys; -- a built-in validator registry and built-in module chain definitions; -- deterministic and LLM-backed validator implementations; -- embedded prompt assets and prompt metadata for LLM-backed validators. - -Current validator ownership is split across two layers: - -- `internal/validators/` - - built-in key constants; - - built-in registry; - - built-in module chain definitions. - -- `internal/framework/validators/` - - request/result/decision models; - - deterministic validator implementation details; - - generic LLM-backed validator runtime; - - batching; - - prompt/response and diagnostics helpers; - - cardinality enforcement and shared validation helpers. - -This refactor should make `internal/validators//` the visible home of each built-in validator, while keeping shared runtime machinery in `internal/framework/validators`. - -## Goals - -Required goals: - -- Mirror the module package pattern by giving each built-in validator its own package under `internal/validators`. -- Preserve the existing `contracts.Validator` runtime contract. -- Keep built-in validator keys stable. -- Keep built-in module validator chains stable. -- Preserve current runtime behavior and report shape unless explicitly noted. -- Remove concrete validator type knowledge from the runner. -- Make production registry construction route through validator-owned packages. -- Localize the `protected_terms` glossary-stage special case inside the `protected_terms` validator package. -- Keep LLM batching, diagnostics, response parsing, cardinality checks, and shared helper behavior centralized. - -Secondary goals: - -- Make validator construction read similarly to module construction. -- Keep validator-specific tests close to validator-specific packages. -- Reduce the impression that `internal/framework/validators` owns the built-in validator catalog. -- Make future built-in validators easier to add without expanding a central framework file. - -## Non-goals - -Do not implement any of the following as part of this refactor: - -- User-defined validator plugins. -- User-configurable validator chains. -- New validator types. -- Changes to validator decision cardinality rules. -- Changes to prompt assets or prompt text. -- Changes to structured response schemas. -- Changes to scheduler behavior. -- Changes to output schemas, reports, diagnostics shape, or correction ledger shape except where a stable validator key already appears. -- A new competing validator interface. - -## Architectural principle - -This should be a package-ownership cleanup over an already sound runtime abstraction. - -The correct bias is: - -- keep runtime semantics stable; -- move validator identity and construction into validator-owned packages; -- keep shared machinery centralized; -- remove concrete implementation leakage from the runner; -- prefer wrappers first and deeper cleanup second. - -## Target package layout - -Target end state: - -```text -internal/framework/validators/ - models.go - runtime.go - llm_runtime.go - llm_batching.go - diagnostics.go - cardinality.go - prompt_helpers.go - protected_vocabulary.go - -internal/validators/ - registry.go - chains.go - metadata.go - interfaces.go - -internal/validators/confidence_threshold/ - validator.go - validator_test.go - -internal/validators/original_text_presence/ - validator.go - validator_test.go - -internal/validators/non_empty_corrected_text/ - validator.go - validator_test.go - -internal/validators/no_effect/ - validator.go - validator_test.go - -internal/validators/protected_terms/ - validator.go - validator_test.go - -internal/validators/spoken_form_plausibility/ - validator.go - validator_test.go - -internal/validators/meaning_reversal_review/ - validator.go - validator_test.go - -internal/validators/editorial_review/ - validator.go - validator_test.go - -internal/validators/grammar_review/ - validator.go - validator_test.go - -internal/validators/spoken_word_review/ - validator.go - validator_test.go -``` - -Notes: - -- `internal/framework/validators` remains the shared runtime layer. -- `internal/validators/` owns construction and validator-specific configuration. -- `internal/validators/registry.go` remains the built-in production registry entrypoint. -- `internal/validators/chains.go` remains the built-in module chain resolver. -- It is acceptable for `internal/framework/validators` to retain a generic shared LLM validator runtime type if only validator-owned packages construct it. - -## Runtime contract - -Keep `contracts.Validator` as the framework-facing contract. - -Do not introduce a second competing validator interface. If helper interfaces are needed, they should supplement the existing contract rather than replace it. - -The runner should continue to operate on validators as opaque components: - -- receive `[]contracts.Validator`; -- call `Validate(ctx, req)`; -- consume returned validator results; -- remove rejected proposals from the eligible set; -- preserve deterministic-before-LLM execution ordering; -- preserve existing decision cardinality rules. - -## Validator metadata - -Add a small metadata surface to avoid concrete implementation checks in the runner. - -Recommended location: - -- `internal/validators/interfaces.go` or `internal/validators/metadata.go` - -This package must stay low-level: - -- it may import `internal/framework/contracts`; -- it must not import validator-specific packages; -- it must not import the registry. - -Suggested API: - -```go -type ExecutionClass string - -const ( - ExecutionClassDeterministic ExecutionClass = "deterministic" - ExecutionClassLLMBacked ExecutionClass = "llm_backed" -) - -type ClassifiedValidator interface { - contracts.Validator - ExecutionClass() ExecutionClass -} -``` - -Runner behavior: - -- if a validator implements `ClassifiedValidator`, use `ExecutionClass()`; -- otherwise treat it as deterministic by default; -- never type-assert against concrete framework validator types such as `*frameworkvalidators.LLMBackedValidator`. - -This removes concrete implementation knowledge from the runner while preserving ordering semantics. - -## Validator package constructors - -Each validator package should expose a constructor that returns `contracts.Validator`. - -Use the simplest constructor that honestly reflects the validator's dependencies. - -Acceptable patterns: - -```go -func New() (contracts.Validator, error) -``` - -```go -func New(opts Options) (contracts.Validator, error) -``` - -```go -func NewGlossaryStage(opts Options) (contracts.Validator, error) -``` - -Guidance: - -- Use `New()` only when construction truly requires no runtime dependencies. -- Use `New(opts Options)` when the existing registry already supplies dependencies such as config, glossary, LLM client, scheduler, diagnostics context, or prompt metadata. -- Keep `Options` package-local unless several validators genuinely share the same option structure. -- Do not force zero-argument constructors if doing so would hide dependencies in globals or cause construction-time behavior to become implicit. -- Constructor names should make module-sensitive behavior explicit, especially for `protected_terms`. - -## Built-in validator registry - -Keep `internal/validators/registry.go` as the production wiring layer. - -After this refactor, registry entries should call validator-package constructors rather than framework concrete implementations. - -Example target shape: - -```go -{Key: KeyConfidenceThreshold, Build: confidencethreshold.New} -{Key: KeyGrammarReview, Build: grammarreview.New} -``` - -The registry should continue to provide: - -- stable validator keys; -- built-in validator metadata; -- clear lookup/build failures for unknown keys; -- stable production construction behavior. - -The registry should not become a user plugin system. - -## Built-in validator keys - -Preserve these keys exactly: - -- `confidence_threshold` -- `original_text_presence` -- `non_empty_corrected_text` -- `no_effect` -- `protected_terms` -- `spoken_form_plausibility` -- `meaning_reversal_review` -- `editorial_review` -- `grammar_review` -- `spoken_word_review` - -These keys must continue to appear consistently in: - -- registry entries; -- built-in chain definitions; -- reports; -- diagnostics metadata or paths where validator identity appears; -- correction ledger entries; -- utilization diagnostics; -- tests; -- documentation. - -## Built-in module chains - -Preserve the current effective built-in module chains exactly unless an existing test proves the documented chain differs from the runtime and the runtime behavior is clearly the intended source of truth. - -Do not use this refactor to revisit validator policy. - -The chain resolver may continue to live in `internal/validators/chains.go`, but it should construct validators through package-owned constructors. - -## Treatment of deterministic validators - -Move ownership into per-validator packages: - -- `internal/validators/confidence_threshold` -- `internal/validators/original_text_presence` -- `internal/validators/non_empty_corrected_text` -- `internal/validators/no_effect` -- `internal/validators/protected_terms` - -Implementation guidance: - -- If the implementation is short and self-contained, it may live directly in the validator package. -- If logic is shared, keep shared helpers in `internal/framework/validators`. -- Do not duplicate cardinality helpers or vocabulary helpers. -- Prefer behavior-preserving wrappers first, then move implementation details only where it clearly improves ownership. - -The goal is not to empty `internal/framework/validators`; the goal is to stop making it look like the built-in validator catalog. - -## Protected terms special case - -`protected_terms` has module-sensitive behavior and must be handled carefully. - -Target ownership: - -- `internal/validators/protected_terms` owns both general and glossary-stage construction. - -Suggested constructors: - -```go -func New(opts Options) (contracts.Validator, error) -func NewGlossaryStage(opts Options) (contracts.Validator, error) -``` - -or, if no options are needed: - -```go -func New() (contracts.Validator, error) -func NewGlossaryStage() (contracts.Validator, error) -``` - -Behavior requirements: - -- non-glossary modules keep existing protected-term behavior; -- glossary-stage behavior remains stricter if that is the current runtime behavior; -- both variants continue to report the stable key `protected_terms`; -- chain resolution should not directly reference framework concrete protected-term validator types. - -Add or preserve explicit tests for: - -- glossary-stage protected-term behavior; -- non-glossary protected-term behavior; -- stable `protected_terms` identity in reports and correction ledger entries. - -## Treatment of LLM-backed validators - -Move ownership into per-validator packages: - -- `internal/validators/spoken_form_plausibility` -- `internal/validators/meaning_reversal_review` -- `internal/validators/editorial_review` -- `internal/validators/grammar_review` -- `internal/validators/spoken_word_review` - -Recommended implementation: - -- each package provides a thin wrapper over shared LLM runtime machinery; -- each package owns the stable validator key and constructor; -- each package selects or configures the appropriate prompt ID/type through existing prompt registry surfaces; -- each package exposes `ExecutionClass() == llm_backed` either directly or through a wrapper; -- shared framework code continues to own batching, diagnostics, structured response parsing, model resolution, and decision mapping. - -Avoid this anti-pattern: - -- five copied versions of the generic LLM-backed validator runtime. - -Acceptable shared runtime: - -- a generic shared `LLMBackedValidator`; -- a shared `NewLLMBackedValidator(...)` factory; -- a shared LLM validator executor configured by validator packages. - -The important boundary is that validator-specific ownership is visible under `internal/validators/`, even if shared execution remains centralized. - -## Prompt ownership - -Do not change prompt assets in this refactor. - -Prompt assets already live under `internal/prompts`. Validator packages may own prompt selection/configuration by referring to existing prompt IDs, but they should not duplicate prompt text or move Markdown assets. - -For example, a package may configure the shared LLM runtime with: - -- validator key: `grammar_review`; -- prompt ID: `validators.grammar_review`; -- structured response schema: `validator_decision_set`; -- execution class: `llm_backed`. - -But this refactor should not alter prompt text, prompt metadata, prompt IDs, prompt hashes, or prompt rendering behavior except where imports must be adjusted. - -## Import-cycle rules - -Avoid import cycles by keeping dependencies acyclic. - -Preferred dependency direction: - -```text -internal/framework/contracts -internal/framework/validators -internal/prompts - ↑ -internal/validators/ - ↑ -internal/validators/registry and chains - ↑ -module construction / runner wiring -``` - -Rules: - -- validator-specific packages must not import `internal/validators/registry`; -- the registry may import validator-specific packages; -- runner may import low-level validator metadata but should not import validator-specific packages; -- metadata interfaces should live in a low-level package that does not import registry or validator-specific packages; -- shared framework runtime must not import the built-in registry. - -## Recommended implementation passes - -Use three implementation passes unless blocked. - -### Pass 1: Metadata, wrappers, and registry switch - -Purpose: -- introduce execution metadata; -- remove runner dependency on concrete framework validator types; -- create per-validator packages as thin wrappers; -- switch the registry to construct validators through those packages. - -Tasks: -- add validator execution classification metadata; -- update runner ordering logic to use metadata; -- create one package per built-in validator under `internal/validators`; -- keep wrappers behavior-preserving; -- preserve stable keys; -- preserve built-in chains; -- switch registry build closures to per-package constructors; -- add tests for registry completeness, metadata classification, and ordering behavior. - -Acceptance criteria: -- runner no longer type-asserts against framework concrete validator types; -- every built-in validator has a package; -- registry builds all validators through package constructors; -- deterministic-before-LLM behavior is unchanged; -- all existing tests pass. - -### Pass 2: Protected terms localization, framework cleanup, and test locality - -Purpose: -- localize the `protected_terms` special case; -- reduce framework built-in ownership; -- move or add tests near validator packages. - -Tasks: -- add explicit protected-terms constructors for general and glossary-stage behavior; -- update chain resolution to call those constructors; -- remove direct chain/registry references to protected-term framework concrete types; -- reduce exported framework validator types where package wrappers fully own construction; -- keep shared runtime helpers in `internal/framework/validators`; -- move or add validator-specific tests under the validator packages where practical; -- retain shared runtime tests in the framework package. - -Acceptance criteria: -- protected-term behavior is unchanged; -- glossary-stage behavior is explicitly tested; -- framework package no longer appears to own the built-in validator catalog; -- validator-specific behavior is tested near validator packages where practical; -- all existing tests pass. - -### Pass 3: Documentation and review - -Purpose: -- align docs with the final structure; -- verify no accidental behavior drift or scope creep. - -Tasks: -- update `docs/architecture.md`; -- update `docs/validators.md`; -- update any public-contract or diagnostics docs only if validator package ownership needs mention there; -- document the distinction between validator-owned packages and shared validator runtime; -- document the 1.0 boundary that validator chains remain built-in and not user-configurable; -- run `go test ./...`; -- review imports and package boundaries for accidental cycles or leakage. - -Acceptance criteria: -- docs describe the new package layout accurately; -- docs do not imply plugin support or user-configurable chains; -- tests pass; -- git diff shows package-ownership refactor only. - -## File-by-file change guide - -Expected direct edits: - -- `internal/framework/runner/runner.go` - - replace concrete LLM validator type check with execution metadata interface check. - -- `internal/validators/interfaces.go` or `internal/validators/metadata.go` - - define execution classification types and optional interface. - -- `internal/validators/registry.go` - - import per-validator packages; - - route construction through package constructors; - - preserve stable keys. - -- `internal/validators/chains.go` - - preserve chain key lists and order; - - use protected-terms glossary-stage constructor where appropriate; - - avoid concrete framework validator types. - -- `internal/framework/validators/deterministic.go` - - reduce or remove exported built-in validator concrete types if wrappers fully own construction; - - keep shared helpers where useful. - -- `internal/framework/validators/llm_validators.go` - - keep shared LLM runtime; - - reduce exported surface only if safe; - - do not duplicate LLM runtime across validator packages. - -Expected new files: - -- `internal/validators/confidence_threshold/validator.go` -- `internal/validators/original_text_presence/validator.go` -- `internal/validators/non_empty_corrected_text/validator.go` -- `internal/validators/no_effect/validator.go` -- `internal/validators/protected_terms/validator.go` -- `internal/validators/spoken_form_plausibility/validator.go` -- `internal/validators/meaning_reversal_review/validator.go` -- `internal/validators/editorial_review/validator.go` -- `internal/validators/grammar_review/validator.go` -- `internal/validators/spoken_word_review/validator.go` -- corresponding package-local tests where practical. - -Expected documentation edits: - -- `docs/architecture.md` -- `docs/validators.md` - -Possibly update: - -- `docs/public-contract.md` -- `docs/diagnostics.md` - -Only update README if the high-level project description would otherwise be inaccurate. - -## Test strategy - -Recommended test distribution: - -- `internal/validators//validator_test.go` - - package-specific constructor and behavior tests. - -- `internal/validators/registry_test.go` - - registry wiring and key coverage. - -- `internal/validators/module_chains_test.go` - - built-in module chain coverage. - -- `internal/framework/runner/runner_test.go` - - orchestration semantics only, including deterministic-before-LLM ordering. - -- `internal/framework/validators/*_test.go` - - shared runtime behavior only. - -Required test coverage: - -- every built-in validator package exists and constructs successfully; -- every built-in key resolves through the registry; -- every built-in module chain resolves; -- unknown validator keys fail deterministically; -- runner ordering uses execution classification, not concrete type checks; -- unclassified validators default to deterministic; -- protected-terms glossary-stage behavior is preserved; -- stable validator keys remain present in reports, diagnostics, utilization diagnostics, and correction ledger entries where currently applicable; -- no live LLM credentials are needed for `go test ./...`. - -## Ordering and safety rules - -Implementation rules: - -- preserve validator keys exactly; -- preserve built-in module chain order exactly; -- preserve `protected_terms` glossary-stage behavior exactly; -- preserve deterministic-before-LLM ordering; -- preserve decision cardinality enforcement; -- preserve report field names and validator identity strings; -- preserve prompt assets and prompt metadata; -- preserve structured response schemas; -- preserve scheduler semantics; -- preserve output schemas; -- avoid duplicating shared LLM runtime code; -- keep tests passing after each implementation pass. - -## Risks and mitigations - -### Risk: accidental behavior drift in protected terms - -Why it matters: -- `protected_terms` has module-sensitive behavior. - -Mitigation: -- add or preserve explicit tests for glossary-stage and non-glossary-stage behavior; -- use named constructors rather than hidden request-based branching where the stage variant is selected at chain build time. - -### Risk: runner ordering regressions - -Why it matters: -- deterministic validators must still run before LLM-backed validators. - -Mitigation: -- add or retain tests proving ordering with a mixed validator list; -- treat unknown or unclassified validators as deterministic. - -### Risk: over-duplicating LLM validator code - -Why it matters: -- LLM-backed validators share batching, diagnostics, parsing, and structured-output handling. - -Mitigation: -- keep one shared LLM runtime implementation; -- use thin validator-package wrappers. - -### Risk: import cycles - -Why it matters: -- registry imports validator packages; -- validator packages import contracts and shared framework runtime; -- runner imports contracts and metadata. - -Mitigation: -- keep metadata low-level; -- keep registry out of validator-specific packages; -- keep shared framework runtime independent of the built-in registry. - -### Risk: excessive cleanup during refactor - -Why it matters: -- broader cleanup can obscure behavior changes and increase review risk. - -Mitigation: -- wrappers first; -- deeper cleanup only after registry construction is package-owned; -- do not combine with prompt, scheduler, report, config, or output-schema changes. - -## Acceptance criteria - -The refactor is complete when all of the following are true: - -- every built-in validator has its own package under `internal/validators`; -- production validator registry constructs validators via those packages; -- module packages still work with `[]contracts.Validator` and require no validator implementation knowledge; -- runner has no concrete-type dependency on framework validator implementations; -- `protected_terms` glossary-stage handling is owned by the validator package; -- built-in validator keys are unchanged; -- built-in module chain order and behavior are unchanged; -- LLM-backed validators still use shared runtime machinery; -- prompt assets and structured response schemas are unchanged; -- tests pass with `go test ./...`; -- docs describe the new package ownership model. - -## Suggested follow-up cleanup after this refactor - -Not required for this refactor, but worth considering later: - -- introduce a validator factory in `internal/framework/validators` only if future dynamic construction becomes necessary; -- consider whether `contracts.ValidationRequest = validators.Request` should remain an alias or become a contract-owned type; -- consider whether chain definitions should eventually move closer to module registry or module packages if module-specific validator policy becomes more configurable; -- consider whether `editorial_review` remains useful if no production module uses it; -- consider a later user-configurable chain system only if there is a clear product need. - -## Summary - -This refactor should make validators mirror modules at the package ownership level without changing Audita's runtime model. - -The desired end state is: - -- validator packages own built-in validator construction; -- the registry owns production wiring; -- the runner owns orchestration only; -- shared framework code owns reusable runtime mechanics; -- tests and docs reflect those boundaries. diff --git a/python/README.md b/python/README.md deleted file mode 100644 index 4052b3c..0000000 --- a/python/README.md +++ /dev/null @@ -1,157 +0,0 @@ -# Audita - -```text -This directory contains the frozen Python implementation of Audita. -It is retained as the behavioral reference for the Go port. - -The canonical implementation is now the Go code at the repository root. -This Python implementation is legacy/reference and is no longer the primary operational path. -Do not add new features here except to fix reference-test issues needed for port validation. -``` - -Audita is a framework-first transcript correction application. The public `audita` package provides: - -- deterministic transcript normalization -- token-batched module orchestration -- concrete `glossary`, `homophones`, `spoken_word`, and `grammar` modules built on reusable proposal / validator contracts -- structured run reporting and work-dir diagnostics - -The previous working implementation has been preserved as `audita_prototype` inside this repository. Its full regression suite lives under `tests/audita_prototype`. - -## Development - -This project is set up for `uv`. - -```sh -uv sync --extra dev -uv run pytest -``` - -## Usage - -Process a transcript with the current framework implementation: - -```sh -uv run audita process transcript.json --glossary glossary.yaml --output corrected.json -``` - -The framework currently runs this default module sequence: - -1. `glossary` -2. `homophones` -3. `glossary` -4. `spoken_word` -5. `grammar` - -Resolved run instance names are auto-numbered for repeats, so the default report pipeline is: - -1. `glossary_1` -2. `homophones` -3. `glossary_2` -4. `spoken_word` -5. `grammar` - -The default module sequence is fully implemented today: - -- `glossary` proposes glossary-supported acoustic corrections -- `homophones` proposes conservative homophone and mistranscription corrections -- `spoken_word` proposes conservative dysfluency cleanup -- `grammar` proposes punctuation, capitalization, and spacing cleanup only - -To run a custom module sequence, pass `--modules`: - -```sh -uv run audita process transcript.json --glossary glossary.yaml --modules grammar --output corrected.json -``` - -To also write a structured JSON report: - -```sh -uv run audita process transcript.json --glossary glossary.yaml --output corrected.json --report-json report.json -``` - -From a checked-out repository, you can also use the root launcher: - -```sh -./audita process transcript.json --glossary glossary.yaml --output corrected.json -``` - -For a system-wide command, install the source tree under `/usr/local/src/audita`, sync dependencies there, and symlink the root launcher into your `PATH`: - -```sh -cd /usr/local/src/audita -uv sync --extra dev -ln -s /usr/local/src/audita/audita /usr/local/bin/audita -audita process transcript.json --glossary glossary.yaml --output corrected.json -``` - -Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr. -`--report-json` writes a separate machine-readable run report and never mixes report data into stdout. - -Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Default OpenRouter runs require LLM API credentials, because the `glossary`, `homophones`, `spoken_word`, and `grammar` modules make real LLM calls. Self-hosted or other non-default OpenAI-compatible endpoints may not require credentials. `AUDITA_LLM_API_KEY` and `--llm-api-key` are the preferred provider-neutral credential surfaces, while `OPENROUTER_API_KEY` remains supported as a backward-compatible fallback. - -| Environment variable | CLI flag | Default | Purpose | -| --- | --- | --- | --- | -| `AUDITA_MODULES` | `--modules` | `glossary,homophones,glossary,spoken_word,grammar` | Comma-separated logical module keys to run; CLI overrides the environment value | -| `AUDITA_LLM_API_KEY` | `--llm-api-key` | unset | Preferred provider-neutral LLM API credential; required for the default OpenRouter endpoint and optional for non-default endpoints; CLI overrides both environment-key variants | -| `AUDITA_VALIDATION_LLM_API_KEY` | `--validation-llm-api-key` | unset | Validation-phase LLM API credential; defaults to the primary LLM API key and is optional for non-default validation endpoints | -| `AUDITA_MODEL` | `--model` | `openrouter/google/gemma-4-31b-it` | LLM model name sent to the configured OpenAI-compatible endpoint | -| `AUDITA_VALIDATION_MODEL` | `--validation-model` | unset | Validation-phase LLM model; defaults to `AUDITA_MODEL` | -| `AUDITA_BASE_URL` | `--base-url` | `https://openrouter.ai/api/v1` | OpenAI-compatible API base URL | -| `AUDITA_VALIDATION_BASE_URL` | `--validation-base-url` | unset | Validation-phase OpenAI-compatible API base URL; defaults to `AUDITA_BASE_URL` | -| `AUDITA_LLM_TIMEOUT_SECONDS` | `--llm-timeout-seconds` | `600` | Per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint | -| `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | `--validation-llm-timeout-seconds` | unset | Validation-phase per-request timeout in seconds; defaults to `AUDITA_LLM_TIMEOUT_SECONDS` | -| `AUDITA_VALIDATION_MAX_PROMPT_TOKENS` | `--validation-max-prompt-tokens` | `2048` | Maximum estimated tokens per validation-phase LLM prompt batch | -| `AUDITA_TARGET_SECTIONS` | `--target-sections` | unset | Exact number of contiguous proposal-stage transcript sections; errors if min/max token bounds cannot be satisfied | -| `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses | -| `AUDITA_VALIDATION_MAX_RETRIES` | `--validation-max-retries` | unset | Validation-phase structured-output retries; defaults to `AUDITA_MAX_RETRIES` | -| `AUDITA_VALIDATION_LLM_CONCURRENCY` | `--validation-llm-concurrency` | unset | Validation-phase LLM concurrency; defaults to `AUDITA_LLM_CONCURRENCY` | -| `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `8192` | Maximum estimated tokens per proposal-stage transcript section | -| `AUDITA_MIN_SECTION_TOKENS` | `--min-section-tokens` | `2048` | Minimum estimated tokens per proposal-stage transcript section when balancing for concurrency | -| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation | -| `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | `--grammar-confidence-threshold` | `0.8` | Minimum confidence required for grammar proposals to survive validation | -| `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | `--homophones-confidence-threshold` | `0.8` | Minimum confidence required for homophone proposals to survive validation | -| `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | `--spoken-word-confidence-threshold` | `0.8` | Minimum confidence required for spoken-word proposals to survive validation | -| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | `--normalize-max-segment-gap` | `4.0` | Same-speaker gaps eligible for deterministic merging | -| `AUDITA_NORMALIZE_ELLIPSIS_GAP` | `--normalize-ellipsis-gap` | `3.5` | Same-speaker gaps above this value are joined with ` ... ` | -| `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | `--normalize-max-segment-duration` | `60.0` | Maximum merged segment duration | -| `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | `--normalize-max-segment-tokens` | `2048` | Maximum merged segment prompt payload size | -| `AUDITA_WORK_DIR` | `--work-dir` | `/tmp/audita` | Per-run scratch diagnostics directory | -| `AUDITA_WORK_DIR_RETENTION` | `--work-dir-retention` | `auto` | Whether to retain the per-run work directory: `auto`, `always`, or `never` | - -Set `AUDITA_MODULES=grammar` to run only the grammar module by default, or override it per command with `--modules`. - -Validation-phase LLM settings inherit from the primary `AUDITA_*` LLM settings by default. Set any of the `AUDITA_VALIDATION_*` values only when you want LLM-backed validators to use a different model, endpoint, credential, timeout, retry budget, or concurrency level. - -OpenRouter remains the default out of the box: - -```sh -export AUDITA_LLM_API_KEY=your-openrouter-key -audita process transcript.json --glossary glossary.yaml --output corrected.json -``` - -You can point Audita at any OpenAI-compatible endpoint by changing `AUDITA_BASE_URL` and, if needed, `AUDITA_MODEL`. For example, a local vLLM server: - -```sh -export AUDITA_BASE_URL=http://localhost:8000/v1 -export AUDITA_MODEL=meta-llama/Llama-3.1-8B-Instruct -audita process transcript.json --glossary glossary.yaml --output corrected.json -``` - -If your self-hosted endpoint requires authentication, you can still set `AUDITA_LLM_API_KEY`; Audita simply no longer requires it for non-default endpoints. - -Or the actual OpenAI API: - -```sh -export AUDITA_LLM_API_KEY=your-openai-key -export AUDITA_BASE_URL=https://api.openai.com/v1 -export AUDITA_MODEL=gpt-4.1-mini -audita process transcript.json --glossary glossary.yaml --output corrected.json -``` - -`AUDITA_WORK_DIR` stores per-run diagnostics while processing. Under the default `AUDITA_WORK_DIR_RETENTION=auto`, clean successful runs are removed, while failed runs and successful runs with final skipped corrections are preserved. Use `always` to keep every run directory and `never` to remove successful run directories even when skips remain. -Failed runs always preserve the run directory and include an authoritative `report.json` alongside normalization and prompt/response diagnostics. - -## Prototype Archive - -The archived prototype remains importable as `audita_prototype` and is still covered by its original regression suite. This is intentional: the new `audita` package is a framework-oriented rewrite, not a thin wrapper around the old code. diff --git a/python/audita b/python/audita deleted file mode 100755 index 658472e..0000000 --- a/python/audita +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python3 -import json -import os -from datetime import datetime -import shutil -import subprocess -import sys -from pathlib import Path -from typing import Optional -from uuid import uuid4 - - -DEFAULT_WORK_DIR = "/tmp/audita" -_SECRET_FLAGS = {"--llm-api-key", "--validation-llm-api-key"} - - -def _redact_argv(argv: list[str]) -> list[str]: - redacted: list[str] = [] - index = 0 - while index < len(argv): - arg = argv[index] - matched_flag = next((flag for flag in _SECRET_FLAGS if arg == flag or arg.startswith(flag + "=")), None) - if matched_flag is None: - redacted.append(arg) - index += 1 - continue - if arg == matched_flag: - redacted.append(arg) - if index + 1 < len(argv): - redacted.append("[REDACTED]") - index += 2 - else: - index += 1 - continue - redacted.append(f"{matched_flag}=[REDACTED]") - index += 1 - return redacted - - -def _resolve_work_root(argv: list[str]) -> Path: - for index, arg in enumerate(argv): - if arg == "--work-dir" and index + 1 < len(argv): - return Path(argv[index + 1]) - if arg.startswith("--work-dir="): - return Path(arg.split("=", 1)[1]) - return Path(os.environ.get("AUDITA_WORK_DIR") or DEFAULT_WORK_DIR) - - -def _create_run_dir(root: Path) -> Path: - root.mkdir(parents=True, exist_ok=True) - timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") - run_dir = root / f"run-{timestamp}-{uuid4().hex[:8]}" - run_dir.mkdir(parents=False, exist_ok=False) - return run_dir - - -def _capture_run_dirs(root: Path) -> set[str]: - if not root.exists(): - return set() - return {path.name for path in root.iterdir() if path.is_dir() and path.name.startswith("run-")} - - -def _find_new_run_dir(root: Path, before: set[str]) -> Optional[Path]: - if not root.exists(): - return None - candidates = [ - path for path in root.iterdir() if path.is_dir() and path.name.startswith("run-") and path.name not in before - ] - if not candidates: - return None - return max(candidates, key=lambda path: path.name) - - -def _write_launcher_error_log( - path: Path, - *, - message: str, - exit_code: int, - argv: list[str], - command: Optional[list[str]], -) -> None: - payload = { - "timestamp": datetime.utcnow().isoformat() + "Z", - "message": message, - "exit_code": exit_code, - "argv": argv, - "cwd": os.getcwd(), - "command": command, - } - path.write_text( - "Audita Launcher Diagnostics\n" + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8", - ) - - -def _emit_console_line(message: str) -> None: - for stream in (sys.stderr, sys.stdout): - if stream is None: - continue - try: - stream.write(f"{message}\n") - stream.flush() - return - except (OSError, ValueError): - continue - - -def main() -> int: - argv = list(sys.argv[1:]) - work_root = _resolve_work_root(argv) - redacted_argv = _redact_argv(argv) - uv = shutil.which("uv") - if uv is None: - run_dir = _create_run_dir(work_root) - error_log = run_dir / "error.log" - message = "uv is required to run this launcher. Install uv and run `uv sync` in the Audita project." - _write_launcher_error_log(error_log, message=message, exit_code=1, argv=redacted_argv, command=None) - _emit_console_line(f"audita: error: {message}") - _emit_console_line("audita: exit code: 1") - _emit_console_line(f"audita: run directory: {run_dir}") - _emit_console_line(f"audita: error log: {error_log}") - return 1 - - project_root = Path(__file__).resolve().parent - command = [uv, "run", "--project", str(project_root), "python", "-m", "audita", *sys.argv[1:]] - before = _capture_run_dirs(work_root) - result = subprocess.run(command, cwd=project_root, check=False) - if result.returncode == 0: - return 0 - if _find_new_run_dir(work_root, before) is None: - run_dir = _create_run_dir(work_root) - error_log = run_dir / "error.log" - _write_launcher_error_log( - error_log, - message=f"Audita subprocess exited with status {result.returncode}.", - exit_code=result.returncode, - argv=redacted_argv, - command=_redact_argv(command), - ) - _emit_console_line(f"audita: subprocess exited with status {result.returncode}") - _emit_console_line(f"audita: run directory: {run_dir}") - _emit_console_line(f"audita: error log: {error_log}") - return result.returncode - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python/pyproject.toml b/python/pyproject.toml deleted file mode 100644 index 8379d9f..0000000 --- a/python/pyproject.toml +++ /dev/null @@ -1,35 +0,0 @@ -[project] -name = "audita" -version = "0.2.0" -description = "Framework-first audio transcript correction pipeline with an archived prototype." -readme = "README.md" -requires-python = ">=3.9" -license = { text = "BSD-3-Clause" } -dependencies = [ - "instructor>=1.7", - "openai>=1.55", - "pydantic>=2.8", - "PyYAML>=6.0", - "tiktoken>=0.8", - "eval-type-backport>=0.2.0; python_version < '3.10'", -] - -[project.optional-dependencies] -dev = [ - "pytest>=8.0", -] - -[project.scripts] -audita = "audita.cli:main" - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["src/audita", "src/audita_prototype"] - -[tool.pytest.ini_options] -pythonpath = ["src"] -testpaths = ["tests"] -addopts = "-ra" diff --git a/python/src/audita/__init__.py b/python/src/audita/__init__.py deleted file mode 100644 index 00fb1a4..0000000 --- a/python/src/audita/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Audita modular transcript correction framework.""" - -__all__ = ["__version__"] - -__version__ = "0.2.0" diff --git a/python/src/audita/__main__.py b/python/src/audita/__main__.py deleted file mode 100644 index a049ad7..0000000 --- a/python/src/audita/__main__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .cli import main - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python/src/audita/cli.py b/python/src/audita/cli.py deleted file mode 100644 index 824c207..0000000 --- a/python/src/audita/cli.py +++ /dev/null @@ -1,253 +0,0 @@ -import argparse -import os -import sys -from pathlib import Path -from typing import Optional, Sequence - -from .core.config import AuditaConfig, ConfigOverrides -from .core.diagnostics import build_error_details, create_run_dir, format_stderr_summary, format_traceback_text, redact_argv, write_error_log -from .core.errors import AuditaError -from .core.io import load_glossary, load_transcript, write_report, write_transcript -from .core.reporting import RunReport -from .core.schemas import transcript_to_json -from .pipeline import process_transcript_result - - -def main(argv: Optional[Sequence[str]] = None) -> int: - parser = _build_parser() - args = parser.parse_args(argv) - if args.command == "process": - return _process(args, raw_argv=argv) - parser.print_help(sys.stderr) - return 2 - - -def _build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(prog="audita") - subparsers = parser.add_subparsers(dest="command", required=True) - process = subparsers.add_parser("process", help="run the modular transcript correction framework") - process.add_argument("transcript", type=Path, help="path to the input transcript JSON") - process.add_argument("--glossary", type=Path, required=True, help="path to the glossary YAML") - process.add_argument("--output", type=Path, help="write corrected transcript JSON to this path") - process.add_argument("--report-json", type=Path, help="write structured run report JSON to this path") - process.add_argument("--llm-api-key", help="LLM API key for the configured OpenAI-compatible endpoint") - process.add_argument("--llm-concurrency", type=int, help="maximum concurrent LLM calls within a module stage") - process.add_argument( - "--llm-timeout-seconds", - type=float, - help="per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint", - ) - process.add_argument( - "--validation-llm-api-key", - help="LLM API key for validation phases; defaults to the primary configured OpenAI-compatible endpoint key", - ) - process.add_argument( - "--validation-llm-concurrency", - type=int, - help="maximum concurrent LLM calls within validation phases; defaults to --llm-concurrency", - ) - process.add_argument( - "--validation-llm-timeout-seconds", - type=float, - help="per-request timeout in seconds for validation-phase LLM calls; defaults to --llm-timeout-seconds", - ) - process.add_argument("--validation-model", help="LLM model name for validation phases; defaults to --model") - process.add_argument( - "--validation-base-url", - help="OpenAI-compatible API base URL for validation phases; defaults to --base-url", - ) - process.add_argument( - "--validation-max-retries", - type=int, - help="maximum structured-output retries for validation phases; defaults to --max-retries", - ) - process.add_argument( - "--validation-max-prompt-tokens", - type=int, - help="maximum estimated tokens per validation-phase LLM prompt batch", - ) - process.add_argument( - "--target-sections", - type=int, - help="exact number of contiguous proposal-stage transcript sections to create", - ) - process.add_argument( - "--modules", - help="comma-separated module keys to run, for example: grammar or glossary,homophones,glossary", - ) - process.add_argument("--model", help="LLM model name for the configured OpenAI-compatible endpoint") - process.add_argument("--base-url", help="OpenAI-compatible API base URL for Audita LLM stages") - process.add_argument("--max-retries", type=int, help="maximum structured-output retries for LLM stages") - process.add_argument("--max-section-tokens", type=int, help="maximum estimated tokens per transcript batch") - process.add_argument( - "--min-section-tokens", - type=int, - help="minimum estimated tokens per transcript batch when balancing proposal-stage sections", - ) - process.add_argument( - "--glossary-confidence-threshold", - type=float, - help="minimum confidence required for glossary proposals to survive validation", - ) - process.add_argument( - "--grammar-confidence-threshold", - type=float, - help="minimum confidence required for grammar proposals to survive validation", - ) - process.add_argument( - "--homophones-confidence-threshold", - type=float, - help="minimum confidence required for homophone proposals to survive validation", - ) - process.add_argument( - "--spoken-word-confidence-threshold", - type=float, - help="minimum confidence required for spoken-word proposals to survive validation", - ) - process.add_argument( - "--normalize-max-segment-gap", - type=float, - help="maximum same-speaker gap in seconds eligible for deterministic merging", - ) - process.add_argument( - "--normalize-ellipsis-gap", - type=float, - help="minimum same-speaker gap in seconds that uses an ellipsis joiner", - ) - process.add_argument( - "--normalize-max-segment-duration", - type=float, - help="maximum merged segment duration in seconds", - ) - process.add_argument( - "--normalize-max-segment-tokens", - type=int, - help="maximum estimated tokens for a merged segment prompt payload", - ) - process.add_argument("--work-dir", type=Path, help="directory for per-run scratch diagnostics") - process.add_argument( - "--work-dir-retention", - choices=("auto", "always", "never"), - help="whether to retain the per-run work directory", - ) - return parser - - -def _process(args: argparse.Namespace, raw_argv: Optional[Sequence[str]] = None) -> int: - argv = list(raw_argv) if raw_argv is not None else list(sys.argv[1:]) - redacted_argv = redact_argv(argv) - run_dir = create_run_dir(_resolve_work_root(args)) - report_path = run_dir / "report.json" - error_log_path = run_dir / "error.log" - config: Optional[AuditaConfig] = None - phase = "config" - try: - config = AuditaConfig.from_sources( - overrides=ConfigOverrides( - llm_api_key=args.llm_api_key, - llm_concurrency=args.llm_concurrency, - llm_timeout_seconds=args.llm_timeout_seconds, - validation_llm_api_key=args.validation_llm_api_key, - validation_llm_concurrency=args.validation_llm_concurrency, - validation_llm_timeout_seconds=args.validation_llm_timeout_seconds, - validation_model=args.validation_model, - validation_base_url=args.validation_base_url, - validation_max_retries=args.validation_max_retries, - validation_max_prompt_tokens=args.validation_max_prompt_tokens, - target_sections=args.target_sections, - module_keys=args.modules, - model=args.model, - base_url=args.base_url, - max_retries=args.max_retries, - max_section_tokens=args.max_section_tokens, - min_section_tokens=args.min_section_tokens, - glossary_confidence_threshold=args.glossary_confidence_threshold, - grammar_confidence_threshold=args.grammar_confidence_threshold, - homophones_confidence_threshold=args.homophones_confidence_threshold, - spoken_word_confidence_threshold=args.spoken_word_confidence_threshold, - normalize_max_segment_gap=args.normalize_max_segment_gap, - normalize_ellipsis_gap=args.normalize_ellipsis_gap, - normalize_max_segment_duration=args.normalize_max_segment_duration, - normalize_max_segment_tokens=args.normalize_max_segment_tokens, - work_dir=args.work_dir, - work_dir_retention=args.work_dir_retention, - ) - ) - phase = "transcript_load" - transcript = load_transcript(args.transcript) - phase = "glossary_load" - glossary = load_glossary(args.glossary) - phase = "pipeline" - result = process_transcript_result( - transcript, - glossary, - config, - progress=_emit_console_line, - run_dir=run_dir, - invocation_details={"argv": redacted_argv, "cwd": os.getcwd()}, - ) - if args.output is not None: - write_transcript(args.output, result.transcript) - else: - sys.stdout.write(transcript_to_json(result.transcript)) - if args.report_json is not None: - write_report(args.report_json, result.report) - return 0 - except Exception as exc: - error_details = getattr(exc, "audita_error_details", None) - if error_details is None: - traceback_text = format_traceback_text(exc) - error_details = build_error_details( - exc=exc, - exit_code=1, - run_dir=run_dir, - report_path=report_path, - error_log_path=error_log_path, - phase=phase, - module_instance=None, - argv=redacted_argv, - cwd=os.getcwd(), - traceback_text=traceback_text, - ) - write_error_log(error_log_path, error_details=error_details, traceback_text=traceback_text) - report_config = config.to_report_dict() if isinstance(config, AuditaConfig) else {} - pipeline = [] if not isinstance(config, AuditaConfig) else list(config.module_keys) - retention = "always" if not isinstance(config, AuditaConfig) else config.work_dir_retention - report = RunReport( - status="failed", - config=report_config, - normalization=None, - pipeline=pipeline, - modules=[], - applied_changes=[], - skipped_corrections=[], - totals={"output_segment_count": 0, "applied_change_count": 0, "skipped_correction_count": 0}, - work_dir_retention=retention, - work_dir_retained=True, - work_dir=str(run_dir), - error=str(exc), - error_details=error_details, - ) - write_report(report_path, report) - if args.report_json is not None and report_path.exists(): - args.report_json.write_text(report_path.read_text(encoding="utf-8"), encoding="utf-8") - _emit_console_line(format_stderr_summary(error_details=error_details)) - return 1 - - -def _resolve_work_root(args: argparse.Namespace) -> Path: - if args.work_dir is not None: - return args.work_dir - return Path(os.environ.get("AUDITA_WORK_DIR") or "/tmp/audita") - - -def _emit_console_line(message: str) -> None: - for stream in (sys.stderr, sys.stdout): - if stream is None: - continue - try: - stream.write(f"{message}\n") - stream.flush() - return - except (OSError, ValueError): - continue diff --git a/python/src/audita/core/__init__.py b/python/src/audita/core/__init__.py deleted file mode 100644 index 6bf3602..0000000 --- a/python/src/audita/core/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Core data models and utilities for the Audita framework.""" diff --git a/python/src/audita/core/chunking.py b/python/src/audita/core/chunking.py deleted file mode 100644 index cc578db..0000000 --- a/python/src/audita/core/chunking.py +++ /dev/null @@ -1,356 +0,0 @@ -import json -from dataclasses import dataclass -from math import ceil -from typing import Any, Callable, Generic, List, Optional, Protocol, Sequence, TypeVar - -from .errors import AuditaValidationError -from .schemas import TranscriptSegment, parse_transcript_json - - -class TokenEstimatorProtocol(Protocol): - def estimate_json(self, value: Any) -> int: - ... - - -class TokenEstimator: - def __init__(self, fallback_chars_per_token: int = 4) -> None: - self._fallback_chars_per_token = fallback_chars_per_token - self._encoding = None - try: - import tiktoken - - self._encoding = tiktoken.get_encoding("cl100k_base") - except Exception: - self._encoding = None - - def estimate_text(self, text: str) -> int: - if self._encoding is not None: - return len(self._encoding.encode(text)) - return max(1, ceil(len(text) / self._fallback_chars_per_token)) - - def estimate_json(self, value: Any) -> int: - return self.estimate_text(json.dumps(value, ensure_ascii=False, separators=(",", ":"))) - - -T = TypeVar("T") - - -@dataclass(frozen=True) -class TokenBatch(Generic[T]): - batch_index: int - items: List[T] - token_count: int - - -def chunk_payload_items( - items: Sequence[T], - max_tokens: int, - payload_fn: Callable[[T], Any], - estimator: Optional[TokenEstimatorProtocol] = None, - empty_error_message: str = "Batch input must contain at least one item.", -) -> List[TokenBatch[T]]: - if max_tokens <= 0: - raise AuditaValidationError("Maximum section token count must be greater than zero.") - if not items: - raise AuditaValidationError(empty_error_message) - - token_estimator = TokenEstimator() if estimator is None else estimator - batches: List[TokenBatch[T]] = [] - current: List[T] = [] - current_tokens = 0 - - for item in items: - single_payload = [payload_fn(item)] - single_tokens = token_estimator.estimate_json(single_payload) - if single_tokens > max_tokens: - raise AuditaValidationError( - "A single transcript segment exceeds the maximum section token limit. " - "Raise the limit or pre-split the transcript." - ) - candidate = current + [item] - candidate_tokens = token_estimator.estimate_json([payload_fn(candidate_item) for candidate_item in candidate]) - if current and candidate_tokens > max_tokens: - batches.append(TokenBatch(batch_index=len(batches), items=list(current), token_count=current_tokens)) - current = [item] - current_tokens = single_tokens - else: - current = candidate - current_tokens = candidate_tokens - - if current: - batches.append(TokenBatch(batch_index=len(batches), items=list(current), token_count=current_tokens)) - return batches - - -@dataclass(frozen=True) -class IndexedSegment: - index: int - segment: TranscriptSegment - - def transcript_payload(self) -> dict: - return self.segment.model_dump(mode="json") - - def prompt_payload(self) -> dict: - payload = {"id": self.segment.id, "original_text": self.segment.text} - if self.segment.categories is not None: - payload["categories"] = list(self.segment.categories) - return payload - - -@dataclass(frozen=True) -class TranscriptSection: - section_index: int - start_index: int - segments: List[IndexedSegment] - token_count: int - - def transcript_payload(self) -> List[dict]: - return [item.transcript_payload() for item in self.segments] - - def prompt_payload(self) -> List[dict]: - return [item.prompt_payload() for item in self.segments] - - def transcript_json(self) -> str: - return json.dumps(self.transcript_payload(), ensure_ascii=False, indent=2) + "\n" - - -def chunk_transcript( - segments: List[TranscriptSegment], - max_section_tokens: int, - min_section_tokens: int = 1, - target_section_count: Optional[int] = None, - exact_target_section_count: Optional[int] = None, - estimator: Optional[TokenEstimatorProtocol] = None, -) -> List[TranscriptSection]: - indexed = [IndexedSegment(index=index, segment=segment) for index, segment in enumerate(segments)] - return chunk_indexed_segments( - indexed, - max_section_tokens, - min_section_tokens=min_section_tokens, - target_section_count=target_section_count, - exact_target_section_count=exact_target_section_count, - estimator=estimator, - ) - - -def chunk_indexed_segments( - indexed_segments: List[IndexedSegment], - max_section_tokens: int, - min_section_tokens: int = 1, - target_section_count: Optional[int] = None, - exact_target_section_count: Optional[int] = None, - estimator: Optional[TokenEstimatorProtocol] = None, -) -> List[TranscriptSection]: - if exact_target_section_count is not None: - batches = _chunk_indexed_segments_for_exact_target_count( - indexed_segments, - max_section_tokens=max_section_tokens, - min_section_tokens=min_section_tokens, - target_section_count=exact_target_section_count, - estimator=estimator, - ) - elif target_section_count is None: - batches = chunk_payload_items( - indexed_segments, - max_section_tokens, - payload_fn=lambda item: item.prompt_payload(), - estimator=estimator, - empty_error_message="Transcript must contain at least one segment.", - ) - else: - batches = _chunk_indexed_segments_for_target_count( - indexed_segments, - max_section_tokens=max_section_tokens, - min_section_tokens=min_section_tokens, - target_section_count=target_section_count, - estimator=estimator, - ) - sections = [ - TranscriptSection( - section_index=batch.batch_index, - start_index=batch.items[0].index, - segments=list(batch.items), - token_count=batch.token_count, - ) - for batch in batches - ] - for section in sections: - parse_transcript_json(section.transcript_json(), require_sequential_ids=False) - return sections - - -def _chunk_indexed_segments_for_target_count( - indexed_segments: List[IndexedSegment], - *, - max_section_tokens: int, - min_section_tokens: int, - target_section_count: int, - estimator: Optional[TokenEstimatorProtocol], -) -> List[TokenBatch[IndexedSegment]]: - if max_section_tokens <= 0: - raise AuditaValidationError("Maximum section token count must be greater than zero.") - if min_section_tokens <= 0: - raise AuditaValidationError("Minimum section token count must be greater than zero.") - if min_section_tokens > max_section_tokens: - raise AuditaValidationError( - "Minimum section token count must be less than or equal to maximum section token count." - ) - if not indexed_segments: - raise AuditaValidationError("Transcript must contain at least one segment.") - if target_section_count <= 0: - raise AuditaValidationError("Target section count must be greater than zero.") - - token_estimator = TokenEstimator() if estimator is None else estimator - single_tokens = [token_estimator.estimate_json([item.prompt_payload()]) for item in indexed_segments] - if any(tokens > max_section_tokens for tokens in single_tokens): - raise AuditaValidationError( - "A single transcript segment exceeds the maximum section token limit. " - "Raise the limit or pre-split the transcript." - ) - - total_tokens = sum(single_tokens) - if total_tokens < min_section_tokens: - return [_build_token_batch(indexed_segments, batch_index=0, estimator=token_estimator)] - - desired_count = min(target_section_count, len(indexed_segments)) - section_count = _resolve_section_count( - indexed_segments=indexed_segments, - single_tokens=single_tokens, - total_tokens=total_tokens, - desired_count=desired_count, - min_section_tokens=min_section_tokens, - max_section_tokens=max_section_tokens, - estimator=token_estimator, - ) - return _build_balanced_batches(indexed_segments, single_tokens, section_count, token_estimator) - - -def _chunk_indexed_segments_for_exact_target_count( - indexed_segments: List[IndexedSegment], - *, - max_section_tokens: int, - min_section_tokens: int, - target_section_count: int, - estimator: Optional[TokenEstimatorProtocol], -) -> List[TokenBatch[IndexedSegment]]: - if max_section_tokens <= 0: - raise AuditaValidationError("Maximum section token count must be greater than zero.") - if min_section_tokens <= 0: - raise AuditaValidationError("Minimum section token count must be greater than zero.") - if min_section_tokens > max_section_tokens: - raise AuditaValidationError( - "Minimum section token count must be less than or equal to maximum section token count." - ) - if not indexed_segments: - raise AuditaValidationError("Transcript must contain at least one segment.") - if target_section_count <= 0: - raise AuditaValidationError("Target section count must be greater than zero.") - if target_section_count > len(indexed_segments): - raise AuditaValidationError( - "Target section count exceeds the number of transcript segments. " - "Lower AUDITA_TARGET_SECTIONS or pre-merge the transcript." - ) - - token_estimator = TokenEstimator() if estimator is None else estimator - single_tokens = [token_estimator.estimate_json([item.prompt_payload()]) for item in indexed_segments] - if any(tokens > max_section_tokens for tokens in single_tokens): - raise AuditaValidationError( - "A single transcript segment exceeds the maximum section token limit. " - "Raise the limit or pre-split the transcript." - ) - - batches = _build_balanced_batches(indexed_segments, single_tokens, target_section_count, token_estimator) - if not _batches_within_bounds(batches, min_section_tokens=min_section_tokens, max_section_tokens=max_section_tokens): - raise AuditaValidationError( - "AUDITA_TARGET_SECTIONS cannot produce contiguous transcript sections within " - "AUDITA_MIN_SECTION_TOKENS and AUDITA_MAX_SECTION_TOKENS." - ) - return batches - - -def _resolve_section_count( - *, - indexed_segments: List[IndexedSegment], - single_tokens: List[int], - total_tokens: int, - desired_count: int, - min_section_tokens: int, - max_section_tokens: int, - estimator: TokenEstimatorProtocol, -) -> int: - batches = _build_balanced_batches(indexed_segments, single_tokens, desired_count, estimator) - if _batches_within_bounds(batches, min_section_tokens=min_section_tokens, max_section_tokens=max_section_tokens): - return desired_count - - average_tokens = total_tokens / desired_count - if average_tokens > max_section_tokens: - candidates = range(desired_count + 1, len(indexed_segments) + 1) - elif average_tokens < min_section_tokens: - candidates = range(desired_count - 1, 0, -1) - else: - candidates = list(range(desired_count + 1, len(indexed_segments) + 1)) + list( - range(desired_count - 1, 0, -1) - ) - - for count in candidates: - batches = _build_balanced_batches(indexed_segments, single_tokens, count, estimator) - if _batches_within_bounds(batches, min_section_tokens=min_section_tokens, max_section_tokens=max_section_tokens): - return count - return 1 - - -def _build_balanced_batches( - indexed_segments: List[IndexedSegment], - single_tokens: List[int], - section_count: int, - estimator: TokenEstimatorProtocol, -) -> List[TokenBatch[IndexedSegment]]: - if section_count == 1: - return [_build_token_batch(indexed_segments, batch_index=0, estimator=estimator)] - - prefix_tokens = [0] - for tokens in single_tokens: - prefix_tokens.append(prefix_tokens[-1] + tokens) - - cuts = [0] - total_tokens = prefix_tokens[-1] - for section_index in range(1, section_count): - target = total_tokens * section_index / section_count - min_cut = cuts[-1] + 1 - max_cut = len(indexed_segments) - (section_count - section_index) - best_cut = min_cut - best_distance = None - for cut in range(min_cut, max_cut + 1): - distance = abs(prefix_tokens[cut] - target) - if best_distance is None or distance < best_distance: - best_cut = cut - best_distance = distance - cuts.append(best_cut) - cuts.append(len(indexed_segments)) - - return [ - _build_token_batch(indexed_segments[cuts[index] : cuts[index + 1]], batch_index=index, estimator=estimator) - for index in range(section_count) - ] - - -def _build_token_batch( - items: Sequence[IndexedSegment], - *, - batch_index: int, - estimator: TokenEstimatorProtocol, -) -> TokenBatch[IndexedSegment]: - return TokenBatch( - batch_index=batch_index, - items=list(items), - token_count=estimator.estimate_json([item.prompt_payload() for item in items]), - ) - - -def _batches_within_bounds( - batches: Sequence[TokenBatch[IndexedSegment]], - *, - min_section_tokens: int, - max_section_tokens: int, -) -> bool: - return all(min_section_tokens <= batch.token_count <= max_section_tokens for batch in batches) diff --git a/python/src/audita/core/config.py b/python/src/audita/core/config.py deleted file mode 100644 index da492cb..0000000 --- a/python/src/audita/core/config.py +++ /dev/null @@ -1,506 +0,0 @@ -import math -import os -from dataclasses import dataclass, replace -from pathlib import Path -from typing import Mapping, Optional, Sequence, Tuple, Union - -from audita.modules import DEFAULT_MODULE_KEYS, normalize_module_keys - -from .errors import AuditaConfigError - - -DEFAULT_MODEL = "openrouter/google/gemma-4-31b-it" -DEFAULT_BASE_URL = "https://openrouter.ai/api/v1" -DEFAULT_LLM_CONCURRENCY = 1 -DEFAULT_LLM_TIMEOUT_SECONDS = 600 -DEFAULT_MAX_RETRIES = 3 -DEFAULT_VALIDATION_MAX_PROMPT_TOKENS = 2048 -DEFAULT_MAX_SECTION_TOKENS = 8192 -DEFAULT_MIN_SECTION_TOKENS = 2048 -DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD = 0.80 -DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD = 0.80 -DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD = 0.80 -DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD = 0.80 -DEFAULT_WORK_DIR = "/tmp/audita" -DEFAULT_WORK_DIR_RETENTION = "auto" -DEFAULT_NORMALIZE_MAX_SEGMENT_GAP = 4.0 -DEFAULT_NORMALIZE_ELLIPSIS_GAP = 3.5 -DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION = 60.0 -DEFAULT_NORMALIZE_MAX_SEGMENT_TOKENS = 2048 - - -@dataclass(frozen=True) -class ConfigOverrides: - llm_api_key: Optional[str] = None - llm_concurrency: Optional[int] = None - llm_timeout_seconds: Optional[float] = None - validation_llm_api_key: Optional[str] = None - validation_llm_concurrency: Optional[int] = None - validation_llm_timeout_seconds: Optional[float] = None - validation_model: Optional[str] = None - validation_base_url: Optional[str] = None - validation_max_retries: Optional[int] = None - validation_max_prompt_tokens: Optional[int] = None - target_sections: Optional[int] = None - module_keys: Optional[Union[str, Sequence[str]]] = None - model: Optional[str] = None - base_url: Optional[str] = None - max_retries: Optional[int] = None - max_section_tokens: Optional[int] = None - min_section_tokens: Optional[int] = None - glossary_confidence_threshold: Optional[float] = None - grammar_confidence_threshold: Optional[float] = None - homophones_confidence_threshold: Optional[float] = None - spoken_word_confidence_threshold: Optional[float] = None - normalize_max_segment_gap: Optional[float] = None - normalize_ellipsis_gap: Optional[float] = None - normalize_max_segment_duration: Optional[float] = None - normalize_max_segment_tokens: Optional[int] = None - work_dir: Optional[Path] = None - work_dir_retention: Optional[str] = None - - -@dataclass(frozen=True) -class AuditaConfig: - api_key: Optional[str] = None - llm_concurrency: int = DEFAULT_LLM_CONCURRENCY - llm_timeout_seconds: float = DEFAULT_LLM_TIMEOUT_SECONDS - validation_llm_api_key: Optional[str] = None - validation_llm_concurrency: Optional[int] = None - validation_llm_timeout_seconds: Optional[float] = None - validation_model: Optional[str] = None - validation_base_url: Optional[str] = None - validation_max_retries: Optional[int] = None - validation_max_prompt_tokens: int = DEFAULT_VALIDATION_MAX_PROMPT_TOKENS - target_sections: Optional[int] = None - module_keys: Tuple[str, ...] = DEFAULT_MODULE_KEYS - model: str = DEFAULT_MODEL - base_url: str = DEFAULT_BASE_URL - max_retries: int = DEFAULT_MAX_RETRIES - max_section_tokens: int = DEFAULT_MAX_SECTION_TOKENS - min_section_tokens: int = DEFAULT_MIN_SECTION_TOKENS - glossary_confidence_threshold: float = DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD - grammar_confidence_threshold: float = DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD - homophones_confidence_threshold: float = DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD - spoken_word_confidence_threshold: float = DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD - normalize_max_segment_gap: float = DEFAULT_NORMALIZE_MAX_SEGMENT_GAP - normalize_ellipsis_gap: float = DEFAULT_NORMALIZE_ELLIPSIS_GAP - normalize_max_segment_duration: float = DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION - normalize_max_segment_tokens: int = DEFAULT_NORMALIZE_MAX_SEGMENT_TOKENS - work_dir: Path = Path(DEFAULT_WORK_DIR) - work_dir_retention: str = DEFAULT_WORK_DIR_RETENTION - - @classmethod - def from_sources( - cls, - env: Optional[Mapping[str, str]] = None, - overrides: Optional[ConfigOverrides] = None, - ) -> "AuditaConfig": - source = os.environ if env is None else env - selected = ConfigOverrides() if overrides is None else overrides - config = cls( - api_key=_select_api_key( - selected.llm_api_key, - source.get("AUDITA_LLM_API_KEY"), - source.get("OPENROUTER_API_KEY"), - ), - llm_concurrency=_select_int( - selected.llm_concurrency, - source.get("AUDITA_LLM_CONCURRENCY"), - DEFAULT_LLM_CONCURRENCY, - "AUDITA_LLM_CONCURRENCY", - ), - llm_timeout_seconds=_select_float( - selected.llm_timeout_seconds, - source.get("AUDITA_LLM_TIMEOUT_SECONDS"), - DEFAULT_LLM_TIMEOUT_SECONDS, - "AUDITA_LLM_TIMEOUT_SECONDS", - ), - validation_llm_api_key=_select_optional_api_key_override( - selected.validation_llm_api_key, - source.get("AUDITA_VALIDATION_LLM_API_KEY"), - ), - validation_llm_concurrency=_select_optional_int( - selected.validation_llm_concurrency, - source.get("AUDITA_VALIDATION_LLM_CONCURRENCY"), - "AUDITA_VALIDATION_LLM_CONCURRENCY", - ), - validation_llm_timeout_seconds=_select_optional_float( - selected.validation_llm_timeout_seconds, - source.get("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"), - "AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS", - ), - validation_model=_select_optional_string_override( - selected.validation_model, - source.get("AUDITA_VALIDATION_MODEL"), - ), - validation_base_url=_select_optional_string_override( - selected.validation_base_url, - source.get("AUDITA_VALIDATION_BASE_URL"), - ), - validation_max_retries=_select_optional_int( - selected.validation_max_retries, - source.get("AUDITA_VALIDATION_MAX_RETRIES"), - "AUDITA_VALIDATION_MAX_RETRIES", - ), - validation_max_prompt_tokens=_select_int( - selected.validation_max_prompt_tokens, - source.get("AUDITA_VALIDATION_MAX_PROMPT_TOKENS"), - DEFAULT_VALIDATION_MAX_PROMPT_TOKENS, - "AUDITA_VALIDATION_MAX_PROMPT_TOKENS", - ), - target_sections=_select_optional_int( - selected.target_sections, - source.get("AUDITA_TARGET_SECTIONS"), - "AUDITA_TARGET_SECTIONS", - ), - module_keys=_select_module_keys( - selected.module_keys, - source.get("AUDITA_MODULES"), - DEFAULT_MODULE_KEYS, - "AUDITA_MODULES", - ), - model=selected.model or source.get("AUDITA_MODEL") or DEFAULT_MODEL, - base_url=selected.base_url or source.get("AUDITA_BASE_URL") or DEFAULT_BASE_URL, - max_retries=_select_int( - selected.max_retries, - source.get("AUDITA_MAX_RETRIES"), - DEFAULT_MAX_RETRIES, - "AUDITA_MAX_RETRIES", - ), - max_section_tokens=_select_int( - selected.max_section_tokens, - source.get("AUDITA_MAX_SECTION_TOKENS"), - DEFAULT_MAX_SECTION_TOKENS, - "AUDITA_MAX_SECTION_TOKENS", - ), - min_section_tokens=_select_int( - selected.min_section_tokens, - source.get("AUDITA_MIN_SECTION_TOKENS"), - DEFAULT_MIN_SECTION_TOKENS, - "AUDITA_MIN_SECTION_TOKENS", - ), - glossary_confidence_threshold=_select_float( - selected.glossary_confidence_threshold, - source.get("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD"), - DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD, - "AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD", - ), - grammar_confidence_threshold=_select_float( - selected.grammar_confidence_threshold, - source.get("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD"), - DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD, - "AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD", - ), - homophones_confidence_threshold=_select_float( - selected.homophones_confidence_threshold, - source.get("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD"), - DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD, - "AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD", - ), - spoken_word_confidence_threshold=_select_float( - selected.spoken_word_confidence_threshold, - source.get("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD"), - DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD, - "AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD", - ), - normalize_max_segment_gap=_select_float( - selected.normalize_max_segment_gap, - source.get("AUDITA_NORMALIZE_MAX_SEGMENT_GAP"), - DEFAULT_NORMALIZE_MAX_SEGMENT_GAP, - "AUDITA_NORMALIZE_MAX_SEGMENT_GAP", - ), - normalize_ellipsis_gap=_select_float( - selected.normalize_ellipsis_gap, - source.get("AUDITA_NORMALIZE_ELLIPSIS_GAP"), - DEFAULT_NORMALIZE_ELLIPSIS_GAP, - "AUDITA_NORMALIZE_ELLIPSIS_GAP", - ), - normalize_max_segment_duration=_select_float( - selected.normalize_max_segment_duration, - source.get("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION"), - DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION, - "AUDITA_NORMALIZE_MAX_SEGMENT_DURATION", - ), - normalize_max_segment_tokens=_select_int( - selected.normalize_max_segment_tokens, - source.get("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS"), - DEFAULT_NORMALIZE_MAX_SEGMENT_TOKENS, - "AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS", - ), - work_dir=selected.work_dir or Path(source.get("AUDITA_WORK_DIR") or DEFAULT_WORK_DIR), - work_dir_retention=_select_choice( - selected.work_dir_retention, - source.get("AUDITA_WORK_DIR_RETENTION"), - DEFAULT_WORK_DIR_RETENTION, - "AUDITA_WORK_DIR_RETENTION", - ("auto", "always", "never"), - ), - ) - config.validate() - return config - - def validate(self) -> None: - normalize_module_keys(self.module_keys) - if self.llm_concurrency <= 0: - raise AuditaConfigError("AUDITA_LLM_CONCURRENCY must be greater than zero.") - if not math.isfinite(self.llm_timeout_seconds): - raise AuditaConfigError("AUDITA_LLM_TIMEOUT_SECONDS must be finite.") - if self.llm_timeout_seconds <= 0: - raise AuditaConfigError("AUDITA_LLM_TIMEOUT_SECONDS must be greater than zero.") - if self.validation_llm_concurrency is not None and self.validation_llm_concurrency <= 0: - raise AuditaConfigError("AUDITA_VALIDATION_LLM_CONCURRENCY must be greater than zero.") - if self.validation_llm_timeout_seconds is not None and not math.isfinite(self.validation_llm_timeout_seconds): - raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be finite.") - if self.validation_llm_timeout_seconds is not None and self.validation_llm_timeout_seconds <= 0: - raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be greater than zero.") - if self.validation_max_retries is not None and self.validation_max_retries < 0: - raise AuditaConfigError("AUDITA_VALIDATION_MAX_RETRIES must be greater than or equal to zero.") - if self.validation_max_prompt_tokens <= 0: - raise AuditaConfigError("AUDITA_VALIDATION_MAX_PROMPT_TOKENS must be greater than zero.") - if self.target_sections is not None and self.target_sections <= 0: - raise AuditaConfigError("AUDITA_TARGET_SECTIONS must be greater than zero.") - if not self.model.strip(): - raise AuditaConfigError("AUDITA_MODEL must not be empty.") - if not self.base_url.strip(): - raise AuditaConfigError("AUDITA_BASE_URL must not be empty.") - if self.validation_model is not None and not self.validation_model.strip(): - raise AuditaConfigError("AUDITA_VALIDATION_MODEL must not be empty.") - if self.validation_base_url is not None and not self.validation_base_url.strip(): - raise AuditaConfigError("AUDITA_VALIDATION_BASE_URL must not be empty.") - if self.max_retries < 0: - raise AuditaConfigError("AUDITA_MAX_RETRIES must be greater than or equal to zero.") - if self.max_section_tokens <= 0: - raise AuditaConfigError("AUDITA_MAX_SECTION_TOKENS must be greater than zero.") - if self.min_section_tokens <= 0: - raise AuditaConfigError("AUDITA_MIN_SECTION_TOKENS must be greater than zero.") - if self.min_section_tokens > self.max_section_tokens: - raise AuditaConfigError( - "AUDITA_MIN_SECTION_TOKENS must be less than or equal to AUDITA_MAX_SECTION_TOKENS." - ) - if not 0.0 <= self.glossary_confidence_threshold <= 1.0: - raise AuditaConfigError("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.") - if not 0.0 <= self.grammar_confidence_threshold <= 1.0: - raise AuditaConfigError("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.") - if not 0.0 <= self.homophones_confidence_threshold <= 1.0: - raise AuditaConfigError("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.") - if not 0.0 <= self.spoken_word_confidence_threshold <= 1.0: - raise AuditaConfigError("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.") - if not math.isfinite(self.normalize_max_segment_gap): - raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_GAP must be finite.") - if not math.isfinite(self.normalize_ellipsis_gap): - raise AuditaConfigError("AUDITA_NORMALIZE_ELLIPSIS_GAP must be finite.") - if not math.isfinite(self.normalize_max_segment_duration): - raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION must be finite.") - if self.normalize_max_segment_gap < 0: - raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_GAP must be greater than or equal to zero.") - if self.normalize_ellipsis_gap < 0: - raise AuditaConfigError("AUDITA_NORMALIZE_ELLIPSIS_GAP must be greater than or equal to zero.") - if self.normalize_ellipsis_gap > self.normalize_max_segment_gap: - raise AuditaConfigError( - "AUDITA_NORMALIZE_ELLIPSIS_GAP must be less than or equal to AUDITA_NORMALIZE_MAX_SEGMENT_GAP." - ) - if self.normalize_max_segment_duration <= 0: - raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION must be greater than zero.") - if self.normalize_max_segment_tokens <= 0: - raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS must be greater than zero.") - if self.work_dir_retention not in ("auto", "always", "never"): - raise AuditaConfigError("AUDITA_WORK_DIR_RETENTION must be one of auto, always, or never.") - validation_config = self.validation_llm_config() - if validation_config.llm_concurrency <= 0: - raise AuditaConfigError("AUDITA_VALIDATION_LLM_CONCURRENCY must be greater than zero.") - if not math.isfinite(validation_config.llm_timeout_seconds): - raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be finite.") - if validation_config.llm_timeout_seconds <= 0: - raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be greater than zero.") - if not validation_config.model.strip(): - raise AuditaConfigError("AUDITA_VALIDATION_MODEL must not be empty.") - if not validation_config.base_url.strip(): - raise AuditaConfigError("AUDITA_VALIDATION_BASE_URL must not be empty.") - if validation_config.max_retries < 0: - raise AuditaConfigError("AUDITA_VALIDATION_MAX_RETRIES must be greater than or equal to zero.") - - def proposal_llm_config(self) -> "AuditaConfig": - return self - - def effective_validation_llm_api_key(self) -> Optional[str]: - return self.validation_llm_api_key if self.validation_llm_api_key is not None else self.api_key - - def effective_validation_llm_concurrency(self) -> int: - return self.validation_llm_concurrency if self.validation_llm_concurrency is not None else self.llm_concurrency - - def effective_validation_llm_timeout_seconds(self) -> float: - if self.validation_llm_timeout_seconds is not None: - return self.validation_llm_timeout_seconds - return self.llm_timeout_seconds - - def effective_validation_model(self) -> str: - return self.validation_model if self.validation_model is not None else self.model - - def effective_validation_base_url(self) -> str: - return self.validation_base_url if self.validation_base_url is not None else self.base_url - - def effective_validation_max_retries(self) -> int: - return self.validation_max_retries if self.validation_max_retries is not None else self.max_retries - - def validation_llm_config(self) -> "AuditaConfig": - return replace( - self, - api_key=self.effective_validation_llm_api_key(), - llm_concurrency=self.effective_validation_llm_concurrency(), - llm_timeout_seconds=self.effective_validation_llm_timeout_seconds(), - model=self.effective_validation_model(), - base_url=self.effective_validation_base_url(), - max_retries=self.effective_validation_max_retries(), - ) - - def to_report_dict(self) -> dict: - effective_validation_config = self.validation_llm_config() - return { - "api_key_configured": bool(self.api_key), - "llm_concurrency": self.llm_concurrency, - "llm_timeout_seconds": self.llm_timeout_seconds, - "validation_llm_api_key_configured": bool(self.validation_llm_api_key), - "validation_llm_concurrency": self.validation_llm_concurrency, - "validation_llm_timeout_seconds": self.validation_llm_timeout_seconds, - "validation_model": self.validation_model, - "validation_base_url": self.validation_base_url, - "validation_max_retries": self.validation_max_retries, - "validation_max_prompt_tokens": self.validation_max_prompt_tokens, - "target_sections": self.target_sections, - "module_keys": list(self.module_keys), - "model": self.model, - "base_url": self.base_url, - "max_retries": self.max_retries, - "effective_validation_llm": { - "api_key_configured": bool(effective_validation_config.api_key), - "llm_concurrency": effective_validation_config.llm_concurrency, - "llm_timeout_seconds": effective_validation_config.llm_timeout_seconds, - "model": effective_validation_config.model, - "base_url": effective_validation_config.base_url, - "max_retries": effective_validation_config.max_retries, - }, - "max_section_tokens": self.max_section_tokens, - "min_section_tokens": self.min_section_tokens, - "glossary_confidence_threshold": self.glossary_confidence_threshold, - "grammar_confidence_threshold": self.grammar_confidence_threshold, - "homophones_confidence_threshold": self.homophones_confidence_threshold, - "spoken_word_confidence_threshold": self.spoken_word_confidence_threshold, - "normalize_max_segment_gap": self.normalize_max_segment_gap, - "normalize_ellipsis_gap": self.normalize_ellipsis_gap, - "normalize_max_segment_duration": self.normalize_max_segment_duration, - "normalize_max_segment_tokens": self.normalize_max_segment_tokens, - "work_dir_retention": self.work_dir_retention, - } - - -def _select_optional_string(value: Optional[str]) -> Optional[str]: - if value is None: - return None - stripped = value.strip() - return stripped or None - - -def _select_optional_string_override(cli_value: Optional[str], env_value: Optional[str]) -> Optional[str]: - if cli_value is not None: - return _select_optional_string(cli_value) - return _select_optional_string(env_value) - - -def _select_optional_api_key_override(cli_value: Optional[str], env_value: Optional[str]) -> Optional[str]: - if cli_value is not None: - return cli_value.strip() - if env_value is not None: - return env_value.strip() - return None - - -def _select_api_key( - cli_value: Optional[str], - generic_env_value: Optional[str], - legacy_env_value: Optional[str], -) -> Optional[str]: - if cli_value is not None: - return _select_optional_string(cli_value) - generic = _select_optional_string(generic_env_value) - if generic is not None: - return generic - return _select_optional_string(legacy_env_value) - - -def _select_int(cli_value: Optional[int], env_value: Optional[str], default: int, name: str) -> int: - if cli_value is not None: - return cli_value - if env_value is None: - return default - try: - return int(env_value) - except ValueError as exc: - raise AuditaConfigError(f"{name} must be an integer.") from exc - - -def _select_optional_int(cli_value: Optional[int], env_value: Optional[str], name: str) -> Optional[int]: - if cli_value is not None: - return cli_value - if env_value is None: - return None - try: - return int(env_value) - except ValueError as exc: - raise AuditaConfigError(f"{name} must be an integer.") from exc - - -def _select_optional_float(cli_value: Optional[float], env_value: Optional[str], name: str) -> Optional[float]: - if cli_value is not None: - return cli_value - if env_value is None: - return None - try: - return float(env_value) - except ValueError as exc: - raise AuditaConfigError(f"{name} must be a number.") from exc - - -def _select_float(cli_value: Optional[float], env_value: Optional[str], default: float, name: str) -> float: - if cli_value is not None: - return cli_value - if env_value is None: - return default - try: - return float(env_value) - except ValueError as exc: - raise AuditaConfigError(f"{name} must be a number.") from exc - - -def _select_choice( - cli_value: Optional[str], - env_value: Optional[str], - default: str, - name: str, - valid_choices: tuple[str, ...], -) -> str: - value = cli_value or env_value or default - if value not in valid_choices: - raise AuditaConfigError(f"{name} must be one of {', '.join(valid_choices)}.") - return value - - -def _select_module_keys( - cli_value: Optional[Union[str, Sequence[str]]], - env_value: Optional[str], - default: Tuple[str, ...], - name: str, -) -> Tuple[str, ...]: - if cli_value is not None: - return _coerce_module_keys(cli_value, name) - if env_value is not None: - return _coerce_module_keys(env_value, name) - return default - - -def _coerce_module_keys(value: Union[str, Sequence[str]], name: str) -> Tuple[str, ...]: - try: - raw_items = value.split(",") if isinstance(value, str) else list(value) - return normalize_module_keys(raw_items) - except AuditaConfigError as exc: - raise AuditaConfigError(f"{name} is invalid: {exc}") from exc diff --git a/python/src/audita/core/diagnostics.py b/python/src/audita/core/diagnostics.py deleted file mode 100644 index c073bbb..0000000 --- a/python/src/audita/core/diagnostics.py +++ /dev/null @@ -1,148 +0,0 @@ -from __future__ import annotations - -import json -import traceback -from datetime import datetime -from pathlib import Path -import re -from typing import Any, Mapping, Optional, Sequence -from uuid import uuid4 - - -_SECRET_FLAGS = {"--llm-api-key", "--validation-llm-api-key"} -_STAGE_PATTERN = re.compile(r"stage '([^']+)'") - - -def create_run_dir(root: Path) -> Path: - root.mkdir(parents=True, exist_ok=True) - timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") - run_dir = root / f"run-{timestamp}-{uuid4().hex[:8]}" - run_dir.mkdir(parents=False, exist_ok=False) - return run_dir - - -def redact_argv(argv: Sequence[str]) -> list[str]: - redacted: list[str] = [] - index = 0 - while index < len(argv): - arg = argv[index] - matched_flag = next((flag for flag in _SECRET_FLAGS if arg == flag or arg.startswith(flag + "=")), None) - if matched_flag is None: - redacted.append(arg) - index += 1 - continue - if arg == matched_flag: - redacted.append(arg) - if index + 1 < len(argv): - redacted.append("[REDACTED]") - index += 2 - else: - index += 1 - continue - redacted.append(f"{matched_flag}=[REDACTED]") - index += 1 - return redacted - - -def format_traceback_text(exc: BaseException) -> str: - return "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)) - - -def extract_stage_name(message: str) -> Optional[str]: - match = _STAGE_PATTERN.search(message) - return match.group(1) if match is not None else None - - -def build_error_details( - *, - exc: BaseException, - exit_code: int, - run_dir: Path, - report_path: Path, - error_log_path: Path, - phase: str, - module_instance: Optional[str], - argv: Optional[Sequence[str]] = None, - cwd: Optional[str] = None, - traceback_text: Optional[str] = None, -) -> dict[str, Any]: - message = str(exc) - stage_name = extract_stage_name(message) - excerpt_source = traceback_text or format_traceback_text(exc) - excerpt_lines = excerpt_source.strip().splitlines() - traceback_excerpt = "\n".join(excerpt_lines[-12:]) if excerpt_lines else "" - details: dict[str, Any] = { - "type": type(exc).__name__, - "message": message, - "exit_code": exit_code, - "phase": phase, - "stage": stage_name, - "module_instance": module_instance, - "run_dir": str(run_dir), - "report_path": str(report_path), - "error_log": str(error_log_path), - "traceback_excerpt": traceback_excerpt, - } - if argv is not None: - details["argv"] = list(argv) - if cwd is not None: - details["cwd"] = cwd - return details - - -def write_error_log( - path: Path, - *, - error_details: Mapping[str, Any], - traceback_text: str, -) -> None: - lines = [ - "Audita Error Diagnostics", - f"timestamp: {datetime.utcnow().isoformat()}Z", - f"type: {error_details.get('type')}", - f"message: {error_details.get('message')}", - f"exit_code: {error_details.get('exit_code')}", - f"phase: {error_details.get('phase')}", - f"stage: {error_details.get('stage') or ''}", - f"module_instance: {error_details.get('module_instance') or ''}", - f"run_dir: {error_details.get('run_dir')}", - f"report_path: {error_details.get('report_path')}", - f"error_log: {error_details.get('error_log')}", - ] - argv = error_details.get("argv") - if argv is not None: - lines.append(f"argv: {json.dumps(argv, ensure_ascii=False)}") - cwd = error_details.get("cwd") - if cwd is not None: - lines.append(f"cwd: {cwd}") - lines.extend( - [ - "", - "Traceback:", - traceback_text.rstrip(), - "", - ] - ) - path.write_text("\n".join(lines), encoding="utf-8") - - -def format_stderr_summary(*, error_details: Mapping[str, Any]) -> str: - lines = [ - f"audita: error: {error_details.get('message')}", - f"audita: exit code: {error_details.get('exit_code')}", - f"audita: phase: {error_details.get('phase')}", - ] - stage = error_details.get("stage") - if stage: - lines.append(f"audita: stage: {stage}") - module_instance = error_details.get("module_instance") - if module_instance: - lines.append(f"audita: module: {module_instance}") - lines.extend( - [ - f"audita: run directory: {error_details.get('run_dir')}", - f"audita: error log: {error_details.get('error_log')}", - f"audita: report: {error_details.get('report_path')}", - ] - ) - return "\n".join(lines) diff --git a/python/src/audita/core/errors.py b/python/src/audita/core/errors.py deleted file mode 100644 index f017433..0000000 --- a/python/src/audita/core/errors.py +++ /dev/null @@ -1,15 +0,0 @@ -class AuditaError(Exception): - """Base exception for user-facing Audita failures.""" - - -class AuditaValidationError(AuditaError): - """Raised when input data does not match Audita's expected schema.""" - - -class AuditaConfigError(AuditaError): - """Raised when runtime configuration is invalid or incomplete.""" - - -class AuditaLLMError(AuditaError): - """Raised when an LLM request or structured response fails.""" - diff --git a/python/src/audita/core/io.py b/python/src/audita/core/io.py deleted file mode 100644 index 44c658e..0000000 --- a/python/src/audita/core/io.py +++ /dev/null @@ -1,22 +0,0 @@ -from pathlib import Path -from typing import List - -from .reporting import RunReport -from .schemas import Glossary, SourceTranscriptSegment, TranscriptSegment -from .schemas import parse_glossary_yaml, parse_source_transcript_json, transcript_to_json - - -def load_transcript(path: Path) -> List[SourceTranscriptSegment]: - return parse_source_transcript_json(path.read_text(encoding="utf-8")) - - -def load_glossary(path: Path) -> Glossary: - return parse_glossary_yaml(path.read_text(encoding="utf-8")) - - -def write_transcript(path: Path, segments: List[TranscriptSegment]) -> None: - path.write_text(transcript_to_json(segments), encoding="utf-8") - - -def write_report(path: Path, report: RunReport) -> None: - path.write_text(report.to_json(), encoding="utf-8") diff --git a/python/src/audita/core/normalization.py b/python/src/audita/core/normalization.py deleted file mode 100644 index 959a3ca..0000000 --- a/python/src/audita/core/normalization.py +++ /dev/null @@ -1,184 +0,0 @@ -from dataclasses import asdict, dataclass -from typing import List, Optional - -from .chunking import TokenEstimator, TokenEstimatorProtocol -from .schemas import SourceTranscriptSegment, TranscriptSegment - - -@dataclass(frozen=True) -class NormalizationSummary: - source_segment_count: int - normalized_segment_count: int - merge_count: int - max_segment_gap: float - ellipsis_gap: float - max_segment_duration: float - max_segment_tokens: int - - def to_dict(self) -> dict: - return asdict(self) - - -@dataclass(frozen=True) -class NormalizationResult: - transcript: List[TranscriptSegment] - summary: NormalizationSummary - - -@dataclass(frozen=True) -class _WorkingSegment: - speaker: str - start: float - end: float - text: str - categories: Optional[List[str]] - order: int - - -def normalize_transcript( - segments: List[SourceTranscriptSegment], - max_segment_gap: float, - ellipsis_gap: float, - max_segment_duration: float, - max_segment_tokens: int, - estimator: Optional[TokenEstimatorProtocol] = None, -) -> NormalizationResult: - token_estimator = TokenEstimator() if estimator is None else estimator - working = [ - _WorkingSegment( - speaker=segment.speaker, - start=segment.start, - end=segment.end, - text=segment.text, - categories=None if segment.categories is None else list(segment.categories), - order=index, - ) - for index, segment in enumerate(segments) - ] - working.sort(key=lambda segment: (segment.start, segment.end, segment.order)) - - merge_count = 0 - while True: - candidate_index = _shortest_mergeable_gap_index( - working, - max_segment_gap, - ellipsis_gap, - max_segment_duration, - max_segment_tokens, - token_estimator, - ) - if candidate_index is None: - break - left = working[candidate_index] - right = working[candidate_index + 1] - working[candidate_index : candidate_index + 2] = [_merge_segments(left, right, ellipsis_gap)] - merge_count += 1 - - normalized = _assign_ids(working) - return NormalizationResult( - transcript=normalized, - summary=NormalizationSummary( - source_segment_count=len(segments), - normalized_segment_count=len(normalized), - merge_count=merge_count, - max_segment_gap=max_segment_gap, - ellipsis_gap=ellipsis_gap, - max_segment_duration=max_segment_duration, - max_segment_tokens=max_segment_tokens, - ), - ) - - -def _shortest_mergeable_gap_index( - segments: List[_WorkingSegment], - max_segment_gap: float, - ellipsis_gap: float, - max_segment_duration: float, - max_segment_tokens: int, - estimator: TokenEstimatorProtocol, -) -> Optional[int]: - best_index = None - best_gap = None - for index in range(len(segments) - 1): - left = segments[index] - right = segments[index + 1] - gap = right.start - left.end - if not _can_merge( - left, - right, - gap, - max_segment_gap, - ellipsis_gap, - max_segment_duration, - max_segment_tokens, - estimator, - ): - continue - if best_gap is None or gap < best_gap: - best_index = index - best_gap = gap - return best_index - - -def _can_merge( - left: _WorkingSegment, - right: _WorkingSegment, - gap: float, - max_segment_gap: float, - ellipsis_gap: float, - max_segment_duration: float, - max_segment_tokens: int, - estimator: TokenEstimatorProtocol, -) -> bool: - if left.speaker != right.speaker: - return False - if gap < 0 or gap > max_segment_gap: - return False - if right.end - left.start > max_segment_duration: - return False - merged_text = _joined_text(left.text, right.text, gap, ellipsis_gap) - return _estimate_prompt_tokens(merged_text, estimator) <= max_segment_tokens - - -def _merge_segments(left: _WorkingSegment, right: _WorkingSegment, ellipsis_gap: float) -> _WorkingSegment: - gap = right.start - left.end - return _WorkingSegment( - speaker=left.speaker, - start=left.start, - end=right.end, - text=_joined_text(left.text, right.text, gap, ellipsis_gap), - categories=_merged_categories(left.categories, right.categories), - order=left.order, - ) - - -def _joined_text(left_text: str, right_text: str, gap: float, ellipsis_gap: float) -> str: - joiner = " " if gap <= ellipsis_gap else " ... " - return f"{left_text.rstrip()}{joiner}{right_text.lstrip()}" - - -def _estimate_prompt_tokens(text: str, estimator: TokenEstimatorProtocol) -> int: - return estimator.estimate_json([{"id": 1, "original_text": text}]) - - -def _merged_categories(left: Optional[List[str]], right: Optional[List[str]]) -> Optional[List[str]]: - merged: List[str] = [] - for category in (left or []) + (right or []): - if category not in merged: - merged.append(category) - return merged or None - - -def _assign_ids(segments: List[_WorkingSegment]) -> List[TranscriptSegment]: - ordered = sorted(segments, key=lambda segment: (segment.start, segment.end, segment.order)) - return [ - TranscriptSegment( - id=index + 1, - speaker=segment.speaker, - start=segment.start, - end=segment.end, - text=segment.text, - categories=None if segment.categories is None else list(segment.categories), - ) - for index, segment in enumerate(ordered) - ] diff --git a/python/src/audita/core/reporting.py b/python/src/audita/core/reporting.py deleted file mode 100644 index 8b75273..0000000 --- a/python/src/audita/core/reporting.py +++ /dev/null @@ -1,122 +0,0 @@ -import json -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import List, Optional - -from .schemas import TranscriptSegment - - -@dataclass(frozen=True) -class AppliedChange: - module_instance: str - module_key: str - proposal_index: int - id: int - original_text: str - corrected_text: str - confidence: float - segment_text_before: str - segment_text_after: str - - def to_dict(self) -> dict: - return asdict(self) - - -@dataclass(frozen=True) -class ReportedSkip: - module_instance: str - module_key: str - proposal_index: int - id: int - reason: str - original_text: str - corrected_text: str - confidence: float - actual_text: Optional[str] = None - source: Optional[str] = None - - def to_dict(self) -> dict: - return asdict(self) - - -@dataclass(frozen=True) -class ValidatorReport: - name: str - execution_kind: str - candidate_count: int - approved_count: int - rejected_count: int - - def to_dict(self) -> dict: - return asdict(self) - - -@dataclass(frozen=True) -class ModuleRunReport: - instance_name: str - module_key: str - replacement_policy: str - section_count: int - proposal_count: int - validators: List[ValidatorReport] - approved_count: int - applied_count: int - skipped_count: int - - def to_dict(self) -> dict: - return { - "instance_name": self.instance_name, - "module_key": self.module_key, - "replacement_policy": self.replacement_policy, - "section_count": self.section_count, - "proposal_count": self.proposal_count, - "validators": [item.to_dict() for item in self.validators], - "approved_count": self.approved_count, - "applied_count": self.applied_count, - "skipped_count": self.skipped_count, - } - - -@dataclass(frozen=True) -class RunReport: - status: str - config: dict - normalization: Optional[dict] - pipeline: List[str] - modules: List[ModuleRunReport] - applied_changes: List[AppliedChange] - skipped_corrections: List[ReportedSkip] - totals: dict - work_dir_retention: str - work_dir_retained: bool - work_dir: Optional[str] - error: Optional[str] = None - error_details: Optional[dict] = None - - def to_dict(self) -> dict: - return { - "status": self.status, - "config": self.config, - "normalization": self.normalization, - "pipeline": self.pipeline, - "modules": [item.to_dict() for item in self.modules], - "applied_changes": [item.to_dict() for item in self.applied_changes], - "skipped_corrections": [item.to_dict() for item in self.skipped_corrections], - "totals": self.totals, - "work_dir_retention": self.work_dir_retention, - "work_dir_retained": self.work_dir_retained, - "work_dir": self.work_dir, - "error": self.error, - "error_details": self.error_details, - } - - def to_json(self) -> str: - return json.dumps(self.to_dict(), ensure_ascii=False, indent=2) + "\n" - - -@dataclass(frozen=True) -class ProcessResult: - transcript: List[TranscriptSegment] - report: RunReport - run_dir: Path - work_dir_retained: bool diff --git a/python/src/audita/core/schemas.py b/python/src/audita/core/schemas.py deleted file mode 100644 index dd8a911..0000000 --- a/python/src/audita/core/schemas.py +++ /dev/null @@ -1,252 +0,0 @@ -import json -import math -from typing import Any, List, Optional - -from pydantic import BaseModel, ConfigDict, Field, StrictStr, TypeAdapter, ValidationError, field_validator, model_validator - -from .errors import AuditaValidationError - - -class TranscriptSegment(BaseModel): - model_config = ConfigDict(extra="ignore") - - id: int = Field(ge=1) - speaker: StrictStr - start: float - end: float - text: StrictStr - categories: Optional[List[StrictStr]] = None - - @field_validator("id", mode="before") - @classmethod - def require_integer_id(cls, value: Any) -> int: - if isinstance(value, bool) or not isinstance(value, int): - raise ValueError("must be an integer") - return value - - @field_validator("speaker", "text") - @classmethod - def require_non_empty_text(cls, value: str) -> str: - if not value.strip(): - raise ValueError("must not be empty") - return value - - @field_validator("categories") - @classmethod - def require_non_empty_categories(cls, categories: Optional[List[str]]) -> Optional[List[str]]: - if categories is None: - return None - for category in categories: - if not category.strip(): - raise ValueError("categories must not contain empty strings") - return categories - - @field_validator("start", "end", mode="before") - @classmethod - def require_number(cls, value: Any) -> float: - if isinstance(value, bool) or not isinstance(value, (int, float)): - raise ValueError("must be a JSON number") - number = float(value) - if not math.isfinite(number): - raise ValueError("must be finite") - if number < 0: - raise ValueError("must be non-negative") - return number - - @model_validator(mode="after") - def require_ordered_times(self) -> "TranscriptSegment": - if self.end < self.start: - raise ValueError("end must be greater than or equal to start") - return self - - -class SourceTranscriptSegment(BaseModel): - model_config = ConfigDict(extra="ignore") - - id: Optional[int] = Field(default=None, ge=1) - speaker: StrictStr - start: float - end: float - text: StrictStr - categories: Optional[List[StrictStr]] = None - - @field_validator("id", mode="before") - @classmethod - def require_optional_integer_id(cls, value: Any) -> Optional[int]: - if value is None: - return None - if isinstance(value, bool) or not isinstance(value, int): - raise ValueError("must be an integer") - return value - - @field_validator("speaker", "text") - @classmethod - def require_non_empty_text(cls, value: str) -> str: - if not value.strip(): - raise ValueError("must not be empty") - return value - - @field_validator("categories") - @classmethod - def require_non_empty_categories(cls, categories: Optional[List[str]]) -> Optional[List[str]]: - if categories is None: - return None - for category in categories: - if not category.strip(): - raise ValueError("categories must not contain empty strings") - return categories - - @field_validator("start", "end", mode="before") - @classmethod - def require_number(cls, value: Any) -> float: - if isinstance(value, bool) or not isinstance(value, (int, float)): - raise ValueError("must be a JSON number") - number = float(value) - if not math.isfinite(number): - raise ValueError("must be finite") - if number < 0: - raise ValueError("must be non-negative") - return number - - @model_validator(mode="after") - def require_ordered_times(self) -> "SourceTranscriptSegment": - if self.end < self.start: - raise ValueError("end must be greater than or equal to start") - return self - - -class GlossaryEntry(BaseModel): - model_config = ConfigDict(extra="forbid") - - name: StrictStr - aliases: List[StrictStr] = Field(default_factory=list) - plural: Optional[StrictStr] = None - category: StrictStr - summary: StrictStr - - @field_validator("name", "category", "summary") - @classmethod - def require_non_empty_text(cls, value: str) -> str: - if not value.strip(): - raise ValueError("must not be empty") - return value - - @field_validator("aliases") - @classmethod - def require_non_empty_aliases(cls, aliases: List[str]) -> List[str]: - for alias in aliases: - if not alias.strip(): - raise ValueError("aliases must not contain empty strings") - return aliases - - @field_validator("plural") - @classmethod - def require_non_empty_plural(cls, plural: Optional[str]) -> Optional[str]: - if plural is not None and not plural.strip(): - raise ValueError("plural must not be empty") - return plural - - -class Glossary(BaseModel): - model_config = ConfigDict(extra="forbid") - - glossary: List[GlossaryEntry] - - @field_validator("glossary") - @classmethod - def require_entries(cls, entries: List[GlossaryEntry]) -> List[GlossaryEntry]: - if not entries: - raise ValueError("glossary must contain at least one entry") - return entries - - -_TRANSCRIPT_ADAPTER = TypeAdapter(List[TranscriptSegment]) -_SOURCE_TRANSCRIPT_ADAPTER = TypeAdapter(List[SourceTranscriptSegment]) - - -def validate_transcript_data( - data: Any, - require_sequential_ids: bool = True, -) -> List[TranscriptSegment]: - data = _extract_transcript_segments(data) - if not isinstance(data, list): - raise AuditaValidationError("Transcript must be a JSON array or an object with a segments array.") - if not data: - raise AuditaValidationError("Transcript must contain at least one segment.") - try: - transcript = _TRANSCRIPT_ADAPTER.validate_python(data) - except ValidationError as exc: - raise AuditaValidationError(f"Transcript schema validation failed: {exc}") from exc - if require_sequential_ids: - _validate_sequential_ids(transcript) - return transcript - - -def validate_source_transcript_data(data: Any) -> List[SourceTranscriptSegment]: - data = _extract_transcript_segments(data) - if not isinstance(data, list): - raise AuditaValidationError("Transcript must be a JSON array or an object with a segments array.") - if not data: - raise AuditaValidationError("Transcript must contain at least one segment.") - try: - return _SOURCE_TRANSCRIPT_ADAPTER.validate_python(data) - except ValidationError as exc: - raise AuditaValidationError(f"Transcript schema validation failed: {exc}") from exc - - -def parse_transcript_json(raw: str, require_sequential_ids: bool = True) -> List[TranscriptSegment]: - try: - data = json.loads(raw) - except json.JSONDecodeError as exc: - raise AuditaValidationError(f"Transcript is not valid JSON: {exc}") from exc - return validate_transcript_data(data, require_sequential_ids=require_sequential_ids) - - -def parse_source_transcript_json(raw: str) -> List[SourceTranscriptSegment]: - try: - data = json.loads(raw) - except json.JSONDecodeError as exc: - raise AuditaValidationError(f"Transcript is not valid JSON: {exc}") from exc - return validate_source_transcript_data(data) - - -def parse_glossary_yaml(raw: str) -> Glossary: - try: - import yaml - except ImportError as exc: - raise AuditaValidationError("PyYAML is required to read glossary files.") from exc - - try: - data = yaml.safe_load(raw) - except yaml.YAMLError as exc: - raise AuditaValidationError(f"Glossary is not valid YAML: {exc}") from exc - - if data is None: - data = {} - try: - return Glossary.model_validate(data) - except ValidationError as exc: - raise AuditaValidationError(f"Glossary schema validation failed: {exc}") from exc - - -def transcript_to_json(segments: List[TranscriptSegment]) -> str: - payload = [segment.model_dump(mode="json", exclude_none=True) for segment in segments] - return json.dumps(payload, ensure_ascii=False, indent=2) + "\n" - - -def source_transcript_to_json(segments: List[SourceTranscriptSegment]) -> str: - payload = [segment.model_dump(mode="json", exclude_none=True) for segment in segments] - return json.dumps(payload, ensure_ascii=False, indent=2) + "\n" - - -def _extract_transcript_segments(data: Any) -> Any: - if isinstance(data, dict) and "segments" in data: - return data["segments"] - return data - - -def _validate_sequential_ids(transcript: List[TranscriptSegment]) -> None: - ids = [segment.id for segment in transcript] - expected = list(range(1, len(transcript) + 1)) - if ids != expected: - raise AuditaValidationError("Transcript segment ids must be sequential starting at 1.") diff --git a/python/src/audita/framework/__init__.py b/python/src/audita/framework/__init__.py deleted file mode 100644 index e8cd49e..0000000 --- a/python/src/audita/framework/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Reusable module, filter, review, and runner contracts.""" diff --git a/python/src/audita/framework/llm.py b/python/src/audita/framework/llm.py deleted file mode 100644 index f2d88b9..0000000 --- a/python/src/audita/framework/llm.py +++ /dev/null @@ -1,108 +0,0 @@ -from __future__ import annotations - -import threading -from typing import Any, Optional, Sequence, Tuple - -from audita.core.config import AuditaConfig, DEFAULT_BASE_URL -from audita.core.errors import AuditaLLMError - -_NO_KEY_PLACEHOLDER = "audita-no-key-required" - - -class OpenAICompatibleStructuredLLMClient: - def __init__(self) -> None: - self._client = None - self._client_identity: Optional[Tuple[str, str, float]] = None - self._client_lock = threading.Lock() - - def run_structured( - self, - *, - stage_name: str, - messages: Sequence[dict], - response_model: Any, - config: AuditaConfig, - ) -> Any: - if _requires_api_key(config) and not config.api_key: - raise AuditaLLMError( - f"Structured LLM stage '{stage_name}' requires LLM API credentials for the configured OpenRouter endpoint " - "via --llm-api-key, --validation-llm-api-key, AUDITA_LLM_API_KEY, " - "AUDITA_VALIDATION_LLM_API_KEY, or OPENROUTER_API_KEY." - ) - - client = self._get_client(config) - request = { - "model": _request_model_name(config), - "messages": list(messages), - "response_model": response_model, - "max_retries": config.max_retries, - } - if _uses_openrouter_shape(config): - request["extra_body"] = {"provider": {"require_parameters": True}} - try: - return client.chat.completions.create(**request) - except Exception as exc: - raise AuditaLLMError( - f"Structured LLM stage '{stage_name}' failed. Confirm the configured model and " - "OpenAI-compatible endpoint support tool calling or structured outputs." - ) from exc - - def _get_client(self, config: AuditaConfig) -> Any: - identity = (config.api_key or "", config.base_url, config.llm_timeout_seconds) - with self._client_lock: - if self._client is not None and self._client_identity == identity: - return self._client - - try: - import instructor - from openai import OpenAI - except ImportError as exc: - raise AuditaLLMError( - "The LLM dependencies are not installed. Run `uv sync` before using Audita LLM stages." - ) from exc - - openai_client = OpenAI( - api_key=_client_api_key(config), - base_url=config.base_url, - timeout=config.llm_timeout_seconds, - ) - self._client = instructor.patch(openai_client, mode=instructor.Mode.TOOLS) - self._client_identity = identity - return self._client - - -def _normalize_openrouter_model(model: str) -> str: - prefix = "openrouter/" - if model.startswith(prefix): - return model[len(prefix) :] - return model - - -def _request_model_name(config: AuditaConfig) -> str: - if _uses_openrouter_shape(config): - return _normalize_openrouter_model(config.model) - return config.model - - -def _uses_openrouter_shape(config: AuditaConfig) -> bool: - return _normalized_base_url(config.base_url) == _normalized_base_url(DEFAULT_BASE_URL) or config.model.startswith( - "openrouter/" - ) - - -def _requires_api_key(config: AuditaConfig) -> bool: - return _normalized_base_url(config.base_url) == _normalized_base_url(DEFAULT_BASE_URL) - - -def _client_api_key(config: AuditaConfig) -> str: - if config.api_key: - return config.api_key - if _requires_api_key(config): - raise AuditaLLMError( - "OpenRouter credentials are required but missing for the configured endpoint." - ) - return _NO_KEY_PLACEHOLDER - - -def _normalized_base_url(base_url: str) -> str: - return base_url.rstrip("/") diff --git a/python/src/audita/framework/llm_scheduler.py b/python/src/audita/framework/llm_scheduler.py deleted file mode 100644 index 546e716..0000000 --- a/python/src/audita/framework/llm_scheduler.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import annotations - -import threading -from typing import Callable, TypeVar - - -T = TypeVar("T") - - -class ModuleLLMScheduler: - def __init__(self, max_concurrency: int) -> None: - self.max_concurrency = max_concurrency - self._semaphore = threading.BoundedSemaphore(max_concurrency) - - def run_backend_call(self, fn: Callable[[], T]) -> T: - with self._semaphore: - return fn() diff --git a/python/src/audita/framework/models.py b/python/src/audita/framework/models.py deleted file mode 100644 index c00593b..0000000 --- a/python/src/audita/framework/models.py +++ /dev/null @@ -1,80 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Optional, Protocol, Sequence - -from audita.core.chunking import TranscriptSection -from audita.core.config import AuditaConfig -from audita.core.schemas import Glossary -from audita.validators.base import Validator - -from .llm_scheduler import ModuleLLMScheduler - - -ReplacementPolicy = str - - -@dataclass(frozen=True) -class ModuleRunSpec: - instance_name: str - module_key: str - module: "TranscriptModule" - - -@dataclass(frozen=True) -class CorrectionProposal: - proposal_index: int - module_instance: str - module_key: str - id: int - original_text: str - corrected_text: str - confidence: float - - def to_prompt_payload(self) -> dict: - return { - "proposal_index": self.proposal_index, - "id": self.id, - "original_text": self.original_text, - "corrected_text": self.corrected_text, - "confidence": self.confidence, - } - - -@dataclass(frozen=True) -class ModuleContext: - run_spec: ModuleRunSpec - glossary: Glossary - config: AuditaConfig - run_dir: Path - llm_client: Optional["StructuredLLMClient"] = None - llm_scheduler: Optional[ModuleLLMScheduler] = None - validation_llm_scheduler: Optional[ModuleLLMScheduler] = None - - -class StructuredLLMClient(Protocol): - def run_structured( - self, - *, - stage_name: str, - messages: Sequence[dict], - response_model: Any, - config: AuditaConfig, - ) -> Any: - ... - - -class TranscriptModule(Protocol): - module_key: str - replacement_policy: ReplacementPolicy - - def validators(self) -> Sequence[Validator]: - ... - - def propose( - self, - transcript_section: TranscriptSection, - context: ModuleContext, - ) -> Sequence[CorrectionProposal]: - ... diff --git a/python/src/audita/framework/proposal_generation.py b/python/src/audita/framework/proposal_generation.py deleted file mode 100644 index 0eafa43..0000000 --- a/python/src/audita/framework/proposal_generation.py +++ /dev/null @@ -1,96 +0,0 @@ -from __future__ import annotations - -import json -import math -from typing import Any, Callable, Dict, List - -from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator - -from audita.core.chunking import TranscriptSection -from audita.core.errors import AuditaLLMError -from audita.core.schemas import Glossary - -from .models import CorrectionProposal, ModuleContext - - -Message = Dict[str, str] -ProposalPromptBuilder = Callable[[TranscriptSection, Glossary], List[Message]] - - -class StructuredCorrectionCandidate(BaseModel): - model_config = ConfigDict(extra="forbid") - - id: int = Field(ge=1) - original_text: StrictStr - corrected_text: StrictStr - confidence: float = Field(ge=0.0, le=1.0) - - @field_validator("id", mode="before") - @classmethod - def require_integer_id(cls, value: Any) -> int: - if isinstance(value, bool) or not isinstance(value, int): - raise ValueError("must be an integer") - return value - - @field_validator("confidence", mode="before") - @classmethod - def require_number(cls, value: Any) -> float: - if isinstance(value, bool) or not isinstance(value, (int, float)): - raise ValueError("must be a JSON number") - number = float(value) - if not math.isfinite(number): - raise ValueError("must be finite") - return number - - -class StructuredCorrectionSet(BaseModel): - model_config = ConfigDict(extra="forbid") - - corrections: List[StructuredCorrectionCandidate] = Field(default_factory=list) - - -def generate_llm_correction_proposals( - *, - section: TranscriptSection, - context: ModuleContext, - prompt_builder: ProposalPromptBuilder, -) -> List[CorrectionProposal]: - llm_client = context.llm_client - if llm_client is None: - raise AuditaLLMError(f"Module '{context.run_spec.instance_name}' requires a structured LLM client.") - - messages = prompt_builder(section, context.glossary) - prompt_path = context.run_dir / f"prompt-{section.section_index:04d}.json" - prompt_path.write_text(json.dumps(messages, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - - if context.llm_scheduler is not None: - response = context.llm_scheduler.run_backend_call( - lambda: llm_client.run_structured( - stage_name=f"{context.run_spec.instance_name}:proposal", - messages=messages, - response_model=StructuredCorrectionSet, - config=context.config.proposal_llm_config(), - ) - ) - else: - response = llm_client.run_structured( - stage_name=f"{context.run_spec.instance_name}:proposal", - messages=messages, - response_model=StructuredCorrectionSet, - config=context.config.proposal_llm_config(), - ) - response_path = context.run_dir / f"corrections-{section.section_index:04d}.json" - response_path.write_text(response.model_dump_json(indent=2) + "\n", encoding="utf-8") - - return [ - CorrectionProposal( - proposal_index=index, - module_instance=context.run_spec.instance_name, - module_key=context.run_spec.module_key, - id=correction.id, - original_text=correction.original_text, - corrected_text=correction.corrected_text, - confidence=correction.confidence, - ) - for index, correction in enumerate(response.corrections) - ] diff --git a/python/src/audita/framework/proposals.py b/python/src/audita/framework/proposals.py deleted file mode 100644 index d0ac8f4..0000000 --- a/python/src/audita/framework/proposals.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import TYPE_CHECKING, Optional, Sequence, Union - -from audita.core.schemas import TranscriptSegment - -if TYPE_CHECKING: - from .models import CorrectionProposal - - -ReplacementPolicy = str - - -@dataclass(frozen=True) -class ProposalPreview: - proposal: CorrectionProposal - segment_index: int - segment: TranscriptSegment - corrected_segment_text: str - - @property - def original_segment_text(self) -> str: - return self.segment.text - - def to_prompt_payload(self) -> dict: - payload = { - "correction_index": self.proposal.proposal_index, - "id": self.proposal.id, - "original_segment_text": self.original_segment_text, - "corrected_segment_text": self.corrected_segment_text, - "original_text": self.proposal.original_text, - "corrected_text": self.proposal.corrected_text, - } - if self.segment.categories is not None: - payload["categories"] = list(self.segment.categories) - return payload - - -@dataclass(frozen=True) -class ProposalPreviewError: - reason: str - actual_text: Optional[str] = None - - -def preview_proposal( - transcript: Sequence[TranscriptSegment], - proposal: CorrectionProposal, - replacement_policy: ReplacementPolicy, -) -> Union[ProposalPreview, ProposalPreviewError]: - index_by_id = {segment.id: index for index, segment in enumerate(transcript)} - segment_index = index_by_id.get(proposal.id) - if segment_index is None: - return ProposalPreviewError(reason="proposal references unknown segment id") - - segment = transcript[segment_index] - if proposal.original_text == "": - return ProposalPreviewError( - reason="proposal original_text must not be empty", - actual_text=segment.text, - ) - - match_count = segment.text.count(proposal.original_text) - if match_count == 0: - return ProposalPreviewError( - reason="proposal original_text does not match segment text", - actual_text=segment.text, - ) - - if replacement_policy == "require_unique" and match_count != 1: - return ProposalPreviewError( - reason="proposal original_text must match exactly once", - actual_text=segment.text, - ) - - corrected_segment_text = segment.text.replace(proposal.original_text, proposal.corrected_text) - return ProposalPreview( - proposal=proposal, - segment_index=segment_index, - segment=segment, - corrected_segment_text=corrected_segment_text, - ) diff --git a/python/src/audita/framework/runner.py b/python/src/audita/framework/runner.py deleted file mode 100644 index 540c249..0000000 --- a/python/src/audita/framework/runner.py +++ /dev/null @@ -1,485 +0,0 @@ -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass -from pathlib import Path -from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union - -from audita.core.chunking import TranscriptSection, chunk_transcript -from audita.core.config import AuditaConfig -from audita.core.reporting import AppliedChange, ModuleRunReport, ReportedSkip, ValidatorReport -from audita.core.schemas import Glossary, TranscriptSegment -from audita.validators.base import ValidationContext, ValidationDecision, ValidationResult - -from .llm_scheduler import ModuleLLMScheduler -from .models import CorrectionProposal, ModuleContext, ModuleRunSpec, StructuredLLMClient -from .proposals import ProposalPreviewError, preview_proposal - - -ProgressCallback = Callable[[str], None] - - -@dataclass(frozen=True) -class PipelineRunResult: - transcript: List[TranscriptSegment] - module_reports: List[ModuleRunReport] - applied_changes: List[AppliedChange] - skipped_corrections: List[ReportedSkip] - - -class ModuleExecutionError(Exception): - def __init__( - self, - *, - transcript: List[TranscriptSegment], - applied_changes: List[AppliedChange], - skipped_corrections: List[ReportedSkip], - cause: Exception, - ) -> None: - super().__init__(str(cause)) - self.transcript = transcript - self.applied_changes = applied_changes - self.skipped_corrections = skipped_corrections - self.cause = cause - - -class PipelineRunError(Exception): - def __init__( - self, - *, - transcript: List[TranscriptSegment], - module_reports: List[ModuleRunReport], - applied_changes: List[AppliedChange], - skipped_corrections: List[ReportedSkip], - cause: Exception, - module_instance: Optional[str], - ) -> None: - super().__init__(str(cause)) - self.transcript = transcript - self.module_reports = module_reports - self.applied_changes = applied_changes - self.skipped_corrections = skipped_corrections - self.cause = cause - self.module_instance = module_instance - - -class PipelineRunner: - def run( - self, - *, - transcript: List[TranscriptSegment], - glossary: Glossary, - module_specs: Sequence[ModuleRunSpec], - config: AuditaConfig, - run_dir: Path, - llm_client: Optional[StructuredLLMClient] = None, - progress: Optional[ProgressCallback] = None, - ) -> PipelineRunResult: - working = list(transcript) - module_reports: List[ModuleRunReport] = [] - applied_changes: List[AppliedChange] = [] - skipped_corrections: List[ReportedSkip] = [] - - for run_spec in module_specs: - try: - module_dir = run_dir / run_spec.instance_name - module_dir.mkdir(parents=True, exist_ok=True) - context = ModuleContext( - run_spec=run_spec, - glossary=glossary, - config=config, - run_dir=module_dir, - llm_client=llm_client, - llm_scheduler=ModuleLLMScheduler(config.llm_concurrency), - validation_llm_scheduler=ModuleLLMScheduler(config.effective_validation_llm_concurrency()), - ) - sections = chunk_transcript( - working, - config.max_section_tokens, - min_section_tokens=config.min_section_tokens, - target_section_count=config.llm_concurrency if config.target_sections is None else None, - exact_target_section_count=config.target_sections, - ) - if progress is not None: - progress( - f"Running module {run_spec.instance_name} " - f"({len(sections)} sections, {len(working)} segments)" - ) - result = _run_module( - working=working, - sections=sections, - context=context, - llm_client=llm_client, - ) - except ModuleExecutionError as exc: - raise PipelineRunError( - transcript=exc.transcript, - module_reports=list(module_reports), - applied_changes=[*applied_changes, *exc.applied_changes], - skipped_corrections=[*skipped_corrections, *exc.skipped_corrections], - cause=exc.cause, - module_instance=run_spec.instance_name, - ) from exc - except Exception as exc: - raise PipelineRunError( - transcript=list(working), - module_reports=list(module_reports), - applied_changes=list(applied_changes), - skipped_corrections=list(skipped_corrections), - cause=exc, - module_instance=run_spec.instance_name, - ) from exc - working = result.transcript - module_reports.append(result.module_report) - applied_changes.extend(result.applied_changes) - skipped_corrections.extend(result.skipped_corrections) - - return PipelineRunResult( - transcript=working, - module_reports=module_reports, - applied_changes=applied_changes, - skipped_corrections=skipped_corrections, - ) - - -@dataclass(frozen=True) -class _ModuleExecutionResult: - transcript: List[TranscriptSegment] - module_report: ModuleRunReport - applied_changes: List[AppliedChange] - skipped_corrections: List[ReportedSkip] - - -def _run_module( - *, - working: List[TranscriptSegment], - sections: Sequence[TranscriptSection], - context: ModuleContext, - llm_client: Optional[StructuredLLMClient], -) -> _ModuleExecutionResult: - module = context.run_spec.module - validators = list(module.validators()) - raw_proposals: List[CorrectionProposal] = [] - validator_reports: List[ValidatorReport] = [] - skipped: List[ReportedSkip] = [] - applied_changes: List[AppliedChange] = [] - updated_transcript = list(working) - try: - section_proposals = _collect_module_proposals(sections=sections, context=context) - for proposed in section_proposals: - raw_proposals.extend(proposed) - - proposals = [ - CorrectionProposal( - proposal_index=index, - module_instance=context.run_spec.instance_name, - module_key=context.run_spec.module_key, - id=proposal.id, - original_text=proposal.original_text, - corrected_text=proposal.corrected_text, - confidence=proposal.confidence, - ) - for index, proposal in enumerate(raw_proposals) - ] - - surviving = proposals - validator_index = 0 - while validator_index < len(validators): - validator = validators[validator_index] - candidate_count = len(surviving) - if not surviving: - validator_reports.append( - ValidatorReport( - name=validator.name, - execution_kind=validator.execution_kind, - candidate_count=0, - approved_count=0, - rejected_count=0, - ) - ) - validator_index += 1 - continue - - if validator.execution_kind != "llm": - result = _run_single_validator( - validator=validator, - proposals=surviving, - working=working, - context=context, - llm_client=llm_client, - ) - validator_reports.append(result.report) - skipped.extend(result.skipped) - surviving = result.approved - validator_index += 1 - continue - - group_start = validator_index - llm_group = [] - while validator_index < len(validators) and validators[validator_index].execution_kind == "llm": - llm_group.append(validators[validator_index]) - validator_index += 1 - group_result = _run_parallel_llm_validator_group( - validators=llm_group, - proposals=surviving, - working=working, - context=context, - llm_client=llm_client, - ) - validator_reports.extend(group_result.reports) - skipped.extend(group_result.skipped) - surviving = group_result.approved - - for proposal in surviving: - apply_result = _apply_proposal(updated_transcript, proposal, module.replacement_policy) - if isinstance(apply_result, ReportedSkip): - skipped.append(apply_result) - continue - updated_transcript, applied_change = apply_result - applied_changes.append(applied_change) - - module_report = ModuleRunReport( - instance_name=context.run_spec.instance_name, - module_key=context.run_spec.module_key, - replacement_policy=module.replacement_policy, - section_count=len(sections), - proposal_count=len(proposals), - validators=validator_reports, - approved_count=len(surviving), - applied_count=len(applied_changes), - skipped_count=len(skipped), - ) - return _ModuleExecutionResult( - transcript=updated_transcript, - module_report=module_report, - applied_changes=applied_changes, - skipped_corrections=skipped, - ) - except Exception as exc: - raise ModuleExecutionError( - transcript=updated_transcript, - applied_changes=applied_changes, - skipped_corrections=skipped, - cause=exc, - ) from exc - - -def _collect_module_proposals( - *, - sections: Sequence[TranscriptSection], - context: ModuleContext, -) -> List[List[CorrectionProposal]]: - module = context.run_spec.module - max_workers = context.llm_scheduler.max_concurrency if context.llm_scheduler is not None else context.config.llm_concurrency - if max_workers == 1 or len(sections) <= 1: - return [list(module.propose(section, context)) for section in sections] - - with ThreadPoolExecutor(max_workers=max_workers) as executor: - return list(executor.map(lambda section: list(module.propose(section, context)), sections)) - - -@dataclass(frozen=True) -class _SingleValidatorResult: - approved: List[CorrectionProposal] - report: ValidatorReport - skipped: List[ReportedSkip] - - -@dataclass(frozen=True) -class _ParallelValidatorGroupResult: - approved: List[CorrectionProposal] - reports: List[ValidatorReport] - skipped: List[ReportedSkip] - - -def _build_validation_context( - *, - proposals: Sequence[CorrectionProposal], - working: Sequence[TranscriptSegment], - context: ModuleContext, - llm_client: Optional[StructuredLLMClient], -) -> ValidationContext: - return ValidationContext( - proposals=proposals, - transcript=working, - glossary=context.glossary, - config=context.config.validation_llm_config(), - run_spec=context.run_spec, - run_dir=context.run_dir, - llm_client=llm_client, - llm_scheduler=context.validation_llm_scheduler, - ) - - -def _run_single_validator( - *, - validator, - proposals: Sequence[CorrectionProposal], - working: Sequence[TranscriptSegment], - context: ModuleContext, - llm_client: Optional[StructuredLLMClient], -) -> _SingleValidatorResult: - candidate_count = len(proposals) - validation_context = _build_validation_context( - proposals=proposals, - working=working, - context=context, - llm_client=llm_client, - ) - result = validator.validate(validation_context) - decisions_by_index = _index_validation_decisions(result, proposals, validator.name) - approved: List[CorrectionProposal] = [] - skipped: List[ReportedSkip] = [] - rejected_count = 0 - for proposal in proposals: - decision = decisions_by_index[proposal.proposal_index] - if decision.approved: - approved.append(proposal) - continue - rejected_count += 1 - skipped.append(_reported_skip_from_decision(proposal, working, validator.name, decision)) - return _SingleValidatorResult( - approved=approved, - report=ValidatorReport( - name=validator.name, - execution_kind=validator.execution_kind, - candidate_count=candidate_count, - approved_count=len(approved), - rejected_count=rejected_count, - ), - skipped=skipped, - ) - - -def _run_parallel_llm_validator_group( - *, - validators: Sequence, - proposals: Sequence[CorrectionProposal], - working: Sequence[TranscriptSegment], - context: ModuleContext, - llm_client: Optional[StructuredLLMClient], -) -> _ParallelValidatorGroupResult: - validation_context = _build_validation_context( - proposals=proposals, - working=working, - context=context, - llm_client=llm_client, - ) - scheduler = context.validation_llm_scheduler - max_workers = scheduler.max_concurrency if scheduler is not None else context.config.effective_validation_llm_concurrency() - if max_workers == 1 or len(validators) <= 1: - results = [validator.validate(validation_context) for validator in validators] - else: - with ThreadPoolExecutor(max_workers=max_workers) as executor: - results = list(executor.map(lambda validator: validator.validate(validation_context), validators)) - - indexed_results = [ - _index_validation_decisions(result, proposals, validator.name) - for validator, result in zip(validators, results) - ] - reports = [ - ValidatorReport( - name=validator.name, - execution_kind=validator.execution_kind, - candidate_count=len(proposals), - approved_count=sum(1 for decision in decisions.values() if decision.approved), - rejected_count=sum(1 for decision in decisions.values() if not decision.approved), - ) - for validator, decisions in zip(validators, indexed_results) - ] - - approved: List[CorrectionProposal] = [] - skipped: List[ReportedSkip] = [] - for proposal in proposals: - rejection = None - for validator, decisions in zip(validators, indexed_results): - decision = decisions[proposal.proposal_index] - if not decision.approved: - rejection = (validator.name, decision) - break - if rejection is None: - approved.append(proposal) - continue - validator_name, decision = rejection - skipped.append(_reported_skip_from_decision(proposal, working, validator_name, decision)) - - return _ParallelValidatorGroupResult(approved=approved, reports=reports, skipped=skipped) - - -def _reported_skip_from_decision( - proposal: CorrectionProposal, - working: Sequence[TranscriptSegment], - validator_name: str, - decision: ValidationDecision, -) -> ReportedSkip: - return ReportedSkip( - module_instance=proposal.module_instance, - module_key=proposal.module_key, - proposal_index=proposal.proposal_index, - id=proposal.id, - reason=decision.reason or f"{validator_name} rejected proposal", - original_text=proposal.original_text, - corrected_text=proposal.corrected_text, - confidence=proposal.confidence, - actual_text=_segment_text_by_id(working).get(proposal.id), - source=f"validator:{validator_name}", - ) - - -def _index_validation_decisions( - result: ValidationResult, - proposals: Sequence[CorrectionProposal], - validator_name: str, -) -> Dict[int, ValidationDecision]: - expected_indexes = {proposal.proposal_index for proposal in proposals} - indexed: Dict[int, ValidationDecision] = {} - for decision in result.decisions: - if decision.proposal_index in indexed: - raise ValueError(f"Validator '{validator_name}' returned duplicate proposal indexes.") - if decision.proposal_index not in expected_indexes: - raise ValueError(f"Validator '{validator_name}' returned an unknown proposal index.") - indexed[decision.proposal_index] = decision - missing = expected_indexes - set(indexed) - if missing: - raise ValueError(f"Validator '{validator_name}' omitted proposal indexes: {sorted(missing)}") - return indexed - - -def _apply_proposal( - transcript: List[TranscriptSegment], - proposal: CorrectionProposal, - replacement_policy: str, -) -> Union[Tuple[List[TranscriptSegment], AppliedChange], ReportedSkip]: - preview = preview_proposal(transcript, proposal, replacement_policy) - if isinstance(preview, ProposalPreviewError): - return ReportedSkip( - module_instance=proposal.module_instance, - module_key=proposal.module_key, - proposal_index=proposal.proposal_index, - id=proposal.id, - reason=preview.reason, - original_text=proposal.original_text, - corrected_text=proposal.corrected_text, - confidence=proposal.confidence, - actual_text=preview.actual_text, - source="application", - ) - - updated = list(transcript) - updated[preview.segment_index] = preview.segment.model_copy(update={"text": preview.corrected_segment_text}) - return ( - updated, - AppliedChange( - module_instance=proposal.module_instance, - module_key=proposal.module_key, - proposal_index=proposal.proposal_index, - id=proposal.id, - original_text=proposal.original_text, - corrected_text=proposal.corrected_text, - confidence=proposal.confidence, - segment_text_before=preview.original_segment_text, - segment_text_after=preview.corrected_segment_text, - ), - ) - - -def _segment_text_by_id(transcript: Sequence[TranscriptSegment]) -> Dict[int, str]: - return {segment.id: segment.text for segment in transcript} diff --git a/python/src/audita/modules/__init__.py b/python/src/audita/modules/__init__.py deleted file mode 100644 index 117fd51..0000000 --- a/python/src/audita/modules/__init__.py +++ /dev/null @@ -1,80 +0,0 @@ -from __future__ import annotations - -from collections import Counter, defaultdict -from typing import Sequence - -from audita.core.errors import AuditaConfigError - - -SUPPORTED_MODULE_KEYS = ("glossary", "homophones", "spoken_word", "grammar") -DEFAULT_MODULE_KEYS = ("glossary", "homophones", "glossary", "spoken_word", "grammar") - - -def normalize_module_keys(module_keys: Sequence[str]) -> tuple[str, ...]: - normalized = tuple(_normalize_module_key(item) for item in module_keys) - if not normalized: - raise AuditaConfigError("Module sequence must contain at least one module key.") - - invalid = tuple(item for item in normalized if item not in SUPPORTED_MODULE_KEYS) - if invalid: - valid = ", ".join(SUPPORTED_MODULE_KEYS) - unknown = ", ".join(invalid) - raise AuditaConfigError(f"Unknown module key(s): {unknown}. Valid module keys: {valid}.") - - return normalized - - -def resolve_module_specs(module_keys: Sequence[str]) -> list["ModuleRunSpec"]: - from audita.framework.models import ModuleRunSpec - - normalized = normalize_module_keys(module_keys) - counts = Counter(normalized) - seen: defaultdict[str, int] = defaultdict(int) - return [ - ModuleRunSpec( - instance_name=_instance_name(module_key, counts, seen), - module_key=module_key, - module=_instantiate_module(module_key), - ) - for module_key in normalized - ] - - -def default_module_specs() -> list["ModuleRunSpec"]: - return resolve_module_specs(DEFAULT_MODULE_KEYS) - - -def _instance_name(module_key: str, counts: Counter[str], seen: defaultdict[str, int]) -> str: - seen[module_key] += 1 - if counts[module_key] == 1: - return module_key - return f"{module_key}_{seen[module_key]}" - - -def _instantiate_module(module_key: str): - if module_key == "glossary": - from .glossary import GlossaryModule - - return GlossaryModule() - if module_key == "homophones": - from .homophones import HomophonesModule - - return HomophonesModule() - if module_key == "spoken_word": - from .spoken_word import SpokenWordModule - - return SpokenWordModule() - if module_key == "grammar": - from .grammar import GrammarModule - - return GrammarModule() - raise AuditaConfigError(f"Unknown module key '{module_key}'.") - - -def _normalize_module_key(value: str) -> str: - if not isinstance(value, str): - raise AuditaConfigError("Module keys must be strings.") - normalized = value.strip() - if not normalized: - raise AuditaConfigError("Module keys must not contain empty values.") - return normalized diff --git a/python/src/audita/modules/glossary.py b/python/src/audita/modules/glossary.py deleted file mode 100644 index bcb91a9..0000000 --- a/python/src/audita/modules/glossary.py +++ /dev/null @@ -1,43 +0,0 @@ -from typing import Sequence - -from audita.core.chunking import TranscriptSection -from audita.framework.models import CorrectionProposal, ModuleContext -from audita.framework.proposal_generation import generate_llm_correction_proposals -from audita.modules.prompts import build_glossary_proposal_messages -from audita.validators import ( - GlossaryStageProtectedGlossaryTermsValidator, - IdenticalTextValidator, - MeaningReversalValidator, - NonEmptySegmentValidator, - OriginalTextPresentValidator, - ProposalConfidenceValidator, - SpokenFormPlausibilityValidator, - Validator, -) - - -class GlossaryModule: - module_key = "glossary" - replacement_policy = "replace_all" - - def validators(self) -> Sequence[Validator]: - return [ - IdenticalTextValidator("identical_text_guard"), - OriginalTextPresentValidator("original_text_present_guard"), - ProposalConfidenceValidator("proposal_confidence_guard", "glossary_confidence_threshold"), - GlossaryStageProtectedGlossaryTermsValidator("glossary_stage_protected_glossary_guard"), - NonEmptySegmentValidator("non_empty_segment_guard"), - SpokenFormPlausibilityValidator("spoken_form_plausibility_review"), - MeaningReversalValidator("meaning_reversal_review"), - ] - - def propose( - self, - transcript_section: TranscriptSection, - context: ModuleContext, - ) -> Sequence[CorrectionProposal]: - return generate_llm_correction_proposals( - section=transcript_section, - context=context, - prompt_builder=build_glossary_proposal_messages, - ) diff --git a/python/src/audita/modules/grammar.py b/python/src/audita/modules/grammar.py deleted file mode 100644 index b9e7c26..0000000 --- a/python/src/audita/modules/grammar.py +++ /dev/null @@ -1,43 +0,0 @@ -from typing import Sequence - -from audita.core.chunking import TranscriptSection -from audita.framework.models import CorrectionProposal, ModuleContext -from audita.framework.proposal_generation import generate_llm_correction_proposals -from audita.modules.prompts import build_grammar_proposal_messages -from audita.validators import ( - GrammarOnlyValidator, - IdenticalTextValidator, - MeaningReversalValidator, - NonEmptySegmentValidator, - OriginalTextPresentValidator, - ProposalConfidenceValidator, - ProtectedGlossaryTermsValidator, - Validator, -) - - -class GrammarModule: - module_key = "grammar" - replacement_policy = "require_unique" - - def validators(self) -> Sequence[Validator]: - return [ - IdenticalTextValidator("identical_text_guard"), - OriginalTextPresentValidator("original_text_present_guard"), - ProposalConfidenceValidator("proposal_confidence_guard", "grammar_confidence_threshold"), - ProtectedGlossaryTermsValidator("protected_glossary_guard"), - NonEmptySegmentValidator("non_empty_segment_guard"), - GrammarOnlyValidator("grammar_only_guard"), - MeaningReversalValidator("meaning_reversal_review"), - ] - - def propose( - self, - transcript_section: TranscriptSection, - context: ModuleContext, - ) -> Sequence[CorrectionProposal]: - return generate_llm_correction_proposals( - section=transcript_section, - context=context, - prompt_builder=build_grammar_proposal_messages, - ) diff --git a/python/src/audita/modules/homophones.py b/python/src/audita/modules/homophones.py deleted file mode 100644 index e68fd46..0000000 --- a/python/src/audita/modules/homophones.py +++ /dev/null @@ -1,43 +0,0 @@ -from typing import Sequence - -from audita.core.chunking import TranscriptSection -from audita.framework.models import CorrectionProposal, ModuleContext -from audita.framework.proposal_generation import generate_llm_correction_proposals -from audita.modules.prompts import build_homophones_proposal_messages -from audita.validators import ( - IdenticalTextValidator, - MeaningReversalValidator, - NonEmptySegmentValidator, - OriginalTextPresentValidator, - ProposalConfidenceValidator, - ProtectedGlossaryTermsValidator, - SpokenFormPlausibilityValidator, - Validator, -) - - -class HomophonesModule: - module_key = "homophones" - replacement_policy = "require_unique" - - def validators(self) -> Sequence[Validator]: - return [ - IdenticalTextValidator("identical_text_guard"), - OriginalTextPresentValidator("original_text_present_guard"), - ProposalConfidenceValidator("proposal_confidence_guard", "homophones_confidence_threshold"), - ProtectedGlossaryTermsValidator("protected_glossary_guard"), - NonEmptySegmentValidator("non_empty_segment_guard"), - SpokenFormPlausibilityValidator("spoken_form_plausibility_review"), - MeaningReversalValidator("meaning_reversal_review"), - ] - - def propose( - self, - transcript_section: TranscriptSection, - context: ModuleContext, - ) -> Sequence[CorrectionProposal]: - return generate_llm_correction_proposals( - section=transcript_section, - context=context, - prompt_builder=build_homophones_proposal_messages, - ) diff --git a/python/src/audita/modules/prompts.py b/python/src/audita/modules/prompts.py deleted file mode 100644 index 4a9bcf6..0000000 --- a/python/src/audita/modules/prompts.py +++ /dev/null @@ -1,169 +0,0 @@ -import json -from typing import Dict, List - -from audita.core.chunking import TranscriptSection -from audita.core.schemas import Glossary - - -Message = Dict[str, str] - - -def build_glossary_proposal_messages(section: TranscriptSection, glossary: Glossary) -> List[Message]: - glossary_json = json.dumps(glossary.model_dump(mode="json", exclude_none=True), ensure_ascii=False, indent=2) - section_json = json.dumps(section.prompt_payload(), ensure_ascii=False, indent=2) - - system = ( - "You are Audita, a careful transcript correction assistant. " - "Identify only transcription errors that are strongly supported by the glossary. " - "A valid correction must be acoustically plausible: the original transcript text " - "should sound similar to the proposed correction when spoken aloud. " - "Do not make generic grammar, spelling, capitalization, style, or filler-word edits. " - "Do not substitute an unrelated glossary term just because it could fit the topic. " - "Preserve speaker names, timestamps, and meaning." - ) - user = ( - "Review this transcript section and return only glossary-supported corrections that should be applied.\n\n" - "Rules:\n" - "- Correct domain-specific names, aliases, jargon, deities, locations, NPCs, players, factions, and similar terms only when both the glossary and surrounding transcript context support the correction.\n" - "- The correction must plausibly fix a transcription error: the original words should be phonetically or acoustically similar to the corrected words in spoken English.\n" - "- Appropriate example: correcting \"gestures\" to \"Jesters\" can be valid if \"Jesters\" appears in the glossary and nearby context supports that inference.\n" - "- Inappropriate example: correcting \"Lyra\" to \"Jesters\" should be omitted because those words are not similar in spoken English, even if \"Jesters\" appears in the glossary.\n" - "- Do not replace one clear glossary term, character name, location, or ordinary word with a different glossary term unless it is a plausible mishearing.\n" - "- Treat glossary names and aliases already present in the transcript as protected spellings.\n" - "- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.\n" - "- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n" - "- If a segment includes categories, treat them as additional transcript context.\n" - "- Plural forms of glossary names and aliases are allowed targets when spoken similarity and context support them, even if the plural is not explicitly listed in the glossary.\n" - "- Use the exact id from the input segment.\n" - "- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n" - "- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n" - "- Each returned correction must contain only id, original_text, corrected_text, and confidence.\n" - "- Do not return corrections where original_text and corrected_text are identical.\n" - "- Do not return speaker, start, or end fields.\n" - "- Return only changed segments; do not return entries for unchanged segments.\n" - "- confidence must be between 0.0 and 1.0.\n" - "- If no corrections are needed, return an empty corrections list.\n\n" - f"Glossary:\n{glossary_json}\n\n" - f"Transcript section:\n{section_json}" - ) - return [{"role": "system", "content": system}, {"role": "user", "content": user}] - - -def build_homophones_proposal_messages(section: TranscriptSection, glossary: Glossary) -> List[Message]: - glossary_json = json.dumps(glossary.model_dump(mode="json", exclude_none=True), ensure_ascii=False, indent=2) - section_json = json.dumps(section.prompt_payload(), ensure_ascii=False, indent=2) - - system = ( - "You are Audita, a conservative homophone correction assistant. " - "Identify only transcript changes that plausibly reflect homophones, phonetic similarity, " - "or common mistranscriptions of spoken English. " - "Do not make punctuation, capitalization, spacing, filler-word, repetition, style, or grammar edits. " - "Do not paraphrase, summarize, or rewrite content." - ) - user = ( - "Review this transcript section and return only homophone or spoken-form corrections that should be applied.\n\n" - "Rules:\n" - "- Approve only corrections where the original text is plausibly a mistaken homophone, phonetic rendering, or mistranscription of what was likely spoken.\n" - "- Allow examples such as changing \"dam\" to \"damn\", \"rank\" to \"Hrank\", or \"gestures\" to \"Jesters\" when local context supports the correction.\n" - "- Reject unrelated substitutions like changing \"Lyra\" to \"Jesters\".\n" - "- Reject antonyms or reversals such as changing \"visible\" to \"invisible\".\n" - "- Do not add or remove punctuation, alter capitalization only, normalize spacing, remove filler words, collapse repetitions, or make general readability edits.\n" - "- Treat glossary names and aliases as protected spellings and context.\n" - "- You may correct toward glossary names, aliases, or their plural forms when the correction is acoustically plausible and supported by local context.\n" - "- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.\n" - "- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n" - "- If a segment includes categories, treat them as additional transcript context.\n" - "- Use the exact id from the input segment.\n" - "- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n" - "- Choose an original_text span that appears exactly once in the current segment text.\n" - "- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n" - "- Each returned correction must contain only id, original_text, corrected_text, and confidence.\n" - "- Do not return corrections where original_text and corrected_text are identical.\n" - "- Do not return speaker, start, or end fields.\n" - "- Return only changed segments; do not return entries for unchanged segments.\n" - "- confidence must be between 0.0 and 1.0.\n" - "- If no corrections are needed, return an empty corrections list.\n\n" - f"Protected glossary/context:\n{glossary_json}\n\n" - f"Transcript section:\n{section_json}" - ) - return [{"role": "system", "content": system}, {"role": "user", "content": user}] - - -def build_spoken_word_proposal_messages(section: TranscriptSection, glossary: Glossary) -> List[Message]: - glossary_json = json.dumps(glossary.model_dump(mode="json", exclude_none=True), ensure_ascii=False, indent=2) - section_json = json.dumps(section.prompt_payload(), ensure_ascii=False, indent=2) - - system = ( - "You are Audita, a conservative spoken-word cleanup assistant. " - "Identify only low-risk cleanup of repeated words or short phrases, filler words, hesitation artifacts, " - "and similar dysfluencies that commonly appear in spoken English transcripts. " - "Preserve substantive meaning, named entities, and transcript content." - ) - user = ( - "Review this transcript section and return only spoken-word cleanup corrections that should be applied.\n\n" - "Rules:\n" - "- Approve only conservative cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar spoken dysfluencies.\n" - "- You may collapse adjacent repetition such as \"I I think\" to \"I think\" or remove filler spans such as \"you know\" or \"uh\" when local context supports that cleanup.\n" - "- Do not collapse repeated words or short phrases when the repetition plausibly expresses urgency, excitement, insistence, or deliberate rhetorical emphasis rather than dysfluency.\n" - "- Phrases such as \"Help! Help! Help!\", \"Stop! Stop! Stop!\", \"No! No! No!\", \"Yes! Yes! Yes!\", and \"Go! Go! Go!\" are often intentional emphasis and should usually be preserved.\n" - "- Only collapse repetition when local context supports it as accidental spoken repetition, hesitation, or verbal restart.\n" - "- You may include low-risk punctuation, spacing, or capitalization cleanup when it is part of removing a dysfluency, such as removing ellipses or hesitation punctuation that no longer belongs after the cleanup.\n" - "- Do not paraphrase, summarize, reorder ideas, replace content with different wording, or make substantive semantic edits.\n" - "- Do not change clear content words just because a different phrasing reads better.\n" - "- Treat glossary names and aliases as protected spellings and context.\n" - "- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.\n" - "- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n" - "- If a segment includes categories, treat them as additional transcript context.\n" - "- Use the exact id from the input segment.\n" - "- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n" - "- Choose an original_text span that appears exactly once in the current segment text.\n" - "- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n" - "- Each returned correction must contain only id, original_text, corrected_text, and confidence.\n" - "- Do not return corrections where original_text and corrected_text are identical.\n" - "- Do not return speaker, start, or end fields.\n" - "- Return only changed segments; do not return entries for unchanged segments.\n" - "- confidence must be between 0.0 and 1.0.\n" - "- If no corrections are needed, return an empty corrections list.\n\n" - f"Protected glossary/context:\n{glossary_json}\n\n" - f"Transcript section:\n{section_json}" - ) - return [{"role": "system", "content": system}, {"role": "user", "content": user}] - - -def build_grammar_proposal_messages(section: TranscriptSection, glossary: Glossary) -> List[Message]: - glossary_json = json.dumps(glossary.model_dump(mode="json", exclude_none=True), ensure_ascii=False, indent=2) - section_json = json.dumps(section.prompt_payload(), ensure_ascii=False, indent=2) - - system = ( - "You are Audita, a conservative grammar cleanup assistant. " - "Identify only punctuation, capitalization, and spacing cleanup that preserves the same underlying words. " - "Do not change content, substitute words, or rewrite the speaker's phrasing." - ) - user = ( - "Review this transcript section and return only grammar cleanup corrections that should be applied.\n\n" - "Rules:\n" - "- Allowed changes are punctuation, capitalization, spacing, and article cleanup only.\n" - "- You may add, remove, or adjust commas, periods, quotation marks, apostrophes, dashes, ellipses, spacing, and capitalization when the underlying words stay the same.\n" - "- You may change the whole-word article \"a\" to \"an\" or \"an\" to \"a\" when the surrounding text otherwise stays the same.\n" - "- Homophone, spoken-form, and mistranscription corrections are handled during a later review stage; do not propose them here.\n" - "- Do not make word substitutions, spelling fixes, homophone fixes, filler cleanup, repetition cleanup, paraphrases, or other semantic rewrites.\n" - "- Do not change one written word into a different written word, except for capitalization changes to the same letters.\n" - "- If a possible correction depends on changing a content word into a different word, omit it here rather than bundling it together with formatting cleanup.\n" - "- Treat glossary names and aliases as protected spellings and context.\n" - "- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.\n" - "- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n" - "- If a segment includes categories, treat them as additional transcript context.\n" - "- Use the exact id from the input segment.\n" - "- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n" - "- Choose an original_text span that appears exactly once in the current segment text.\n" - "- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n" - "- Each returned correction must contain only id, original_text, corrected_text, and confidence.\n" - "- Do not return corrections where original_text and corrected_text are identical.\n" - "- Do not return speaker, start, or end fields.\n" - "- Return only changed segments; do not return entries for unchanged segments.\n" - "- confidence must be between 0.0 and 1.0.\n" - "- If no corrections are needed, return an empty corrections list.\n\n" - f"Protected glossary/context:\n{glossary_json}\n\n" - f"Transcript section:\n{section_json}" - ) - return [{"role": "system", "content": system}, {"role": "user", "content": user}] diff --git a/python/src/audita/modules/spoken_word.py b/python/src/audita/modules/spoken_word.py deleted file mode 100644 index a44db56..0000000 --- a/python/src/audita/modules/spoken_word.py +++ /dev/null @@ -1,43 +0,0 @@ -from typing import Sequence - -from audita.core.chunking import TranscriptSection -from audita.framework.models import CorrectionProposal, ModuleContext -from audita.framework.proposal_generation import generate_llm_correction_proposals -from audita.modules.prompts import build_spoken_word_proposal_messages -from audita.validators import ( - IdenticalTextValidator, - MeaningReversalValidator, - NonEmptySegmentValidator, - OriginalTextPresentValidator, - ProposalConfidenceValidator, - ProtectedGlossaryTermsValidator, - SpokenWordValidator, - Validator, -) - - -class SpokenWordModule: - module_key = "spoken_word" - replacement_policy = "require_unique" - - def validators(self) -> Sequence[Validator]: - return [ - IdenticalTextValidator("identical_text_guard"), - OriginalTextPresentValidator("original_text_present_guard"), - ProposalConfidenceValidator("proposal_confidence_guard", "spoken_word_confidence_threshold"), - ProtectedGlossaryTermsValidator("protected_glossary_guard"), - NonEmptySegmentValidator("non_empty_segment_guard"), - SpokenWordValidator("spoken_word_review"), - MeaningReversalValidator("meaning_reversal_review"), - ] - - def propose( - self, - transcript_section: TranscriptSection, - context: ModuleContext, - ) -> Sequence[CorrectionProposal]: - return generate_llm_correction_proposals( - section=transcript_section, - context=context, - prompt_builder=build_spoken_word_proposal_messages, - ) diff --git a/python/src/audita/pipeline.py b/python/src/audita/pipeline.py deleted file mode 100644 index 50b1f3d..0000000 --- a/python/src/audita/pipeline.py +++ /dev/null @@ -1,258 +0,0 @@ -import json -import shutil -from pathlib import Path -from typing import Any, Callable, List, Optional, Sequence - -from .core.config import AuditaConfig -from .core.diagnostics import build_error_details, create_run_dir, format_traceback_text, write_error_log -from .core.errors import AuditaError -from .core.normalization import normalize_transcript -from .core.reporting import AppliedChange, ModuleRunReport, ProcessResult, ReportedSkip, RunReport -from .core.schemas import Glossary, SourceTranscriptSegment, TranscriptSegment, source_transcript_to_json, transcript_to_json -from .framework.llm import OpenAICompatibleStructuredLLMClient -from .framework.models import StructuredLLMClient -from .framework.runner import PipelineRunError, PipelineRunner -from .modules import resolve_module_specs - - -ProgressCallback = Callable[[str], None] - - -def process_transcript( - transcript: List[SourceTranscriptSegment], - glossary: Glossary, - config: AuditaConfig, - module_keys: Optional[Sequence[str]] = None, - llm_client: Optional[StructuredLLMClient] = None, - progress: Optional[ProgressCallback] = None, -) -> List[TranscriptSegment]: - return process_transcript_result( - transcript, - glossary, - config, - module_keys=module_keys, - llm_client=llm_client, - progress=progress, - ).transcript - - -def process_transcript_result( - transcript: List[SourceTranscriptSegment], - glossary: Glossary, - config: AuditaConfig, - module_keys: Optional[Sequence[str]] = None, - llm_client: Optional[StructuredLLMClient] = None, - progress: Optional[ProgressCallback] = None, - run_dir: Optional[Path] = None, - invocation_details: Optional[dict[str, Any]] = None, -) -> ProcessResult: - run_dir = create_run_dir(config.work_dir) if run_dir is None else run_dir - module_specs = resolve_module_specs(config.module_keys if module_keys is None else module_keys) - normalized_transcript: List[TranscriptSegment] = [] - normalization_summary: Optional[dict] = None - report: Optional[RunReport] = None - report_path = run_dir / "report.json" - error_log_path = run_dir / "error.log" - try: - _log(progress, f"Created work directory {run_dir}") - normalization_result = normalize_transcript( - transcript, - max_segment_gap=config.normalize_max_segment_gap, - ellipsis_gap=config.normalize_ellipsis_gap, - max_segment_duration=config.normalize_max_segment_duration, - max_segment_tokens=config.normalize_max_segment_tokens, - ) - normalized_transcript = list(normalization_result.transcript) - normalization_summary = normalization_result.summary.to_dict() - _write_normalization_diagnostics(run_dir, transcript, normalization_result) - _log( - progress, - "Normalized transcript from " - f"{normalization_result.summary.source_segment_count} to " - f"{normalization_result.summary.normalized_segment_count} segments", - ) - - pipeline_runner = PipelineRunner() - effective_llm_client = OpenAICompatibleStructuredLLMClient() if llm_client is None else llm_client - pipeline_result = pipeline_runner.run( - transcript=normalized_transcript, - glossary=glossary, - module_specs=module_specs, - config=config, - run_dir=run_dir, - llm_client=effective_llm_client, - progress=progress, - ) - revised = _sort_transcript_chronologically(pipeline_result.transcript) - work_dir_retained = _should_retain_run_dir(config.work_dir_retention, bool(pipeline_result.skipped_corrections)) - report = _build_run_report( - status="success", - config=config.to_report_dict(), - normalization=normalization_summary, - pipeline=[spec.instance_name for spec in module_specs], - modules=pipeline_result.module_reports, - applied_changes=pipeline_result.applied_changes, - skipped_corrections=pipeline_result.skipped_corrections, - transcript=revised, - work_dir_retention=config.work_dir_retention, - work_dir_retained=work_dir_retained, - work_dir=str(run_dir) if work_dir_retained else None, - error=None, - error_details=None, - ) - _write_run_report(report_path, report) - if work_dir_retained: - _log(progress, f"Work directory preserved at {run_dir}") - else: - shutil.rmtree(run_dir) - _log(progress, "Removed work directory after successful run") - return ProcessResult( - transcript=revised, - report=report, - run_dir=run_dir, - work_dir_retained=work_dir_retained, - ) - except Exception as exc: - failed_report_data = _failure_report_data(exc, normalized_transcript) - root_error = failed_report_data["error"] - traceback_text = format_traceback_text(root_error) - error_details = build_error_details( - exc=root_error, - exit_code=1, - run_dir=run_dir, - report_path=report_path, - error_log_path=error_log_path, - phase=failed_report_data["phase"], - module_instance=failed_report_data["module_instance"], - argv=None if invocation_details is None else invocation_details.get("argv"), - cwd=None if invocation_details is None else invocation_details.get("cwd"), - traceback_text=traceback_text, - ) - write_error_log(error_log_path, error_details=error_details, traceback_text=traceback_text) - report = _build_run_report( - status="failed", - config=config.to_report_dict(), - normalization=normalization_summary, - pipeline=[spec.instance_name for spec in module_specs], - modules=failed_report_data["modules"], - applied_changes=failed_report_data["applied_changes"], - skipped_corrections=failed_report_data["skipped_corrections"], - transcript=failed_report_data["transcript"], - work_dir_retention=config.work_dir_retention, - work_dir_retained=True, - work_dir=str(run_dir), - error=str(root_error), - error_details=error_details, - ) - _write_run_report(report_path, report) - error = root_error - if isinstance(error, AuditaError): - reraised = type(error)(f"{error} Diagnostics preserved at {run_dir}") - setattr(reraised, "audita_run_dir", run_dir) - setattr(reraised, "audita_report_path", report_path) - setattr(reraised, "audita_error_log_path", error_log_path) - setattr(reraised, "audita_error_details", error_details) - raise reraised from error - reraised = AuditaError(f"{error} Diagnostics preserved at {run_dir}") - setattr(reraised, "audita_run_dir", run_dir) - setattr(reraised, "audita_report_path", report_path) - setattr(reraised, "audita_error_log_path", error_log_path) - setattr(reraised, "audita_error_details", error_details) - raise reraised from error - - -def _should_retain_run_dir(retention: str, has_skips: bool) -> bool: - if retention == "always": - return True - if retention == "never": - return False - return has_skips - - -def _sort_transcript_chronologically(transcript: List[TranscriptSegment]) -> List[TranscriptSegment]: - return list(sorted(transcript, key=lambda segment: (segment.start, segment.end, segment.id))) - - -def _write_normalization_diagnostics(run_dir: Path, transcript, normalization_result) -> None: - normalization_dir = run_dir / "normalization" - normalization_dir.mkdir(parents=True, exist_ok=True) - (normalization_dir / "source-transcript.json").write_text(source_transcript_to_json(transcript), encoding="utf-8") - (normalization_dir / "normalized-transcript.json").write_text( - transcript_to_json(normalization_result.transcript), - encoding="utf-8", - ) - (normalization_dir / "summary.json").write_text( - json.dumps(normalization_result.summary.to_dict(), indent=2) + "\n", - encoding="utf-8", - ) - - -def _write_run_report(path: Path, report: RunReport) -> None: - path.write_text(report.to_json(), encoding="utf-8") - - -def _log(progress: Optional[ProgressCallback], message: str) -> None: - if progress is not None: - progress(message) - - -def _build_run_report( - *, - status: str, - config: dict, - normalization: Optional[dict], - pipeline: List[str], - modules: List[ModuleRunReport], - applied_changes: List[AppliedChange], - skipped_corrections: List[ReportedSkip], - transcript: List[TranscriptSegment], - work_dir_retention: str, - work_dir_retained: bool, - work_dir: Optional[str], - error: Optional[str], - error_details: Optional[dict], -) -> RunReport: - return RunReport( - status=status, - config=config, - normalization=normalization, - pipeline=pipeline, - modules=modules, - applied_changes=applied_changes, - skipped_corrections=skipped_corrections, - totals={ - "output_segment_count": len(transcript), - "applied_change_count": len(applied_changes), - "skipped_correction_count": len(skipped_corrections), - }, - work_dir_retention=work_dir_retention, - work_dir_retained=work_dir_retained, - work_dir=work_dir, - error=error, - error_details=error_details, - ) - - -def _failure_report_data( - exc: Exception, - normalized_transcript: List[TranscriptSegment], -) -> dict: - if isinstance(exc, PipelineRunError): - return { - "modules": exc.module_reports, - "applied_changes": exc.applied_changes, - "skipped_corrections": exc.skipped_corrections, - "transcript": _sort_transcript_chronologically(exc.transcript), - "error": exc.cause, - "phase": "pipeline", - "module_instance": exc.module_instance, - } - return { - "modules": [], - "applied_changes": [], - "skipped_corrections": [], - "transcript": _sort_transcript_chronologically(normalized_transcript), - "error": exc, - "phase": "normalization", - "module_instance": None, - } diff --git a/python/src/audita/py.typed b/python/src/audita/py.typed deleted file mode 100644 index 8b13789..0000000 --- a/python/src/audita/py.typed +++ /dev/null @@ -1 +0,0 @@ - diff --git a/python/src/audita/validators/__init__.py b/python/src/audita/validators/__init__.py deleted file mode 100644 index 740a97c..0000000 --- a/python/src/audita/validators/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -from .base import ValidationContext, ValidationDecision, ValidationResult, Validator -from .deterministic import ( - GlossaryStageProtectedGlossaryTermsValidator, - IdenticalTextValidator, - NonEmptySegmentValidator, - OriginalTextPresentValidator, - ProposalConfidenceValidator, - ProtectedGlossaryTermsValidator, -) -from .llm import EditorialValidator, GrammarOnlyValidator, MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator -from .protection import ProtectedVocabulary - -__all__ = [ - "ValidationContext", - "ValidationDecision", - "ValidationResult", - "Validator", - "IdenticalTextValidator", - "OriginalTextPresentValidator", - "ProposalConfidenceValidator", - "ProtectedGlossaryTermsValidator", - "GlossaryStageProtectedGlossaryTermsValidator", - "NonEmptySegmentValidator", - "EditorialValidator", - "GrammarOnlyValidator", - "ProtectedVocabulary", - "SpokenFormPlausibilityValidator", - "SpokenWordValidator", - "MeaningReversalValidator", -] diff --git a/python/src/audita/validators/base.py b/python/src/audita/validators/base.py deleted file mode 100644 index e51f3d3..0000000 --- a/python/src/audita/validators/base.py +++ /dev/null @@ -1,47 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -from typing import TYPE_CHECKING, List, Optional, Protocol, Sequence - -from audita.core.config import AuditaConfig -from audita.core.schemas import Glossary, TranscriptSegment - -if TYPE_CHECKING: - from audita.framework.models import CorrectionProposal, ModuleRunSpec, StructuredLLMClient - from audita.framework.llm_scheduler import ModuleLLMScheduler - - -@dataclass(frozen=True) -class ValidationContext: - proposals: Sequence["CorrectionProposal"] - transcript: Sequence[TranscriptSegment] - glossary: Glossary - config: AuditaConfig - run_spec: "ModuleRunSpec" - run_dir: Path - llm_client: Optional["StructuredLLMClient"] = None - llm_scheduler: Optional["ModuleLLMScheduler"] = None - - -@dataclass(frozen=True) -class ValidationDecision: - proposal_index: int - approved: bool - confidence: Optional[float] = None - reason: Optional[str] = None - - -@dataclass(frozen=True) -class ValidationResult: - validator_name: str - execution_kind: str - decisions: List[ValidationDecision] - - -class Validator(Protocol): - name: str - execution_kind: str - - def validate(self, context: ValidationContext) -> ValidationResult: - ... diff --git a/python/src/audita/validators/deterministic.py b/python/src/audita/validators/deterministic.py deleted file mode 100644 index 3b900f8..0000000 --- a/python/src/audita/validators/deterministic.py +++ /dev/null @@ -1,156 +0,0 @@ -from dataclasses import dataclass - -from audita.framework.proposals import ProposalPreviewError, preview_proposal - -from .base import ValidationContext, ValidationDecision, ValidationResult -from .protection import ProtectedVocabulary - - -@dataclass(frozen=True) -class IdenticalTextValidator: - name: str - execution_kind: str = "deterministic" - - def validate(self, context: ValidationContext) -> ValidationResult: - return ValidationResult( - validator_name=self.name, - execution_kind=self.execution_kind, - decisions=[ - ValidationDecision( - proposal_index=proposal.proposal_index, - approved=proposal.original_text != proposal.corrected_text, - reason=( - None - if proposal.original_text != proposal.corrected_text - else "proposal original_text and corrected_text are identical" - ), - ) - for proposal in context.proposals - ], - ) - - -@dataclass(frozen=True) -class OriginalTextPresentValidator: - name: str - execution_kind: str = "deterministic" - - def validate(self, context: ValidationContext) -> ValidationResult: - segment_text_by_id = {segment.id: segment.text for segment in context.transcript} - return ValidationResult( - validator_name=self.name, - execution_kind=self.execution_kind, - decisions=[ - ValidationDecision( - proposal_index=proposal.proposal_index, - approved=( - (segment_text := segment_text_by_id.get(proposal.id)) is None - or proposal.original_text in segment_text - ), - reason=( - None - if (segment_text := segment_text_by_id.get(proposal.id)) is None or proposal.original_text in segment_text - else "proposal original_text does not match segment text" - ), - ) - for proposal in context.proposals - ], - ) - - -@dataclass(frozen=True) -class ProposalConfidenceValidator: - name: str - threshold_attr: str - execution_kind: str = "deterministic" - - def validate(self, context: ValidationContext) -> ValidationResult: - threshold = getattr(context.config, self.threshold_attr) - return ValidationResult( - validator_name=self.name, - execution_kind=self.execution_kind, - decisions=[ - ValidationDecision( - proposal_index=proposal.proposal_index, - approved=proposal.confidence >= threshold, - reason=None if proposal.confidence >= threshold else "proposal confidence below threshold", - ) - for proposal in context.proposals - ], - ) - - -@dataclass(frozen=True) -class ProtectedGlossaryTermsValidator: - name: str - execution_kind: str = "deterministic" - - def validate(self, context: ValidationContext) -> ValidationResult: - vocabulary = ProtectedVocabulary.from_glossary(context.glossary) - return ValidationResult( - validator_name=self.name, - execution_kind=self.execution_kind, - decisions=[ - ValidationDecision( - proposal_index=proposal.proposal_index, - approved=(reason := vocabulary.violation_reason( - proposal.original_text, - proposal.corrected_text, - )) is None, - reason=reason, - ) - for proposal in context.proposals - ], - ) - - -@dataclass(frozen=True) -class GlossaryStageProtectedGlossaryTermsValidator: - name: str - execution_kind: str = "deterministic" - - def validate(self, context: ValidationContext) -> ValidationResult: - vocabulary = ProtectedVocabulary.from_glossary(context.glossary) - return ValidationResult( - validator_name=self.name, - execution_kind=self.execution_kind, - decisions=[ - ValidationDecision( - proposal_index=proposal.proposal_index, - approved=(reason := vocabulary.glossary_stage_violation_reason( - proposal.original_text, - proposal.corrected_text, - )) is None, - reason=reason, - ) - for proposal in context.proposals - ], - ) - - -@dataclass(frozen=True) -class NonEmptySegmentValidator: - name: str - execution_kind: str = "deterministic" - - def validate(self, context: ValidationContext) -> ValidationResult: - replacement_policy = context.run_spec.module.replacement_policy - decisions: list[ValidationDecision] = [] - for proposal in context.proposals: - preview = preview_proposal(context.transcript, proposal, replacement_policy) - if isinstance(preview, ProposalPreviewError): - decisions.append(ValidationDecision(proposal_index=proposal.proposal_index, approved=True)) - continue - is_non_empty = preview.corrected_segment_text.strip() != "" - decisions.append( - ValidationDecision( - proposal_index=proposal.proposal_index, - approved=is_non_empty, - reason=None if is_non_empty else "correction would leave the segment empty", - ) - ) - return ValidationResult( - validator_name=self.name, - execution_kind=self.execution_kind, - decisions=decisions, - ) diff --git a/python/src/audita/validators/llm.py b/python/src/audita/validators/llm.py deleted file mode 100644 index f351bef..0000000 --- a/python/src/audita/validators/llm.py +++ /dev/null @@ -1,195 +0,0 @@ -from __future__ import annotations - -from concurrent.futures import ThreadPoolExecutor -import json -from dataclasses import dataclass -from pathlib import Path -from typing import Callable, Dict, List, Sequence - -from pydantic import BaseModel, ConfigDict, Field, StrictStr - -from audita.core.chunking import chunk_payload_items -from audita.core.errors import AuditaLLMError -from audita.framework.proposals import ProposalPreview, ProposalPreviewError, preview_proposal - -from .base import ValidationContext, ValidationDecision, ValidationResult -from .prompts import ( - build_editorial_messages, - build_meaning_reversal_messages, - build_spoken_form_plausibility_messages, -) - - -class _LLMValidationDecisionModel(BaseModel): - model_config = ConfigDict(extra="forbid") - - correction_index: int - approved: bool - confidence: float = Field(ge=0.0, le=1.0) - reason: StrictStr - - -class _LLMValidationSetModel(BaseModel): - model_config = ConfigDict(extra="forbid") - - validations: List[_LLMValidationDecisionModel] - - -PromptBuilder = Callable[[List[dict]], List[dict]] - - -@dataclass(frozen=True) -class _BaseLLMValidator: - name: str - prompt_builder: PromptBuilder - execution_kind: str = "llm" - - def validate(self, context: ValidationContext) -> ValidationResult: - if not context.proposals: - return ValidationResult( - validator_name=self.name, - execution_kind=self.execution_kind, - decisions=[], - ) - - previewable: List[ProposalPreview] = [] - decisions: List[ValidationDecision] = [] - replacement_policy = context.run_spec.module.replacement_policy - for proposal in context.proposals: - preview = preview_proposal(context.transcript, proposal, replacement_policy) - if isinstance(preview, ProposalPreviewError): - decisions.append( - ValidationDecision( - proposal_index=proposal.proposal_index, - approved=False, - reason=preview.reason, - ) - ) - continue - previewable.append(preview) - - if previewable: - llm_client = context.llm_client - if llm_client is None: - raise AuditaLLMError(f"Validator '{self.name}' requires a structured LLM client.") - batches = chunk_payload_items( - previewable, - context.config.validation_max_prompt_tokens, - payload_fn=lambda item: item.to_prompt_payload(), - empty_error_message="Validation input must contain at least one proposal.", - ) - max_workers = context.llm_scheduler.max_concurrency if context.llm_scheduler is not None else context.config.llm_concurrency - if max_workers == 1 or len(batches) <= 1: - for batch in batches: - decisions.extend(self._run_batch(context, llm_client, batch)) - else: - with ThreadPoolExecutor(max_workers=max_workers) as executor: - for batch_decisions in executor.map( - lambda batch: self._run_batch(context, llm_client, batch), - batches, - ): - decisions.extend(batch_decisions) - - return ValidationResult( - validator_name=self.name, - execution_kind=self.execution_kind, - decisions=sorted(decisions, key=lambda decision: decision.proposal_index), - ) - - def _validate_batch_response( - self, - response: _LLMValidationSetModel, - proposals: Sequence[ProposalPreview], - ) -> List[ValidationDecision]: - expected_indexes = {proposal.proposal.proposal_index for proposal in proposals} - indexed: Dict[int, _LLMValidationDecisionModel] = {} - for decision in response.validations: - if decision.correction_index in indexed: - raise AuditaLLMError( - f"Validator '{self.name}' returned duplicate correction_index values." - ) - if decision.correction_index not in expected_indexes: - raise AuditaLLMError( - f"Validator '{self.name}' returned an unknown correction_index." - ) - indexed[decision.correction_index] = decision - - missing_indexes = sorted(expected_indexes - set(indexed)) - if missing_indexes: - raise AuditaLLMError( - f"Validator '{self.name}' omitted correction_index values: {missing_indexes}" - ) - - return [ - ValidationDecision( - proposal_index=proposal.proposal.proposal_index, - approved=indexed[proposal.proposal.proposal_index].approved, - confidence=indexed[proposal.proposal.proposal_index].confidence, - reason=indexed[proposal.proposal.proposal_index].reason, - ) - for proposal in proposals - ] - - def _run_batch( - self, - context: ValidationContext, - llm_client, - batch, - ) -> List[ValidationDecision]: - payload = [item.to_prompt_payload() for item in batch.items] - messages = self.prompt_builder(payload) - prompt_path = context.run_dir / f"{self.name}-prompt-{batch.batch_index:04d}.json" - response_path = context.run_dir / f"{self.name}-response-{batch.batch_index:04d}.json" - _write_json(prompt_path, {"messages": messages}) - if context.llm_scheduler is not None: - response = context.llm_scheduler.run_backend_call( - lambda: llm_client.run_structured( - stage_name=f"{context.run_spec.instance_name}:{self.name}", - messages=messages, - response_model=_LLMValidationSetModel, - config=context.config, - ) - ) - else: - response = llm_client.run_structured( - stage_name=f"{context.run_spec.instance_name}:{self.name}", - messages=messages, - response_model=_LLMValidationSetModel, - config=context.config, - ) - _write_json(response_path, response.model_dump(mode="json")) - return self._validate_batch_response(response, batch.items) - - -@dataclass(frozen=True) -class SpokenFormPlausibilityValidator(_BaseLLMValidator): - name: str = "spoken_form_plausibility_review" - prompt_builder: PromptBuilder = build_spoken_form_plausibility_messages - - -@dataclass(frozen=True) -class MeaningReversalValidator(_BaseLLMValidator): - name: str = "meaning_reversal_review" - prompt_builder: PromptBuilder = build_meaning_reversal_messages - - -@dataclass(frozen=True) -class EditorialValidator(_BaseLLMValidator): - name: str = "editorial_review" - prompt_builder: PromptBuilder = build_editorial_messages - - -@dataclass(frozen=True) -class SpokenWordValidator(_BaseLLMValidator): - name: str = "spoken_word_review" - prompt_builder: PromptBuilder = build_editorial_messages - - -@dataclass(frozen=True) -class GrammarOnlyValidator(_BaseLLMValidator): - name: str = "grammar_only_guard" - prompt_builder: PromptBuilder = build_editorial_messages - - -def _write_json(path: Path, payload: dict) -> None: - path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") diff --git a/python/src/audita/validators/prompts.py b/python/src/audita/validators/prompts.py deleted file mode 100644 index 5f753f6..0000000 --- a/python/src/audita/validators/prompts.py +++ /dev/null @@ -1,96 +0,0 @@ -import json -from typing import Dict, List - - -Message = Dict[str, str] - - -def build_spoken_form_plausibility_messages(validation_payload: List[dict]) -> List[Message]: - payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2) - system = ( - "You are Audita, a conservative spoken-form validation assistant. " - "Evaluate whether each proposed correction is plausibly explained by a homophone, phonetic similarity, " - "or a common mistranscription of spoken English. " - "Your job is not to improve style or readability. " - "Approve only when the corrected text is a plausible recovery of the words that were likely spoken." - ) - user = ( - "Review these proposed transcript corrections and decide whether each one is a plausible spoken-form correction.\n\n" - "Rules:\n" - "- Return one validation decision for every correction_index in the input.\n" - "- Approve when the original text and corrected text are plausibly related by homophone confusion, " - "phonetic similarity, or a common spoken-word mistranscription, and the surrounding segment context supports the correction.\n" - "- Examples that may be approved when context supports them: changing \"gestures\" to \"Jesters\", " - "\"rank\" to \"Hrank\", or \"dam\" to \"damn\".\n" - "- Reject unrelated substitutions like changing \"Lyra\" to \"Jesters\".\n" - "- Judge spoken-form plausibility, not whether the correction is cleaner, more formal, or more grammatical.\n" - "- Do not approve paraphrases, stylistic rewrites, or arbitrary semantic substitutions.\n" - "- If a correction includes categories, treat them as additional segment context.\n" - "- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" - "- confidence must be between 0.0 and 1.0.\n\n" - f"Corrections to validate:\n{payload_json}" - ) - return [{"role": "system", "content": system}, {"role": "user", "content": user}] - - -def build_meaning_reversal_messages(validation_payload: List[dict]) -> List[Message]: - payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2) - system = ( - "You are Audita, a narrow semantic-reversal validation assistant. " - "Evaluate whether each proposed correction changes a word to its antonym or otherwise reverses the meaning " - "of the full segment. " - "Do not treat every word substitution as a problem; focus specifically on antonyms and meaning reversals." - ) - user = ( - "Review these proposed transcript corrections and decide whether each one avoids reversing the segment meaning.\n\n" - "Rules:\n" - "- Return one validation decision for every correction_index in the input.\n" - "- Reject corrections that introduce antonyms or otherwise reverse the meaning of the original segment.\n" - "- Reject examples like changing \"visible\" to \"invisible\" or \"up\" to \"down\" when that reverses the segment meaning.\n" - "- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.\n" - "- Do not reject a correction merely because the literal written word changes.\n" - "- Do not act as a general semantic-style reviewer; this validator is only a guard against antonyms and meaning reversals.\n" - "- If a correction includes categories, treat them as additional segment context.\n" - "- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" - "- confidence must be between 0.0 and 1.0.\n\n" - f"Corrections to validate:\n{payload_json}" - ) - return [{"role": "system", "content": system}, {"role": "user", "content": user}] - - -def build_editorial_messages(validation_payload: List[dict]) -> List[Message]: - payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2) - system = ( - "You are Audita, a conservative editorial validation assistant. " - "Evaluate whether each proposed correction is editorial in nature and preserves the segment's underlying meaning. " - "Editorial revisions may include dysfluency cleanup, punctuation changes, capitalization changes, " - "homophone or mistranscription corrections, and similar low-risk editorial cleanup." - ) - user = ( - "Review these proposed transcript corrections and decide whether each one is an acceptable editorial revision.\n\n" - "Rules:\n" - "- Return one validation decision for every correction_index in the input.\n" - "- Approve editorial revisions that preserve the underlying meaning of the segment.\n" - "- Approve dysfluency cleanup, including cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies.\n" - "- Approve punctuation, spacing, capitalization, and article cleanup when they preserve meaning.\n" - "- Approve low-risk homophone or mistranscription corrections when they are contextually well supported.\n" - "- Approve conservative combinations of these editorial changes when the overall revision remains meaning-preserving.\n" - "- Reject repetition cleanup when the repetition plausibly serves urgency, excitement, insistence, or deliberate rhetorical emphasis rather than dysfluency.\n" - "- Phrases such as \"Help! Help! Help!\", \"Stop! Stop! Stop!\", \"No! No! No!\", \"Yes! Yes! Yes!\", and \"Go! Go! Go!\" are often intentional emphasis and should usually be preserved.\n" - "- Approve repetition cleanup only when local context supports it as accidental repetition, hesitation, or verbal restart.\n" - "- Reject broad paraphrase, substantive semantic changes, substantive meaning changes, and revisions that materially change, obscure, distort, or reverse the segment's meaning.\n" - "- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.\n" - "- If a correction includes categories, treat them as additional segment context.\n" - "- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" - "- confidence must be between 0.0 and 1.0.\n\n" - f"Corrections to validate:\n{payload_json}" - ) - return [{"role": "system", "content": system}, {"role": "user", "content": user}] - - -def build_grammar_only_messages(validation_payload: List[dict]) -> List[Message]: - return build_editorial_messages(validation_payload) - - -def build_spoken_word_messages(validation_payload: List[dict]) -> List[Message]: - return build_editorial_messages(validation_payload) diff --git a/python/src/audita/validators/protection.py b/python/src/audita/validators/protection.py deleted file mode 100644 index 14491b3..0000000 --- a/python/src/audita/validators/protection.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -from dataclasses import dataclass -from typing import Dict, List, Optional, Pattern - -from audita.core.schemas import Glossary - - -@dataclass(frozen=True) -class ProtectedVocabulary: - terms_by_folded: Dict[str, "_ProtectedTermDefinition"] - pattern: Optional[Pattern[str]] - - @classmethod - def from_glossary(cls, glossary: Glossary) -> "ProtectedVocabulary": - terms_by_folded: Dict[str, _ProtectedTermDefinition] = {} - for identity, entry in enumerate(glossary.glossary): - entry_terms = [entry.name, *entry.aliases] - for term in entry_terms: - _add_term(terms_by_folded, term, identity) - _add_term(terms_by_folded, f"{term}s", identity) - if entry.plural is not None: - _add_term(terms_by_folded, entry.plural, identity) - - terms = [definition.canonical for definition in terms_by_folded.values()] - if not terms: - return cls(terms_by_folded=terms_by_folded, pattern=None) - - alternatives = sorted((re.escape(term) for term in terms), key=len, reverse=True) - pattern = re.compile(r"(? Optional[str]: - before_occurrences = self._occurrences_by_identity(before) - after_occurrences = self._occurrences_by_identity(after) - - reason = self._validate_identity_preservation(before_occurrences, after_occurrences) - if reason is not None: - return reason - return self._validate_capitalization_transitions(before_occurrences, after_occurrences) - - def glossary_stage_violation_reason(self, before: str, after: str) -> Optional[str]: - before_occurrences = self._occurrences_by_identity(before) - after_occurrences = self._occurrences_by_identity(after) - - reason = self._validate_glossary_stage_identity_preservation(before_occurrences, after_occurrences) - if reason is not None: - return reason - return self._validate_capitalization_transitions(before_occurrences, after_occurrences) - - def _occurrences(self, text: str) -> List["_ProtectedOccurrence"]: - if self.pattern is None: - return [] - occurrences = [] - for match in self.pattern.finditer(text): - matched_text = match.group(0) - definition = self.terms_by_folded[matched_text.casefold()] - occurrences.append( - _ProtectedOccurrence( - text=matched_text, - identity=definition.identity, - canonical=definition.canonical, - ) - ) - return occurrences - - def _occurrences_by_identity(self, text: str) -> Dict[int, List["_ProtectedOccurrence"]]: - occurrences_by_identity: Dict[int, List["_ProtectedOccurrence"]] = {} - for occurrence in self._occurrences(text): - occurrences_by_identity.setdefault(occurrence.identity, []).append(occurrence) - return occurrences_by_identity - - def _validate_identity_preservation( - self, - before_occurrences: Dict[int, List["_ProtectedOccurrence"]], - after_occurrences: Dict[int, List["_ProtectedOccurrence"]], - ) -> Optional[str]: - for identity, before_items in before_occurrences.items(): - if len(after_occurrences.get(identity, [])) < len(before_items): - return "correction changes protected glossary term usage" - return None - - def _validate_glossary_stage_identity_preservation( - self, - before_occurrences: Dict[int, List["_ProtectedOccurrence"]], - after_occurrences: Dict[int, List["_ProtectedOccurrence"]], - ) -> Optional[str]: - before_total = sum(len(items) for items in before_occurrences.values()) - after_total = sum(len(items) for items in after_occurrences.values()) - if after_total < before_total: - return "correction changes protected glossary term usage" - return None - - def _validate_capitalization_transitions( - self, - before_occurrences: Dict[int, List["_ProtectedOccurrence"]], - after_occurrences: Dict[int, List["_ProtectedOccurrence"]], - ) -> Optional[str]: - for identity, after_items in after_occurrences.items(): - before_items = before_occurrences.get(identity, []) - before_count = len(before_items) - for index, after_item in enumerate(after_items): - if index < before_count: - before_item = before_items[index] - if after_item.text == before_item.text: - continue - if after_item.text == after_item.canonical: - continue - return "correction changes protected glossary term capitalization" - return None - - -@dataclass(frozen=True) -class _ProtectedOccurrence: - text: str - identity: int - canonical: str - - -@dataclass(frozen=True) -class _ProtectedTermDefinition: - identity: int - canonical: str - - -def _add_term( - terms_by_folded: Dict[str, _ProtectedTermDefinition], - term: str, - identity: int, -) -> None: - stripped = term.strip() - if not stripped: - return - terms_by_folded.setdefault( - stripped.casefold(), - _ProtectedTermDefinition(identity=identity, canonical=stripped), - ) diff --git a/python/tests/__init__.py b/python/tests/__init__.py deleted file mode 100644 index 9c432b5..0000000 --- a/python/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Test package root.""" diff --git a/python/tests/test_framework_chunking.py b/python/tests/test_framework_chunking.py deleted file mode 100644 index 6935856..0000000 --- a/python/tests/test_framework_chunking.py +++ /dev/null @@ -1,206 +0,0 @@ -import pytest - -from audita.core.chunking import TokenEstimatorProtocol, chunk_transcript -from audita.core.errors import AuditaValidationError -from audita.core.schemas import parse_transcript_json - - -class FakeEstimator(TokenEstimatorProtocol): - def estimate_json(self, value): - if len(value) == 1: - return 4 - return len(value) * 4 - - -def test_chunk_transcript_batches_sections_by_token_limit(): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"}, - {"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "three"} - ] - """ - ) - - sections = chunk_transcript(transcript, max_section_tokens=8, estimator=FakeEstimator()) - - assert len(sections) == 2 - assert [segment.segment.id for segment in sections[0].segments] == [1, 2] - assert [segment.segment.id for segment in sections[1].segments] == [3] - - -def test_chunk_transcript_targets_llm_concurrency_when_feasible(): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"}, - {"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "three"}, - {"id": 4, "speaker": "A", "start": 3.0, "end": 4.0, "text": "four"} - ] - """ - ) - - sections = chunk_transcript( - transcript, - max_section_tokens=8, - min_section_tokens=4, - target_section_count=2, - estimator=FakeEstimator(), - ) - - assert len(sections) == 2 - assert [segment.segment.id for segment in sections[0].segments] == [1, 2] - assert [segment.segment.id for segment in sections[1].segments] == [3, 4] - - -def test_chunk_transcript_increases_section_count_when_target_sections_exceed_max_tokens(): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"}, - {"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "three"}, - {"id": 4, "speaker": "A", "start": 3.0, "end": 4.0, "text": "four"}, - {"id": 5, "speaker": "A", "start": 4.0, "end": 5.0, "text": "five"} - ] - """ - ) - - sections = chunk_transcript( - transcript, - max_section_tokens=8, - min_section_tokens=4, - target_section_count=1, - estimator=FakeEstimator(), - ) - - assert len(sections) > 1 - assert all(section.token_count <= 8 for section in sections) - - -def test_chunk_transcript_reduces_section_count_when_target_sections_fall_below_min_tokens(): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"}, - {"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "three"} - ] - """ - ) - - sections = chunk_transcript( - transcript, - max_section_tokens=12, - min_section_tokens=8, - target_section_count=3, - estimator=FakeEstimator(), - ) - - assert len(sections) == 1 - assert sections[0].token_count >= 8 - - -def test_chunk_transcript_exact_target_sections_returns_exact_count_when_feasible(): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"}, - {"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "three"}, - {"id": 4, "speaker": "A", "start": 3.0, "end": 4.0, "text": "four"} - ] - """ - ) - - sections = chunk_transcript( - transcript, - max_section_tokens=8, - min_section_tokens=4, - exact_target_section_count=2, - estimator=FakeEstimator(), - ) - - assert len(sections) == 2 - assert [segment.segment.id for segment in sections[0].segments] == [1, 2] - assert [segment.segment.id for segment in sections[1].segments] == [3, 4] - - -def test_chunk_transcript_exact_target_sections_errors_when_target_exceeds_segment_count(): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"} - ] - """ - ) - - with pytest.raises(AuditaValidationError, match="Target section count exceeds the number of transcript segments"): - chunk_transcript( - transcript, - max_section_tokens=8, - min_section_tokens=4, - exact_target_section_count=3, - estimator=FakeEstimator(), - ) - - -def test_chunk_transcript_exact_target_sections_errors_when_section_would_exceed_max(): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"}, - {"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "three"} - ] - """ - ) - - with pytest.raises(AuditaValidationError, match="AUDITA_TARGET_SECTIONS cannot produce contiguous transcript sections"): - chunk_transcript( - transcript, - max_section_tokens=8, - min_section_tokens=4, - exact_target_section_count=1, - estimator=FakeEstimator(), - ) - - -def test_chunk_transcript_exact_target_sections_errors_when_section_would_fall_below_min(): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"}, - {"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "three"} - ] - """ - ) - - with pytest.raises(AuditaValidationError, match="AUDITA_TARGET_SECTIONS cannot produce contiguous transcript sections"): - chunk_transcript( - transcript, - max_section_tokens=12, - min_section_tokens=8, - exact_target_section_count=3, - estimator=FakeEstimator(), - ) - - -def test_chunk_transcript_prompt_payload_includes_categories_when_present(): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one", "categories": ["intro", "aside"]} - ] - """ - ) - - sections = chunk_transcript(transcript, max_section_tokens=8, estimator=FakeEstimator()) - - assert sections[0].prompt_payload() == [ - {"id": 1, "original_text": "one", "categories": ["intro", "aside"]} - ] diff --git a/python/tests/test_framework_llm.py b/python/tests/test_framework_llm.py deleted file mode 100644 index 80f2402..0000000 --- a/python/tests/test_framework_llm.py +++ /dev/null @@ -1,255 +0,0 @@ -from concurrent.futures import ThreadPoolExecutor -import sys -import types - -import pytest - -from audita.core.config import AuditaConfig -from audita.core.errors import AuditaLLMError -from audita.framework.llm import OpenAICompatibleStructuredLLMClient - - -class DummyResponseModel: - pass - - -def _config(**overrides): - base = AuditaConfig.from_sources(env={}) - data = { - "api_key": "test-key", - "llm_concurrency": base.llm_concurrency, - "llm_timeout_seconds": base.llm_timeout_seconds, - "validation_llm_api_key": base.validation_llm_api_key, - "validation_llm_concurrency": base.validation_llm_concurrency, - "validation_llm_timeout_seconds": base.validation_llm_timeout_seconds, - "validation_model": base.validation_model, - "validation_base_url": base.validation_base_url, - "validation_max_retries": base.validation_max_retries, - "module_keys": base.module_keys, - "model": base.model, - "base_url": base.base_url, - "max_retries": base.max_retries, - "max_section_tokens": base.max_section_tokens, - "glossary_confidence_threshold": base.glossary_confidence_threshold, - "grammar_confidence_threshold": base.grammar_confidence_threshold, - "homophones_confidence_threshold": base.homophones_confidence_threshold, - "spoken_word_confidence_threshold": base.spoken_word_confidence_threshold, - "normalize_max_segment_gap": base.normalize_max_segment_gap, - "normalize_ellipsis_gap": base.normalize_ellipsis_gap, - "normalize_max_segment_duration": base.normalize_max_segment_duration, - "normalize_max_segment_tokens": base.normalize_max_segment_tokens, - "work_dir": base.work_dir, - "work_dir_retention": base.work_dir_retention, - } - data.update(overrides) - return AuditaConfig(**data) - - -def _install_fake_llm_modules(monkeypatch): - create_calls = [] - openai_inits = [] - - class FakePatchedClient: - def __init__(self): - self.chat = types.SimpleNamespace(completions=types.SimpleNamespace(create=self._create)) - - def _create(self, **kwargs): - create_calls.append(kwargs) - return {"ok": True} - - class FakeOpenAI: - def __init__(self, *, api_key, base_url, timeout): - openai_inits.append({"api_key": api_key, "base_url": base_url, "timeout": timeout}) - - fake_instructor = types.SimpleNamespace( - Mode=types.SimpleNamespace(TOOLS="TOOLS"), - patch=lambda client, mode: FakePatchedClient(), - ) - fake_openai = types.SimpleNamespace(OpenAI=FakeOpenAI) - monkeypatch.setitem(sys.modules, "instructor", fake_instructor) - monkeypatch.setitem(sys.modules, "openai", fake_openai) - return create_calls, openai_inits - - -def test_openrouter_requests_strip_prefix_and_include_extra_body(monkeypatch): - create_calls, _ = _install_fake_llm_modules(monkeypatch) - client = OpenAICompatibleStructuredLLMClient() - - client.run_structured( - stage_name="test-stage", - messages=[{"role": "user", "content": "Hello"}], - response_model=DummyResponseModel, - config=_config(model="openrouter/google/gemma-4-31b-it"), - ) - - assert create_calls == [ - { - "model": "google/gemma-4-31b-it", - "messages": [{"role": "user", "content": "Hello"}], - "response_model": DummyResponseModel, - "max_retries": 3, - "extra_body": {"provider": {"require_parameters": True}}, - } - ] - - -def test_openrouter_default_base_url_uses_openrouter_request_shape(monkeypatch): - create_calls, _ = _install_fake_llm_modules(monkeypatch) - client = OpenAICompatibleStructuredLLMClient() - - client.run_structured( - stage_name="test-stage", - messages=[{"role": "user", "content": "Hello"}], - response_model=DummyResponseModel, - config=_config(model="google/gemma-4-31b-it"), - ) - - assert create_calls == [ - { - "model": "google/gemma-4-31b-it", - "messages": [{"role": "user", "content": "Hello"}], - "response_model": DummyResponseModel, - "max_retries": 3, - "extra_body": {"provider": {"require_parameters": True}}, - } - ] - - -def test_generic_endpoint_requests_keep_model_and_omit_extra_body(monkeypatch): - create_calls, _ = _install_fake_llm_modules(monkeypatch) - client = OpenAICompatibleStructuredLLMClient() - - client.run_structured( - stage_name="test-stage", - messages=[{"role": "user", "content": "Hello"}], - response_model=DummyResponseModel, - config=_config( - model="meta-llama/Llama-3.1-8B-Instruct", - base_url="http://localhost:8000/v1", - ), - ) - - assert create_calls == [ - { - "model": "meta-llama/Llama-3.1-8B-Instruct", - "messages": [{"role": "user", "content": "Hello"}], - "response_model": DummyResponseModel, - "max_retries": 3, - } - ] - - -def test_client_cache_identity_uses_api_key_and_base_url(monkeypatch): - _, openai_inits = _install_fake_llm_modules(monkeypatch) - client = OpenAICompatibleStructuredLLMClient() - first = _config(api_key="key-1", base_url="http://localhost:8000/v1") - second = _config(api_key="key-1", base_url="http://localhost:8000/v1") - third = _config(api_key="key-1", base_url="https://api.openai.com/v1") - - client.run_structured( - stage_name="one", - messages=[{"role": "user", "content": "Hello"}], - response_model=DummyResponseModel, - config=first, - ) - client.run_structured( - stage_name="two", - messages=[{"role": "user", "content": "Hello"}], - response_model=DummyResponseModel, - config=second, - ) - client.run_structured( - stage_name="three", - messages=[{"role": "user", "content": "Hello"}], - response_model=DummyResponseModel, - config=third, - ) - - assert openai_inits == [ - {"api_key": "key-1", "base_url": "http://localhost:8000/v1", "timeout": 600}, - {"api_key": "key-1", "base_url": "https://api.openai.com/v1", "timeout": 600}, - ] - - -def test_client_cache_identity_uses_timeout(monkeypatch): - _, openai_inits = _install_fake_llm_modules(monkeypatch) - client = OpenAICompatibleStructuredLLMClient() - first = _config(api_key="key-1", base_url="http://localhost:8000/v1", llm_timeout_seconds=600) - second = _config(api_key="key-1", base_url="http://localhost:8000/v1", llm_timeout_seconds=1200) - - client.run_structured( - stage_name="one", - messages=[{"role": "user", "content": "Hello"}], - response_model=DummyResponseModel, - config=first, - ) - client.run_structured( - stage_name="two", - messages=[{"role": "user", "content": "Hello"}], - response_model=DummyResponseModel, - config=second, - ) - - assert openai_inits == [ - {"api_key": "key-1", "base_url": "http://localhost:8000/v1", "timeout": 600}, - {"api_key": "key-1", "base_url": "http://localhost:8000/v1", "timeout": 1200}, - ] - - -def test_missing_api_key_error_is_provider_neutral(): - client = OpenAICompatibleStructuredLLMClient() - - with pytest.raises(AuditaLLMError, match="OpenRouter endpoint"): - client.run_structured( - stage_name="test-stage", - messages=[{"role": "user", "content": "Hello"}], - response_model=DummyResponseModel, - config=_config(api_key=None), - ) - - -def test_missing_api_key_is_allowed_for_nondefault_endpoint(monkeypatch): - create_calls, openai_inits = _install_fake_llm_modules(monkeypatch) - client = OpenAICompatibleStructuredLLMClient() - - client.run_structured( - stage_name="test-stage", - messages=[{"role": "user", "content": "Hello"}], - response_model=DummyResponseModel, - config=_config( - api_key=None, - model="meta-llama/Llama-3.1-8B-Instruct", - base_url="http://localhost:8000/v1", - ), - ) - - assert openai_inits == [{"api_key": "audita-no-key-required", "base_url": "http://localhost:8000/v1", "timeout": 600}] - assert create_calls == [ - { - "model": "meta-llama/Llama-3.1-8B-Instruct", - "messages": [{"role": "user", "content": "Hello"}], - "response_model": DummyResponseModel, - "max_retries": 3, - } - ] - - -def test_client_initialization_is_safe_under_concurrent_calls(monkeypatch): - _, openai_inits = _install_fake_llm_modules(monkeypatch) - client = OpenAICompatibleStructuredLLMClient() - config = _config(api_key="key-1", base_url="http://localhost:8000/v1") - - with ThreadPoolExecutor(max_workers=4) as executor: - list( - executor.map( - lambda _: client.run_structured( - stage_name="test-stage", - messages=[{"role": "user", "content": "Hello"}], - response_model=DummyResponseModel, - config=config, - ), - range(4), - ) - ) - - assert openai_inits == [{"api_key": "key-1", "base_url": "http://localhost:8000/v1", "timeout": 600}] diff --git a/python/tests/test_framework_runner.py b/python/tests/test_framework_runner.py deleted file mode 100644 index 0531b28..0000000 --- a/python/tests/test_framework_runner.py +++ /dev/null @@ -1,642 +0,0 @@ -import threading - -from audita.core.chunking import IndexedSegment, TranscriptSection -from audita.core.config import AuditaConfig, ConfigOverrides -from audita.core.errors import AuditaLLMError -from audita.core.schemas import parse_glossary_yaml, parse_transcript_json -from audita.framework.models import CorrectionProposal, ModuleContext, ModuleRunSpec -from audita.framework.runner import PipelineRunner -from audita.validators import ( - MeaningReversalValidator, - ProposalConfidenceValidator, - ProtectedGlossaryTermsValidator, - SpokenFormPlausibilityValidator, -) -from audita.validators.base import ValidationContext, ValidationDecision, ValidationResult - - -class RecordingValidator: - execution_kind = "deterministic" - - def __init__(self, name, recorder, approve=True): - self.name = name - self._recorder = recorder - self._approve = approve - - def validate(self, context: ValidationContext) -> ValidationResult: - self._recorder.append((self.name, [proposal.corrected_text for proposal in context.proposals])) - return ValidationResult( - validator_name=self.name, - execution_kind=self.execution_kind, - decisions=[ - ValidationDecision( - proposal_index=proposal.proposal_index, - approved=self._approve, - reason=None if self._approve else f"{self.name} rejected proposal", - ) - for proposal in context.proposals - ], - ) - - -class RecordingLLMValidator(RecordingValidator): - execution_kind = "llm" - - -class FakeStructuredLLMClient: - def __init__(self, responses): - self._responses = responses - self._lock = threading.Lock() - self.calls = [] - - def run_structured(self, *, stage_name, messages, response_model, config): - with self._lock: - self.calls.append( - { - "stage_name": stage_name, - "messages": list(messages), - "response_model": response_model, - } - ) - payload = _pop_llm_response(self._responses, stage_name) - return response_model.model_validate(payload) - - -def _pop_llm_response(responses, stage_name): - if isinstance(responses, dict): - if stage_name not in responses: - raise AuditaLLMError(f"FakeStructuredLLMClient received unexpected stage_name: {stage_name}") - payloads = responses[stage_name] - if isinstance(payloads, list): - if not payloads: - raise AuditaLLMError(f"FakeStructuredLLMClient received too many calls for stage_name: {stage_name}") - return payloads.pop(0) - payload = payloads - del responses[stage_name] - return payload - if not responses: - raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.") - return responses.pop(0) - - -class TrackingStructuredLLMClient: - def __init__(self, responses, barrier=None): - self._responses = responses - self._barrier = barrier - self._lock = threading.Lock() - self.calls = [] - self.in_flight = 0 - self.max_in_flight = 0 - - def run_structured(self, *, stage_name, messages, response_model, config): - if self._barrier is not None: - self._barrier.wait() - with self._lock: - self.in_flight += 1 - self.max_in_flight = max(self.max_in_flight, self.in_flight) - self.calls.append( - { - "stage_name": stage_name, - "messages": list(messages), - "response_model": response_model, - } - ) - payload = _pop_llm_response(self._responses, stage_name) - try: - return response_model.model_validate(payload) - finally: - with self._lock: - self.in_flight -= 1 - - -class RecordingModule: - replacement_policy = "require_unique" - - def __init__(self, module_key, proposals, validators, recorder): - self.module_key = module_key - self._proposals = proposals - self._validators = validators - self._recorder = recorder - - def validators(self): - return list(self._validators) - - def propose(self, transcript_section, context: ModuleContext): - self._recorder.append(("propose", [item.segment.text for item in transcript_section.segments])) - return list(self._proposals) - - -class ConcurrentRecordingModule: - replacement_policy = "require_unique" - - def __init__(self, recorder, barrier): - self.module_key = "concurrent" - self._recorder = recorder - self._barrier = barrier - - def validators(self): - return [] - - def propose(self, transcript_section, context: ModuleContext): - texts = [item.segment.text for item in transcript_section.segments] - self._recorder.append(("start", transcript_section.section_index, texts)) - self._barrier.wait() - segment = transcript_section.segments[0].segment - return [ - CorrectionProposal( - proposal_index=0, - module_instance=context.run_spec.instance_name, - module_key=context.run_spec.module_key, - id=segment.id, - original_text=segment.text.rstrip("."), - corrected_text=f"{segment.text.rstrip('.')} revised", - confidence=0.9, - ) - ] - - -def test_pipeline_runner_applies_modules_sequentially(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Alpha."} - ] - """ - ) - glossary = parse_glossary_yaml( - """ - glossary: - - name: "Alpha" - category: noun - summary: "Alpha." - """ - ) - seen = [] - first = RecordingModule( - "first", - [ - CorrectionProposal( - proposal_index=0, - module_instance="first", - module_key="first", - id=1, - original_text="Alpha", - corrected_text="Beta", - confidence=0.9, - ) - ], - [RecordingValidator("first_validator", seen)], - seen, - ) - second = RecordingModule( - "second", - [ - CorrectionProposal( - proposal_index=0, - module_instance="second", - module_key="second", - id=1, - original_text="Beta", - corrected_text="Gamma", - confidence=0.9, - ) - ], - [RecordingValidator("second_validator", seen)], - seen, - ) - - runner = PipelineRunner() - result = runner.run( - transcript=transcript, - glossary=glossary, - module_specs=[ - ModuleRunSpec(instance_name="first", module_key="first", module=first), - ModuleRunSpec(instance_name="second", module_key="second", module=second), - ], - config=AuditaConfig.from_sources(env={}), - run_dir=tmp_path / "run", - ) - - assert seen[0] == ("propose", ["Alpha."]) - assert seen[1] == ("first_validator", ["Beta"]) - assert seen[2] == ("propose", ["Beta."]) - assert seen[3] == ("second_validator", ["Gamma"]) - assert result.transcript[0].text == "Gamma." - assert len(result.applied_changes) == 2 - - -def test_pipeline_runner_validator_order_respects_survivors(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Hello."} - ] - """ - ) - glossary = parse_glossary_yaml( - """ - glossary: - - name: "Hello" - category: noun - summary: "Hello." - """ - ) - seen = [] - module = RecordingModule( - "mod", - [ - CorrectionProposal( - proposal_index=0, - module_instance="mod", - module_key="mod", - id=1, - original_text="Hello", - corrected_text="Goodbye", - confidence=0.9, - ) - ], - [ - RecordingValidator("first", seen, approve=False), - RecordingLLMValidator("second", seen, approve=True), - ], - seen, - ) - - runner = PipelineRunner() - result = runner.run( - transcript=transcript, - glossary=glossary, - module_specs=[ModuleRunSpec(instance_name="mod", module_key="mod", module=module)], - config=AuditaConfig.from_sources(env={}), - run_dir=tmp_path / "run", - ) - - assert ("first", ["Goodbye"]) in seen - assert all(entry[0] != "second" for entry in seen) - assert result.module_reports[0].validators[0].rejected_count == 1 - assert result.module_reports[0].validators[1].candidate_count == 0 - assert result.skipped_corrections[0].source == "validator:first" - - -def test_pipeline_runner_supports_deterministic_and_llm_validators_in_one_chain(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Alpha."} - ] - """ - ) - glossary = parse_glossary_yaml( - """ - glossary: - - name: "Alpha" - category: noun - summary: "Alpha." - """ - ) - seen = [] - module = RecordingModule( - "mixed", - [ - CorrectionProposal( - proposal_index=0, - module_instance="mixed", - module_key="mixed", - id=1, - original_text="Alpha", - corrected_text="Beta", - confidence=0.9, - ) - ], - [ - RecordingValidator("deterministic_guard", seen), - RecordingLLMValidator("llm_review", seen), - ], - seen, - ) - - runner = PipelineRunner() - result = runner.run( - transcript=transcript, - glossary=glossary, - module_specs=[ModuleRunSpec(instance_name="mixed", module_key="mixed", module=module)], - config=AuditaConfig.from_sources(env={}), - run_dir=tmp_path / "run", - ) - - assert result.transcript[0].text == "Beta." - assert [report.execution_kind for report in result.module_reports[0].validators] == [ - "deterministic", - "llm", - ] - - -def test_pipeline_runner_supports_real_llm_validators_in_one_chain(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the dam."} - ] - """ - ) - glossary = parse_glossary_yaml( - """ - glossary: - - name: "Jesters" - category: faction - summary: "The Jesters are a faction." - """ - ) - module = RecordingModule( - "mixed_real", - [ - CorrectionProposal( - proposal_index=0, - module_instance="mixed_real", - module_key="glossary", - id=1, - original_text="gestures", - corrected_text="Jesters", - confidence=0.9, - ) - ], - [ - ProposalConfidenceValidator("proposal_confidence_guard", "glossary_confidence_threshold"), - ProtectedGlossaryTermsValidator("protected_glossary_guard"), - SpokenFormPlausibilityValidator("spoken_form_plausibility_review"), - MeaningReversalValidator("meaning_reversal_review"), - ], - [], - ) - llm_client = FakeStructuredLLMClient( - { - "mixed_real:spoken_form_plausibility_review": { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.95, - "reason": "Likely phonetic mistranscription in context.", - } - ] - }, - "mixed_real:meaning_reversal_review": { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.98, - "reason": "Does not reverse the segment meaning.", - } - ] - }, - } - ) - - runner = PipelineRunner() - result = runner.run( - transcript=transcript, - glossary=glossary, - module_specs=[ModuleRunSpec(instance_name="mixed_real", module_key="glossary", module=module)], - config=AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}), - run_dir=tmp_path / "run", - llm_client=llm_client, - ) - - assert result.transcript[0].text == "There were Jesters at the dam." - assert [report.execution_kind for report in result.module_reports[0].validators] == [ - "deterministic", - "deterministic", - "llm", - "llm", - ] - assert {call["stage_name"] for call in llm_client.calls} == { - "mixed_real:spoken_form_plausibility_review", - "mixed_real:meaning_reversal_review", - } - - -def test_pipeline_runner_uses_real_protected_glossary_validator(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Hrank moves."} - ] - """ - ) - glossary = parse_glossary_yaml( - """ - glossary: - - name: "Hrank" - category: pc - summary: "Hrank is a player character." - """ - ) - module = RecordingModule( - "protected", - [ - CorrectionProposal( - proposal_index=0, - module_instance="protected", - module_key="protected", - id=1, - original_text="Hrank", - corrected_text="Frank", - confidence=0.9, - ) - ], - [ProtectedGlossaryTermsValidator("protected_glossary_guard")], - [], - ) - - runner = PipelineRunner() - result = runner.run( - transcript=transcript, - glossary=glossary, - module_specs=[ModuleRunSpec(instance_name="protected", module_key="protected", module=module)], - config=AuditaConfig.from_sources(env={}), - run_dir=tmp_path / "run", - ) - - assert result.transcript[0].text == "Hrank moves." - assert result.skipped_corrections[0].source == "validator:protected_glossary_guard" - assert result.skipped_corrections[0].reason == "correction changes protected glossary term usage" - - -def test_pipeline_runner_collects_section_proposals_concurrently_and_preserves_section_order(tmp_path, monkeypatch): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Alpha."}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "Beta."} - ] - """ - ) - glossary = parse_glossary_yaml( - """ - glossary: - - name: "Alpha" - category: noun - summary: "Alpha." - """ - ) - sections = [ - TranscriptSection( - section_index=0, - start_index=0, - segments=[IndexedSegment(index=0, segment=transcript[0])], - token_count=1, - ), - TranscriptSection( - section_index=1, - start_index=1, - segments=[IndexedSegment(index=1, segment=transcript[1])], - token_count=1, - ), - ] - seen = [] - module = ConcurrentRecordingModule(seen, threading.Barrier(2, timeout=1.0)) - monkeypatch.setattr( - "audita.framework.runner.chunk_transcript", - lambda working, max_tokens, min_section_tokens=1, target_section_count=None, exact_target_section_count=None: sections, - ) - - runner = PipelineRunner() - result = runner.run( - transcript=transcript, - glossary=glossary, - module_specs=[ModuleRunSpec(instance_name="concurrent", module_key="concurrent", module=module)], - config=AuditaConfig.from_sources(env={}, overrides=ConfigOverrides(llm_concurrency=2)), - run_dir=tmp_path / "run", - ) - - assert [change.id for change in result.applied_changes] == [1, 2] - assert result.applied_changes[0].corrected_text == "Alpha revised" - assert result.applied_changes[1].corrected_text == "Beta revised" - assert result.transcript[0].text == "Alpha revised." - assert result.transcript[1].text == "Beta revised." - - -def test_pipeline_runner_passes_exact_target_sections_to_chunker(tmp_path, monkeypatch): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Alpha."} - ] - """ - ) - glossary = parse_glossary_yaml( - """ - glossary: - - name: "Alpha" - category: noun - summary: "Alpha." - """ - ) - seen_args = {} - sections = [ - TranscriptSection( - section_index=0, - start_index=0, - segments=[IndexedSegment(index=0, segment=transcript[0])], - token_count=1, - ) - ] - module = RecordingModule("noop", [], [], []) - - def _fake_chunk_transcript(working, max_tokens, min_section_tokens=1, target_section_count=None, exact_target_section_count=None): - seen_args["target_section_count"] = target_section_count - seen_args["exact_target_section_count"] = exact_target_section_count - return sections - - monkeypatch.setattr("audita.framework.runner.chunk_transcript", _fake_chunk_transcript) - - runner = PipelineRunner() - runner.run( - transcript=transcript, - glossary=glossary, - module_specs=[ModuleRunSpec(instance_name="noop", module_key="noop", module=module)], - config=AuditaConfig.from_sources(env={}, overrides=ConfigOverrides(target_sections=3)), - run_dir=tmp_path / "run", - ) - - assert seen_args == { - "target_section_count": None, - "exact_target_section_count": 3, - } - - -def test_pipeline_runner_reports_first_llm_validator_rejection_in_chain_order(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the dam."} - ] - """ - ) - glossary = parse_glossary_yaml( - """ - glossary: - - name: "Jesters" - category: faction - summary: "The Jesters are a faction." - """ - ) - module = RecordingModule( - "ordered_real", - [ - CorrectionProposal( - proposal_index=0, - module_instance="ordered_real", - module_key="glossary", - id=1, - original_text="gestures", - corrected_text="Jesters", - confidence=0.9, - ) - ], - [ - ProposalConfidenceValidator("proposal_confidence_guard", "glossary_confidence_threshold"), - ProtectedGlossaryTermsValidator("protected_glossary_guard"), - SpokenFormPlausibilityValidator("spoken_form_plausibility_review"), - MeaningReversalValidator("meaning_reversal_review"), - ], - [], - ) - llm_client = FakeStructuredLLMClient( - { - "ordered_real:spoken_form_plausibility_review": { - "validations": [ - { - "correction_index": 0, - "approved": False, - "confidence": 0.95, - "reason": "Not plausibly supported by spoken-form context.", - } - ] - }, - "ordered_real:meaning_reversal_review": { - "validations": [ - { - "correction_index": 0, - "approved": False, - "confidence": 0.98, - "reason": "Changes meaning too much.", - } - ] - }, - } - ) - - runner = PipelineRunner() - result = runner.run( - transcript=transcript, - glossary=glossary, - module_specs=[ModuleRunSpec(instance_name="ordered_real", module_key="glossary", module=module)], - config=AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}), - run_dir=tmp_path / "run", - llm_client=llm_client, - ) - - assert result.skipped_corrections[0].source == "validator:spoken_form_plausibility_review" - assert result.skipped_corrections[0].reason == "Not plausibly supported by spoken-form context." diff --git a/python/tests/test_llm_validators.py b/python/tests/test_llm_validators.py deleted file mode 100644 index 557fbcc..0000000 --- a/python/tests/test_llm_validators.py +++ /dev/null @@ -1,1441 +0,0 @@ -import threading - -import pytest - -from audita.core.chunking import TokenBatch -from audita.core.config import AuditaConfig, ConfigOverrides -from audita.core.errors import AuditaLLMError -from audita.core.schemas import parse_glossary_yaml, parse_transcript_json -from audita.framework.models import CorrectionProposal, ModuleRunSpec -from audita.validators import ( - EditorialValidator, - GrammarOnlyValidator, - IdenticalTextValidator, - NonEmptySegmentValidator, - OriginalTextPresentValidator, -) -from audita.validators.base import ValidationContext -from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator -from audita.validators.prompts import ( - build_grammar_only_messages, - build_meaning_reversal_messages, - build_spoken_form_plausibility_messages, - build_spoken_word_messages, -) -import audita.validators.llm as llm_module - - -class _Module: - def __init__(self, replacement_policy: str) -> None: - self.replacement_policy = replacement_policy - - -class FakeStructuredLLMClient: - def __init__(self, responses): - self._responses = list(responses) - self.calls = [] - - def run_structured(self, *, stage_name, messages, response_model, config): - self.calls.append( - { - "stage_name": stage_name, - "messages": list(messages), - "response_model": response_model, - "config": config, - } - ) - if not self._responses: - raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.") - return response_model.model_validate(self._responses.pop(0)) - - -class CoordinatedStructuredLLMClient: - def __init__(self, responses, barrier): - self._responses = list(responses) - self._barrier = barrier - self._lock = threading.Lock() - self.calls = [] - - def run_structured(self, *, stage_name, messages, response_model, config): - self._barrier.wait() - with self._lock: - self.calls.append( - { - "stage_name": stage_name, - "messages": list(messages), - "response_model": response_model, - "config": config, - } - ) - if not self._responses: - raise AuditaLLMError("CoordinatedStructuredLLMClient received more calls than expected.") - payload = self._responses.pop(0) - if callable(payload): - payload = payload(stage_name=stage_name, messages=messages, response_model=response_model, config=config) - return response_model.model_validate(payload) - - -def _glossary(): - return parse_glossary_yaml( - """ - glossary: - - name: "Jesters" - category: faction - summary: "The Jesters are a faction." - - name: "Hrank" - category: pc - summary: "Hrank is a player character." - - name: "Lyra" - category: npc - summary: "Lyra is an NPC." - """ - ) - - -def _context( - *, - proposals, - transcript, - llm_client, - tmp_path, - replacement_policy="require_unique", - llm_concurrency=1, - validation_llm_concurrency=None, -): - config = AuditaConfig.from_sources( - env={"OPENROUTER_API_KEY": "test-key"}, - overrides=ConfigOverrides( - llm_concurrency=llm_concurrency, - validation_llm_concurrency=validation_llm_concurrency, - ), - ) - return ValidationContext( - proposals=proposals, - transcript=transcript, - glossary=_glossary(), - config=config.validation_llm_config(), - run_spec=ModuleRunSpec( - instance_name="homophones", - module_key="homophones", - module=_Module(replacement_policy), - ), - run_dir=tmp_path, - llm_client=llm_client, - ) - - -def test_spoken_form_plausibility_validator_approves_plausible_and_rejects_implausible_corrections(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple.", "categories": ["narration"]}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "Lyra moved first.", "categories": ["combat"]} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="homophones", - module_key="homophones", - id=1, - original_text="gestures", - corrected_text="Jesters", - confidence=0.95, - ), - CorrectionProposal( - proposal_index=1, - module_instance="homophones", - module_key="homophones", - id=2, - original_text="Lyra", - corrected_text="Jesters", - confidence=0.95, - ), - ] - client = FakeStructuredLLMClient( - [ - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.97, - "reason": "Likely spoken-form correction in context.", - }, - { - "correction_index": 1, - "approved": False, - "confidence": 0.99, - "reason": "Not plausibly related by homophone or mistranscription.", - }, - ] - } - ] - ) - - result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate( - _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) - ) - - assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [ - (0, True), - (1, False), - ] - prompt_text = client.calls[0]["messages"][1]["content"] - assert '"original_segment_text"' in prompt_text - assert '"categories"' in prompt_text - assert "There were Jesters at the temple." in prompt_text - assert "Lyra moved first." in prompt_text - - -def test_meaning_reversal_validator_rejects_reversal_and_approves_nonreversal(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam, but Claude actually can."}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "The figure became visible in the doorway."} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="homophones", - module_key="homophones", - id=1, - original_text="dam", - corrected_text="damn", - confidence=0.95, - ), - CorrectionProposal( - proposal_index=1, - module_instance="homophones", - module_key="homophones", - id=2, - original_text="visible", - corrected_text="invisible", - confidence=0.95, - ), - ] - client = FakeStructuredLLMClient( - [ - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.94, - "reason": "Does not reverse the segment meaning.", - }, - { - "correction_index": 1, - "approved": False, - "confidence": 1.0, - "reason": "Changes visible to invisible and reverses the segment meaning.", - }, - ] - } - ] - ) - - result = MeaningReversalValidator("meaning_reversal_review").validate( - _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) - ) - - assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [ - (0, True), - (1, False), - ] - prompt_text = client.calls[0]["messages"][1]["content"] - assert "The figure became visible in the doorway." in prompt_text - assert "The figure became invisible in the doorway." in prompt_text - - -def test_spoken_word_validator_approves_cleanup_and_rejects_rewrite(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "We should maybe proceed carefully."} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="spoken_word", - module_key="spoken_word", - id=1, - original_text="I, uh, I think", - corrected_text="I think", - confidence=0.95, - ), - CorrectionProposal( - proposal_index=1, - module_instance="spoken_word", - module_key="spoken_word", - id=2, - original_text="maybe proceed carefully", - corrected_text="go now", - confidence=0.95, - ), - ] - client = FakeStructuredLLMClient( - [ - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.97, - "reason": "Reasonable dysfluency cleanup that preserves meaning.", - }, - { - "correction_index": 1, - "approved": False, - "confidence": 0.99, - "reason": "This changes the substance of the segment rather than cleaning a dysfluency.", - }, - ] - } - ] - ) - - result = SpokenWordValidator("spoken_word_review").validate( - _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) - ) - - assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [ - (0, True), - (1, False), - ] - prompt_text = client.calls[0]["messages"][1]["content"] - assert "I think we should go." in prompt_text - assert "go now" in prompt_text - assert "acceptable editorial revision" in prompt_text - assert "homophone or mistranscription corrections" in prompt_text - - -def test_spoken_word_validator_allows_punctuation_cleanup_tied_to_dysfluency(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Well ... I think we should go."} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="spoken_word", - module_key="spoken_word", - id=1, - original_text="Well ... ", - corrected_text="", - confidence=0.95, - ) - ] - client = FakeStructuredLLMClient( - [ - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.95, - "reason": "Removes a hesitation artifact without changing substantive meaning.", - } - ] - } - ] - ) - - result = SpokenWordValidator("spoken_word_review").validate( - _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) - ) - - assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)] - - -def test_grammar_only_validator_approves_editorial_cleanup(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello there"}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "cant we go"}, - {"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "Hello,world"} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="grammar", - module_key="grammar", - id=1, - original_text="hello there", - corrected_text="Hello there.", - confidence=0.95, - ), - CorrectionProposal( - proposal_index=1, - module_instance="grammar", - module_key="grammar", - id=2, - original_text="cant", - corrected_text="can't", - confidence=0.95, - ), - CorrectionProposal( - proposal_index=2, - module_instance="grammar", - module_key="grammar", - id=3, - original_text="Hello,world", - corrected_text="Hello, world", - confidence=0.95, - ), - ] - client = FakeStructuredLLMClient( - [ - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.98, - "reason": "Conservative punctuation and capitalization cleanup.", - }, - { - "correction_index": 1, - "approved": True, - "confidence": 0.95, - "reason": "Conservative apostrophe insertion within grammar cleanup.", - }, - { - "correction_index": 2, - "approved": True, - "confidence": 0.97, - "reason": "Conservative spacing cleanup.", - }, - ] - } - ] - ) - - result = GrammarOnlyValidator("grammar_only_guard").validate( - _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) - ) - - assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [ - (0, True), - (1, True), - (2, True), - ] - prompt_text = client.calls[0]["messages"][1]["content"] - assert "acceptable editorial revision" in prompt_text - assert "article cleanup" in prompt_text - assert "dysfluency cleanup" in prompt_text - - -def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "a intelligence saving throw"}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "an owl"} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="grammar", - module_key="grammar", - id=1, - original_text="a intelligence saving throw", - corrected_text="an intelligence saving throw", - confidence=0.95, - ), - CorrectionProposal( - proposal_index=1, - module_instance="grammar", - module_key="grammar", - id=2, - original_text="an owl", - corrected_text="a owl", - confidence=0.95, - ), - ] - client = FakeStructuredLLMClient( - [ - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.96, - "reason": "Allowed whole-word article cleanup.", - }, - { - "correction_index": 1, - "approved": True, - "confidence": 0.93, - "reason": "Allowed whole-word article cleanup.", - }, - ] - } - ] - ) - - result = GrammarOnlyValidator("grammar_only_guard").validate( - _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) - ) - - assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [ - (0, True), - (1, True), - ] - - -def test_grammar_only_validator_allows_editorial_overlap_and_rejects_meaning_change(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "their plan"}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "dam"}, - {"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "uh"}, - {"id": 4, "speaker": "A", "start": 3.0, "end": 4.0, "text": "I I agree"}, - {"id": 5, "speaker": "A", "start": 4.0, "end": 5.0, "text": "Eric"} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="grammar", - module_key="grammar", - id=1, - original_text="their", - corrected_text="there", - confidence=0.95, - ), - CorrectionProposal( - proposal_index=1, - module_instance="grammar", - module_key="grammar", - id=2, - original_text="dam", - corrected_text="damn", - confidence=0.95, - ), - CorrectionProposal( - proposal_index=2, - module_instance="grammar", - module_key="grammar", - id=3, - original_text="uh", - corrected_text="", - confidence=0.95, - ), - CorrectionProposal( - proposal_index=3, - module_instance="grammar", - module_key="grammar", - id=4, - original_text="I I", - corrected_text="I", - confidence=0.95, - ), - CorrectionProposal( - proposal_index=4, - module_instance="grammar", - module_key="grammar", - id=5, - original_text="Eric", - corrected_text="Eric's", - confidence=0.95, - ), - ] - client = FakeStructuredLLMClient( - [ - { - "validations": [ - { - "correction_index": 0, - "approved": False, - "confidence": 0.99, - "reason": "This changes meaning rather than preserving it through editorial cleanup.", - }, - { - "correction_index": 1, - "approved": True, - "confidence": 0.93, - "reason": "Low-risk editorial homophone correction that preserves meaning.", - }, - { - "correction_index": 2, - "approved": True, - "confidence": 0.95, - "reason": "Editorial dysfluency cleanup that preserves meaning.", - }, - { - "correction_index": 3, - "approved": True, - "confidence": 0.95, - "reason": "Editorial repetition cleanup that preserves meaning.", - }, - { - "correction_index": 4, - "approved": True, - "confidence": 0.91, - "reason": "Conservative editorial possessive correction that preserves meaning.", - }, - ] - } - ] - ) - - result = GrammarOnlyValidator("grammar_only_guard").validate( - _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) - ) - - assert [decision.approved for decision in result.decisions] == [False, True, True, True, True] - - -def test_grammar_only_validator_allows_embedded_homophone_fix_within_grammar_revision(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't"} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="grammar", - module_key="grammar", - id=1, - original_text="question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't", - corrected_text="question: If he dies, does he stay there? The way that, yeah, the way that it's written, it's like, so if it stops, then he goes down; but what if it doesn't?", - confidence=0.95, - ) - ] - client = FakeStructuredLLMClient( - [ - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.94, - "reason": "Primarily a grammatical revision with an embedded likely mistranscription recovery.", - } - ] - } - ] - ) - - result = GrammarOnlyValidator("grammar_only_guard").validate( - _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) - ) - - assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)] - - -def test_non_empty_segment_validator_rejects_empty_and_whitespace_only_segments(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "uh"}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "um"} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="spoken_word", - module_key="spoken_word", - id=1, - original_text="uh", - corrected_text="", - confidence=0.95, - ), - CorrectionProposal( - proposal_index=1, - module_instance="spoken_word", - module_key="spoken_word", - id=2, - original_text="um", - corrected_text=" ", - confidence=0.95, - ), - ] - - result = NonEmptySegmentValidator("non_empty_segment_guard").validate( - _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path) - ) - - assert [decision.approved for decision in result.decisions] == [False, False] - assert all(decision.reason == "correction would leave the segment empty" for decision in result.decisions) - - -def test_non_empty_segment_validator_allows_punctuation_only_and_unpreviewable_proposals(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "uh"}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "There were gestures at the temple."} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="spoken_word", - module_key="spoken_word", - id=1, - original_text="uh", - corrected_text=".", - confidence=0.95, - ), - CorrectionProposal( - proposal_index=1, - module_instance="spoken_word", - module_key="spoken_word", - id=2, - original_text="rank", - corrected_text="Hrank", - confidence=0.95, - ), - ] - - result = NonEmptySegmentValidator("non_empty_segment_guard").validate( - _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path) - ) - - assert [(decision.proposal_index, decision.approved, decision.reason) for decision in result.decisions] == [ - (0, True, None), - (1, True, None), - ] - - -def test_identical_text_validator_rejects_exact_noops_only(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello"}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "hello"} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="grammar", - module_key="grammar", - id=1, - original_text="hello", - corrected_text="hello", - confidence=0.95, - ), - CorrectionProposal( - proposal_index=1, - module_instance="grammar", - module_key="grammar", - id=2, - original_text="hello", - corrected_text="Hello", - confidence=0.95, - ), - ] - - result = IdenticalTextValidator("identical_text_guard").validate( - _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path) - ) - - assert [(decision.proposal_index, decision.approved, decision.reason) for decision in result.decisions] == [ - (0, False, "proposal original_text and corrected_text are identical"), - (1, True, None), - ] - - -def test_original_text_present_validator_rejects_missing_spans_and_allows_present_or_unknown_segments(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello hello"}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "goodbye"} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="homophones", - module_key="homophones", - id=1, - original_text="hello", - corrected_text="hi", - confidence=0.95, - ), - CorrectionProposal( - proposal_index=1, - module_instance="homophones", - module_key="homophones", - id=2, - original_text="rank", - corrected_text="Hrank", - confidence=0.95, - ), - CorrectionProposal( - proposal_index=2, - module_instance="homophones", - module_key="homophones", - id=99, - original_text="missing", - corrected_text="present", - confidence=0.95, - ), - ] - - result = OriginalTextPresentValidator("original_text_present_guard").validate( - _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path) - ) - - assert [(decision.proposal_index, decision.approved, decision.reason) for decision in result.decisions] == [ - (0, True, None), - (1, False, "proposal original_text does not match segment text"), - (2, True, None), - ] - - -@pytest.mark.parametrize( - ("validator", "payload", "message_fragment"), - [ - ( - SpokenFormPlausibilityValidator("spoken_form_plausibility_review"), - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.9, - "reason": "ok", - }, - { - "correction_index": 0, - "approved": True, - "confidence": 0.9, - "reason": "duplicate", - }, - ] - }, - "duplicate correction_index values", - ), - ( - MeaningReversalValidator("meaning_reversal_review"), - {"validations": []}, - "omitted correction_index values", - ), - ( - SpokenFormPlausibilityValidator("spoken_form_plausibility_review"), - { - "validations": [ - { - "correction_index": 99, - "approved": True, - "confidence": 0.9, - "reason": "unknown", - } - ] - }, - "unknown correction_index", - ), - ( - EditorialValidator("editorial_review"), - { - "validations": [ - { - "correction_index": 99, - "approved": True, - "confidence": 0.9, - "reason": "unknown", - } - ] - }, - "unknown correction_index", - ), - ( - SpokenWordValidator("spoken_word_review"), - { - "validations": [ - { - "correction_index": 99, - "approved": True, - "confidence": 0.9, - "reason": "unknown", - } - ] - }, - "unknown correction_index", - ), - ], -) -def test_llm_validators_reject_bad_correction_indexes(tmp_path, validator, payload, message_fragment): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="homophones", - module_key="homophones", - id=1, - original_text="gestures", - corrected_text="Jesters", - confidence=0.95, - ) - ] - client = FakeStructuredLLMClient([payload]) - - with pytest.raises(AuditaLLMError, match=message_fragment): - validator.validate(_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)) - - -def test_llm_validator_rejects_unpreviewable_proposals_without_calling_llm(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="homophones", - module_key="homophones", - id=1, - original_text="rank", - corrected_text="Hrank", - confidence=0.95, - ) - ] - client = FakeStructuredLLMClient([]) - - result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate( - _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) - ) - - assert result.decisions[0].approved is False - assert result.decisions[0].reason == "proposal original_text does not match segment text" - assert client.calls == [] - - -def test_llm_validator_skips_empty_candidate_sets_without_calling_llm(tmp_path): - client = FakeStructuredLLMClient([]) - - result = MeaningReversalValidator("meaning_reversal_review").validate( - _context(proposals=[], transcript=[], llm_client=client, tmp_path=tmp_path) - ) - - assert result.decisions == [] - assert client.calls == [] - - -def test_spoken_form_prompt_emphasizes_acoustic_plausibility(): - messages = build_spoken_form_plausibility_messages( - [ - { - "correction_index": 0, - "id": 1, - "original_segment_text": "There were gestures at the temple.", - "corrected_segment_text": "There were Jesters at the temple.", - "original_text": "gestures", - "corrected_text": "Jesters", - } - ] - ) - - combined = messages[0]["content"] + messages[1]["content"] - assert "homophone" in combined - assert "gestures" in combined - assert "Lyra" in combined - assert '"original_segment_text"' in messages[1]["content"] - - -def test_meaning_reversal_prompt_emphasizes_antonyms_and_segment_context(): - messages = build_meaning_reversal_messages( - [ - { - "correction_index": 0, - "id": 1, - "original_segment_text": "The figure became visible in the doorway.", - "corrected_segment_text": "The figure became invisible in the doorway.", - "original_text": "visible", - "corrected_text": "invisible", - } - ] - ) - - combined = messages[0]["content"] + messages[1]["content"] - assert "antonym" in combined - assert "visible" in combined - assert "up" in combined - assert "original_segment_text" in messages[1]["content"] - - -def test_spoken_word_prompt_emphasizes_dysfluency_cleanup(): - messages = build_spoken_word_messages( - [ - { - "correction_index": 0, - "id": 1, - "original_segment_text": "I, uh, I think we should go.", - "corrected_segment_text": "I think we should go.", - "original_text": "I, uh, I think", - "corrected_text": "I think", - } - ] - ) - - combined = messages[0]["content"] + messages[1]["content"] - assert "dysfluencies" in combined - assert "intentional emphasis" in combined - assert "Stop! Stop! Stop!" in combined - assert "punctuation" in combined - assert "substantive meaning" in combined - assert "original_segment_text" in messages[1]["content"] - - -def test_llm_validators_use_shared_token_batching_helper(monkeypatch, tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "We saw rank near the gate."}, - {"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "ChatGPT still can't do that with a dam."} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="homophones", - module_key="homophones", - id=1, - original_text="gestures", - corrected_text="Jesters", - confidence=0.95, - ), - CorrectionProposal( - proposal_index=1, - module_instance="homophones", - module_key="homophones", - id=2, - original_text="rank", - corrected_text="Hrank", - confidence=0.95, - ), - CorrectionProposal( - proposal_index=2, - module_instance="homophones", - module_key="homophones", - id=3, - original_text="dam", - corrected_text="damn", - confidence=0.95, - ), - ] - client = FakeStructuredLLMClient( - [ - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.9, - "reason": "ok", - } - ] - }, - { - "validations": [ - { - "correction_index": 1, - "approved": True, - "confidence": 0.9, - "reason": "ok", - }, - { - "correction_index": 2, - "approved": True, - "confidence": 0.9, - "reason": "ok", - }, - ] - }, - ] - ) - chunk_calls = [] - - def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message): - chunk_calls.append( - { - "max_tokens": max_tokens, - "payloads": [payload_fn(item) for item in items], - "empty_error_message": empty_error_message, - } - ) - return [ - TokenBatch(batch_index=0, items=list(items[:1]), token_count=1), - TokenBatch(batch_index=1, items=list(items[1:]), token_count=1), - ] - - monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items) - - result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate( - _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) - ) - - assert [decision.approved for decision in result.decisions] == [True, True, True] - assert len(client.calls) == 2 - assert len(chunk_calls) == 1 - assert ( - chunk_calls[0]["max_tokens"] - == AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}).validation_max_prompt_tokens - ) - assert all("corrected_segment_text" in payload for payload in chunk_calls[0]["payloads"]) - - -def test_grammar_validation_prompt_is_scoped_to_editorial_cleanup(): - messages = build_grammar_only_messages( - [ - { - "correction_index": 0, - "id": 1, - "original_text": "dam", - "corrected_text": "damn", - "confidence": 0.95, - "original_segment_text": "ChatGPT still can't do that with a dam.", - "corrected_segment_text": "ChatGPT still can't do that with a damn.", - } - ] - ) - - combined = messages[0]["content"] + messages[1]["content"] - assert "acceptable editorial revision" in combined - assert "article cleanup" in combined - assert "homophone or mistranscription corrections" in combined - assert "dysfluency cleanup" in combined - assert "original_segment_text" in messages[1]["content"] - - -def test_llm_validators_process_batches_concurrently_and_preserve_proposal_order(monkeypatch, tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "ChatGPT still can't do that with a dam."} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="homophones", - module_key="homophones", - id=1, - original_text="gestures", - corrected_text="Jesters", - confidence=0.95, - ), - CorrectionProposal( - proposal_index=1, - module_instance="homophones", - module_key="homophones", - id=2, - original_text="dam", - corrected_text="damn", - confidence=0.95, - ), - ] - client = CoordinatedStructuredLLMClient( - [ - lambda **kwargs: { - "validations": [ - { - "correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1, - "approved": True, - "confidence": 0.97, - "reason": "ok", - } - ] - }, - lambda **kwargs: { - "validations": [ - { - "correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1, - "approved": True, - "confidence": 0.94, - "reason": "ok", - } - ] - }, - ], - threading.Barrier(2, timeout=1.0), - ) - - def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message): - return [ - TokenBatch(batch_index=0, items=list(items[:1]), token_count=1), - TokenBatch(batch_index=1, items=list(items[1:]), token_count=1), - ] - - monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items) - - result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate( - _context( - proposals=proposals, - transcript=transcript, - llm_client=client, - tmp_path=tmp_path, - llm_concurrency=2, - ) - ) - - assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [ - (0, True), - (1, True), - ] - assert len(client.calls) == 2 - - -def test_validator_uses_validation_llm_config(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="homophones", - module_key="homophones", - id=1, - original_text="gestures", - corrected_text="Jesters", - confidence=0.95, - ) - ] - client = FakeStructuredLLMClient( - [ - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.97, - "reason": "ok", - } - ] - } - ] - ) - config = AuditaConfig.from_sources( - env={"OPENROUTER_API_KEY": "primary-key"}, - overrides=ConfigOverrides( - model="primary-model", - base_url="http://localhost:8000/v1", - max_retries=7, - llm_timeout_seconds=120, - validation_llm_api_key="validation-key", - validation_model="validation-model", - validation_base_url="http://localhost:9000/v1", - validation_max_retries=2, - validation_llm_timeout_seconds=240, - validation_llm_concurrency=3, - ), - ) - context = ValidationContext( - proposals=proposals, - transcript=transcript, - glossary=_glossary(), - config=config.validation_llm_config(), - run_spec=ModuleRunSpec( - instance_name="homophones", - module_key="homophones", - module=_Module("require_unique"), - ), - run_dir=tmp_path, - llm_client=client, - ) - - SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(context) - - validation_config = client.calls[0]["config"] - assert validation_config.api_key == "validation-key" - assert validation_config.model == "validation-model" - assert validation_config.base_url == "http://localhost:9000/v1" - assert validation_config.max_retries == 2 - assert validation_config.llm_timeout_seconds == 240 - assert validation_config.llm_concurrency == 3 - - -def test_validator_batches_use_validation_max_prompt_tokens(tmp_path, monkeypatch): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="homophones", - module_key="homophones", - id=1, - original_text="gestures", - corrected_text="Jesters", - confidence=0.95, - ) - ] - client = FakeStructuredLLMClient( - [ - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.97, - "reason": "ok", - } - ] - } - ] - ) - chunk_calls = [] - - def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message): - chunk_calls.append( - { - "count": len(items), - "max_tokens": max_tokens, - } - ) - return [TokenBatch(batch_index=0, items=list(items), token_count=1)] - - monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items) - config = AuditaConfig.from_sources( - env={"OPENROUTER_API_KEY": "test-key"}, - overrides=ConfigOverrides(validation_max_prompt_tokens=1024), - ) - context = ValidationContext( - proposals=proposals, - transcript=transcript, - glossary=_glossary(), - config=config.validation_llm_config(), - run_spec=ModuleRunSpec( - instance_name="homophones", - module_key="homophones", - module=_Module("require_unique"), - ), - run_dir=tmp_path, - llm_client=client, - ) - - SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(context) - - assert chunk_calls[0]["max_tokens"] == 1024 - - -def test_validator_uses_validation_llm_concurrency_override(tmp_path, monkeypatch): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}, - {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "ChatGPT still can't do that with a dam."} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="homophones", - module_key="homophones", - id=1, - original_text="gestures", - corrected_text="Jesters", - confidence=0.95, - ), - CorrectionProposal( - proposal_index=1, - module_instance="homophones", - module_key="homophones", - id=2, - original_text="dam", - corrected_text="damn", - confidence=0.95, - ), - ] - client = CoordinatedStructuredLLMClient( - [ - lambda **kwargs: { - "validations": [ - { - "correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1, - "approved": True, - "confidence": 0.97, - "reason": "ok", - } - ] - }, - lambda **kwargs: { - "validations": [ - { - "correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1, - "approved": True, - "confidence": 0.94, - "reason": "ok", - } - ] - }, - ], - threading.Barrier(2, timeout=1.0), - ) - - def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message): - return [ - TokenBatch(batch_index=0, items=list(items[:1]), token_count=1), - TokenBatch(batch_index=1, items=list(items[1:]), token_count=1), - ] - - monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items) - - result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate( - _context( - proposals=proposals, - transcript=transcript, - llm_client=client, - tmp_path=tmp_path, - llm_concurrency=1, - validation_llm_concurrency=2, - ) - ) - - assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [ - (0, True), - (1, True), - ] - assert len(client.calls) == 2 diff --git a/python/tests/test_module_proposals.py b/python/tests/test_module_proposals.py deleted file mode 100644 index 88572ce..0000000 --- a/python/tests/test_module_proposals.py +++ /dev/null @@ -1,1173 +0,0 @@ -import threading -from pathlib import Path - -from audita.core.chunking import chunk_transcript -from audita.core.config import AuditaConfig, ConfigOverrides -from audita.core.errors import AuditaLLMError -from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_json, parse_transcript_json -from audita.framework.models import ModuleContext, ModuleRunSpec -from audita.modules.grammar import GrammarModule -from audita.modules.glossary import GlossaryModule -from audita.modules.homophones import HomophonesModule -from audita.modules.prompts import ( - build_grammar_proposal_messages, - build_homophones_proposal_messages, - build_spoken_word_proposal_messages, -) -from audita.modules.spoken_word import SpokenWordModule -from audita.pipeline import process_transcript_result - - -class FakeStructuredLLMClient: - def __init__(self, responses): - self._responses = responses - self._lock = threading.Lock() - self.calls = [] - - def run_structured(self, *, stage_name, messages, response_model, config): - with self._lock: - self.calls.append( - { - "stage_name": stage_name, - "messages": list(messages), - "response_model": response_model, - "config": config, - } - ) - payload = _pop_llm_response(self._responses, stage_name) - return response_model.model_validate(payload) - - -def _pop_llm_response(responses, stage_name): - if isinstance(responses, dict): - if stage_name not in responses: - raise AuditaLLMError(f"FakeStructuredLLMClient received unexpected stage_name: {stage_name}") - payloads = responses[stage_name] - if isinstance(payloads, list): - if not payloads: - raise AuditaLLMError(f"FakeStructuredLLMClient received too many calls for stage_name: {stage_name}") - return payloads.pop(0) - payload = payloads - del responses[stage_name] - return payload - if not responses: - raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.") - return responses.pop(0) - - -def _glossary(): - return parse_glossary_yaml( - """ - glossary: - - name: "Jesters" - aliases: - - "Jester" - category: faction - summary: "A faction." - - name: "Hrank" - category: pc - summary: "A player character." - """ - ) - - -def test_glossary_module_propose_writes_diagnostics_and_returns_proposals_without_api_key(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."} - ] - """ - ) - section = chunk_transcript(transcript, max_section_tokens=1000)[0] - module = GlossaryModule() - client = FakeStructuredLLMClient( - [ - { - "corrections": [ - { - "id": 1, - "original_text": "gestures", - "corrected_text": "Jesters", - "confidence": 0.95, - } - ] - } - ] - ) - context = ModuleContext( - run_spec=ModuleRunSpec(instance_name="glossary_primary", module_key="glossary", module=module), - glossary=_glossary(), - config=AuditaConfig.from_sources(env={}), - run_dir=tmp_path, - llm_client=client, - ) - - proposals = list(module.propose(section, context)) - - assert [(proposal.id, proposal.original_text, proposal.corrected_text, proposal.confidence) for proposal in proposals] == [ - (1, "gestures", "Jesters", 0.95) - ] - assert (tmp_path / "prompt-0000.json").exists() - assert (tmp_path / "corrections-0000.json").exists() - prompt_text = client.calls[0]["messages"][1]["content"] - assert "Glossary:" in prompt_text - assert "exact text span" in prompt_text - assert "gestures" in prompt_text - - -def test_module_propose_uses_primary_llm_config(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."} - ] - """ - ) - section = chunk_transcript(transcript, max_section_tokens=1000)[0] - module = GlossaryModule() - client = FakeStructuredLLMClient( - [ - { - "corrections": [ - { - "id": 1, - "original_text": "gestures", - "corrected_text": "Jesters", - "confidence": 0.95, - } - ] - } - ] - ) - config = AuditaConfig.from_sources( - env={"OPENROUTER_API_KEY": "primary-key"}, - overrides=ConfigOverrides( - model="primary-model", - base_url="http://localhost:8000/v1", - max_retries=7, - llm_timeout_seconds=120, - validation_llm_api_key="validation-key", - validation_model="validation-model", - validation_base_url="http://localhost:9000/v1", - validation_max_retries=2, - validation_llm_timeout_seconds=240, - validation_llm_concurrency=3, - ), - ) - context = ModuleContext( - run_spec=ModuleRunSpec(instance_name="glossary_primary", module_key="glossary", module=module), - glossary=_glossary(), - config=config, - run_dir=tmp_path, - llm_client=client, - ) - - module.propose(section, context) - - proposal_config = client.calls[0]["config"] - assert proposal_config.api_key == "primary-key" - assert proposal_config.model == "primary-model" - assert proposal_config.base_url == "http://localhost:8000/v1" - assert proposal_config.max_retries == 7 - assert proposal_config.llm_timeout_seconds == 120 - - -def test_homophones_prompt_is_explicitly_scoped_to_spoken_form_corrections(): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't do that with a dam."} - ] - """ - ) - section = chunk_transcript(transcript, max_section_tokens=1000)[0] - - messages = build_homophones_proposal_messages(section, _glossary()) - combined = messages[0]["content"] + messages[1]["content"] - - assert "homophone" in combined - assert "mistranscription" in combined - assert "Do not add or remove punctuation" in combined - assert "visible" in combined - assert '"id": 1' in messages[1]["content"] - - -def test_spoken_word_module_propose_writes_diagnostics_and_returns_proposals_without_api_key(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."} - ] - """ - ) - section = chunk_transcript(transcript, max_section_tokens=1000)[0] - module = SpokenWordModule() - client = FakeStructuredLLMClient( - [ - { - "corrections": [ - { - "id": 1, - "original_text": "I, uh, I think", - "corrected_text": "I think", - "confidence": 0.95, - } - ] - } - ] - ) - context = ModuleContext( - run_spec=ModuleRunSpec(instance_name="spoken_word", module_key="spoken_word", module=module), - glossary=_glossary(), - config=AuditaConfig.from_sources(env={}), - run_dir=tmp_path, - llm_client=client, - ) - - proposals = list(module.propose(section, context)) - - assert [(proposal.id, proposal.original_text, proposal.corrected_text, proposal.confidence) for proposal in proposals] == [ - (1, "I, uh, I think", "I think", 0.95) - ] - assert (tmp_path / "prompt-0000.json").exists() - assert (tmp_path / "corrections-0000.json").exists() - prompt_text = client.calls[0]["messages"][1]["content"] - assert "spoken-word cleanup" in prompt_text - assert "exact text span" in prompt_text - assert "uh" in prompt_text - - -def test_spoken_word_prompt_is_explicitly_scoped_to_dysfluency_cleanup(): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Well ... I think we should go."} - ] - """ - ) - section = chunk_transcript(transcript, max_section_tokens=1000)[0] - - messages = build_spoken_word_proposal_messages(section, _glossary()) - combined = messages[0]["content"] + messages[1]["content"] - - assert "dysfluencies" in combined - assert "intentional emphasis" in combined - assert "Help! Help! Help!" in combined - assert "punctuation" in combined - assert "paraphrase" in combined - assert '"id": 1' in messages[1]["content"] - - -def test_grammar_module_propose_writes_diagnostics_and_returns_proposals_without_api_key(tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} - ] - """ - ) - section = chunk_transcript(transcript, max_section_tokens=1000)[0] - module = GrammarModule() - client = FakeStructuredLLMClient( - [ - { - "corrections": [ - { - "id": 1, - "original_text": "hello world", - "corrected_text": "Hello world.", - "confidence": 0.95, - } - ] - } - ] - ) - context = ModuleContext( - run_spec=ModuleRunSpec(instance_name="grammar", module_key="grammar", module=module), - glossary=_glossary(), - config=AuditaConfig.from_sources(env={}), - run_dir=tmp_path, - llm_client=client, - ) - - proposals = list(module.propose(section, context)) - - assert [(proposal.id, proposal.original_text, proposal.corrected_text, proposal.confidence) for proposal in proposals] == [ - (1, "hello world", "Hello world.", 0.95) - ] - assert (tmp_path / "prompt-0000.json").exists() - assert (tmp_path / "corrections-0000.json").exists() - prompt_text = client.calls[0]["messages"][1]["content"] - assert "punctuation, capitalization, spacing, and article cleanup" in prompt_text - assert "exact text span" in prompt_text - assert "word substitutions" in prompt_text - assert "later review stage" in prompt_text - - -def test_grammar_prompt_is_explicitly_scoped_to_formatting_cleanup(): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} - ] - """ - ) - section = chunk_transcript(transcript, max_section_tokens=1000)[0] - - messages = build_grammar_proposal_messages(section, _glossary()) - combined = messages[0]["content"] + messages[1]["content"] - - assert "punctuation, capitalization, spacing, and article cleanup" in combined - assert "word substitutions" in combined - assert "homophone fixes" in combined - assert "later review stage" in combined - assert "mistranscription" in combined - assert '"id": 1' in messages[1]["content"] - - -def test_process_transcript_result_uses_injected_fake_client_and_applies_sequential_module_updates(tmp_path): - transcript = parse_source_transcript_json( - """ - [ - {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the dam."} - ] - """ - ) - config = AuditaConfig.from_sources( - env={}, - overrides=None, - ) - config = AuditaConfig( - api_key=config.api_key, - model=config.model, - base_url=config.base_url, - max_retries=config.max_retries, - max_section_tokens=config.max_section_tokens, - glossary_confidence_threshold=config.glossary_confidence_threshold, - grammar_confidence_threshold=config.grammar_confidence_threshold, - homophones_confidence_threshold=config.homophones_confidence_threshold, - spoken_word_confidence_threshold=config.spoken_word_confidence_threshold, - normalize_max_segment_gap=config.normalize_max_segment_gap, - normalize_ellipsis_gap=config.normalize_ellipsis_gap, - normalize_max_segment_duration=config.normalize_max_segment_duration, - normalize_max_segment_tokens=config.normalize_max_segment_tokens, - work_dir=tmp_path / "work", - work_dir_retention="always", - ) - client = FakeStructuredLLMClient( - [ - { - "corrections": [ - { - "id": 1, - "original_text": "gestures", - "corrected_text": "Jesters", - "confidence": 0.95, - } - ] - }, - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.97, - "reason": "Likely spoken-form correction in context.", - } - ] - }, - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.99, - "reason": "Does not reverse the segment meaning.", - } - ] - }, - { - "corrections": [ - { - "id": 1, - "original_text": "dam", - "corrected_text": "damn", - "confidence": 0.92, - } - ] - }, - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.98, - "reason": "Likely spoken-form correction in context.", - } - ] - }, - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.99, - "reason": "Does not reverse the segment meaning.", - } - ] - }, - {"corrections": []}, - {"corrections": []}, - {"corrections": []}, - ] - ) - - result = process_transcript_result(transcript, _glossary(), config, llm_client=client) - - assert result.transcript[0].text == "There were Jesters at the damn." - assert [call["stage_name"] for call in client.calls] == [ - "glossary_1:proposal", - "glossary_1:spoken_form_plausibility_review", - "glossary_1:meaning_reversal_review", - "homophones:proposal", - "homophones:spoken_form_plausibility_review", - "homophones:meaning_reversal_review", - "glossary_2:proposal", - "spoken_word:proposal", - "grammar:proposal", - ] - assert "There were Jesters at the dam." in client.calls[3]["messages"][1]["content"] - - -def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_validators(tmp_path): - transcript = parse_source_transcript_json( - """ - [ - {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."} - ] - """ - ) - base_config = AuditaConfig.from_sources(env={}) - config = AuditaConfig( - api_key=base_config.api_key, - model=base_config.model, - base_url=base_config.base_url, - max_retries=base_config.max_retries, - max_section_tokens=base_config.max_section_tokens, - glossary_confidence_threshold=0.96, - grammar_confidence_threshold=base_config.grammar_confidence_threshold, - homophones_confidence_threshold=base_config.homophones_confidence_threshold, - spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, - normalize_max_segment_gap=base_config.normalize_max_segment_gap, - normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, - normalize_max_segment_duration=base_config.normalize_max_segment_duration, - normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, - work_dir=tmp_path / "work", - work_dir_retention="always", - ) - client = FakeStructuredLLMClient( - [ - { - "corrections": [ - { - "id": 1, - "original_text": "gestures", - "corrected_text": "Jesters", - "confidence": 0.95, - } - ] - }, - {"corrections": []}, - {"corrections": []}, - {"corrections": []}, - {"corrections": []}, - ] - ) - - result = process_transcript_result(transcript, _glossary(), config, llm_client=client) - - assert result.transcript[0].text == "There were gestures at the temple." - assert [call["stage_name"] for call in client.calls] == [ - "glossary_1:proposal", - "homophones:proposal", - "glossary_2:proposal", - "spoken_word:proposal", - "grammar:proposal", - ] - assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold" - assert result.report.skipped_corrections[0].source == "validator:proposal_confidence_guard" - assert result.report.modules[0].validators[2].rejected_count == 1 - assert result.report.modules[0].validators[3].candidate_count == 0 - - -def test_process_transcript_result_runs_spoken_word_module_with_full_validator_chain(tmp_path): - transcript = parse_source_transcript_json( - """ - [ - {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."} - ] - """ - ) - base_config = AuditaConfig.from_sources(env={}) - config = AuditaConfig( - api_key=base_config.api_key, - model=base_config.model, - base_url=base_config.base_url, - max_retries=base_config.max_retries, - max_section_tokens=base_config.max_section_tokens, - glossary_confidence_threshold=base_config.glossary_confidence_threshold, - grammar_confidence_threshold=base_config.grammar_confidence_threshold, - homophones_confidence_threshold=base_config.homophones_confidence_threshold, - spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, - normalize_max_segment_gap=base_config.normalize_max_segment_gap, - normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, - normalize_max_segment_duration=base_config.normalize_max_segment_duration, - normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, - work_dir=tmp_path / "work", - work_dir_retention="always", - ) - client = FakeStructuredLLMClient( - [ - { - "corrections": [ - { - "id": 1, - "original_text": "I, uh, I think", - "corrected_text": "I think", - "confidence": 0.95, - } - ] - }, - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.97, - "reason": "Reasonable dysfluency cleanup that preserves meaning.", - } - ] - }, - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.99, - "reason": "Does not reverse the segment meaning.", - } - ] - }, - ] - ) - - result = process_transcript_result( - transcript, - _glossary(), - config, - module_keys=["spoken_word"], - llm_client=client, - ) - - assert result.transcript[0].text == "I think we should go." - assert [call["stage_name"] for call in client.calls] == [ - "spoken_word:proposal", - "spoken_word:spoken_word_review", - "spoken_word:meaning_reversal_review", - ] - assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ - "identical_text_guard", - "original_text_present_guard", - "proposal_confidence_guard", - "protected_glossary_guard", - "non_empty_segment_guard", - "spoken_word_review", - "meaning_reversal_review", - ] - - -def test_process_transcript_result_rejects_spoken_word_below_threshold_before_llm_validators(tmp_path): - transcript = parse_source_transcript_json( - """ - [ - {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."} - ] - """ - ) - base_config = AuditaConfig.from_sources(env={}) - config = AuditaConfig( - api_key=base_config.api_key, - model=base_config.model, - base_url=base_config.base_url, - max_retries=base_config.max_retries, - max_section_tokens=base_config.max_section_tokens, - glossary_confidence_threshold=base_config.glossary_confidence_threshold, - grammar_confidence_threshold=base_config.grammar_confidence_threshold, - homophones_confidence_threshold=base_config.homophones_confidence_threshold, - spoken_word_confidence_threshold=0.96, - normalize_max_segment_gap=base_config.normalize_max_segment_gap, - normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, - normalize_max_segment_duration=base_config.normalize_max_segment_duration, - normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, - work_dir=tmp_path / "work", - work_dir_retention="always", - ) - client = FakeStructuredLLMClient( - [ - { - "corrections": [ - { - "id": 1, - "original_text": "I, uh, I think", - "corrected_text": "I think", - "confidence": 0.95, - } - ] - } - ] - ) - - result = process_transcript_result( - transcript, - _glossary(), - config, - module_keys=["spoken_word"], - llm_client=client, - ) - - assert result.transcript[0].text == "I, uh, I think we should go." - assert [call["stage_name"] for call in client.calls] == ["spoken_word:proposal"] - assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold" - assert result.report.modules[0].validators[3].candidate_count == 0 - - -def test_process_transcript_result_runs_grammar_module_with_full_validator_chain(tmp_path): - transcript = parse_source_transcript_json( - """ - [ - {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} - ] - """ - ) - base_config = AuditaConfig.from_sources(env={}) - config = AuditaConfig( - api_key=base_config.api_key, - model=base_config.model, - base_url=base_config.base_url, - max_retries=base_config.max_retries, - max_section_tokens=base_config.max_section_tokens, - glossary_confidence_threshold=base_config.glossary_confidence_threshold, - grammar_confidence_threshold=base_config.grammar_confidence_threshold, - homophones_confidence_threshold=base_config.homophones_confidence_threshold, - spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, - normalize_max_segment_gap=base_config.normalize_max_segment_gap, - normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, - normalize_max_segment_duration=base_config.normalize_max_segment_duration, - normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, - work_dir=tmp_path / "work", - work_dir_retention="always", - ) - client = FakeStructuredLLMClient( - [ - { - "corrections": [ - { - "id": 1, - "original_text": "hello world", - "corrected_text": "Hello world.", - "confidence": 0.95, - } - ] - }, - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.98, - "reason": "Conservative grammar cleanup.", - } - ] - }, - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.99, - "reason": "Does not reverse the segment meaning.", - } - ] - }, - ] - ) - - result = process_transcript_result( - transcript, - _glossary(), - config, - module_keys=["grammar"], - llm_client=client, - ) - - assert result.transcript[0].text == "Hello world." - assert [call["stage_name"] for call in client.calls] == [ - "grammar:proposal", - "grammar:grammar_only_guard", - "grammar:meaning_reversal_review", - ] - assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ - "identical_text_guard", - "original_text_present_guard", - "proposal_confidence_guard", - "protected_glossary_guard", - "non_empty_segment_guard", - "grammar_only_guard", - "meaning_reversal_review", - ] - - -def test_process_transcript_result_grammar_module_applies_indefinite_article_cleanup(tmp_path): - transcript = parse_source_transcript_json( - """ - [ - {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Give me a intelligence saving throw."} - ] - """ - ) - base_config = AuditaConfig.from_sources(env={}) - config = AuditaConfig( - api_key=base_config.api_key, - model=base_config.model, - base_url=base_config.base_url, - max_retries=base_config.max_retries, - max_section_tokens=base_config.max_section_tokens, - glossary_confidence_threshold=base_config.glossary_confidence_threshold, - grammar_confidence_threshold=base_config.grammar_confidence_threshold, - homophones_confidence_threshold=base_config.homophones_confidence_threshold, - spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, - normalize_max_segment_gap=base_config.normalize_max_segment_gap, - normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, - normalize_max_segment_duration=base_config.normalize_max_segment_duration, - normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, - work_dir=tmp_path / "work", - work_dir_retention="always", - ) - client = FakeStructuredLLMClient( - [ - { - "corrections": [ - { - "id": 1, - "original_text": "a intelligence saving throw", - "corrected_text": "an intelligence saving throw", - "confidence": 0.95, - } - ] - }, - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.97, - "reason": "Allowed article cleanup.", - } - ] - }, - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.99, - "reason": "Does not reverse the segment meaning.", - } - ] - }, - ] - ) - - result = process_transcript_result( - transcript, - _glossary(), - config, - module_keys=["grammar"], - llm_client=client, - ) - - assert result.transcript[0].text == "Give me an intelligence saving throw." - assert [call["stage_name"] for call in client.calls] == [ - "grammar:proposal", - "grammar:grammar_only_guard", - "grammar:meaning_reversal_review", - ] - - -def test_process_transcript_result_rejects_grammar_below_threshold_before_later_validators(tmp_path): - transcript = parse_source_transcript_json( - """ - [ - {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} - ] - """ - ) - base_config = AuditaConfig.from_sources(env={}) - config = AuditaConfig( - api_key=base_config.api_key, - model=base_config.model, - base_url=base_config.base_url, - max_retries=base_config.max_retries, - max_section_tokens=base_config.max_section_tokens, - glossary_confidence_threshold=base_config.glossary_confidence_threshold, - grammar_confidence_threshold=0.96, - homophones_confidence_threshold=base_config.homophones_confidence_threshold, - spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, - normalize_max_segment_gap=base_config.normalize_max_segment_gap, - normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, - normalize_max_segment_duration=base_config.normalize_max_segment_duration, - normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, - work_dir=tmp_path / "work", - work_dir_retention="always", - ) - client = FakeStructuredLLMClient( - [ - { - "corrections": [ - { - "id": 1, - "original_text": "hello world", - "corrected_text": "Hello world.", - "confidence": 0.95, - } - ] - } - ] - ) - - result = process_transcript_result( - transcript, - _glossary(), - config, - module_keys=["grammar"], - llm_client=client, - ) - - assert result.transcript[0].text == "hello world" - assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"] - assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold" - assert result.report.modules[0].validators[3].candidate_count == 0 - - -def test_process_transcript_result_grammar_module_allows_editorial_homophone_style_proposals(tmp_path): - transcript = parse_source_transcript_json( - """ - [ - {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."} - ] - """ - ) - base_config = AuditaConfig.from_sources(env={}) - config = AuditaConfig( - api_key=base_config.api_key, - model=base_config.model, - base_url=base_config.base_url, - max_retries=base_config.max_retries, - max_section_tokens=base_config.max_section_tokens, - glossary_confidence_threshold=base_config.glossary_confidence_threshold, - grammar_confidence_threshold=base_config.grammar_confidence_threshold, - homophones_confidence_threshold=base_config.homophones_confidence_threshold, - spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, - normalize_max_segment_gap=base_config.normalize_max_segment_gap, - normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, - normalize_max_segment_duration=base_config.normalize_max_segment_duration, - normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, - work_dir=tmp_path / "work", - work_dir_retention="always", - ) - client = FakeStructuredLLMClient( - { - "grammar:proposal": { - "corrections": [ - { - "id": 1, - "original_text": "dam", - "corrected_text": "damn", - "confidence": 0.95, - } - ] - }, - "grammar:grammar_only_guard": { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.94, - "reason": "A low-risk editorial homophone correction that preserves meaning.", - } - ] - }, - "grammar:meaning_reversal_review": { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.99, - "reason": "Does not reverse the segment meaning.", - } - ] - }, - } - ) - - result = process_transcript_result( - transcript, - _glossary(), - config, - module_keys=["grammar"], - llm_client=client, - ) - - assert result.transcript[0].text == "ChatGPT still can't really do that with a damn." - assert [call["stage_name"] for call in client.calls] == [ - "grammar:proposal", - "grammar:grammar_only_guard", - "grammar:meaning_reversal_review", - ] - assert result.report.skipped_corrections == [] - - -def test_process_transcript_result_grammar_module_allows_embedded_homophone_fix_with_grammar_cleanup(tmp_path): - transcript = parse_source_transcript_json( - """ - [ - {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't"} - ] - """ - ) - base_config = AuditaConfig.from_sources(env={}) - config = AuditaConfig( - api_key=base_config.api_key, - llm_concurrency=base_config.llm_concurrency, - module_keys=base_config.module_keys, - model=base_config.model, - base_url=base_config.base_url, - max_retries=base_config.max_retries, - max_section_tokens=base_config.max_section_tokens, - glossary_confidence_threshold=base_config.glossary_confidence_threshold, - grammar_confidence_threshold=base_config.grammar_confidence_threshold, - homophones_confidence_threshold=base_config.homophones_confidence_threshold, - spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, - normalize_max_segment_gap=base_config.normalize_max_segment_gap, - normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, - normalize_max_segment_duration=base_config.normalize_max_segment_duration, - normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, - work_dir=tmp_path / "work", - work_dir_retention="always", - ) - client = FakeStructuredLLMClient( - [ - { - "corrections": [ - { - "id": 1, - "original_text": "question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't", - "corrected_text": "question: If he dies, does he stay there? The way that, yeah, the way that it's written, it's like, so if it stops, then he goes down; but what if it doesn't?", - "confidence": 0.95, - } - ] - }, - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.95, - "reason": "Primarily a grammatical revision with an embedded likely mistranscription recovery.", - } - ] - }, - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.99, - "reason": "Does not reverse the segment meaning.", - } - ] - }, - ] - ) - - result = process_transcript_result( - transcript, - _glossary(), - config, - module_keys=["grammar"], - llm_client=client, - ) - - assert result.transcript[0].text == ( - "question: If he dies, does he stay there? The way that, yeah, the way that it's written, " - "it's like, so if it stops, then he goes down; but what if it doesn't?" - ) - assert [call["stage_name"] for call in client.calls] == [ - "grammar:proposal", - "grammar:grammar_only_guard", - "grammar:meaning_reversal_review", - ] - - -def test_process_transcript_result_rejects_spoken_word_whole_segment_deletion_before_llm_validators(tmp_path): - transcript = parse_source_transcript_json( - """ - [ - {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "uh"} - ] - """ - ) - base_config = AuditaConfig.from_sources(env={}) - config = AuditaConfig( - api_key=base_config.api_key, - model=base_config.model, - base_url=base_config.base_url, - max_retries=base_config.max_retries, - max_section_tokens=base_config.max_section_tokens, - glossary_confidence_threshold=base_config.glossary_confidence_threshold, - grammar_confidence_threshold=base_config.grammar_confidence_threshold, - homophones_confidence_threshold=base_config.homophones_confidence_threshold, - spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, - normalize_max_segment_gap=base_config.normalize_max_segment_gap, - normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, - normalize_max_segment_duration=base_config.normalize_max_segment_duration, - normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, - work_dir=tmp_path / "work", - work_dir_retention="always", - ) - client = FakeStructuredLLMClient( - [ - { - "corrections": [ - { - "id": 1, - "original_text": "uh", - "corrected_text": "", - "confidence": 0.95, - } - ] - } - ] - ) - - result = process_transcript_result( - transcript, - _glossary(), - config, - module_keys=["spoken_word"], - llm_client=client, - ) - - assert result.transcript[0].text == "uh" - assert [call["stage_name"] for call in client.calls] == ["spoken_word:proposal"] - assert result.report.skipped_corrections[0].source == "validator:non_empty_segment_guard" - assert result.report.skipped_corrections[0].reason == "correction would leave the segment empty" - assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ - "identical_text_guard", - "original_text_present_guard", - "proposal_confidence_guard", - "protected_glossary_guard", - "non_empty_segment_guard", - "spoken_word_review", - "meaning_reversal_review", - ] - assert result.report.modules[0].validators[4].rejected_count == 1 - assert result.report.modules[0].validators[5].candidate_count == 0 - - -def test_process_transcript_result_rejects_identical_text_before_later_validators(tmp_path): - transcript = parse_source_transcript_json( - """ - [ - {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} - ] - """ - ) - config = AuditaConfig.from_sources( - env={}, - overrides=ConfigOverrides(work_dir=tmp_path / "work", work_dir_retention="always"), - ) - client = FakeStructuredLLMClient( - [ - { - "corrections": [ - { - "id": 1, - "original_text": "hello", - "corrected_text": "hello", - "confidence": 0.95, - } - ] - } - ] - ) - - result = process_transcript_result( - transcript, - _glossary(), - config, - module_keys=["grammar"], - llm_client=client, - ) - - assert result.transcript[0].text == "hello world" - assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"] - assert result.report.skipped_corrections[0].source == "validator:identical_text_guard" - assert result.report.skipped_corrections[0].reason == "proposal original_text and corrected_text are identical" - assert result.report.modules[0].validators[0].rejected_count == 1 - assert result.report.modules[0].validators[1].candidate_count == 0 - - -def test_process_transcript_result_rejects_missing_original_text_before_later_validators(tmp_path): - transcript = parse_source_transcript_json( - """ - [ - {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} - ] - """ - ) - config = AuditaConfig.from_sources( - env={}, - overrides=ConfigOverrides(work_dir=tmp_path / "work", work_dir_retention="always"), - ) - client = FakeStructuredLLMClient( - [ - { - "corrections": [ - { - "id": 1, - "original_text": "goodbye", - "corrected_text": "farewell", - "confidence": 0.95, - } - ] - } - ] - ) - - result = process_transcript_result( - transcript, - _glossary(), - config, - module_keys=["grammar"], - llm_client=client, - ) - - assert result.transcript[0].text == "hello world" - assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"] - assert result.report.skipped_corrections[0].source == "validator:original_text_present_guard" - assert result.report.skipped_corrections[0].reason == "proposal original_text does not match segment text" - assert result.report.modules[0].validators[0].approved_count == 1 - assert result.report.modules[0].validators[1].rejected_count == 1 - assert result.report.modules[0].validators[2].candidate_count == 0 diff --git a/python/tests/test_new_cli.py b/python/tests/test_new_cli.py deleted file mode 100644 index 8fd464e..0000000 --- a/python/tests/test_new_cli.py +++ /dev/null @@ -1,659 +0,0 @@ -import json -import io -import pytest - -from audita.cli import main -from audita.core.errors import AuditaConfigError -from audita.core.reporting import ProcessResult, RunReport -from audita.core.schemas import parse_transcript_json - - -def test_cli_help_uses_audita_program_name(capsys): - with pytest.raises(SystemExit) as exc: - main(["--help"]) - - assert exc.value.code == 0 - assert capsys.readouterr().out.startswith("usage: audita ") - - -def test_process_help_exposes_framework_flags(capsys): - with pytest.raises(SystemExit) as exc: - main(["process", "--help"]) - - assert exc.value.code == 0 - output = capsys.readouterr().out - assert "--report-json" in output - assert "--llm-api-key" in output - assert "--llm-concurrency" in output - assert "--llm-timeout-seconds" in output - assert "--validation-llm-api-key" in output - assert "--validation-llm-concurrency" in output - assert "--validation-llm-timeout-seconds" in output - assert "--validation-model" in output - assert "--validation-base-url" in output - assert "--validation-max-retries" in output - assert "--validation-max-prompt-tokens" in output - assert "--target-sections" in output - assert "--modules" in output - assert "--model" in output - assert "--base-url" in output - assert "--max-retries" in output - assert "--max-section-tokens" in output - assert "--min-section-tokens" in output - assert "--glossary-confidence-threshold" in output - assert "--grammar-confidence-threshold" in output - assert "--homophones-confidence-threshold" in output - assert "--spoken-word-confidence-threshold" in output - assert "--work-dir-retention" in output - assert "--normalize-max-segment-gap" in output - assert "--grammar-validation-enabled" not in output - - -def test_cli_process_writes_report_json(monkeypatch, tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."} - ] - """ - ) - report = RunReport( - status="success", - config={"model": "m", "base_url": "b"}, - normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0}, - pipeline=["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"], - modules=[], - applied_changes=[], - skipped_corrections=[], - totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0}, - work_dir_retention="auto", - work_dir_retained=False, - work_dir=None, - error=None, - ) - result = ProcessResult( - transcript=transcript, - report=report, - run_dir=tmp_path / "run", - work_dir_retained=False, - ) - - monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", lambda overrides=None: object()) - monkeypatch.setattr("audita.cli.load_transcript", lambda path: []) - monkeypatch.setattr("audita.cli.load_glossary", lambda path: object()) - monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result) - - output_path = tmp_path / "out.json" - report_path = tmp_path / "report.json" - exit_code = main( - [ - "process", - "transcript.json", - "--glossary", - "glossary.yaml", - "--output", - str(output_path), - "--report-json", - str(report_path), - ] - ) - - assert exit_code == 0 - assert report_path.exists() - - -def test_cli_process_passes_modules_override_to_config(monkeypatch, tmp_path): - captured = {} - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."} - ] - """ - ) - report = RunReport( - status="success", - config={"model": "m", "base_url": "b"}, - normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0}, - pipeline=["grammar"], - modules=[], - applied_changes=[], - skipped_corrections=[], - totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0}, - work_dir_retention="auto", - work_dir_retained=False, - work_dir=None, - error=None, - ) - result = ProcessResult( - transcript=transcript, - report=report, - run_dir=tmp_path / "run", - work_dir_retained=False, - ) - - def _fake_from_sources(*, overrides=None): - captured["module_keys"] = overrides.module_keys - return object() - - monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources) - monkeypatch.setattr("audita.cli.load_transcript", lambda path: []) - monkeypatch.setattr("audita.cli.load_glossary", lambda path: object()) - monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result) - - exit_code = main( - [ - "process", - "transcript.json", - "--glossary", - "glossary.yaml", - "--modules", - "grammar", - ] - ) - - assert exit_code == 0 - assert captured["module_keys"] == "grammar" - - -def test_cli_process_passes_llm_api_key_override_to_config(monkeypatch, tmp_path): - captured = {} - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."} - ] - """ - ) - report = RunReport( - status="success", - config={"model": "m", "base_url": "b"}, - normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0}, - pipeline=["grammar"], - modules=[], - applied_changes=[], - skipped_corrections=[], - totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0}, - work_dir_retention="auto", - work_dir_retained=False, - work_dir=None, - error=None, - ) - result = ProcessResult( - transcript=transcript, - report=report, - run_dir=tmp_path / "run", - work_dir_retained=False, - ) - - def _fake_from_sources(*, overrides=None): - captured["llm_api_key"] = overrides.llm_api_key - return object() - - monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources) - monkeypatch.setattr("audita.cli.load_transcript", lambda path: []) - monkeypatch.setattr("audita.cli.load_glossary", lambda path: object()) - monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result) - - exit_code = main( - [ - "process", - "transcript.json", - "--glossary", - "glossary.yaml", - "--llm-api-key", - "cli-key", - ] - ) - - assert exit_code == 0 - assert captured["llm_api_key"] == "cli-key" - - -def test_cli_process_passes_llm_concurrency_override_to_config(monkeypatch, tmp_path): - captured = {} - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."} - ] - """ - ) - report = RunReport( - status="success", - config={"model": "m", "base_url": "b"}, - normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0}, - pipeline=["grammar"], - modules=[], - applied_changes=[], - skipped_corrections=[], - totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0}, - work_dir_retention="auto", - work_dir_retained=False, - work_dir=None, - error=None, - ) - result = ProcessResult( - transcript=transcript, - report=report, - run_dir=tmp_path / "run", - work_dir_retained=False, - ) - - def _fake_from_sources(*, overrides=None): - captured["llm_concurrency"] = overrides.llm_concurrency - return object() - - monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources) - monkeypatch.setattr("audita.cli.load_transcript", lambda path: []) - monkeypatch.setattr("audita.cli.load_glossary", lambda path: object()) - monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result) - - exit_code = main( - [ - "process", - "transcript.json", - "--glossary", - "glossary.yaml", - "--llm-concurrency", - "3", - ] - ) - - assert exit_code == 0 - assert captured["llm_concurrency"] == 3 - - -def test_cli_process_passes_llm_timeout_seconds_override_to_config(monkeypatch, tmp_path): - captured = {} - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."} - ] - """ - ) - report = RunReport( - status="success", - config={"model": "m", "base_url": "b"}, - normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0}, - pipeline=["grammar"], - modules=[], - applied_changes=[], - skipped_corrections=[], - totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0}, - work_dir_retention="auto", - work_dir_retained=False, - work_dir=None, - error=None, - ) - result = ProcessResult( - transcript=transcript, - report=report, - run_dir=tmp_path / "run", - work_dir_retained=False, - ) - - def _fake_from_sources(*, overrides=None): - captured["llm_timeout_seconds"] = overrides.llm_timeout_seconds - return object() - - monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources) - monkeypatch.setattr("audita.cli.load_transcript", lambda path: []) - monkeypatch.setattr("audita.cli.load_glossary", lambda path: object()) - monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result) - - exit_code = main( - [ - "process", - "transcript.json", - "--glossary", - "glossary.yaml", - "--llm-timeout-seconds", - "900", - ] - ) - - assert exit_code == 0 - assert captured["llm_timeout_seconds"] == 900.0 - - -def test_cli_process_passes_validation_llm_overrides_to_config(monkeypatch, tmp_path): - captured = {} - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."} - ] - """ - ) - report = RunReport( - status="success", - config={"model": "m", "base_url": "b"}, - normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0}, - pipeline=["grammar"], - modules=[], - applied_changes=[], - skipped_corrections=[], - totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0}, - work_dir_retention="auto", - work_dir_retained=False, - work_dir=None, - error=None, - ) - result = ProcessResult( - transcript=transcript, - report=report, - run_dir=tmp_path / "run", - work_dir_retained=False, - ) - - def _fake_from_sources(*, overrides=None): - captured["validation_llm_api_key"] = overrides.validation_llm_api_key - captured["validation_llm_concurrency"] = overrides.validation_llm_concurrency - captured["validation_llm_timeout_seconds"] = overrides.validation_llm_timeout_seconds - captured["validation_model"] = overrides.validation_model - captured["validation_base_url"] = overrides.validation_base_url - captured["validation_max_retries"] = overrides.validation_max_retries - captured["validation_max_prompt_tokens"] = overrides.validation_max_prompt_tokens - return object() - - monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources) - monkeypatch.setattr("audita.cli.load_transcript", lambda path: []) - monkeypatch.setattr("audita.cli.load_glossary", lambda path: object()) - monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result) - - exit_code = main( - [ - "process", - "transcript.json", - "--glossary", - "glossary.yaml", - "--validation-llm-api-key", - "validator-key", - "--validation-llm-concurrency", - "4", - "--validation-llm-timeout-seconds", - "180", - "--validation-model", - "validator-model", - "--validation-base-url", - "http://localhost:9000/v1", - "--validation-max-retries", - "2", - "--validation-max-prompt-tokens", - "1024", - ] - ) - - assert exit_code == 0 - assert captured == { - "validation_llm_api_key": "validator-key", - "validation_llm_concurrency": 4, - "validation_llm_timeout_seconds": 180.0, - "validation_model": "validator-model", - "validation_base_url": "http://localhost:9000/v1", - "validation_max_retries": 2, - "validation_max_prompt_tokens": 1024, - } - - -def test_cli_process_passes_target_sections_override_to_config(monkeypatch, tmp_path): - captured = {} - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."} - ] - """ - ) - report = RunReport( - status="success", - config={"model": "m", "base_url": "b"}, - normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0}, - pipeline=["grammar"], - modules=[], - applied_changes=[], - skipped_corrections=[], - totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0}, - work_dir_retention="auto", - work_dir_retained=False, - work_dir=None, - error=None, - ) - result = ProcessResult( - transcript=transcript, - report=report, - run_dir=tmp_path / "run", - work_dir_retained=False, - ) - - def _fake_from_sources(*, overrides=None): - captured["target_sections"] = overrides.target_sections - return object() - - monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources) - monkeypatch.setattr("audita.cli.load_transcript", lambda path: []) - monkeypatch.setattr("audita.cli.load_glossary", lambda path: object()) - monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result) - - exit_code = main( - [ - "process", - "transcript.json", - "--glossary", - "glossary.yaml", - "--target-sections", - "4", - ] - ) - - assert exit_code == 0 - assert captured["target_sections"] == 4 - - -def test_cli_process_passes_min_section_tokens_override_to_config(monkeypatch, tmp_path): - captured = {} - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."} - ] - """ - ) - report = RunReport( - status="success", - config={"model": "m", "base_url": "b"}, - normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0}, - pipeline=["grammar"], - modules=[], - applied_changes=[], - skipped_corrections=[], - totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0}, - work_dir_retention="auto", - work_dir_retained=False, - work_dir=None, - error=None, - ) - result = ProcessResult( - transcript=transcript, - report=report, - run_dir=tmp_path / "run", - work_dir_retained=False, - ) - - def _fake_from_sources(*, overrides=None): - captured["min_section_tokens"] = overrides.min_section_tokens - return object() - - monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources) - monkeypatch.setattr("audita.cli.load_transcript", lambda path: []) - monkeypatch.setattr("audita.cli.load_glossary", lambda path: object()) - monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result) - - exit_code = main( - [ - "process", - "transcript.json", - "--glossary", - "glossary.yaml", - "--min-section-tokens", - "5000", - ] - ) - - assert exit_code == 0 - assert captured["min_section_tokens"] == 5000 - - -def test_cli_process_writes_failure_diagnostics_and_keeps_stdout_empty(monkeypatch, tmp_path, capsys): - monkeypatch.setattr( - "audita.cli.AuditaConfig.from_sources", - lambda overrides=None: (_ for _ in ()).throw(AuditaConfigError("bad config")), - ) - - report_path = tmp_path / "external-report.json" - exit_code = main( - [ - "process", - "transcript.json", - "--glossary", - "glossary.yaml", - "--work-dir", - str(tmp_path / "work"), - "--report-json", - str(report_path), - ] - ) - - captured = capsys.readouterr() - assert exit_code == 1 - assert captured.out == "" - assert "audita: error: bad config" in captured.err - assert "audita: exit code: 1" in captured.err - run_dir = next((tmp_path / "work").iterdir()) - assert f"audita: run directory: {run_dir}" in captured.err - assert f"audita: error log: {run_dir / 'error.log'}" in captured.err - assert f"audita: report: {run_dir / 'report.json'}" in captured.err - assert (run_dir / "error.log").exists() - report = json.loads((run_dir / "report.json").read_text(encoding="utf-8")) - assert report["status"] == "failed" - assert report["error"] == "bad config" - assert report["error_details"]["phase"] == "config" - assert report["error_details"]["type"] == "AuditaConfigError" - external = json.loads(report_path.read_text(encoding="utf-8")) - assert external["error"] == "bad config" - - -def test_cli_process_writes_failure_diagnostics_for_unexpected_exceptions(monkeypatch, tmp_path, capsys): - monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", lambda overrides=None: object()) - monkeypatch.setattr("audita.cli.load_transcript", lambda path: (_ for _ in ()).throw(RuntimeError("boom"))) - - exit_code = main( - [ - "process", - "transcript.json", - "--glossary", - "glossary.yaml", - "--work-dir", - str(tmp_path / "work"), - ] - ) - - captured = capsys.readouterr() - assert exit_code == 1 - assert captured.out == "" - assert "audita: error: boom" in captured.err - run_dir = next((tmp_path / "work").iterdir()) - report = json.loads((run_dir / "report.json").read_text(encoding="utf-8")) - assert report["status"] == "failed" - assert report["error_details"]["phase"] == "transcript_load" - assert report["error_details"]["type"] == "RuntimeError" - assert "RuntimeError: boom" in (run_dir / "error.log").read_text(encoding="utf-8") - - -class _BrokenWriter: - def write(self, _message): - raise OSError(9, "Bad file descriptor") - - def flush(self): - raise OSError(9, "Bad file descriptor") - - -def test_cli_process_progress_falls_back_to_stdout_when_stderr_is_invalid(monkeypatch, tmp_path): - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."} - ] - """ - ) - report = RunReport( - status="success", - config={"model": "m", "base_url": "b"}, - normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0}, - pipeline=["grammar"], - modules=[], - applied_changes=[], - skipped_corrections=[], - totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0}, - work_dir_retention="auto", - work_dir_retained=False, - work_dir=None, - error=None, - ) - result = ProcessResult( - transcript=transcript, - report=report, - run_dir=tmp_path / "run", - work_dir_retained=False, - ) - - def _fake_pipeline(*args, **kwargs): - kwargs["progress"]("progress-line") - return result - - stdout_capture = io.StringIO() - monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", lambda overrides=None: object()) - monkeypatch.setattr("audita.cli.load_transcript", lambda path: []) - monkeypatch.setattr("audita.cli.load_glossary", lambda path: object()) - monkeypatch.setattr("audita.cli.process_transcript_result", _fake_pipeline) - monkeypatch.setattr("audita.cli.sys.stderr", _BrokenWriter()) - monkeypatch.setattr("audita.cli.sys.stdout", stdout_capture) - - output_path = tmp_path / "out.json" - exit_code = main( - [ - "process", - "transcript.json", - "--glossary", - "glossary.yaml", - "--output", - str(output_path), - ] - ) - - assert exit_code == 0 - assert "progress-line" in stdout_capture.getvalue() - - -def test_cli_process_failure_summary_falls_back_to_stdout_when_stderr_is_invalid(monkeypatch, tmp_path): - stdout_capture = io.StringIO() - monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", lambda overrides=None: object()) - monkeypatch.setattr("audita.cli.load_transcript", lambda path: (_ for _ in ()).throw(RuntimeError("boom"))) - monkeypatch.setattr("audita.cli.sys.stderr", _BrokenWriter()) - monkeypatch.setattr("audita.cli.sys.stdout", stdout_capture) - - exit_code = main( - [ - "process", - "transcript.json", - "--glossary", - "glossary.yaml", - "--work-dir", - str(tmp_path / "work"), - ] - ) - - assert exit_code == 1 - assert "audita: error: boom" in stdout_capture.getvalue() diff --git a/python/tests/test_new_config.py b/python/tests/test_new_config.py deleted file mode 100644 index 4bdceee..0000000 --- a/python/tests/test_new_config.py +++ /dev/null @@ -1,435 +0,0 @@ -import pytest - -from audita.core.config import ( - AuditaConfig, - ConfigOverrides, - DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD, - DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD, - DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD, - DEFAULT_LLM_CONCURRENCY, - DEFAULT_LLM_TIMEOUT_SECONDS, - DEFAULT_MAX_SECTION_TOKENS, - DEFAULT_MIN_SECTION_TOKENS, - DEFAULT_NORMALIZE_MAX_SEGMENT_GAP, - DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD, - DEFAULT_VALIDATION_MAX_PROMPT_TOKENS, - DEFAULT_WORK_DIR_RETENTION, -) -from audita.core.errors import AuditaConfigError -from audita.modules import DEFAULT_MODULE_KEYS - - -def test_default_config_allows_missing_api_key(): - config = AuditaConfig.from_sources(env={}) - - assert config.api_key is None - assert config.llm_concurrency == DEFAULT_LLM_CONCURRENCY - assert config.llm_timeout_seconds == DEFAULT_LLM_TIMEOUT_SECONDS - assert config.validation_llm_api_key is None - assert config.validation_llm_concurrency is None - assert config.validation_llm_timeout_seconds is None - assert config.validation_model is None - assert config.validation_base_url is None - assert config.validation_max_retries is None - assert config.validation_max_prompt_tokens == DEFAULT_VALIDATION_MAX_PROMPT_TOKENS - assert config.target_sections is None - assert config.max_section_tokens == DEFAULT_MAX_SECTION_TOKENS - assert config.min_section_tokens == DEFAULT_MIN_SECTION_TOKENS - assert config.module_keys == DEFAULT_MODULE_KEYS - assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD - assert config.grammar_confidence_threshold == DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD - assert config.homophones_confidence_threshold == DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD - assert config.spoken_word_confidence_threshold == DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD - assert config.normalize_max_segment_gap == DEFAULT_NORMALIZE_MAX_SEGMENT_GAP - assert config.work_dir_retention == DEFAULT_WORK_DIR_RETENTION - - -def test_cli_overrides_take_precedence(): - config = AuditaConfig.from_sources( - env={"AUDITA_MAX_SECTION_TOKENS": "1000"}, - overrides=ConfigOverrides(max_section_tokens=2000, min_section_tokens=1000), - ) - - assert config.max_section_tokens == 2000 - - -def test_min_section_tokens_cli_override_takes_precedence(): - config = AuditaConfig.from_sources( - env={"AUDITA_MIN_SECTION_TOKENS": "2000"}, - overrides=ConfigOverrides(min_section_tokens=6000), - ) - - assert config.min_section_tokens == 6000 - - -def test_llm_concurrency_cli_override_takes_precedence(): - config = AuditaConfig.from_sources( - env={"AUDITA_LLM_CONCURRENCY": "2"}, - overrides=ConfigOverrides(llm_concurrency=4), - ) - - assert config.llm_concurrency == 4 - - -def test_llm_concurrency_env_is_parsed(): - config = AuditaConfig.from_sources(env={"AUDITA_LLM_CONCURRENCY": "3"}) - - assert config.llm_concurrency == 3 - - -def test_llm_timeout_seconds_cli_override_takes_precedence(): - config = AuditaConfig.from_sources( - env={"AUDITA_LLM_TIMEOUT_SECONDS": "120"}, - overrides=ConfigOverrides(llm_timeout_seconds=900.0), - ) - - assert config.llm_timeout_seconds == 900.0 - - -def test_llm_timeout_seconds_env_is_parsed(): - config = AuditaConfig.from_sources(env={"AUDITA_LLM_TIMEOUT_SECONDS": "120.5"}) - - assert config.llm_timeout_seconds == 120.5 - - -def test_validation_llm_concurrency_cli_override_takes_precedence(): - config = AuditaConfig.from_sources( - env={"AUDITA_VALIDATION_LLM_CONCURRENCY": "2"}, - overrides=ConfigOverrides(validation_llm_concurrency=4), - ) - - assert config.validation_llm_concurrency == 4 - - -def test_validation_llm_concurrency_env_is_parsed(): - config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_CONCURRENCY": "3"}) - - assert config.validation_llm_concurrency == 3 - - -def test_validation_llm_timeout_seconds_cli_override_takes_precedence(): - config = AuditaConfig.from_sources( - env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "120"}, - overrides=ConfigOverrides(validation_llm_timeout_seconds=900.0), - ) - - assert config.validation_llm_timeout_seconds == 900.0 - - -def test_validation_llm_timeout_seconds_env_is_parsed(): - config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "120.5"}) - - assert config.validation_llm_timeout_seconds == 120.5 - - -def test_validation_model_env_is_parsed(): - config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MODEL": "validator-model"}) - - assert config.validation_model == "validator-model" - - -def test_validation_base_url_env_is_parsed(): - config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_BASE_URL": "http://localhost:9000/v1"}) - - assert config.validation_base_url == "http://localhost:9000/v1" - - -def test_validation_max_retries_env_is_parsed(): - config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_RETRIES": "7"}) - - assert config.validation_max_retries == 7 - - -def test_validation_max_prompt_tokens_cli_override_takes_precedence(): - config = AuditaConfig.from_sources( - env={"AUDITA_VALIDATION_MAX_PROMPT_TOKENS": "1024"}, - overrides=ConfigOverrides(validation_max_prompt_tokens=4096), - ) - - assert config.validation_max_prompt_tokens == 4096 - - -def test_validation_max_prompt_tokens_env_is_parsed(): - config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_PROMPT_TOKENS": "1024"}) - - assert config.validation_max_prompt_tokens == 1024 - - -def test_target_sections_cli_override_takes_precedence(): - config = AuditaConfig.from_sources( - env={"AUDITA_TARGET_SECTIONS": "2"}, - overrides=ConfigOverrides(target_sections=5), - ) - - assert config.target_sections == 5 - - -def test_target_sections_env_is_parsed(): - config = AuditaConfig.from_sources(env={"AUDITA_TARGET_SECTIONS": "3"}) - - assert config.target_sections == 3 - - -def test_min_section_tokens_env_is_parsed(): - config = AuditaConfig.from_sources(env={"AUDITA_MIN_SECTION_TOKENS": "3000"}) - - assert config.min_section_tokens == 3000 - - -def test_generic_llm_api_key_env_is_read(): - config = AuditaConfig.from_sources(env={"AUDITA_LLM_API_KEY": "generic-key"}) - - assert config.api_key == "generic-key" - - -def test_generic_llm_api_key_takes_precedence_over_openrouter_env(): - config = AuditaConfig.from_sources( - env={ - "AUDITA_LLM_API_KEY": "generic-key", - "OPENROUTER_API_KEY": "legacy-key", - } - ) - - assert config.api_key == "generic-key" - - -def test_llm_api_key_cli_override_takes_precedence_over_env(): - config = AuditaConfig.from_sources( - env={ - "AUDITA_LLM_API_KEY": "generic-key", - "OPENROUTER_API_KEY": "legacy-key", - }, - overrides=ConfigOverrides(llm_api_key="cli-key"), - ) - - assert config.api_key == "cli-key" - - -def test_blank_llm_api_key_override_resolves_to_none(): - config = AuditaConfig.from_sources( - env={"OPENROUTER_API_KEY": "legacy-key"}, - overrides=ConfigOverrides(llm_api_key=" "), - ) - - assert config.api_key is None - - -def test_validation_llm_api_key_env_is_read(): - config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_API_KEY": "validation-key"}) - - assert config.validation_llm_api_key == "validation-key" - - -def test_blank_validation_llm_api_key_override_disables_primary_fallback(): - config = AuditaConfig.from_sources( - env={"AUDITA_LLM_API_KEY": "primary-key"}, - overrides=ConfigOverrides(validation_llm_api_key=" "), - ) - - assert config.validation_llm_api_key == "" - assert config.validation_llm_config().api_key == "" - - -def test_effective_validation_fields_fall_back_to_primary_settings(): - config = AuditaConfig.from_sources( - env={ - "AUDITA_LLM_API_KEY": "primary-key", - "AUDITA_MODEL": "primary-model", - "AUDITA_BASE_URL": "http://localhost:8000/v1", - "AUDITA_LLM_TIMEOUT_SECONDS": "120", - "AUDITA_LLM_CONCURRENCY": "5", - "AUDITA_MAX_RETRIES": "9", - } - ) - - validation = config.validation_llm_config() - - assert validation.api_key == "primary-key" - assert validation.model == "primary-model" - assert validation.base_url == "http://localhost:8000/v1" - assert validation.llm_timeout_seconds == 120.0 - assert validation.llm_concurrency == 5 - assert validation.max_retries == 9 - - -def test_effective_validation_fields_use_overrides_when_set(): - config = AuditaConfig.from_sources( - env={ - "AUDITA_LLM_API_KEY": "primary-key", - "AUDITA_MODEL": "primary-model", - "AUDITA_BASE_URL": "http://localhost:8000/v1", - "AUDITA_LLM_TIMEOUT_SECONDS": "120", - "AUDITA_LLM_CONCURRENCY": "5", - "AUDITA_MAX_RETRIES": "9", - "AUDITA_VALIDATION_LLM_API_KEY": "validation-key", - "AUDITA_VALIDATION_MODEL": "validation-model", - "AUDITA_VALIDATION_BASE_URL": "http://localhost:9000/v1", - "AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "240", - "AUDITA_VALIDATION_LLM_CONCURRENCY": "3", - "AUDITA_VALIDATION_MAX_RETRIES": "2", - } - ) - - validation = config.validation_llm_config() - - assert validation.api_key == "validation-key" - assert validation.model == "validation-model" - assert validation.base_url == "http://localhost:9000/v1" - assert validation.llm_timeout_seconds == 240.0 - assert validation.llm_concurrency == 3 - assert validation.max_retries == 2 - - -def test_module_key_overrides_take_precedence(): - config = AuditaConfig.from_sources( - env={"AUDITA_MODULES": "grammar"}, - overrides=ConfigOverrides(module_keys="homophones,grammar"), - ) - - assert config.module_keys == ("homophones", "grammar") - - -def test_module_key_env_is_parsed_and_trimmed(): - config = AuditaConfig.from_sources(env={"AUDITA_MODULES": " glossary , grammar "}) - - assert config.module_keys == ("glossary", "grammar") - - -def test_invalid_work_dir_retention_is_rejected(): - with pytest.raises(AuditaConfigError): - AuditaConfig.from_sources(env={"AUDITA_WORK_DIR_RETENTION": "sometimes"}) - - -def test_report_dict_includes_llm_timeout_seconds(): - config = AuditaConfig.from_sources(env={"AUDITA_LLM_TIMEOUT_SECONDS": "321"}) - - assert config.to_report_dict()["llm_timeout_seconds"] == 321.0 - - -def test_report_dict_includes_target_sections(): - config = AuditaConfig.from_sources(env={"AUDITA_TARGET_SECTIONS": "7"}) - - assert config.to_report_dict()["target_sections"] == 7 - - -def test_report_dict_includes_effective_validation_llm_config(): - config = AuditaConfig.from_sources( - env={ - "AUDITA_LLM_API_KEY": "primary-key", - "AUDITA_VALIDATION_MODEL": "validation-model", - } - ) - - report = config.to_report_dict() - - assert report["validation_model"] == "validation-model" - assert report["validation_max_prompt_tokens"] == DEFAULT_VALIDATION_MAX_PROMPT_TOKENS - assert report["effective_validation_llm"]["api_key_configured"] is True - assert report["effective_validation_llm"]["model"] == "validation-model" - - -def test_threshold_overrides_take_precedence(): - config = AuditaConfig.from_sources( - env={ - "AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.6", - "AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.65", - "AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.7", - "AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.75", - }, - overrides=ConfigOverrides( - glossary_confidence_threshold=0.85, - grammar_confidence_threshold=0.88, - homophones_confidence_threshold=0.9, - spoken_word_confidence_threshold=0.95, - ), - ) - - assert config.glossary_confidence_threshold == 0.85 - assert config.grammar_confidence_threshold == 0.88 - assert config.homophones_confidence_threshold == 0.9 - assert config.spoken_word_confidence_threshold == 0.95 - - -@pytest.mark.parametrize( - "value", - [ - "", - "grammar,,homophones", - "bogus", - ], -) -def test_invalid_module_sequences_are_rejected(value): - with pytest.raises(AuditaConfigError, match="AUDITA_MODULES"): - AuditaConfig.from_sources(env={"AUDITA_MODULES": value}) - - -@pytest.mark.parametrize( - "env_name", - [ - "AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD", - "AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD", - "AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD", - "AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD", - ], -) -def test_invalid_thresholds_are_rejected(env_name): - with pytest.raises(AuditaConfigError): - AuditaConfig.from_sources(env={env_name: "1.5"}) - - -@pytest.mark.parametrize("value", ["0", "-1", "many"]) -def test_invalid_llm_concurrency_is_rejected(value): - with pytest.raises(AuditaConfigError, match="AUDITA_LLM_CONCURRENCY"): - AuditaConfig.from_sources(env={"AUDITA_LLM_CONCURRENCY": value}) - - -@pytest.mark.parametrize("value", ["0", "-1", "many"]) -def test_invalid_llm_timeout_seconds_is_rejected(value): - with pytest.raises(AuditaConfigError, match="AUDITA_LLM_TIMEOUT_SECONDS"): - AuditaConfig.from_sources(env={"AUDITA_LLM_TIMEOUT_SECONDS": value}) - - -@pytest.mark.parametrize("value", ["0", "-1", "many"]) -def test_invalid_validation_llm_concurrency_is_rejected(value): - with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_LLM_CONCURRENCY"): - AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_CONCURRENCY": value}) - - -@pytest.mark.parametrize("value", ["0", "-1", "many"]) -def test_invalid_validation_llm_timeout_seconds_is_rejected(value): - with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"): - AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": value}) - - -@pytest.mark.parametrize("value", ["-1", "many"]) -def test_invalid_validation_max_retries_is_rejected(value): - with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_MAX_RETRIES"): - AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_RETRIES": value}) - - -@pytest.mark.parametrize("value", ["0", "-1", "many"]) -def test_invalid_validation_max_prompt_tokens_is_rejected(value): - with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_MAX_PROMPT_TOKENS"): - AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_PROMPT_TOKENS": value}) - - -@pytest.mark.parametrize("value", ["0", "-1", "many"]) -def test_invalid_target_sections_is_rejected(value): - with pytest.raises(AuditaConfigError, match="AUDITA_TARGET_SECTIONS"): - AuditaConfig.from_sources(env={"AUDITA_TARGET_SECTIONS": value}) - - -@pytest.mark.parametrize("value", ["0", "-1", "many"]) -def test_invalid_min_section_tokens_is_rejected(value): - with pytest.raises(AuditaConfigError, match="AUDITA_MIN_SECTION_TOKENS"): - AuditaConfig.from_sources(env={"AUDITA_MIN_SECTION_TOKENS": value}) - - -def test_min_section_tokens_must_not_exceed_max_section_tokens(): - with pytest.raises(AuditaConfigError, match="AUDITA_MIN_SECTION_TOKENS"): - AuditaConfig.from_sources( - env={ - "AUDITA_MIN_SECTION_TOKENS": "9000", - "AUDITA_MAX_SECTION_TOKENS": "8000", - } - ) diff --git a/python/tests/test_new_launcher.py b/python/tests/test_new_launcher.py deleted file mode 100644 index c9b46b5..0000000 --- a/python/tests/test_new_launcher.py +++ /dev/null @@ -1,117 +0,0 @@ -import importlib.machinery -import importlib.util -import io -import os -import shutil -import subprocess -import sys -from pathlib import Path - -import pytest - - -ROOT = Path(__file__).resolve().parents[1] -LAUNCHER = ROOT / "audita" - - -def _load_launcher_module(): - loader = importlib.machinery.SourceFileLoader("audita_launcher", str(LAUNCHER)) - spec = importlib.util.spec_from_file_location("audita_launcher", LAUNCHER, loader=loader) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def test_root_launcher_exists_and_is_executable(): - assert LAUNCHER.is_file() - assert os.access(LAUNCHER, os.X_OK) - - -def test_root_launcher_help_smoke(): - if shutil.which("uv") is None: - pytest.skip("uv is not installed") - - result = subprocess.run( - [str(LAUNCHER), "--help"], - cwd=ROOT, - text=True, - capture_output=True, - check=False, - ) - - assert result.returncode == 0 - assert result.stdout.startswith("usage: audita ") - - -def test_root_launcher_writes_error_log_when_uv_is_missing(tmp_path): - work_dir = tmp_path / "work" - result = subprocess.run( - [sys.executable, str(LAUNCHER), "process", "transcript.json", "--glossary", "glossary.yaml", "--work-dir", str(work_dir)], - cwd=ROOT, - text=True, - capture_output=True, - env={**os.environ, "PATH": ""}, - check=False, - ) - - assert result.returncode == 1 - assert result.stdout == "" - assert "audita: error: uv is required to run this launcher" in result.stderr - run_dir = next(work_dir.iterdir()) - assert f"audita: run directory: {run_dir}" in result.stderr - assert (run_dir / "error.log").exists() - - -def test_root_launcher_preserves_child_exit_code_and_writes_fallback_error_log(tmp_path): - work_dir = tmp_path / "work" - fake_bin = tmp_path / "bin" - fake_bin.mkdir() - fake_uv = fake_bin / "uv" - fake_uv.write_text("#!/bin/sh\nexit 120\n", encoding="utf-8") - fake_uv.chmod(0o755) - - result = subprocess.run( - [sys.executable, str(LAUNCHER), "process", "transcript.json", "--glossary", "glossary.yaml", "--work-dir", str(work_dir)], - cwd=ROOT, - text=True, - capture_output=True, - env={**os.environ, "PATH": str(fake_bin)}, - check=False, - ) - - assert result.returncode == 120 - assert result.stdout == "" - assert "audita: subprocess exited with status 120" in result.stderr - run_dir = next(work_dir.iterdir()) - assert (run_dir / "error.log").exists() - - -class _BrokenWriter: - def write(self, _message): - raise OSError(9, "Bad file descriptor") - - def flush(self): - raise OSError(9, "Bad file descriptor") - - -def test_root_launcher_falls_back_to_stdout_when_stderr_is_invalid(monkeypatch, tmp_path): - launcher = _load_launcher_module() - work_dir = tmp_path / "work" - stdout_capture = io.StringIO() - - monkeypatch.setattr(launcher.shutil, "which", lambda _name: None) - monkeypatch.setattr( - launcher.sys, - "argv", - ["audita", "process", "transcript.json", "--glossary", "glossary.yaml", "--work-dir", str(work_dir)], - ) - monkeypatch.setattr(launcher.sys, "stderr", _BrokenWriter()) - monkeypatch.setattr(launcher.sys, "stdout", stdout_capture) - - exit_code = launcher.main() - - assert exit_code == 1 - assert "audita: error: uv is required to run this launcher" in stdout_capture.getvalue() - run_dir = next(work_dir.iterdir()) - assert (run_dir / "error.log").exists() diff --git a/python/tests/test_new_pipeline.py b/python/tests/test_new_pipeline.py deleted file mode 100644 index f3f9703..0000000 --- a/python/tests/test_new_pipeline.py +++ /dev/null @@ -1,698 +0,0 @@ -import json -import threading - -import pytest - -from audita.core.config import AuditaConfig, ConfigOverrides -from audita.core.errors import AuditaLLMError -from audita.core.io import write_report -from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_json -from audita.modules import DEFAULT_MODULE_KEYS, default_module_specs, resolve_module_specs -from audita.pipeline import process_transcript, process_transcript_result - - -class FakeStructuredLLMClient: - def __init__(self, responses): - self._responses = responses - self._lock = threading.Lock() - self.calls = [] - - def run_structured(self, *, stage_name, messages, response_model, config): - with self._lock: - self.calls.append( - { - "stage_name": stage_name, - "messages": list(messages), - "response_model": response_model, - "config": config, - } - ) - response = _pop_llm_response(self._responses, stage_name) - if isinstance(response, Exception): - raise response - return response_model.model_validate(response) - - -def _pop_llm_response(responses, stage_name): - if isinstance(responses, dict): - if stage_name not in responses: - raise AuditaLLMError(f"FakeStructuredLLMClient received unexpected stage_name: {stage_name}") - payloads = responses[stage_name] - if isinstance(payloads, list): - if not payloads: - raise AuditaLLMError(f"FakeStructuredLLMClient received too many calls for stage_name: {stage_name}") - return payloads.pop(0) - payload = payloads - del responses[stage_name] - return payload - if not responses: - raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.") - return responses.pop(0) - - -def _glossary(): - return parse_glossary_yaml( - """ - glossary: - - name: "Jesters" - category: faction - summary: "A faction." - """ - ) - - -def _transcript(): - return parse_source_transcript_json( - """ - [ - {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hello.", "categories": ["intro"]}, - {"speaker": "Eric", "start": 1.5, "end": 2.0, "text": "Again.", "categories": ["intro", "aside"]}, - {"speaker": "Mike", "start": 10.0, "end": 11.0, "text": "Done.", "categories": ["response"]} - ] - """ - ) - - -def test_process_transcript_runs_noop_framework(tmp_path): - llm_client = FakeStructuredLLMClient( - [ - {"corrections": []}, - {"corrections": []}, - {"corrections": []}, - {"corrections": []}, - {"corrections": []}, - ] - ) - revised = process_transcript( - _transcript(), - _glossary(), - AuditaConfig.from_sources(env={}, overrides=None), - llm_client=llm_client, - ) - - assert [segment.id for segment in revised] == [1, 2] - assert revised[0].text == "Hello. Again." - assert revised[1].text == "Done." - assert revised[0].categories == ["intro", "aside"] - assert revised[1].categories == ["response"] - assert [call["stage_name"] for call in llm_client.calls] == [ - "glossary_1:proposal", - "homophones:proposal", - "glossary_2:proposal", - "spoken_word:proposal", - "grammar:proposal", - ] - - -def test_process_transcript_result_can_use_different_validation_llm_settings(tmp_path): - llm_client = FakeStructuredLLMClient( - { - "grammar:proposal": { - "corrections": [ - { - "id": 1, - "original_text": "hello world", - "corrected_text": "Hello world.", - "confidence": 0.95, - } - ] - }, - "grammar:grammar_only_guard": { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.98, - "reason": "ok", - } - ] - }, - "grammar:meaning_reversal_review": { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.99, - "reason": "ok", - } - ] - }, - } - ) - config = AuditaConfig.from_sources( - env={"OPENROUTER_API_KEY": "primary-key"}, - overrides=ConfigOverrides( - model="primary-model", - base_url="http://localhost:8000/v1", - max_retries=7, - llm_timeout_seconds=120, - validation_llm_api_key="validation-key", - validation_model="validation-model", - validation_base_url="http://localhost:9000/v1", - validation_max_retries=2, - validation_llm_timeout_seconds=240, - validation_llm_concurrency=3, - work_dir=tmp_path / "work", - work_dir_retention="always", - ), - ) - - result = process_transcript_result( - parse_source_transcript_json( - """ - [ - {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} - ] - """ - ), - _glossary(), - config, - module_keys=["grammar"], - llm_client=llm_client, - ) - - assert result.transcript[0].text == "Hello world." - calls_by_stage = {call["stage_name"]: call["config"] for call in llm_client.calls} - assert calls_by_stage["grammar:proposal"].model == "primary-model" - assert calls_by_stage["grammar:proposal"].base_url == "http://localhost:8000/v1" - assert calls_by_stage["grammar:proposal"].api_key == "primary-key" - assert calls_by_stage["grammar:grammar_only_guard"].model == "validation-model" - assert calls_by_stage["grammar:grammar_only_guard"].base_url == "http://localhost:9000/v1" - assert calls_by_stage["grammar:grammar_only_guard"].api_key == "validation-key" - assert calls_by_stage["grammar:grammar_only_guard"].max_retries == 2 - assert calls_by_stage["grammar:grammar_only_guard"].llm_timeout_seconds == 240 - - -def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(tmp_path): - config = AuditaConfig.from_sources( - env={}, - overrides=None, - ) - config = AuditaConfig( - api_key=config.api_key, - model=config.model, - base_url=config.base_url, - max_retries=config.max_retries, - max_section_tokens=config.max_section_tokens, - glossary_confidence_threshold=config.glossary_confidence_threshold, - grammar_confidence_threshold=config.grammar_confidence_threshold, - homophones_confidence_threshold=config.homophones_confidence_threshold, - spoken_word_confidence_threshold=config.spoken_word_confidence_threshold, - normalize_max_segment_gap=config.normalize_max_segment_gap, - normalize_ellipsis_gap=config.normalize_ellipsis_gap, - normalize_max_segment_duration=config.normalize_max_segment_duration, - normalize_max_segment_tokens=config.normalize_max_segment_tokens, - work_dir=tmp_path / "work", - work_dir_retention="always", - ) - - llm_client = FakeStructuredLLMClient( - [ - {"corrections": []}, - {"corrections": []}, - {"corrections": []}, - {"corrections": []}, - {"corrections": []}, - ] - ) - result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client) - - assert result.work_dir_retained is True - assert result.report.pipeline == [ - "glossary_1", - "homophones", - "glossary_2", - "spoken_word", - "grammar", - ] - assert result.report.totals["applied_change_count"] == 0 - assert (result.run_dir / "report.json").exists() - assert (result.run_dir / "normalization" / "summary.json").exists() - assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ - "identical_text_guard", - "original_text_present_guard", - "proposal_confidence_guard", - "glossary_stage_protected_glossary_guard", - "non_empty_segment_guard", - "spoken_form_plausibility_review", - "meaning_reversal_review", - ] - assert [validator["name"] for validator in result.report.modules[2].to_dict()["validators"]] == [ - "identical_text_guard", - "original_text_present_guard", - "proposal_confidence_guard", - "glossary_stage_protected_glossary_guard", - "non_empty_segment_guard", - "spoken_form_plausibility_review", - "meaning_reversal_review", - ] - assert [validator["name"] for validator in result.report.modules[3].to_dict()["validators"]] == [ - "identical_text_guard", - "original_text_present_guard", - "proposal_confidence_guard", - "protected_glossary_guard", - "non_empty_segment_guard", - "spoken_word_review", - "meaning_reversal_review", - ] - assert [validator["name"] for validator in result.report.modules[4].to_dict()["validators"]] == [ - "identical_text_guard", - "original_text_present_guard", - "proposal_confidence_guard", - "protected_glossary_guard", - "non_empty_segment_guard", - "grammar_only_guard", - "meaning_reversal_review", - ] - - -def test_external_report_can_be_written(tmp_path): - config = AuditaConfig.from_sources(env={}) - llm_client = FakeStructuredLLMClient( - [ - {"corrections": []}, - {"corrections": []}, - {"corrections": []}, - {"corrections": []}, - {"corrections": []}, - ] - ) - result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client) - report_path = tmp_path / "report.json" - write_report(report_path, result.report) - - payload = json.loads(report_path.read_text(encoding="utf-8")) - assert payload["pipeline"][0] == "glossary_1" - assert payload["totals"]["applied_change_count"] == 0 - - -def test_process_transcript_preserves_categories_in_llm_prompt_payloads(tmp_path): - llm_client = FakeStructuredLLMClient( - [ - {"corrections": []}, - {"corrections": []}, - {"corrections": []}, - {"corrections": []}, - {"corrections": []}, - ] - ) - - process_transcript( - _transcript(), - _glossary(), - AuditaConfig.from_sources(env={}, overrides=None), - llm_client=llm_client, - ) - - proposal_prompt = llm_client.calls[0]["messages"][1]["content"] - assert '"categories": [' in proposal_prompt - assert '"intro"' in proposal_prompt - assert '"aside"' in proposal_prompt - - -def test_default_module_specs_expose_final_validator_order(): - specs = default_module_specs() - - assert DEFAULT_MODULE_KEYS == ("glossary", "homophones", "glossary", "spoken_word", "grammar") - assert [spec.instance_name for spec in specs] == [ - "glossary_1", - "homophones", - "glossary_2", - "spoken_word", - "grammar", - ] - assert [validator.name for validator in specs[0].module.validators()] == [ - "identical_text_guard", - "original_text_present_guard", - "proposal_confidence_guard", - "glossary_stage_protected_glossary_guard", - "non_empty_segment_guard", - "spoken_form_plausibility_review", - "meaning_reversal_review", - ] - assert [validator.name for validator in specs[1].module.validators()] == [ - "identical_text_guard", - "original_text_present_guard", - "proposal_confidence_guard", - "protected_glossary_guard", - "non_empty_segment_guard", - "spoken_form_plausibility_review", - "meaning_reversal_review", - ] - assert [validator.name for validator in specs[2].module.validators()] == [ - "identical_text_guard", - "original_text_present_guard", - "proposal_confidence_guard", - "glossary_stage_protected_glossary_guard", - "non_empty_segment_guard", - "spoken_form_plausibility_review", - "meaning_reversal_review", - ] - assert [validator.name for validator in specs[3].module.validators()] == [ - "identical_text_guard", - "original_text_present_guard", - "proposal_confidence_guard", - "protected_glossary_guard", - "non_empty_segment_guard", - "spoken_word_review", - "meaning_reversal_review", - ] - assert [validator.name for validator in specs[4].module.validators()] == [ - "identical_text_guard", - "original_text_present_guard", - "proposal_confidence_guard", - "protected_glossary_guard", - "non_empty_segment_guard", - "grammar_only_guard", - "meaning_reversal_review", - ] - - -def test_process_transcript_result_missing_api_key_writes_failed_report(tmp_path): - config = AuditaConfig.from_sources( - env={}, - overrides=None, - ) - config = AuditaConfig( - api_key=None, - model=config.model, - base_url=config.base_url, - max_retries=config.max_retries, - max_section_tokens=config.max_section_tokens, - glossary_confidence_threshold=config.glossary_confidence_threshold, - grammar_confidence_threshold=config.grammar_confidence_threshold, - homophones_confidence_threshold=config.homophones_confidence_threshold, - spoken_word_confidence_threshold=config.spoken_word_confidence_threshold, - normalize_max_segment_gap=config.normalize_max_segment_gap, - normalize_ellipsis_gap=config.normalize_ellipsis_gap, - normalize_max_segment_duration=config.normalize_max_segment_duration, - normalize_max_segment_tokens=config.normalize_max_segment_tokens, - work_dir=tmp_path / "work", - work_dir_retention="never", - ) - - with pytest.raises(AuditaLLMError, match="OpenRouter endpoint"): - process_transcript_result(_transcript(), _glossary(), config) - - run_dir = next((tmp_path / "work").iterdir()) - report = json.loads((run_dir / "report.json").read_text(encoding="utf-8")) - - assert report["status"] == "failed" - assert report["normalization"]["normalized_segment_count"] == 2 - assert report["pipeline"] == [ - "glossary_1", - "homophones", - "glossary_2", - "spoken_word", - "grammar", - ] - assert report["modules"] == [] - assert report["applied_changes"] == [] - assert report["skipped_corrections"] == [] - assert report["work_dir_retained"] is True - assert report["work_dir"] == str(run_dir) - assert "AUDITA_LLM_API_KEY" in report["error"] - assert "OPENROUTER_API_KEY" in report["error"] - assert "OpenRouter endpoint" in report["error"] - assert report["error_details"]["type"] == "AuditaLLMError" - assert report["error_details"]["phase"] == "pipeline" - assert report["error_details"]["error_log"] == str(run_dir / "error.log") - assert (run_dir / "error.log").exists() - - -def test_process_transcript_result_allows_missing_api_key_for_nondefault_proposal_endpoint(tmp_path): - llm_client = FakeStructuredLLMClient( - [ - {"corrections": []}, - {"corrections": []}, - {"corrections": []}, - {"corrections": []}, - {"corrections": []}, - ] - ) - config = AuditaConfig.from_sources( - env={}, - overrides=ConfigOverrides( - base_url="http://localhost:8000/v1", - model="meta-llama/Llama-3.1-8B-Instruct", - work_dir=tmp_path / "work", - work_dir_retention="always", - ), - ) - - result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client) - - assert result.report.status == "success" - assert llm_client.calls[0]["config"].api_key is None - assert llm_client.calls[0]["config"].base_url == "http://localhost:8000/v1" - - -def test_process_transcript_result_allows_missing_validation_api_key_for_nondefault_validation_endpoint(tmp_path): - llm_client = FakeStructuredLLMClient( - { - "grammar:proposal": { - "corrections": [ - { - "id": 1, - "original_text": "hello world", - "corrected_text": "Hello world.", - "confidence": 0.95, - } - ] - }, - "grammar:grammar_only_guard": { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.98, - "reason": "ok", - } - ] - }, - "grammar:meaning_reversal_review": { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.99, - "reason": "ok", - } - ] - }, - } - ) - config = AuditaConfig.from_sources( - env={"OPENROUTER_API_KEY": "primary-key"}, - overrides=ConfigOverrides( - work_dir=tmp_path / "work", - work_dir_retention="always", - validation_llm_api_key=" ", - validation_base_url="http://localhost:9000/v1", - validation_model="meta-llama/Llama-3.1-8B-Instruct", - ), - ) - - result = process_transcript_result( - parse_source_transcript_json( - """ - [ - {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} - ] - """ - ), - _glossary(), - config, - module_keys=["grammar"], - llm_client=llm_client, - ) - - assert result.transcript[0].text == "Hello world." - calls_by_stage = {call["stage_name"]: call["config"] for call in llm_client.calls} - assert calls_by_stage["grammar:proposal"].api_key == "primary-key" - assert calls_by_stage["grammar:grammar_only_guard"].api_key == "" - assert calls_by_stage["grammar:grammar_only_guard"].base_url == "http://localhost:9000/v1" - - -def test_process_transcript_result_preserves_partial_progress_when_later_module_fails(tmp_path): - transcript = parse_source_transcript_json( - """ - [ - {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the dam."} - ] - """ - ) - config = AuditaConfig.from_sources( - env={}, - overrides=None, - ) - config = AuditaConfig( - api_key=config.api_key, - model=config.model, - base_url=config.base_url, - max_retries=config.max_retries, - max_section_tokens=config.max_section_tokens, - glossary_confidence_threshold=config.glossary_confidence_threshold, - grammar_confidence_threshold=config.grammar_confidence_threshold, - homophones_confidence_threshold=config.homophones_confidence_threshold, - spoken_word_confidence_threshold=config.spoken_word_confidence_threshold, - normalize_max_segment_gap=config.normalize_max_segment_gap, - normalize_ellipsis_gap=config.normalize_ellipsis_gap, - normalize_max_segment_duration=config.normalize_max_segment_duration, - normalize_max_segment_tokens=config.normalize_max_segment_tokens, - work_dir=tmp_path / "work", - work_dir_retention="never", - ) - llm_client = FakeStructuredLLMClient( - [ - { - "corrections": [ - { - "id": 1, - "original_text": "gestures", - "corrected_text": "Jesters", - "confidence": 0.95, - } - ] - }, - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.97, - "reason": "Likely spoken-form correction in context.", - } - ] - }, - { - "validations": [ - { - "correction_index": 0, - "approved": True, - "confidence": 0.99, - "reason": "Does not reverse the segment meaning.", - } - ] - }, - AuditaLLMError("Simulated homophones proposal failure."), - ] - ) - - with pytest.raises(AuditaLLMError, match="Simulated homophones proposal failure"): - process_transcript_result(transcript, _glossary(), config, llm_client=llm_client) - - run_dir = next((tmp_path / "work").iterdir()) - report = json.loads((run_dir / "report.json").read_text(encoding="utf-8")) - - assert report["status"] == "failed" - assert report["normalization"]["normalized_segment_count"] == 1 - assert [module["instance_name"] for module in report["modules"]] == ["glossary_1"] - assert report["applied_changes"][0]["corrected_text"] == "Jesters" - assert report["applied_changes"][0]["segment_text_after"] == "There were Jesters at the dam." - assert report["totals"]["applied_change_count"] == 1 - assert report["skipped_corrections"] == [] - assert report["pipeline"][1] == "homophones" - assert "Simulated homophones proposal failure." in report["error"] - assert report["error_details"]["module_instance"] == "homophones" - assert report["error_details"]["phase"] == "pipeline" - assert (run_dir / "error.log").exists() - - -def test_process_transcript_result_preserves_partial_skips_and_validator_diagnostics_on_failure(tmp_path): - config = AuditaConfig.from_sources( - env={}, - overrides=None, - ) - config = AuditaConfig( - api_key=config.api_key, - model=config.model, - base_url=config.base_url, - max_retries=config.max_retries, - max_section_tokens=config.max_section_tokens, - glossary_confidence_threshold=0.8, - grammar_confidence_threshold=config.grammar_confidence_threshold, - homophones_confidence_threshold=config.homophones_confidence_threshold, - spoken_word_confidence_threshold=config.spoken_word_confidence_threshold, - normalize_max_segment_gap=config.normalize_max_segment_gap, - normalize_ellipsis_gap=config.normalize_ellipsis_gap, - normalize_max_segment_duration=config.normalize_max_segment_duration, - normalize_max_segment_tokens=config.normalize_max_segment_tokens, - work_dir=tmp_path / "work", - work_dir_retention="never", - ) - llm_client = FakeStructuredLLMClient( - [ - { - "corrections": [ - { - "id": 1, - "original_text": "Hello", - "corrected_text": "Jesters", - "confidence": 0.40, - }, - { - "id": 1, - "original_text": "Hello", - "corrected_text": "Jesters", - "confidence": 0.95, - }, - ] - }, - { - "validations": [ - { - "correction_index": 99, - "approved": True, - "confidence": 0.98, - "reason": "Malformed response for testing.", - } - ] - }, - ] - ) - - with pytest.raises(AuditaLLMError, match="unknown correction_index"): - process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client) - - run_dir = next((tmp_path / "work").iterdir()) - report = json.loads((run_dir / "report.json").read_text(encoding="utf-8")) - validator_dir = run_dir / "glossary_1" - - assert report["status"] == "failed" - assert report["modules"] == [] - assert len(report["skipped_corrections"]) == 1 - assert report["skipped_corrections"][0]["reason"] == "proposal confidence below threshold" - assert report["skipped_corrections"][0]["source"] == "validator:proposal_confidence_guard" - assert "unknown correction_index" in report["error"] - assert report["error_details"]["module_instance"] == "glossary_1" - assert report["error_details"]["phase"] == "pipeline" - assert (run_dir / "error.log").exists() - assert (validator_dir / "spoken_form_plausibility_review-prompt-0000.json").exists() - assert (validator_dir / "spoken_form_plausibility_review-response-0000.json").exists() - - -def test_resolve_module_specs_numbers_repeated_keys(): - specs = resolve_module_specs(["glossary", "homophones", "glossary"]) - - assert [spec.instance_name for spec in specs] == ["glossary_1", "homophones", "glossary_2"] - assert [spec.module_key for spec in specs] == ["glossary", "homophones", "glossary"] - - -def test_process_transcript_result_supports_grammar_only_module_override(tmp_path): - config = AuditaConfig.from_sources( - env={}, - overrides=ConfigOverrides(work_dir=tmp_path / "work", work_dir_retention="always"), - ) - - result = process_transcript_result( - _transcript(), - _glossary(), - config, - module_keys=["grammar"], - llm_client=FakeStructuredLLMClient([{"corrections": []}]), - ) - - assert [segment.id for segment in result.transcript] == [1, 2] - assert result.report.pipeline == ["grammar"] - assert result.report.totals["applied_change_count"] == 0 diff --git a/python/tests/test_new_transcript_schema.py b/python/tests/test_new_transcript_schema.py deleted file mode 100644 index f62ad15..0000000 --- a/python/tests/test_new_transcript_schema.py +++ /dev/null @@ -1,84 +0,0 @@ -import json - -from audita.core.schemas import parse_source_transcript_json, parse_transcript_json, transcript_to_json - - -SERIATIM_TRANSCRIPT = """ -{ - "metadata": { - "application": "seriatim", - "version": "dev", - "input_reader": "json-files", - "input_files": ["eric.json", "mike.json"], - "preprocessing_modules": ["validate-raw", "normalize-speakers", "trim-text"], - "postprocessing_modules": ["detect-overlaps", "resolve-overlaps", "backchannel"], - "output_modules": ["json"] - }, - "segments": [ - { - "id": 1, - "source": "eric.json", - "source_segment_index": 0, - "speaker": "Eric Rakestraw", - "start": 1.25, - "end": 3.5, - "text": "Hello there.", - "overlap_group_id": 1 - }, - { - "id": 2, - "source": "eric.json", - "source_ref": "word-run:1:1:1", - "derived_from": ["eric.json#0"], - "speaker": "Eric Rakestraw", - "start": 4.0, - "end": 4.5, - "text": "Resolved word run", - "categories": ["backchannel"] - } - ], - "overlap_groups": [ - { - "id": 1, - "start": 1.25, - "end": 4.0, - "segments": ["eric.json#0", "mike.json#0"], - "speakers": ["Eric Rakestraw", "Mike Brown"], - "class": "unknown", - "resolution": "unresolved" - } - ] -} -""" - - -def test_parse_source_transcript_json_accepts_seriatim_transcript_object(): - segments = parse_source_transcript_json(SERIATIM_TRANSCRIPT) - - assert len(segments) == 2 - assert segments[0].id == 1 - assert segments[0].speaker == "Eric Rakestraw" - assert segments[0].start == 1.25 - assert segments[0].end == 3.5 - assert segments[0].text == "Hello there." - assert segments[1].text == "Resolved word run" - assert segments[0].categories is None - assert segments[1].categories == ["backchannel"] - - -def test_parse_transcript_json_accepts_seriatim_transcript_object_and_ignores_unused_fields(): - segments = parse_transcript_json(SERIATIM_TRANSCRIPT) - - assert [segment.id for segment in segments] == [1, 2] - assert [segment.text for segment in segments] == ["Hello there.", "Resolved word run"] - assert segments[0].categories is None - assert segments[1].categories == ["backchannel"] - - -def test_transcript_to_json_emits_categories_only_when_present(): - segments = parse_transcript_json(SERIATIM_TRANSCRIPT) - - payload = json.loads(transcript_to_json(segments)) - - assert "categories" not in payload[0] - assert payload[1]["categories"] == ["backchannel"] diff --git a/python/tests/test_protected_validator.py b/python/tests/test_protected_validator.py deleted file mode 100644 index 0f637fb..0000000 --- a/python/tests/test_protected_validator.py +++ /dev/null @@ -1,320 +0,0 @@ -from audita.core.schemas import parse_glossary_yaml, parse_transcript_json -from audita.framework.models import CorrectionProposal, ModuleRunSpec -from audita.validators import ( - GlossaryStageProtectedGlossaryTermsValidator, - ProtectedGlossaryTermsValidator, - ProtectedVocabulary, -) -from audita.validators.base import ValidationContext - - -def _glossary(): - return parse_glossary_yaml( - """ - glossary: - - name: "Hrank" - aliases: - - "Greenfield" - category: pc - summary: "Hrank Greenfield is a player character." - - name: "Popov" - category: npc - summary: "Popov is an allied NPC." - - name: "Jesters" - aliases: - - "Jester" - category: faction - summary: "The Jesters are a faction." - - name: "Svend" - category: pc - summary: "Svend is a player character." - - name: "Godfrey" - category: npc - summary: "Godfrey is an NPC." - - name: "Lyra" - category: npc - summary: "Lyra is an NPC." - - name: "Loviator" - category: deity - summary: "Loviator is a deity." - """ - ) - - -def test_protected_vocabulary_blocks_replacing_protected_term(): - vocabulary = ProtectedVocabulary.from_glossary(_glossary()) - - assert ( - vocabulary.violation_reason("Hrank moves.", "Frank moves.") - == "correction changes protected glossary term usage" - ) - - -def test_protected_vocabulary_glossary_stage_allows_glossary_to_glossary_changes(): - vocabulary = ProtectedVocabulary.from_glossary(_glossary()) - - assert vocabulary.violation_reason("Hrank moves.", "Popov moves.") == "correction changes protected glossary term usage" - assert vocabulary.glossary_stage_violation_reason("Hrank moves.", "Popov moves.") is None - - -def test_protected_vocabulary_glossary_stage_still_blocks_glossary_to_nonglossary_changes(): - vocabulary = ProtectedVocabulary.from_glossary(_glossary()) - - assert ( - vocabulary.glossary_stage_violation_reason("Hrank moves.", "Frank moves.") - == "correction changes protected glossary term usage" - ) - - -def test_protected_vocabulary_blocks_noncanonical_capitalization(): - vocabulary = ProtectedVocabulary.from_glossary(_glossary()) - - assert ( - vocabulary.violation_reason("Popov moves.", "POPOV moves.") - == "correction changes protected glossary term capitalization" - ) - assert ( - vocabulary.violation_reason("Jesters", "jesters") - == "correction changes protected glossary term capitalization" - ) - assert ( - vocabulary.glossary_stage_violation_reason("Popov moves.", "POPOV moves.") - == "correction changes protected glossary term capitalization" - ) - - -def test_protected_vocabulary_allows_corrections_toward_protected_terms(): - vocabulary = ProtectedVocabulary.from_glossary(_glossary()) - - assert vocabulary.violation_reason("Pawpaw moves.", "Popov moves.") is None - assert vocabulary.violation_reason("gestures", "Jesters") is None - assert vocabulary.violation_reason("gestures", "jesters") is None - assert vocabulary.violation_reason("rank", "Hrank") is None - assert vocabulary.violation_reason("rank", "hrank") is None - assert vocabulary.violation_reason("spend", "Svend") is None - - -def test_protected_vocabulary_allows_unchanged_noncanonical_terms_and_quote_wrapping(): - vocabulary = ProtectedVocabulary.from_glossary(_glossary()) - - assert vocabulary.violation_reason("jesters advance.", "jesters advance.") is None - before = ( - "When you say that, Popov will say, when I was in that room with the jesters, " - "I just knew that Godfrey and Lyra came directly from Loviator herself." - ) - after = ( - 'When you say that, Popov will say, "When I was in that room with the jesters, ' - 'I just knew that Godfrey and Lyra came directly from Loviator herself."' - ) - assert vocabulary.violation_reason(before, after) is None - - -def test_protected_vocabulary_allows_inferred_and_explicit_plurals(): - vocabulary = ProtectedVocabulary.from_glossary(_glossary()) - explicit = ProtectedVocabulary.from_glossary( - parse_glossary_yaml( - """ - glossary: - - name: "Mox" - plural: "Moxen" - category: faction - summary: "The Mox are a faction." - """ - ) - ) - - assert vocabulary.violation_reason("Godfrey's", "Godfreys") is None - assert vocabulary.violation_reason("gesture", "Jesters") is None - assert explicit.violation_reason("Mox's", "Moxen") is None - - -def test_protected_vocabulary_does_not_match_embedded_substrings(): - vocabulary = ProtectedVocabulary.from_glossary(_glossary()) - - assert vocabulary.violation_reason("The shrank spell worked.", "The shrank spell works.") is None - - -def test_protected_glossary_terms_validator_returns_proposal_indexed_decisions(): - validator = ProtectedGlossaryTermsValidator("protected_glossary_guard") - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Frank moves."}, - {"id": 2, "speaker": "Eric", "start": 1.0, "end": 2.0, "text": "Pawpaw waits."} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="glossary_primary", - module_key="glossary", - id=1, - original_text="Hrank", - corrected_text="Frank", - confidence=0.9, - ), - CorrectionProposal( - proposal_index=1, - module_instance="glossary_primary", - module_key="glossary", - id=2, - original_text="Pawpaw", - corrected_text="Popov", - confidence=0.9, - ), - ] - result = validator.validate( - ValidationContext( - proposals=proposals, - transcript=transcript, - glossary=_glossary(), - config=None, # type: ignore[arg-type] - run_spec=ModuleRunSpec(instance_name="glossary_primary", module_key="glossary", module=None), # type: ignore[arg-type] - run_dir=transcript[0].__class__.__module__ and __import__("pathlib").Path("."), - ) - ) - - assert [decision.proposal_index for decision in result.decisions] == [0, 1] - assert result.decisions[0].approved is False - assert result.decisions[0].reason == "correction changes protected glossary term usage" - assert result.decisions[1].approved is True - - -def test_protected_glossary_terms_validator_allows_nonglossary_to_lowercase_glossary_replacement(): - validator = ProtectedGlossaryTermsValidator("protected_glossary_guard") - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "gestures advance."}, - {"id": 2, "speaker": "Eric", "start": 1.0, "end": 2.0, "text": "rank moves."} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="homophones", - module_key="homophones", - id=1, - original_text="gestures", - corrected_text="jesters", - confidence=0.9, - ), - CorrectionProposal( - proposal_index=1, - module_instance="homophones", - module_key="homophones", - id=2, - original_text="rank", - corrected_text="hrank", - confidence=0.9, - ), - ] - - result = validator.validate( - ValidationContext( - proposals=proposals, - transcript=transcript, - glossary=_glossary(), - config=None, # type: ignore[arg-type] - run_spec=ModuleRunSpec(instance_name="homophones", module_key="homophones", module=None), # type: ignore[arg-type] - run_dir=__import__("pathlib").Path("."), - ) - ) - - assert [decision.approved for decision in result.decisions] == [True, True] - - -def test_glossary_stage_protected_glossary_terms_validator_allows_glossary_to_glossary_replacement(): - validator = GlossaryStageProtectedGlossaryTermsValidator("glossary_stage_protected_glossary_guard") - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hrank moves."}, - {"id": 2, "speaker": "Eric", "start": 1.0, "end": 2.0, "text": "Hrank moves."} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="glossary_1", - module_key="glossary", - id=1, - original_text="Hrank", - corrected_text="Popov", - confidence=0.9, - ), - CorrectionProposal( - proposal_index=1, - module_instance="glossary_1", - module_key="glossary", - id=2, - original_text="Hrank", - corrected_text="POPOV", - confidence=0.9, - ), - ] - - result = validator.validate( - ValidationContext( - proposals=proposals, - transcript=transcript, - glossary=_glossary(), - config=None, # type: ignore[arg-type] - run_spec=ModuleRunSpec(instance_name="glossary_1", module_key="glossary", module=None), # type: ignore[arg-type] - run_dir=__import__("pathlib").Path("."), - ) - ) - - assert [decision.proposal_index for decision in result.decisions] == [0, 1] - assert result.decisions[0].approved is True - assert result.decisions[1].approved is True - assert result.decisions[1].reason is None - - -def test_protected_glossary_terms_validator_uses_proposal_span_only(): - validator = ProtectedGlossaryTermsValidator("protected_glossary_guard") - glossary = parse_glossary_yaml( - """ - glossary: - - name: "Svend" - category: pc - summary: "Svend is a player character." - - name: "Jesters" - category: faction - summary: "The Jesters are a faction." - """ - ) - transcript = parse_transcript_json( - """ - [ - {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "You have to keep it bind. Svend sees the jesters."} - ] - """ - ) - proposals = [ - CorrectionProposal( - proposal_index=0, - module_instance="spoken_word", - module_key="spoken_word", - id=1, - original_text="keep it bind", - corrected_text="keep in mind", - confidence=0.9, - ) - ] - - result = validator.validate( - ValidationContext( - proposals=proposals, - transcript=transcript, - glossary=glossary, - config=None, # type: ignore[arg-type] - run_spec=ModuleRunSpec(instance_name="spoken_word", module_key="spoken_word", module=None), # type: ignore[arg-type] - run_dir=__import__("pathlib").Path("."), - ) - ) - - assert result.decisions[0].approved is True