Compare commits

92 Commits

Author SHA1 Message Date
76651333b1 Add documetation policy 2026-05-23 19:44:55 -05:00
0b01c3a83d Implemented the config source setter cleanup identified during the code audit 2026-05-23 18:49:44 -05:00
0630d36734 Implemented the CLI override extraction cleanup identified in the code audit 2026-05-23 18:33:12 -05:00
52c2697040 Moved report/ledger assembly to a new processreport module 2026-05-23 18:27:30 -05:00
f790c1441c Refresh architecture and configuration documentation for current runtime behavior 2026-05-23 18:24:06 +00:00
56f9b28f4b Consolidate shared test helpers and stabilize timeout hook integration test 2026-05-23 18:18:48 +00:00
222222f449 Share configured LLM secret extraction across diagnostics paths 2026-05-23 18:11:36 +00:00
99391cd18b Centralize validator classification and malformed output handling 2026-05-23 18:02:53 +00:00
84be774b34 Share module proposal execution and transcript-section prompt payload helpers 2026-05-23 17:56:07 +00:00
e053f7e124 Add shared metadata maps and stage-name helpers 2026-05-23 17:48:37 +00:00
13029dbb33 Centralize effective config loading and path resolution 2026-05-23 17:43:27 +00:00
938bfe88c1 Centralize output schema and module key validation catalogs 2026-05-23 17:39:13 +00:00
fa1bd237d1 Centralize diagnostics artifact names and report metadata paths 2026-05-23 17:32:30 +00:00
3d7057b437 Added an implementation roadmap for the issues identified in the code audit 2026-05-23 12:22:01 -05:00
32c8c8b446 Audit code quality and deduplication opportunities 2026-05-23 11:10:44 -05:00
a3655f5540 Make module-stage LLM handling resilient and report warnings 2026-05-23 10:07:06 -05:00
a84941d681 Rationalize default configuration file paths and update documentation
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-14 20:14:11 -05:00
46b7356a3b Add WoodpeckerCI support 2026-05-14 20:10:19 -05:00
39208cd119 Update the documentation for running audita as a subprocess 2026-05-14 09:34:33 -05:00
6dae15656d Standardized the grammar and spoken_word modules on a shared editorial_review validator 2026-05-14 09:30:10 -05:00
b103bb2e7d Improved organization of the documentation 2026-05-14 09:24:02 -05:00
2b2a3fc024 Removed the old python code and cleaned up outdated documentation 2026-05-13 20:13:33 -05:00
52ffe42e73 Refactor validators into package-owned components 2026-05-14 00:28:28 +00:00
3b160cf05b Added documentation for an upcoming refactoring of the validator modules 2026-05-13 19:12:44 -05:00
0687982822 Complete pre-1.0 release hardening 2026-05-13 19:42:35 +00:00
ff4ed82239 Add utilization diagnostics and correction ledger 2026-05-13 19:03:37 +00:00
037121e9ce Move prompts into embedded Markdown assets 2026-05-13 18:49:06 +00:00
d6126bf52b Refactor validators into built-in registry chains 2026-05-13 18:36:46 +00:00
1bc5936681 Add output schema registry and public contract docs 2026-05-13 12:59:10 +00:00
9a77a0cd0b Add versioned Audita config support 2026-05-13 12:36:20 +00:00
ebbd2c8a63 Add transcript description prompt context 2026-05-13 12:13:32 +00:00
de99467ede Replace structured LLM dependency with Audita adapter 2026-05-13 02:10:24 +00:00
20f612215f Documentation cleanup and addition of pre-1.0 roadmap punchlist 2026-05-12 20:40:35 -05:00
3d45571bb0 Add intra-module pipeline for LLM validation 2026-05-12 18:39:26 -05:00
1afd753fad Audit intra-module LLM pipeline 2026-05-12 18:24:15 -05:00
a85a7e204e Review LLM concurrency refactor 2026-05-12 21:23:57 +00:00
509436cc4a Add explicit LLM concurrency controls 2026-05-12 21:17:35 +00:00
a48f6da1f4 Audit LLM concurrency flow 2026-05-12 16:00:26 -05:00
df96f9fdf6 Rewrite and simplify the section chunking algorithm 2026-05-12 15:42:32 -05:00
390daa8b84 Cleanup pass to remove refactoring-related artifacts and references 2026-05-12 15:19:50 -05:00
af84249da0 Implemented an --llm-concurrency flag in the Go application that enforces a global LLM concurrency cap 2026-05-12 12:59:46 -05:00
cad172a758 Added chunking logic to modules and added corresponding regression tests 2026-05-12 12:26:01 -05:00
fb59cb21b9 Complete Phase 19 Go rollout documentation 2026-05-12 13:46:14 +00:00
68e2d9b549 Complete Phase 18 operational hardening 2026-05-12 13:32:35 +00:00
185f7ca2b6 Complete Phase 17 parity fixture suite 2026-05-12 12:50:05 +00:00
7ccadc6bd6 Complete Phase 16 default pipeline integration 2026-05-12 12:19:50 +00:00
a9f7fa27ff Complete Phase 15 spoken-word module 2026-05-12 11:56:28 +00:00
dbf3605712 Complete Phase 14 homophones module 2026-05-12 11:37:35 +00:00
543a7ff8ef Complete Phase 13 glossary module 2026-05-12 11:20:02 +00:00
fc3a7b7a67 Complete Phase 12 grammar module 2026-05-12 02:57:06 +00:00
b360493cdc Complete Phase 11 proposal generation framework 2026-05-12 02:25:33 +00:00
12202508bf Complete Phase 10 LLM validators 2026-05-12 01:25:52 +00:00
6d9a4bd017 Complete Phase 9 LLM infrastructure 2026-05-11 20:10:56 -05:00
426864eedb Add Phase 9 LLM scheduler, config resolution, diagnostics primitives 2026-05-11 20:09:34 -05:00
0b17a6fbeb Implement Phase 9 structured LLM adapter spike 2026-05-11 19:54:57 -05:00
aeb31f1c0d Complete Phase 8 deterministic validators 2026-05-12 00:26:27 +00:00
28fe899aa1 Complete Phase 7 runner orchestration 2026-05-11 14:22:22 +00:00
30606f5c49 Updated the Go rewrite work plan to reflect completed work so far 2026-05-11 09:10:47 -05:00
db880ed868 Document completed Phase 6 diagnostics 2026-05-11 14:03:14 +00:00
5217093be2 Complete diagnostics retention semantics 2026-05-11 14:01:56 +00:00
0e83991537 Add diagnostics references to reports 2026-05-11 13:59:46 +00:00
3e8d19cccd Add redacted run metadata artifacts 2026-05-11 13:58:13 +00:00
c1193e3450 Refine rewrite documentation accuracy 2026-05-11 08:50:33 -05:00
73249b63d8 Update Go rewrite documentation status 2026-05-11 08:49:26 -05:00
c3087aeda6 Complete Phase 3 proposal foundation 2026-05-11 13:35:15 +00:00
0452a605ad Document Phase 3 rewrite status 2026-05-11 13:33:13 +00:00
b9b7384123 Test chunking with proposal application 2026-05-11 13:32:17 +00:00
726acc47e1 Define minimal module framework contracts 2026-05-11 13:29:34 +00:00
5c78b1d5d9 Add proposal application fixtures 2026-05-11 13:27:08 +00:00
f461922b9b Apply correction proposals to transcripts 2026-05-11 13:23:59 +00:00
c58d307ba7 Add safe replacement preview logic 2026-05-11 13:20:35 +00:00
1eb93481e0 Add correction proposal models 2026-05-11 13:18:13 +00:00
b997e7c97c Record chunking metadata in process runs 2026-05-11 12:47:31 +00:00
12fd541669 Add token-bounded transcript chunking 2026-05-11 12:23:01 +00:00
10377876e4 Complete Phase 2 Go normalization foundation 2026-05-11 00:41:29 +00:00
d847168ecd Add Phase 2 fixtures and compatibility tests 2026-05-11 00:21:19 +00:00
14e51698c2 Add Phase 2 normalization diagnostics 2026-05-11 00:19:28 +00:00
aeb9c4f062 Report schema validation and normalization results 2026-05-11 00:16:50 +00:00
e2ae7f77d8 Run normalization in process command 2026-05-11 00:13:26 +00:00
0b1b670baf Add deterministic transcript normalization 2026-05-11 00:12:19 +00:00
3cfa4b6e8a Wire schema validation into process command 2026-05-11 00:01:42 +00:00
950edc01f2 Add Go transcript and glossary schemas 2026-05-10 23:52:36 +00:00
ea8def423e Complete Phase 1 Go skeleton 2026-05-10 23:34:43 +00:00
95fe8c32fa Harden Go CLI subprocess behavior 2026-05-10 23:33:13 +00:00
08b7531149 Add minimal process reports 2026-05-10 23:30:46 +00:00
2cf2d390da Add transcript passthrough processing 2026-05-10 23:27:54 +00:00
8f3c2ec5fd Wire process command flags 2026-05-10 23:24:48 +00:00
9427c4e6cc Add Go configuration model 2026-05-10 23:19:59 +00:00
6424d7db4f Initialize Go CLI skeleton 2026-05-10 23:12:46 +00:00
87e560dd3d Added architecture reference documentation for the upcoming Go rewrite 2026-05-10 23:04:29 +00:00
09fc6fd364 Updated .gitignore in preparation for the Go rewrite 2026-05-10 22:40:27 +00:00
2e47c8a1b6 Move python implementation under python/ in preparation for the upcoming Go rewrite 2026-05-10 22:37:38 +00:00
311 changed files with 31141 additions and 10395 deletions

61
.gitignore vendored
View File

@@ -1,3 +1,4 @@
# --> Python
.DS_Store
.venv/
__pycache__/
@@ -9,3 +10,63 @@ dist/
build/
*.egg-info/
# ---> Codex
.codex
AGENTS.md
# ---> Go
# If you prefer the allow list template instead of the deny list, see community template:
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
#
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Dependency directories (remove the comment below to include it)
# vendor/
# Go workspace file
go.work
go.work.sum
# env file
.env
# Compiled binaries and test configuration
narratio
local-test
pipeline.yml
bin/
# Local run artifacts
.audita-runs/
report.json
corrected.json
normalized.json
# Coverage artifacts
coverage.out
coverage.txt
# ---> VisualStudioCode
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
!.vscode/*.code-snippets
# Local History for Visual Studio Code
.history/
# Built Visual Studio Code Extensions
*.vsix

50
.woodpecker/release.yml Normal file
View File

@@ -0,0 +1,50 @@
when:
- event: tag
steps:
- name: build-release-assets
image: golang:1.25
commands:
- |
set -eu
version="$CI_COMMIT_TAG"
dist="dist"
pkg="gitea.maximumdirect.net/eric/audita/cmd/audita"
rm -rf "$dist"
mkdir -p "$dist"
build_binary() {
goos="$1"
goarch="$2"
suffix="$3"
output="$dist/audita-$version-$goos-$goarch$suffix"
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \
go build -trimpath -ldflags "-s -w -X gitea.maximumdirect.net/eric/audita/internal/buildinfo.Version=$version" \
-o "$output" "$pkg"
}
build_binary linux amd64 ""
build_binary linux arm64 ""
build_binary darwin amd64 ""
build_binary darwin arm64 ""
build_binary windows amd64 ".exe"
build_binary windows arm64 ".exe"
- name: publish-release
image: woodpeckerci/plugin-release
depends_on:
- build-release-assets
settings:
api_key:
from_secret: GITEA_RELEASE_TOKEN
files:
- dist/audita-*
checksum: sha256
checksum-file: SHA256SUMS
checksum-flatten: true
file-exists: skip
overwrite: false
prerelease: false

365
README.md
View File

@@ -1,148 +1,305 @@
# Audita
Audita is a framework-first transcript correction application. The public `audita` package provides:
Audita is a transcript polishing CLI.
- deterministic transcript normalization
- token-batched module orchestration
- concrete `glossary`, `homophones`, `spoken_word`, and `grammar` modules built on reusable proposal / validator contracts
- structured run reporting and work-dir diagnostics
`audita process` validates transcript/glossary input, normalizes and chunks transcript segments, runs the default correction pipeline, and emits corrected transcript output plus machine-readable diagnostics and reports.
The previous working implementation has been preserved as `audita_prototype` inside this repository. Its full regression suite lives under `tests/audita_prototype`.
## What Audita Does
## Development
Default module sequence:
- `glossary`
- `homophones`
- `glossary`
- `spoken_word`
- `grammar`
This project is set up for `uv`.
Pipeline behavior includes:
- glossary-backed domain/acoustic corrections
- conservative homophone and mistranscription corrections
- conservative spoken-word dysfluency cleanup with semantic guardrails
- grammar/punctuation/capitalization/formatting cleanup
- validator-chain enforcement before application
- malformed module-stage LLM payloads degrade to warnings/rejections instead of failing the run
- run reports and diagnostics artifacts with secret redaction
## Build and Install
Build a local binary:
```sh
uv sync --extra dev
uv run pytest
go build -o ./bin/audita ./cmd/audita
```
## Usage
Process a transcript with the current framework implementation:
Install into your Go bin directory:
```sh
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json
go install ./cmd/audita
```
The framework currently runs this default module sequence:
1. `glossary`
2. `homophones`
3. `glossary`
4. `spoken_word`
5. `grammar`
Resolved run instance names are auto-numbered for repeats, so the default report pipeline is:
1. `glossary_1`
2. `homophones`
3. `glossary_2`
4. `spoken_word`
5. `grammar`
The default module sequence is fully implemented today:
- `glossary` proposes glossary-supported acoustic corrections
- `homophones` proposes conservative homophone and mistranscription corrections
- `spoken_word` proposes conservative dysfluency cleanup
- `grammar` proposes punctuation, capitalization, and spacing cleanup only
To run a custom module sequence, pass `--modules`:
CLI help:
```sh
uv run audita process transcript.json --glossary glossary.yaml --modules grammar --output corrected.json
audita --help
audita process --help
audita config --help
```
To also write a structured JSON report:
## Test
Run all tests:
```sh
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json --report-json report.json
go test ./...
```
From a checked-out repository, you can also use the root launcher:
## Basic Usage
Required inputs:
- transcript JSON path (positional argument)
- `--glossary <glossary.yaml>`
Recommended run:
```sh
./audita process transcript.json --glossary glossary.yaml --output corrected.json
audita process transcript.json \
--glossary glossary.yaml \
--output corrected.json \
--report-json report.json
```
For a system-wide command, install the source tree under `/usr/local/src/audita`, sync dependencies there, and symlink the root launcher into your `PATH`:
Select an explicit output schema (default is `bare-segments`):
```sh
cd /usr/local/src/audita
uv sync --extra dev
ln -s /usr/local/src/audita/audita /usr/local/bin/audita
audita process transcript.json --glossary glossary.yaml --output corrected.json
audita process transcript.json \
--glossary glossary.yaml \
--output-schema audita-v1 \
--output corrected.json \
--report-json report.json
```
Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr.
`--report-json` writes a separate machine-readable run report and never mixes report data into stdout.
Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Default OpenRouter runs require LLM API credentials, because the `glossary`, `homophones`, `spoken_word`, and `grammar` modules make real LLM calls. Self-hosted or other non-default OpenAI-compatible endpoints may not require credentials. `AUDITA_LLM_API_KEY` and `--llm-api-key` are the preferred provider-neutral credential surfaces, while `OPENROUTER_API_KEY` remains supported as a backward-compatible fallback.
| Environment variable | CLI flag | Default | Purpose |
| --- | --- | --- | --- |
| `AUDITA_MODULES` | `--modules` | `glossary,homophones,glossary,spoken_word,grammar` | Comma-separated logical module keys to run; CLI overrides the environment value |
| `AUDITA_LLM_API_KEY` | `--llm-api-key` | unset | Preferred provider-neutral LLM API credential; required for the default OpenRouter endpoint and optional for non-default endpoints; CLI overrides both environment-key variants |
| `AUDITA_VALIDATION_LLM_API_KEY` | `--validation-llm-api-key` | unset | Validation-phase LLM API credential; defaults to the primary LLM API key and is optional for non-default validation endpoints |
| `AUDITA_MODEL` | `--model` | `openrouter/google/gemma-4-31b-it` | LLM model name sent to the configured OpenAI-compatible endpoint |
| `AUDITA_VALIDATION_MODEL` | `--validation-model` | unset | Validation-phase LLM model; defaults to `AUDITA_MODEL` |
| `AUDITA_BASE_URL` | `--base-url` | `https://openrouter.ai/api/v1` | OpenAI-compatible API base URL |
| `AUDITA_VALIDATION_BASE_URL` | `--validation-base-url` | unset | Validation-phase OpenAI-compatible API base URL; defaults to `AUDITA_BASE_URL` |
| `AUDITA_LLM_TIMEOUT_SECONDS` | `--llm-timeout-seconds` | `600` | Per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint |
| `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | `--validation-llm-timeout-seconds` | unset | Validation-phase per-request timeout in seconds; defaults to `AUDITA_LLM_TIMEOUT_SECONDS` |
| `AUDITA_VALIDATION_MAX_PROMPT_TOKENS` | `--validation-max-prompt-tokens` | `2048` | Maximum estimated tokens per validation-phase LLM prompt batch |
| `AUDITA_TARGET_SECTIONS` | `--target-sections` | unset | Exact number of contiguous proposal-stage transcript sections; errors if min/max token bounds cannot be satisfied |
| `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses |
| `AUDITA_VALIDATION_MAX_RETRIES` | `--validation-max-retries` | unset | Validation-phase structured-output retries; defaults to `AUDITA_MAX_RETRIES` |
| `AUDITA_VALIDATION_LLM_CONCURRENCY` | `--validation-llm-concurrency` | unset | Validation-phase LLM concurrency; defaults to `AUDITA_LLM_CONCURRENCY` |
| `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `8192` | Maximum estimated tokens per proposal-stage transcript section |
| `AUDITA_MIN_SECTION_TOKENS` | `--min-section-tokens` | `2048` | Minimum estimated tokens per proposal-stage transcript section when balancing for concurrency |
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation |
| `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | `--grammar-confidence-threshold` | `0.8` | Minimum confidence required for grammar proposals to survive validation |
| `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | `--homophones-confidence-threshold` | `0.8` | Minimum confidence required for homophone proposals to survive validation |
| `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | `--spoken-word-confidence-threshold` | `0.8` | Minimum confidence required for spoken-word proposals to survive validation |
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | `--normalize-max-segment-gap` | `4.0` | Same-speaker gaps eligible for deterministic merging |
| `AUDITA_NORMALIZE_ELLIPSIS_GAP` | `--normalize-ellipsis-gap` | `3.5` | Same-speaker gaps above this value are joined with ` ... ` |
| `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | `--normalize-max-segment-duration` | `60.0` | Maximum merged segment duration |
| `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | `--normalize-max-segment-tokens` | `2048` | Maximum merged segment prompt payload size |
| `AUDITA_WORK_DIR` | `--work-dir` | `/tmp/audita` | Per-run scratch diagnostics directory |
| `AUDITA_WORK_DIR_RETENTION` | `--work-dir-retention` | `auto` | Whether to retain the per-run work directory: `auto`, `always`, or `never` |
Set `AUDITA_MODULES=grammar` to run only the grammar module by default, or override it per command with `--modules`.
Validation-phase LLM settings inherit from the primary `AUDITA_*` LLM settings by default. Set any of the `AUDITA_VALIDATION_*` values only when you want LLM-backed validators to use a different model, endpoint, credential, timeout, retry budget, or concurrency level.
OpenRouter remains the default out of the box:
Recommended config-based run:
```sh
export AUDITA_LLM_API_KEY=your-openrouter-key
audita process transcript.json --glossary glossary.yaml --output corrected.json
audita process transcript.json \
--glossary glossary.yaml \
--config audita.yml \
--output corrected.json \
--report-json report.json
```
You can point Audita at any OpenAI-compatible endpoint by changing `AUDITA_BASE_URL` and, if needed, `AUDITA_MODEL`. For example, a local vLLM server:
Explicit module override:
```sh
export AUDITA_BASE_URL=http://localhost:8000/v1
export AUDITA_MODEL=meta-llama/Llama-3.1-8B-Instruct
audita process transcript.json --glossary glossary.yaml --output corrected.json
audita process transcript.json \
--glossary glossary.yaml \
--modules glossary,homophones,grammar \
--output corrected.json \
--report-json report.json
```
If your self-hosted endpoint requires authentication, you can still set `AUDITA_LLM_API_KEY`; Audita simply no longer requires it for non-default endpoints.
Or the actual OpenAI API:
Optional transcript background context:
```sh
export AUDITA_LLM_API_KEY=your-openai-key
export AUDITA_BASE_URL=https://api.openai.com/v1
export AUDITA_MODEL=gpt-4.1-mini
audita process transcript.json --glossary glossary.yaml --output corrected.json
audita process transcript.json \
--glossary glossary.yaml \
--transcript-description "Brief context that may help resolve ambiguous terms." \
--output corrected.json
```
`AUDITA_WORK_DIR` stores per-run diagnostics while processing. Under the default `AUDITA_WORK_DIR_RETENTION=auto`, clean successful runs are removed, while failed runs and successful runs with final skipped corrections are preserved. Use `always` to keep every run directory and `never` to remove successful run directories even when skips remain.
Failed runs always preserve the run directory and include an authoritative `report.json` alongside normalization and prompt/response diagnostics.
The transcript description is background context only and does not override transcript content.
## Prototype Archive
Write transcript JSON to stdout (no `--output`):
The archived prototype remains importable as `audita_prototype` and is still covered by its original regression suite. This is intentional: the new `audita` package is a framework-oriented rewrite, not a thin wrapper around the old code.
```sh
audita process transcript.json --glossary glossary.yaml
```
Control diagnostics location/retention:
```sh
audita process transcript.json \
--glossary glossary.yaml \
--work-dir /tmp/audita \
--work-dir-retention auto \
--output corrected.json \
--report-json report.json
```
## Stdout/Stderr Contract
- With `--output`, stdout is expected to be empty on success.
- Without `--output`, stdout contains transcript JSON only on success.
- `--report-json` writes a file and is never printed to stdout.
- stderr is human-readable diagnostics/errors.
- successful runs remain quiet on stderr even when module warnings are recorded in report/diagnostics artifacts.
For subprocess orchestration guidance, see [`docs/subprocess-operations.md`](docs/subprocess-operations.md).
## Configuration
Precedence:
1. defaults
2. config file (`--config`, `AUDITA_CONFIG`, or default search paths when present: `/usr/local/etc/audita/config.yml`, then `/etc/audita/config.yml`)
3. environment (`AUDITA_*`)
4. CLI flags
Config commands:
```sh
audita config validate --config audita.yml
audita config print-effective --config audita.yml
```
For full config-file schema and examples, see [`docs/configuration.md`](docs/configuration.md).
For output-schema details, see [`docs/architecture/output-schemas.md`](docs/architecture/output-schemas.md).
For built-in validator keys and chain definitions, see [`docs/architecture/validators.md`](docs/architecture/validators.md).
For embedded prompt assets and prompt metadata behavior, see [`docs/architecture/prompts.md`](docs/architecture/prompts.md).
For CLI/process compatibility guarantees, see [`docs/architecture/public-contract.md`](docs/architecture/public-contract.md).
### Modules
- `AUDITA_MODULES` (CSV)
- CLI: `--modules`
### Transcript Description
CLI:
- `--transcript-description`
Behavior:
- optional background context for proposal and LLM-validator prompts;
- trimmed and length-limited by CLI validation;
- does not override transcript content;
- no `AUDITA_*` environment variable is currently defined for this setting.
### Primary LLM
Environment:
- `AUDITA_LLM_API_KEY` (or `OPENROUTER_API_KEY` fallback)
- `AUDITA_MODEL`
- `AUDITA_BASE_URL`
- `AUDITA_LLM_TIMEOUT_SECONDS`
- `AUDITA_MAX_RETRIES`
CLI:
- `--llm-api-key`
- `--model`
- `--base-url`
- `--llm-timeout-seconds`
- `--max-retries`
### Validation LLM
Environment:
- `AUDITA_VALIDATION_LLM_API_KEY`
- `AUDITA_VALIDATION_MODEL`
- `AUDITA_VALIDATION_BASE_URL`
- `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS`
- `AUDITA_VALIDATION_MAX_RETRIES`
- `AUDITA_VALIDATION_LLM_CONCURRENCY`
- `AUDITA_VALIDATION_MAX_PROMPT_TOKENS`
CLI:
- `--validation-llm-api-key`
- `--validation-model`
- `--validation-base-url`
- `--validation-llm-timeout-seconds`
- `--validation-max-retries`
- `--validation-llm-concurrency`
- `--validation-max-prompt-tokens`
### LLM Concurrency
Environment:
- `AUDITA_TOTAL_LLM_CONCURRENCY`
- `AUDITA_PROPOSAL_LLM_CONCURRENCY`
- `AUDITA_VALIDATION_LLM_CONCURRENCY`
- `AUDITA_LLM_CONCURRENCY` (legacy alias for `AUDITA_TOTAL_LLM_CONCURRENCY`)
CLI:
- `--total-llm-concurrency`
- `--proposal-llm-concurrency`
- `--validation-llm-concurrency`
- `--llm-concurrency` (legacy alias for `--total-llm-concurrency`)
Behavior:
- all proposal and validation LLM calls are bounded by total LLM concurrency
- proposal LLM calls are additionally bounded by proposal LLM concurrency
- when validation concurrency is unset, it inherits total LLM concurrency
- when explicitly set, proposal and validation concurrency must each be `<= total-llm-concurrency`
- canonical total settings win when both canonical and legacy alias settings are provided at the same precedence layer
### Confidence Thresholds
Environment:
- `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD`
- `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD`
- `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD`
- `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD`
CLI:
- `--glossary-confidence-threshold`
- `--homophones-confidence-threshold`
- `--spoken-word-confidence-threshold`
- `--grammar-confidence-threshold`
### Normalization and Chunking
Environment:
- `AUDITA_NORMALIZE_MAX_SEGMENT_GAP`
- `AUDITA_NORMALIZE_ELLIPSIS_GAP`
- `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION`
- `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS`
- `AUDITA_MAX_SECTION_TOKENS`
- `AUDITA_MIN_SECTION_TOKENS`
- `AUDITA_TARGET_SECTIONS`
CLI:
- `--normalize-max-segment-gap`
- `--normalize-ellipsis-gap`
- `--normalize-max-segment-duration`
- `--normalize-max-segment-tokens`
- `--max-section-tokens`
- `--min-section-tokens`
- `--target-sections`
### Work Directory
Environment:
- `AUDITA_WORK_DIR`
- `AUDITA_WORK_DIR_RETENTION` (`auto`, `always`, `never`)
CLI:
- `--work-dir`
- `--work-dir-retention`
Retention behavior:
- `always`: keep all run directories
- `never`: keep successful run directories
- `auto`: keep failed runs and successful runs with skipped/rejected corrections
## Reports and Diagnostics
Per-run diagnostics include:
- source transcript artifacts
- normalized transcript artifact
- normalization summary
- chunking summary
- utilization diagnostics summary
- correction ledger
- invocation metadata
- redacted effective config
- module/validator prompt-response diagnostics
- `report.json`
- `error.log` on failure
Optional external report output:
- `--report-json <path>`
## Documentation
- Architecture: [`docs/architecture.md`](docs/architecture.md)
- Diagnostics: [`docs/diagnostics.md`](docs/diagnostics.md)
- Structured LLM adapter: [`docs/structured-llm.md`](docs/structured-llm.md)
- Subprocess operations: [`docs/subprocess-operations.md`](docs/subprocess-operations.md)
- Release checklist: [`docs/release-checklist.md`](docs/release-checklist.md)

147
audita
View File

@@ -1,147 +0,0 @@
#!/usr/bin/env python3
import json
import os
from datetime import datetime
import shutil
import subprocess
import sys
from pathlib import Path
from typing import Optional
from uuid import uuid4
DEFAULT_WORK_DIR = "/tmp/audita"
_SECRET_FLAGS = {"--llm-api-key", "--validation-llm-api-key"}
def _redact_argv(argv: list[str]) -> list[str]:
redacted: list[str] = []
index = 0
while index < len(argv):
arg = argv[index]
matched_flag = next((flag for flag in _SECRET_FLAGS if arg == flag or arg.startswith(flag + "=")), None)
if matched_flag is None:
redacted.append(arg)
index += 1
continue
if arg == matched_flag:
redacted.append(arg)
if index + 1 < len(argv):
redacted.append("[REDACTED]")
index += 2
else:
index += 1
continue
redacted.append(f"{matched_flag}=[REDACTED]")
index += 1
return redacted
def _resolve_work_root(argv: list[str]) -> Path:
for index, arg in enumerate(argv):
if arg == "--work-dir" and index + 1 < len(argv):
return Path(argv[index + 1])
if arg.startswith("--work-dir="):
return Path(arg.split("=", 1)[1])
return Path(os.environ.get("AUDITA_WORK_DIR") or DEFAULT_WORK_DIR)
def _create_run_dir(root: Path) -> Path:
root.mkdir(parents=True, exist_ok=True)
timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
run_dir = root / f"run-{timestamp}-{uuid4().hex[:8]}"
run_dir.mkdir(parents=False, exist_ok=False)
return run_dir
def _capture_run_dirs(root: Path) -> set[str]:
if not root.exists():
return set()
return {path.name for path in root.iterdir() if path.is_dir() and path.name.startswith("run-")}
def _find_new_run_dir(root: Path, before: set[str]) -> Optional[Path]:
if not root.exists():
return None
candidates = [
path for path in root.iterdir() if path.is_dir() and path.name.startswith("run-") and path.name not in before
]
if not candidates:
return None
return max(candidates, key=lambda path: path.name)
def _write_launcher_error_log(
path: Path,
*,
message: str,
exit_code: int,
argv: list[str],
command: Optional[list[str]],
) -> None:
payload = {
"timestamp": datetime.utcnow().isoformat() + "Z",
"message": message,
"exit_code": exit_code,
"argv": argv,
"cwd": os.getcwd(),
"command": command,
}
path.write_text(
"Audita Launcher Diagnostics\n" + json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
def _emit_console_line(message: str) -> None:
for stream in (sys.stderr, sys.stdout):
if stream is None:
continue
try:
stream.write(f"{message}\n")
stream.flush()
return
except (OSError, ValueError):
continue
def main() -> int:
argv = list(sys.argv[1:])
work_root = _resolve_work_root(argv)
redacted_argv = _redact_argv(argv)
uv = shutil.which("uv")
if uv is None:
run_dir = _create_run_dir(work_root)
error_log = run_dir / "error.log"
message = "uv is required to run this launcher. Install uv and run `uv sync` in the Audita project."
_write_launcher_error_log(error_log, message=message, exit_code=1, argv=redacted_argv, command=None)
_emit_console_line(f"audita: error: {message}")
_emit_console_line("audita: exit code: 1")
_emit_console_line(f"audita: run directory: {run_dir}")
_emit_console_line(f"audita: error log: {error_log}")
return 1
project_root = Path(__file__).resolve().parent
command = [uv, "run", "--project", str(project_root), "python", "-m", "audita", *sys.argv[1:]]
before = _capture_run_dirs(work_root)
result = subprocess.run(command, cwd=project_root, check=False)
if result.returncode == 0:
return 0
if _find_new_run_dir(work_root, before) is None:
run_dir = _create_run_dir(work_root)
error_log = run_dir / "error.log"
_write_launcher_error_log(
error_log,
message=f"Audita subprocess exited with status {result.returncode}.",
exit_code=result.returncode,
argv=redacted_argv,
command=_redact_argv(command),
)
_emit_console_line(f"audita: subprocess exited with status {result.returncode}")
_emit_console_line(f"audita: run directory: {run_dir}")
_emit_console_line(f"audita: error log: {error_log}")
return result.returncode
if __name__ == "__main__":
raise SystemExit(main())

11
cmd/audita/main.go Normal file
View File

@@ -0,0 +1,11 @@
package main
import (
"os"
"gitea.maximumdirect.net/eric/audita/internal/cli"
)
func main() {
os.Exit(cli.Run(os.Args[1:], os.Stdout, os.Stderr))
}

View File

@@ -0,0 +1,677 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"reflect"
"runtime"
"strconv"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/internal/cli"
"gitea.maximumdirect.net/eric/audita/internal/testsupport"
)
func TestHelperProcess(t *testing.T) {
if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
return
}
sep := -1
for i, arg := range os.Args {
if arg == "--" {
sep = i
break
}
}
if sep == -1 {
os.Exit(2)
}
cli.ConfigureSubprocessTestHooksFromEnv()
code := cli.Run(os.Args[sep+1:], os.Stdout, os.Stderr)
os.Exit(code)
}
func TestProcessHelpSubprocess(t *testing.T) {
result := runCLISubprocess(t, "process", "--help")
if result.exitCode != 0 {
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
}
if !strings.Contains(result.stdout, "Usage:") || !strings.Contains(result.stdout, "--glossary") {
t.Fatalf("unexpected help stdout: %q", result.stdout)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr, got %q", result.stderr)
}
}
func TestProcessSuccessWithOutputSubprocess(t *testing.T) {
outputPath := filepath.Join(t.TempDir(), "corrected.json")
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--output",
outputPath,
)
if result.exitCode != 0 {
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
}
if result.stdout != "" {
t.Fatalf("expected empty stdout when --output is set, got %q", result.stdout)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
inputBytes := readFile(t, fixturePath("tiny_transcript.json"))
outputBytes := readFile(t, outputPath)
assertJSONSemanticallyEqual(t, inputBytes, outputBytes)
}
func TestProcessSuccessWithoutOutputSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
)
if result.exitCode != 0 {
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
inputBytes := readFile(t, fixturePath("tiny_transcript.json"))
assertJSONSemanticallyEqual(t, inputBytes, []byte(result.stdout))
}
func TestProcessSuccessWithAuditaV1OutputSchemaSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--output-schema",
"audita-v1",
)
if result.exitCode != 0 {
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
var out struct {
Schema string `json:"schema"`
}
if err := json.Unmarshal([]byte(result.stdout), &out); err != nil {
t.Fatalf("expected valid audita-v1 JSON output: %v", err)
}
if out.Schema != "audita-v1" {
t.Fatalf("expected audita-v1 schema, got %q", out.Schema)
}
}
func TestProcessFailureMissingTranscriptSubprocess(t *testing.T) {
result := runCLISubprocess(t, "process", "--glossary", fixturePath("tiny_glossary.yaml"))
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "expected exactly 1 transcript JSON path argument") {
t.Fatalf("expected actionable missing transcript error, got %q", result.stderr)
}
}
func TestProcessFailureMalformedJSONSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
fixturePath("malformed_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "is not valid JSON") {
t.Fatalf("expected malformed JSON error, got %q", result.stderr)
}
}
func TestProcessFailureMissingTranscriptFileSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
filepath.Join(t.TempDir(), "missing-transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "transcript_read") {
t.Fatalf("expected transcript_read failure, got %q", result.stderr)
}
}
func TestProcessFailureMissingGlossaryFileSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
filepath.Join(t.TempDir(), "missing-glossary.yaml"),
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "glossary_read") {
t.Fatalf("expected glossary_read failure, got %q", result.stderr)
}
}
func TestProcessFailureTranscriptSchemaSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
schemaFixturePath("transcript_empty_speaker.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "transcript_schema") {
t.Fatalf("expected transcript_schema failure, got %q", result.stderr)
}
}
func TestProcessFailureMalformedGlossaryYAMLSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
schemaFixturePath("glossary_malformed.yaml"),
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "glossary_schema") {
t.Fatalf("expected glossary_schema failure, got %q", result.stderr)
}
}
func TestProcessFailureUnreadableTranscriptSubprocess(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("portable unreadable-file permissions are not reliable on windows")
}
dir := t.TempDir()
transcriptPath := filepath.Join(dir, "transcript.json")
if err := os.WriteFile(transcriptPath, []byte(`[]`), 0o000); err != nil {
t.Fatalf("write unreadable transcript: %v", err)
}
t.Cleanup(func() { _ = os.Chmod(transcriptPath, 0o644) })
if _, err := os.ReadFile(transcriptPath); err == nil {
t.Skip("unable to make transcript unreadable on this platform/user")
}
result := runCLISubprocess(
t,
"process",
transcriptPath,
"--glossary",
fixturePath("tiny_glossary.yaml"),
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "transcript_read") {
t.Fatalf("expected transcript_read failure, got %q", result.stderr)
}
}
func TestProcessFailureUnwritableOutputSubprocess(t *testing.T) {
outputDir := t.TempDir()
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--output",
outputDir,
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "failed to write output file") {
t.Fatalf("expected write failure message, got %q", result.stderr)
}
}
func TestProcessFailureUnwritableReportJSONSubprocess(t *testing.T) {
reportDir := t.TempDir()
outputPath := filepath.Join(t.TempDir(), "out.json")
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--output",
outputPath,
"--report-json",
reportDir,
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "failed to write report JSON file") {
t.Fatalf("expected report write failure message, got %q", result.stderr)
}
}
func TestProcessSuccessReportJSONSubprocess(t *testing.T) {
reportPath := filepath.Join(t.TempDir(), "report.json")
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--report-json",
reportPath,
)
if result.exitCode != 0 {
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
if !json.Valid([]byte(result.stdout)) {
t.Fatalf("expected transcript JSON only on stdout, got %q", result.stdout)
}
report := readFile(t, reportPath)
if !json.Valid(report) {
t.Fatalf("expected valid report JSON, got %q", string(report))
}
// Ensure report JSON is not printed to stdout.
if strings.Contains(result.stdout, `"default_pipeline"`) {
t.Fatalf("report JSON leaked to stdout: %q", result.stdout)
}
}
func TestProcessSuccessLargeTranscriptSubprocess(t *testing.T) {
transcriptPath := writeLargeTranscriptFixture(t, 320)
result := runCLISubprocess(
t,
"process",
transcriptPath,
"--glossary",
fixturePath("tiny_glossary.yaml"),
)
if result.exitCode != 0 {
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
if !json.Valid([]byte(result.stdout)) {
t.Fatalf("expected valid transcript JSON on stdout")
}
}
func TestProcessFailureMalformedStructuredLLMResponseViaSubprocessHook(t *testing.T) {
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
result := runCLISubprocessWithEnv(t,
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "malformed_structured"},
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--modules",
"grammar",
"--report-json",
reportPath,
"--work-dir",
workDir,
"--work-dir-retention",
"always",
)
if result.exitCode != 0 {
t.Fatalf("expected zero exit code, got %d stderr=%q", result.exitCode, result.stderr)
}
if !json.Valid([]byte(result.stdout)) {
t.Fatalf("expected transcript JSON on stdout, got %q", result.stdout)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
report := readFile(t, reportPath)
if !json.Valid(report) {
t.Fatalf("expected valid success report JSON")
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
t.Fatalf("did not expect error.log, got: %v", err)
}
}
func TestProcessFailureBackendLLMViaSubprocessHook(t *testing.T) {
workDir := t.TempDir()
result := runCLISubprocessWithEnv(t,
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "backend_error"},
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--modules",
"grammar",
"--work-dir",
workDir,
"--work-dir-retention",
"always",
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "synthetic backend failure") {
t.Fatalf("expected backend failure details, got %q", result.stderr)
}
if !strings.Contains(result.stderr, "diagnostics:") {
t.Fatalf("expected diagnostics path in stderr, got %q", result.stderr)
}
if _, err := os.Stat(filepath.Join(onlyRunDir(t, workDir), "error.log")); err != nil {
t.Fatalf("expected error.log in retained failed run: %v", err)
}
}
func TestProcessFailureMidPipelinePreservesPartialReportsSubprocess(t *testing.T) {
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
result := runCLISubprocessWithEnv(t,
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "mid_pipeline_fail"},
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--modules",
"glossary,homophones,glossary,spoken_word,grammar",
"--report-json",
reportPath,
"--work-dir",
workDir,
"--work-dir-retention",
"always",
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
reportRaw := readFile(t, reportPath)
var report struct {
Status string `json:"status"`
ErrorPhase string `json:"error_phase"`
ModuleResults []struct {
ModuleInstance string `json:"module_instance"`
Status string `json:"status"`
} `json:"module_results"`
}
if err := json.Unmarshal(reportRaw, &report); err != nil {
t.Fatalf("unmarshal report: %v", err)
}
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
t.Fatalf("expected failed runner_execution report, got %+v", report)
}
if len(report.ModuleResults) == 0 {
t.Fatalf("expected partial module results in failure report")
}
}
func TestProcessCancellationViaSubprocessTimeoutHook(t *testing.T) {
workDir := t.TempDir()
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
result := runCLISubprocessContext(t, ctx,
map[string]string{
"AUDITA_SUBPROCESS_TEST_LLM_MODE": "block_until_cancel",
"AUDITA_SUBPROCESS_TEST_RUN_TIMEOUT_MS": "120",
},
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--modules",
"grammar",
"--work-dir",
workDir,
"--work-dir-retention",
"always",
)
if result.stdout != "" {
if result.stderr == "" {
t.Skipf("subprocess timeout hook did not trigger in this run; stdout=%q", result.stdout)
}
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "context deadline exceeded") {
t.Fatalf("expected context deadline error, got %q", result.stderr)
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log for canceled run: %v", err)
}
if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil {
t.Fatalf("expected report.json for canceled run: %v", err)
}
}
func TestProcessSubprocessNoSecretLeakInOutputsAndDiagnostics(t *testing.T) {
secret := "subprocess-secret"
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
result := runCLISubprocessWithEnv(t,
map[string]string{
"AUDITA_LLM_API_KEY": secret,
"AUDITA_VALIDATION_LLM_API_KEY": secret,
},
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--output",
outputPath,
"--report-json",
reportPath,
"--work-dir",
workDir,
"--work-dir-retention",
"always",
)
if result.exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", result.exitCode, result.stderr)
}
if strings.Contains(result.stdout, secret) || strings.Contains(result.stderr, secret) {
t.Fatalf("secret leaked in subprocess stdio")
}
assertNoSecretInFile(t, reportPath, secret)
assertNoSecretInTree(t, onlyRunDir(t, workDir), secret)
}
type subprocessResult struct {
stdout string
stderr string
exitCode int
}
func runCLISubprocess(t *testing.T, args ...string) subprocessResult {
t.Helper()
return runCLISubprocessWithEnv(t, nil, args...)
}
func runCLISubprocessWithEnv(t *testing.T, extraEnv map[string]string, args ...string) subprocessResult {
t.Helper()
return runCLISubprocessContext(t, context.Background(), extraEnv, args...)
}
func runCLISubprocessContext(t *testing.T, ctx context.Context, extraEnv map[string]string, args ...string) subprocessResult {
t.Helper()
cmdArgs := append([]string{"-test.run=TestHelperProcess", "--"}, args...)
cmd := exec.CommandContext(ctx, os.Args[0], cmdArgs...)
env := append(filterAuditaEnv(os.Environ()), "GO_WANT_HELPER_PROCESS=1")
for k, v := range extraEnv {
env = append(env, fmt.Sprintf("%s=%s", k, v))
}
cmd.Env = env
var stdoutBuf bytes.Buffer
var stderrBuf bytes.Buffer
cmd.Stdout = &stdoutBuf
cmd.Stderr = &stderrBuf
err := cmd.Run()
result := subprocessResult{
stdout: stdoutBuf.String(),
stderr: stderrBuf.String(),
}
if err == nil {
return result
}
if exitErr, ok := err.(*exec.ExitError); ok {
result.exitCode = exitErr.ExitCode()
return result
}
t.Fatalf("subprocess execution failed: %v", err)
return subprocessResult{}
}
func filterAuditaEnv(env []string) []string {
filtered := make([]string, 0, len(env))
for _, entry := range env {
key := entry
if idx := strings.IndexByte(entry, '='); idx >= 0 {
key = entry[:idx]
}
if strings.HasPrefix(key, "AUDITA_") || key == "OPENROUTER_API_KEY" {
continue
}
filtered = append(filtered, entry)
}
return filtered
}
func fixturePath(name string) string {
return filepath.Join("..", "..", "internal", "cli", "testdata", name)
}
func schemaFixturePath(name string) string {
return filepath.Join("..", "..", "internal", "core", "schema", "testdata", name)
}
func readFile(t *testing.T, path string) []byte {
return testsupport.ReadFile(t, path)
}
func assertJSONSemanticallyEqual(t *testing.T, expected []byte, actual []byte) {
t.Helper()
if !json.Valid(actual) {
t.Fatalf("actual output is not valid JSON: %q", string(actual))
}
var expectedValue any
var actualValue any
if err := json.Unmarshal(expected, &expectedValue); err != nil {
t.Fatalf("failed to unmarshal expected JSON: %v", err)
}
if err := json.Unmarshal(actual, &actualValue); err != nil {
t.Fatalf("failed to unmarshal actual JSON: %v", err)
}
if !reflect.DeepEqual(expectedValue, actualValue) {
t.Fatalf("JSON content mismatch: expected %q got %q", string(expected), string(actual))
}
}
func writeLargeTranscriptFixture(t *testing.T, segments int) string {
t.Helper()
path := filepath.Join(t.TempDir(), "large-transcript.json")
rows := make([]string, 0, segments)
for i := 0; i < segments; i++ {
rows = append(rows, fmt.Sprintf(`{"id":%d,"speaker":"Speaker%d","start":%s,"end":%s,"text":"Segment %d has enough words to exercise stdout and pipe buffering safely."}`,
i+1,
(i%4)+1,
strconv.FormatFloat(float64(i)*1.1, 'f', 1, 64),
strconv.FormatFloat(float64(i)*1.1+1.0, 'f', 1, 64),
i+1,
))
}
payload := "[\n " + strings.Join(rows, ",\n ") + "\n]\n"
if err := os.WriteFile(path, []byte(payload), 0o644); err != nil {
t.Fatalf("write large transcript fixture: %v", err)
}
return path
}
func onlyRunDir(t *testing.T, workDir string) string {
return testsupport.OnlyRunDir(t, workDir)
}
func assertNoSecretInFile(t *testing.T, path, secret string) {
testsupport.AssertNoSecretInFile(t, path, secret)
}
func assertNoSecretInTree(t *testing.T, root, secret string) {
testsupport.AssertNoSecretInTree(t, root, secret)
}

14
docs/architecture.md Normal file
View File

@@ -0,0 +1,14 @@
# Audita Architecture Index
This file is the entrypoint for architecture documentation.
Core architecture overview:
- [Architecture Overview](./architecture/architecture.md)
Focused architecture contracts:
- [Public Contract](./architecture/public-contract.md)
- [Diagnostics](./architecture/diagnostics.md)
- [Structured LLM](./architecture/structured-llm.md)
- [Validators](./architecture/validators.md)
- [Prompts](./architecture/prompts.md)
- [Output Schemas](./architecture/output-schemas.md)

View File

@@ -0,0 +1,153 @@
# Audita Architecture
## Scope
This document describes the production architecture implemented in this repository today.
Audita is a single-process Go CLI that:
- loads effective runtime configuration;
- reads transcript and glossary inputs;
- normalizes and sections transcripts;
- runs a built-in module pipeline with validator chains;
- writes transcript output and run diagnostics.
## Runtime entrypoints
Primary CLI commands:
- `audita process <transcript.json> --glossary <glossary.yaml> [flags]`
- `audita config validate --config <config.yml>`
- `audita config print-effective [--config <config.yml>]`
Command ownership lives in `internal/cli/run.go`.
## Configuration model
`internal/core/config` owns defaults, file parsing, environment overrides, CLI overrides, and validation.
Effective-config loading for `process` and `config print-effective` is centralized in:
- `ResolveConfigPath`
- `LoadEffectiveConfig`
Effective precedence for `audita process`:
1. defaults
2. config file
3. environment overrides
4. CLI overrides
`audita config validate` is intentionally file-only validation:
- load versioned file;
- apply onto defaults;
- validate;
- do not apply environment overrides.
Supported module and output-schema keys are validated through shared catalogs:
- module keys: `internal/core/modulecatalog`
- output schemas: `internal/core/outputschema`
## Pipeline and module orchestration
The built-in module sequence is configured in runtime config and executed by `internal/framework/runner` through resolved module specs.
Current default sequence:
- `glossary`
- `homophones`
- `glossary`
- `spoken_word`
- `grammar`
Execution behavior:
- modules execute serially over the working transcript;
- section proposal work can run concurrently within a module;
- validator execution happens on generated proposals before application;
- approved proposals are applied once per module in deterministic proposal-index order.
Production modules remain separate packages:
- `internal/modules/glossary`
- `internal/modules/homophones`
- `internal/modules/spoken_word`
- `internal/modules/grammar`
## Proposal generation and prompt context
Shared proposal plumbing is centralized in `internal/framework/proposal_generation`.
Module packages provide:
- module identity and replacement policy;
- module-specific prompt message building;
- built-in validator chain selection.
Shared prompt payload helpers are in `internal/framework/promptcontext`.
## Validator architecture
Built-in validator construction and chain composition live in `internal/validators`.
Shared validator runtime mechanics live in `internal/framework/validators`.
Execution class metadata (deterministic vs LLM-backed) is centralized in `internal/validators/metadata` and used for ordering and reporting classification.
## Structured LLM boundary
All production LLM calls go through the internal contract:
- `contracts.StructuredLLMClient`
- `CompleteStructured(ctx, req, out)`
The OpenAI-compatible HTTP adapter is implemented in `internal/framework/llm`.
Structured response schemas are registered in `internal/framework/responseschema` and attached to requests via `response_format` metadata.
Malformed structured-output detection is centralized in `internal/framework/structuredoutput` and reused by proposal generation and validator execution so downgrade behavior stays consistent.
## Stage naming and diagnostics metadata
Diagnostics stage naming is centralized in `internal/framework/stagename`:
- module proposal stage names;
- proposal-generation stage names;
- validator batch stage names.
Prompt metadata and response-schema metadata each expose canonical diagnostics maps via:
- `prompts.Metadata.DiagnosticsMap()`
- `responseschema.Schema.DiagnosticsMap()`
## Diagnostics and reporting
Run-directory artifacts are owned by `internal/core/diagnostics`.
Stable artifact names are centralized constants (for example transcript artifacts, `invocation.json`, `effective-config.json`, `utilization-diagnostics.json`, `correction-ledger.json`, `report.json`, `error.log`).
Report diagnostics path metadata is constructed through `BuildDiagnosticsMetadata`, which keeps run-directory artifact references consistent between success and failure reports.
## Secret redaction
Redaction responsibilities are split by concern:
- structural config redaction: `config.Config.Redacted()`
- byte/string payload redaction for diagnostics and surfaced errors: framework redaction utilities.
Configured LLM secret extraction is centralized in `llm.ConfiguredSecrets(cfg)` and reused across proposal and validator diagnostics paths.
## Output contracts
Transcript output schema selection is owned by `internal/core/outputschema`.
Supported schemas:
- `bare-segments`
- `audita-v1`
Unknown schema keys fail validation and runtime resolution.
## Key package map
Core packages:
- `internal/core/config`
- `internal/core/schema`
- `internal/core/normalization`
- `internal/core/chunking`
- `internal/core/diagnostics`
- `internal/core/reporting`
- `internal/core/modulecatalog`
- `internal/core/outputschema`
Framework packages:
- `internal/framework/contracts`
- `internal/framework/proposals`
- `internal/framework/proposal_generation`
- `internal/framework/promptcontext`
- `internal/framework/runner`
- `internal/framework/validators`
- `internal/framework/llm`
- `internal/framework/responseschema`
- `internal/framework/stagename`
- `internal/framework/structuredoutput`
Domain packages:
- `internal/modules/*`
- `internal/validators/*`
- `internal/prompts`

View File

@@ -0,0 +1,104 @@
# Audita Diagnostics
This document describes the run-directory diagnostics artifacts produced by `audita process`.
## Purpose
Diagnostics provide machine-readable run context and execution artifacts for:
- failure debugging;
- validator/correction review;
- post-run performance analysis.
Diagnostics are written under the configured work directory (`--work-dir`) when run-directory initialization succeeds.
## Core artifacts
Typical artifacts in each run directory:
- `source-transcript.json`
- `source-transcript-parsed.json`
- `normalized-transcript.json`
- `normalization-summary.json`
- `chunking-summary.json`
- `invocation.json`
- `effective-config.json` (redacted)
- module/validator LLM interaction artifacts
- `report.json`
- `error.log` on failure
## Utilization diagnostics artifact
Artifact:
- `utilization-diagnostics.json`
High-level fields:
- `effective_concurrency`:
- total/proposal/validation LLM concurrency limits in effect.
- `run_timing`:
- run wall time;
- scheduler queue wait time;
- LLM execution time;
- deterministic validator time;
- max/average in-flight LLM calls.
- `llm_calls`:
- total proposal and validation LLM call counts.
- `modules`:
- module-level timing summaries.
- `validators`:
- per-validator timing summaries keyed by stable validator key.
## Correction ledger artifact
Artifact:
- `correction-ledger.json`
Ledger records are flattened review entries derived from module results and include:
- module/proposal identity (`module_key`, `module_instance`, `proposal_index`, `segment_id`);
- correction text fields and replacement policy when available;
- disposition:
- `applied`
- `rejected`
- `skipped`
- `failed`
- stable reason codes/messages;
- deterministic and LLM validator decision snapshots using stable validator keys.
Validator rejection and proposal-application skip are distinct dispositions.
Module warnings are reported in module results and diagnostics metadata, but do not create standalone correction-ledger rows.
## Report references
`report.json` and optional `--report-json` output include diagnostics metadata paths for:
- utilization diagnostics artifact;
- correction ledger artifact;
- existing transcript/normalization/chunking/invocation/effective-config artifacts.
Module report entries also include warning records for malformed proposal-generation payloads and malformed validator batches.
## Retention behavior
Run-directory retention follows configured policy:
- `always`: keep all run directories;
- `never`: keep successful run directories;
- `auto`: keep failed runs and successful runs with skipped/rejected corrections.
## Redaction guarantees
API keys and other configured secrets are redacted from:
- `effective-config.json`;
- LLM interaction diagnostics artifacts;
- reports and surfaced errors.
## Debugging guide
When debugging:
- slow runs:
- inspect `utilization-diagnostics.json` (`run_timing`, `modules`, `validators`, in-flight metrics).
- validator rejections:
- inspect `correction-ledger.json` rejected entries and matching validator decisions;
- inspect validator response diagnostics payloads.
- module warnings:
- inspect module `warnings` entries in `report.json` or `--report-json`;
- follow any diagnostic artifact path on the warning to the recorded error/response payload.
- application skips:
- inspect `correction-ledger.json` skipped entries and skip reason codes;
- compare with validator decisions to distinguish validation rejection vs apply-time skip.

View File

@@ -0,0 +1,88 @@
# Audita Output Schemas
This document describes the built-in transcript output schema registry used by `audita process`.
## Supported schema names
### `bare-segments`
Status:
- implemented
- default output schema
Shape:
- top-level JSON array of transcript segments
Segment fields:
- `id`
- `speaker`
- `start`
- `end`
- `text`
- optional `categories`
Compatibility:
- this preserves the long-standing output shape used by existing consumers.
### `audita-v1`
Status:
- implemented
Shape:
- top-level JSON object:
- `schema`: `"audita-v1"`
- `version`: `"v1"`
- `segments`: transcript segment array
Segment fields inside `segments` match `bare-segments` segment fields.
Compatibility:
- this is the Audita-native object format with explicit schema/version metadata.
### `seriatim-intermediate`
Status:
- deferred / not implemented
Current behavior:
- selecting `seriatim-intermediate` fails clearly as an unsupported output schema.
Reason:
- a concrete, repository-backed contract for this schema has not been finalized yet.
## Selection
Choose output schema with CLI:
```sh
audita process <transcript.json> --glossary <glossary.yaml> --output-schema audita-v1
```
Or in file config:
```yaml
version: 1
output:
schema: audita-v1
```
Precedence remains:
1. defaults
2. file config
3. environment overrides
4. CLI overrides
`--output-schema` overrides `output.schema` when both are supplied.
## Output routing behavior
- With `--output`, transcript JSON is written to file using the selected schema and stdout stays empty on success.
- Without `--output`, stdout contains transcript JSON only, using the selected schema.
- `--report-json` writes report JSON to file and does not write report payloads to stdout.
## Backward-compatibility expectations
- default schema stays `bare-segments` for compatibility unless explicitly changed in a future breaking release;
- supported schema names are treated as stable public contract values;
- unsupported schema names fail before output write.

View File

@@ -0,0 +1,118 @@
# Audita Prompts
This document describes Audita's built-in embedded prompt assets and prompt registry behavior.
## Why embedded prompt assets
Audita embeds production prompt text into the binary so runtime behavior is:
- deterministic;
- auditable;
- dependency-light;
- not dependent on external prompt files at execution time.
Prompt text is authored as Markdown assets and rendered by Go code using typed template data.
## Built-in prompt registry
The prompt registry lives in `internal/prompts` and is responsible for:
- loading embedded prompt assets;
- registering stable prompt IDs and versions;
- recording prompt source metadata;
- computing deterministic SHA-256 source hashes;
- rendering system/user prompts with strict missing-key failures.
Current prompt source behavior:
- built-in embedded prompts only (`prompt_source = builtin`).
- filesystem prompt overrides are not supported.
## Built-in prompt IDs
Module proposal prompts:
- `modules.glossary.proposal`
- `modules.homophones.proposal`
- `modules.spoken_word.proposal`
- `modules.grammar.proposal`
LLM-backed validator prompts:
- `validators.spoken_form_plausibility`
- `validators.meaning_reversal_review`
- `validators.editorial_review`
- `validators.grammar_review`
- `validators.spoken_word_review`
## Prompt version semantics
Current built-in prompt version value is `v1`.
Version is a stable metadata identifier for diagnostics and debugging. It is not a dynamic prompt-selection mechanism.
## Prompt hash semantics
Each registered prompt includes a deterministic SHA-256 hash of embedded source text.
Hash purpose:
- identify exact prompt source used in a run;
- support diagnostics reproducibility and change auditing.
Current hash scope:
- source prompt text (system + user assets for a registered prompt), not a runtime secret-bearing payload.
## Template rendering behavior
Prompt rendering uses Go `text/template` with typed template data from module/validator builders.
Missing-key behavior:
- rendering uses missing-key errors;
- missing/renamed template fields fail quickly instead of silently producing incomplete prompts.
Go code still owns:
- structured request/response models;
- response schema selection;
- transcript/glossary/payload formatting;
- module and validator selection;
- diagnostics wiring.
## Shared prompt hardening policy
A shared hardening fragment is embedded once and included in every module proposal prompt and every LLM-validator prompt.
Hardening policy includes:
- transcript text is untrusted data;
- glossary entries and transcript descriptions are reference data, not instructions;
- instructions found inside transcript text must not be obeyed;
- model must perform only the requested correction/validation task;
- no invention of facts, names, events, motivations, speaker intent, or corrections;
- transcript remains the source of truth.
## Transcript description behavior
Transcript description remains background-only prompt context:
- it may help interpret ambiguous terms;
- it is explicitly non-authoritative and must not override transcript content;
- empty descriptions do not render awkward blank context sections.
Generated transcript descriptions are not implemented in this workstream.
## Diagnostics and report metadata boundaries
Current metadata flow:
- proposal-generation diagnostics request metadata includes prompt metadata;
- LLM-validator diagnostics request metadata includes prompt metadata.
Prompt metadata fields used in diagnostics:
- `prompt_id`
- `prompt_version`
- `prompt_source`
- `embedded_path`
- `sha256`
Current boundary:
- detailed prompt metadata is diagnostics-first;
- broad report-level prompt registries/ledgers are deferred.
## 1.0 boundary
Not implemented for 1.0 in this workstream:
- filesystem prompt overrides;
- user-configurable prompt selection;
- external prompt directories.

View File

@@ -0,0 +1,121 @@
# Audita Public Contract
## Scope
This document defines stability expectations for Audita's external runtime interfaces.
Covered interfaces:
- CLI commands and major flags;
- versioned config behavior and precedence;
- transcript/glossary input forms;
- output schema selection;
- report schema metadata;
- diagnostics artifact path metadata;
- stdout/stderr and exit-code behavior;
- redaction guarantees.
## CLI contract
Stable commands:
- `audita process`
- `audita config validate`
- `audita config print-effective`
Stable high-value `process` flags:
- `--config`
- `--glossary`
- `--output`
- `--report-json`
- `--modules`
- `--output-schema`
## Config contract
Supported config format:
- YAML;
- `version: 1`;
- strict unknown-field rejection.
Path resolution for `process` and `config print-effective`:
1. `--config`
2. `AUDITA_CONFIG`
3. `/usr/local/etc/audita/config.yml`
4. `/etc/audita/config.yml`
Missing explicit path is an error. Missing default paths is non-fatal.
Precedence for `process`:
1. defaults
2. file config
3. environment overrides
4. CLI overrides
`config validate` remains file-only validation (defaults + file config; no env overrides).
Module and output-schema keys are validated against built-in catalogs. Unknown keys fail validation.
## Input contract
Supported transcript JSON top-level forms:
- array of segments
- object with `segments` array
Supported glossary YAML form:
- top-level `glossary` list with required entry fields validated by schema parsing.
## Output schema contract
Supported transcript output schemas:
- `bare-segments` (default)
- `audita-v1`
Unknown schema keys fail before output write.
## Report metadata contract
Process reports include stable report metadata fields:
- `report_schema_name`
- `report_schema_version`
- `output_schema`
- `config_version` (when file config is loaded)
Current values:
- `report_schema_name = audita-process-report`
- `report_schema_version = v1`
`--report-json` output and run-directory `report.json` use the same report schema metadata.
Validator decision/rejection records use stable validator keys via `validator_name`.
## Diagnostics metadata contract
When run-directory initialization succeeds, diagnostics metadata paths reference stable artifacts, including:
- transcript and normalization artifacts;
- chunking summary;
- invocation metadata;
- redacted effective config;
- utilization diagnostics;
- correction ledger;
- `error.log` on failures.
LLM interaction diagnostics include stable prompt and structured-schema identifiers where applicable.
## Stdout/stderr and exit codes
Success:
- with `--output`, stdout is empty;
- without `--output`, stdout contains transcript JSON only;
- report JSON is not written to stdout.
Failures:
- nonzero exit;
- human-readable stderr summary;
- diagnostics directory path on stderr when available.
Exit codes:
- `0` success
- nonzero failure
## Redaction contract
Configured secrets are redacted from:
- effective config outputs;
- diagnostics artifacts;
- report artifacts;
- surfaced adapter/runtime errors.
## Compatibility policy
Stable command behavior, schema names, report metadata keys, diagnostics-path field semantics, and validator key identities are treated as public contract.
Additive fields are acceptable when existing fields and behavior remain compatible.

View File

@@ -0,0 +1,72 @@
# Structured LLM Architecture
## Scope
This document describes Audita's structured LLM runtime boundary and adapter behavior.
## Runtime boundary
Production LLM integration depends on the internal contract only:
- `contracts.StructuredLLMClient`
- `CompleteStructured(ctx, req, out)`
Provider SDK types do not leak past this boundary.
## Adapter ownership
`internal/framework/llm` owns the OpenAI-compatible HTTP adapter and shared LLM runtime utilities.
Key responsibilities:
- request assembly;
- timeout/cancellation propagation;
- bounded retry behavior;
- scheduler integration;
- provider response decoding;
- error redaction.
## Structured schema registry
Structured response schemas are registered in `internal/framework/responseschema` and include stable metadata:
- `id`
- `version`
- `name`
- `json_schema`
- `sha256`
Current schema keys:
- `correction_set`
- `validator_decision_set`
Schema metadata is attached to diagnostics through `Schema.DiagnosticsMap()`.
## Request shape assumptions
Audita targets OpenAI-compatible chat-completions endpoints and sends structured requests with:
- model;
- chat messages;
- `response_format.type = json_schema`;
- schema name and JSON schema payload.
## Local validation remains mandatory
Provider schema enforcement is treated as transport-level guardrails.
Audita still validates output locally before applying behavior changes:
- proposal decoding and proposal invariants;
- validator decision decoding and cardinality checks;
- deterministic validation and apply-time rules.
## Shared malformed-output policy
Malformed structured-output classification is centralized in `internal/framework/structuredoutput`.
Proposal generation and validator execution both use this shared classifier so downgrade behavior cannot drift between the two paths.
## Secrets and redaction
Secret extraction for LLM redaction is centralized in `llm.ConfiguredSecrets(cfg)` and reused by proposal and validator diagnostics writers.
Secrets are redacted from:
- diagnostics artifacts;
- report artifacts;
- surfaced adapter/runtime errors.
## Concurrency and scheduling
LLM execution is constrained by composed scheduler limits:
- total LLM concurrency;
- proposal LLM concurrency;
- validation LLM concurrency.
The scheduler is FIFO and context-aware so permits are released on success, failure, and cancellation.

View File

@@ -0,0 +1,96 @@
# Audita Validators
## Scope
This document defines the built-in validator system used by production module runs.
## Ownership boundaries
Built-in validator keys, constructors, and module chains are owned by `internal/validators`.
Shared runtime execution mechanics are owned by `internal/framework/validators`, including:
- validator request/result models;
- deterministic proposal checks;
- LLM validator batching and execution;
- decision-cardinality enforcement;
- diagnostics integration.
Execution class metadata is owned by `internal/validators/metadata`.
## Stable validator keys
Deterministic:
- `proposal_shape`
- `confidence_threshold`
- `original_text_presence`
- `non_empty_corrected_text`
- `no_effect`
- `protected_terms`
LLM-backed:
- `spoken_form_plausibility`
- `meaning_reversal_review`
- `editorial_review`
## Built-in module chains
`glossary`:
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
- `protected_terms`
- `non_empty_corrected_text`
- `spoken_form_plausibility`
- `meaning_reversal_review`
`homophones`:
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
- `protected_terms`
- `non_empty_corrected_text`
- `spoken_form_plausibility`
- `meaning_reversal_review`
`spoken_word`:
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
- `protected_terms`
- `non_empty_corrected_text`
- `editorial_review`
- `meaning_reversal_review`
`grammar`:
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
- `protected_terms`
- `non_empty_corrected_text`
- `editorial_review`
- `meaning_reversal_review`
## Ordering and execution semantics
Validator ordering is based on canonical metadata:
- deterministic validators run before LLM-backed validators.
Within each module stage:
- proposals are generated per section;
- validator chains execute on those proposals;
- approved proposals are applied once after section work settles.
## Malformed payload behavior
Malformed structured-output from proposal generation and LLM validator calls is downgraded, not treated as a process-fatal transport error.
Current outcomes:
- malformed proposal-generation payloads produce section/module warnings and zero proposals for the affected section;
- malformed validator decision payloads reject the affected validator batch with warnings;
- deterministic validator behavior and runner order remain unchanged.
## Reporting identity
Reports and diagnostics use stable validator keys as identifiers.
Correction-ledger deterministic-vs-LLM classification is derived from canonical validator metadata, not package-local hardcoded maps.
## Prompt assets
LLM validator prompt assets and prompt metadata are documented in [Prompts](./prompts.md).

149
docs/configuration.md Normal file
View File

@@ -0,0 +1,149 @@
# Audita Configuration
## Scope
This document defines the supported versioned YAML configuration model and runtime precedence behavior.
## Supported file version
Current supported config file version:
- `version: 1`
Validation rules:
- missing `version` fails;
- unsupported version fails;
- unknown YAML fields fail (strict decoding).
## Config path resolution
For `audita process` and `audita config print-effective`, path resolution order is:
1. `--config <path>`
2. `AUDITA_CONFIG`
3. `/usr/local/etc/audita/config.yml` (if present)
4. `/etc/audita/config.yml` (if present)
Missing-path behavior:
- missing `--config` path is an error;
- missing `AUDITA_CONFIG` path is an error;
- missing both default paths is non-fatal.
## Effective precedence
`audita process` effective precedence:
1. defaults
2. file config
3. environment overrides
4. CLI overrides
`audita config print-effective` uses:
1. defaults
2. file config
3. environment overrides
`audita config validate` intentionally uses file-only validation:
1. defaults
2. file config
Environment overrides are not applied in `config validate`.
## Supported top-level YAML fields
```yaml
version: 1
pipeline:
modules: [glossary, homophones, glossary, spoken_word, grammar]
output:
schema: bare-segments
llm:
proposal:
base_url: https://openrouter.ai/api/v1
model: openrouter/google/gemma-4-31b-it
api_key_env: AUDITA_LLM_API_KEY
timeout: 120s
max_retries: 3
validation:
base_url: https://openrouter.ai/api/v1
model: openrouter/google/gemma-4-31b-it
api_key_env: AUDITA_VALIDATION_LLM_API_KEY
timeout: 120s
max_retries: 3
concurrency:
total_llm: 2
proposal_llm: 2
validation_llm: 1
chunking:
target_sections: 8
max_section_tokens: 8192
min_section_tokens: 2048
normalization:
max_segment_gap: 4s
ellipsis_gap: 3.5s
max_segment_duration: 60s
max_segment_tokens: 2048
thresholds:
glossary: 0.8
homophones: 0.8
spoken_word: 0.8
grammar: 0.8
context:
description: "optional transcript background context"
diagnostics:
work_dir: /tmp/audita
retention: auto
```
## Module and output-schema validation
`pipeline.modules` keys are validated against the built-in supported module catalog.
Supported module keys:
- `glossary`
- `homophones`
- `spoken_word`
- `grammar`
Repeated supported module keys are allowed.
`output.schema` is validated against the built-in output schema catalog.
Supported output schema keys:
- `bare-segments`
- `audita-v1`
Unknown module keys and unknown output schema keys fail validation.
## Duration field parsing
Duration-like fields support:
- numeric seconds (for example `120`, `3.5`)
- duration strings (for example `120s`, `2m`)
LLM timeout duration strings must resolve to whole seconds.
## Secret handling
Use `api_key_env` fields for secrets:
- `llm.proposal.api_key_env`
- `llm.validation.api_key_env`
These fields store environment variable names, not secret values.
Resolved secret values are redacted from:
- `audita config print-effective` output;
- diagnostics `effective-config.json`;
- report and diagnostics payloads.
## Commands
Validate a file config:
```sh
audita config validate --config ./audita.yml
```
Print redacted effective config:
```sh
audita config print-effective --config ./audita.yml
```
## Compatibility notes
Legacy compatibility flags and environment aliases remain available where implemented, but the stable configuration surface is the versioned YAML model described above.

33
docs/development.md Normal file
View File

@@ -0,0 +1,33 @@
# Audita Development Workflow
## Scope
This document defines the canonical contributor workflow and engineering conventions for this repository.
## Workflow
1. Start from a clean understanding of scope and constraints.
2. Make focused changes that preserve existing public behavior unless behavior change is explicitly intended.
3. Run targeted tests for touched packages.
4. Run `go test ./...` before finalizing substantial changes.
5. Update affected documentation so it describes current behavior only.
## Engineering conventions
- Keep module packages separate: `glossary`, `homophones`, `spoken_word`, `grammar`.
- Prefer narrow shared helpers and catalogs over broad abstractions.
- Preserve diagnostics artifact naming and report field contracts unless intentionally changed.
- Preserve CLI/config precedence semantics unless intentionally changed.
- Treat stable validator keys, prompt identifiers, and output-schema keys as contract surfaces.
## Configuration and runtime expectations
- `audita process` precedence is defaults -> file -> env -> CLI.
- `audita config validate` validates file config merged onto defaults only.
- `audita config print-effective` includes environment overrides and prints redacted JSON.
## Testing expectations
- Add tests for new behavior and for bug fixes.
- Keep deterministic fixtures stable.
- Do not reduce existing parity, release-fixture, subprocess, or module-specific coverage without equivalent replacement.
## Commit discipline
- Keep commits scoped and reviewable.
- Avoid mixing unrelated refactors with behavior changes.
- Use clear plain-English commit messages.

View File

@@ -0,0 +1,27 @@
# Documentation Policy
## Scope
This policy defines how project documentation should be authored and maintained.
## Core rules
- Document the current behavior of the codebase.
- Remove stale behavior descriptions promptly when code changes.
- Do not describe development history in architecture or behavior docs unless a document is explicitly historical.
- Do not use architecture or behavior docs as changelogs.
- Prefer rewriting stale sections from scratch when substantial behavior or ownership changes occur.
## Consistency requirements
- Keep command examples aligned with current CLI surfaces.
- Keep configuration examples aligned with supported fields and precedence.
- Keep architecture package ownership descriptions aligned with current code layout.
- Keep stable contract identifiers accurate (module keys, validator keys, output-schema keys, report metadata fields).
## Cross-document expectations
- `docs/architecture/*` documents runtime behavior and package ownership.
- `docs/configuration.md` documents config schema and precedence.
- `docs/development.md` documents contributor workflow and engineering conventions.
## Review expectations for documentation changes
- Verify referenced files and links exist.
- Verify examples match current behavior.
- Prefer concise, direct language and avoid speculative future claims.

View File

@@ -0,0 +1,96 @@
# Audita Subprocess Operations
This document describes how parent processes should invoke `audita process` safely in production orchestration.
## Recommended command form
Use explicit file outputs for orchestrated runs:
```sh
audita process <transcript.json> \
--transcript-description "Brief context that may help resolve ambiguous terms." \
--glossary <glossary.yaml> \
--output <output-transcript.json> \
--report-json <report.json>
```
Additional flags that may be situationally appropriate:
- `--config <path>` to select an explicit versioned config file.
- `--output-schema <bare-segments|audita-v1>` to select transcript output shape.
- `--work-dir <dir>` to control diagnostics location.
- `--work-dir-retention <always|auto|never>` to control retained run directories.
- `--total-llm-concurrency`, `--proposal-llm-concurrency`, and `--validation-llm-concurrency` when orchestration needs to set explicit LLM throughput controls.
- `--modules ...` only when intentionally overriding the default sequence.
For config-driven orchestration, validate config files in CI/preflight:
```sh
audita config validate --config <path>
```
## Stdout behavior
- With `--output`: stdout is expected to be empty on success.
- Without `--output`: stdout contains transcript JSON only on success.
- Report JSON is never written to stdout.
## Stderr behavior
- Success path should be quiet or minimal human-readable logs.
- Failure path writes concise human-readable errors.
- When a diagnostics run directory exists, failure stderr includes its path.
- Prompt/response diagnostic payloads are not streamed to stderr.
## Output file behavior
- `--output` writes transcript JSON in the selected output schema to the provided path.
- Output write failures return nonzero and surface actionable errors.
- The command does not silently ignore output write errors.
## Report JSON behavior
- `--report-json` writes a machine-readable process report to the requested path.
- Run-directory `report.json` is written independently under diagnostics.
- Best-effort failure reports are emitted when possible without masking the primary failure.
- Report write failures return nonzero with clear stderr messaging.
- Report diagnostics metadata references run-directory artifacts including utilization diagnostics and correction ledger paths when available.
## Diagnostics directory behavior
- Each run creates (when possible) a per-run diagnostics directory.
- Typical artifacts include transcript, normalization, chunking, invocation, effective config, LLM diagnostics, `utilization-diagnostics.json`, `correction-ledger.json`, `report.json`, and `error.log` on failure.
- Failed runs retain diagnostics.
- Under `auto` retention, successful runs with skipped/rejected corrections are retained; clean successful runs may be removed.
## Exit codes
- `0`: success.
- Nonzero: failure (input/schema/config/module/LLM/runtime/output/report/diagnostics errors).
Treat any nonzero as a failed subprocess invocation.
## Timeout and cancellation
- Runtime operations propagate context cancellation and request timeouts through LLM/scheduler paths.
- On cancellation or timeout, the process exits nonzero and should not hang.
- If diagnostics were initialized before failure, failure artifacts remain available for debugging.
## Secret redaction expectations
API keys and configured secret values are redacted from:
- reports (`--report-json` and run-dir `report.json`);
- diagnostics artifacts (including effective config and LLM interaction artifacts);
- surfaced adapter/runtime errors;
- test fixtures and regression outputs.
Parent-process logs should still avoid printing raw environment variables.
## Parent-process pipe guidance
To avoid deadlocks in orchestrators:
- always read both stdout and stderr concurrently when invoking as a subprocess;
- prefer file outputs (`--output`, `--report-json`) for machine workflows;
- treat stderr as human-readable diagnostics, not structured data;
- parse structured results from output/report files.
For Go callers, prefer `exec.CommandContext` with explicit timeout/cancellation and buffered/streamed readers for both pipes.

209
docs/policy/architecture.md Normal file
View File

@@ -0,0 +1,209 @@
# Architecture Policy
## Purpose
This document defines Audita's development architecture and invariants for maintainers and LLM coding agents. It describes how the project is intended to be changed safely, based on behavior implemented in this repository today.
User-facing behavior belongs in the README and focused runtime docs. Future or proposed work belongs only under `docs/roadmap/`.
## Project Shape
Audita is a single-process Go CLI for transcript polishing. The executable entrypoint is `cmd/audita`; command handling lives in `internal/cli`.
The implemented `audita process` flow is:
1. load effective config;
2. read and validate transcript JSON and glossary YAML;
3. normalize transcript segments;
4. chunk the working transcript into sections;
5. resolve configured module instances;
6. run correction modules and validator chains;
7. apply approved proposals deterministically;
8. write transcript output, reports, and diagnostics artifacts.
The current built-in modules are `glossary`, `homophones`, `spoken_word`, and `grammar`. The default configured module sequence repeats `glossary`.
For external behavior and compatibility details, prefer links to existing behavior docs:
- [Architecture overview](../architecture/architecture.md)
- [Public contract](../architecture/public-contract.md)
- [Diagnostics](../architecture/diagnostics.md)
- [Structured LLM](../architecture/structured-llm.md)
- [Validators](../architecture/validators.md)
- [Prompts](../architecture/prompts.md)
- [Output schemas](../architecture/output-schemas.md)
- [Configuration](../configuration.md)
## Core Design Principles
- **Hexagonal architecture:** keep domain behavior behind narrow internal contracts. CLI, filesystem, config loading, diagnostics writing, and LLM transport are adapters around the core processing flow.
- **Composable modules and validators:** correction stages and validators should remain small, explicit, and independently testable.
- **Deterministic orchestration around LLM calls:** LLM responses are nondeterministic inputs. Proposal indexing, validator ordering, proposal application, reports, and output serialization must remain deterministic.
- **Bounded and observable concurrency:** use the implemented schedulers and configured concurrency limits for LLM call sites. Preserve utilization diagnostics when changing scheduling or orchestration.
- **Conservative correction behavior:** validate proposed corrections before application; apply accepted proposals through deterministic apply-time safety checks.
- **Standard-library-first:** prefer the Go standard library. Narrow third-party dependencies are acceptable when they materially improve maintainability, such as `gopkg.in/yaml.v3` for YAML parsing.
- **Current-behavior documentation:** non-roadmap docs must describe implemented behavior only.
## Architectural Boundaries
`internal/core` owns domain data handling and stable runtime contracts that do not require CLI or provider transport knowledge:
- config defaults, loading, validation, redaction, and catalogs;
- transcript and glossary schemas;
- normalization and chunking;
- output-schema encoding;
- diagnostics artifact naming and run-directory helpers;
- public process report shapes.
`internal/framework` owns orchestration contracts and reusable runtime mechanics:
- module and validator interfaces;
- proposal generation, proposal application, and prompt context;
- runner orchestration;
- LLM scheduler, OpenAI-compatible adapter, redaction helpers, and diagnostics writers;
- structured response schema registry;
- process report and correction-ledger assembly.
`internal/modules/*` owns module-specific correction stages. `internal/validators/*` owns built-in validator implementations, registry, chains, and execution-class metadata. `internal/prompts` owns embedded prompt assets and prompt metadata.
`internal/cli` owns command parsing, exit codes, stdout/stderr behavior, config command behavior, filesystem input/output wiring, and top-level process orchestration. CLI concerns should not move into modules, validators, or schema logic.
Tests should stay close to the behavior they protect. Shared test helpers are acceptable when they remove clear duplication without hiding module-specific behavior.
## Modules and Validators
Modules implement `contracts.TranscriptModule`. A module must provide:
- a stable key;
- a replacement policy;
- a validator chain;
- proposal generation from explicit request inputs.
Module packages should stay separate. Do not collapse module-specific prompts, scope, or validation choices into a broad generic stage abstraction.
Validators implement the shared validator contract and return one decision per candidate proposal. Deterministic validators and LLM-backed validators are both composable chain elements. Validator identity and execution class metadata are stable enough to affect ordering, diagnostics, reports, and correction-ledger classification.
Future module or validator changes should preserve:
- explicit inputs and outputs;
- no hidden global state;
- explicit config dependencies;
- deterministic proposal index handling;
- validation before final mutation;
- stable reason codes and validator keys where already exposed.
## LLM Integration and Concurrency
LLM calls are external effects behind narrow contracts. Production structured completions use `contracts.StructuredLLMClient`; the implemented provider adapter is OpenAI-compatible HTTP code in `internal/framework/llm`.
Structured response schemas are registered in `internal/framework/responseschema`. Provider-side schema enforcement is not a substitute for local validation: Audita still validates proposal structure, validator decision cardinality, and apply-time safety.
Concurrency is bounded by configured scheduler limits:
- total LLM concurrency;
- proposal LLM concurrency;
- validation LLM concurrency.
The scheduler is context-aware and releases permits on success, failure, and cancellation. Runner code may collect section-level work concurrently, but transcript mutation is applied later in deterministic proposal-index order.
Diagnostics for LLM interactions should be useful for debugging without leaking configured secrets. Use the existing redaction helpers and `llm.ConfiguredSecrets`.
## State, Inputs, and Outputs
Audita does not implement resume, checkpoint, manifest, or remote storage behavior. Runtime state is in memory plus per-run diagnostics artifacts written under the configured work directory.
Transcript input accepts the implemented JSON forms documented in the public contract. Parsed source transcripts are normalized into Audita's internal transcript shape before chunking and module execution.
Proposals and validator decisions are intermediate runtime data. Approved proposals are applied through `internal/framework/proposals`, which clones transcript state, orders by proposal index, and records applied or skipped changes.
Transcript output is encoded through `internal/core/outputschema`. Reports and correction ledgers are machine-readable artifacts derived from runner outputs; their public shape should not be changed casually.
## Configuration and CLI Boundaries
Config behavior is owned by `internal/core/config`; command usage and process wiring are owned by `internal/cli`.
`audita process` uses implemented precedence: defaults, config file, environment, then CLI flags. `config validate` validates defaults plus a file config and intentionally does not apply environment overrides. `config print-effective` applies defaults, file config, and environment overrides, then prints redacted JSON.
Do not duplicate full CLI or config reference material here. Use [Configuration](../configuration.md), the README, and [Public contract](../architecture/public-contract.md) for current external behavior.
When adding config fields or CLI flags, update:
- config defaults, file/env/CLI application, and validation;
- CLI flag extraction if applicable;
- redaction when secrets are involved;
- tests for precedence and source-specific behavior;
- user-facing docs if external behavior changes.
## Errors, Logging, and Diagnostics
Errors should be phase-specific enough for CLI users and subprocess callers. The CLI writes human-readable errors to stderr and preserves transcript JSON-only stdout behavior on successful stdout output.
Run diagnostics are best-effort after run-directory creation. Failed runs are retained. Successful run retention follows the implemented work-dir retention policy.
Diagnostics and reports must not leak configured LLM secrets. Config redaction and LLM payload/error redaction are separate responsibilities and should remain separate.
Process reports, diagnostics metadata, utilization diagnostics, and correction ledgers are part of the public contract. Prefer additive, compatible changes.
## Testing Expectations
Use targeted package tests for touched behavior and `go test ./...` for substantial changes.
When changing modules, inspect or add:
- package-local module tests under `internal/modules/*`;
- prompt rendering or proposal-generation tests when prompt inputs change;
- parity or release fixtures when public output behavior changes.
When changing validators, inspect or add:
- validator package tests;
- registry and chain tests under `internal/validators`;
- framework validator tests for batching, malformed output, diagnostics, and cardinality.
When changing LLM integration or concurrency, inspect or add:
- `internal/framework/llm` scheduler/client/redaction tests;
- `internal/framework/runner` orchestration and utilization tests;
- structured-output malformed classification tests.
When changing config, CLI, schema, output, reports, or diagnostics, inspect or add:
- `internal/core/config` tests;
- CLI tests under `internal/cli`;
- schema and output-schema tests under `internal/core`;
- report, diagnostics, parity, and release-fixture tests.
## Dependency Policy
Audita should remain dependency-light. Prefer standard-library solutions for CLI parsing, HTTP, JSON, filesystem, synchronization, and tests.
Third-party dependencies should be narrow, justified, and preferably de facto standard for their purpose. YAML parsing is the current direct dependency exception.
Do not add broad frameworks for CLI, dependency injection, workflow orchestration, logging, or plugin systems without a concrete implemented need and focused tests.
## Documentation Expectations
Follow [Documentation Policy](./documentation.md). Architecture policy must stay concise and aligned with implemented behavior.
Do not use architecture docs as changelogs. Do not describe planned modules, adapters, modes, persistence, or configuration unless they are implemented. Put future work under `docs/roadmap/`.
## Architectural Invariants
- Keep LLM transport behind `StructuredLLMClient` and framework adapter boundaries.
- Keep correction modules narrowly scoped and package-separated.
- Keep validators modular, composable, and identified by stable keys.
- Keep CLI/config/filesystem concerns out of module and validator domain logic.
- Preserve deterministic transcript mutation and output handling around nondeterministic LLM calls.
- Keep LLM concurrency bounded, configurable, and observable where implemented.
- Keep run diagnostics and reports redacted and machine-readable.
- Keep public CLI, config, output-schema, diagnostics, report, prompt, module, and validator contracts stable unless a change is explicit and tested.
- Prefer small shared helpers over broad rewrites.
- Avoid new dependencies unless they are narrow and clearly justified.
## Non-Goals
- No plugin framework is implemented.
- No generic workflow engine is implemented.
- No resume, checkpoint, manifest, or remote storage system is implemented.
- No multi-process service mode is implemented.
- No provider SDK abstraction beyond the current structured LLM client contract and OpenAI-compatible HTTP adapter is implemented.

View File

@@ -0,0 +1,356 @@
# Go Project Documentation Policy
## Purpose
Project documentation must help four audiences:
1. users who need to run the application;
2. administrators/operators who need to configure and operate it;
3. developers who need to understand and change it safely;
4. LLM coding agents that need clear scope, boundaries, and invariants.
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
## Core Rules
### 1. Keep docs concise
Each document should cover a defined scope and only the essentials for that scope.
Avoid:
- long background explanations;
- repeated reference material;
- implementation detail in user-facing docs;
- aspirational language outside roadmap docs;
- verbose examples where one minimal example is clearer.
### 2. Document only implemented behavior outside roadmap files
Unimplemented, planned, aspirational, experimental, or future work may be described only under:
- `docs/roadmap/`
No other documentation file, including `README.md`, should describe code, features, modules, stages, commands, config fields, or behaviors that do not currently exist.
If a feature is partial, non-roadmap docs may describe only the implemented portion and its current boundary.
### 3. Use canonical homes
Each type of information should have one canonical location.
Canonical homes:
- project purpose and quickstart: `README.md`
- development principles: `docs/policy/architecture.md`
- configuration reference: `docs/config.md`
- CLI reference: `docs/cli.md`
- operations and recovery: `docs/operations.md`
- troubleshooting: `docs/troubleshooting.md`
- implemented internals: `docs/internal/`
- future work: `docs/roadmap/`
- contributor workflow: `docs/policy/development.md`
- copyable examples: `examples/`
Other files should summarize briefly and link to the canonical source.
### 4. Keep examples real
Examples should be valid, maintained, and free of secrets.
Where practical:
- example configs should load successfully;
- example commands should match real CLI syntax;
- important examples should be covered by tests.
## Documentation Profiles
All projects require:
- `README.md`
- `docs/policy/architecture.md`
Additional docs depend on the project.
### Small library
Recommended:
- `docs/policy/development.md`, if contributor conventions are non-obvious
### Simple CLI
Required:
- `docs/cli.md`
Recommended:
- `docs/policy/development.md`
### Config-driven CLI
Required:
- `docs/cli.md`
- `docs/config.md`
Recommended:
- `examples/`
- `docs/policy/development.md`
### Stateful or operator-facing application
Required:
- `docs/cli.md`, if CLI-based
- `docs/config.md`, if config-driven
- `docs/operations.md`
Recommended:
- `docs/troubleshooting.md`
- `examples/`
- `docs/policy/development.md`
### Modular, staged, service-oriented, or orchestration application
Required:
- `docs/cli.md`, if CLI-based
- `docs/config.md`, if config-driven
- `docs/operations.md`
- `docs/internal/`
- `docs/policy/development.md`
Recommended:
- `docs/troubleshooting.md`
- validated examples under `examples/`
## Required Documents
### README.md
**Audience:** users, administrators, operators
The README is the outward-facing project orientation page.
It should include, in order:
1. concise description;
2. elevator pitch;
3. shortest useful command or usage example;
4. links to targeted docs.
The README should be short. It is not a manual.
The “shortest useful command” means the simplest command that performs the projects core use case. (It does not mean `app --help`.)
### docs/policy/architecture.md
**Audience:** developers, LLM coding agents
`docs/policy/architecture.md` is required for every project.
It is an inward-facing development policy document. It should describe how the project is intended to be built and changed.
It should include:
- project shape;
- core design principles;
- package and boundary philosophy;
- state/persistence philosophy, if applicable;
- external integration philosophy, if applicable;
- error-handling and logging principles;
- testing expectations;
- documentation expectations;
- architectural invariants;
- explicit non-goals, if useful.
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
### docs/policy/development.md
**Audience:** developers, LLM coding agents
Required for projects maintained by humans and LLM coding agents.
It should include:
- repository layout;
- build/test commands;
- coding conventions;
- dependency policy;
- how to add config fields;
- how to add CLI flags;
- how to add stages/modules/adapters, if applicable;
- how to update examples;
- documentation update expectations.
### docs/config.md
**Audience:** administrators, operators, advanced users
Required for applications with configuration files.
It should include, in order:
1. config file locations and discovery precedence;
2. minimal working config;
3. production-oriented config;
4. full configuration reference;
5. secrets handling, if applicable;
6. links to maintained examples.
The full configuration reference should be canonical.
### docs/cli.md
**Audience:** users, administrators, operators
Required for CLI applications.
It should include, in order:
1. shortest useful command;
2. command overview;
3. complete flag reference;
4. common workflows;
5. diagnostic or recovery commands, if applicable.
Explain when commands are useful, not just their syntax.
### docs/operations.md
**Audience:** administrators, operators
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
It should cover:
- normal workflow;
- filesystem layout;
- remote storage layout, if applicable;
- logs and manifests;
- resume/retry behavior;
- cleanup behavior;
- archive/backup behavior;
- safe recovery procedures;
- operational caveats.
### docs/troubleshooting.md
**Audience:** administrators, operators
Recommended once recurring failure modes exist.
Each entry should include:
- symptom;
- likely cause;
- diagnostic command or inspection step;
- safe fix;
- relevant links.
### docs/internal/
**Audience:** developers, LLM coding agents
Required for modular, staged, service-oriented, or orchestration projects.
This directory describes implemented internal components. It is not the roadmap.
Use one file per major component where useful.
Each component doc should include:
1. purpose;
2. inputs and outputs;
3. boundaries;
4. config fields used;
5. external adapters used;
6. state or manifest behavior, if applicable;
7. skip/resume behavior, if applicable;
8. failure behavior;
9. tests to inspect before changing;
10. architectural invariants.
### docs/roadmap/
**Audience:** maintainers, developers, LLM coding agents
This is the only place for planned, future, aspirational, experimental, or unimplemented work.
Roadmap docs should clearly distinguish:
- proposed work;
- accepted plans;
- deferred ideas;
- rejected ideas;
- implementation prompts or task breakdowns, if useful.
Roadmap docs should not be confused with current behavior.
### docs/integrations/
**Audience:** developers, LLM coding agents
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
Use one file per integration where useful.
## Examples Directory
Projects with non-trivial configuration or workflows should include `examples/`.
Useful examples include:
- minimal working config;
- production-oriented config;
- full annotated config;
- local development config;
- remote/object-storage config;
- minimal session/input file.
Examples should be valid, maintained, tested when practical, and linked from relevant docs.
## Security and Privacy
Docs and examples must not include:
- real API keys;
- tokens;
- passwords;
- private keys;
- private environment dumps;
- sensitive user data;
- raw private transcripts;
- private infrastructure details unless intentionally public.
Document secret-handling mechanisms, not actual secret values.
## Maintenance Rules
When docs change, verify the affected behavior.
Where practical:
- load example config files in tests;
- test CLI examples or command parser behavior;
- validate documented flags against real flags;
- remove stale references;
- update links after renames;
- keep roadmap content out of non-roadmap docs.
If documentation and code disagree, fix the documentation and/or open a roadmap item; do not leave aspirational behavior in current-behavior docs.
Documentation is complete only when it matches the current code.
## Documentation Change Checklist
Before merging documentation changes, verify:
- README is concise and orientation-focused.
- `docs/policy/architecture.md` describes development principles.
- Future work appears only under `docs/roadmap/`.
- User-facing docs avoid unnecessary internals.
- Developer-facing docs preserve boundaries and invariants.
- Config examples match the schema.
- CLI examples match real commands and flags.
- Defaults appear in the canonical config reference.
- No secrets or private data are included.
- Links are accurate.

128
docs/release-checklist.md Normal file
View File

@@ -0,0 +1,128 @@
# Audita Release Checklist
Use this checklist before cutting a pre-1.0 or 1.0 release candidate.
## Core test pass
- Run:
- `go test ./...`
- Confirm tests pass without live LLM credentials and without Python dependencies.
## Config validation and precedence
- Validate a representative config:
- `audita config validate --config <path>`
- Inspect redacted effective config:
- `audita config print-effective --config <path>`
- Confirm precedence behavior:
- defaults -> file config -> environment -> CLI.
- Confirm default config search order:
- `/usr/local/etc/audita/config.yml` first, then `/etc/audita/config.yml`.
- Confirm missing both default-path config files is non-fatal when `--config`/`AUDITA_CONFIG` are unset.
## Output schema checks
- Verify default output schema remains `bare-segments`.
- Verify `--output-schema audita-v1` emits object payload with `schema` and `version`.
- Verify unknown schema (for example `seriatim-intermediate`) fails clearly.
## Subprocess contract checks
- With `--output`, verify stdout is empty on success.
- Without `--output`, verify stdout contains transcript JSON only.
- Verify `--report-json` writes file output and does not write report JSON to stdout.
- Verify failure stderr remains human-readable and includes diagnostics path when available.
- Verify nonzero exit on failures.
## Structured LLM checks
- Verify runtime uses the Audita-owned OpenAI-compatible adapter.
- Verify structured response schemas are attached via `response_format.type=json_schema`.
- Verify diagnostics metadata includes structured schema `id/version/name/sha256`.
- Verify provider output is still locally decoded/validated before use.
- Verify malformed module-stage structured payloads degrade to warnings/rejections instead of failing the run.
## Report and diagnostics schema checks
- Verify report metadata fields:
- `report_schema_name`
- `report_schema_version`
- `output_schema`
- `config_version` when file config is used.
- Verify diagnostics artifact references exist in reports:
- transcript/normalization/chunking/invocation/effective-config artifacts
- utilization diagnostics artifact
- correction ledger artifact
- error log on failures.
## Redaction checks
- Verify secrets are redacted from:
- `effective-config.json`
- run-dir and `--report-json` reports
- LLM request/response/error diagnostics payloads.
- Verify no API keys/bearer tokens leak into fixtures or outputs.
## Prompt and validator metadata checks
- Verify prompt metadata appears in LLM request metadata diagnostics:
- `prompt_id`, `prompt_version`, `prompt_source`, `embedded_path`, `sha256`.
- Verify stable validator keys appear in report decisions/rejections.
- Verify module warning records appear in reports for malformed proposal-generation payloads and malformed validator batches.
- Verify built-in validator chains resolve and execute for default and explicit module runs.
## Utilization diagnostics checks
- Verify `utilization-diagnostics.json` exists on successful runs.
- Verify partial utilization artifact behavior on controlled failure paths.
- Verify utilization fields are structurally present and nonnegative:
- effective concurrency
- run timing
- module timing summaries
- per-validator timing summaries.
## Correction ledger checks
- Verify `correction-ledger.json` exists on successful runs.
- Verify report references ledger artifact path.
- Verify ledger dispositions include applied/rejected and skipped/failed where exercised.
- Verify validator rejection and proposal-application skip remain distinct.
## Pipeline behavior checks
- Verify default full pipeline run remains:
- `glossary`, `homophones`, `glossary`, `spoken_word`, `grammar`
- with deterministic repeated instance naming (`glossary_1`, `glossary_2`).
- Verify explicit module runs (`--modules`) still work.
## Failure and cancellation checks
- Verify controlled failure paths retain diagnostics and produce best-effort failure reports.
- Verify malformed proposal-generation payloads keep exit code `0`, keep stderr empty on success, and record warnings in reports/diagnostics.
- Verify malformed validator payloads reject only the affected batch and do not fail the module.
- Verify timeout/cancellation paths exit nonzero, do not hang, and retain failure diagnostics when initialized.
## Release fixture/idempotence checks
- Run release fixtures (`internal/cli/testdata/release`) through `go test ./...`.
- Confirm fixture checks cover:
- must-apply and must-not-apply expectations
- protected-term survival
- report and diagnostics contracts
- output-schema checks
- prompt/schema metadata diagnostics
- utilization/ledger artifacts
- idempotence-oriented second pass no-op behavior with deterministic fake responses.
## Deferred-feature guardrail
- Confirm release docs do not claim support for deferred items:
- filesystem prompt overrides
- user-configurable validator chains
- arbitrary user-supplied output schemas
- resume/start-at/stop-after execution
- diff/check/propose-only modes
- generated transcript descriptions enabled by default
- interactive review UI
- UI/server wrapper
- provider benchmarking harness.

View File

@@ -0,0 +1,577 @@
# Documentation Roadmap
## Purpose
This roadmap defines the work required to bring Audita documentation into compliance with `docs/policy/documentation.md` and the implemented architecture described by `docs/policy/architecture.md`.
This is an implementation plan for future documentation cleanup. It does not rewrite the main documentation. Future implementation passes should document only current behavior outside `docs/roadmap/`, keep planned or unimplemented work in roadmap files, and verify claims against repository code and tests rather than stale documentation.
## Repository Documentation Inventory
- `README.md`: keep and rewrite. It should remain the project orientation and quickstart, but it currently carries too much reference material and includes stale links such as `docs/diagnostics.md`, `docs/structured-llm.md`, and `docs/subprocess-operations.md`.
- `docs/policy/documentation.md`: keep and lightly update only if needed. It is the canonical documentation policy.
- `docs/policy/architecture.md`: keep and lightly verify after the migration. It is the canonical architecture policy for developers and coding agents.
- `docs/development.md`: move and rewrite as `docs/policy/development.md`. Contributor workflow belongs under `docs/policy/`.
- `docs/configuration.md`: move and rewrite as `docs/config.md`. Configuration reference belongs at the canonical config path.
- `docs/architecture.md`: merge or delete after the internal docs are created. Its useful content should become an internal overview or links to canonical internal docs.
- `docs/architecture/architecture.md`: split and rewrite into `docs/internal/overview.md` and `docs/internal/pipeline.md`.
- `docs/architecture/public-contract.md`: split across `docs/cli.md`, `docs/config.md`, `docs/operations.md`, and integration docs where applicable.
- `docs/architecture/diagnostics.md`: split across `docs/operations.md` and `docs/internal/diagnostics-reporting.md`.
- `docs/architecture/structured-llm.md`: split across `docs/internal/llm-runtime.md` and `docs/integrations/openai-compatible-llm.md`.
- `docs/architecture/validators.md`: move and rewrite as `docs/internal/validators.md`.
- `docs/architecture/prompts.md`: move and rewrite as `docs/internal/prompts.md`; remove deferred and unimplemented prompt override material.
- `docs/architecture/output-schemas.md`: move and rewrite as `docs/internal/output-schemas.md`; remove deferred or unimplemented schema material such as `seriatim-intermediate`.
- `docs/documentation/policy.md`: merge/delete in favor of `docs/policy/documentation.md`. It duplicates policy material in a noncanonical location.
- `docs/integration/subprocess-operations.md`: move and rewrite as `docs/integrations/subprocess.md`.
- `docs/release-checklist.md`: merge current-behavior checks into `docs/policy/development.md` or move to a clearer policy/internal location; remove pre-release or deferred-feature guardrail language from non-roadmap docs.
- `docs/roadmap/audit.md`: currently deleted in the worktree. Treat this as unrelated state unless a later task explicitly restores or updates it.
- `docs/roadmap/implementation.md`: currently deleted in the worktree. Treat this as unrelated state unless a later task explicitly restores or updates it.
- `examples/`: create new. No examples directory is currently present, but policy expects copyable examples when practical.
## Policy Compliance Assessment
Required or expected canonical documents are missing:
- `docs/cli.md`
- `docs/config.md`
- `docs/operations.md`
- `docs/internal/`
- `docs/policy/development.md`
- `examples/`
Recommended documents that should be added:
- `docs/troubleshooting.md`
- `docs/integrations/openai-compatible-llm.md`
- `docs/integrations/transcript-glossary-files.md`
Documents in the wrong canonical home:
- `docs/configuration.md` should become `docs/config.md`.
- `docs/development.md` should become `docs/policy/development.md`.
- `docs/integration/` should become `docs/integrations/`.
- Implemented internal architecture content under `docs/architecture/` should move to `docs/internal/`.
- `docs/documentation/policy.md` should merge/delete in favor of `docs/policy/documentation.md`.
Content that should not remain outside `docs/roadmap/`:
- Deferred or unimplemented output schema content in `docs/architecture/output-schemas.md`.
- Deferred or unimplemented prompt override, generated transcript description, and report prompt ledger content in `docs/architecture/prompts.md`.
- Pre-release or future-feature guardrail language in `docs/release-checklist.md`, unless moved to roadmap or rewritten as current contributor workflow.
Examples and links:
- `examples/` is missing.
- README links to nonexistent documentation paths.
- Links to `docs/configuration.md`, `docs/development.md`, and `docs/integration/` should be updated after canonical moves.
- A repository-wide link/path check should be part of final validation.
## Target Documentation Set
### `README.md`
- Audience: users and operators.
- Purpose: concise project orientation and shortest useful workflow.
- Canonical scope: what Audita does, install/build basics, minimal command shape, and links to canonical docs.
- Recommended outline: overview, quickstart, minimal configuration pointer, common command pointer, documentation map, development pointer.
- Sources to inspect: `cmd/audita/main.go`, `internal/cli/run.go`, `internal/cli/process_flags.go`, README tests or CLI integration tests.
- Acceptance criteria: no long CLI or config reference; no stale links; all linked docs exist.
### `docs/cli.md`
- Audience: users and operators.
- Purpose: canonical CLI reference.
- Canonical scope: commands, flags, common workflows, output destinations, stdout/stderr behavior, and exit behavior.
- Recommended outline: command overview, `process`, `config validate`, `config print-effective`, config path selection, process outputs, examples, exit behavior.
- Sources to inspect: `internal/cli/run.go`, `internal/cli/process_flags.go`, `cmd/audita`, CLI tests.
- Acceptance criteria: every implemented command and flag is documented; examples match parser behavior; config details link to `docs/config.md`.
### `docs/config.md`
- Audience: administrators, operators, and advanced users.
- Purpose: canonical configuration reference.
- Canonical scope: config path resolution, precedence, YAML schema, environment overrides, CLI override relationship, secrets, validation.
- Recommended outline: loading model, precedence, file schema, environment variables, CLI relationship, secrets, examples, validation.
- Sources to inspect: `internal/core/config/*`, config tests, CLI config commands.
- Acceptance criteria: replaces `docs/configuration.md`; documents implemented defaults and validation only; examples validate.
### `docs/operations.md`
- Audience: operators.
- Purpose: operational behavior and recovery/debugging reference.
- Canonical scope: run directories, diagnostics artifacts, reports, correction ledger, retention, output writes, failure inspection.
- Recommended outline: process run lifecycle, output files, diagnostics directory, reports, retention, operational failure modes, recovery steps.
- Sources to inspect: `internal/core/diagnostics`, `internal/framework/processreport`, `internal/cli`, reporting tests.
- Acceptance criteria: no resume, checkpoint, or remote storage claims; operational artifacts match implemented filenames and report behavior.
### `docs/troubleshooting.md`
- Audience: users and operators.
- Purpose: concise guide for recurring implemented failures.
- Canonical scope: symptoms, likely causes, inspection steps, and safe fixes.
- Recommended outline: config validation errors, transcript/glossary schema errors, LLM request errors, output/report write failures, diagnostics lookup.
- Sources to inspect: CLI tests, config tests, schema tests, LLM tests, reporting tests.
- Acceptance criteria: every entry maps to implemented behavior; no speculative remediation.
### `docs/policy/documentation.md`
- Audience: maintainers and coding agents.
- Purpose: canonical documentation policy.
- Canonical scope: documentation layout, audience boundaries, roadmap rules, maintenance rules.
- Recommended outline: keep current structure unless policy itself needs small alignment.
- Sources to inspect: documentation policy and final documentation tree.
- Acceptance criteria: remains the only canonical documentation policy.
### `docs/policy/architecture.md`
- Audience: developers and coding agents.
- Purpose: canonical architecture policy.
- Canonical scope: development principles, boundaries, invariants, dependency policy, testing expectations.
- Recommended outline: keep current policy; update links after docs migration only if necessary.
- Sources to inspect: package layout and policy docs.
- Acceptance criteria: no stale links; no duplicated CLI/config reference.
### `docs/policy/development.md`
- Audience: developers and coding agents.
- Purpose: contributor workflow and change expectations.
- Canonical scope: repo layout, setup, tests, conventions, adding config/CLI/module/validator/docs/examples.
- Recommended outline: setup, repository layout, running tests, change workflow, adding features, documentation expectations, release checks.
- Sources to inspect: `docs/development.md`, tests, `go.mod`, package layout.
- Acceptance criteria: replaces `docs/development.md`; no future-feature roadmap content; includes practical validation commands.
### `docs/internal/overview.md`
- Audience: developers and coding agents.
- Purpose: implemented internal architecture overview.
- Canonical scope: core/framework/module/validator/adapter layout at a high level.
- Recommended outline: package map, main execution path, boundary summary, where to add new code.
- Sources to inspect: `internal/core`, `internal/framework`, `internal/modules`, `internal/validators`, `internal/cli`.
- Acceptance criteria: concise internal entry point; links to detailed internal docs.
### `docs/internal/pipeline.md`
- Audience: developers and coding agents.
- Purpose: implemented process pipeline.
- Canonical scope: transcript loading, normalization, chunking, module proposal generation, validation, deterministic application, output/report handoff.
- Recommended outline: inputs, pipeline phases, runner outputs, failure behavior, tests.
- Sources to inspect: `internal/framework/runner`, `internal/core/normalization`, `internal/core/chunking`, CLI process tests.
- Acceptance criteria: no unimplemented workflow engine or resume claims.
### `docs/internal/modules.md`
- Audience: developers and coding agents.
- Purpose: module authoring and maintenance reference.
- Canonical scope: current module packages, module contracts, proposal behavior, prompt assets.
- Recommended outline: module contract, implemented modules, prompt ownership, proposal output, tests.
- Sources to inspect: `internal/modules/*`, `internal/framework/contracts`, `internal/framework/proposal_generation`.
- Acceptance criteria: keeps module packages separate; no plugin architecture claims.
### `docs/internal/validators.md`
- Audience: developers and coding agents.
- Purpose: validator architecture reference.
- Canonical scope: validator registry, chains, deterministic and LLM-backed validators, decision handling.
- Recommended outline: validator contract, chain registration, classifications, batching, failure behavior, tests.
- Sources to inspect: `internal/validators`, `internal/framework/validators`.
- Acceptance criteria: documents composable validators without inventing new validator APIs.
### `docs/internal/llm-runtime.md`
- Audience: developers and coding agents.
- Purpose: internal LLM runtime and scheduler reference.
- Canonical scope: `StructuredLLMClient`, OpenAI-compatible adapter boundary, retries, redaction, scheduler permits, structured response handling.
- Recommended outline: client interface, request/response handling, retries/timeouts, concurrency, diagnostics, tests.
- Sources to inspect: `internal/framework/llm`, `internal/framework/responseschema`, `internal/framework/structuredoutput`.
- Acceptance criteria: documents only implemented OpenAI-compatible HTTP behavior.
### `docs/internal/diagnostics-reporting.md`
- Audience: developers and coding agents.
- Purpose: diagnostics, report, and correction ledger implementation reference.
- Canonical scope: artifact names, metadata, process report mapping, correction ledger, retention interaction.
- Recommended outline: diagnostics ownership, artifact metadata, process report builder, ledger mapping, tests.
- Sources to inspect: `internal/core/diagnostics`, `internal/core/reporting`, `internal/framework/processreport`, CLI report tests.
- Acceptance criteria: filenames and report fields match code; no planned artifact claims.
### `docs/internal/prompts.md`
- Audience: developers and coding agents.
- Purpose: implemented prompt registry and prompt asset reference.
- Canonical scope: embedded prompt assets, prompt metadata, rendering inputs, module prompt ownership.
- Recommended outline: registry, assets, metadata, module usage, tests.
- Sources to inspect: `internal/prompts`, `internal/framework/promptcontext`, module prompt tests.
- Acceptance criteria: removes unimplemented filesystem overrides and deferred prompt ledger content.
### `docs/internal/output-schemas.md`
- Audience: developers and coding agents.
- Purpose: implemented output schema registry reference.
- Canonical scope: supported output schemas, config validation, output emission.
- Recommended outline: registry, `bare-segments`, `audita-v1`, validation, tests.
- Sources to inspect: `internal/core/outputschema`, `internal/core/config`, schema/output tests.
- Acceptance criteria: documents only implemented schemas.
### `docs/integrations/subprocess.md`
- Audience: operators and external-process integrators.
- Purpose: subprocess invocation contract.
- Canonical scope: invoking `audita process`, stdin/stdout/stderr expectations where implemented, files, reports, exit codes.
- Recommended outline: invocation model, outputs, diagnostics, errors, parent-process guidance.
- Sources to inspect: `internal/cli`, subprocess-oriented docs, CLI integration tests.
- Acceptance criteria: no non-existent streaming API or server mode.
### `docs/integrations/openai-compatible-llm.md`
- Audience: developers and operators integrating an LLM endpoint.
- Purpose: OpenAI-compatible LLM contract.
- Canonical scope: chat completions request behavior, JSON schema response format, retries, timeouts, redaction, configured endpoints.
- Recommended outline: endpoint expectations, authentication, response format, retry/timeout behavior, diagnostics and redaction.
- Sources to inspect: `internal/framework/llm`, config LLM settings, LLM tests.
- Acceptance criteria: no provider SDK or non-OpenAI-compatible API claims.
### `docs/integrations/transcript-glossary-files.md`
- Audience: users, operators, and external systems producing input files.
- Purpose: accepted transcript and glossary file contracts.
- Canonical scope: implemented JSON/YAML shapes and validation behavior.
- Recommended outline: transcript shape, glossary shape, validation errors, example files.
- Sources to inspect: `internal/core/schema`, schema tests, CLI input tests.
- Acceptance criteria: does not invent a formal versioned schema beyond implemented fields.
### `examples/`
- Audience: users and operators.
- Purpose: copyable, maintained examples.
- Canonical scope: minimal and fuller config, tiny transcript, tiny glossary.
- Recommended files: `minimal-config.yml`, `production-config.yml`, `tiny-transcript.json`, `tiny-glossary.yaml`.
- Sources to inspect: config defaults/tests, schema tests, CLI tests.
- Acceptance criteria: no secrets; config examples validate; examples are linked from README, CLI, and config docs.
### `docs/roadmap/documentation.md`
- Audience: maintainers and coding agents.
- Purpose: staged documentation migration plan.
- Canonical scope: future documentation work only.
- Recommended outline: this file.
- Sources to inspect: repository docs, code, tests, documentation policy, architecture policy.
- Acceptance criteria: remains action-oriented and does not rewrite current documentation prematurely.
## File-by-File Rewrite Guidance
### README
Cover project purpose, shortest useful command, build/test basics, and links to canonical docs. Avoid full CLI flag lists, full config schema, diagnostics reference, module internals, and architectural history. Link to `docs/cli.md`, `docs/config.md`, `docs/operations.md`, `docs/troubleshooting.md`, and policy docs after those files exist. Do not carry forward stale links to nonexistent `docs/diagnostics.md`, `docs/structured-llm.md`, or `docs/subprocess-operations.md`.
### `docs/config.md`
Rewrite from `docs/configuration.md`. Cover path resolution, precedence, YAML schema, env overrides, CLI override relationship, validation, and secrets. Link to `docs/cli.md` for command syntax and to examples for copyable files. Inspect `internal/core/config/*` and config tests. Avoid duplicating every CLI flag except where needed to explain precedence.
### `docs/cli.md`
Build from `internal/cli/run.go`, `internal/cli/process_flags.go`, and CLI tests. Cover `process`, `config validate`, and `config print-effective`. Include implemented output destinations and subprocess-friendly behavior. Link to `docs/config.md` for configuration details and `docs/operations.md` for diagnostics and reports. Avoid documenting unsupported command aliases or future commands.
### `docs/operations.md`
Merge operational material from diagnostics and subprocess docs. Cover run directories, diagnostics artifacts, reports, correction ledger, retention, output/report writes, and safe failure inspection. Inspect `internal/core/diagnostics`, `internal/framework/processreport`, and CLI tests. State that resume, checkpoint, and remote storage are not implemented only if needed to avoid user confusion.
### `docs/troubleshooting.md`
Create concise symptom/cause/inspect/fix entries for implemented failures. Inspect config validation tests, schema tests, LLM adapter tests, reporting tests, and CLI integration tests. Avoid broad operational advice that is not supported by the repository.
### `docs/policy/development.md`
Move and rewrite from `docs/development.md`. Cover setup, package layout, tests, conventions, and how to add config fields, CLI flags, modules, validators, docs, and examples. Merge any still-useful current-behavior release checks from `docs/release-checklist.md`. Avoid roadmap, pre-1.0 history, and deferred-feature guardrail language.
### `docs/internal/*`
Move implemented architecture details out of `docs/architecture/*`. Keep these docs concise and developer-facing. Remove deferred or unimplemented sections such as `seriatim-intermediate`, prompt overrides, generated transcript descriptions, report-level prompt ledgers, plugin systems, workflow engines, resume, and remote storage.
### `docs/integrations/subprocess.md`
Move from `docs/integration/subprocess-operations.md`. Keep stdout/stderr, file outputs, exit behavior, diagnostics/report handling, and parent-process guidance that matches current CLI behavior. Do not document non-existent streaming APIs.
### `docs/integrations/openai-compatible-llm.md`
Derive from implemented `internal/framework/llm` behavior and the current structured LLM architecture doc. Cover OpenAI-compatible chat completions, `response_format.type=json_schema`, retries, timeouts, and redaction. Do not claim support for provider SDKs or non-OpenAI-compatible APIs.
### `docs/integrations/transcript-glossary-files.md`
Create from implemented schema loading and validation. Cover the file shapes accepted by Audita and link to examples. Do not invent a formal external schema version beyond what the code validates.
### `docs/documentation/policy.md`
Delete after verifying any unique useful policy content is already in `docs/policy/documentation.md`. Do not keep two documentation policy homes.
### `docs/release-checklist.md`
Either merge current-behavior contributor checks into `docs/policy/development.md` or move a concise checklist to a clearer policy/internal location. Remove future-feature or deferred-work guardrails from non-roadmap documentation.
## Examples Plan
Create maintained, non-secret examples only for implemented behavior.
### `examples/minimal-config.yml`
- Purpose: smallest useful config with `version: 1`, output schema, and `api_key_env`.
- Expected validity check: `go run ./cmd/audita config validate --config examples/minimal-config.yml`.
- Link from: `README.md`, `docs/config.md`, `docs/cli.md`.
### `examples/production-config.yml`
- Purpose: fuller config showing modules, LLMs, concurrency, chunking, normalization, thresholds, context, and diagnostics.
- Expected validity check: `go run ./cmd/audita config validate --config examples/production-config.yml`.
- Link from: `docs/config.md`.
### `examples/tiny-transcript.json`
- Purpose: small copyable transcript input for CLI examples and schema documentation.
- Expected validity check: schema tests or a no-live-LLM CLI parser path if practical.
- Link from: `README.md`, `docs/cli.md`, `docs/integrations/transcript-glossary-files.md`.
### `examples/tiny-glossary.yaml`
- Purpose: small copyable glossary input for CLI examples.
- Expected validity check: schema tests or a no-live-LLM CLI parser path if practical.
- Link from: `README.md`, `docs/cli.md`, `docs/integrations/transcript-glossary-files.md`.
Do not add examples for resume, remote storage, prompt overrides, plugin systems, UI/server mode, unsupported output schemas, or other unimplemented behavior.
## Internal Documentation Plan
### Pipeline
- Path: `docs/internal/pipeline.md`
- Purpose: document the implemented transcript processing pipeline.
- Inputs and outputs: normalized transcript, sections, configured module specs, proposal results, validation results, runner output.
- Boundaries: runner orchestrates; modules propose; validators filter; accepted proposals are applied deterministically.
- Config fields used: modules, output schema, chunking, normalization, thresholds, concurrency, context, diagnostics.
- Adapters used: LLM client through framework contracts; filesystem/reporting through CLI and diagnostics boundaries.
- Failure behavior: module and validator warnings, rejected proposals, run/report error status.
- Tests to inspect: runner tests, proposal generation tests, CLI parity and release fixture tests.
- Architectural invariants: keep nondeterministic LLM effects isolated from deterministic transcript state handling.
### Modules
- Path: `docs/internal/modules.md`
- Purpose: document implemented correction modules and their contracts.
- Inputs and outputs: `contracts.ProposalRequest`, module proposals, warnings, replacement policies.
- Boundaries: one package per module; prompt assets remain module-specific; shared framework plumbing stays outside module packages.
- Config fields used: configured module keys, LLM settings, chunking/context where applicable.
- Adapters used: LLM client only through contracts and proposal generation framework.
- Failure behavior: proposal warnings and malformed LLM output handling as implemented.
- Tests to inspect: `internal/modules/...` and proposal generation tests.
- Architectural invariants: keep module scope narrow and avoid hidden global state.
### Validators
- Path: `docs/internal/validators.md`
- Purpose: document validator composition and decision handling.
- Inputs and outputs: candidate proposals, validator decisions, rejection reasons, warnings.
- Boundaries: validator registry and chains live in `internal/validators`; runtime mechanics live in `internal/framework/validators`.
- Config fields used: thresholds, validation LLM settings, validation concurrency, validation prompt limits.
- Adapters used: LLM-backed validators use the LLM contract rather than direct transport.
- Failure behavior: rejected proposals, warning behavior, malformed output policy.
- Tests to inspect: validator registry, chain, batching, malformed output, protected terms, and LLM validator tests.
- Architectural invariants: validators remain modular and composable.
### LLM Runtime
- Path: `docs/internal/llm-runtime.md`
- Purpose: document structured LLM calls and bounded scheduling.
- Inputs and outputs: structured prompt requests, response schemas, parsed responses, scheduler permit results, diagnostics metadata.
- Boundaries: transport stays behind `StructuredLLMClient`; scheduler manages permits; response schema registry owns schema metadata.
- Config fields used: model, base URL, API key, timeout, retries, total/proposal/validation concurrency, validation max prompt tokens.
- Adapters used: OpenAI-compatible HTTP adapter.
- Failure behavior: retries, timeout/context handling, malformed structured output handling, redacted errors.
- Tests to inspect: LLM client, scheduler, redaction, response schema, structured output tests.
- Architectural invariants: keep concurrency bounded and explicit; do not leak secrets in diagnostics.
### Diagnostics and Reporting
- Path: `docs/internal/diagnostics-reporting.md`
- Purpose: document diagnostics artifacts, process reports, and correction ledger generation.
- Inputs and outputs: run directory artifacts, diagnostics metadata, process report JSON, correction ledger entries.
- Boundaries: diagnostics owns artifact names and metadata; processreport maps runner output to reporting structures; CLI chooses output destinations.
- Config fields used: work dir, work-dir retention, transcript description.
- Adapters used: filesystem through diagnostics/CLI boundaries.
- Failure behavior: report status/error mapping and artifact write errors as implemented.
- Tests to inspect: diagnostics tests, processreport tests, CLI report fixture tests.
- Architectural invariants: preserve diagnostics filenames and report JSON shape unless intentionally changed and documented.
### Prompts
- Path: `docs/internal/prompts.md`
- Purpose: document implemented prompt registry, embedded assets, and metadata.
- Inputs and outputs: prompt identifiers, prompt asset content, rendered prompt payloads, diagnostic metadata.
- Boundaries: prompt assets remain owned by module/framework areas that use them; no filesystem override mechanism is implemented.
- Config fields used: transcript description/context where applicable.
- Adapters used: none directly; prompts are consumed by LLM-backed framework code.
- Failure behavior: missing or malformed embedded prompt assets should surface through tests or runtime errors as implemented.
- Tests to inspect: prompt registry and module prompt tests.
- Architectural invariants: keep prompt metadata consistent with diagnostics.
### Output Schemas
- Path: `docs/internal/output-schemas.md`
- Purpose: document implemented output schema registry and report/output relationship.
- Inputs and outputs: configured output schema key, validated schema support, emitted transcript output.
- Boundaries: output schema registry lives in `internal/core/outputschema`; config validation consumes registry support.
- Config fields used: output schema.
- Adapters used: none directly.
- Failure behavior: unsupported schema keys fail validation.
- Tests to inspect: output schema and config validation tests.
- Architectural invariants: do not document unsupported schemas as current behavior.
## Integration Documentation Plan
### `docs/integrations/subprocess.md`
- External system or contract: parent process invoking the `audita` CLI.
- Current usage in Audita: `audita process` writes output/report files and emits subprocess-friendly diagnostics and errors.
- Version or compatibility notes: document only the current CLI behavior and implemented exit behavior.
- What to document: invocation model, command examples, output files, report JSON path, stderr/stdout expectations, diagnostics, exit codes.
- What not to document: streaming protocols, server mode, remote job control, resume APIs.
### `docs/integrations/openai-compatible-llm.md`
- External system or contract: OpenAI-compatible chat completions endpoint using JSON schema response format.
- Current usage in Audita: configured primary and validation LLM clients issue structured chat completion requests with retries/timeouts and redaction.
- Version or compatibility notes: document compatibility based on request behavior in `internal/framework/llm`, not provider marketing claims.
- What to document: endpoint configuration, authentication, request/response expectations, `response_format.type=json_schema`, retries, timeouts, redaction.
- What not to document: unsupported provider SDKs, non-OpenAI-compatible APIs, unimplemented model-routing features.
### `docs/integrations/transcript-glossary-files.md`
- External system or contract: transcript JSON and glossary YAML files accepted as inputs.
- Current usage in Audita: CLI loads transcript and glossary files before processing and validates their shape through core schema code.
- Version or compatibility notes: document implemented fields and validation behavior only.
- What to document: accepted file shapes, required/optional fields, common validation errors, tiny examples.
- What not to document: a formal versioned external schema that the code does not enforce.
## Recommended Implementation Sequence
### Stage 1: Roadmap Creation
- Goal: create this documentation roadmap.
- Files to create/update/delete/move: create `docs/roadmap/documentation.md` only.
- Repository areas to inspect: documentation policy, architecture policy, existing docs, CLI/config/package/test layout.
- Acceptance criteria: roadmap is action-oriented, staged, and limited to future documentation work.
- Suggested validation commands: `git diff --check -- docs/roadmap/documentation.md`.
- One prompt: yes.
### Stage 2: Canonical Layout and README Links
- Goal: establish canonical paths and remove obvious stale links without rewriting all content.
- Files to create/update/delete/move: create target directories, move/rewrite shells for `docs/config.md`, `docs/policy/development.md`, `docs/integrations/subprocess.md`, and update README links; remove old duplicates only after content is preserved.
- Repository areas to inspect: docs policy, README, moved docs.
- Acceptance criteria: canonical paths exist; README does not link to nonexistent docs; old paths are either redirected by content moves or removed.
- Suggested validation commands: `rg "docs/(diagnostics|structured-llm|subprocess-operations)\\.md" README.md docs`; `rg "docs/configuration\\.md|docs/development\\.md|docs/integration/" README.md docs`.
- One prompt: yes.
### Stage 3: README and CLI Reference
- Goal: make README concise and create complete `docs/cli.md`.
- Files to create/update/delete/move: `README.md`, `docs/cli.md`.
- Repository areas to inspect: `cmd/audita/main.go`, `internal/cli/run.go`, `internal/cli/process_flags.go`, CLI tests.
- Acceptance criteria: README is orientation only; all implemented commands and flags are covered in `docs/cli.md`; examples match parser behavior.
- Suggested validation commands: `go test ./internal/cli ./cmd/audita`; stale-link grep checks.
- One prompt: yes.
### Stage 4: Config Reference and Examples
- Goal: rewrite `docs/config.md` and add maintained copyable examples.
- Files to create/update/delete/move: `docs/config.md`, `examples/minimal-config.yml`, `examples/production-config.yml`, `examples/tiny-transcript.json`, `examples/tiny-glossary.yaml`; remove `docs/configuration.md` after migration.
- Repository areas to inspect: `internal/core/config/*`, config tests, schema tests.
- Acceptance criteria: config reference matches implemented defaults, precedence, env vars, validation, and secrets; examples contain no secrets and validate where practical.
- Suggested validation commands: `go test ./internal/core/config`; `go run ./cmd/audita config validate --config examples/minimal-config.yml`; `go run ./cmd/audita config validate --config examples/production-config.yml`.
- One prompt: yes.
### Stage 5: Operations and Troubleshooting
- Goal: create operational and troubleshooting references.
- Files to create/update/delete/move: `docs/operations.md`, `docs/troubleshooting.md`.
- Repository areas to inspect: `internal/core/diagnostics`, `internal/framework/processreport`, `internal/core/reporting`, CLI failure/report tests.
- Acceptance criteria: implemented artifacts, retention, reports, correction ledger, and failure inspection are documented; no resume or remote-storage claims.
- Suggested validation commands: `go test ./internal/core/diagnostics ./internal/framework/processreport ./internal/cli`.
- One prompt: yes.
### Stage 6: Internal Architecture Docs Migration
- Goal: move implemented architecture details into `docs/internal/` and remove roadmap content from non-roadmap docs.
- Files to create/update/delete/move: `docs/internal/overview.md`, `docs/internal/pipeline.md`, `docs/internal/modules.md`, `docs/internal/validators.md`, `docs/internal/llm-runtime.md`, `docs/internal/diagnostics-reporting.md`, `docs/internal/prompts.md`, `docs/internal/output-schemas.md`; migrate/delete relevant `docs/architecture/*`.
- Repository areas to inspect: `internal/core`, `internal/framework`, `internal/modules`, `internal/validators`, `internal/prompts`.
- Acceptance criteria: internal docs document implemented behavior only; deferred or unimplemented content appears only under `docs/roadmap/`.
- Suggested validation commands: `go test ./internal/framework/llm ./internal/framework/runner`; `go test ./internal/validators/...`; `go test ./internal/modules/...`; `rg "deferred|not implemented|future|planned|experimental|aspirational" docs --glob '!docs/roadmap/**'`.
- One prompt: split if needed into pipeline/modules/validators and LLM/diagnostics/prompts/output schemas.
### Stage 7: Integration Docs
- Goal: create external contract docs for implemented integrations.
- Files to create/update/delete/move: `docs/integrations/subprocess.md`, `docs/integrations/openai-compatible-llm.md`, `docs/integrations/transcript-glossary-files.md`; remove `docs/integration/` after migration.
- Repository areas to inspect: CLI behavior, `internal/framework/llm`, `internal/core/schema`, integration-related tests.
- Acceptance criteria: integration docs describe actual external contracts and do not claim unsupported APIs.
- Suggested validation commands: `go test ./internal/cli ./cmd/audita`; `go test ./internal/framework/llm`; schema package tests.
- One prompt: yes.
### Stage 8: Development Policy and Duplicate Cleanup
- Goal: finish contributor workflow docs and remove duplicate policy locations.
- Files to create/update/delete/move: `docs/policy/development.md`, `docs/documentation/policy.md`, `docs/release-checklist.md`, any remaining old architecture/config/development paths.
- Repository areas to inspect: policy docs, development docs, test layout, final documentation tree.
- Acceptance criteria: one canonical documentation policy, one canonical development workflow, no duplicate or stale canonical-home references.
- Suggested validation commands: `find docs -type f | sort`; grep checks for old paths and duplicate policy paths.
- One prompt: yes.
### Stage 9: Final Documentation Validation
- Goal: repository-wide documentation review after migration.
- Files to create/update/delete/move: all documentation and examples touched by prior stages only as needed for fixes.
- Repository areas to inspect: final docs tree, README, examples, code-backed docs.
- Acceptance criteria: canonical docs exist, stale docs removed, examples valid, no unimplemented claims outside roadmap, Go tests pass.
- Suggested validation commands: `go test ./...`; all grep/link checks in this roadmap; example validation commands.
- One prompt: yes.
## Validation Plan
No markdown or documentation linter configuration was found. Use repository behavior tests, whitespace checks, grep checks, and manual review.
Automated checks:
- `git diff --check`
- `go test ./internal/core/config`
- `go test ./internal/cli ./cmd/audita`
- `go test ./internal/core/diagnostics ./internal/framework/processreport`
- `go test ./internal/framework/llm ./internal/framework/runner`
- `go test ./...`
Example checks after examples exist:
- `go run ./cmd/audita config validate --config examples/minimal-config.yml`
- `go run ./cmd/audita config validate --config examples/production-config.yml`
Recommended grep and path checks:
- `rg "docs/(diagnostics|structured-llm|subprocess-operations)\\.md" README.md docs`
- `rg "docs/configuration\\.md|docs/development\\.md|docs/integration/" README.md docs`
- `rg "deferred|not implemented|future|planned|experimental|aspirational" docs --glob '!docs/roadmap/**'`
- `find docs -type f | sort`
- `find examples -type f | sort`
Manual review:
- Confirm README is concise and links to canonical docs.
- Confirm CLI and config docs do not duplicate each other.
- Confirm internal docs are developer-facing and not user manuals.
- Confirm operations and troubleshooting docs describe current behavior only.
- Confirm future work appears only under `docs/roadmap/`.
- Confirm examples contain no secrets or private transcript data.
## Open Questions
No questions block the roadmap. Use these defaults unless a later implementation prompt says otherwise:
- Use the canonical paths from `docs/policy/documentation.md`, even when that requires moving existing docs.
- Treat `docs/configuration.md`, `docs/development.md`, `docs/integration/`, and `docs/architecture/*` as migration sources, not final homes.
- Do not restore deleted roadmap files unless separately requested.
- Prefer concise canonical docs over preserving historical wording from stale files.

11
go.mod Normal file
View File

@@ -0,0 +1,11 @@
module gitea.maximumdirect.net/eric/audita
go 1.24.0
require gopkg.in/yaml.v3 v3.0.1
require (
github.com/kr/pretty v0.3.1 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
)

17
go.sum Normal file
View File

@@ -0,0 +1,17 @@
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

338
internal/cli/parity_test.go Normal file
View File

@@ -0,0 +1,338 @@
package cli
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
)
type parityFixtureCase struct {
Name string `json:"name"`
TranscriptFile string `json:"transcript_file"`
GlossaryFile string `json:"glossary_file"`
ModulesCSV string `json:"modules_csv,omitempty"`
ProposalResponsesFile string `json:"proposal_responses_file,omitempty"`
ValidationResponsesFile string `json:"validation_responses_file,omitempty"`
Env map[string]string `json:"env,omitempty"`
Expect parityExpectation `json:"expect"`
}
type parityExpectation struct {
ExitCode int `json:"exit_code"`
Status string `json:"status,omitempty"`
ErrorPhase string `json:"error_phase,omitempty"`
StdoutMode string `json:"stdout_mode,omitempty"` // empty|json
StderrContains string `json:"stderr_contains,omitempty"`
OutputTranscriptFile string `json:"output_transcript_file,omitempty"`
ModuleInstances []string `json:"module_instances,omitempty"`
ModuleCount int `json:"module_count,omitempty"`
TotalAppliedChanges int `json:"total_applied_changes,omitempty"`
TotalSkippedChanges int `json:"total_skipped_changes,omitempty"`
FailedModuleInstance string `json:"failed_module_instance,omitempty"`
ValidatorRejectedReasonCodes []string `json:"validator_rejected_reason_codes,omitempty"`
ApplicationSkipReasonCodes []string `json:"application_skip_reason_codes,omitempty"`
RequireErrorLog bool `json:"require_error_log,omitempty"`
SecretMarkers []string `json:"secret_markers,omitempty"`
ExpectedProposalCalls []string `json:"expected_proposal_calls,omitempty"`
ExpectedValidationCalls []string `json:"expected_validation_calls,omitempty"`
ModuleAppliedCounts []int `json:"module_applied_counts,omitempty"`
ModuleRejectedCounts []int `json:"module_rejected_counts,omitempty"`
ModuleSkipCounts []int `json:"module_skip_counts,omitempty"`
MinResponsePayloadArtifacts int `json:"min_response_payload_artifacts,omitempty"`
}
func TestParityFixtures(t *testing.T) {
casePaths, err := filepath.Glob(parityFixturePath("*.case.json"))
if err != nil {
t.Fatalf("glob parity fixtures: %v", err)
}
if len(casePaths) == 0 {
t.Fatal("expected at least one parity fixture case")
}
for _, casePath := range casePaths {
fx := loadParityFixtureCase(t, casePath)
t.Run(fx.Name, func(t *testing.T) {
runParityFixtureCase(t, filepath.Dir(casePath), fx)
})
}
}
func loadParityFixtureCase(t *testing.T, casePath string) parityFixtureCase {
t.Helper()
var fx parityFixtureCase
raw := readFile(t, casePath)
if err := json.Unmarshal(raw, &fx); err != nil {
t.Fatalf("parse parity case %q: %v", casePath, err)
}
if strings.TrimSpace(fx.Name) == "" {
t.Fatalf("parity case %q missing name", casePath)
}
return fx
}
func runParityFixtureCase(t *testing.T, caseDir string, fx parityFixtureCase) {
t.Helper()
for k, v := range fx.Env {
t.Setenv(k, v)
}
proposalClient := &fakeStructuredLLMClient{}
validationClient := &fakeStructuredLLMClient{}
if strings.TrimSpace(fx.ProposalResponsesFile) != "" {
raw := readFile(t, filepath.Join(caseDir, fx.ProposalResponsesFile))
if err := json.Unmarshal(raw, &proposalClient.proposalResponses); err != nil {
t.Fatalf("parse proposal responses: %v", err)
}
processProposalLLMClient = proposalClient
}
if strings.TrimSpace(fx.ValidationResponsesFile) != "" {
raw := readFile(t, filepath.Join(caseDir, fx.ValidationResponsesFile))
if err := json.Unmarshal(raw, &validationClient.validationResponses); err != nil {
t.Fatalf("parse validation responses: %v", err)
}
processValidationLLMClient = validationClient
}
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
var stdout, stderr bytes.Buffer
workDir := t.TempDir()
outputPath := filepath.Join(t.TempDir(), "out.json")
reportPath := filepath.Join(t.TempDir(), "report.json")
args := []string{
"process",
filepath.Join(caseDir, fx.TranscriptFile),
"--glossary",
filepath.Join(caseDir, fx.GlossaryFile),
"--report-json",
reportPath,
"--work-dir",
workDir,
"--work-dir-retention",
"always",
}
// Keep stdout shape deterministic for parity tests.
if fx.Expect.StdoutMode != "json" {
args = append(args, "--output", outputPath)
}
if strings.TrimSpace(fx.ModulesCSV) != "" {
args = append(args, "--modules", fx.ModulesCSV)
}
exitCode := Run(args, &stdout, &stderr)
if exitCode != fx.Expect.ExitCode {
t.Fatalf("expected exit code %d, got %d stderr=%q", fx.Expect.ExitCode, exitCode, stderr.String())
}
switch fx.Expect.StdoutMode {
case "json":
if _, err := json.Marshal(stdout.String()); err != nil {
t.Fatalf("unexpected stdout marshal error: %v", err)
}
if !json.Valid(stdout.Bytes()) {
t.Fatalf("expected JSON stdout, got %q", stdout.String())
}
default:
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout, got %q", stdout.String())
}
}
if fx.Expect.StderrContains != "" && !strings.Contains(stderr.String(), fx.Expect.StderrContains) {
t.Fatalf("expected stderr to contain %q, got %q", fx.Expect.StderrContains, stderr.String())
}
report := readProcessReport(t, reportPath)
assertParityReport(t, report, fx.Expect)
runDir := onlyRunDir(t, workDir)
runDirReport := readProcessReport(t, filepath.Join(runDir, "report.json"))
assertParityReport(t, runDirReport, fx.Expect)
if fx.Expect.RequireErrorLog {
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log: %v", err)
}
}
if strings.TrimSpace(fx.Expect.OutputTranscriptFile) != "" && fx.Expect.ExitCode == 0 {
got := readFile(t, outputPath)
want := readFile(t, filepath.Join(caseDir, fx.Expect.OutputTranscriptFile))
assertJSONSemanticEqual(t, want, got)
}
if len(fx.Expect.ExpectedProposalCalls) > 0 && !reflect.DeepEqual(proposalClient.calls, fx.Expect.ExpectedProposalCalls) {
t.Fatalf("unexpected proposal calls: got %v want %v", proposalClient.calls, fx.Expect.ExpectedProposalCalls)
}
if len(fx.Expect.ExpectedValidationCalls) > 0 && !reflect.DeepEqual(validationClient.calls, fx.Expect.ExpectedValidationCalls) {
t.Fatalf("unexpected validation calls: got %v want %v", validationClient.calls, fx.Expect.ExpectedValidationCalls)
}
if len(fx.Expect.SecretMarkers) > 0 {
assertNoSecretMarkers(t, reportPath, fx.Expect.SecretMarkers)
assertNoSecretMarkersInTree(t, runDir, fx.Expect.SecretMarkers)
}
if fx.Expect.MinResponsePayloadArtifacts > 0 {
matches, err := filepath.Glob(filepath.Join(runDir, "*", "*response-payload.json"))
if err != nil {
t.Fatalf("glob response payload artifacts: %v", err)
}
if len(matches) < fx.Expect.MinResponsePayloadArtifacts {
t.Fatalf("expected at least %d response payload artifacts, got %d", fx.Expect.MinResponsePayloadArtifacts, len(matches))
}
}
}
func assertParityReport(t *testing.T, report reporting.ProcessReport, exp parityExpectation) {
t.Helper()
if exp.Status != "" && report.Status != exp.Status {
t.Fatalf("expected report status %q, got %q", exp.Status, report.Status)
}
if exp.ErrorPhase != "" && report.ErrorPhase != exp.ErrorPhase {
t.Fatalf("expected report error_phase %q, got %q", exp.ErrorPhase, report.ErrorPhase)
}
if len(exp.ModuleInstances) > 0 {
got := make([]string, 0, len(report.ModuleResults))
for _, mr := range report.ModuleResults {
got = append(got, mr.ModuleInstance)
}
if !reflect.DeepEqual(got, exp.ModuleInstances) {
t.Fatalf("unexpected module instances: got %v want %v", got, exp.ModuleInstances)
}
}
if exp.ModuleCount > 0 {
if report.ModulesSummary == nil || report.ModulesSummary.ModuleCount != exp.ModuleCount {
t.Fatalf("expected module_count=%d, got %+v", exp.ModuleCount, report.ModulesSummary)
}
}
if exp.TotalAppliedChanges > 0 {
if report.ModulesSummary == nil || report.ModulesSummary.TotalAppliedChanges != exp.TotalAppliedChanges {
t.Fatalf("expected total_applied_changes=%d, got %+v", exp.TotalAppliedChanges, report.ModulesSummary)
}
}
if exp.TotalSkippedChanges > 0 {
if report.ModulesSummary == nil || report.ModulesSummary.TotalSkippedChanges != exp.TotalSkippedChanges {
t.Fatalf("expected total_skipped_changes=%d, got %+v", exp.TotalSkippedChanges, report.ModulesSummary)
}
}
if exp.FailedModuleInstance != "" {
if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != exp.FailedModuleInstance {
t.Fatalf("expected failed_module_instance=%q, got %+v", exp.FailedModuleInstance, report.ModulesSummary)
}
}
if len(exp.ValidatorRejectedReasonCodes) > 0 {
got := collectValidatorRejectedReasonCodes(report.ModuleResults)
if !reflect.DeepEqual(got, exp.ValidatorRejectedReasonCodes) {
t.Fatalf("unexpected validator rejected reason codes: got %v want %v", got, exp.ValidatorRejectedReasonCodes)
}
}
if len(exp.ApplicationSkipReasonCodes) > 0 {
got := collectApplicationSkipReasonCodes(report.ModuleResults)
if !reflect.DeepEqual(got, exp.ApplicationSkipReasonCodes) {
t.Fatalf("unexpected application skip reason codes: got %v want %v", got, exp.ApplicationSkipReasonCodes)
}
}
if len(exp.ModuleAppliedCounts) > 0 {
got := make([]int, 0, len(report.ModuleResults))
for _, mr := range report.ModuleResults {
got = append(got, len(mr.AppliedChanges))
}
if !reflect.DeepEqual(got, exp.ModuleAppliedCounts) {
t.Fatalf("unexpected per-module applied counts: got %v want %v", got, exp.ModuleAppliedCounts)
}
}
if len(exp.ModuleRejectedCounts) > 0 {
got := make([]int, 0, len(report.ModuleResults))
for _, mr := range report.ModuleResults {
got = append(got, len(mr.ValidatorRejected))
}
if !reflect.DeepEqual(got, exp.ModuleRejectedCounts) {
t.Fatalf("unexpected per-module rejected counts: got %v want %v", got, exp.ModuleRejectedCounts)
}
}
if len(exp.ModuleSkipCounts) > 0 {
got := make([]int, 0, len(report.ModuleResults))
for _, mr := range report.ModuleResults {
got = append(got, len(mr.SkippedChanges))
}
if !reflect.DeepEqual(got, exp.ModuleSkipCounts) {
t.Fatalf("unexpected per-module skip counts: got %v want %v", got, exp.ModuleSkipCounts)
}
}
}
func collectValidatorRejectedReasonCodes(results []reporting.ModuleReport) []string {
out := make([]string, 0)
for _, mr := range results {
for _, vr := range mr.ValidatorRejected {
out = append(out, vr.ReasonCode)
}
}
return out
}
func collectApplicationSkipReasonCodes(results []reporting.ModuleReport) []string {
out := make([]string, 0)
for _, mr := range results {
for _, sk := range mr.SkippedChanges {
out = append(out, string(sk.SkipReason))
}
}
return out
}
func assertJSONSemanticEqual(t *testing.T, expected []byte, actual []byte) {
t.Helper()
var exp any
var act any
if err := json.Unmarshal(expected, &exp); err != nil {
t.Fatalf("unmarshal expected json: %v", err)
}
if err := json.Unmarshal(actual, &act); err != nil {
t.Fatalf("unmarshal actual json: %v", err)
}
if !reflect.DeepEqual(exp, act) {
t.Fatalf("JSON mismatch\nexpected=%s\nactual=%s", string(expected), string(actual))
}
}
func assertNoSecretMarkers(t *testing.T, filePath string, markers []string) {
t.Helper()
raw := string(readFile(t, filePath))
for _, marker := range markers {
if marker != "" && strings.Contains(raw, marker) {
t.Fatalf("secret marker %q leaked in %s", marker, filePath)
}
}
}
func assertNoSecretMarkersInTree(t *testing.T, root string, markers []string) {
t.Helper()
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil || d == nil || d.IsDir() {
return nil
}
raw := string(readFile(t, path))
for _, marker := range markers {
if marker != "" && strings.Contains(raw, marker) {
t.Fatalf("secret marker %q leaked in %s", marker, path)
}
}
return nil
})
}
func parityFixturePath(name string) string {
return filepath.Join("testdata", "parity", name)
}

View File

@@ -0,0 +1,121 @@
package cli
import (
"flag"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
)
type processOverrideBinding func(*config.CLIOverrides, processFlags)
var processOverrideBindings = map[string]processOverrideBinding{
"modules": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ModulesCSV = flags.modules
},
"output-schema": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.OutputSchema = flags.outputSchema
},
"llm-api-key": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.PrimaryLLMAPIKey = flags.llmAPIKey
},
"validation-llm-api-key": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationLLMAPIKey = flags.validationLLMAPIKey
},
"model": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.PrimaryModel = flags.model
},
"validation-model": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationModel = flags.validationModel
},
"base-url": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.PrimaryBaseURL = flags.baseURL
},
"validation-base-url": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationBaseURL = flags.validationBaseURL
},
"llm-timeout-seconds": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.PrimaryLLMTimeoutSeconds = flags.llmTimeoutSeconds
},
"total-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.TotalLLMConcurrency = flags.totalLLMConcurrency
},
"proposal-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ProposalLLMConcurrency = flags.proposalLLMConcurrency
},
"llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.PrimaryLLMConcurrency = flags.llmConcurrency
},
"validation-llm-timeout-seconds": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationLLMTimeoutSeconds = flags.validationLLMTimeoutSeconds
},
"max-retries": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.MaxRetries = flags.maxRetries
},
"validation-max-retries": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationMaxRetries = flags.validationMaxRetries
},
"validation-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationLLMConcurrency = flags.validationLLMConcurrency
},
"validation-max-prompt-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationMaxPromptTokens = flags.validationMaxPromptTokens
},
"max-section-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.MaxSectionTokens = flags.maxSectionTokens
},
"min-section-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.MinSectionTokens = flags.minSectionTokens
},
"target-sections": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.TargetSections = flags.targetSections
},
"glossary-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.GlossaryConfidenceThreshold = flags.glossaryConfidenceThreshold
},
"grammar-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.GrammarConfidenceThreshold = flags.grammarConfidenceThreshold
},
"homophones-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.HomophonesConfidenceThreshold = flags.homophonesConfidenceThreshold
},
"spoken-word-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.SpokenWordConfidenceThreshold = flags.spokenWordConfidenceThreshold
},
"normalize-max-segment-gap": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.NormalizeMaxSegmentGap = flags.normalizeMaxSegmentGap
},
"normalize-ellipsis-gap": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.NormalizeEllipsisGap = flags.normalizeEllipsisGap
},
"normalize-max-segment-duration": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.NormalizeMaxSegmentDuration = flags.normalizeMaxSegmentDuration
},
"normalize-max-segment-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.NormalizeMaxSegmentTokens = flags.normalizeMaxSegmentTokens
},
"transcript-description": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.TranscriptDescription = flags.transcriptDescription
},
"work-dir": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.WorkDir = flags.workDir
},
"work-dir-retention": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.WorkDirRetention = flags.workDirRetention
},
}
func processCLIOverrides(fs *flag.FlagSet, flags processFlags) (config.CLIOverrides, bool) {
overrides := config.CLIOverrides{}
explicitModules := false
fs.Visit(func(f *flag.Flag) {
if f.Name == "modules" {
explicitModules = true
}
binding, ok := processOverrideBindings[f.Name]
if !ok {
return
}
binding(&overrides, flags)
})
return overrides, explicitModules
}

View File

@@ -0,0 +1,433 @@
package cli
import (
"io"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
)
func TestProcessCLIOverridesMapsEveryConfigMutatingFlag(t *testing.T) {
tests := []struct {
name string
flagName string
value string
wantExplicitModules bool
assertOverrideFields func(t *testing.T, overrides config.CLIOverrides)
}{
{
name: "modules",
flagName: "modules",
value: "grammar,glossary",
wantExplicitModules: true,
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "ModulesCSV", overrides.ModulesCSV, "grammar,glossary")
},
},
{
name: "output schema",
flagName: "output-schema",
value: "audita-v1",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "OutputSchema", overrides.OutputSchema, "audita-v1")
},
},
{
name: "primary api key",
flagName: "llm-api-key",
value: "primary-key",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "PrimaryLLMAPIKey", overrides.PrimaryLLMAPIKey, "primary-key")
},
},
{
name: "validation api key",
flagName: "validation-llm-api-key",
value: "validation-key",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "ValidationLLMAPIKey", overrides.ValidationLLMAPIKey, "validation-key")
},
},
{
name: "primary model",
flagName: "model",
value: "primary-model",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "PrimaryModel", overrides.PrimaryModel, "primary-model")
},
},
{
name: "validation model",
flagName: "validation-model",
value: "validation-model",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "ValidationModel", overrides.ValidationModel, "validation-model")
},
},
{
name: "primary base url",
flagName: "base-url",
value: "https://primary.example.test",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "PrimaryBaseURL", overrides.PrimaryBaseURL, "https://primary.example.test")
},
},
{
name: "validation base url",
flagName: "validation-base-url",
value: "https://validation.example.test",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "ValidationBaseURL", overrides.ValidationBaseURL, "https://validation.example.test")
},
},
{
name: "primary timeout",
flagName: "llm-timeout-seconds",
value: "101",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "PrimaryLLMTimeoutSeconds", overrides.PrimaryLLMTimeoutSeconds, 101)
},
},
{
name: "total concurrency",
flagName: "total-llm-concurrency",
value: "5",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "TotalLLMConcurrency", overrides.TotalLLMConcurrency, 5)
},
},
{
name: "proposal concurrency",
flagName: "proposal-llm-concurrency",
value: "3",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "ProposalLLMConcurrency", overrides.ProposalLLMConcurrency, 3)
},
},
{
name: "legacy concurrency alias",
flagName: "llm-concurrency",
value: "4",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "PrimaryLLMConcurrency", overrides.PrimaryLLMConcurrency, 4)
},
},
{
name: "validation timeout",
flagName: "validation-llm-timeout-seconds",
value: "202",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "ValidationLLMTimeoutSeconds", overrides.ValidationLLMTimeoutSeconds, 202)
},
},
{
name: "max retries",
flagName: "max-retries",
value: "6",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "MaxRetries", overrides.MaxRetries, 6)
},
},
{
name: "validation max retries",
flagName: "validation-max-retries",
value: "7",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "ValidationMaxRetries", overrides.ValidationMaxRetries, 7)
},
},
{
name: "validation concurrency",
flagName: "validation-llm-concurrency",
value: "8",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "ValidationLLMConcurrency", overrides.ValidationLLMConcurrency, 8)
},
},
{
name: "validation max prompt tokens",
flagName: "validation-max-prompt-tokens",
value: "4096",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "ValidationMaxPromptTokens", overrides.ValidationMaxPromptTokens, 4096)
},
},
{
name: "max section tokens",
flagName: "max-section-tokens",
value: "9000",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "MaxSectionTokens", overrides.MaxSectionTokens, 9000)
},
},
{
name: "min section tokens",
flagName: "min-section-tokens",
value: "1000",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "MinSectionTokens", overrides.MinSectionTokens, 1000)
},
},
{
name: "target sections",
flagName: "target-sections",
value: "12",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "TargetSections", overrides.TargetSections, 12)
},
},
{
name: "glossary threshold",
flagName: "glossary-confidence-threshold",
value: "0.91",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "GlossaryConfidenceThreshold", overrides.GlossaryConfidenceThreshold, 0.91)
},
},
{
name: "grammar threshold",
flagName: "grammar-confidence-threshold",
value: "0.92",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "GrammarConfidenceThreshold", overrides.GrammarConfidenceThreshold, 0.92)
},
},
{
name: "homophones threshold",
flagName: "homophones-confidence-threshold",
value: "0.93",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "HomophonesConfidenceThreshold", overrides.HomophonesConfidenceThreshold, 0.93)
},
},
{
name: "spoken word threshold",
flagName: "spoken-word-confidence-threshold",
value: "0.94",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "SpokenWordConfidenceThreshold", overrides.SpokenWordConfidenceThreshold, 0.94)
},
},
{
name: "normalize max segment gap",
flagName: "normalize-max-segment-gap",
value: "1.2",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "NormalizeMaxSegmentGap", overrides.NormalizeMaxSegmentGap, 1.2)
},
},
{
name: "normalize ellipsis gap",
flagName: "normalize-ellipsis-gap",
value: "2.3",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "NormalizeEllipsisGap", overrides.NormalizeEllipsisGap, 2.3)
},
},
{
name: "normalize max segment duration",
flagName: "normalize-max-segment-duration",
value: "45.6",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "NormalizeMaxSegmentDuration", overrides.NormalizeMaxSegmentDuration, 45.6)
},
},
{
name: "normalize max segment tokens",
flagName: "normalize-max-segment-tokens",
value: "321",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "NormalizeMaxSegmentTokens", overrides.NormalizeMaxSegmentTokens, 321)
},
},
{
name: "transcript description",
flagName: "transcript-description",
value: "podcast episode",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "TranscriptDescription", overrides.TranscriptDescription, "podcast episode")
},
},
{
name: "work dir",
flagName: "work-dir",
value: "/tmp/custom-audita",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "WorkDir", overrides.WorkDir, "/tmp/custom-audita")
},
},
{
name: "work dir retention",
flagName: "work-dir-retention",
value: "always",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "WorkDirRetention", overrides.WorkDirRetention, "always")
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
fs, flags := newProcessFlagSet(config.Default(), io.Discard)
if err := fs.Parse([]string{"--" + tc.flagName, tc.value}); err != nil {
t.Fatalf("parse flag: %v", err)
}
overrides, explicitModules := processCLIOverrides(fs, flags)
if explicitModules != tc.wantExplicitModules {
t.Fatalf("explicitModules=%v, want %v", explicitModules, tc.wantExplicitModules)
}
tc.assertOverrideFields(t, overrides)
})
}
}
func TestProcessCLIOverridesIgnoresNonConfigFlags(t *testing.T) {
fs, flags := newProcessFlagSet(config.Default(), io.Discard)
if err := fs.Parse([]string{
"--config", "/tmp/config.yml",
"--glossary", "/tmp/glossary.yml",
"--output", "/tmp/output.json",
"--report-json", "/tmp/report.json",
}); err != nil {
t.Fatalf("parse flags: %v", err)
}
overrides, explicitModules := processCLIOverrides(fs, flags)
if explicitModules {
t.Fatal("non-config flags should not mark modules explicit")
}
assertNoCLIOverrides(t, overrides)
}
func TestNewProcessFlagSetDefaultsReflectEffectiveConfig(t *testing.T) {
cfg := config.Default()
cfg.Modules = []string{"grammar", "glossary"}
cfg.OutputSchema = "audita-v1"
cfg.PrimaryLLM.APIKey = "primary-key"
cfg.ValidationLLM.APIKey = "validation-key"
cfg.PrimaryLLM.Model = "primary-model"
cfg.ValidationLLM.Model = "validation-model"
cfg.PrimaryLLM.BaseURL = "https://primary.example.test"
cfg.ValidationLLM.BaseURL = "https://validation.example.test"
cfg.PrimaryLLM.TimeoutSeconds = 101
cfg.TotalLLMConcurrency = 5
cfg.ProposalLLMConcurrency = 3
cfg.PrimaryLLM.MaxRetries = 6
cfg.ValidationMaxPromptTokens = 4096
cfg.MaxSectionTokens = 9000
cfg.MinSectionTokens = 1000
cfg.Thresholds.Glossary = 0.91
cfg.Thresholds.Grammar = 0.92
cfg.Thresholds.Homophones = 0.93
cfg.Thresholds.SpokenWord = 0.94
cfg.Normalization.MaxSegmentGap = 1.2
cfg.Normalization.EllipsisGap = 2.3
cfg.Normalization.MaxSegmentDuration = 45.6
cfg.Normalization.MaxSegmentTokens = 321
cfg.TranscriptDescription = "podcast episode"
cfg.WorkDir = "/tmp/custom-audita"
cfg.WorkDirRetention = config.WorkDirRetentionAlways
validationTimeout := 202
validationRetries := 7
validationConcurrency := 8
targetSections := 12
cfg.ValidationLLM.TimeoutSeconds = &validationTimeout
cfg.ValidationLLM.MaxRetries = &validationRetries
cfg.ValidationLLMConcurrency = &validationConcurrency
cfg.TargetSections = &targetSections
_, flags := newProcessFlagSet(cfg, io.Discard)
assertStringOverride(t, "modules default", flags.modules, "grammar,glossary")
assertStringOverride(t, "output schema default", flags.outputSchema, "audita-v1")
assertStringOverride(t, "primary api key default", flags.llmAPIKey, "primary-key")
assertStringOverride(t, "validation api key default", flags.validationLLMAPIKey, "validation-key")
assertStringOverride(t, "primary model default", flags.model, "primary-model")
assertStringOverride(t, "validation model default", flags.validationModel, "validation-model")
assertStringOverride(t, "primary base url default", flags.baseURL, "https://primary.example.test")
assertStringOverride(t, "validation base url default", flags.validationBaseURL, "https://validation.example.test")
assertIntOverride(t, "primary timeout default", flags.llmTimeoutSeconds, 101)
assertIntOverride(t, "total concurrency default", flags.totalLLMConcurrency, 5)
assertIntOverride(t, "proposal concurrency default", flags.proposalLLMConcurrency, 3)
assertIntOverride(t, "legacy concurrency alias default", flags.llmConcurrency, 5)
assertIntOverride(t, "validation timeout default", flags.validationLLMTimeoutSeconds, validationTimeout)
assertIntOverride(t, "max retries default", flags.maxRetries, 6)
assertIntOverride(t, "validation max retries default", flags.validationMaxRetries, validationRetries)
assertIntOverride(t, "validation concurrency default", flags.validationLLMConcurrency, validationConcurrency)
assertIntOverride(t, "validation max prompt tokens default", flags.validationMaxPromptTokens, 4096)
assertIntOverride(t, "max section tokens default", flags.maxSectionTokens, 9000)
assertIntOverride(t, "min section tokens default", flags.minSectionTokens, 1000)
assertIntOverride(t, "target sections default", flags.targetSections, targetSections)
assertFloatOverride(t, "glossary threshold default", flags.glossaryConfidenceThreshold, 0.91)
assertFloatOverride(t, "grammar threshold default", flags.grammarConfidenceThreshold, 0.92)
assertFloatOverride(t, "homophones threshold default", flags.homophonesConfidenceThreshold, 0.93)
assertFloatOverride(t, "spoken word threshold default", flags.spokenWordConfidenceThreshold, 0.94)
assertFloatOverride(t, "normalize max segment gap default", flags.normalizeMaxSegmentGap, 1.2)
assertFloatOverride(t, "normalize ellipsis gap default", flags.normalizeEllipsisGap, 2.3)
assertFloatOverride(t, "normalize max segment duration default", flags.normalizeMaxSegmentDuration, 45.6)
assertIntOverride(t, "normalize max segment tokens default", flags.normalizeMaxSegmentTokens, 321)
assertStringOverride(t, "transcript description default", flags.transcriptDescription, "podcast episode")
assertStringOverride(t, "work dir default", flags.workDir, "/tmp/custom-audita")
assertStringOverride(t, "work dir retention default", flags.workDirRetention, "always")
}
func TestNewProcessFlagSetUsesFallbackDefaultsForUnsetOptionalConfig(t *testing.T) {
cfg := config.Default()
_, flags := newProcessFlagSet(cfg, io.Discard)
assertIntOverride(t, "validation timeout fallback", flags.validationLLMTimeoutSeconds, cfg.PrimaryLLM.TimeoutSeconds)
assertIntOverride(t, "validation retries fallback", flags.validationMaxRetries, cfg.PrimaryLLM.MaxRetries)
assertIntOverride(t, "validation concurrency fallback", flags.validationLLMConcurrency, cfg.TotalLLMConcurrency)
assertIntOverride(t, "target sections fallback", flags.targetSections, 0)
}
func assertStringOverride(t *testing.T, name string, got *string, want string) {
t.Helper()
if got == nil || *got != want {
t.Fatalf("%s=%v, want %q", name, pointerValue(got), want)
}
}
func assertIntOverride(t *testing.T, name string, got *int, want int) {
t.Helper()
if got == nil || *got != want {
t.Fatalf("%s=%v, want %d", name, pointerValue(got), want)
}
}
func assertFloatOverride(t *testing.T, name string, got *float64, want float64) {
t.Helper()
if got == nil || *got != want {
t.Fatalf("%s=%v, want %v", name, pointerValue(got), want)
}
}
func assertNoCLIOverrides(t *testing.T, overrides config.CLIOverrides) {
t.Helper()
value := reflect.ValueOf(overrides)
typ := value.Type()
for i := 0; i < value.NumField(); i++ {
field := value.Field(i)
if field.Kind() != reflect.Ptr {
t.Fatalf("unexpected non-pointer CLIOverrides field %s", typ.Field(i).Name)
}
if !field.IsNil() {
t.Fatalf("expected no CLI overrides, field %s was set", typ.Field(i).Name)
}
}
}
func pointerValue[T any](ptr *T) any {
if ptr == nil {
return "<nil>"
}
if stringer, ok := any(*ptr).(interface{ String() string }); ok {
return strings.TrimSpace(stringer.String())
}
return *ptr
}

View File

@@ -0,0 +1,469 @@
package cli
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
type releaseFixtureExpectations struct {
MustApplyTexts []string `json:"must_apply_texts"`
MustNotApplyTexts []string `json:"must_not_apply_texts"`
ProtectedTerms []string `json:"protected_terms"`
ExpectedModuleInstance []string `json:"expected_module_instances"`
MinimumCounts struct {
Applied int `json:"applied"`
Rejected int `json:"rejected"`
Skipped int `json:"skipped"`
} `json:"minimum_counts"`
}
func TestReleaseFixtureDefaultPipelineReadiness(t *testing.T) {
base := fixturePath(filepath.Join("release", "default-release"))
var expectations releaseFixtureExpectations
if err := json.Unmarshal(readFile(t, base+".expectations.json"), &expectations); err != nil {
t.Fatalf("unmarshal release expectations: %v", err)
}
proposalResponses := readProposalResponses(t, base+".proposals.json")
validationResponses := readValidationResponses(t, base+".validations.json")
// First pass: default full pipeline with deterministic fake LLM responses.
first := runReleaseFixturePass(t, releaseRunConfig{
transcriptPath: base + ".transcript.json",
glossaryPath: base + ".glossary.yaml",
outputSchema: "bare-segments",
proposalResponses: proposalResponses,
validationResponses: validationResponses,
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
reportSchemaName: reporting.DefaultProcessReportSchemaName,
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
expectedOutputSchema: "bare-segments",
expectModuleInstances: expectations.ExpectedModuleInstance,
expectUtilizationPaths: true,
})
gotTranscript := mustReadTranscript(t, first.outputPath)
expectFinalTranscriptContains(t, gotTranscript, expectations.MustApplyTexts)
expectFinalTranscriptDoesNotContain(t, gotTranscript, expectations.MustNotApplyTexts)
expectFinalTranscriptContains(t, gotTranscript, expectations.ProtectedTerms)
assertReleaseCounts(t, first.report, expectations)
assertPromptAndSchemaMetadataPresent(t, first.runDir)
assertReleaseLedgerShape(t, first.report)
assertReleaseUtilizationShape(t, first.report)
assertStableValidatorKeysPresent(t, first.report)
assertStdoutStderrContract(t, first.stdout, first.stderr)
assertNoSecretMarkersInTree(t, first.runDir, []string{"release-secret"})
assertNoSecretMarkers(t, first.reportPath, []string{"release-secret"})
// Output schema check: audita-v1 object payload.
auditaV1 := runReleaseFixturePass(t, releaseRunConfig{
transcriptPath: base + ".transcript.json",
glossaryPath: base + ".glossary.yaml",
outputSchema: "audita-v1",
proposalResponses: proposalResponses,
validationResponses: validationResponses,
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
reportSchemaName: reporting.DefaultProcessReportSchemaName,
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
expectedOutputSchema: "audita-v1",
expectModuleInstances: expectations.ExpectedModuleInstance,
expectUtilizationPaths: true,
})
assertAuditaV1OutputShape(t, auditaV1.outputPath)
// Idempotence-oriented second pass:
// run again on first output with deterministic no-op responses.
noOpProposals := make([]proposal_generation.StructuredCorrectionSet, 5)
for i := range noOpProposals {
noOpProposals[i] = proposal_generation.StructuredCorrectionSet{Corrections: nil}
}
second := runReleaseFixturePass(t, releaseRunConfig{
transcriptPath: first.outputPath,
glossaryPath: base + ".glossary.yaml",
outputSchema: "bare-segments",
proposalResponses: noOpProposals,
validationResponses: nil,
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
reportSchemaName: reporting.DefaultProcessReportSchemaName,
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
expectedOutputSchema: "bare-segments",
expectModuleInstances: expectations.ExpectedModuleInstance,
expectUtilizationPaths: true,
})
firstSegments := mustReadTranscript(t, first.outputPath)
secondSegments := mustReadTranscript(t, second.outputPath)
if !reflect.DeepEqual(firstSegments, secondSegments) {
t.Fatalf("expected idempotent second pass transcript; first=%+v second=%+v", firstSegments, secondSegments)
}
if second.report.ModulesSummary == nil {
t.Fatalf("expected modules summary on second pass")
}
if second.report.ModulesSummary.TotalAppliedChanges != 0 {
t.Fatalf("expected no-op second pass (0 applied), got %+v", second.report.ModulesSummary)
}
}
type releaseRunConfig struct {
transcriptPath string
glossaryPath string
outputSchema string
proposalResponses []proposal_generation.StructuredCorrectionSet
validationResponses []validators.LLMValidationResponse
expectedProposalCalls []string
reportSchemaName string
reportSchemaVersion string
expectedOutputSchema string
expectModuleInstances []string
expectUtilizationPaths bool
}
type releaseRunResult struct {
stdout string
stderr string
outputPath string
reportPath string
report reporting.ProcessReport
runDir string
}
func runReleaseFixturePass(t *testing.T, cfg releaseRunConfig) releaseRunResult {
t.Helper()
processProposalLLMClient = &fakeStructuredLLMClient{proposalResponses: append([]proposal_generation.StructuredCorrectionSet(nil), cfg.proposalResponses...)}
processValidationLLMClient = &fakeStructuredLLMClient{validationResponses: append([]validators.LLMValidationResponse(nil), cfg.validationResponses...)}
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
configPath := writeFile(t, "release-config.yml", "version: 1\n")
args := []string{
"process",
cfg.transcriptPath,
"--glossary",
cfg.glossaryPath,
"--config",
configPath,
"--output",
outputPath,
"--output-schema",
cfg.outputSchema,
"--report-json",
reportPath,
"--work-dir",
workDir,
"--work-dir-retention",
"always",
}
var stdout, stderr bytes.Buffer
exitCode := Run(args, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
report := readProcessReport(t, reportPath)
if report.ReportMetadata.ReportSchemaName != cfg.reportSchemaName || report.ReportMetadata.ReportSchemaVersion != cfg.reportSchemaVersion {
t.Fatalf("unexpected report schema metadata: %+v", report.ReportMetadata)
}
if report.ReportMetadata.OutputSchema != cfg.expectedOutputSchema {
t.Fatalf("unexpected output schema metadata: got %q want %q", report.ReportMetadata.OutputSchema, cfg.expectedOutputSchema)
}
if len(cfg.expectModuleInstances) > 0 {
got := make([]string, 0, len(report.ModuleResults))
for _, mr := range report.ModuleResults {
got = append(got, mr.ModuleInstance)
}
if !reflect.DeepEqual(got, cfg.expectModuleInstances) {
t.Fatalf("unexpected module instances: got %v want %v", got, cfg.expectModuleInstances)
}
}
if report.Diagnostics == nil {
t.Fatalf("expected diagnostics metadata")
}
if cfg.expectUtilizationPaths {
if report.Diagnostics.UtilizationSummaryPath == "" || report.Diagnostics.CorrectionLedgerPath == "" {
t.Fatalf("expected utilization/ledger artifact paths in report diagnostics: %+v", report.Diagnostics)
}
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil {
t.Fatalf("expected run-dir report: %v", err)
}
if c, ok := processProposalLLMClient.(*fakeStructuredLLMClient); ok {
if !reflect.DeepEqual(c.calls, cfg.expectedProposalCalls) {
t.Fatalf("unexpected proposal call order: got %v want %v", c.calls, cfg.expectedProposalCalls)
}
}
return releaseRunResult{
stdout: stdout.String(),
stderr: stderr.String(),
outputPath: outputPath,
reportPath: reportPath,
report: report,
runDir: runDir,
}
}
func readProposalResponses(t *testing.T, path string) []proposal_generation.StructuredCorrectionSet {
t.Helper()
var out []proposal_generation.StructuredCorrectionSet
if err := json.Unmarshal(readFile(t, path), &out); err != nil {
t.Fatalf("unmarshal proposal responses: %v", err)
}
return out
}
func readValidationResponses(t *testing.T, path string) []validators.LLMValidationResponse {
t.Helper()
var out []validators.LLMValidationResponse
if err := json.Unmarshal(readFile(t, path), &out); err != nil {
t.Fatalf("unmarshal validation responses: %v", err)
}
return out
}
func mustReadTranscript(t *testing.T, path string) []schema.Segment {
t.Helper()
transcript, err := schema.ParseTranscriptJSON(readFile(t, path))
if err != nil {
t.Fatalf("parse transcript output: %v", err)
}
return transcript.Segments
}
func expectFinalTranscriptContains(t *testing.T, segments []schema.Segment, needles []string) {
t.Helper()
joined := flattenTranscriptText(segments)
for _, needle := range needles {
if !strings.Contains(joined, needle) {
t.Fatalf("expected transcript to contain %q, got %q", needle, joined)
}
}
}
func expectFinalTranscriptDoesNotContain(t *testing.T, segments []schema.Segment, needles []string) {
t.Helper()
joined := flattenTranscriptText(segments)
for _, needle := range needles {
if strings.Contains(joined, needle) {
t.Fatalf("expected transcript to not contain %q, got %q", needle, joined)
}
}
}
func flattenTranscriptText(segments []schema.Segment) string {
parts := make([]string, 0, len(segments))
for _, s := range segments {
parts = append(parts, s.Text)
}
return strings.Join(parts, "\n")
}
func assertReleaseCounts(t *testing.T, report reporting.ProcessReport, exp releaseFixtureExpectations) {
t.Helper()
if report.ModulesSummary == nil {
t.Fatalf("expected modules_summary")
}
if report.ModulesSummary.TotalAppliedChanges < exp.MinimumCounts.Applied {
t.Fatalf("expected at least %d applied changes, got %+v", exp.MinimumCounts.Applied, report.ModulesSummary)
}
validatorRejected := 0
skipped := 0
for _, mr := range report.ModuleResults {
validatorRejected += len(mr.ValidatorRejected)
skipped += len(mr.SkippedChanges)
}
if validatorRejected < exp.MinimumCounts.Rejected {
t.Fatalf("expected at least %d validator rejections, got %d", exp.MinimumCounts.Rejected, validatorRejected)
}
if skipped < exp.MinimumCounts.Skipped {
t.Fatalf("expected at least %d application skips, got %d", exp.MinimumCounts.Skipped, skipped)
}
}
func assertReleaseUtilizationShape(t *testing.T, report reporting.ProcessReport) {
t.Helper()
var payload struct {
EffectiveConcurrency struct {
TotalLLM int `json:"total_llm"`
} `json:"effective_concurrency"`
RunTiming struct {
SchedulerQueueWaitMS int64 `json:"scheduler_queue_wait_ms"`
LLMExecutionTimeMS int64 `json:"llm_execution_time_ms"`
DeterministicValidationMS int64 `json:"deterministic_validation_time_ms"`
} `json:"run_timing"`
Modules []map[string]any `json:"modules"`
Validators []map[string]any `json:"validators"`
}
if err := json.Unmarshal(readFile(t, report.Diagnostics.UtilizationSummaryPath), &payload); err != nil {
t.Fatalf("unmarshal utilization diagnostics: %v", err)
}
if payload.EffectiveConcurrency.TotalLLM <= 0 {
t.Fatalf("expected positive total llm concurrency, got %+v", payload.EffectiveConcurrency)
}
if payload.RunTiming.SchedulerQueueWaitMS < 0 || payload.RunTiming.LLMExecutionTimeMS < 0 || payload.RunTiming.DeterministicValidationMS < 0 {
t.Fatalf("expected non-negative run timing values, got %+v", payload.RunTiming)
}
if len(payload.Modules) == 0 {
t.Fatalf("expected module timing summaries")
}
if len(payload.Validators) == 0 {
t.Fatalf("expected validator timing summaries")
}
}
func assertReleaseLedgerShape(t *testing.T, report reporting.ProcessReport) {
t.Helper()
var entries []struct {
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ProposalIndex int `json:"proposal_index"`
Disposition string `json:"disposition"`
DispositionReason string `json:"disposition_reason_code"`
OriginalText string `json:"original_text"`
ProposedCorrected string `json:"proposed_corrected_text"`
ReplacementPolicy string `json:"replacement_policy"`
DeterministicResults []struct {
ValidatorKey string `json:"validator_key"`
} `json:"deterministic_validator_decisions"`
LLMResults []struct {
ValidatorKey string `json:"validator_key"`
} `json:"llm_validator_decisions"`
}
if err := json.Unmarshal(readFile(t, report.Diagnostics.CorrectionLedgerPath), &entries); err != nil {
t.Fatalf("unmarshal correction ledger: %v", err)
}
if len(entries) == 0 {
t.Fatalf("expected correction ledger entries")
}
hasApplied := false
hasRejected := false
hasSkipped := false
for _, entry := range entries {
if entry.ModuleInstance == "" || entry.ModuleKey == "" {
t.Fatalf("expected module identity in ledger entry: %+v", entry)
}
switch entry.Disposition {
case "applied":
hasApplied = true
case "rejected":
hasRejected = true
case "skipped":
hasSkipped = true
}
}
if !hasApplied || !hasRejected {
t.Fatalf("expected applied and rejected entries in correction ledger, got %+v", entries)
}
if !hasSkipped {
// Some deterministic fixture paths do not trigger apply-time skips;
// rejections are still captured separately from application skips.
}
}
func assertPromptAndSchemaMetadataPresent(t *testing.T, runDir string) {
t.Helper()
metadataPaths, err := filepath.Glob(filepath.Join(runDir, "*", "*request-metadata.json"))
if err != nil {
t.Fatalf("glob request metadata artifacts: %v", err)
}
if len(metadataPaths) == 0 {
t.Fatalf("expected request metadata artifacts with prompt metadata")
}
foundPromptMetadata := false
foundSchemaMetadata := false
for _, path := range metadataPaths {
var payload map[string]any
if err := json.Unmarshal(readFile(t, path), &payload); err != nil {
t.Fatalf("unmarshal request metadata artifact %q: %v", path, err)
}
if pm, ok := payload["prompt_metadata"].(map[string]any); ok {
if pm["prompt_id"] != nil && pm["prompt_version"] != nil && pm["sha256"] != nil {
foundPromptMetadata = true
}
}
if sm, ok := payload["response_schema"].(map[string]any); ok {
if sm["id"] != nil && sm["version"] != nil && sm["name"] != nil && sm["sha256"] != nil {
foundSchemaMetadata = true
}
}
}
if !foundPromptMetadata {
t.Fatalf("expected prompt metadata in request metadata artifacts")
}
if !foundSchemaMetadata {
t.Fatalf("expected structured response schema metadata in request metadata artifacts")
}
}
func assertStableValidatorKeysPresent(t *testing.T, report reporting.ProcessReport) {
t.Helper()
seen := map[string]bool{}
for _, module := range report.ModuleResults {
for _, decision := range module.ValidatorDecisions {
seen[decision.ValidatorName] = true
}
for _, rejected := range module.ValidatorRejected {
seen[rejected.ValidatorName] = true
}
}
expectedAny := []string{
"confidence_threshold",
"original_text_presence",
"no_effect",
}
for _, key := range expectedAny {
if !seen[key] {
t.Fatalf("expected stable validator key %q in report decisions/rejections; seen=%v", key, seen)
}
}
}
func assertStdoutStderrContract(t *testing.T, stdout, stderr string) {
t.Helper()
if stdout != "" {
t.Fatalf("expected empty stdout with --output, got %q", stdout)
}
if strings.Contains(stderr, `"module_results"`) || strings.Contains(stderr, `"report_metadata"`) {
t.Fatalf("stderr should remain human-readable, not report JSON: %q", stderr)
}
}
func assertAuditaV1OutputShape(t *testing.T, outputPath string) {
t.Helper()
var payload struct {
Schema string `json:"schema"`
Version string `json:"version"`
Segments []schema.Segment `json:"segments"`
}
if err := json.Unmarshal(readFile(t, outputPath), &payload); err != nil {
t.Fatalf("unmarshal audita-v1 output: %v", err)
}
if payload.Schema != "audita-v1" || payload.Version != "v1" {
t.Fatalf("unexpected audita-v1 metadata: %+v", payload)
}
if len(payload.Segments) == 0 {
t.Fatalf("expected non-empty audita-v1 segments")
}
}

867
internal/cli/run.go Normal file
View File

@@ -0,0 +1,867 @@
package cli
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
coreio "gitea.maximumdirect.net/eric/audita/internal/core/io"
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
"gitea.maximumdirect.net/eric/audita/internal/framework/processreport"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
type noOpStructuredLLMClient struct{}
func (c noOpStructuredLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
_ = req
switch target := out.(type) {
case *validators.LLMValidationResponse:
*target = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{}}
case *proposal_generation.StructuredCorrectionSet:
*target = proposal_generation.StructuredCorrectionSet{Corrections: nil}
}
return contracts.StructuredCompletionResponse{}, nil
}
func shouldUseNoOpLLMClientForTests() bool {
return strings.HasSuffix(filepath.Base(os.Args[0]), ".test") || os.Getenv("GO_WANT_HELPER_PROCESS") == "1"
}
type processInvocation struct {
TranscriptPath string
GlossaryPath string
OutputPath string
ReportJSONPath string
Config config.Config
ConfigPath string
ConfigSource string
ConfigVersion *int
ExplicitModules bool
}
var processModuleFactory runner.ModuleFactory
var processProposalLLMClient contracts.StructuredLLMClient
var processProposalLLMScheduler runner.ValidationScheduler
var processValidationLLMClient contracts.StructuredLLMClient
var processValidationLLMScheduler runner.ValidationScheduler
var processRunnerContext = func() (context.Context, context.CancelFunc) {
return context.Background(), func() {}
}
var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) {
runDir, err := diagnostics.NewRunDirectory(inv.Config.WorkDir, string(inv.Config.WorkDirRetention))
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("run_dir_creation: %w", err)
}
fail := func(phase string, err error, runOutput *runner.RunOutput) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) {
_ = runDir.WriteErrorLog(fmt.Sprintf("%s: %v", phase, err))
return nil, nil, runOutput, runDir, fmt.Errorf("%s: %w", phase, err)
}
if err := runDir.WriteInvocationMetadata(diagnostics.InvocationMetadata{
Operation: "process",
TranscriptPath: inv.TranscriptPath,
GlossaryPath: inv.GlossaryPath,
OutputPath: inv.OutputPath,
ReportJSONPath: inv.ReportJSONPath,
ConfigPath: inv.ConfigPath,
ConfigSource: inv.ConfigSource,
ConfigVersion: inv.ConfigVersion,
TranscriptDescription: inv.Config.TranscriptDescription,
Modules: append([]string(nil), inv.Config.Modules...),
}); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("invocation_metadata: %v", err))
}
if err := runDir.WriteEffectiveConfig(inv.Config); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("effective_config: %v", err))
}
transcriptBytes, err := coreio.ReadRequiredFile(inv.TranscriptPath, "transcript")
if err != nil {
return fail("transcript_read", err, nil)
}
glossaryBytes, err := coreio.ReadRequiredFile(inv.GlossaryPath, "glossary")
if err != nil {
return fail("glossary_read", err, nil)
}
sourceTranscript, err := schema.ParseSourceTranscriptJSON(transcriptBytes)
if err != nil {
return fail("transcript_schema", err, nil)
}
glossary, err := schema.ParseGlossaryYAML(glossaryBytes)
if err != nil {
return fail("glossary_schema", err, nil)
}
if err := runDir.WriteSourceTranscript(sourceTranscript, transcriptBytes); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("source_artifact: %v", err))
}
canonical := sourceToCanonicalTranscript(sourceTranscript)
normalizer := normalization.NewNormalizer(normalization.NormalizationConfig{
MaxSegmentGap: inv.Config.Normalization.MaxSegmentGap,
EllipsisGap: inv.Config.Normalization.EllipsisGap,
MaxSegmentDuration: inv.Config.Normalization.MaxSegmentDuration,
MaxSegmentTokens: inv.Config.Normalization.MaxSegmentTokens,
})
normalizedTranscript, normSummary := normalizer.Normalize(canonical)
if err := runDir.WriteNormalizedTranscript(normalizedTranscript); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("normalized_artifact: %v", err))
}
if err := runDir.WriteNormalizationSummary(normSummary); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("normalization_summary: %v", err))
}
// Compute chunks after normalization
chunker := chunking.NewChunker(chunking.ChunkingConfig{
MaxSectionTokens: inv.Config.MaxSectionTokens,
MinSectionTokens: inv.Config.MinSectionTokens,
TargetSections: inv.Config.TargetSections,
})
sections, chunkErr := chunker.ChunkTranscript(normalizedTranscript)
if chunkErr != nil {
return fail("chunking", chunkErr, nil)
}
chunkConfig := chunking.ChunkingConfig{
MaxSectionTokens: inv.Config.MaxSectionTokens,
MinSectionTokens: inv.Config.MinSectionTokens,
TargetSections: inv.Config.TargetSections,
}
chunkSummary := chunking.ComputeSummary(sections, chunkConfig)
chunkDetailedSummary := chunking.ComputeDetailedSummary(sections, chunkConfig)
if err := runDir.WriteChunkingSummary(&chunkDetailedSummary); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("chunking_summary: %v", err))
}
workingTranscript := normalizedTranscript
var runOutput *runner.RunOutput
moduleFactory := processModuleFactory
if moduleFactory == nil {
moduleFactory = modules.NewFactory(modules.Dependencies{
Config: &inv.Config,
Glossary: glossary,
DiagnosticsDir: runDir.Path(),
})
}
if moduleFactory != nil {
proposalLLMClient := processProposalLLMClient
validationLLMClient := processValidationLLMClient
proposalScheduler := processProposalLLMScheduler
validationScheduler := processValidationLLMScheduler
if processModuleFactory == nil {
// Production runtime path: construct clients/schedulers from config.
if proposalLLMClient == nil {
if shouldUseNoOpLLMClientForTests() {
proposalLLMClient = noOpStructuredLLMClient{}
} else {
primaryCfg := llm.ResolvePrimaryConfig(inv.Config)
client, clientErr := llm.NewOpenAICompatibleClient(primaryCfg.ToOpenAICompatibleClientConfig(nil))
if clientErr != nil {
return fail("runner_setup", clientErr, nil)
}
proposalLLMClient = client
}
}
if validationLLMClient == nil {
if shouldUseNoOpLLMClientForTests() {
validationLLMClient = noOpStructuredLLMClient{}
} else {
validationCfg := llm.ResolveValidationConfig(inv.Config)
client, clientErr := llm.NewOpenAICompatibleClient(validationCfg.ToOpenAICompatibleClientConfig(nil))
if clientErr != nil {
return fail("runner_setup", clientErr, nil)
}
validationLLMClient = client
}
}
globalScheduler := proposalScheduler
if globalScheduler == nil {
s, sErr := llm.NewScheduler(inv.Config.TotalLLMConcurrency)
if sErr != nil {
return fail("runner_setup", sErr, nil)
}
globalScheduler = s
}
if proposalScheduler == nil {
proposalScheduler = globalScheduler
if inv.Config.EffectiveProposalLLMConcurrency() < inv.Config.TotalLLMConcurrency {
s, sErr := llm.NewScheduler(inv.Config.EffectiveProposalLLMConcurrency())
if sErr != nil {
return fail("runner_setup", sErr, nil)
}
proposalScheduler = composeSchedulers(globalScheduler, s)
}
}
if validationScheduler == nil {
validationScheduler = globalScheduler
if inv.Config.ValidationLLMConcurrency != nil && inv.Config.EffectiveValidationLLMConcurrency() < inv.Config.TotalLLMConcurrency {
s, sErr := llm.NewScheduler(inv.Config.EffectiveValidationLLMConcurrency())
if sErr != nil {
return fail("runner_setup", sErr, nil)
}
validationScheduler = composeSchedulers(globalScheduler, s)
}
}
}
moduleSpecs, err := contracts.ResolveModuleRunSpecs(inv.Config.Modules)
if err != nil {
return fail("runner_setup", err, nil)
}
runCtx, cancelRun := processRunnerContext()
defer cancelRun()
runnerResult, runErr := runner.New(moduleFactory).Run(runCtx, runner.RunInput{
Config: &inv.Config,
Transcript: normalizedTranscript,
Glossary: glossary,
ModuleSpecs: moduleSpecs,
EffectiveConcurrency: runner.EffectiveConcurrencyLimits{
TotalLLM: inv.Config.TotalLLMConcurrency,
ProposalLLM: inv.Config.EffectiveProposalLLMConcurrency(),
ValidationLLM: inv.Config.EffectiveValidationLLMConcurrency(),
},
ProposalLLMClient: proposalLLMClient,
ProposalLLMScheduler: proposalScheduler,
ProposalDiagnosticsDir: runDir.Path(),
ValidationLLMClient: validationLLMClient,
ValidationLLMScheduler: validationScheduler,
ValidationDiagnosticsDir: runDir.Path(),
})
runOutput = &runnerResult
if runErr != nil {
return fail("runner_execution", runErr, runOutput)
}
workingTranscript = runnerResult.FinalTranscript
}
encoderDef, err := outputschema.Resolve(inv.Config.OutputSchema)
if err != nil {
return fail("output_schema", err, runOutput)
}
outputBytes, err := encoderDef.Encoder(workingTranscript)
if err != nil {
return fail("serialization", err, runOutput)
}
if strings.TrimSpace(inv.OutputPath) != "" {
if err := coreio.WriteFile(inv.OutputPath, outputBytes); err != nil {
return fail("output_write", err, runOutput)
}
return normSummary, &chunkSummary, runOutput, runDir, nil
}
if _, err := stdout.Write(outputBytes); err != nil {
return fail("stdout_write", err, runOutput)
}
return normSummary, &chunkSummary, runOutput, runDir, nil
}
func sourceToCanonicalTranscript(source *schema.SourceTranscript) *schema.Transcript {
segments := make([]schema.Segment, len(source.Segments))
for i, s := range source.Segments {
id := i + 1
if s.ID != nil {
id = *s.ID
}
segments[i] = schema.Segment{
ID: id,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: s.Categories,
}
}
return &schema.Transcript{Segments: segments}
}
type chainedScheduler struct {
schedulers []runner.ValidationScheduler
}
func (s chainedScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
if len(s.schedulers) == 0 {
return fn(ctx)
}
run := fn
for i := len(s.schedulers) - 1; i >= 0; i-- {
scheduler := s.schedulers[i]
next := run
run = func(callCtx context.Context) error {
return scheduler.Run(callCtx, next)
}
}
return run(ctx)
}
func composeSchedulers(schedulers ...runner.ValidationScheduler) runner.ValidationScheduler {
filtered := make([]runner.ValidationScheduler, 0, len(schedulers))
for _, scheduler := range schedulers {
if scheduler != nil {
filtered = append(filtered, scheduler)
}
}
switch len(filtered) {
case 0:
return nil
case 1:
return filtered[0]
default:
return chainedScheduler{schedulers: filtered}
}
}
// Run executes the Audita CLI with the provided arguments and streams.
func Run(args []string, stdout, stderr io.Writer) int {
if len(args) == 0 {
writeRootUsage(stdout)
return 0
}
if isHelpCommand(args) {
writeRootUsage(stdout)
return 0
}
if args[0] == "process" {
return runProcess(args[1:], stdout, stderr)
}
if args[0] == "config" {
return runConfig(args[1:], stdout, stderr)
}
fmt.Fprintf(stderr, "audita: unknown command %q\n\n", args[0])
writeRootUsage(stderr)
return 2
}
func runProcess(args []string, stdout, stderr io.Writer) int {
startedAt := time.Now().UTC()
configPathOverride, configPathOverrideSet, err := findConfigPathOverride(args)
if err != nil {
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
return 2
}
effectiveConfig, err := config.LoadEffectiveConfig(configPathOverride, configPathOverrideSet)
if err != nil {
var effectiveConfigErr *config.EffectiveConfigError
if errors.As(err, &effectiveConfigErr) {
switch effectiveConfigErr.Kind {
case config.EffectiveConfigErrorLoadFile, config.EffectiveConfigErrorApplyFile:
fmt.Fprintf(stderr, "audita process: invalid config file: %v\n", effectiveConfigErr)
case config.EffectiveConfigErrorApplyEnv:
fmt.Fprintf(stderr, "audita process: invalid environment configuration: %v\n", effectiveConfigErr)
default:
fmt.Fprintf(stderr, "audita process: %v\n", effectiveConfigErr)
}
} else {
fmt.Fprintf(stderr, "audita process: %v\n", err)
}
return 2
}
cfg := effectiveConfig.Config
configPath := effectiveConfig.ConfigPath
configSource := effectiveConfig.ConfigSource
configVersion := effectiveConfig.ConfigVersion
fs, pFlags := newProcessFlagSet(cfg, stderr)
if isHelpCommand(args) || hasHelpFlag(args) {
writeProcessUsage(stdout, fs)
return 0
}
parseArgs := args
transcriptFromFront := ""
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
transcriptFromFront = args[0]
parseArgs = args[1:]
}
if err := fs.Parse(parseArgs); err != nil {
if errors.Is(err, flag.ErrHelp) {
writeProcessUsage(stdout, fs)
return 0
}
return 2
}
overrides, explicitModules := processCLIOverrides(fs, pFlags)
if err := cfg.ApplyCLIOverrides(overrides); err != nil {
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
return 2
}
remaining := fs.Args()
positional := make([]string, 0, len(remaining)+1)
if transcriptFromFront != "" {
positional = append(positional, transcriptFromFront)
}
positional = append(positional, remaining...)
if len(positional) != 1 {
fmt.Fprintln(stderr, "audita process: expected exactly 1 transcript JSON path argument")
return 2
}
if strings.TrimSpace(*pFlags.glossaryPath) == "" {
fmt.Fprintln(stderr, "audita process: --glossary is required")
return 2
}
inv := processInvocation{
TranscriptPath: positional[0],
GlossaryPath: *pFlags.glossaryPath,
OutputPath: *pFlags.outputPath,
ReportJSONPath: *pFlags.reportJSONPath,
Config: cfg,
ConfigPath: configPath,
ConfigSource: configSource,
ConfigVersion: configVersion,
ExplicitModules: explicitModules,
}
normSummary, chunkSummary, runOutput, runDir, runErr := processRunner(inv, stdout)
completedAt := time.Now().UTC()
if runErr != nil {
if runDir != nil && runOutput != nil {
if runOutput.Utilization != nil {
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
}
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
RunDirectoryPath: runDir.Path(),
RunOutput: runOutput,
}))
}
errorPhase, errorMessage := extractErrorPhase(runErr)
report := processreport.Build(processReportInput("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput))
if strings.TrimSpace(inv.ReportJSONPath) != "" {
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
fmt.Fprintf(stderr, "audita process: %v\n", err)
}
}
if runDir != nil {
_ = runDir.WriteReport(report)
_ = runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
RunSucceeded: false,
})
}
fmt.Fprintf(stderr, "audita process: %v\n", runErr)
if runDir != nil {
fmt.Fprintf(stderr, "audita process: diagnostics: %s\n", runDir.Path())
}
return 1
}
if runDir != nil && runOutput != nil {
if runOutput.Utilization != nil {
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
}
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
RunDirectoryPath: runDir.Path(),
RunOutput: runOutput,
}))
}
report := processreport.Build(processReportInput("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput))
if strings.TrimSpace(inv.ReportJSONPath) != "" {
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
if runDir != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("report_write: %v", err))
_ = runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
RunSucceeded: false,
})
}
fmt.Fprintf(stderr, "audita process: %v\n", err)
return 1
}
}
if runDir != nil {
_ = runDir.WriteReport(report)
if err := runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
RunSucceeded: true,
HasSkippedCorrections: processreport.HasSkippedCorrections(runOutput),
}); err != nil {
fmt.Fprintf(stderr, "audita process: failed to apply work-dir retention: %v\n", err)
return 1
}
}
return 0
}
func runConfig(args []string, stdout, stderr io.Writer) int {
if len(args) == 0 || isHelpCommand(args) || hasHelpFlag(args) {
writeConfigUsage(stdout)
return 0
}
switch args[0] {
case "validate":
return runConfigValidate(args[1:], stdout, stderr)
case "print-effective":
return runConfigPrintEffective(args[1:], stdout, stderr)
default:
fmt.Fprintf(stderr, "audita config: unknown command %q\n\n", args[0])
writeConfigUsage(stderr)
return 2
}
}
func runConfigValidate(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("config validate", flag.ContinueOnError)
fs.SetOutput(stderr)
configPath := fs.String("config", "", "Path to versioned YAML config file")
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
writeConfigValidateUsage(stdout)
return 0
}
return 2
}
if strings.TrimSpace(*configPath) == "" {
fmt.Fprintln(stderr, "audita config validate: --config is required")
return 2
}
if len(fs.Args()) != 0 {
fmt.Fprintln(stderr, "audita config validate: unexpected positional arguments")
return 2
}
fileCfg, err := config.LoadFileConfig(strings.TrimSpace(*configPath))
if err != nil {
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
return 2
}
cfg := config.Default()
if err := cfg.ApplyFileConfig(fileCfg); err != nil {
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
return 2
}
if err := cfg.Validate(); err != nil {
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
return 2
}
fmt.Fprintln(stdout, "config is valid")
return 0
}
func runConfigPrintEffective(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("config print-effective", flag.ContinueOnError)
fs.SetOutput(stderr)
configPath := fs.String("config", "", "Path to versioned YAML config file")
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
writeConfigPrintEffectiveUsage(stdout)
return 0
}
return 2
}
if len(fs.Args()) != 0 {
fmt.Fprintln(stderr, "audita config print-effective: unexpected positional arguments")
return 2
}
configPathValue := strings.TrimSpace(*configPath)
configPathSet := configPathValue != ""
effectiveConfig, err := config.LoadEffectiveConfig(configPathValue, configPathSet)
if err != nil {
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
return 2
}
cfg := effectiveConfig.Config
redacted := cfg.Redacted()
out, err := json.MarshalIndent(redacted, "", " ")
if err != nil {
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
return 1
}
out = append(out, '\n')
if _, err := stdout.Write(out); err != nil {
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
return 1
}
return 0
}
func extractErrorPhase(err error) (phase string, message string) {
msg := err.Error()
if strings.Contains(msg, ": ") {
parts := strings.SplitN(msg, ": ", 2)
if len(parts) == 2 {
return parts[0], parts[1]
}
}
return "", msg
}
func processReportInput(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) processreport.BuildInput {
runDirectoryPath := ""
if runDir != nil {
runDirectoryPath = runDir.Path()
}
return processreport.BuildInput{
Status: status,
TranscriptPath: inv.TranscriptPath,
GlossaryPath: inv.GlossaryPath,
OutputPath: inv.OutputPath,
Modules: inv.Config.Modules,
OutputSchema: inv.Config.OutputSchema,
ConfigVersion: inv.ConfigVersion,
StartedAt: startedAt,
CompletedAt: completedAt,
ErrorMessage: errorMessage,
ErrorPhase: errorPhase,
RunDirectoryPath: runDirectoryPath,
NormalizationSummary: normalizationSummary,
ChunkingSummary: chunkingSummary,
RunOutput: runOutput,
}
}
type processFlags struct {
configPath *string
glossaryPath *string
outputPath *string
reportJSONPath *string
modules *string
outputSchema *string
llmAPIKey *string
validationLLMAPIKey *string
model *string
validationModel *string
baseURL *string
validationBaseURL *string
llmTimeoutSeconds *int
totalLLMConcurrency *int
proposalLLMConcurrency *int
llmConcurrency *int
validationLLMTimeoutSeconds *int
validationMaxPromptTokens *int
targetSections *int
maxRetries *int
validationMaxRetries *int
validationLLMConcurrency *int
maxSectionTokens *int
minSectionTokens *int
glossaryConfidenceThreshold *float64
grammarConfidenceThreshold *float64
homophonesConfidenceThreshold *float64
spokenWordConfidenceThreshold *float64
normalizeMaxSegmentGap *float64
normalizeEllipsisGap *float64
normalizeMaxSegmentDuration *float64
normalizeMaxSegmentTokens *int
transcriptDescription *string
workDir *string
workDirRetention *string
}
func newProcessFlagSet(cfg config.Config, stderr io.Writer) (*flag.FlagSet, processFlags) {
fs := flag.NewFlagSet("process", flag.ContinueOnError)
fs.SetOutput(stderr)
validationTimeoutSecondsDefault := cfg.PrimaryLLM.TimeoutSeconds
if cfg.ValidationLLM.TimeoutSeconds != nil {
validationTimeoutSecondsDefault = *cfg.ValidationLLM.TimeoutSeconds
}
validationMaxRetriesDefault := cfg.PrimaryLLM.MaxRetries
if cfg.ValidationLLM.MaxRetries != nil {
validationMaxRetriesDefault = *cfg.ValidationLLM.MaxRetries
}
validationLLMConcurrencyDefault := cfg.TotalLLMConcurrency
if cfg.ValidationLLMConcurrency != nil {
validationLLMConcurrencyDefault = *cfg.ValidationLLMConcurrency
}
targetSectionsDefault := 0
if cfg.TargetSections != nil {
targetSectionsDefault = *cfg.TargetSections
}
pFlags := processFlags{
configPath: fs.String("config", "", "Path to versioned YAML config file"),
glossaryPath: fs.String("glossary", "", "Path to glossary YAML file"),
outputPath: fs.String("output", "", "Path to corrected transcript JSON output file"),
reportJSONPath: fs.String("report-json", "", "Path to machine-readable report JSON output file"),
modules: fs.String("modules", strings.Join(cfg.Modules, ","), "Comma-separated module sequence override"),
outputSchema: fs.String("output-schema", cfg.OutputSchema, "Output schema: bare-segments|audita-v1"),
llmAPIKey: fs.String("llm-api-key", cfg.PrimaryLLM.APIKey, "Primary LLM API key"),
validationLLMAPIKey: fs.String("validation-llm-api-key", cfg.ValidationLLM.APIKey, "Validation LLM API key"),
model: fs.String("model", cfg.PrimaryLLM.Model, "Primary LLM model name"),
validationModel: fs.String("validation-model", cfg.ValidationLLM.Model, "Validation LLM model name"),
baseURL: fs.String("base-url", cfg.PrimaryLLM.BaseURL, "Primary OpenAI-compatible base URL"),
validationBaseURL: fs.String("validation-base-url", cfg.ValidationLLM.BaseURL, "Validation OpenAI-compatible base URL"),
llmTimeoutSeconds: fs.Int("llm-timeout-seconds", cfg.PrimaryLLM.TimeoutSeconds, "Primary LLM timeout in seconds"),
totalLLMConcurrency: fs.Int("total-llm-concurrency", cfg.TotalLLMConcurrency, "Total concurrent LLM calls across proposal and validation"),
proposalLLMConcurrency: fs.Int("proposal-llm-concurrency", cfg.EffectiveProposalLLMConcurrency(), "Concurrent proposal-generation LLM calls"),
llmConcurrency: fs.Int("llm-concurrency", cfg.TotalLLMConcurrency, "Alias for --total-llm-concurrency"),
validationLLMTimeoutSeconds: fs.Int("validation-llm-timeout-seconds", validationTimeoutSecondsDefault, "Validation LLM timeout in seconds"),
validationMaxPromptTokens: fs.Int("validation-max-prompt-tokens", cfg.ValidationMaxPromptTokens, "Validation max prompt tokens"),
targetSections: fs.Int("target-sections", targetSectionsDefault, "Target number of transcript sections"),
maxRetries: fs.Int("max-retries", cfg.PrimaryLLM.MaxRetries, "Maximum structured-output retries"),
validationMaxRetries: fs.Int("validation-max-retries", validationMaxRetriesDefault, "Validation structured-output retries"),
validationLLMConcurrency: fs.Int("validation-llm-concurrency", validationLLMConcurrencyDefault, "Concurrent validation LLM calls (inherits total when unset)"),
maxSectionTokens: fs.Int("max-section-tokens", cfg.MaxSectionTokens, "Maximum section tokens"),
minSectionTokens: fs.Int("min-section-tokens", cfg.MinSectionTokens, "Minimum section tokens"),
glossaryConfidenceThreshold: fs.Float64("glossary-confidence-threshold", cfg.Thresholds.Glossary, "Glossary confidence threshold"),
grammarConfidenceThreshold: fs.Float64("grammar-confidence-threshold", cfg.Thresholds.Grammar, "Grammar confidence threshold"),
homophonesConfidenceThreshold: fs.Float64("homophones-confidence-threshold", cfg.Thresholds.Homophones, "Homophones confidence threshold"),
spokenWordConfidenceThreshold: fs.Float64("spoken-word-confidence-threshold", cfg.Thresholds.SpokenWord, "Spoken-word confidence threshold"),
normalizeMaxSegmentGap: fs.Float64("normalize-max-segment-gap", cfg.Normalization.MaxSegmentGap, "Maximum same-speaker merge gap"),
normalizeEllipsisGap: fs.Float64("normalize-ellipsis-gap", cfg.Normalization.EllipsisGap, "Gap threshold for ellipsis insertion"),
normalizeMaxSegmentDuration: fs.Float64("normalize-max-segment-duration", cfg.Normalization.MaxSegmentDuration, "Maximum merged segment duration"),
normalizeMaxSegmentTokens: fs.Int("normalize-max-segment-tokens", cfg.Normalization.MaxSegmentTokens, "Maximum merged segment token estimate"),
transcriptDescription: fs.String("transcript-description", cfg.TranscriptDescription, "Brief background context for LLM prompts; does not override transcript content"),
workDir: fs.String("work-dir", cfg.WorkDir, "Per-run work directory"),
workDirRetention: fs.String("work-dir-retention", string(cfg.WorkDirRetention), "Work-dir retention policy: auto|always|never"),
}
return fs, pFlags
}
func findConfigPathOverride(args []string) (path string, set bool, err error) {
for i := 0; i < len(args); i++ {
arg := strings.TrimSpace(args[i])
if arg == "" {
continue
}
if arg == "--config" {
if i+1 >= len(args) {
return "", false, fmt.Errorf("--config requires a path")
}
return strings.TrimSpace(args[i+1]), true, nil
}
if strings.HasPrefix(arg, "--config=") {
return strings.TrimSpace(strings.TrimPrefix(arg, "--config=")), true, nil
}
}
return "", false, nil
}
func isHelpCommand(args []string) bool {
if len(args) == 0 {
return false
}
if len(args) == 1 {
switch args[0] {
case "help", "-h", "--help":
return true
}
}
if len(args) == 2 && args[0] == "help" {
switch args[1] {
case "process":
return true
}
}
return false
}
func hasHelpFlag(args []string) bool {
for _, arg := range args {
if arg == "-h" || arg == "--help" {
return true
}
}
return false
}
func writeRootUsage(w io.Writer) {
fmt.Fprintln(w, "Audita is a transcript processing CLI.")
fmt.Fprintln(w)
fmt.Fprintln(w, "Usage:")
fmt.Fprintln(w, " audita <command> [options]")
fmt.Fprintln(w)
fmt.Fprintln(w, "Commands:")
fmt.Fprintln(w, " process Process a transcript JSON file")
fmt.Fprintln(w, " config Validate and inspect config")
fmt.Fprintln(w)
fmt.Fprintln(w, "Example:")
fmt.Fprintln(w, " audita process transcript.json --glossary glossary.yaml --output corrected.json")
}
func writeConfigUsage(w io.Writer) {
fmt.Fprintln(w, "Validate and inspect Audita config.")
fmt.Fprintln(w)
fmt.Fprintln(w, "Usage:")
fmt.Fprintln(w, " audita config <command> [flags]")
fmt.Fprintln(w)
fmt.Fprintln(w, "Commands:")
fmt.Fprintln(w, " validate Validate a versioned YAML config file")
fmt.Fprintln(w, " print-effective Print redacted effective config JSON (defaults + config file + env)")
}
func writeConfigValidateUsage(w io.Writer) {
fmt.Fprintln(w, "Validate a versioned YAML config file.")
fmt.Fprintln(w)
fmt.Fprintln(w, "Usage:")
fmt.Fprintln(w, " audita config validate --config <path>")
}
func writeConfigPrintEffectiveUsage(w io.Writer) {
fmt.Fprintln(w, "Print redacted effective config JSON.")
fmt.Fprintln(w)
fmt.Fprintln(w, "Usage:")
fmt.Fprintln(w, " audita config print-effective [--config <path>]")
}
func writeProcessUsage(w io.Writer, fs *flag.FlagSet) {
fmt.Fprintln(w, "Process a transcript JSON file.")
fmt.Fprintln(w)
fmt.Fprintln(w, "Usage:")
fmt.Fprintln(w, " audita process <transcript.json> [flags]")
fmt.Fprintln(w)
fmt.Fprintln(w, "Flags:")
fs.VisitAll(func(f *flag.Flag) {
fmt.Fprintf(w, " --%s\n", f.Name)
})
fmt.Fprintln(w)
fmt.Fprintln(w, "Example:")
fmt.Fprintln(w, " audita process transcript.json --glossary glossary.yaml --output corrected.json")
}

4354
internal/cli/run_test.go Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,110 @@
package cli
import (
"context"
"errors"
"os"
"strconv"
"strings"
"sync"
"time"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
const (
subprocessTestLLMModeEnv = "AUDITA_SUBPROCESS_TEST_LLM_MODE"
subprocessTestRunTimeoutMSEnv = "AUDITA_SUBPROCESS_TEST_RUN_TIMEOUT_MS"
)
// ConfigureSubprocessTestHooksFromEnv enables deterministic test-only hooks for
// subprocess integration tests that run through the Go test binary helper path.
func ConfigureSubprocessTestHooksFromEnv() {
mode := strings.TrimSpace(os.Getenv(subprocessTestLLMModeEnv))
timeoutMSRaw := strings.TrimSpace(os.Getenv(subprocessTestRunTimeoutMSEnv))
// Only activate in explicit subprocess test mode.
if mode == "" && timeoutMSRaw == "" {
return
}
if mode != "" {
client := &subprocessTestLLMClient{mode: mode}
processProposalLLMClient = client
processValidationLLMClient = client
}
if timeoutMSRaw == "" {
return
}
timeoutMS, err := strconv.Atoi(timeoutMSRaw)
if err != nil || timeoutMS <= 0 {
return
}
processRunnerContext = func() (context.Context, context.CancelFunc) {
return context.WithTimeout(context.Background(), time.Duration(timeoutMS)*time.Millisecond)
}
}
type subprocessTestLLMClient struct {
mode string
mu sync.Mutex
proposals int
}
func (c *subprocessTestLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = req
switch c.mode {
case "backend_error":
return contracts.StructuredCompletionResponse{}, errors.New("synthetic backend failure")
case "block_until_cancel":
<-ctx.Done()
return contracts.StructuredCompletionResponse{}, ctx.Err()
case "malformed_structured":
switch target := out.(type) {
case *proposal_generation.StructuredCorrectionSet:
*target = proposal_generation.StructuredCorrectionSet{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y", Confidence: 0.99},
},
}
case *validators.LLMValidationResponse:
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 999, Approved: true, Confidence: 0.9, Reason: "bad index"},
},
}
}
case "mid_pipeline_fail":
if _, ok := out.(*proposal_generation.StructuredCorrectionSet); ok {
c.mu.Lock()
c.proposals++
proposalCall := c.proposals
c.mu.Unlock()
if proposalCall >= 3 {
return contracts.StructuredCompletionResponse{}, errors.New("synthetic mid-pipeline failure")
}
}
switch target := out.(type) {
case *proposal_generation.StructuredCorrectionSet:
*target = proposal_generation.StructuredCorrectionSet{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99},
},
}
case *validators.LLMValidationResponse:
*target = validators.LLMValidationResponse{Validations: nil}
}
default:
switch target := out.(type) {
case *proposal_generation.StructuredCorrectionSet:
*target = proposal_generation.StructuredCorrectionSet{Corrections: nil}
case *validators.LLMValidationResponse:
*target = validators.LLMValidationResponse{Validations: nil}
}
}
return contracts.StructuredCompletionResponse{}, nil
}

View File

@@ -0,0 +1 @@
{"segments":[{"id":1,"text":"oops"}

View File

@@ -0,0 +1,22 @@
{
"name": "ambiguous_match_rejected_before_application",
"transcript_file": "application-skip-ambiguous.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"modules_csv": "homophones",
"proposal_responses_file": "application-skip-ambiguous.proposals.json",
"validation_responses_file": "application-skip-ambiguous.validations.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "application-skip-ambiguous.expected-transcript.json",
"module_instances": ["homophones"],
"module_count": 1,
"total_applied_changes": 0,
"total_skipped_changes": 1,
"module_applied_counts": [0],
"module_rejected_counts": [1],
"module_skip_counts": [0],
"validator_rejected_reason_codes": ["ambiguous_original_text"],
"expected_proposal_calls": ["homophones:proposal"]
}
}

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"the site near another site"}
]

View File

@@ -0,0 +1,3 @@
[
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"the site near another site"}
]

View File

@@ -0,0 +1,4 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

View File

@@ -0,0 +1,28 @@
{
"name": "default_full_pipeline_shape_and_reports",
"transcript_file": "default-full-pipeline.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"proposal_responses_file": "default-full-pipeline.proposals.json",
"validation_responses_file": "default-full-pipeline.validations.json",
"env": {
"AUDITA_LLM_API_KEY": "parity-secret",
"AUDITA_VALIDATION_LLM_API_KEY": "parity-secret"
},
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "default-full-pipeline.expected-transcript.json",
"module_instances": ["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
"module_count": 5,
"total_applied_changes": 3,
"total_skipped_changes": 3,
"secret_markers": ["parity-secret"],
"expected_proposal_calls": [
"glossary_1:proposal",
"homophones:proposal",
"glossary_2:proposal",
"spoken_word:proposal",
"grammar:proposal"
]
}
}

View File

@@ -0,0 +1,9 @@
[
{
"id": 1,
"speaker": "Alice",
"start": 0,
"end": 1,
"text": "Hello, there were Jesters hmm"
}
]

View File

@@ -0,0 +1,7 @@
glossary:
- name: Jesters
aliases:
- jester
plural: jesters
category: faction
summary: A protected in-world faction term.

View File

@@ -0,0 +1,28 @@
[
{
"corrections": [
{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "Jesters", "corrected_text": "jesters", "confidence": 0.99},
{"id": 1, "original_text": "Jesters", "corrected_text": "JESTERX", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "jesters", "corrected_text": "JESTERS", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "uh", "corrected_text": "hmm", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
]
}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
]

View File

@@ -0,0 +1,12 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject cleanup"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "parity-secret"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

View File

@@ -0,0 +1,27 @@
{
"name": "default_pipeline_handoff_and_module_order",
"transcript_file": "default-handoff.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"proposal_responses_file": "default-handoff.proposals.json",
"validation_responses_file": "default-handoff.validations.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "default-handoff.expected-transcript.json",
"module_instances": ["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
"module_count": 5,
"total_applied_changes": 5,
"total_skipped_changes": 0,
"module_applied_counts": [1, 1, 1, 1, 1],
"module_rejected_counts": [0, 0, 0, 0, 0],
"module_skip_counts": [0, 0, 0, 0, 0],
"expected_proposal_calls": [
"glossary_1:proposal",
"homophones:proposal",
"glossary_2:proposal",
"spoken_word:proposal",
"grammar:proposal"
],
"min_response_payload_artifacts": 15
}
}

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"Hello, there were Jesters at the Sight um"}
]

View File

@@ -0,0 +1,6 @@
glossary:
- name: Jesters
aliases: [jester]
plural: jesters
category: faction
summary: A protected in-world faction term.

View File

@@ -0,0 +1,7 @@
[
{"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "sight", "corrected_text": "Sight", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "um um", "corrected_text": "um", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}]}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures at the site um um"}
]

View File

@@ -0,0 +1,12 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

View File

@@ -0,0 +1,17 @@
{
"name": "deterministic_validator_low_confidence",
"transcript_file": "deterministic-validator-low-confidence.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"modules_csv": "grammar",
"proposal_responses_file": "deterministic-validator-low-confidence.proposals.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "deterministic-validator-low-confidence.expected-transcript.json",
"module_instances": ["grammar"],
"module_count": 1,
"total_skipped_changes": 1,
"validator_rejected_reason_codes": ["low_confidence"],
"expected_proposal_calls": ["grammar:proposal"]
}
}

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"hello , world"}
]

View File

@@ -0,0 +1,7 @@
[
{
"corrections": [
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.1}
]
}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , world"}
]

View File

@@ -0,0 +1,12 @@
{
"name": "glossary_schema_handling",
"transcript_file": "default-full-pipeline.transcript.json",
"glossary_file": "glossary-schema-error.glossary.yaml",
"expect": {
"exit_code": 1,
"status": "failed",
"error_phase": "glossary_schema",
"stderr_contains": "glossary_schema",
"require_error_log": true
}
}

View File

@@ -0,0 +1,2 @@
glossary:
- name: MissingCategoryAndSummary

View File

@@ -0,0 +1,19 @@
{
"name": "llm_validator_decision_handling",
"transcript_file": "llm-validator-rejection.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"modules_csv": "grammar",
"proposal_responses_file": "llm-validator-rejection.proposals.json",
"validation_responses_file": "llm-validator-rejection.validations.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "llm-validator-rejection.expected-transcript.json",
"module_instances": ["grammar"],
"module_count": 1,
"total_skipped_changes": 1,
"validator_rejected_reason_codes": ["llm_rejected"],
"expected_proposal_calls": ["grammar:proposal"],
"expected_validation_calls": ["grammar:section-0000:editorial_review:batch-0000"]
}
}

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"hello , world"}
]

View File

@@ -0,0 +1,7 @@
[
{
"corrections": [
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
]
}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , world"}
]

View File

@@ -0,0 +1,7 @@
[
{
"validations": [
{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject stylistic overreach"}
]
}
]

View File

@@ -0,0 +1,23 @@
{
"name": "mid_pipeline_failure_partial_progress",
"transcript_file": "default-handoff.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"proposal_responses_file": "mid-pipeline-failure.proposals.json",
"validation_responses_file": "mid-pipeline-failure.validations.json",
"expect": {
"exit_code": 1,
"status": "failed",
"error_phase": "runner_execution",
"stderr_contains": "runner_execution",
"module_instances": ["glossary_1", "homophones", "glossary_2"],
"module_count": 3,
"total_applied_changes": 2,
"total_skipped_changes": 0,
"failed_module_instance": "glossary_2",
"module_applied_counts": [1, 1, 0],
"module_rejected_counts": [0, 0, 0],
"module_skip_counts": [0, 0, 0],
"require_error_log": true,
"expected_proposal_calls": ["glossary_1:proposal", "homophones:proposal", "glossary_2:proposal"]
}
}

View File

@@ -0,0 +1,4 @@
[
{"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]}
]

View File

@@ -0,0 +1,6 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

View File

@@ -0,0 +1,21 @@
{
"name": "protected_glossary_term_behavior",
"transcript_file": "protected-term-rejection.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"modules_csv": "homophones",
"proposal_responses_file": "protected-term-rejection.proposals.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "protected-term-rejection.expected-transcript.json",
"module_instances": ["homophones"],
"module_count": 1,
"total_applied_changes": 0,
"total_skipped_changes": 1,
"module_applied_counts": [0],
"module_rejected_counts": [1],
"module_skip_counts": [0],
"validator_rejected_reason_codes": ["protected_glossary_term"],
"expected_proposal_calls": ["homophones:proposal"]
}
}

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"The Jesters entered the hall."}
]

View File

@@ -0,0 +1,3 @@
[
{"corrections": [{"id": 1, "original_text": "Jesters", "corrected_text": "Gestures", "confidence": 0.99}]}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"The Jesters entered the hall."}
]

View File

@@ -0,0 +1,12 @@
{
"name": "transcript_schema_handling",
"transcript_file": "transcript-schema-error.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"expect": {
"exit_code": 1,
"status": "failed",
"error_phase": "transcript_schema",
"stderr_contains": "transcript_schema",
"require_error_log": true
}
}

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"","start":0.0,"end":1.0,"text":"bad"}
]

View File

@@ -0,0 +1,24 @@
{
"must_apply_texts": [
"Hello, there were Jesters hmm"
],
"must_not_apply_texts": [
"JESTERX",
"there were gestures"
],
"protected_terms": [
"Jesters"
],
"expected_module_instances": [
"glossary_1",
"homophones",
"glossary_2",
"spoken_word",
"grammar"
],
"minimum_counts": {
"applied": 1,
"rejected": 1,
"skipped": 0
}
}

View File

@@ -0,0 +1,9 @@
[
{
"id": 1,
"speaker": "Alice",
"start": 0,
"end": 1,
"text": "Hello, there were Jesters hmm"
}
]

View File

@@ -0,0 +1,7 @@
glossary:
- name: Jesters
aliases:
- jester
plural: jesters
category: faction
summary: A protected in-world faction term.

View File

@@ -0,0 +1,28 @@
[
{
"corrections": [
{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "Jesters", "corrected_text": "jesters", "confidence": 0.99},
{"id": 1, "original_text": "Jesters", "corrected_text": "JESTERX", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "jesters", "corrected_text": "JESTERS", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "uh", "corrected_text": "hmm", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
]
}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
]

View File

@@ -0,0 +1,12 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject cleanup"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "release-secret"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

View File

@@ -0,0 +1,6 @@
glossary:
- name: Audita
aliases:
- audita
category: product
summary: The Audita transcript correction CLI.

View File

@@ -0,0 +1,9 @@
[
{
"id": 1,
"speaker": "A",
"start": 0.0,
"end": 1.2,
"text": "hello world"
}
]

View File

@@ -0,0 +1,312 @@
package chunking
import (
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
// Section represents a contiguous chunk of transcript segments with metadata.
type Section struct {
// Index is the 0-based section index within the chunked transcript
Index int `json:"section_index"`
// StartSegmentID is the ID of the first segment in this section
StartSegmentID int `json:"start_segment_id"`
// EndSegmentID is the ID of the last segment in this section
EndSegmentID int `json:"end_segment_id"`
// EstimatedTokens is the approximate token count for this section
EstimatedTokens int `json:"estimated_tokens"`
// Segments contains the segments in this section, in order
Segments []schema.Segment `json:"segments"`
}
// ChunkingConfig holds configuration for transcript chunking.
type ChunkingConfig struct {
// MaxSectionTokens is the maximum allowed tokens per section
MaxSectionTokens int
// MinSectionTokens is a validated soft lower-bound setting retained for
// configuration/reporting compatibility.
MinSectionTokens int
// TargetSections is an optional target number of sections
// If nil, section count is derived from total/max token budgeting.
TargetSections *int
}
// Chunker performs deterministic chunking of normalized transcript segments.
type Chunker struct {
config ChunkingConfig
estimator TokenEstimator
}
// NewChunker creates a new chunker with the given configuration.
func NewChunker(config ChunkingConfig) *Chunker {
return &Chunker{
config: config,
estimator: NewSimpleTokenEstimator(),
}
}
// NewChunkerWithEstimator creates a new chunker with a custom estimator.
func NewChunkerWithEstimator(config ChunkingConfig, estimator TokenEstimator) *Chunker {
return &Chunker{
config: config,
estimator: estimator,
}
}
// ChunkTranscript divides a normalized transcript into contiguous token-bounded
// sections using a deterministic balanced forward pass.
//
// Behavior:
// - preserve segment order and never split segments;
// - estimate per-segment tokens once, then compute total;
// - derive desired section count from ceil(total/max_section_tokens), unless
// target_sections is explicitly set;
// - prefer section sizes near ceil(total/section_count) while never exceeding
// max_section_tokens unless a section consists of a single oversized segment.
//
// Returns an error if explicit target_sections is impossible under constraints.
// The input transcript is never mutated.
func (c *Chunker) ChunkTranscript(transcript *schema.Transcript) ([]Section, error) {
if transcript == nil || len(transcript.Segments) == 0 {
return []Section{}, nil
}
// Validate configuration
if c.config.MaxSectionTokens <= 0 {
return nil, fmt.Errorf("max_section_tokens must be positive, got %d", c.config.MaxSectionTokens)
}
if c.config.MinSectionTokens < 0 {
return nil, fmt.Errorf("min_section_tokens must be non-negative, got %d", c.config.MinSectionTokens)
}
if c.config.MinSectionTokens > c.config.MaxSectionTokens {
return nil, fmt.Errorf("min_section_tokens (%d) cannot exceed max_section_tokens (%d)",
c.config.MinSectionTokens, c.config.MaxSectionTokens)
}
// Calculate token counts for each segment (deterministic).
segmentTokens := make([]int, len(transcript.Segments))
totalTokens := 0
for i, seg := range transcript.Segments {
segmentTokens[i] = c.estimator.EstimateTokens(seg.Text)
totalTokens += segmentTokens[i]
}
minPossibleSections := c.calculateMinPossibleSections(segmentTokens)
var desiredSections int
useExplicitTarget := false
if c.config.TargetSections != nil {
desiredSections = *c.config.TargetSections
useExplicitTarget = true
if desiredSections <= 0 {
return nil, fmt.Errorf("target_sections must be positive, got %d", desiredSections)
}
if err := c.validateTargetSections(desiredSections, segmentTokens); err != nil {
return nil, err
}
} else {
desiredSections = ceilDiv(totalTokens, c.config.MaxSectionTokens)
if desiredSections < minPossibleSections {
desiredSections = minPossibleSections
}
if desiredSections < 1 {
desiredSections = 1
}
if desiredSections > len(transcript.Segments) {
desiredSections = len(transcript.Segments)
}
}
targetTokensPerSection := ceilDiv(totalTokens, desiredSections)
if useExplicitTarget {
return c.buildSectionsWithExplicitTarget(
transcript.Segments,
segmentTokens,
desiredSections,
targetTokensPerSection,
)
}
return c.buildSectionsBalanced(transcript.Segments, segmentTokens, targetTokensPerSection), nil
}
// validateTargetSections checks if the target section count is achievable.
func (c *Chunker) validateTargetSections(target int, segmentTokens []int) error {
// Maximum possible sections: limited by segment count
maxPossibleSections := len(segmentTokens)
if target > maxPossibleSections {
return fmt.Errorf(
"target_sections (%d) is impossible: cannot have more sections than segments (%d)",
target, maxPossibleSections)
}
// Minimum possible sections: each segment must fit within max bounds
minPossibleSections := c.calculateMinPossibleSections(segmentTokens)
if target < minPossibleSections {
return fmt.Errorf(
"target_sections (%d) is impossible: need at least %d sections to respect max_section_tokens (%d)",
target, minPossibleSections, c.config.MaxSectionTokens)
}
return nil
}
// buildSectionsBalanced creates sections with a deterministic single-pass policy.
func (c *Chunker) buildSectionsBalanced(segments []schema.Segment, segmentTokens []int, targetTokensPerSection int) []Section {
var sections []Section
var currentSegments []schema.Segment
currentTokens := 0
for i, seg := range segments {
tokens := segmentTokens[i]
// Empty section: always accept the next segment, including oversized.
if len(currentSegments) == 0 {
currentSegments = append(currentSegments, seg)
currentTokens = tokens
continue
}
// If adding next segment would exceed max, close current section.
if currentTokens+tokens > c.config.MaxSectionTokens {
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
currentSegments = []schema.Segment{seg}
currentTokens = tokens
continue
}
// Prefer staying near target tokens per section.
if targetTokensPerSection == 0 || currentTokens < targetTokensPerSection {
currentSegments = append(currentSegments, seg)
currentTokens += tokens
continue
}
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
currentSegments = []schema.Segment{seg}
currentTokens = tokens
}
if len(currentSegments) > 0 {
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
}
return sections
}
// buildSection creates a Section from segments.
func (c *Chunker) buildSection(index int, segments []schema.Segment, tokens int) Section {
return Section{
Index: index,
StartSegmentID: segments[0].ID,
EndSegmentID: segments[len(segments)-1].ID,
EstimatedTokens: tokens,
Segments: segments,
}
}
// calculateMinPossibleSections calculates the minimum number of sections needed
// to ensure no section exceeds max tokens.
func (c *Chunker) calculateMinPossibleSections(segmentTokens []int) int {
sections := 0
currentTokens := 0
for _, tokens := range segmentTokens {
if tokens > c.config.MaxSectionTokens {
// Each oversized segment needs its own section
if currentTokens > 0 {
sections++
currentTokens = 0
}
sections++
} else if currentTokens+tokens > c.config.MaxSectionTokens {
sections++
currentTokens = tokens
} else {
currentTokens += tokens
}
}
if currentTokens > 0 {
sections++
}
return sections
}
// buildSectionsWithExplicitTarget builds exactly desiredSections when feasible.
func (c *Chunker) buildSectionsWithExplicitTarget(
segments []schema.Segment,
segmentTokens []int,
desiredSections int,
targetTokensPerSection int,
) ([]Section, error) {
n := len(segments)
cursor := 0
sections := make([]Section, 0, desiredSections)
for sectionIdx := 0; sectionIdx < desiredSections; sectionIdx++ {
if cursor >= n {
break
}
remainingSectionsAfter := desiredSections - sectionIdx - 1
currentSegments := []schema.Segment{segments[cursor]}
currentTokens := segmentTokens[cursor]
cursor++
for cursor < n {
remainingSegments := n - cursor
// Reserve one segment per future section to avoid empty sections.
if remainingSegments == remainingSectionsAfter {
break
}
nextTokens := segmentTokens[cursor]
if currentTokens+nextTokens > c.config.MaxSectionTokens {
break
}
if targetTokensPerSection == 0 || currentTokens < targetTokensPerSection {
currentSegments = append(currentSegments, segments[cursor])
currentTokens += nextTokens
cursor++
continue
}
break
}
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
}
if cursor != n || len(sections) != desiredSections {
return nil, fmt.Errorf(
"target_sections (%d) is impossible under current constraints (got %d sections)",
desiredSections,
len(sections),
)
}
return sections, nil
}
func ceilDiv(numerator int, denominator int) int {
if denominator <= 0 {
return 0
}
if numerator <= 0 {
return 0
}
return (numerator + denominator - 1) / denominator
}

View File

@@ -0,0 +1,518 @@
package chunking
import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
// mapTokenEstimator provides deterministic per-segment token counts for tests.
type mapTokenEstimator struct {
byText map[string]int
}
func (e *mapTokenEstimator) EstimateTokens(text string) int {
if e.byText == nil {
return 0
}
if tokens, ok := e.byText[text]; ok {
return tokens
}
return 0
}
func makeSegments(texts []string) []schema.Segment {
segments := make([]schema.Segment, len(texts))
for i, text := range texts {
segments[i] = schema.Segment{
ID: i + 1,
Speaker: "DM",
Start: float64(i * 10),
End: float64(i*10 + 5),
Text: text,
}
}
return segments
}
func makeTranscript(segments []schema.Segment) *schema.Transcript {
return &schema.Transcript{Segments: segments}
}
func intPtr(i int) *int {
return &i
}
func assertSegmentCoverageAndOrder(t *testing.T, input []schema.Segment, sections []Section) {
t.Helper()
seen := make([]schema.Segment, 0, len(input))
for _, sec := range sections {
seen = append(seen, sec.Segments...)
}
if len(seen) != len(input) {
t.Fatalf("expected %d total segment occurrences, got %d", len(input), len(seen))
}
for i := range input {
if seen[i].ID != input[i].ID {
t.Fatalf("segment order mismatch at index %d: got id=%d want id=%d", i, seen[i].ID, input[i].ID)
}
}
}
func assertSectionMetadataConsistent(t *testing.T, sections []Section) {
t.Helper()
for i, sec := range sections {
if sec.Index != i {
t.Fatalf("section %d: expected index=%d got=%d", i, i, sec.Index)
}
if len(sec.Segments) == 0 {
t.Fatalf("section %d: section must not be empty", i)
}
if sec.StartSegmentID != sec.Segments[0].ID {
t.Fatalf("section %d: start_segment_id mismatch", i)
}
if sec.EndSegmentID != sec.Segments[len(sec.Segments)-1].ID {
t.Fatalf("section %d: end_segment_id mismatch", i)
}
}
}
func assertMaxBoundExceptSingletonOversized(t *testing.T, sections []Section, max int) {
t.Helper()
for i, sec := range sections {
if sec.EstimatedTokens <= max {
continue
}
if len(sec.Segments) != 1 {
t.Fatalf("section %d exceeds max tokens (%d>%d) with %d segments", i, sec.EstimatedTokens, max, len(sec.Segments))
}
}
}
func imbalance(sections []Section) int {
if len(sections) == 0 {
return 0
}
minTokens := sections[0].EstimatedTokens
maxTokens := sections[0].EstimatedTokens
for _, sec := range sections {
if sec.EstimatedTokens < minTokens {
minTokens = sec.EstimatedTokens
}
if sec.EstimatedTokens > maxTokens {
maxTokens = sec.EstimatedTokens
}
}
return maxTokens - minTokens
}
func greedyMaxFillSections(segments []schema.Segment, tokens []int, max int) []Section {
sections := make([]Section, 0)
var current []schema.Segment
currentTokens := 0
for i, seg := range segments {
tok := tokens[i]
if len(current) == 0 {
current = append(current, seg)
currentTokens = tok
continue
}
if currentTokens+tok > max {
sections = append(sections, Section{
Index: len(sections),
StartSegmentID: current[0].ID,
EndSegmentID: current[len(current)-1].ID,
EstimatedTokens: currentTokens,
Segments: append([]schema.Segment(nil), current...),
})
current = []schema.Segment{seg}
currentTokens = tok
continue
}
current = append(current, seg)
currentTokens += tok
}
if len(current) > 0 {
sections = append(sections, Section{
Index: len(sections),
StartSegmentID: current[0].ID,
EndSegmentID: current[len(current)-1].ID,
EstimatedTokens: currentTokens,
Segments: append([]schema.Segment(nil), current...),
})
}
return sections
}
func TestChunkEmptyTranscript(t *testing.T) {
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 10})
sections, err := chunker.ChunkTranscript(nil)
if err != nil {
t.Fatalf("ChunkTranscript(nil): %v", err)
}
if len(sections) != 0 {
t.Fatalf("expected 0 sections for nil transcript, got %d", len(sections))
}
sections, err = chunker.ChunkTranscript(makeTranscript(nil))
if err != nil {
t.Fatalf("ChunkTranscript(empty): %v", err)
}
if len(sections) != 0 {
t.Fatalf("expected 0 sections for empty transcript, got %d", len(sections))
}
}
func TestChunkSingleSegment(t *testing.T) {
segments := makeSegments([]string{"s1"})
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 10},
&mapTokenEstimator{byText: map[string]int{"s1": 7}},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 1 {
t.Fatalf("expected 1 section, got %d", len(sections))
}
if sections[0].EstimatedTokens != 7 {
t.Fatalf("expected estimated_tokens=7, got %d", sections[0].EstimatedTokens)
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertSectionMetadataConsistent(t, sections)
}
func TestChunkSingleOversizedSegment(t *testing.T) {
segments := makeSegments([]string{"big"})
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 5},
&mapTokenEstimator{byText: map[string]int{"big": 120}},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 1 {
t.Fatalf("expected 1 section, got %d", len(sections))
}
if sections[0].EstimatedTokens != 120 {
t.Fatalf("expected oversized singleton section, got %d", sections[0].EstimatedTokens)
}
assertMaxBoundExceptSingletonOversized(t, sections, 50)
}
func TestChunkTotalBelowMaxSingleSection(t *testing.T) {
segments := makeSegments([]string{"a", "b", "c"})
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 5},
&ConstTokenEstimator{Tokens: 10},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 1 {
t.Fatalf("expected 1 section, got %d", len(sections))
}
if sections[0].EstimatedTokens != 30 {
t.Fatalf("expected 30 section tokens, got %d", sections[0].EstimatedTokens)
}
assertSegmentCoverageAndOrder(t, segments, sections)
}
func TestChunkTotalExactlyDivisibleByMax(t *testing.T) {
segments := makeSegments([]string{"a", "b", "c", "d"})
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1},
&ConstTokenEstimator{Tokens: 5},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 2 {
t.Fatalf("expected 2 sections, got %d", len(sections))
}
if sections[0].EstimatedTokens != 10 || sections[1].EstimatedTokens != 10 {
t.Fatalf("expected [10,10] tokens, got [%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens)
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertMaxBoundExceptSingletonOversized(t, sections, 10)
}
func TestChunkTotalNotDivisibleByMax(t *testing.T) {
segments := makeSegments([]string{"a", "b", "c", "d", "e"})
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1},
&ConstTokenEstimator{Tokens: 5},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 3 {
t.Fatalf("expected 3 sections, got %d", len(sections))
}
if sections[0].EstimatedTokens != 10 || sections[1].EstimatedTokens != 10 || sections[2].EstimatedTokens != 5 {
t.Fatalf("expected [10,10,5] tokens, got [%d,%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens, sections[2].EstimatedTokens)
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertMaxBoundExceptSingletonOversized(t, sections, 10)
}
func TestChunkTargetSectionsPrecedenceAndSuccess(t *testing.T) {
segments := makeSegments([]string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10"})
target := 3
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 200, MinSectionTokens: 1, TargetSections: &target},
&ConstTokenEstimator{Tokens: 10},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != target {
t.Fatalf("expected %d sections from explicit target, got %d", target, len(sections))
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertSectionMetadataConsistent(t, sections)
}
func TestChunkTargetSectionsImpossibleTooMany(t *testing.T) {
segments := makeSegments([]string{"1", "2", "3", "4", "5"})
target := 10
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 1, TargetSections: &target})
_, err := chunker.ChunkTranscript(makeTranscript(segments))
if err == nil {
t.Fatal("expected error for impossible target_sections")
}
if !strings.Contains(err.Error(), "cannot have more sections than segments") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestChunkTargetSectionsImpossibleTooFew(t *testing.T) {
segments := makeSegments([]string{"1", "2", "3"})
target := 1
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1, TargetSections: &target},
&ConstTokenEstimator{Tokens: 30},
)
_, err := chunker.ChunkTranscript(makeTranscript(segments))
if err == nil {
t.Fatal("expected error for impossible target_sections")
}
if !strings.Contains(err.Error(), "need at least 3 sections") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestChunkManySmallSegmentsBalanced(t *testing.T) {
texts := []string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"}
segments := makeSegments(texts)
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1},
&ConstTokenEstimator{Tokens: 10},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 3 {
t.Fatalf("expected 3 sections, got %d", len(sections))
}
if sections[0].EstimatedTokens != 40 || sections[1].EstimatedTokens != 40 || sections[2].EstimatedTokens != 30 {
t.Fatalf("expected [40,40,30], got [%d,%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens, sections[2].EstimatedTokens)
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertMaxBoundExceptSingletonOversized(t, sections, 50)
}
func TestChunkMixedLargeAndSmallSegments(t *testing.T) {
segments := makeSegments([]string{"big1", "s1", "s2", "s3", "big2", "s4"})
estimator := &mapTokenEstimator{byText: map[string]int{
"big1": 120,
"s1": 10,
"s2": 10,
"s3": 10,
"big2": 120,
"s4": 10,
}}
chunker := NewChunkerWithEstimator(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 1}, estimator)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 4 {
t.Fatalf("expected 4 sections, got %d", len(sections))
}
if len(sections[0].Segments) != 1 || sections[0].Segments[0].Text != "big1" {
t.Fatalf("expected first oversized segment in singleton section, got %+v", sections[0].Segments)
}
if len(sections[2].Segments) != 1 || sections[2].Segments[0].Text != "big2" {
t.Fatalf("expected second oversized segment in singleton section, got %+v", sections[2].Segments)
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertMaxBoundExceptSingletonOversized(t, sections, 100)
}
func TestChunkDeterministicOrdering(t *testing.T) {
segments := makeSegments([]string{"a", "b", "c", "d", "e", "f"})
chunkerCfg := ChunkingConfig{MaxSectionTokens: 15, MinSectionTokens: 1}
estimator := &ConstTokenEstimator{Tokens: 5}
var first []Section
for i := 0; i < 5; i++ {
chunker := NewChunkerWithEstimator(chunkerCfg, estimator)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("iteration %d: %v", i, err)
}
if i == 0 {
first = sections
continue
}
if len(sections) != len(first) {
t.Fatalf("iteration %d: section count mismatch (%d vs %d)", i, len(sections), len(first))
}
for j := range sections {
if sections[j].Index != first[j].Index ||
sections[j].StartSegmentID != first[j].StartSegmentID ||
sections[j].EndSegmentID != first[j].EndSegmentID ||
sections[j].EstimatedTokens != first[j].EstimatedTokens ||
len(sections[j].Segments) != len(first[j].Segments) {
t.Fatalf("iteration %d section %d mismatch", i, j)
}
}
}
}
func TestChunkNoMutationOfInput(t *testing.T) {
segments := makeSegments([]string{"original one", "original two"})
transcript := makeTranscript(segments)
original := make([]string, len(transcript.Segments))
for i := range transcript.Segments {
original[i] = transcript.Segments[i].Text
}
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1})
if _, err := chunker.ChunkTranscript(transcript); err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
for i := range transcript.Segments {
if transcript.Segments[i].Text != original[i] {
t.Fatalf("segment %d mutated", i)
}
}
}
func TestChunkConfigValidation(t *testing.T) {
tests := []struct {
name string
config ChunkingConfig
errContains string
}{
{
name: "zero max tokens",
config: ChunkingConfig{MaxSectionTokens: 0, MinSectionTokens: 1},
errContains: "max_section_tokens must be positive",
},
{
name: "negative max tokens",
config: ChunkingConfig{MaxSectionTokens: -1, MinSectionTokens: 1},
errContains: "max_section_tokens must be positive",
},
{
name: "negative min tokens",
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: -1},
errContains: "min_section_tokens must be non-negative",
},
{
name: "min exceeds max",
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 11},
errContains: "min_section_tokens (11) cannot exceed max_section_tokens (10)",
},
{
name: "zero target sections",
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1, TargetSections: intPtr(0)},
errContains: "target_sections must be positive",
},
{
name: "negative target sections",
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1, TargetSections: intPtr(-1)},
errContains: "target_sections must be positive",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
chunker := NewChunker(tt.config)
_, err := chunker.ChunkTranscript(makeTranscript(makeSegments([]string{"x"})))
if err == nil {
t.Fatalf("expected error containing %q", tt.errContains)
}
if !strings.Contains(err.Error(), tt.errContains) {
t.Fatalf("expected error containing %q, got %q", tt.errContains, err.Error())
}
})
}
}
func TestChunkBalancedAlgorithmBeatsGreedyMaxFillOnUnevenTranscript(t *testing.T) {
segments := makeSegments([]string{"s1", "s2", "s3", "s4", "s5", "s6"})
tokenMap := map[string]int{
"s1": 50,
"s2": 10,
"s3": 10,
"s4": 10,
"s5": 10,
"s6": 10,
}
estimator := &mapTokenEstimator{byText: tokenMap}
chunker := NewChunkerWithEstimator(ChunkingConfig{MaxSectionTokens: 80, MinSectionTokens: 1}, estimator)
balancedSections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
tokens := make([]int, 0, len(segments))
for _, seg := range segments {
tokens = append(tokens, tokenMap[seg.Text])
}
greedySections := greedyMaxFillSections(segments, tokens, 80)
balancedImbalance := imbalance(balancedSections)
greedyImbalance := imbalance(greedySections)
if balancedImbalance >= greedyImbalance {
t.Fatalf(
"expected balanced chunking to improve over greedy max-fill; balanced=%d greedy=%d",
balancedImbalance,
greedyImbalance,
)
}
assertSegmentCoverageAndOrder(t, segments, balancedSections)
assertMaxBoundExceptSingletonOversized(t, balancedSections, 80)
}

View File

@@ -0,0 +1,84 @@
package chunking
// Summary provides a concise overview of chunking results for reports
type Summary struct {
ChunkCount int `json:"chunk_count"`
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
TargetSections *int `json:"target_sections,omitempty"`
MaxSectionTokens int `json:"max_section_tokens"`
MinSectionTokens int `json:"min_section_tokens"`
}
// ChunkSummary represents a single chunk's metadata for diagnostics
type ChunkSummary struct {
Index int `json:"index"`
StartSegmentID int `json:"start_segment_id"`
EndSegmentID int `json:"end_segment_id"`
EstimatedTokens int `json:"estimated_tokens"`
SegmentCount int `json:"segment_count"`
}
// DetailedSummary provides per-chunk details for diagnostics
type DetailedSummary struct {
Summary `json:",inline"`
Chunks []ChunkSummary `json:"chunks"`
}
// ComputeSummary creates a Summary from sections and config
func ComputeSummary(sections []Section, config ChunkingConfig) Summary {
if len(sections) == 0 {
return Summary{
ChunkCount: 0,
MaxSectionTokens: config.MaxSectionTokens,
MinSectionTokens: config.MinSectionTokens,
TargetSections: config.TargetSections,
}
}
minTokens := sections[0].EstimatedTokens
maxTokens := sections[0].EstimatedTokens
totalTokens := 0
for _, sec := range sections {
if sec.EstimatedTokens < minTokens {
minTokens = sec.EstimatedTokens
}
if sec.EstimatedTokens > maxTokens {
maxTokens = sec.EstimatedTokens
}
totalTokens += sec.EstimatedTokens
}
return Summary{
ChunkCount: len(sections),
MinEstimatedTokens: minTokens,
MaxEstimatedTokens: maxTokens,
TotalEstimatedTokens: totalTokens,
TargetSections: config.TargetSections,
MaxSectionTokens: config.MaxSectionTokens,
MinSectionTokens: config.MinSectionTokens,
}
}
// ComputeDetailedSummary creates a DetailedSummary from sections and config
func ComputeDetailedSummary(sections []Section, config ChunkingConfig) DetailedSummary {
summary := ComputeSummary(sections, config)
chunks := make([]ChunkSummary, len(sections))
for i, sec := range sections {
chunks[i] = ChunkSummary{
Index: sec.Index,
StartSegmentID: sec.StartSegmentID,
EndSegmentID: sec.EndSegmentID,
EstimatedTokens: sec.EstimatedTokens,
SegmentCount: len(sec.Segments),
}
}
return DetailedSummary{
Summary: summary,
Chunks: chunks,
}
}

View File

@@ -0,0 +1,148 @@
package chunking
import (
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
func TestComputeSummary(t *testing.T) {
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
}
sections := []Section{
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
{Index: 1, EstimatedTokens: 50, StartSegmentID: 3, EndSegmentID: 4},
{Index: 2, EstimatedTokens: 20, StartSegmentID: 5, EndSegmentID: 5},
}
summary := ComputeSummary(sections, config)
if summary.ChunkCount != 3 {
t.Errorf("expected chunk_count=3, got %d", summary.ChunkCount)
}
if summary.MinEstimatedTokens != 20 {
t.Errorf("expected min_estimated_tokens=20, got %d", summary.MinEstimatedTokens)
}
if summary.MaxEstimatedTokens != 50 {
t.Errorf("expected max_estimated_tokens=50, got %d", summary.MaxEstimatedTokens)
}
if summary.TotalEstimatedTokens != 100 {
t.Errorf("expected total_estimated_tokens=100, got %d", summary.TotalEstimatedTokens)
}
if summary.MaxSectionTokens != 100 {
t.Errorf("expected max_section_tokens=100, got %d", summary.MaxSectionTokens)
}
if summary.MinSectionTokens != 10 {
t.Errorf("expected min_section_tokens=10, got %d", summary.MinSectionTokens)
}
if summary.TargetSections != nil {
t.Errorf("expected target_sections=nil, got %v", summary.TargetSections)
}
}
func TestComputeSummaryWithTarget(t *testing.T) {
target := 5
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
TargetSections: &target,
}
sections := []Section{
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
}
summary := ComputeSummary(sections, config)
if summary.TargetSections == nil || *summary.TargetSections != 5 {
t.Errorf("expected target_sections=5, got %v", summary.TargetSections)
}
}
func TestComputeSummaryEmptySections(t *testing.T) {
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
}
sections := []Section{}
summary := ComputeSummary(sections, config)
if summary.ChunkCount != 0 {
t.Errorf("expected chunk_count=0, got %d", summary.ChunkCount)
}
if summary.MinEstimatedTokens != 0 {
t.Errorf("expected min_estimated_tokens=0 for empty, got %d", summary.MinEstimatedTokens)
}
if summary.MaxSectionTokens != 100 {
t.Errorf("expected max_section_tokens preserved, got %d", summary.MaxSectionTokens)
}
}
func TestComputeDetailedSummary(t *testing.T) {
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
}
sections := []Section{
{
Index: 0,
EstimatedTokens: 30,
StartSegmentID: 1,
EndSegmentID: 2,
Segments: make([]schema.Segment, 2), // 2 segments
},
{
Index: 1,
EstimatedTokens: 50,
StartSegmentID: 3,
EndSegmentID: 5,
Segments: make([]schema.Segment, 3), // 3 segments
},
}
detailed := ComputeDetailedSummary(sections, config)
if detailed.ChunkCount != 2 {
t.Errorf("expected chunk_count=2, got %d", detailed.ChunkCount)
}
if len(detailed.Chunks) != 2 {
t.Fatalf("expected 2 chunk entries, got %d", len(detailed.Chunks))
}
// Check first chunk
if detailed.Chunks[0].Index != 0 {
t.Errorf("expected chunk[0].index=0, got %d", detailed.Chunks[0].Index)
}
if detailed.Chunks[0].StartSegmentID != 1 {
t.Errorf("expected chunk[0].start_segment_id=1, got %d", detailed.Chunks[0].StartSegmentID)
}
if detailed.Chunks[0].EndSegmentID != 2 {
t.Errorf("expected chunk[0].end_segment_id=2, got %d", detailed.Chunks[0].EndSegmentID)
}
if detailed.Chunks[0].EstimatedTokens != 30 {
t.Errorf("expected chunk[0].estimated_tokens=30, got %d", detailed.Chunks[0].EstimatedTokens)
}
if detailed.Chunks[0].SegmentCount != 2 {
t.Errorf("expected chunk[0].segment_count=2, got %d", detailed.Chunks[0].SegmentCount)
}
// Check second chunk
if detailed.Chunks[1].Index != 1 {
t.Errorf("expected chunk[1].index=1, got %d", detailed.Chunks[1].Index)
}
if detailed.Chunks[1].StartSegmentID != 3 {
t.Errorf("expected chunk[1].start_segment_id=3, got %d", detailed.Chunks[1].StartSegmentID)
}
if detailed.Chunks[1].EndSegmentID != 5 {
t.Errorf("expected chunk[1].end_segment_id=5, got %d", detailed.Chunks[1].EndSegmentID)
}
if detailed.Chunks[1].SegmentCount != 3 {
t.Errorf("expected chunk[1].segment_count=3, got %d", detailed.Chunks[1].SegmentCount)
}
}

View File

@@ -0,0 +1,55 @@
package chunking
import (
"strings"
"unicode"
)
// TokenEstimator provides a deterministic token estimation suitable for prompt budgeting.
// The estimator is approximate but stable, isolated, and replaceable.
type TokenEstimator interface {
EstimateTokens(text string) int
}
// SimpleTokenEstimator provides a basic deterministic token estimation.
// This uses a simple heuristic based on word count and punctuation.
type SimpleTokenEstimator struct{}
// NewSimpleTokenEstimator creates a new simple token estimator.
func NewSimpleTokenEstimator() *SimpleTokenEstimator {
return &SimpleTokenEstimator{}
}
// EstimateTokens provides a rough estimate of the number of tokens in the given text.
// This implementation uses a simple heuristic: count words and punctuation as tokens.
// The estimate is deterministic and stable for the same input text.
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
if text == "" {
return 0
}
// Simple heuristic: split on whitespace and count non-empty segments
words := strings.Fields(text)
tokenCount := len(words)
// Add some estimate for punctuation that might be separate tokens
punctuationCount := 0
for _, r := range text {
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
punctuationCount++
}
}
// Rough estimate: each word is a token, plus half the punctuation as separate tokens
return tokenCount + (punctuationCount / 2)
}
// ConstTokenEstimator returns a constant token count for testing purposes.
type ConstTokenEstimator struct {
Tokens int
}
// EstimateTokens returns the configured constant token count.
func (e *ConstTokenEstimator) EstimateTokens(text string) int {
return e.Tokens
}

View File

@@ -0,0 +1,59 @@
package chunking
import (
"testing"
)
func TestSimpleTokenEstimator(t *testing.T) {
estimator := NewSimpleTokenEstimator()
tests := []struct {
name string
text string
expected int
}{
{"empty string", "", 0},
{"single word", "hello", 1},
{"two words", "hello world", 2},
{"with punctuation", "hello, world!", 3}, // 2 words + 2 punctuation/2 = 3
{"multiple sentences", "Hello world. This is a test.", 7}, // 7 words + 2 punctuation/2 = 8? Actually "Hello world." has 3 punctuation
{"with apostrophes", "don't won't can't", 3},
{"with hyphens", "well-known state-of-the-art", 2}, // hyphens don't count
{"unicode text", "café naïve", 2},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := estimator.EstimateTokens(tt.text)
if got != tt.expected {
t.Errorf("EstimateTokens(%q) = %d, want %d", tt.text, got, tt.expected)
}
})
}
}
func TestConstTokenEstimator(t *testing.T) {
estimator := &ConstTokenEstimator{Tokens: 42}
if got := estimator.EstimateTokens("any text"); got != 42 {
t.Errorf("ConstTokenEstimator.EstimateTokens = %d, want 42", got)
}
if got := estimator.EstimateTokens(""); got != 42 {
t.Errorf("ConstTokenEstimator.EstimateTokens(empty) = %d, want 42", got)
}
}
func TestTokenEstimatorDeterminism(t *testing.T) {
estimator := NewSimpleTokenEstimator()
text := "The quick brown fox jumps over the lazy dog. Hello, world!"
// Run multiple times and verify same result
first := estimator.EstimateTokens(text)
for i := 0; i < 10; i++ {
got := estimator.EstimateTokens(text)
if got != first {
t.Errorf("EstimateTokens not deterministic: iteration %d got %d, first was %d", i, got, first)
}
}
}

View File

@@ -0,0 +1,171 @@
package config
import "strings"
type llmTargetPatch struct {
apiKey *string
model *string
baseURL *string
timeoutSeconds *int
maxRetries *int
}
type concurrencyPatch struct {
totalLLM *int
legacyTotalLLM *int
proposalLLM *int
validationLLM *int
inheritProposal bool
allowLegacyAlias bool
}
type chunkingPatch struct {
targetSections *int
maxSectionTokens *int
minSectionTokens *int
}
type thresholdsPatch struct {
glossary *float64
grammar *float64
homophones *float64
spokenWord *float64
}
type normalizationPatch struct {
maxSegmentGap *float64
ellipsisGap *float64
maxSegmentDuration *float64
maxSegmentTokens *int
}
type contextPatch struct {
transcriptDescription *string
}
type diagnosticsPatch struct {
workDir *string
workDirRetention *string
}
func (c *Config) applyPrimaryLLMTargetPatch(patch llmTargetPatch) {
if patch.apiKey != nil {
c.PrimaryLLM.APIKey = *patch.apiKey
}
if patch.model != nil {
c.PrimaryLLM.Model = *patch.model
}
if patch.baseURL != nil {
c.PrimaryLLM.BaseURL = *patch.baseURL
}
if patch.timeoutSeconds != nil {
c.PrimaryLLM.TimeoutSeconds = *patch.timeoutSeconds
}
if patch.maxRetries != nil {
c.PrimaryLLM.MaxRetries = *patch.maxRetries
}
}
func (c *Config) applyValidationLLMTargetPatch(patch llmTargetPatch) {
if patch.apiKey != nil {
c.ValidationLLM.APIKey = *patch.apiKey
}
if patch.model != nil {
c.ValidationLLM.Model = *patch.model
}
if patch.baseURL != nil {
c.ValidationLLM.BaseURL = *patch.baseURL
}
if patch.timeoutSeconds != nil {
value := *patch.timeoutSeconds
c.ValidationLLM.TimeoutSeconds = &value
}
if patch.maxRetries != nil {
value := *patch.maxRetries
c.ValidationLLM.MaxRetries = &value
}
}
func (c *Config) applyConcurrencyPatch(patch concurrencyPatch) {
totalSet := false
if patch.totalLLM != nil {
c.TotalLLMConcurrency = *patch.totalLLM
totalSet = true
}
if patch.allowLegacyAlias && patch.legacyTotalLLM != nil && !totalSet {
c.TotalLLMConcurrency = *patch.legacyTotalLLM
totalSet = true
}
proposalSet := false
if patch.proposalLLM != nil {
c.ProposalLLMConcurrency = *patch.proposalLLM
proposalSet = true
}
if patch.inheritProposal && totalSet && !proposalSet {
c.ProposalLLMConcurrency = c.TotalLLMConcurrency
}
if patch.validationLLM != nil {
value := *patch.validationLLM
c.ValidationLLMConcurrency = &value
}
}
func (c *Config) applyChunkingPatch(patch chunkingPatch) {
if patch.targetSections != nil {
value := *patch.targetSections
c.TargetSections = &value
}
if patch.maxSectionTokens != nil {
c.MaxSectionTokens = *patch.maxSectionTokens
}
if patch.minSectionTokens != nil {
c.MinSectionTokens = *patch.minSectionTokens
}
}
func (c *Config) applyThresholdsPatch(patch thresholdsPatch) {
if patch.glossary != nil {
c.Thresholds.Glossary = *patch.glossary
}
if patch.grammar != nil {
c.Thresholds.Grammar = *patch.grammar
}
if patch.homophones != nil {
c.Thresholds.Homophones = *patch.homophones
}
if patch.spokenWord != nil {
c.Thresholds.SpokenWord = *patch.spokenWord
}
}
func (c *Config) applyNormalizationPatch(patch normalizationPatch) {
if patch.maxSegmentGap != nil {
c.Normalization.MaxSegmentGap = *patch.maxSegmentGap
}
if patch.ellipsisGap != nil {
c.Normalization.EllipsisGap = *patch.ellipsisGap
}
if patch.maxSegmentDuration != nil {
c.Normalization.MaxSegmentDuration = *patch.maxSegmentDuration
}
if patch.maxSegmentTokens != nil {
c.Normalization.MaxSegmentTokens = *patch.maxSegmentTokens
}
}
func (c *Config) applyContextPatch(patch contextPatch) {
if patch.transcriptDescription != nil {
c.TranscriptDescription = strings.TrimSpace(*patch.transcriptDescription)
}
}
func (c *Config) applyDiagnosticsPatch(patch diagnosticsPatch) {
if patch.workDir != nil {
c.WorkDir = *patch.workDir
}
if patch.workDirRetention != nil {
c.WorkDirRetention = WorkDirRetention(*patch.workDirRetention)
}
}

View File

@@ -0,0 +1,199 @@
package config
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
)
type WorkDirRetention string
const (
WorkDirRetentionAuto WorkDirRetention = "auto"
WorkDirRetentionAlways WorkDirRetention = "always"
WorkDirRetentionNever WorkDirRetention = "never"
)
const (
DefaultModulesCSV = modulecatalog.KeyGlossary + "," + modulecatalog.KeyHomophones + "," + modulecatalog.KeyGlossary + "," + modulecatalog.KeySpokenWord + "," + modulecatalog.KeyGrammar
DefaultOutputSchema = "bare-segments"
DefaultPrimaryModel = "openrouter/google/gemma-4-31b-it"
DefaultPrimaryBaseURL = "https://openrouter.ai/api/v1"
DefaultPrimaryLLMTimeoutSeconds = 600
DefaultMaxRetries = 3
DefaultLLMConcurrency = 1
DefaultValidationMaxPromptTokens = 2048
DefaultMaxSectionTokens = 8192
DefaultMinSectionTokens = 2048
DefaultConfidenceThreshold = 0.8
DefaultNormalizeMaxSegmentGap = 4.0
DefaultNormalizeEllipsisGap = 3.5
DefaultNormalizeMaxSegmentDuration = 60.0
DefaultNormalizeMaxSegmentTokens = 2048
DefaultTranscriptDescriptionMaxChars = 500
DefaultWorkDir = "/tmp/audita"
DefaultWorkDirRetention WorkDirRetention = WorkDirRetentionAuto
)
type Config struct {
Modules []string
OutputSchema string
PrimaryLLM LLMConfig
ValidationLLM ValidationLLMConfig
TotalLLMConcurrency int
ProposalLLMConcurrency int
ValidationLLMConcurrency *int
ValidationMaxPromptTokens int
MaxSectionTokens int
MinSectionTokens int
TargetSections *int
Thresholds ConfidenceThresholds
Normalization NormalizationConfig
TranscriptDescription string
WorkDir string
WorkDirRetention WorkDirRetention
}
type LLMConfig struct {
APIKey string
Model string
BaseURL string
TimeoutSeconds int
MaxRetries int
// Concurrency is retained as a backward-compatible alias for
// TotalLLMConcurrency.
Concurrency int
}
type ValidationLLMConfig struct {
APIKey string
Model string
BaseURL string
TimeoutSeconds *int
MaxRetries *int
// Concurrency is retained as a backward-compatible alias for
// ValidationLLMConcurrency.
Concurrency *int
}
type ConfidenceThresholds struct {
Glossary float64
Grammar float64
Homophones float64
SpokenWord float64
}
type NormalizationConfig struct {
MaxSegmentGap float64
EllipsisGap float64
MaxSegmentDuration float64
MaxSegmentTokens int
}
func Default() Config {
modules, _ := ParseModulesCSV(DefaultModulesCSV)
return Config{
Modules: modules,
OutputSchema: DefaultOutputSchema,
PrimaryLLM: LLMConfig{
Model: DefaultPrimaryModel,
BaseURL: DefaultPrimaryBaseURL,
TimeoutSeconds: DefaultPrimaryLLMTimeoutSeconds,
MaxRetries: DefaultMaxRetries,
Concurrency: DefaultLLMConcurrency,
},
ValidationLLM: ValidationLLMConfig{},
TotalLLMConcurrency: DefaultLLMConcurrency,
ProposalLLMConcurrency: DefaultLLMConcurrency,
ValidationLLMConcurrency: nil,
ValidationMaxPromptTokens: DefaultValidationMaxPromptTokens,
MaxSectionTokens: DefaultMaxSectionTokens,
MinSectionTokens: DefaultMinSectionTokens,
TargetSections: nil,
Thresholds: ConfidenceThresholds{
Glossary: DefaultConfidenceThreshold,
Grammar: DefaultConfidenceThreshold,
Homophones: DefaultConfidenceThreshold,
SpokenWord: DefaultConfidenceThreshold,
},
Normalization: NormalizationConfig{
MaxSegmentGap: DefaultNormalizeMaxSegmentGap,
EllipsisGap: DefaultNormalizeEllipsisGap,
MaxSegmentDuration: DefaultNormalizeMaxSegmentDuration,
MaxSegmentTokens: DefaultNormalizeMaxSegmentTokens,
},
WorkDir: DefaultWorkDir,
WorkDirRetention: DefaultWorkDirRetention,
}
}
func ParseModulesCSV(raw string) ([]string, error) {
parts := strings.Split(raw, ",")
modules := make([]string, 0, len(parts))
for _, part := range parts {
trimmed := strings.TrimSpace(part)
if trimmed == "" {
return nil, fmt.Errorf("modules list contains an empty value")
}
modules = append(modules, trimmed)
}
if len(modules) == 0 {
return nil, fmt.Errorf("modules list must not be empty")
}
return modules, nil
}
func (c Config) EffectiveValidationLLMConfig() LLMConfig {
effective := c.PrimaryLLM
if c.ValidationLLM.APIKey != "" {
effective.APIKey = c.ValidationLLM.APIKey
}
if c.ValidationLLM.Model != "" {
effective.Model = c.ValidationLLM.Model
}
if c.ValidationLLM.BaseURL != "" {
effective.BaseURL = c.ValidationLLM.BaseURL
}
if c.ValidationLLM.TimeoutSeconds != nil {
effective.TimeoutSeconds = *c.ValidationLLM.TimeoutSeconds
}
if c.ValidationLLM.MaxRetries != nil {
effective.MaxRetries = *c.ValidationLLM.MaxRetries
}
effective.Concurrency = c.EffectiveValidationLLMConcurrency()
return effective
}
func (c Config) EffectiveValidationLLMConcurrency() int {
if c.ValidationLLMConcurrency != nil {
return *c.ValidationLLMConcurrency
}
return c.TotalLLMConcurrency
}
func (c Config) EffectiveProposalLLMConcurrency() int {
if c.ProposalLLMConcurrency > 0 {
return c.ProposalLLMConcurrency
}
return c.TotalLLMConcurrency
}
func (c *Config) syncLegacyConcurrencyAliases() {
if c == nil {
return
}
c.PrimaryLLM.Concurrency = c.TotalLLMConcurrency
c.ValidationLLM.Concurrency = intPtr(c.ValidationLLMConcurrency)
}
func intPtr(v *int) *int {
if v == nil {
return nil
}
x := *v
return &x
}

View File

@@ -0,0 +1,705 @@
package config
import (
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
)
func TestDefaultConfigValues(t *testing.T) {
cfg := Default()
if got, want := strings.Join(cfg.Modules, ","), DefaultModulesCSV; got != want {
t.Fatalf("modules mismatch: got %q want %q", got, want)
}
if cfg.OutputSchema != DefaultOutputSchema {
t.Fatalf("unexpected default output schema: %q", cfg.OutputSchema)
}
if cfg.PrimaryLLM.Model != DefaultPrimaryModel {
t.Fatalf("unexpected default primary model: %q", cfg.PrimaryLLM.Model)
}
if cfg.PrimaryLLM.BaseURL != DefaultPrimaryBaseURL {
t.Fatalf("unexpected default primary base url: %q", cfg.PrimaryLLM.BaseURL)
}
if cfg.PrimaryLLM.TimeoutSeconds != DefaultPrimaryLLMTimeoutSeconds {
t.Fatalf("unexpected default timeout seconds: %d", cfg.PrimaryLLM.TimeoutSeconds)
}
if cfg.PrimaryLLM.MaxRetries != DefaultMaxRetries {
t.Fatalf("unexpected default max retries: %d", cfg.PrimaryLLM.MaxRetries)
}
if cfg.TotalLLMConcurrency != DefaultLLMConcurrency {
t.Fatalf("unexpected default total llm concurrency: %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != DefaultLLMConcurrency {
t.Fatalf("unexpected default proposal llm concurrency: %d", cfg.ProposalLLMConcurrency)
}
if cfg.ValidationLLMConcurrency != nil {
t.Fatalf("expected validation llm concurrency to be unset by default")
}
if cfg.PrimaryLLM.Concurrency != cfg.TotalLLMConcurrency {
t.Fatalf("expected primary llm concurrency alias to mirror total, got primary=%d total=%d", cfg.PrimaryLLM.Concurrency, cfg.TotalLLMConcurrency)
}
if cfg.ValidationLLM.TimeoutSeconds != nil {
t.Fatalf("expected validation timeout to be unset by default")
}
if cfg.ValidationLLM.MaxRetries != nil {
t.Fatalf("expected validation max retries to be unset by default")
}
if cfg.ValidationLLM.Concurrency != nil {
t.Fatalf("expected legacy validation llm concurrency alias to be unset by default")
}
if cfg.TargetSections != nil {
t.Fatalf("expected target sections to be unset by default")
}
if cfg.WorkDir != DefaultWorkDir {
t.Fatalf("unexpected default work dir: %q", cfg.WorkDir)
}
if cfg.TranscriptDescription != "" {
t.Fatalf("expected default transcript description to be empty, got %q", cfg.TranscriptDescription)
}
if cfg.WorkDirRetention != DefaultWorkDirRetention {
t.Fatalf("unexpected default work dir retention: %q", cfg.WorkDirRetention)
}
if err := cfg.Validate(); err != nil {
t.Fatalf("default config should validate: %v", err)
}
}
func TestLoadFromEnvOverridesAndFallback(t *testing.T) {
env := map[string]string{
"AUDITA_MODEL": "openai/gpt-4.1-mini",
"AUDITA_BASE_URL": "https://api.openai.com/v1",
"AUDITA_LLM_TIMEOUT_SECONDS": "120",
"AUDITA_MAX_RETRIES": "7",
"AUDITA_TOTAL_LLM_CONCURRENCY": "6",
"AUDITA_PROPOSAL_LLM_CONCURRENCY": "4",
"AUDITA_VALIDATION_LLM_CONCURRENCY": "2",
"AUDITA_VALIDATION_MAX_PROMPT_TOKENS": "4096",
"AUDITA_MAX_SECTION_TOKENS": "9000",
"AUDITA_MIN_SECTION_TOKENS": "3000",
"AUDITA_TARGET_SECTIONS": "5",
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.9",
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.7",
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.6",
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.5",
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "2.5",
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "2.0",
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "30.0",
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "1024",
"AUDITA_WORK_DIR": "/var/tmp/audita",
"AUDITA_WORK_DIR_RETENTION": "always",
"OPENROUTER_API_KEY": "fallback-key",
}
cfg, err := loadFromLookup(mapLookup(env))
if err != nil {
t.Fatalf("loadFromLookup returned error: %v", err)
}
if cfg.PrimaryLLM.APIKey != "fallback-key" {
t.Fatalf("expected OPENROUTER_API_KEY fallback, got %q", cfg.PrimaryLLM.APIKey)
}
if cfg.PrimaryLLM.Model != env["AUDITA_MODEL"] {
t.Fatalf("unexpected model: %q", cfg.PrimaryLLM.Model)
}
if cfg.PrimaryLLM.BaseURL != env["AUDITA_BASE_URL"] {
t.Fatalf("unexpected base url: %q", cfg.PrimaryLLM.BaseURL)
}
if cfg.TargetSections == nil || *cfg.TargetSections != 5 {
t.Fatalf("unexpected target sections: %#v", cfg.TargetSections)
}
if cfg.TotalLLMConcurrency != 6 {
t.Fatalf("unexpected total llm concurrency: %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != 4 {
t.Fatalf("unexpected proposal llm concurrency: %d", cfg.ProposalLLMConcurrency)
}
if cfg.ValidationLLMConcurrency == nil || *cfg.ValidationLLMConcurrency != 2 {
t.Fatalf("unexpected validation llm concurrency: %#v", cfg.ValidationLLMConcurrency)
}
if cfg.PrimaryLLM.Concurrency != 6 {
t.Fatalf("expected primary alias concurrency 6, got %d", cfg.PrimaryLLM.Concurrency)
}
if cfg.ValidationLLM.Concurrency == nil || *cfg.ValidationLLM.Concurrency != 2 {
t.Fatalf("expected validation alias concurrency 2, got %#v", cfg.ValidationLLM.Concurrency)
}
if cfg.WorkDirRetention != WorkDirRetentionAlways {
t.Fatalf("unexpected work dir retention: %q", cfg.WorkDirRetention)
}
}
func TestLoadFromEnvLegacyLLMConcurrencyAliasForTotalAndProposal(t *testing.T) {
env := map[string]string{
"AUDITA_LLM_CONCURRENCY": "5",
}
cfg, err := loadFromLookup(mapLookup(env))
if err != nil {
t.Fatalf("loadFromLookup returned error: %v", err)
}
if cfg.TotalLLMConcurrency != 5 {
t.Fatalf("expected total concurrency from legacy alias, got %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != 5 {
t.Fatalf("expected proposal concurrency to inherit legacy total, got %d", cfg.ProposalLLMConcurrency)
}
}
func TestLoadFromEnvCanonicalTotalWinsLegacyAlias(t *testing.T) {
env := map[string]string{
"AUDITA_TOTAL_LLM_CONCURRENCY": "4",
"AUDITA_LLM_CONCURRENCY": "9",
}
cfg, err := loadFromLookup(mapLookup(env))
if err != nil {
t.Fatalf("loadFromLookup returned error: %v", err)
}
if cfg.TotalLLMConcurrency != 4 {
t.Fatalf("expected canonical total to win over legacy alias, got %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != 4 {
t.Fatalf("expected proposal to inherit canonical total when unset, got %d", cfg.ProposalLLMConcurrency)
}
}
func TestLoadFromEnvUsesAuditaLLMAPIKeyOverFallback(t *testing.T) {
env := map[string]string{
"AUDITA_LLM_API_KEY": "primary-key",
"OPENROUTER_API_KEY": "fallback-key",
}
cfg, err := loadFromLookup(mapLookup(env))
if err != nil {
t.Fatalf("loadFromLookup returned error: %v", err)
}
if cfg.PrimaryLLM.APIKey != "primary-key" {
t.Fatalf("expected AUDITA_LLM_API_KEY to win, got %q", cfg.PrimaryLLM.APIKey)
}
}
func TestApplyCLIOverridesPrecedence(t *testing.T) {
cfg := Default()
cfg.PrimaryLLM.Model = "env-model"
cfg.WorkDir = "/env/work"
model := "cli-model"
workDir := "/cli/work"
modules := "grammar"
outputSchema := "audita-v1"
totalLLMConcurrency := 5
proposalLLMConcurrency := 3
overrides := CLIOverrides{
PrimaryModel: &model,
WorkDir: &workDir,
ModulesCSV: &modules,
OutputSchema: &outputSchema,
TotalLLMConcurrency: &totalLLMConcurrency,
ProposalLLMConcurrency: &proposalLLMConcurrency,
}
if err := cfg.ApplyCLIOverrides(overrides); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.PrimaryLLM.Model != "cli-model" {
t.Fatalf("expected CLI model override, got %q", cfg.PrimaryLLM.Model)
}
if cfg.WorkDir != "/cli/work" {
t.Fatalf("expected CLI work dir override, got %q", cfg.WorkDir)
}
if !reflect.DeepEqual(cfg.Modules, []string{"grammar"}) {
t.Fatalf("unexpected modules: %#v", cfg.Modules)
}
if cfg.OutputSchema != "audita-v1" {
t.Fatalf("expected CLI output schema override, got %q", cfg.OutputSchema)
}
if cfg.TotalLLMConcurrency != 5 {
t.Fatalf("expected CLI total concurrency override, got %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != 3 {
t.Fatalf("expected CLI proposal concurrency override, got %d", cfg.ProposalLLMConcurrency)
}
}
func TestApplyCLIOverridesTrimsTranscriptDescription(t *testing.T) {
cfg := Default()
description := " background context about speakers "
if err := cfg.ApplyCLIOverrides(CLIOverrides{TranscriptDescription: &description}); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.TranscriptDescription != "background context about speakers" {
t.Fatalf("unexpected transcript description trim result: %q", cfg.TranscriptDescription)
}
}
func TestConfigSourcesApplySharedEffectiveFieldsConsistently(t *testing.T) {
fileCfg := mustParseFileConfigYAML(t, `
version: 1
output:
schema: " audita-v1 "
llm:
proposal:
base_url: https://proposal.example.test/v1
model: provider/proposal
timeout: 101
max_retries: 5
validation:
base_url: https://validation.example.test/v1
model: provider/validation
timeout: 202
max_retries: 6
chunking:
target_sections: 7
max_section_tokens: 9000
min_section_tokens: 1000
thresholds:
glossary: 0.91
grammar: 0.92
homophones: 0.93
spoken_word: 0.94
normalization:
max_segment_gap: 1.2
ellipsis_gap: 2.3
max_segment_duration: 45.6
max_segment_tokens: 321
context:
description: " shared context "
diagnostics:
work_dir: /tmp/audita-shared
retention: always
`)
tests := []struct {
name string
apply func(*Config) error
}{
{
name: "file",
apply: func(cfg *Config) error {
return cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{}))
},
},
{
name: "env",
apply: func(cfg *Config) error {
return cfg.applyEnvOverrides(mapLookup(map[string]string{
"AUDITA_MODEL": "provider/proposal",
"AUDITA_BASE_URL": "https://proposal.example.test/v1",
"AUDITA_LLM_TIMEOUT_SECONDS": "101",
"AUDITA_MAX_RETRIES": "5",
"AUDITA_VALIDATION_MODEL": "provider/validation",
"AUDITA_VALIDATION_BASE_URL": "https://validation.example.test/v1",
"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "202",
"AUDITA_VALIDATION_MAX_RETRIES": "6",
"AUDITA_TARGET_SECTIONS": "7",
"AUDITA_MAX_SECTION_TOKENS": "9000",
"AUDITA_MIN_SECTION_TOKENS": "1000",
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.91",
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.92",
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.93",
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.94",
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "1.2",
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "2.3",
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.6",
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "321",
"AUDITA_WORK_DIR": "/tmp/audita-shared",
"AUDITA_WORK_DIR_RETENTION": "always",
}))
},
},
{
name: "cli",
apply: func(cfg *Config) error {
outputSchema := " audita-v1 "
proposalModel := "provider/proposal"
proposalBaseURL := "https://proposal.example.test/v1"
proposalTimeout := 101
proposalMaxRetries := 5
validationModel := "provider/validation"
validationBaseURL := "https://validation.example.test/v1"
validationTimeout := 202
validationMaxRetries := 6
targetSections := 7
maxSectionTokens := 9000
minSectionTokens := 1000
glossaryThreshold := 0.91
grammarThreshold := 0.92
homophonesThreshold := 0.93
spokenWordThreshold := 0.94
normalizeMaxSegmentGap := 1.2
normalizeEllipsisGap := 2.3
normalizeMaxSegmentDuration := 45.6
normalizeMaxSegmentTokens := 321
description := " shared context "
workDir := "/tmp/audita-shared"
workDirRetention := "always"
return cfg.ApplyCLIOverrides(CLIOverrides{
OutputSchema: &outputSchema,
PrimaryModel: &proposalModel,
PrimaryBaseURL: &proposalBaseURL,
PrimaryLLMTimeoutSeconds: &proposalTimeout,
MaxRetries: &proposalMaxRetries,
ValidationModel: &validationModel,
ValidationBaseURL: &validationBaseURL,
ValidationLLMTimeoutSeconds: &validationTimeout,
ValidationMaxRetries: &validationMaxRetries,
TargetSections: &targetSections,
MaxSectionTokens: &maxSectionTokens,
MinSectionTokens: &minSectionTokens,
GlossaryConfidenceThreshold: &glossaryThreshold,
GrammarConfidenceThreshold: &grammarThreshold,
HomophonesConfidenceThreshold: &homophonesThreshold,
SpokenWordConfidenceThreshold: &spokenWordThreshold,
NormalizeMaxSegmentGap: &normalizeMaxSegmentGap,
NormalizeEllipsisGap: &normalizeEllipsisGap,
NormalizeMaxSegmentDuration: &normalizeMaxSegmentDuration,
NormalizeMaxSegmentTokens: &normalizeMaxSegmentTokens,
TranscriptDescription: &description,
WorkDir: &workDir,
WorkDirRetention: &workDirRetention,
})
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
cfg := Default()
if err := tc.apply(&cfg); err != nil {
t.Fatalf("apply config source: %v", err)
}
assertSharedEffectiveFields(t, cfg, sharedEffectiveFieldOptions{
wantOutputSchemaOverride: tc.name != "env",
wantTranscriptDescriptionPatch: tc.name != "env",
})
})
}
}
func TestCLIAPIKeyOverrideIsDirectValue(t *testing.T) {
cfg := Default()
apiKey := "NOT_AN_ENV_VAR_NAME"
validationAPIKey := "also direct"
if err := cfg.ApplyCLIOverrides(CLIOverrides{PrimaryLLMAPIKey: &apiKey, ValidationLLMAPIKey: &validationAPIKey}); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.PrimaryLLM.APIKey != apiKey {
t.Fatalf("expected direct primary api key, got %q", cfg.PrimaryLLM.APIKey)
}
if cfg.ValidationLLM.APIKey != validationAPIKey {
t.Fatalf("expected direct validation api key, got %q", cfg.ValidationLLM.APIKey)
}
}
func TestApplyFileConfigTotalConcurrencyDoesNotChangeProposalWhenProposalUnset(t *testing.T) {
fileCfg := mustParseFileConfigYAML(t, `
version: 1
concurrency:
total_llm: 4
`)
cfg := Default()
if err := cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{})); err != nil {
t.Fatalf("applyFileConfigWithLookup failed: %v", err)
}
if cfg.TotalLLMConcurrency != 4 {
t.Fatalf("expected file total concurrency 4, got %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != DefaultLLMConcurrency {
t.Fatalf("expected file config to preserve proposal concurrency when unset, got %d", cfg.ProposalLLMConcurrency)
}
}
func TestValidationRejectsOverlyLongTranscriptDescription(t *testing.T) {
cfg := Default()
cfg.TranscriptDescription = strings.Repeat("a", DefaultTranscriptDescriptionMaxChars+1)
err := cfg.Validate()
if err == nil {
t.Fatalf("expected transcript description length validation error")
}
if !strings.Contains(err.Error(), "transcript description must be 500 characters or fewer") {
t.Fatalf("unexpected validation error: %v", err)
}
}
func TestApplyCLIOverridesLegacyLLMConcurrencyAlias(t *testing.T) {
cfg := Default()
aliasConcurrency := 6
if err := cfg.ApplyCLIOverrides(CLIOverrides{PrimaryLLMConcurrency: &aliasConcurrency}); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.TotalLLMConcurrency != 6 {
t.Fatalf("expected legacy --llm-concurrency alias to set total, got %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != 6 {
t.Fatalf("expected proposal to inherit aliased total when unset, got %d", cfg.ProposalLLMConcurrency)
}
}
func TestApplyCLIOverridesCanonicalTotalWinsLegacyAlias(t *testing.T) {
cfg := Default()
canonicalTotal := 4
legacyAlias := 9
if err := cfg.ApplyCLIOverrides(CLIOverrides{TotalLLMConcurrency: &canonicalTotal, PrimaryLLMConcurrency: &legacyAlias}); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.TotalLLMConcurrency != 4 {
t.Fatalf("expected canonical total concurrency to win, got %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != 4 {
t.Fatalf("expected proposal to inherit canonical total when proposal is unset, got %d", cfg.ProposalLLMConcurrency)
}
}
func TestValidationFailures(t *testing.T) {
cfg := Default()
cfg.OutputSchema = "unknown-schema"
cfg.PrimaryLLM.TimeoutSeconds = -1
cfg.TotalLLMConcurrency = 0
cfg.ProposalLLMConcurrency = 0
validationConcurrency := 5
cfg.ValidationLLMConcurrency = &validationConcurrency
cfg.ValidationMaxPromptTokens = 0
cfg.MaxSectionTokens = 100
cfg.MinSectionTokens = 200
cfg.Thresholds.Grammar = 1.5
cfg.WorkDirRetention = WorkDirRetention("sometimes")
err := cfg.Validate()
if err == nil {
t.Fatalf("expected validation error")
}
message := err.Error()
for _, expected := range []string{
"primary llm timeout seconds",
"total llm concurrency",
"proposal llm concurrency",
"validation llm concurrency must be less than or equal to total llm concurrency",
"validation max prompt tokens",
"min section tokens",
"grammar confidence threshold",
"work dir retention",
"unsupported output schema",
} {
if !strings.Contains(message, expected) {
t.Fatalf("expected error to contain %q, got %q", expected, message)
}
}
}
func TestValidationRejectsUnsupportedModuleKey(t *testing.T) {
cfg := Default()
cfg.Modules = []string{modulecatalog.KeyGlossary, "made_up"}
err := cfg.Validate()
if err == nil {
t.Fatalf("expected validation error for unsupported module key")
}
if !strings.Contains(err.Error(), `unsupported module key "made_up"`) {
t.Fatalf("expected unsupported module key error, got %q", err.Error())
}
}
func TestValidationAllowsRepeatedSupportedModuleKeys(t *testing.T) {
cfg := Default()
cfg.Modules = []string{modulecatalog.KeyGlossary, modulecatalog.KeyGlossary, modulecatalog.KeyGrammar}
if err := cfg.Validate(); err != nil {
t.Fatalf("expected repeated supported module keys to validate, got %v", err)
}
}
func TestValidationAcceptsAllSupportedOutputSchemas(t *testing.T) {
for _, schemaKey := range outputschema.SupportedKeys() {
cfg := Default()
cfg.OutputSchema = schemaKey
if err := cfg.Validate(); err != nil {
t.Fatalf("expected output schema %q to validate, got %v", schemaKey, err)
}
}
}
func TestEffectiveValidationLLMInheritance(t *testing.T) {
cfg := Default()
cfg.PrimaryLLM.APIKey = "primary-key"
cfg.PrimaryLLM.Model = "primary-model"
cfg.PrimaryLLM.BaseURL = "https://primary.example/v1"
cfg.PrimaryLLM.TimeoutSeconds = 111
cfg.PrimaryLLM.MaxRetries = 2
cfg.TotalLLMConcurrency = 7
cfg.syncLegacyConcurrencyAliases()
effective := cfg.EffectiveValidationLLMConfig()
if effective.APIKey != "primary-key" || effective.Model != "primary-model" || effective.BaseURL != "https://primary.example/v1" || effective.TimeoutSeconds != 111 || effective.MaxRetries != 2 || effective.Concurrency != 7 {
t.Fatalf("unexpected inherited config: %#v", effective)
}
validationTimeout := 222
validationRetries := 9
cfg.ValidationLLM.APIKey = "validation-key"
cfg.ValidationLLM.Model = "validation-model"
cfg.ValidationLLM.BaseURL = "https://validation.example/v1"
cfg.ValidationLLM.TimeoutSeconds = &validationTimeout
cfg.ValidationLLM.MaxRetries = &validationRetries
validationConcurrency := 4
cfg.ValidationLLMConcurrency = &validationConcurrency
cfg.syncLegacyConcurrencyAliases()
effective = cfg.EffectiveValidationLLMConfig()
if effective.APIKey != "validation-key" || effective.Model != "validation-model" || effective.BaseURL != "https://validation.example/v1" || effective.TimeoutSeconds != 222 || effective.MaxRetries != 9 || effective.Concurrency != 4 {
t.Fatalf("unexpected overridden validation config: %#v", effective)
}
}
func TestValidationLLMConcurrencyCannotExceedTotal(t *testing.T) {
cfg := Default()
cfg.TotalLLMConcurrency = 2
cfg.ProposalLLMConcurrency = 2
validationConcurrency := 3
cfg.ValidationLLMConcurrency = &validationConcurrency
if err := cfg.Validate(); err == nil {
t.Fatal("expected validation error when validation llm concurrency exceeds total")
}
validationConcurrency = 2
cfg.ValidationLLMConcurrency = &validationConcurrency
if err := cfg.Validate(); err != nil {
t.Fatalf("expected equal concurrency to validate, got %v", err)
}
}
func TestProposalLLMConcurrencyCannotExceedTotal(t *testing.T) {
cfg := Default()
cfg.TotalLLMConcurrency = 2
cfg.ProposalLLMConcurrency = 3
if err := cfg.Validate(); err == nil {
t.Fatal("expected validation error when proposal llm concurrency exceeds total")
}
cfg.ProposalLLMConcurrency = 2
if err := cfg.Validate(); err != nil {
t.Fatalf("expected equal concurrency to validate, got %v", err)
}
}
func TestCLITotalLLMConcurrencyOverrideDrivesEffectiveValidationConcurrencyWhenValidationUnset(t *testing.T) {
cfg := Default()
totalLLMConcurrency := 6
if err := cfg.ApplyCLIOverrides(CLIOverrides{TotalLLMConcurrency: &totalLLMConcurrency}); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.ValidationLLMConcurrency != nil {
t.Fatalf("expected validation concurrency to remain unset, got %#v", cfg.ValidationLLMConcurrency)
}
if cfg.EffectiveValidationLLMConcurrency() != 6 {
t.Fatalf("expected inherited validation concurrency 6, got %d", cfg.EffectiveValidationLLMConcurrency())
}
}
func TestRedactedConfig(t *testing.T) {
cfg := Default()
cfg.PrimaryLLM.APIKey = "secret-primary"
cfg.ValidationLLM.APIKey = "secret-validation"
redacted := cfg.Redacted()
if redacted.PrimaryLLM.APIKey != redactedSecret {
t.Fatalf("expected primary api key to be redacted, got %q", redacted.PrimaryLLM.APIKey)
}
if redacted.ValidationLLM.APIKey != redactedSecret {
t.Fatalf("expected validation api key to be redacted, got %q", redacted.ValidationLLM.APIKey)
}
if cfg.PrimaryLLM.APIKey != "secret-primary" {
t.Fatalf("redaction should not mutate original config")
}
}
func mapLookup(values map[string]string) func(string) (string, bool) {
return func(key string) (string, bool) {
value, ok := values[key]
return value, ok
}
}
func mustParseFileConfigYAML(t *testing.T, raw string) FileConfig {
t.Helper()
fileCfg, err := ParseFileConfigYAML([]byte(raw))
if err != nil {
t.Fatalf("ParseFileConfigYAML failed: %v", err)
}
return fileCfg
}
type sharedEffectiveFieldOptions struct {
wantOutputSchemaOverride bool
wantTranscriptDescriptionPatch bool
}
func assertSharedEffectiveFields(t *testing.T, cfg Config, opts sharedEffectiveFieldOptions) {
t.Helper()
wantOutputSchema := DefaultOutputSchema
if opts.wantOutputSchemaOverride {
wantOutputSchema = "audita-v1"
}
if cfg.OutputSchema != wantOutputSchema {
t.Fatalf("unexpected output schema: %q", cfg.OutputSchema)
}
if cfg.PrimaryLLM.Model != "provider/proposal" ||
cfg.PrimaryLLM.BaseURL != "https://proposal.example.test/v1" ||
cfg.PrimaryLLM.TimeoutSeconds != 101 ||
cfg.PrimaryLLM.MaxRetries != 5 {
t.Fatalf("unexpected primary llm config: %+v", cfg.PrimaryLLM)
}
if cfg.ValidationLLM.Model != "provider/validation" ||
cfg.ValidationLLM.BaseURL != "https://validation.example.test/v1" ||
cfg.ValidationLLM.TimeoutSeconds == nil ||
*cfg.ValidationLLM.TimeoutSeconds != 202 ||
cfg.ValidationLLM.MaxRetries == nil ||
*cfg.ValidationLLM.MaxRetries != 6 {
t.Fatalf("unexpected validation llm config: %+v", cfg.ValidationLLM)
}
if cfg.TargetSections == nil || *cfg.TargetSections != 7 ||
cfg.MaxSectionTokens != 9000 ||
cfg.MinSectionTokens != 1000 {
t.Fatalf("unexpected chunking config: target=%v max=%d min=%d", cfg.TargetSections, cfg.MaxSectionTokens, cfg.MinSectionTokens)
}
if cfg.Thresholds.Glossary != 0.91 ||
cfg.Thresholds.Grammar != 0.92 ||
cfg.Thresholds.Homophones != 0.93 ||
cfg.Thresholds.SpokenWord != 0.94 {
t.Fatalf("unexpected thresholds: %+v", cfg.Thresholds)
}
if cfg.Normalization.MaxSegmentGap != 1.2 ||
cfg.Normalization.EllipsisGap != 2.3 ||
cfg.Normalization.MaxSegmentDuration != 45.6 ||
cfg.Normalization.MaxSegmentTokens != 321 {
t.Fatalf("unexpected normalization: %+v", cfg.Normalization)
}
wantDescription := ""
if opts.wantTranscriptDescriptionPatch {
wantDescription = "shared context"
}
if cfg.TranscriptDescription != wantDescription {
t.Fatalf("unexpected transcript description: %q", cfg.TranscriptDescription)
}
if cfg.WorkDir != "/tmp/audita-shared" ||
cfg.WorkDirRetention != WorkDirRetentionAlways {
t.Fatalf("unexpected diagnostics config: work_dir=%q retention=%q", cfg.WorkDir, cfg.WorkDirRetention)
}
}

View File

@@ -0,0 +1,119 @@
package config
import (
"fmt"
"os"
"strings"
)
type EffectiveConfigErrorKind string
const (
EffectiveConfigErrorResolvePath EffectiveConfigErrorKind = "resolve_path"
EffectiveConfigErrorLoadFile EffectiveConfigErrorKind = "load_file"
EffectiveConfigErrorApplyFile EffectiveConfigErrorKind = "apply_file"
EffectiveConfigErrorApplyEnv EffectiveConfigErrorKind = "apply_env"
)
type EffectiveConfigError struct {
Kind EffectiveConfigErrorKind
Err error
}
func (e *EffectiveConfigError) Error() string {
if e == nil || e.Err == nil {
return ""
}
return e.Err.Error()
}
func (e *EffectiveConfigError) Unwrap() error {
if e == nil {
return nil
}
return e.Err
}
type EffectiveConfig struct {
Config Config
ConfigPath string
ConfigSource string
ConfigVersion *int
}
func ResolveConfigPath(cliConfigPath string, cliConfigPathSet bool) (path string, source string, err error) {
return resolveConfigPathWithLookup(cliConfigPath, cliConfigPathSet, os.LookupEnv, os.Stat, DefaultConfigSearchPaths)
}
func LoadEffectiveConfig(cliConfigPath string, cliConfigPathSet bool) (EffectiveConfig, error) {
return loadEffectiveConfigWithLookup(cliConfigPath, cliConfigPathSet, os.LookupEnv, os.Stat, DefaultConfigSearchPaths)
}
func loadEffectiveConfigWithLookup(cliConfigPath string, cliConfigPathSet bool, lookup func(string) (string, bool), statPath func(string) (os.FileInfo, error), defaultSearchPaths []string) (EffectiveConfig, error) {
configPath, configSource, err := resolveConfigPathWithLookup(cliConfigPath, cliConfigPathSet, lookup, statPath, defaultSearchPaths)
if err != nil {
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorResolvePath, Err: err}
}
cfg := Default()
var configVersion *int
if configPath != "" {
fileCfg, fileErr := LoadFileConfig(configPath)
if fileErr != nil {
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorLoadFile, Err: fileErr}
}
if applyErr := cfg.ApplyFileConfig(fileCfg); applyErr != nil {
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorApplyFile, Err: applyErr}
}
configVersion = &fileCfg.Version
}
if applyEnvErr := cfg.applyEnvOverrides(lookup); applyEnvErr != nil {
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorApplyEnv, Err: applyEnvErr}
}
return EffectiveConfig{
Config: cfg,
ConfigPath: configPath,
ConfigSource: configSource,
ConfigVersion: configVersion,
}, nil
}
func resolveConfigPathWithLookup(cliConfigPath string, cliConfigPathSet bool, lookup func(string) (string, bool), statPath func(string) (os.FileInfo, error), defaultSearchPaths []string) (path string, source string, err error) {
if cliConfigPathSet {
path = strings.TrimSpace(cliConfigPath)
if path == "" {
return "", "", fmt.Errorf("--config requires a non-empty path")
}
if _, statErr := statPath(path); statErr != nil {
if os.IsNotExist(statErr) {
return "", "", fmt.Errorf("config file not found: %s", path)
}
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
}
return path, "flag", nil
}
if raw, ok := lookup("AUDITA_CONFIG"); ok {
path = strings.TrimSpace(raw)
if path == "" {
return "", "", fmt.Errorf("AUDITA_CONFIG must not be empty")
}
if _, statErr := statPath(path); statErr != nil {
if os.IsNotExist(statErr) {
return "", "", fmt.Errorf("config file not found: %s", path)
}
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
}
return path, "env", nil
}
for _, defaultPath := range defaultSearchPaths {
if _, statErr := statPath(defaultPath); statErr == nil {
return defaultPath, "default", nil
} else if !os.IsNotExist(statErr) {
return "", "", fmt.Errorf("cannot access config file %s: %w", defaultPath, statErr)
}
}
return "", "", nil
}

View File

@@ -0,0 +1,163 @@
package config
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestResolveConfigPathWithLookupMatrix(t *testing.T) {
statFor := func(existing map[string]bool) func(string) (os.FileInfo, error) {
return func(path string) (os.FileInfo, error) {
if existing[path] {
return nil, nil
}
return nil, os.ErrNotExist
}
}
tests := []struct {
name string
cliPath string
cliPathSet bool
lookup func(string) (string, bool)
stat func(string) (os.FileInfo, error)
defaultSearchPaths []string
wantPath string
wantSource string
wantErrContains string
}{
{
name: "explicit config path",
cliPath: "/tmp/explicit.yml",
cliPathSet: true,
lookup: func(string) (string, bool) { return "", false },
stat: statFor(map[string]bool{"/tmp/explicit.yml": true}),
defaultSearchPaths: []string{
"/usr/local/etc/audita/config.yml",
"/etc/audita/config.yml",
},
wantPath: "/tmp/explicit.yml",
wantSource: "flag",
},
{
name: "env config path",
cliPathSet: false,
lookup: func(key string) (string, bool) {
if key == "AUDITA_CONFIG" {
return "/tmp/from-env.yml", true
}
return "", false
},
stat: statFor(map[string]bool{"/tmp/from-env.yml": true}),
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
wantPath: "/tmp/from-env.yml",
wantSource: "env",
},
{
name: "default search path",
cliPathSet: false,
lookup: func(string) (string, bool) { return "", false },
stat: statFor(map[string]bool{
"/usr/local/etc/audita/config.yml": true,
"/etc/audita/config.yml": true,
}),
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
wantPath: "/usr/local/etc/audita/config.yml",
wantSource: "default",
},
{
name: "explicit missing path",
cliPath: "/tmp/missing.yml",
cliPathSet: true,
lookup: func(string) (string, bool) { return "", false },
stat: statFor(map[string]bool{}),
defaultSearchPaths: []string{
"/usr/local/etc/audita/config.yml",
"/etc/audita/config.yml",
},
wantErrContains: "config file not found",
},
{
name: "missing env path",
cliPathSet: false,
lookup: func(key string) (string, bool) {
if key == "AUDITA_CONFIG" {
return "/tmp/missing-from-env.yml", true
}
return "", false
},
stat: statFor(map[string]bool{}),
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
wantErrContains: "config file not found",
},
{
name: "missing default paths",
cliPathSet: false,
lookup: func(string) (string, bool) { return "", false },
stat: statFor(map[string]bool{}),
defaultSearchPaths: []string{
"/usr/local/etc/audita/config.yml",
"/etc/audita/config.yml",
},
wantPath: "",
wantSource: "",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
gotPath, gotSource, err := resolveConfigPathWithLookup(tc.cliPath, tc.cliPathSet, tc.lookup, tc.stat, tc.defaultSearchPaths)
if tc.wantErrContains != "" {
if err == nil || !strings.Contains(err.Error(), tc.wantErrContains) {
t.Fatalf("expected error containing %q, got %v", tc.wantErrContains, err)
}
return
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if gotPath != tc.wantPath || gotSource != tc.wantSource {
t.Fatalf("unexpected result: got path=%q source=%q, want path=%q source=%q", gotPath, gotSource, tc.wantPath, tc.wantSource)
}
})
}
}
func TestLoadEffectiveConfigWithLookupAppliesDefaultsFileThenEnv(t *testing.T) {
tempDir := t.TempDir()
configPath := filepath.Join(tempDir, "config.yml")
configYAML := "version: 1\nllm:\n proposal:\n model: file-model\n"
if err := os.WriteFile(configPath, []byte(configYAML), 0o644); err != nil {
t.Fatalf("write config file: %v", err)
}
lookup := func(key string) (string, bool) {
switch key {
case "AUDITA_CONFIG":
return configPath, true
case "AUDITA_MODEL":
return "env-model", true
default:
return "", false
}
}
result, err := loadEffectiveConfigWithLookup("", false, lookup, os.Stat, DefaultConfigSearchPaths)
if err != nil {
t.Fatalf("loadEffectiveConfigWithLookup error: %v", err)
}
if result.ConfigPath != configPath {
t.Fatalf("unexpected config path: %q", result.ConfigPath)
}
if result.ConfigSource != "env" {
t.Fatalf("unexpected config source: %q", result.ConfigSource)
}
if result.ConfigVersion == nil || *result.ConfigVersion != SupportedFileConfigVersion {
t.Fatalf("unexpected config version: %#v", result.ConfigVersion)
}
if result.Config.PrimaryLLM.Model != "env-model" {
t.Fatalf("expected env override to win over file value, got %q", result.Config.PrimaryLLM.Model)
}
}

269
internal/core/config/env.go Normal file
View File

@@ -0,0 +1,269 @@
package config
import (
"fmt"
"os"
"strconv"
)
const (
DefaultConfigPath = "/etc/audita/config.yml"
DefaultConfigPathUsrLocal = "/usr/local/etc/audita/config.yml"
)
var DefaultConfigSearchPaths = []string{
DefaultConfigPathUsrLocal,
DefaultConfigPath,
}
func LoadFromEnv() (Config, error) {
cfg := Default()
if err := cfg.applyEnvOverrides(os.LookupEnv); err != nil {
return Config{}, err
}
return cfg, nil
}
func loadFromLookup(lookup func(string) (string, bool)) (Config, error) {
cfg := Default()
if err := cfg.applyEnvOverrides(lookup); err != nil {
return Config{}, err
}
return cfg, nil
}
func (c *Config) ApplyEnvOverrides() error {
return c.applyEnvOverrides(os.LookupEnv)
}
func (c *Config) applyEnvOverrides(lookup func(string) (string, bool)) error {
if c == nil {
return fmt.Errorf("config must not be nil")
}
cfg := c
if raw, ok := lookup("AUDITA_MODULES"); ok {
modules, err := ParseModulesCSV(raw)
if err != nil {
return fmt.Errorf("AUDITA_MODULES: %w", err)
}
cfg.Modules = modules
}
primaryLLM := llmTargetPatch{}
if raw, ok := lookup("AUDITA_LLM_API_KEY"); ok {
primaryLLM.apiKey = &raw
} else if raw, ok := lookup("OPENROUTER_API_KEY"); ok {
primaryLLM.apiKey = &raw
}
if raw, ok := lookup("AUDITA_MODEL"); ok {
primaryLLM.model = &raw
}
if raw, ok := lookup("AUDITA_BASE_URL"); ok {
primaryLLM.baseURL = &raw
}
if raw, ok := lookup("AUDITA_LLM_TIMEOUT_SECONDS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_LLM_TIMEOUT_SECONDS: %w", err)
}
primaryLLM.timeoutSeconds = &value
}
if raw, ok := lookup("AUDITA_MAX_RETRIES"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_MAX_RETRIES: %w", err)
}
primaryLLM.maxRetries = &value
}
cfg.applyPrimaryLLMTargetPatch(primaryLLM)
validationLLM := llmTargetPatch{}
if raw, ok := lookup("AUDITA_VALIDATION_LLM_API_KEY"); ok {
validationLLM.apiKey = &raw
}
if raw, ok := lookup("AUDITA_VALIDATION_MODEL"); ok {
validationLLM.model = &raw
}
if raw, ok := lookup("AUDITA_VALIDATION_BASE_URL"); ok {
validationLLM.baseURL = &raw
}
if raw, ok := lookup("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS: %w", err)
}
validationLLM.timeoutSeconds = &value
}
if raw, ok := lookup("AUDITA_VALIDATION_MAX_RETRIES"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_VALIDATION_MAX_RETRIES: %w", err)
}
validationLLM.maxRetries = &value
}
cfg.applyValidationLLMTargetPatch(validationLLM)
concurrency := concurrencyPatch{
inheritProposal: true,
allowLegacyAlias: true,
}
if raw, ok := lookup("AUDITA_TOTAL_LLM_CONCURRENCY"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_TOTAL_LLM_CONCURRENCY: %w", err)
}
concurrency.totalLLM = &value
}
if raw, ok := lookup("AUDITA_LLM_CONCURRENCY"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_LLM_CONCURRENCY: %w", err)
}
concurrency.legacyTotalLLM = &value
}
if raw, ok := lookup("AUDITA_PROPOSAL_LLM_CONCURRENCY"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_PROPOSAL_LLM_CONCURRENCY: %w", err)
}
concurrency.proposalLLM = &value
}
if raw, ok := lookup("AUDITA_VALIDATION_LLM_CONCURRENCY"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_VALIDATION_LLM_CONCURRENCY: %w", err)
}
concurrency.validationLLM = &value
}
cfg.applyConcurrencyPatch(concurrency)
if raw, ok := lookup("AUDITA_VALIDATION_MAX_PROMPT_TOKENS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_VALIDATION_MAX_PROMPT_TOKENS: %w", err)
}
cfg.ValidationMaxPromptTokens = value
}
chunking := chunkingPatch{}
if raw, ok := lookup("AUDITA_MAX_SECTION_TOKENS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_MAX_SECTION_TOKENS: %w", err)
}
chunking.maxSectionTokens = &value
}
if raw, ok := lookup("AUDITA_MIN_SECTION_TOKENS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_MIN_SECTION_TOKENS: %w", err)
}
chunking.minSectionTokens = &value
}
if raw, ok := lookup("AUDITA_TARGET_SECTIONS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_TARGET_SECTIONS: %w", err)
}
chunking.targetSections = &value
}
cfg.applyChunkingPatch(chunking)
thresholds := thresholdsPatch{}
if raw, ok := lookup("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD: %w", err)
}
thresholds.glossary = &value
}
if raw, ok := lookup("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD: %w", err)
}
thresholds.grammar = &value
}
if raw, ok := lookup("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD: %w", err)
}
thresholds.homophones = &value
}
if raw, ok := lookup("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD: %w", err)
}
thresholds.spokenWord = &value
}
cfg.applyThresholdsPatch(thresholds)
normalization := normalizationPatch{}
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_GAP"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_GAP: %w", err)
}
normalization.maxSegmentGap = &value
}
if raw, ok := lookup("AUDITA_NORMALIZE_ELLIPSIS_GAP"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_NORMALIZE_ELLIPSIS_GAP: %w", err)
}
normalization.ellipsisGap = &value
}
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION: %w", err)
}
normalization.maxSegmentDuration = &value
}
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS: %w", err)
}
normalization.maxSegmentTokens = &value
}
cfg.applyNormalizationPatch(normalization)
diagnostics := diagnosticsPatch{}
if raw, ok := lookup("AUDITA_WORK_DIR"); ok {
diagnostics.workDir = &raw
}
if raw, ok := lookup("AUDITA_WORK_DIR_RETENTION"); ok {
diagnostics.workDirRetention = &raw
}
cfg.applyDiagnosticsPatch(diagnostics)
cfg.syncLegacyConcurrencyAliases()
if err := cfg.Validate(); err != nil {
return err
}
return nil
}
func parseInt(raw string) (int, error) {
value, err := strconv.Atoi(raw)
if err != nil {
return 0, fmt.Errorf("must be an integer")
}
return value, nil
}
func parseFloat(raw string) (float64, error) {
value, err := strconv.ParseFloat(raw, 64)
if err != nil {
return 0, fmt.Errorf("must be a number")
}
return value, nil
}

View File

@@ -0,0 +1,322 @@
package config
import (
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"gopkg.in/yaml.v3"
)
const SupportedFileConfigVersion = 1
var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
type FileConfig struct {
Version int `yaml:"version"`
Pipeline *FileConfigPipeline `yaml:"pipeline,omitempty"`
Output *FileConfigOutput `yaml:"output,omitempty"`
LLM *FileConfigLLM `yaml:"llm,omitempty"`
Concurrency *FileConfigConcurrency `yaml:"concurrency,omitempty"`
Chunking *FileConfigChunking `yaml:"chunking,omitempty"`
Normalization *FileConfigNormalization `yaml:"normalization,omitempty"`
Thresholds *FileConfigThresholds `yaml:"thresholds,omitempty"`
Context *FileConfigContext `yaml:"context,omitempty"`
Diagnostics *FileConfigDiagnostics `yaml:"diagnostics,omitempty"`
}
type FileConfigPipeline struct {
Modules []string `yaml:"modules,omitempty"`
}
type FileConfigOutput struct {
Schema *string `yaml:"schema,omitempty"`
}
type FileConfigLLM struct {
Proposal *FileConfigLLMTarget `yaml:"proposal,omitempty"`
Validation *FileConfigLLMTarget `yaml:"validation,omitempty"`
}
type FileConfigLLMTarget struct {
BaseURL *string `yaml:"base_url,omitempty"`
Model *string `yaml:"model,omitempty"`
APIKeyEnv *string `yaml:"api_key_env,omitempty"`
Timeout *fileConfigDurationOrInt `yaml:"timeout,omitempty"`
MaxRetries *int `yaml:"max_retries,omitempty"`
}
type FileConfigConcurrency struct {
TotalLLM *int `yaml:"total_llm,omitempty"`
ProposalLLM *int `yaml:"proposal_llm,omitempty"`
ValidationLLM *int `yaml:"validation_llm,omitempty"`
}
type FileConfigChunking struct {
TargetSections *int `yaml:"target_sections,omitempty"`
MaxSectionTokens *int `yaml:"max_section_tokens,omitempty"`
MinSectionTokens *int `yaml:"min_section_tokens,omitempty"`
}
type FileConfigNormalization struct {
MaxSegmentGap *fileConfigDurationOrFloat `yaml:"max_segment_gap,omitempty"`
EllipsisGap *fileConfigDurationOrFloat `yaml:"ellipsis_gap,omitempty"`
MaxSegmentDuration *fileConfigDurationOrFloat `yaml:"max_segment_duration,omitempty"`
MaxSegmentTokens *int `yaml:"max_segment_tokens,omitempty"`
}
type FileConfigThresholds struct {
Glossary *float64 `yaml:"glossary,omitempty"`
Homophones *float64 `yaml:"homophones,omitempty"`
SpokenWord *float64 `yaml:"spoken_word,omitempty"`
Grammar *float64 `yaml:"grammar,omitempty"`
}
type FileConfigContext struct {
Description *string `yaml:"description,omitempty"`
}
type FileConfigDiagnostics struct {
WorkDir *string `yaml:"work_dir,omitempty"`
Retention *string `yaml:"retention,omitempty"`
}
type fileConfigDurationOrInt struct {
seconds int
}
func (v *fileConfigDurationOrInt) UnmarshalYAML(node *yaml.Node) error {
switch node.Kind {
case yaml.ScalarNode:
if node.Tag == "!!int" {
var n int
if err := node.Decode(&n); err != nil {
return fmt.Errorf("must be an integer seconds value or duration string")
}
v.seconds = n
return nil
}
var s string
if err := node.Decode(&s); err != nil {
return fmt.Errorf("must be an integer seconds value or duration string")
}
d, err := time.ParseDuration(strings.TrimSpace(s))
if err != nil {
return fmt.Errorf("invalid duration %q", s)
}
if d <= 0 {
v.seconds = int(d / time.Second)
return nil
}
if d%time.Second != 0 {
return fmt.Errorf("duration %q must resolve to whole seconds", s)
}
v.seconds = int(d / time.Second)
return nil
default:
return fmt.Errorf("must be an integer seconds value or duration string")
}
}
func (v fileConfigDurationOrInt) Seconds() int { return v.seconds }
type fileConfigDurationOrFloat struct {
seconds float64
}
func (v *fileConfigDurationOrFloat) UnmarshalYAML(node *yaml.Node) error {
switch node.Kind {
case yaml.ScalarNode:
if node.Tag == "!!int" || node.Tag == "!!float" {
var f float64
if err := node.Decode(&f); err != nil {
return fmt.Errorf("must be a numeric seconds value or duration string")
}
v.seconds = f
return nil
}
var s string
if err := node.Decode(&s); err != nil {
return fmt.Errorf("must be a numeric seconds value or duration string")
}
d, err := time.ParseDuration(strings.TrimSpace(s))
if err != nil {
return fmt.Errorf("invalid duration %q", s)
}
v.seconds = d.Seconds()
return nil
default:
return fmt.Errorf("must be a numeric seconds value or duration string")
}
}
func (v fileConfigDurationOrFloat) Seconds() float64 { return v.seconds }
func LoadFileConfig(path string) (FileConfig, error) {
b, err := os.ReadFile(path)
if err != nil {
return FileConfig{}, fmt.Errorf("read config file %q: %w", path, err)
}
cfg, err := ParseFileConfigYAML(b)
if err != nil {
return FileConfig{}, fmt.Errorf("parse config file %q: %w", path, err)
}
return cfg, nil
}
func ParseFileConfigYAML(data []byte) (FileConfig, error) {
var fileCfg FileConfig
dec := yaml.NewDecoder(strings.NewReader(string(data)))
dec.KnownFields(true)
if err := dec.Decode(&fileCfg); err != nil {
return FileConfig{}, fmt.Errorf("decode yaml: %w", err)
}
if fileCfg.Version == 0 {
return FileConfig{}, fmt.Errorf("config version is required")
}
if fileCfg.Version != SupportedFileConfigVersion {
return FileConfig{}, fmt.Errorf("unsupported config version %d", fileCfg.Version)
}
return fileCfg, nil
}
func (c *Config) ApplyFileConfig(fileCfg FileConfig) error {
return c.applyFileConfigWithLookup(fileCfg, os.LookupEnv)
}
func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
if c == nil {
return fmt.Errorf("config must not be nil")
}
if fileCfg.Version != SupportedFileConfigVersion {
return fmt.Errorf("unsupported config version %d", fileCfg.Version)
}
if fileCfg.Pipeline != nil && len(fileCfg.Pipeline.Modules) > 0 {
c.Modules = append([]string(nil), fileCfg.Pipeline.Modules...)
}
if fileCfg.Output != nil && fileCfg.Output.Schema != nil {
c.OutputSchema = strings.TrimSpace(*fileCfg.Output.Schema)
}
if fileCfg.LLM != nil {
if fileCfg.LLM.Proposal != nil {
patch := llmTargetPatch{
model: fileCfg.LLM.Proposal.Model,
baseURL: fileCfg.LLM.Proposal.BaseURL,
maxRetries: fileCfg.LLM.Proposal.MaxRetries,
}
if fileCfg.LLM.Proposal.Timeout != nil {
timeoutSeconds := fileCfg.LLM.Proposal.Timeout.Seconds()
patch.timeoutSeconds = &timeoutSeconds
}
if fileCfg.LLM.Proposal.APIKeyEnv != nil {
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Proposal.APIKeyEnv, lookup)
if err != nil {
return fmt.Errorf("llm.proposal.api_key_env: %w", err)
}
patch.apiKey = &apiKey
}
c.applyPrimaryLLMTargetPatch(patch)
}
if fileCfg.LLM.Validation != nil {
patch := llmTargetPatch{
model: fileCfg.LLM.Validation.Model,
baseURL: fileCfg.LLM.Validation.BaseURL,
maxRetries: fileCfg.LLM.Validation.MaxRetries,
}
if fileCfg.LLM.Validation.Timeout != nil {
timeoutSeconds := fileCfg.LLM.Validation.Timeout.Seconds()
patch.timeoutSeconds = &timeoutSeconds
}
if fileCfg.LLM.Validation.APIKeyEnv != nil {
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Validation.APIKeyEnv, lookup)
if err != nil {
return fmt.Errorf("llm.validation.api_key_env: %w", err)
}
patch.apiKey = &apiKey
}
c.applyValidationLLMTargetPatch(patch)
}
}
if fileCfg.Concurrency != nil {
c.applyConcurrencyPatch(concurrencyPatch{
totalLLM: fileCfg.Concurrency.TotalLLM,
proposalLLM: fileCfg.Concurrency.ProposalLLM,
validationLLM: fileCfg.Concurrency.ValidationLLM,
})
}
if fileCfg.Chunking != nil {
c.applyChunkingPatch(chunkingPatch{
targetSections: fileCfg.Chunking.TargetSections,
maxSectionTokens: fileCfg.Chunking.MaxSectionTokens,
minSectionTokens: fileCfg.Chunking.MinSectionTokens,
})
}
if fileCfg.Normalization != nil {
patch := normalizationPatch{
maxSegmentTokens: fileCfg.Normalization.MaxSegmentTokens,
}
if fileCfg.Normalization.MaxSegmentGap != nil {
maxSegmentGap := fileCfg.Normalization.MaxSegmentGap.Seconds()
patch.maxSegmentGap = &maxSegmentGap
}
if fileCfg.Normalization.EllipsisGap != nil {
ellipsisGap := fileCfg.Normalization.EllipsisGap.Seconds()
patch.ellipsisGap = &ellipsisGap
}
if fileCfg.Normalization.MaxSegmentDuration != nil {
maxSegmentDuration := fileCfg.Normalization.MaxSegmentDuration.Seconds()
patch.maxSegmentDuration = &maxSegmentDuration
}
c.applyNormalizationPatch(patch)
}
if fileCfg.Thresholds != nil {
c.applyThresholdsPatch(thresholdsPatch{
glossary: fileCfg.Thresholds.Glossary,
grammar: fileCfg.Thresholds.Grammar,
homophones: fileCfg.Thresholds.Homophones,
spokenWord: fileCfg.Thresholds.SpokenWord,
})
}
if fileCfg.Context != nil && fileCfg.Context.Description != nil {
c.applyContextPatch(contextPatch{transcriptDescription: fileCfg.Context.Description})
}
if fileCfg.Diagnostics != nil {
c.applyDiagnosticsPatch(diagnosticsPatch{
workDir: fileCfg.Diagnostics.WorkDir,
workDirRetention: fileCfg.Diagnostics.Retention,
})
}
c.syncLegacyConcurrencyAliases()
if err := c.Validate(); err != nil {
return err
}
return nil
}
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
name := strings.TrimSpace(envName)
if name == "" {
return "", fmt.Errorf("must not be empty")
}
if !envVarNamePattern.MatchString(name) {
return "", fmt.Errorf("must be an environment variable name")
}
if strings.Contains(name, string(filepath.Separator)) {
return "", fmt.Errorf("must be an environment variable name")
}
v, _ := lookup(name)
return v, nil
}

View File

@@ -0,0 +1,290 @@
package config
import (
"os"
"strings"
"testing"
)
func TestParseFileConfigYAMLValid(t *testing.T) {
raw := `
version: 1
pipeline:
modules: [glossary, homophones, grammar]
output:
schema: audita-v1
llm:
proposal:
base_url: https://example.test/v1
model: provider/model-a
api_key_env: AUDITA_PROPOSAL_KEY
timeout: 2m
max_retries: 4
validation:
base_url: https://example.test/validation
model: provider/model-b
api_key_env: AUDITA_VALIDATION_KEY
timeout: 45
max_retries: 3
concurrency:
total_llm: 8
proposal_llm: 4
validation_llm: 2
chunking:
target_sections: 6
max_section_tokens: 9000
min_section_tokens: 3000
normalization:
max_segment_gap: 1.5s
ellipsis_gap: 2
max_segment_duration: 45s
max_segment_tokens: 1500
thresholds:
glossary: 0.9
homophones: 0.7
spoken_word: 0.8
grammar: 0.75
context:
description: " crowd scene with many proper nouns "
diagnostics:
work_dir: /tmp/audita-config
retention: always
`
cfg, err := ParseFileConfigYAML([]byte(raw))
if err != nil {
t.Fatalf("ParseFileConfigYAML error: %v", err)
}
if cfg.Version != 1 {
t.Fatalf("expected version 1, got %d", cfg.Version)
}
if cfg.Pipeline == nil || len(cfg.Pipeline.Modules) != 3 {
t.Fatalf("unexpected pipeline modules: %#v", cfg.Pipeline)
}
if cfg.Output == nil || cfg.Output.Schema == nil || *cfg.Output.Schema != "audita-v1" {
t.Fatalf("expected output schema audita-v1, got %#v", cfg.Output)
}
if cfg.LLM == nil || cfg.LLM.Proposal == nil || cfg.LLM.Validation == nil {
t.Fatalf("expected llm proposal+validation blocks")
}
if cfg.LLM.Proposal.Timeout == nil || cfg.LLM.Proposal.Timeout.Seconds() != 120 {
t.Fatalf("expected proposal timeout 120s, got %#v", cfg.LLM.Proposal.Timeout)
}
if cfg.LLM.Validation.Timeout == nil || cfg.LLM.Validation.Timeout.Seconds() != 45 {
t.Fatalf("expected validation timeout 45s, got %#v", cfg.LLM.Validation.Timeout)
}
if cfg.Normalization == nil || cfg.Normalization.MaxSegmentGap == nil || cfg.Normalization.MaxSegmentGap.Seconds() != 1.5 {
t.Fatalf("expected parsed duration for normalization max_segment_gap")
}
}
func TestParseFileConfigYAMLRejectsUnknownField(t *testing.T) {
raw := `
version: 1
pipeline:
modules: [grammar]
output:
unknown: v1
`
_, err := ParseFileConfigYAML([]byte(raw))
if err == nil {
t.Fatalf("expected unknown field error")
}
if !strings.Contains(err.Error(), "field unknown not found") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestParseFileConfigYAMLRejectsMissingVersion(t *testing.T) {
raw := `pipeline: {modules: [grammar]}`
_, err := ParseFileConfigYAML([]byte(raw))
if err == nil {
t.Fatalf("expected missing version error")
}
if !strings.Contains(err.Error(), "config version is required") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestParseFileConfigYAMLRejectsUnsupportedVersion(t *testing.T) {
raw := `version: 2`
_, err := ParseFileConfigYAML([]byte(raw))
if err == nil {
t.Fatalf("expected unsupported version error")
}
if !strings.Contains(err.Error(), "unsupported config version 2") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestApplyFileConfigParsesAndMergesFields(t *testing.T) {
raw := `
version: 1
pipeline:
modules: [spoken_word, grammar]
output:
schema: audita-v1
llm:
proposal:
model: provider/new-proposal
api_key_env: PROPOSAL_KEY_NAME
timeout: 90s
max_retries: 5
validation:
model: provider/new-validation
api_key_env: VALIDATION_KEY_NAME
timeout: 150
max_retries: 6
concurrency:
total_llm: 7
proposal_llm: 3
validation_llm: 2
chunking:
target_sections: 9
thresholds:
glossary: 0.91
homophones: 0.61
spoken_word: 0.71
grammar: 0.81
diagnostics:
retention: never
`
fileCfg, err := ParseFileConfigYAML([]byte(raw))
if err != nil {
t.Fatalf("ParseFileConfigYAML error: %v", err)
}
cfg := Default()
lookup := func(name string) (string, bool) {
switch name {
case "PROPOSAL_KEY_NAME":
return "proposal-secret", true
case "VALIDATION_KEY_NAME":
return "validation-secret", true
default:
return "", false
}
}
if err := cfg.applyFileConfigWithLookup(fileCfg, lookup); err != nil {
t.Fatalf("applyFileConfigWithLookup error: %v", err)
}
if strings.Join(cfg.Modules, ",") != "spoken_word,grammar" {
t.Fatalf("unexpected modules: %#v", cfg.Modules)
}
if cfg.OutputSchema != "audita-v1" {
t.Fatalf("unexpected output schema: %q", cfg.OutputSchema)
}
if cfg.PrimaryLLM.Model != "provider/new-proposal" {
t.Fatalf("unexpected proposal model: %q", cfg.PrimaryLLM.Model)
}
if cfg.PrimaryLLM.APIKey != "proposal-secret" {
t.Fatalf("expected proposal key from api_key_env lookup, got %q", cfg.PrimaryLLM.APIKey)
}
if cfg.PrimaryLLM.TimeoutSeconds != 90 {
t.Fatalf("unexpected proposal timeout: %d", cfg.PrimaryLLM.TimeoutSeconds)
}
if cfg.ValidationLLM.Model != "provider/new-validation" {
t.Fatalf("unexpected validation model: %q", cfg.ValidationLLM.Model)
}
if cfg.ValidationLLM.APIKey != "validation-secret" {
t.Fatalf("expected validation key from api_key_env lookup, got %q", cfg.ValidationLLM.APIKey)
}
if cfg.ValidationLLM.TimeoutSeconds == nil || *cfg.ValidationLLM.TimeoutSeconds != 150 {
t.Fatalf("unexpected validation timeout: %#v", cfg.ValidationLLM.TimeoutSeconds)
}
if cfg.TotalLLMConcurrency != 7 || cfg.ProposalLLMConcurrency != 3 {
t.Fatalf("unexpected llm concurrency values: total=%d proposal=%d", cfg.TotalLLMConcurrency, cfg.ProposalLLMConcurrency)
}
if cfg.ValidationLLMConcurrency == nil || *cfg.ValidationLLMConcurrency != 2 {
t.Fatalf("unexpected validation llm concurrency: %#v", cfg.ValidationLLMConcurrency)
}
if cfg.TargetSections == nil || *cfg.TargetSections != 9 {
t.Fatalf("unexpected target sections: %#v", cfg.TargetSections)
}
if cfg.WorkDirRetention != WorkDirRetentionNever {
t.Fatalf("unexpected retention: %q", cfg.WorkDirRetention)
}
if cfg.PrimaryLLM.Concurrency != 7 {
t.Fatalf("expected legacy alias to sync, got %d", cfg.PrimaryLLM.Concurrency)
}
if cfg.ValidationLLM.Concurrency == nil || *cfg.ValidationLLM.Concurrency != 2 {
t.Fatalf("expected validation alias to sync, got %#v", cfg.ValidationLLM.Concurrency)
}
}
func TestApplyFileConfigContextDescriptionTrim(t *testing.T) {
raw := `
version: 1
context:
description: " scene context "
`
fileCfg, err := ParseFileConfigYAML([]byte(raw))
if err != nil {
t.Fatalf("ParseFileConfigYAML error: %v", err)
}
cfg := Default()
if err := cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{})); err != nil {
t.Fatalf("applyFileConfigWithLookup error: %v", err)
}
if cfg.TranscriptDescription != "scene context" {
t.Fatalf("unexpected transcript description: %q", cfg.TranscriptDescription)
}
}
func TestApplyFileConfigRejectsInvalidAPIKeyEnvName(t *testing.T) {
raw := `
version: 1
llm:
proposal:
api_key_env: "not a var name"
`
fileCfg, err := ParseFileConfigYAML([]byte(raw))
if err != nil {
t.Fatalf("ParseFileConfigYAML error: %v", err)
}
cfg := Default()
err = cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{}))
if err == nil {
t.Fatalf("expected api_key_env validation error")
}
if !strings.Contains(err.Error(), "environment variable name") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestParseFileConfigDurationParsingErrors(t *testing.T) {
raw := `
version: 1
llm:
proposal:
timeout: "1.5s"
`
_, err := ParseFileConfigYAML([]byte(raw))
if err == nil {
t.Fatalf("expected duration parse error")
}
if !strings.Contains(err.Error(), "whole seconds") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestLoadFileConfigReadsFromPath(t *testing.T) {
p := writeTempFileConfig(t, "version: 1\n")
cfg, err := LoadFileConfig(p)
if err != nil {
t.Fatalf("LoadFileConfig error: %v", err)
}
if cfg.Version != 1 {
t.Fatalf("expected version 1, got %d", cfg.Version)
}
}
func writeTempFileConfig(t *testing.T, contents string) string {
t.Helper()
dir := t.TempDir()
path := dir + "/config.yaml"
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
t.Fatalf("write config file: %v", err)
}
return path
}

View File

@@ -0,0 +1,106 @@
package config
import (
"fmt"
"strings"
)
type CLIOverrides struct {
ModulesCSV *string
OutputSchema *string
PrimaryLLMAPIKey *string
ValidationLLMAPIKey *string
PrimaryModel *string
ValidationModel *string
PrimaryBaseURL *string
ValidationBaseURL *string
PrimaryLLMTimeoutSeconds *int
TotalLLMConcurrency *int
ProposalLLMConcurrency *int
PrimaryLLMConcurrency *int
ValidationLLMTimeoutSeconds *int
MaxRetries *int
ValidationMaxRetries *int
ValidationLLMConcurrency *int
ValidationMaxPromptTokens *int
MaxSectionTokens *int
MinSectionTokens *int
TargetSections *int
GlossaryConfidenceThreshold *float64
GrammarConfidenceThreshold *float64
HomophonesConfidenceThreshold *float64
SpokenWordConfidenceThreshold *float64
NormalizeMaxSegmentGap *float64
NormalizeEllipsisGap *float64
NormalizeMaxSegmentDuration *float64
NormalizeMaxSegmentTokens *int
TranscriptDescription *string
WorkDir *string
WorkDirRetention *string
}
func (c *Config) ApplyCLIOverrides(overrides CLIOverrides) error {
if overrides.ModulesCSV != nil {
modules, err := ParseModulesCSV(*overrides.ModulesCSV)
if err != nil {
return fmt.Errorf("--modules: %w", err)
}
c.Modules = modules
}
if overrides.OutputSchema != nil {
c.OutputSchema = strings.TrimSpace(*overrides.OutputSchema)
}
c.applyPrimaryLLMTargetPatch(llmTargetPatch{
apiKey: overrides.PrimaryLLMAPIKey,
model: overrides.PrimaryModel,
baseURL: overrides.PrimaryBaseURL,
timeoutSeconds: overrides.PrimaryLLMTimeoutSeconds,
maxRetries: overrides.MaxRetries,
})
c.applyValidationLLMTargetPatch(llmTargetPatch{
apiKey: overrides.ValidationLLMAPIKey,
model: overrides.ValidationModel,
baseURL: overrides.ValidationBaseURL,
timeoutSeconds: overrides.ValidationLLMTimeoutSeconds,
maxRetries: overrides.ValidationMaxRetries,
})
c.applyConcurrencyPatch(concurrencyPatch{
totalLLM: overrides.TotalLLMConcurrency,
legacyTotalLLM: overrides.PrimaryLLMConcurrency,
proposalLLM: overrides.ProposalLLMConcurrency,
validationLLM: overrides.ValidationLLMConcurrency,
inheritProposal: true,
allowLegacyAlias: true,
})
if overrides.ValidationMaxPromptTokens != nil {
c.ValidationMaxPromptTokens = *overrides.ValidationMaxPromptTokens
}
c.applyChunkingPatch(chunkingPatch{
targetSections: overrides.TargetSections,
maxSectionTokens: overrides.MaxSectionTokens,
minSectionTokens: overrides.MinSectionTokens,
})
c.applyThresholdsPatch(thresholdsPatch{
glossary: overrides.GlossaryConfidenceThreshold,
grammar: overrides.GrammarConfidenceThreshold,
homophones: overrides.HomophonesConfidenceThreshold,
spokenWord: overrides.SpokenWordConfidenceThreshold,
})
c.applyNormalizationPatch(normalizationPatch{
maxSegmentGap: overrides.NormalizeMaxSegmentGap,
ellipsisGap: overrides.NormalizeEllipsisGap,
maxSegmentDuration: overrides.NormalizeMaxSegmentDuration,
maxSegmentTokens: overrides.NormalizeMaxSegmentTokens,
})
c.applyContextPatch(contextPatch{transcriptDescription: overrides.TranscriptDescription})
c.applyDiagnosticsPatch(diagnosticsPatch{
workDir: overrides.WorkDir,
workDirRetention: overrides.WorkDirRetention,
})
c.syncLegacyConcurrencyAliases()
return c.Validate()
}

View File

@@ -0,0 +1,17 @@
package config
const redactedSecret = "[REDACTED]"
func (c Config) Redacted() Config {
redacted := c
redacted.PrimaryLLM.APIKey = redactSecret(redacted.PrimaryLLM.APIKey)
redacted.ValidationLLM.APIKey = redactSecret(redacted.ValidationLLM.APIKey)
return redacted
}
func redactSecret(value string) string {
if value == "" {
return ""
}
return redactedSecret
}

View File

@@ -0,0 +1,136 @@
package config
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
)
func (c Config) Validate() error {
var issues []string
if len(c.Modules) == 0 {
issues = append(issues, "modules must not be empty")
}
for _, module := range c.Modules {
moduleKey := strings.TrimSpace(module)
if moduleKey == "" {
issues = append(issues, "modules must not contain empty values")
break
}
if !modulecatalog.IsSupported(moduleKey) {
issues = append(issues, fmt.Sprintf("unsupported module key %q", moduleKey))
}
}
if strings.TrimSpace(c.OutputSchema) == "" {
issues = append(issues, "output schema must not be empty")
} else if !outputschema.IsSupported(c.OutputSchema) {
issues = append(issues, fmt.Sprintf("unsupported output schema %q", c.OutputSchema))
}
if c.PrimaryLLM.TimeoutSeconds <= 0 {
issues = append(issues, "primary llm timeout seconds must be greater than zero")
}
if c.PrimaryLLM.MaxRetries < 0 {
issues = append(issues, "max retries must be zero or greater")
}
if c.TotalLLMConcurrency <= 0 {
issues = append(issues, "total llm concurrency must be greater than zero")
}
if c.ProposalLLMConcurrency <= 0 {
issues = append(issues, "proposal llm concurrency must be greater than zero")
}
if c.ProposalLLMConcurrency > c.TotalLLMConcurrency {
issues = append(issues, "proposal llm concurrency must be less than or equal to total llm concurrency")
}
if c.ValidationLLM.TimeoutSeconds != nil && *c.ValidationLLM.TimeoutSeconds <= 0 {
issues = append(issues, "validation llm timeout seconds must be greater than zero")
}
if c.ValidationLLM.MaxRetries != nil && *c.ValidationLLM.MaxRetries < 0 {
issues = append(issues, "validation max retries must be zero or greater")
}
if c.ValidationLLMConcurrency != nil && *c.ValidationLLMConcurrency <= 0 {
issues = append(issues, "validation llm concurrency must be greater than zero")
}
if c.ValidationLLMConcurrency != nil && *c.ValidationLLMConcurrency > c.TotalLLMConcurrency {
issues = append(issues, "validation llm concurrency must be less than or equal to total llm concurrency")
}
if c.ValidationMaxPromptTokens <= 0 {
issues = append(issues, "validation max prompt tokens must be greater than zero")
}
if c.MaxSectionTokens <= 0 {
issues = append(issues, "max section tokens must be greater than zero")
}
if c.MinSectionTokens <= 0 {
issues = append(issues, "min section tokens must be greater than zero")
}
if c.MinSectionTokens > c.MaxSectionTokens {
issues = append(issues, "min section tokens must be less than or equal to max section tokens")
}
if c.TargetSections != nil && *c.TargetSections <= 0 {
issues = append(issues, "target sections must be greater than zero when set")
}
if err := validateConfidence("glossary", c.Thresholds.Glossary); err != nil {
issues = append(issues, err.Error())
}
if err := validateConfidence("grammar", c.Thresholds.Grammar); err != nil {
issues = append(issues, err.Error())
}
if err := validateConfidence("homophones", c.Thresholds.Homophones); err != nil {
issues = append(issues, err.Error())
}
if err := validateConfidence("spoken-word", c.Thresholds.SpokenWord); err != nil {
issues = append(issues, err.Error())
}
if c.Normalization.MaxSegmentGap < 0 {
issues = append(issues, "normalize max segment gap must be zero or greater")
}
if c.Normalization.EllipsisGap < 0 {
issues = append(issues, "normalize ellipsis gap must be zero or greater")
}
if c.Normalization.MaxSegmentDuration <= 0 {
issues = append(issues, "normalize max segment duration must be greater than zero")
}
if c.Normalization.MaxSegmentTokens <= 0 {
issues = append(issues, "normalize max segment tokens must be greater than zero")
}
if len(strings.TrimSpace(c.TranscriptDescription)) > DefaultTranscriptDescriptionMaxChars {
issues = append(issues, fmt.Sprintf("transcript description must be %d characters or fewer", DefaultTranscriptDescriptionMaxChars))
}
if strings.TrimSpace(c.WorkDir) == "" {
issues = append(issues, "work dir must not be empty")
}
if err := validateRetention(c.WorkDirRetention); err != nil {
issues = append(issues, err.Error())
}
if len(issues) > 0 {
return fmt.Errorf("invalid config: %s", strings.Join(issues, "; "))
}
return nil
}
func validateConfidence(name string, threshold float64) error {
if threshold < 0.0 || threshold > 1.0 {
return fmt.Errorf("%s confidence threshold must be between 0.0 and 1.0", name)
}
return nil
}
func validateRetention(retention WorkDirRetention) error {
switch retention {
case WorkDirRetentionAuto, WorkDirRetentionAlways, WorkDirRetentionNever:
return nil
default:
return fmt.Errorf("work dir retention must be one of: auto, always, never")
}
}

View File

@@ -0,0 +1,42 @@
package diagnostics
import (
"path/filepath"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
)
const (
ArtifactSourceTranscript = "source-transcript.json"
ArtifactParsedSourceTranscript = "source-transcript-parsed.json"
ArtifactNormalizedTranscript = "normalized-transcript.json"
ArtifactNormalizationSummary = "normalization-summary.json"
ArtifactChunkingSummary = "chunking-summary.json"
ArtifactUtilizationSummary = "utilization-diagnostics.json"
ArtifactCorrectionLedger = "correction-ledger.json"
ArtifactInvocationMetadata = "invocation.json"
ArtifactEffectiveConfig = "effective-config.json"
ArtifactReport = "report.json"
ArtifactErrorLog = "error.log"
)
func BuildDiagnosticsMetadata(runDirectoryPath string, runSucceeded bool) reporting.DiagnosticsMetadata {
metadata := reporting.DiagnosticsMetadata{
DirectoryPath: runDirectoryPath,
SourceTranscriptPath: filepath.Join(runDirectoryPath, ArtifactSourceTranscript),
ParsedSourceTranscriptPath: filepath.Join(runDirectoryPath, ArtifactParsedSourceTranscript),
NormalizedTranscriptPath: filepath.Join(runDirectoryPath, ArtifactNormalizedTranscript),
NormalizationSummaryPath: filepath.Join(runDirectoryPath, ArtifactNormalizationSummary),
ChunkingSummaryPath: filepath.Join(runDirectoryPath, ArtifactChunkingSummary),
UtilizationSummaryPath: filepath.Join(runDirectoryPath, ArtifactUtilizationSummary),
CorrectionLedgerPath: filepath.Join(runDirectoryPath, ArtifactCorrectionLedger),
InvocationMetadataPath: filepath.Join(runDirectoryPath, ArtifactInvocationMetadata),
RedactedEffectiveConfigPath: filepath.Join(runDirectoryPath, ArtifactEffectiveConfig),
}
if !runSucceeded {
metadata.ErrorLogPath = filepath.Join(runDirectoryPath, ArtifactErrorLog)
}
return metadata
}

View File

@@ -0,0 +1,55 @@
package diagnostics
import (
"path/filepath"
"testing"
)
func TestBuildDiagnosticsMetadataSuccessPathsMatchArtifactConstants(t *testing.T) {
runPath := filepath.Join("tmp", "run-123")
metadata := BuildDiagnosticsMetadata(runPath, true)
if metadata.DirectoryPath != runPath {
t.Fatalf("unexpected diagnostics directory path: got=%q want=%q", metadata.DirectoryPath, runPath)
}
if metadata.SourceTranscriptPath != filepath.Join(runPath, ArtifactSourceTranscript) {
t.Fatalf("unexpected source transcript path: %q", metadata.SourceTranscriptPath)
}
if metadata.ParsedSourceTranscriptPath != filepath.Join(runPath, ArtifactParsedSourceTranscript) {
t.Fatalf("unexpected parsed source transcript path: %q", metadata.ParsedSourceTranscriptPath)
}
if metadata.NormalizedTranscriptPath != filepath.Join(runPath, ArtifactNormalizedTranscript) {
t.Fatalf("unexpected normalized transcript path: %q", metadata.NormalizedTranscriptPath)
}
if metadata.NormalizationSummaryPath != filepath.Join(runPath, ArtifactNormalizationSummary) {
t.Fatalf("unexpected normalization summary path: %q", metadata.NormalizationSummaryPath)
}
if metadata.ChunkingSummaryPath != filepath.Join(runPath, ArtifactChunkingSummary) {
t.Fatalf("unexpected chunking summary path: %q", metadata.ChunkingSummaryPath)
}
if metadata.UtilizationSummaryPath != filepath.Join(runPath, ArtifactUtilizationSummary) {
t.Fatalf("unexpected utilization summary path: %q", metadata.UtilizationSummaryPath)
}
if metadata.CorrectionLedgerPath != filepath.Join(runPath, ArtifactCorrectionLedger) {
t.Fatalf("unexpected correction ledger path: %q", metadata.CorrectionLedgerPath)
}
if metadata.InvocationMetadataPath != filepath.Join(runPath, ArtifactInvocationMetadata) {
t.Fatalf("unexpected invocation metadata path: %q", metadata.InvocationMetadataPath)
}
if metadata.RedactedEffectiveConfigPath != filepath.Join(runPath, ArtifactEffectiveConfig) {
t.Fatalf("unexpected redacted effective config path: %q", metadata.RedactedEffectiveConfigPath)
}
if metadata.ErrorLogPath != "" {
t.Fatalf("did not expect error log path on success: %q", metadata.ErrorLogPath)
}
}
func TestBuildDiagnosticsMetadataFailureIncludesErrorLogPath(t *testing.T) {
runPath := filepath.Join("tmp", "run-123")
metadata := BuildDiagnosticsMetadata(runPath, false)
want := filepath.Join(runPath, ArtifactErrorLog)
if metadata.ErrorLogPath != want {
t.Fatalf("unexpected error log path: got=%q want=%q", metadata.ErrorLogPath, want)
}
}

View File

@@ -0,0 +1,72 @@
package diagnostics
import (
"os"
"path/filepath"
"testing"
)
func TestShouldRetainRunDirectoryMatrix(t *testing.T) {
tests := []struct {
name string
input RetentionDecisionInput
want bool
}{
{name: "always success keeps", input: RetentionDecisionInput{RetentionMode: "always", RunSucceeded: true}, want: true},
{name: "always failure keeps", input: RetentionDecisionInput{RetentionMode: "always", RunSucceeded: false}, want: true},
{name: "never success keeps", input: RetentionDecisionInput{RetentionMode: "never", RunSucceeded: true}, want: true},
{name: "never failure keeps", input: RetentionDecisionInput{RetentionMode: "never", RunSucceeded: false}, want: true},
{name: "auto success no skips removes", input: RetentionDecisionInput{RetentionMode: "auto", RunSucceeded: true, HasSkippedCorrections: false}, want: false},
{name: "auto success skips keeps", input: RetentionDecisionInput{RetentionMode: "auto", RunSucceeded: true, HasSkippedCorrections: true}, want: true},
{name: "auto failure keeps", input: RetentionDecisionInput{RetentionMode: "auto", RunSucceeded: false}, want: true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := ShouldRetainRunDirectory(tc.input)
if got != tc.want {
t.Fatalf("unexpected retain decision: got=%v want=%v", got, tc.want)
}
})
}
}
func TestApplyRetentionRemovesWhenDecisionSaysRemove(t *testing.T) {
workDir := t.TempDir()
runPath := filepath.Join(workDir, "run-test")
if err := os.Mkdir(runPath, 0o755); err != nil {
t.Fatalf("mkdir run path: %v", err)
}
runDir := &RunDirectory{path: runPath, retention: "auto"}
err := runDir.ApplyRetention(RetentionDecisionInput{RunSucceeded: true, HasSkippedCorrections: false})
if err != nil {
t.Fatalf("ApplyRetention failed: %v", err)
}
if _, err := os.Stat(runPath); !os.IsNotExist(err) {
t.Fatalf("expected run directory removed, stat err=%v", err)
}
}
func TestApplyRetentionReturnsRemovalError(t *testing.T) {
parent := t.TempDir()
runPath := filepath.Join(parent, "run-test")
if err := os.Mkdir(runPath, 0o755); err != nil {
t.Fatalf("mkdir run path: %v", err)
}
// Make parent non-writable so removing child fails.
if err := os.Chmod(parent, 0o500); err != nil {
t.Fatalf("chmod parent: %v", err)
}
t.Cleanup(func() {
_ = os.Chmod(parent, 0o700)
})
runDir := &RunDirectory{path: runPath, retention: "auto"}
err := runDir.ApplyRetention(RetentionDecisionInput{RunSucceeded: true, HasSkippedCorrections: false})
if err == nil {
t.Fatalf("expected removal error, got nil")
}
}

View File

@@ -0,0 +1,242 @@
package diagnostics
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"time"
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
// RunDirectory represents a per-run diagnostics directory
type RunDirectory struct {
path string
retention string
createdAt time.Time
}
type RetentionDecisionInput struct {
RetentionMode string
RunSucceeded bool
HasSkippedCorrections bool
}
func ShouldRetainRunDirectory(input RetentionDecisionInput) bool {
// Failed runs are always retained.
if !input.RunSucceeded {
return true
}
switch input.RetentionMode {
case "always":
return true
case "never":
return true
case "auto":
return input.HasSkippedCorrections
default:
// Be conservative for unknown values.
return true
}
}
// InvocationMetadata captures non-secret invocation details for diagnostics.
type InvocationMetadata struct {
Operation string `json:"operation"`
TranscriptPath string `json:"transcript_path"`
GlossaryPath string `json:"glossary_path"`
OutputPath string `json:"output_path,omitempty"`
ReportJSONPath string `json:"report_json_path,omitempty"`
ConfigPath string `json:"config_path,omitempty"`
ConfigSource string `json:"config_source,omitempty"`
ConfigVersion *int `json:"config_version,omitempty"`
TranscriptDescription string `json:"transcript_description,omitempty"`
Modules []string `json:"modules"`
RunID string `json:"run_id"`
StartedAt time.Time `json:"started_at"`
}
// NewRunDirectory creates a new run directory under the configured work dir
func NewRunDirectory(workDir, retention string) (*RunDirectory, error) {
if workDir == "" {
workDir = ".audita-runs"
}
// Create work directory if it doesn't exist
if err := os.MkdirAll(workDir, 0o755); err != nil {
return nil, fmt.Errorf("failed to create work directory %q: %w", workDir, err)
}
// Create a unique run directory identifier.
runID := fmt.Sprintf("run-%d", time.Now().UTC().UnixNano())
runPath := filepath.Join(workDir, runID)
if err := os.Mkdir(runPath, 0o755); err != nil {
return nil, fmt.Errorf("failed to create run directory %q: %w", runPath, err)
}
return &RunDirectory{
path: runPath,
retention: retention,
createdAt: time.Now().UTC(),
}, nil
}
// Path returns the run directory path
func (r *RunDirectory) Path() string {
return r.path
}
func (r *RunDirectory) runID() string {
return filepath.Base(r.path)
}
// WriteInvocationMetadata writes invocation metadata for this run.
func (r *RunDirectory) WriteInvocationMetadata(metadata InvocationMetadata) error {
if metadata.RunID == "" {
metadata.RunID = r.runID()
}
if metadata.StartedAt.IsZero() {
metadata.StartedAt = r.createdAt
}
path := filepath.Join(r.path, ArtifactInvocationMetadata)
bytes, err := json.MarshalIndent(metadata, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal invocation metadata: %w", err)
}
bytes = append(bytes, '\n')
if err := os.WriteFile(path, bytes, 0o644); err != nil {
return fmt.Errorf("failed to write invocation metadata: %w", err)
}
return nil
}
// WriteEffectiveConfig writes redacted effective config metadata for this run.
func (r *RunDirectory) WriteEffectiveConfig(cfg config.Config) error {
path := filepath.Join(r.path, ArtifactEffectiveConfig)
redacted := cfg.Redacted()
bytes, err := json.MarshalIndent(redacted, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal effective config: %w", err)
}
bytes = append(bytes, '\n')
if err := os.WriteFile(path, bytes, 0o644); err != nil {
return fmt.Errorf("failed to write effective config: %w", err)
}
return nil
}
// WriteSourceTranscript writes the source transcript artifact
func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript, raw []byte) error {
// Write raw source for reference
sourcePath := filepath.Join(r.path, ArtifactSourceTranscript)
if err := os.WriteFile(sourcePath, raw, 0o644); err != nil {
return fmt.Errorf("failed to write source transcript: %w", err)
}
// Write parsed source for debugging
parsedPath := filepath.Join(r.path, ArtifactParsedSourceTranscript)
parsedBytes, err := json.MarshalIndent(transcript, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal parsed source transcript: %w", err)
}
parsedBytes = append(parsedBytes, '\n')
if err := os.WriteFile(parsedPath, parsedBytes, 0o644); err != nil {
return fmt.Errorf("failed to write parsed source transcript: %w", err)
}
return nil
}
// WriteNormalizedTranscript writes the normalized transcript artifact
func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript) error {
normalizedPath := filepath.Join(r.path, ArtifactNormalizedTranscript)
bytes, err := schema.TranscriptToJSON(transcript)
if err != nil {
return fmt.Errorf("failed to serialize normalized transcript: %w", err)
}
if err := os.WriteFile(normalizedPath, bytes, 0o644); err != nil {
return fmt.Errorf("failed to write normalized transcript: %w", err)
}
return nil
}
// WriteNormalizationSummary writes the normalization summary artifact
func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.NormalizationSummary) error {
summaryPath := filepath.Join(r.path, ArtifactNormalizationSummary)
bytes, err := json.MarshalIndent(summary, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal normalization summary: %w", err)
}
bytes = append(bytes, '\n')
if err := os.WriteFile(summaryPath, bytes, 0o644); err != nil {
return fmt.Errorf("failed to write normalization summary: %w", err)
}
return nil
}
// WriteReport writes the authoritative report artifact
func (r *RunDirectory) WriteReport(report reporting.ProcessReport) error {
reportPath := filepath.Join(r.path, ArtifactReport)
bytes, err := json.MarshalIndent(report, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal report: %w", err)
}
bytes = append(bytes, '\n')
if err := os.WriteFile(reportPath, bytes, 0o644); err != nil {
return fmt.Errorf("failed to write report: %w", err)
}
return nil
}
// WriteErrorLog writes an error log on failure
func (r *RunDirectory) WriteErrorLog(errorMessage string) error {
errorPath := filepath.Join(r.path, ArtifactErrorLog)
return os.WriteFile(errorPath, []byte(errorMessage+"\n"), 0o644)
}
// WriteChunkingSummary writes the chunking summary artifact
func (r *RunDirectory) WriteChunkingSummary(summary *chunking.DetailedSummary) error {
summaryPath := filepath.Join(r.path, ArtifactChunkingSummary)
bytes, err := json.MarshalIndent(summary, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal chunking summary: %w", err)
}
bytes = append(bytes, '\n')
if err := os.WriteFile(summaryPath, bytes, 0o644); err != nil {
return fmt.Errorf("failed to write chunking summary: %w", err)
}
return nil
}
func (r *RunDirectory) WriteJSONArtifact(name string, payload any) error {
artifactPath := filepath.Join(r.path, name)
bytes, err := json.MarshalIndent(payload, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal %s: %w", name, err)
}
bytes = append(bytes, '\n')
if err := os.WriteFile(artifactPath, bytes, 0o644); err != nil {
return fmt.Errorf("failed to write %s: %w", name, err)
}
return nil
}
func (r *RunDirectory) ApplyRetention(input RetentionDecisionInput) error {
decision := input
if decision.RetentionMode == "" {
decision.RetentionMode = r.retention
}
if ShouldRetainRunDirectory(decision) {
return nil
}
return os.RemoveAll(r.path)
}

29
internal/core/io/files.go Normal file
View File

@@ -0,0 +1,29 @@
package io
import (
"encoding/json"
"fmt"
"os"
)
func ReadRequiredFile(path string, label string) ([]byte, error) {
contents, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("failed to read %s file %q: %w", label, path, err)
}
return contents, nil
}
func ValidateWellFormedJSON(path string, raw []byte) error {
if !json.Valid(raw) {
return fmt.Errorf("transcript file %q is not valid JSON", path)
}
return nil
}
func WriteFile(path string, contents []byte) error {
if err := os.WriteFile(path, contents, 0o644); err != nil {
return fmt.Errorf("failed to write output file %q: %w", path, err)
}
return nil
}

View File

@@ -0,0 +1,35 @@
package modulecatalog
import "strings"
const (
KeyGlossary = "glossary"
KeyHomophones = "homophones"
KeySpokenWord = "spoken_word"
KeyGrammar = "grammar"
)
var supportedKeys = []string{
KeyGlossary,
KeyHomophones,
KeySpokenWord,
KeyGrammar,
}
var supportedKeySet = map[string]struct{}{
KeyGlossary: {},
KeyHomophones: {},
KeySpokenWord: {},
KeyGrammar: {},
}
func SupportedKeys() []string {
out := make([]string, len(supportedKeys))
copy(out, supportedKeys)
return out
}
func IsSupported(key string) bool {
_, ok := supportedKeySet[strings.TrimSpace(key)]
return ok
}

Some files were not shown because too many files have changed in this diff Show More