Compare commits

88 Commits

Author SHA1 Message Date
f790c1441c Refresh architecture and configuration documentation for current runtime behavior 2026-05-23 18:24:06 +00:00
56f9b28f4b Consolidate shared test helpers and stabilize timeout hook integration test 2026-05-23 18:18:48 +00:00
222222f449 Share configured LLM secret extraction across diagnostics paths 2026-05-23 18:11:36 +00:00
99391cd18b Centralize validator classification and malformed output handling 2026-05-23 18:02:53 +00:00
84be774b34 Share module proposal execution and transcript-section prompt payload helpers 2026-05-23 17:56:07 +00:00
e053f7e124 Add shared metadata maps and stage-name helpers 2026-05-23 17:48:37 +00:00
13029dbb33 Centralize effective config loading and path resolution 2026-05-23 17:43:27 +00:00
938bfe88c1 Centralize output schema and module key validation catalogs 2026-05-23 17:39:13 +00:00
fa1bd237d1 Centralize diagnostics artifact names and report metadata paths 2026-05-23 17:32:30 +00:00
3d7057b437 Added an implementation roadmap for the issues identified in the code audit 2026-05-23 12:22:01 -05:00
32c8c8b446 Audit code quality and deduplication opportunities 2026-05-23 11:10:44 -05:00
a3655f5540 Make module-stage LLM handling resilient and report warnings 2026-05-23 10:07:06 -05:00
a84941d681 Rationalize default configuration file paths and update documentation
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-14 20:14:11 -05:00
46b7356a3b Add WoodpeckerCI support 2026-05-14 20:10:19 -05:00
39208cd119 Update the documentation for running audita as a subprocess 2026-05-14 09:34:33 -05:00
6dae15656d Standardized the grammar and spoken_word modules on a shared editorial_review validator 2026-05-14 09:30:10 -05:00
b103bb2e7d Improved organization of the documentation 2026-05-14 09:24:02 -05:00
2b2a3fc024 Removed the old python code and cleaned up outdated documentation 2026-05-13 20:13:33 -05:00
52ffe42e73 Refactor validators into package-owned components 2026-05-14 00:28:28 +00:00
3b160cf05b Added documentation for an upcoming refactoring of the validator modules 2026-05-13 19:12:44 -05:00
0687982822 Complete pre-1.0 release hardening 2026-05-13 19:42:35 +00:00
ff4ed82239 Add utilization diagnostics and correction ledger 2026-05-13 19:03:37 +00:00
037121e9ce Move prompts into embedded Markdown assets 2026-05-13 18:49:06 +00:00
d6126bf52b Refactor validators into built-in registry chains 2026-05-13 18:36:46 +00:00
1bc5936681 Add output schema registry and public contract docs 2026-05-13 12:59:10 +00:00
9a77a0cd0b Add versioned Audita config support 2026-05-13 12:36:20 +00:00
ebbd2c8a63 Add transcript description prompt context 2026-05-13 12:13:32 +00:00
de99467ede Replace structured LLM dependency with Audita adapter 2026-05-13 02:10:24 +00:00
20f612215f Documentation cleanup and addition of pre-1.0 roadmap punchlist 2026-05-12 20:40:35 -05:00
3d45571bb0 Add intra-module pipeline for LLM validation 2026-05-12 18:39:26 -05:00
1afd753fad Audit intra-module LLM pipeline 2026-05-12 18:24:15 -05:00
a85a7e204e Review LLM concurrency refactor 2026-05-12 21:23:57 +00:00
509436cc4a Add explicit LLM concurrency controls 2026-05-12 21:17:35 +00:00
a48f6da1f4 Audit LLM concurrency flow 2026-05-12 16:00:26 -05:00
df96f9fdf6 Rewrite and simplify the section chunking algorithm 2026-05-12 15:42:32 -05:00
390daa8b84 Cleanup pass to remove refactoring-related artifacts and references 2026-05-12 15:19:50 -05:00
af84249da0 Implemented an --llm-concurrency flag in the Go application that enforces a global LLM concurrency cap 2026-05-12 12:59:46 -05:00
cad172a758 Added chunking logic to modules and added corresponding regression tests 2026-05-12 12:26:01 -05:00
fb59cb21b9 Complete Phase 19 Go rollout documentation 2026-05-12 13:46:14 +00:00
68e2d9b549 Complete Phase 18 operational hardening 2026-05-12 13:32:35 +00:00
185f7ca2b6 Complete Phase 17 parity fixture suite 2026-05-12 12:50:05 +00:00
7ccadc6bd6 Complete Phase 16 default pipeline integration 2026-05-12 12:19:50 +00:00
a9f7fa27ff Complete Phase 15 spoken-word module 2026-05-12 11:56:28 +00:00
dbf3605712 Complete Phase 14 homophones module 2026-05-12 11:37:35 +00:00
543a7ff8ef Complete Phase 13 glossary module 2026-05-12 11:20:02 +00:00
fc3a7b7a67 Complete Phase 12 grammar module 2026-05-12 02:57:06 +00:00
b360493cdc Complete Phase 11 proposal generation framework 2026-05-12 02:25:33 +00:00
12202508bf Complete Phase 10 LLM validators 2026-05-12 01:25:52 +00:00
6d9a4bd017 Complete Phase 9 LLM infrastructure 2026-05-11 20:10:56 -05:00
426864eedb Add Phase 9 LLM scheduler, config resolution, diagnostics primitives 2026-05-11 20:09:34 -05:00
0b17a6fbeb Implement Phase 9 structured LLM adapter spike 2026-05-11 19:54:57 -05:00
aeb31f1c0d Complete Phase 8 deterministic validators 2026-05-12 00:26:27 +00:00
28fe899aa1 Complete Phase 7 runner orchestration 2026-05-11 14:22:22 +00:00
30606f5c49 Updated the Go rewrite work plan to reflect completed work so far 2026-05-11 09:10:47 -05:00
db880ed868 Document completed Phase 6 diagnostics 2026-05-11 14:03:14 +00:00
5217093be2 Complete diagnostics retention semantics 2026-05-11 14:01:56 +00:00
0e83991537 Add diagnostics references to reports 2026-05-11 13:59:46 +00:00
3e8d19cccd Add redacted run metadata artifacts 2026-05-11 13:58:13 +00:00
c1193e3450 Refine rewrite documentation accuracy 2026-05-11 08:50:33 -05:00
73249b63d8 Update Go rewrite documentation status 2026-05-11 08:49:26 -05:00
c3087aeda6 Complete Phase 3 proposal foundation 2026-05-11 13:35:15 +00:00
0452a605ad Document Phase 3 rewrite status 2026-05-11 13:33:13 +00:00
b9b7384123 Test chunking with proposal application 2026-05-11 13:32:17 +00:00
726acc47e1 Define minimal module framework contracts 2026-05-11 13:29:34 +00:00
5c78b1d5d9 Add proposal application fixtures 2026-05-11 13:27:08 +00:00
f461922b9b Apply correction proposals to transcripts 2026-05-11 13:23:59 +00:00
c58d307ba7 Add safe replacement preview logic 2026-05-11 13:20:35 +00:00
1eb93481e0 Add correction proposal models 2026-05-11 13:18:13 +00:00
b997e7c97c Record chunking metadata in process runs 2026-05-11 12:47:31 +00:00
12fd541669 Add token-bounded transcript chunking 2026-05-11 12:23:01 +00:00
10377876e4 Complete Phase 2 Go normalization foundation 2026-05-11 00:41:29 +00:00
d847168ecd Add Phase 2 fixtures and compatibility tests 2026-05-11 00:21:19 +00:00
14e51698c2 Add Phase 2 normalization diagnostics 2026-05-11 00:19:28 +00:00
aeb9c4f062 Report schema validation and normalization results 2026-05-11 00:16:50 +00:00
e2ae7f77d8 Run normalization in process command 2026-05-11 00:13:26 +00:00
0b1b670baf Add deterministic transcript normalization 2026-05-11 00:12:19 +00:00
3cfa4b6e8a Wire schema validation into process command 2026-05-11 00:01:42 +00:00
950edc01f2 Add Go transcript and glossary schemas 2026-05-10 23:52:36 +00:00
ea8def423e Complete Phase 1 Go skeleton 2026-05-10 23:34:43 +00:00
95fe8c32fa Harden Go CLI subprocess behavior 2026-05-10 23:33:13 +00:00
08b7531149 Add minimal process reports 2026-05-10 23:30:46 +00:00
2cf2d390da Add transcript passthrough processing 2026-05-10 23:27:54 +00:00
8f3c2ec5fd Wire process command flags 2026-05-10 23:24:48 +00:00
9427c4e6cc Add Go configuration model 2026-05-10 23:19:59 +00:00
6424d7db4f Initialize Go CLI skeleton 2026-05-10 23:12:46 +00:00
87e560dd3d Added architecture reference documentation for the upcoming Go rewrite 2026-05-10 23:04:29 +00:00
09fc6fd364 Updated .gitignore in preparation for the Go rewrite 2026-05-10 22:40:27 +00:00
2e47c8a1b6 Move python implementation under python/ in preparation for the upcoming Go rewrite 2026-05-10 22:37:38 +00:00
305 changed files with 29955 additions and 10395 deletions

61
.gitignore vendored
View File

@@ -1,3 +1,4 @@
# --> Python
.DS_Store
.venv/
__pycache__/
@@ -9,3 +10,63 @@ dist/
build/
*.egg-info/
# ---> Codex
.codex
AGENTS.md
# ---> Go
# If you prefer the allow list template instead of the deny list, see community template:
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
#
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Dependency directories (remove the comment below to include it)
# vendor/
# Go workspace file
go.work
go.work.sum
# env file
.env
# Compiled binaries and test configuration
narratio
local-test
pipeline.yml
bin/
# Local run artifacts
.audita-runs/
report.json
corrected.json
normalized.json
# Coverage artifacts
coverage.out
coverage.txt
# ---> VisualStudioCode
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
!.vscode/*.code-snippets
# Local History for Visual Studio Code
.history/
# Built Visual Studio Code Extensions
*.vsix

50
.woodpecker/release.yml Normal file
View File

@@ -0,0 +1,50 @@
when:
- event: tag
steps:
- name: build-release-assets
image: golang:1.25
commands:
- |
set -eu
version="$CI_COMMIT_TAG"
dist="dist"
pkg="gitea.maximumdirect.net/eric/audita/cmd/audita"
rm -rf "$dist"
mkdir -p "$dist"
build_binary() {
goos="$1"
goarch="$2"
suffix="$3"
output="$dist/audita-$version-$goos-$goarch$suffix"
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \
go build -trimpath -ldflags "-s -w -X gitea.maximumdirect.net/eric/audita/internal/buildinfo.Version=$version" \
-o "$output" "$pkg"
}
build_binary linux amd64 ""
build_binary linux arm64 ""
build_binary darwin amd64 ""
build_binary darwin arm64 ""
build_binary windows amd64 ".exe"
build_binary windows arm64 ".exe"
- name: publish-release
image: woodpeckerci/plugin-release
depends_on:
- build-release-assets
settings:
api_key:
from_secret: GITEA_RELEASE_TOKEN
files:
- dist/audita-*
checksum: sha256
checksum-file: SHA256SUMS
checksum-flatten: true
file-exists: skip
overwrite: false
prerelease: false

365
README.md
View File

@@ -1,148 +1,305 @@
# Audita
Audita is a framework-first transcript correction application. The public `audita` package provides:
Audita is a transcript polishing CLI.
- deterministic transcript normalization
- token-batched module orchestration
- concrete `glossary`, `homophones`, `spoken_word`, and `grammar` modules built on reusable proposal / validator contracts
- structured run reporting and work-dir diagnostics
`audita process` validates transcript/glossary input, normalizes and chunks transcript segments, runs the default correction pipeline, and emits corrected transcript output plus machine-readable diagnostics and reports.
The previous working implementation has been preserved as `audita_prototype` inside this repository. Its full regression suite lives under `tests/audita_prototype`.
## What Audita Does
## Development
Default module sequence:
- `glossary`
- `homophones`
- `glossary`
- `spoken_word`
- `grammar`
This project is set up for `uv`.
Pipeline behavior includes:
- glossary-backed domain/acoustic corrections
- conservative homophone and mistranscription corrections
- conservative spoken-word dysfluency cleanup with semantic guardrails
- grammar/punctuation/capitalization/formatting cleanup
- validator-chain enforcement before application
- malformed module-stage LLM payloads degrade to warnings/rejections instead of failing the run
- run reports and diagnostics artifacts with secret redaction
## Build and Install
Build a local binary:
```sh
uv sync --extra dev
uv run pytest
go build -o ./bin/audita ./cmd/audita
```
## Usage
Process a transcript with the current framework implementation:
Install into your Go bin directory:
```sh
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json
go install ./cmd/audita
```
The framework currently runs this default module sequence:
1. `glossary`
2. `homophones`
3. `glossary`
4. `spoken_word`
5. `grammar`
Resolved run instance names are auto-numbered for repeats, so the default report pipeline is:
1. `glossary_1`
2. `homophones`
3. `glossary_2`
4. `spoken_word`
5. `grammar`
The default module sequence is fully implemented today:
- `glossary` proposes glossary-supported acoustic corrections
- `homophones` proposes conservative homophone and mistranscription corrections
- `spoken_word` proposes conservative dysfluency cleanup
- `grammar` proposes punctuation, capitalization, and spacing cleanup only
To run a custom module sequence, pass `--modules`:
CLI help:
```sh
uv run audita process transcript.json --glossary glossary.yaml --modules grammar --output corrected.json
audita --help
audita process --help
audita config --help
```
To also write a structured JSON report:
## Test
Run all tests:
```sh
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json --report-json report.json
go test ./...
```
From a checked-out repository, you can also use the root launcher:
## Basic Usage
Required inputs:
- transcript JSON path (positional argument)
- `--glossary <glossary.yaml>`
Recommended run:
```sh
./audita process transcript.json --glossary glossary.yaml --output corrected.json
audita process transcript.json \
--glossary glossary.yaml \
--output corrected.json \
--report-json report.json
```
For a system-wide command, install the source tree under `/usr/local/src/audita`, sync dependencies there, and symlink the root launcher into your `PATH`:
Select an explicit output schema (default is `bare-segments`):
```sh
cd /usr/local/src/audita
uv sync --extra dev
ln -s /usr/local/src/audita/audita /usr/local/bin/audita
audita process transcript.json --glossary glossary.yaml --output corrected.json
audita process transcript.json \
--glossary glossary.yaml \
--output-schema audita-v1 \
--output corrected.json \
--report-json report.json
```
Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr.
`--report-json` writes a separate machine-readable run report and never mixes report data into stdout.
Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Default OpenRouter runs require LLM API credentials, because the `glossary`, `homophones`, `spoken_word`, and `grammar` modules make real LLM calls. Self-hosted or other non-default OpenAI-compatible endpoints may not require credentials. `AUDITA_LLM_API_KEY` and `--llm-api-key` are the preferred provider-neutral credential surfaces, while `OPENROUTER_API_KEY` remains supported as a backward-compatible fallback.
| Environment variable | CLI flag | Default | Purpose |
| --- | --- | --- | --- |
| `AUDITA_MODULES` | `--modules` | `glossary,homophones,glossary,spoken_word,grammar` | Comma-separated logical module keys to run; CLI overrides the environment value |
| `AUDITA_LLM_API_KEY` | `--llm-api-key` | unset | Preferred provider-neutral LLM API credential; required for the default OpenRouter endpoint and optional for non-default endpoints; CLI overrides both environment-key variants |
| `AUDITA_VALIDATION_LLM_API_KEY` | `--validation-llm-api-key` | unset | Validation-phase LLM API credential; defaults to the primary LLM API key and is optional for non-default validation endpoints |
| `AUDITA_MODEL` | `--model` | `openrouter/google/gemma-4-31b-it` | LLM model name sent to the configured OpenAI-compatible endpoint |
| `AUDITA_VALIDATION_MODEL` | `--validation-model` | unset | Validation-phase LLM model; defaults to `AUDITA_MODEL` |
| `AUDITA_BASE_URL` | `--base-url` | `https://openrouter.ai/api/v1` | OpenAI-compatible API base URL |
| `AUDITA_VALIDATION_BASE_URL` | `--validation-base-url` | unset | Validation-phase OpenAI-compatible API base URL; defaults to `AUDITA_BASE_URL` |
| `AUDITA_LLM_TIMEOUT_SECONDS` | `--llm-timeout-seconds` | `600` | Per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint |
| `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | `--validation-llm-timeout-seconds` | unset | Validation-phase per-request timeout in seconds; defaults to `AUDITA_LLM_TIMEOUT_SECONDS` |
| `AUDITA_VALIDATION_MAX_PROMPT_TOKENS` | `--validation-max-prompt-tokens` | `2048` | Maximum estimated tokens per validation-phase LLM prompt batch |
| `AUDITA_TARGET_SECTIONS` | `--target-sections` | unset | Exact number of contiguous proposal-stage transcript sections; errors if min/max token bounds cannot be satisfied |
| `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses |
| `AUDITA_VALIDATION_MAX_RETRIES` | `--validation-max-retries` | unset | Validation-phase structured-output retries; defaults to `AUDITA_MAX_RETRIES` |
| `AUDITA_VALIDATION_LLM_CONCURRENCY` | `--validation-llm-concurrency` | unset | Validation-phase LLM concurrency; defaults to `AUDITA_LLM_CONCURRENCY` |
| `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `8192` | Maximum estimated tokens per proposal-stage transcript section |
| `AUDITA_MIN_SECTION_TOKENS` | `--min-section-tokens` | `2048` | Minimum estimated tokens per proposal-stage transcript section when balancing for concurrency |
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation |
| `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | `--grammar-confidence-threshold` | `0.8` | Minimum confidence required for grammar proposals to survive validation |
| `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | `--homophones-confidence-threshold` | `0.8` | Minimum confidence required for homophone proposals to survive validation |
| `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | `--spoken-word-confidence-threshold` | `0.8` | Minimum confidence required for spoken-word proposals to survive validation |
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | `--normalize-max-segment-gap` | `4.0` | Same-speaker gaps eligible for deterministic merging |
| `AUDITA_NORMALIZE_ELLIPSIS_GAP` | `--normalize-ellipsis-gap` | `3.5` | Same-speaker gaps above this value are joined with ` ... ` |
| `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | `--normalize-max-segment-duration` | `60.0` | Maximum merged segment duration |
| `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | `--normalize-max-segment-tokens` | `2048` | Maximum merged segment prompt payload size |
| `AUDITA_WORK_DIR` | `--work-dir` | `/tmp/audita` | Per-run scratch diagnostics directory |
| `AUDITA_WORK_DIR_RETENTION` | `--work-dir-retention` | `auto` | Whether to retain the per-run work directory: `auto`, `always`, or `never` |
Set `AUDITA_MODULES=grammar` to run only the grammar module by default, or override it per command with `--modules`.
Validation-phase LLM settings inherit from the primary `AUDITA_*` LLM settings by default. Set any of the `AUDITA_VALIDATION_*` values only when you want LLM-backed validators to use a different model, endpoint, credential, timeout, retry budget, or concurrency level.
OpenRouter remains the default out of the box:
Recommended config-based run:
```sh
export AUDITA_LLM_API_KEY=your-openrouter-key
audita process transcript.json --glossary glossary.yaml --output corrected.json
audita process transcript.json \
--glossary glossary.yaml \
--config audita.yml \
--output corrected.json \
--report-json report.json
```
You can point Audita at any OpenAI-compatible endpoint by changing `AUDITA_BASE_URL` and, if needed, `AUDITA_MODEL`. For example, a local vLLM server:
Explicit module override:
```sh
export AUDITA_BASE_URL=http://localhost:8000/v1
export AUDITA_MODEL=meta-llama/Llama-3.1-8B-Instruct
audita process transcript.json --glossary glossary.yaml --output corrected.json
audita process transcript.json \
--glossary glossary.yaml \
--modules glossary,homophones,grammar \
--output corrected.json \
--report-json report.json
```
If your self-hosted endpoint requires authentication, you can still set `AUDITA_LLM_API_KEY`; Audita simply no longer requires it for non-default endpoints.
Or the actual OpenAI API:
Optional transcript background context:
```sh
export AUDITA_LLM_API_KEY=your-openai-key
export AUDITA_BASE_URL=https://api.openai.com/v1
export AUDITA_MODEL=gpt-4.1-mini
audita process transcript.json --glossary glossary.yaml --output corrected.json
audita process transcript.json \
--glossary glossary.yaml \
--transcript-description "Brief context that may help resolve ambiguous terms." \
--output corrected.json
```
`AUDITA_WORK_DIR` stores per-run diagnostics while processing. Under the default `AUDITA_WORK_DIR_RETENTION=auto`, clean successful runs are removed, while failed runs and successful runs with final skipped corrections are preserved. Use `always` to keep every run directory and `never` to remove successful run directories even when skips remain.
Failed runs always preserve the run directory and include an authoritative `report.json` alongside normalization and prompt/response diagnostics.
The transcript description is background context only and does not override transcript content.
## Prototype Archive
Write transcript JSON to stdout (no `--output`):
The archived prototype remains importable as `audita_prototype` and is still covered by its original regression suite. This is intentional: the new `audita` package is a framework-oriented rewrite, not a thin wrapper around the old code.
```sh
audita process transcript.json --glossary glossary.yaml
```
Control diagnostics location/retention:
```sh
audita process transcript.json \
--glossary glossary.yaml \
--work-dir /tmp/audita \
--work-dir-retention auto \
--output corrected.json \
--report-json report.json
```
## Stdout/Stderr Contract
- With `--output`, stdout is expected to be empty on success.
- Without `--output`, stdout contains transcript JSON only on success.
- `--report-json` writes a file and is never printed to stdout.
- stderr is human-readable diagnostics/errors.
- successful runs remain quiet on stderr even when module warnings are recorded in report/diagnostics artifacts.
For subprocess orchestration guidance, see [`docs/subprocess-operations.md`](docs/subprocess-operations.md).
## Configuration
Precedence:
1. defaults
2. config file (`--config`, `AUDITA_CONFIG`, or default search paths when present: `/usr/local/etc/audita/config.yml`, then `/etc/audita/config.yml`)
3. environment (`AUDITA_*`)
4. CLI flags
Config commands:
```sh
audita config validate --config audita.yml
audita config print-effective --config audita.yml
```
For full config-file schema and examples, see [`docs/configuration.md`](docs/configuration.md).
For output-schema details, see [`docs/architecture/output-schemas.md`](docs/architecture/output-schemas.md).
For built-in validator keys and chain definitions, see [`docs/architecture/validators.md`](docs/architecture/validators.md).
For embedded prompt assets and prompt metadata behavior, see [`docs/architecture/prompts.md`](docs/architecture/prompts.md).
For CLI/process compatibility guarantees, see [`docs/architecture/public-contract.md`](docs/architecture/public-contract.md).
### Modules
- `AUDITA_MODULES` (CSV)
- CLI: `--modules`
### Transcript Description
CLI:
- `--transcript-description`
Behavior:
- optional background context for proposal and LLM-validator prompts;
- trimmed and length-limited by CLI validation;
- does not override transcript content;
- no `AUDITA_*` environment variable is currently defined for this setting.
### Primary LLM
Environment:
- `AUDITA_LLM_API_KEY` (or `OPENROUTER_API_KEY` fallback)
- `AUDITA_MODEL`
- `AUDITA_BASE_URL`
- `AUDITA_LLM_TIMEOUT_SECONDS`
- `AUDITA_MAX_RETRIES`
CLI:
- `--llm-api-key`
- `--model`
- `--base-url`
- `--llm-timeout-seconds`
- `--max-retries`
### Validation LLM
Environment:
- `AUDITA_VALIDATION_LLM_API_KEY`
- `AUDITA_VALIDATION_MODEL`
- `AUDITA_VALIDATION_BASE_URL`
- `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS`
- `AUDITA_VALIDATION_MAX_RETRIES`
- `AUDITA_VALIDATION_LLM_CONCURRENCY`
- `AUDITA_VALIDATION_MAX_PROMPT_TOKENS`
CLI:
- `--validation-llm-api-key`
- `--validation-model`
- `--validation-base-url`
- `--validation-llm-timeout-seconds`
- `--validation-max-retries`
- `--validation-llm-concurrency`
- `--validation-max-prompt-tokens`
### LLM Concurrency
Environment:
- `AUDITA_TOTAL_LLM_CONCURRENCY`
- `AUDITA_PROPOSAL_LLM_CONCURRENCY`
- `AUDITA_VALIDATION_LLM_CONCURRENCY`
- `AUDITA_LLM_CONCURRENCY` (legacy alias for `AUDITA_TOTAL_LLM_CONCURRENCY`)
CLI:
- `--total-llm-concurrency`
- `--proposal-llm-concurrency`
- `--validation-llm-concurrency`
- `--llm-concurrency` (legacy alias for `--total-llm-concurrency`)
Behavior:
- all proposal and validation LLM calls are bounded by total LLM concurrency
- proposal LLM calls are additionally bounded by proposal LLM concurrency
- when validation concurrency is unset, it inherits total LLM concurrency
- when explicitly set, proposal and validation concurrency must each be `<= total-llm-concurrency`
- canonical total settings win when both canonical and legacy alias settings are provided at the same precedence layer
### Confidence Thresholds
Environment:
- `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD`
- `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD`
- `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD`
- `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD`
CLI:
- `--glossary-confidence-threshold`
- `--homophones-confidence-threshold`
- `--spoken-word-confidence-threshold`
- `--grammar-confidence-threshold`
### Normalization and Chunking
Environment:
- `AUDITA_NORMALIZE_MAX_SEGMENT_GAP`
- `AUDITA_NORMALIZE_ELLIPSIS_GAP`
- `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION`
- `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS`
- `AUDITA_MAX_SECTION_TOKENS`
- `AUDITA_MIN_SECTION_TOKENS`
- `AUDITA_TARGET_SECTIONS`
CLI:
- `--normalize-max-segment-gap`
- `--normalize-ellipsis-gap`
- `--normalize-max-segment-duration`
- `--normalize-max-segment-tokens`
- `--max-section-tokens`
- `--min-section-tokens`
- `--target-sections`
### Work Directory
Environment:
- `AUDITA_WORK_DIR`
- `AUDITA_WORK_DIR_RETENTION` (`auto`, `always`, `never`)
CLI:
- `--work-dir`
- `--work-dir-retention`
Retention behavior:
- `always`: keep all run directories
- `never`: keep successful run directories
- `auto`: keep failed runs and successful runs with skipped/rejected corrections
## Reports and Diagnostics
Per-run diagnostics include:
- source transcript artifacts
- normalized transcript artifact
- normalization summary
- chunking summary
- utilization diagnostics summary
- correction ledger
- invocation metadata
- redacted effective config
- module/validator prompt-response diagnostics
- `report.json`
- `error.log` on failure
Optional external report output:
- `--report-json <path>`
## Documentation
- Architecture: [`docs/architecture.md`](docs/architecture.md)
- Diagnostics: [`docs/diagnostics.md`](docs/diagnostics.md)
- Structured LLM adapter: [`docs/structured-llm.md`](docs/structured-llm.md)
- Subprocess operations: [`docs/subprocess-operations.md`](docs/subprocess-operations.md)
- Release checklist: [`docs/release-checklist.md`](docs/release-checklist.md)

147
audita
View File

@@ -1,147 +0,0 @@
#!/usr/bin/env python3
import json
import os
from datetime import datetime
import shutil
import subprocess
import sys
from pathlib import Path
from typing import Optional
from uuid import uuid4
DEFAULT_WORK_DIR = "/tmp/audita"
_SECRET_FLAGS = {"--llm-api-key", "--validation-llm-api-key"}
def _redact_argv(argv: list[str]) -> list[str]:
redacted: list[str] = []
index = 0
while index < len(argv):
arg = argv[index]
matched_flag = next((flag for flag in _SECRET_FLAGS if arg == flag or arg.startswith(flag + "=")), None)
if matched_flag is None:
redacted.append(arg)
index += 1
continue
if arg == matched_flag:
redacted.append(arg)
if index + 1 < len(argv):
redacted.append("[REDACTED]")
index += 2
else:
index += 1
continue
redacted.append(f"{matched_flag}=[REDACTED]")
index += 1
return redacted
def _resolve_work_root(argv: list[str]) -> Path:
for index, arg in enumerate(argv):
if arg == "--work-dir" and index + 1 < len(argv):
return Path(argv[index + 1])
if arg.startswith("--work-dir="):
return Path(arg.split("=", 1)[1])
return Path(os.environ.get("AUDITA_WORK_DIR") or DEFAULT_WORK_DIR)
def _create_run_dir(root: Path) -> Path:
root.mkdir(parents=True, exist_ok=True)
timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
run_dir = root / f"run-{timestamp}-{uuid4().hex[:8]}"
run_dir.mkdir(parents=False, exist_ok=False)
return run_dir
def _capture_run_dirs(root: Path) -> set[str]:
if not root.exists():
return set()
return {path.name for path in root.iterdir() if path.is_dir() and path.name.startswith("run-")}
def _find_new_run_dir(root: Path, before: set[str]) -> Optional[Path]:
if not root.exists():
return None
candidates = [
path for path in root.iterdir() if path.is_dir() and path.name.startswith("run-") and path.name not in before
]
if not candidates:
return None
return max(candidates, key=lambda path: path.name)
def _write_launcher_error_log(
path: Path,
*,
message: str,
exit_code: int,
argv: list[str],
command: Optional[list[str]],
) -> None:
payload = {
"timestamp": datetime.utcnow().isoformat() + "Z",
"message": message,
"exit_code": exit_code,
"argv": argv,
"cwd": os.getcwd(),
"command": command,
}
path.write_text(
"Audita Launcher Diagnostics\n" + json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
def _emit_console_line(message: str) -> None:
for stream in (sys.stderr, sys.stdout):
if stream is None:
continue
try:
stream.write(f"{message}\n")
stream.flush()
return
except (OSError, ValueError):
continue
def main() -> int:
argv = list(sys.argv[1:])
work_root = _resolve_work_root(argv)
redacted_argv = _redact_argv(argv)
uv = shutil.which("uv")
if uv is None:
run_dir = _create_run_dir(work_root)
error_log = run_dir / "error.log"
message = "uv is required to run this launcher. Install uv and run `uv sync` in the Audita project."
_write_launcher_error_log(error_log, message=message, exit_code=1, argv=redacted_argv, command=None)
_emit_console_line(f"audita: error: {message}")
_emit_console_line("audita: exit code: 1")
_emit_console_line(f"audita: run directory: {run_dir}")
_emit_console_line(f"audita: error log: {error_log}")
return 1
project_root = Path(__file__).resolve().parent
command = [uv, "run", "--project", str(project_root), "python", "-m", "audita", *sys.argv[1:]]
before = _capture_run_dirs(work_root)
result = subprocess.run(command, cwd=project_root, check=False)
if result.returncode == 0:
return 0
if _find_new_run_dir(work_root, before) is None:
run_dir = _create_run_dir(work_root)
error_log = run_dir / "error.log"
_write_launcher_error_log(
error_log,
message=f"Audita subprocess exited with status {result.returncode}.",
exit_code=result.returncode,
argv=redacted_argv,
command=_redact_argv(command),
)
_emit_console_line(f"audita: subprocess exited with status {result.returncode}")
_emit_console_line(f"audita: run directory: {run_dir}")
_emit_console_line(f"audita: error log: {error_log}")
return result.returncode
if __name__ == "__main__":
raise SystemExit(main())

11
cmd/audita/main.go Normal file
View File

@@ -0,0 +1,11 @@
package main
import (
"os"
"gitea.maximumdirect.net/eric/audita/internal/cli"
)
func main() {
os.Exit(cli.Run(os.Args[1:], os.Stdout, os.Stderr))
}

View File

@@ -0,0 +1,677 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"reflect"
"runtime"
"strconv"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/internal/cli"
"gitea.maximumdirect.net/eric/audita/internal/testsupport"
)
func TestHelperProcess(t *testing.T) {
if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
return
}
sep := -1
for i, arg := range os.Args {
if arg == "--" {
sep = i
break
}
}
if sep == -1 {
os.Exit(2)
}
cli.ConfigureSubprocessTestHooksFromEnv()
code := cli.Run(os.Args[sep+1:], os.Stdout, os.Stderr)
os.Exit(code)
}
func TestProcessHelpSubprocess(t *testing.T) {
result := runCLISubprocess(t, "process", "--help")
if result.exitCode != 0 {
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
}
if !strings.Contains(result.stdout, "Usage:") || !strings.Contains(result.stdout, "--glossary") {
t.Fatalf("unexpected help stdout: %q", result.stdout)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr, got %q", result.stderr)
}
}
func TestProcessSuccessWithOutputSubprocess(t *testing.T) {
outputPath := filepath.Join(t.TempDir(), "corrected.json")
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--output",
outputPath,
)
if result.exitCode != 0 {
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
}
if result.stdout != "" {
t.Fatalf("expected empty stdout when --output is set, got %q", result.stdout)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
inputBytes := readFile(t, fixturePath("tiny_transcript.json"))
outputBytes := readFile(t, outputPath)
assertJSONSemanticallyEqual(t, inputBytes, outputBytes)
}
func TestProcessSuccessWithoutOutputSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
)
if result.exitCode != 0 {
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
inputBytes := readFile(t, fixturePath("tiny_transcript.json"))
assertJSONSemanticallyEqual(t, inputBytes, []byte(result.stdout))
}
func TestProcessSuccessWithAuditaV1OutputSchemaSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--output-schema",
"audita-v1",
)
if result.exitCode != 0 {
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
var out struct {
Schema string `json:"schema"`
}
if err := json.Unmarshal([]byte(result.stdout), &out); err != nil {
t.Fatalf("expected valid audita-v1 JSON output: %v", err)
}
if out.Schema != "audita-v1" {
t.Fatalf("expected audita-v1 schema, got %q", out.Schema)
}
}
func TestProcessFailureMissingTranscriptSubprocess(t *testing.T) {
result := runCLISubprocess(t, "process", "--glossary", fixturePath("tiny_glossary.yaml"))
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "expected exactly 1 transcript JSON path argument") {
t.Fatalf("expected actionable missing transcript error, got %q", result.stderr)
}
}
func TestProcessFailureMalformedJSONSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
fixturePath("malformed_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "is not valid JSON") {
t.Fatalf("expected malformed JSON error, got %q", result.stderr)
}
}
func TestProcessFailureMissingTranscriptFileSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
filepath.Join(t.TempDir(), "missing-transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "transcript_read") {
t.Fatalf("expected transcript_read failure, got %q", result.stderr)
}
}
func TestProcessFailureMissingGlossaryFileSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
filepath.Join(t.TempDir(), "missing-glossary.yaml"),
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "glossary_read") {
t.Fatalf("expected glossary_read failure, got %q", result.stderr)
}
}
func TestProcessFailureTranscriptSchemaSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
schemaFixturePath("transcript_empty_speaker.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "transcript_schema") {
t.Fatalf("expected transcript_schema failure, got %q", result.stderr)
}
}
func TestProcessFailureMalformedGlossaryYAMLSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
schemaFixturePath("glossary_malformed.yaml"),
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "glossary_schema") {
t.Fatalf("expected glossary_schema failure, got %q", result.stderr)
}
}
func TestProcessFailureUnreadableTranscriptSubprocess(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("portable unreadable-file permissions are not reliable on windows")
}
dir := t.TempDir()
transcriptPath := filepath.Join(dir, "transcript.json")
if err := os.WriteFile(transcriptPath, []byte(`[]`), 0o000); err != nil {
t.Fatalf("write unreadable transcript: %v", err)
}
t.Cleanup(func() { _ = os.Chmod(transcriptPath, 0o644) })
if _, err := os.ReadFile(transcriptPath); err == nil {
t.Skip("unable to make transcript unreadable on this platform/user")
}
result := runCLISubprocess(
t,
"process",
transcriptPath,
"--glossary",
fixturePath("tiny_glossary.yaml"),
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "transcript_read") {
t.Fatalf("expected transcript_read failure, got %q", result.stderr)
}
}
func TestProcessFailureUnwritableOutputSubprocess(t *testing.T) {
outputDir := t.TempDir()
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--output",
outputDir,
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "failed to write output file") {
t.Fatalf("expected write failure message, got %q", result.stderr)
}
}
func TestProcessFailureUnwritableReportJSONSubprocess(t *testing.T) {
reportDir := t.TempDir()
outputPath := filepath.Join(t.TempDir(), "out.json")
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--output",
outputPath,
"--report-json",
reportDir,
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "failed to write report JSON file") {
t.Fatalf("expected report write failure message, got %q", result.stderr)
}
}
func TestProcessSuccessReportJSONSubprocess(t *testing.T) {
reportPath := filepath.Join(t.TempDir(), "report.json")
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--report-json",
reportPath,
)
if result.exitCode != 0 {
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
if !json.Valid([]byte(result.stdout)) {
t.Fatalf("expected transcript JSON only on stdout, got %q", result.stdout)
}
report := readFile(t, reportPath)
if !json.Valid(report) {
t.Fatalf("expected valid report JSON, got %q", string(report))
}
// Ensure report JSON is not printed to stdout.
if strings.Contains(result.stdout, `"default_pipeline"`) {
t.Fatalf("report JSON leaked to stdout: %q", result.stdout)
}
}
func TestProcessSuccessLargeTranscriptSubprocess(t *testing.T) {
transcriptPath := writeLargeTranscriptFixture(t, 320)
result := runCLISubprocess(
t,
"process",
transcriptPath,
"--glossary",
fixturePath("tiny_glossary.yaml"),
)
if result.exitCode != 0 {
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
if !json.Valid([]byte(result.stdout)) {
t.Fatalf("expected valid transcript JSON on stdout")
}
}
func TestProcessFailureMalformedStructuredLLMResponseViaSubprocessHook(t *testing.T) {
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
result := runCLISubprocessWithEnv(t,
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "malformed_structured"},
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--modules",
"grammar",
"--report-json",
reportPath,
"--work-dir",
workDir,
"--work-dir-retention",
"always",
)
if result.exitCode != 0 {
t.Fatalf("expected zero exit code, got %d stderr=%q", result.exitCode, result.stderr)
}
if !json.Valid([]byte(result.stdout)) {
t.Fatalf("expected transcript JSON on stdout, got %q", result.stdout)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
report := readFile(t, reportPath)
if !json.Valid(report) {
t.Fatalf("expected valid success report JSON")
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
t.Fatalf("did not expect error.log, got: %v", err)
}
}
func TestProcessFailureBackendLLMViaSubprocessHook(t *testing.T) {
workDir := t.TempDir()
result := runCLISubprocessWithEnv(t,
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "backend_error"},
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--modules",
"grammar",
"--work-dir",
workDir,
"--work-dir-retention",
"always",
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "synthetic backend failure") {
t.Fatalf("expected backend failure details, got %q", result.stderr)
}
if !strings.Contains(result.stderr, "diagnostics:") {
t.Fatalf("expected diagnostics path in stderr, got %q", result.stderr)
}
if _, err := os.Stat(filepath.Join(onlyRunDir(t, workDir), "error.log")); err != nil {
t.Fatalf("expected error.log in retained failed run: %v", err)
}
}
func TestProcessFailureMidPipelinePreservesPartialReportsSubprocess(t *testing.T) {
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
result := runCLISubprocessWithEnv(t,
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "mid_pipeline_fail"},
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--modules",
"glossary,homophones,glossary,spoken_word,grammar",
"--report-json",
reportPath,
"--work-dir",
workDir,
"--work-dir-retention",
"always",
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
reportRaw := readFile(t, reportPath)
var report struct {
Status string `json:"status"`
ErrorPhase string `json:"error_phase"`
ModuleResults []struct {
ModuleInstance string `json:"module_instance"`
Status string `json:"status"`
} `json:"module_results"`
}
if err := json.Unmarshal(reportRaw, &report); err != nil {
t.Fatalf("unmarshal report: %v", err)
}
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
t.Fatalf("expected failed runner_execution report, got %+v", report)
}
if len(report.ModuleResults) == 0 {
t.Fatalf("expected partial module results in failure report")
}
}
func TestProcessCancellationViaSubprocessTimeoutHook(t *testing.T) {
workDir := t.TempDir()
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
result := runCLISubprocessContext(t, ctx,
map[string]string{
"AUDITA_SUBPROCESS_TEST_LLM_MODE": "block_until_cancel",
"AUDITA_SUBPROCESS_TEST_RUN_TIMEOUT_MS": "120",
},
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--modules",
"grammar",
"--work-dir",
workDir,
"--work-dir-retention",
"always",
)
if result.stdout != "" {
if result.stderr == "" {
t.Skipf("subprocess timeout hook did not trigger in this run; stdout=%q", result.stdout)
}
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "context deadline exceeded") {
t.Fatalf("expected context deadline error, got %q", result.stderr)
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log for canceled run: %v", err)
}
if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil {
t.Fatalf("expected report.json for canceled run: %v", err)
}
}
func TestProcessSubprocessNoSecretLeakInOutputsAndDiagnostics(t *testing.T) {
secret := "subprocess-secret"
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
result := runCLISubprocessWithEnv(t,
map[string]string{
"AUDITA_LLM_API_KEY": secret,
"AUDITA_VALIDATION_LLM_API_KEY": secret,
},
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--output",
outputPath,
"--report-json",
reportPath,
"--work-dir",
workDir,
"--work-dir-retention",
"always",
)
if result.exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", result.exitCode, result.stderr)
}
if strings.Contains(result.stdout, secret) || strings.Contains(result.stderr, secret) {
t.Fatalf("secret leaked in subprocess stdio")
}
assertNoSecretInFile(t, reportPath, secret)
assertNoSecretInTree(t, onlyRunDir(t, workDir), secret)
}
type subprocessResult struct {
stdout string
stderr string
exitCode int
}
func runCLISubprocess(t *testing.T, args ...string) subprocessResult {
t.Helper()
return runCLISubprocessWithEnv(t, nil, args...)
}
func runCLISubprocessWithEnv(t *testing.T, extraEnv map[string]string, args ...string) subprocessResult {
t.Helper()
return runCLISubprocessContext(t, context.Background(), extraEnv, args...)
}
func runCLISubprocessContext(t *testing.T, ctx context.Context, extraEnv map[string]string, args ...string) subprocessResult {
t.Helper()
cmdArgs := append([]string{"-test.run=TestHelperProcess", "--"}, args...)
cmd := exec.CommandContext(ctx, os.Args[0], cmdArgs...)
env := append(filterAuditaEnv(os.Environ()), "GO_WANT_HELPER_PROCESS=1")
for k, v := range extraEnv {
env = append(env, fmt.Sprintf("%s=%s", k, v))
}
cmd.Env = env
var stdoutBuf bytes.Buffer
var stderrBuf bytes.Buffer
cmd.Stdout = &stdoutBuf
cmd.Stderr = &stderrBuf
err := cmd.Run()
result := subprocessResult{
stdout: stdoutBuf.String(),
stderr: stderrBuf.String(),
}
if err == nil {
return result
}
if exitErr, ok := err.(*exec.ExitError); ok {
result.exitCode = exitErr.ExitCode()
return result
}
t.Fatalf("subprocess execution failed: %v", err)
return subprocessResult{}
}
func filterAuditaEnv(env []string) []string {
filtered := make([]string, 0, len(env))
for _, entry := range env {
key := entry
if idx := strings.IndexByte(entry, '='); idx >= 0 {
key = entry[:idx]
}
if strings.HasPrefix(key, "AUDITA_") || key == "OPENROUTER_API_KEY" {
continue
}
filtered = append(filtered, entry)
}
return filtered
}
func fixturePath(name string) string {
return filepath.Join("..", "..", "internal", "cli", "testdata", name)
}
func schemaFixturePath(name string) string {
return filepath.Join("..", "..", "internal", "core", "schema", "testdata", name)
}
func readFile(t *testing.T, path string) []byte {
return testsupport.ReadFile(t, path)
}
func assertJSONSemanticallyEqual(t *testing.T, expected []byte, actual []byte) {
t.Helper()
if !json.Valid(actual) {
t.Fatalf("actual output is not valid JSON: %q", string(actual))
}
var expectedValue any
var actualValue any
if err := json.Unmarshal(expected, &expectedValue); err != nil {
t.Fatalf("failed to unmarshal expected JSON: %v", err)
}
if err := json.Unmarshal(actual, &actualValue); err != nil {
t.Fatalf("failed to unmarshal actual JSON: %v", err)
}
if !reflect.DeepEqual(expectedValue, actualValue) {
t.Fatalf("JSON content mismatch: expected %q got %q", string(expected), string(actual))
}
}
func writeLargeTranscriptFixture(t *testing.T, segments int) string {
t.Helper()
path := filepath.Join(t.TempDir(), "large-transcript.json")
rows := make([]string, 0, segments)
for i := 0; i < segments; i++ {
rows = append(rows, fmt.Sprintf(`{"id":%d,"speaker":"Speaker%d","start":%s,"end":%s,"text":"Segment %d has enough words to exercise stdout and pipe buffering safely."}`,
i+1,
(i%4)+1,
strconv.FormatFloat(float64(i)*1.1, 'f', 1, 64),
strconv.FormatFloat(float64(i)*1.1+1.0, 'f', 1, 64),
i+1,
))
}
payload := "[\n " + strings.Join(rows, ",\n ") + "\n]\n"
if err := os.WriteFile(path, []byte(payload), 0o644); err != nil {
t.Fatalf("write large transcript fixture: %v", err)
}
return path
}
func onlyRunDir(t *testing.T, workDir string) string {
return testsupport.OnlyRunDir(t, workDir)
}
func assertNoSecretInFile(t *testing.T, path, secret string) {
testsupport.AssertNoSecretInFile(t, path, secret)
}
func assertNoSecretInTree(t *testing.T, root, secret string) {
testsupport.AssertNoSecretInTree(t, root, secret)
}

14
docs/architecture.md Normal file
View File

@@ -0,0 +1,14 @@
# Audita Architecture Index
This file is the entrypoint for architecture documentation.
Core architecture overview:
- [Architecture Overview](./architecture/architecture.md)
Focused architecture contracts:
- [Public Contract](./architecture/public-contract.md)
- [Diagnostics](./architecture/diagnostics.md)
- [Structured LLM](./architecture/structured-llm.md)
- [Validators](./architecture/validators.md)
- [Prompts](./architecture/prompts.md)
- [Output Schemas](./architecture/output-schemas.md)

View File

@@ -0,0 +1,153 @@
# Audita Architecture
## Scope
This document describes the production architecture implemented in this repository today.
Audita is a single-process Go CLI that:
- loads effective runtime configuration;
- reads transcript and glossary inputs;
- normalizes and sections transcripts;
- runs a built-in module pipeline with validator chains;
- writes transcript output and run diagnostics.
## Runtime entrypoints
Primary CLI commands:
- `audita process <transcript.json> --glossary <glossary.yaml> [flags]`
- `audita config validate --config <config.yml>`
- `audita config print-effective [--config <config.yml>]`
Command ownership lives in `internal/cli/run.go`.
## Configuration model
`internal/core/config` owns defaults, file parsing, environment overrides, CLI overrides, and validation.
Effective-config loading for `process` and `config print-effective` is centralized in:
- `ResolveConfigPath`
- `LoadEffectiveConfig`
Effective precedence for `audita process`:
1. defaults
2. config file
3. environment overrides
4. CLI overrides
`audita config validate` is intentionally file-only validation:
- load versioned file;
- apply onto defaults;
- validate;
- do not apply environment overrides.
Supported module and output-schema keys are validated through shared catalogs:
- module keys: `internal/core/modulecatalog`
- output schemas: `internal/core/outputschema`
## Pipeline and module orchestration
The built-in module sequence is configured in runtime config and executed by `internal/framework/runner` through resolved module specs.
Current default sequence:
- `glossary`
- `homophones`
- `glossary`
- `spoken_word`
- `grammar`
Execution behavior:
- modules execute serially over the working transcript;
- section proposal work can run concurrently within a module;
- validator execution happens on generated proposals before application;
- approved proposals are applied once per module in deterministic proposal-index order.
Production modules remain separate packages:
- `internal/modules/glossary`
- `internal/modules/homophones`
- `internal/modules/spoken_word`
- `internal/modules/grammar`
## Proposal generation and prompt context
Shared proposal plumbing is centralized in `internal/framework/proposal_generation`.
Module packages provide:
- module identity and replacement policy;
- module-specific prompt message building;
- built-in validator chain selection.
Shared prompt payload helpers are in `internal/framework/promptcontext`.
## Validator architecture
Built-in validator construction and chain composition live in `internal/validators`.
Shared validator runtime mechanics live in `internal/framework/validators`.
Execution class metadata (deterministic vs LLM-backed) is centralized in `internal/validators/metadata` and used for ordering and reporting classification.
## Structured LLM boundary
All production LLM calls go through the internal contract:
- `contracts.StructuredLLMClient`
- `CompleteStructured(ctx, req, out)`
The OpenAI-compatible HTTP adapter is implemented in `internal/framework/llm`.
Structured response schemas are registered in `internal/framework/responseschema` and attached to requests via `response_format` metadata.
Malformed structured-output detection is centralized in `internal/framework/structuredoutput` and reused by proposal generation and validator execution so downgrade behavior stays consistent.
## Stage naming and diagnostics metadata
Diagnostics stage naming is centralized in `internal/framework/stagename`:
- module proposal stage names;
- proposal-generation stage names;
- validator batch stage names.
Prompt metadata and response-schema metadata each expose canonical diagnostics maps via:
- `prompts.Metadata.DiagnosticsMap()`
- `responseschema.Schema.DiagnosticsMap()`
## Diagnostics and reporting
Run-directory artifacts are owned by `internal/core/diagnostics`.
Stable artifact names are centralized constants (for example transcript artifacts, `invocation.json`, `effective-config.json`, `utilization-diagnostics.json`, `correction-ledger.json`, `report.json`, `error.log`).
Report diagnostics path metadata is constructed through `BuildDiagnosticsMetadata`, which keeps run-directory artifact references consistent between success and failure reports.
## Secret redaction
Redaction responsibilities are split by concern:
- structural config redaction: `config.Config.Redacted()`
- byte/string payload redaction for diagnostics and surfaced errors: framework redaction utilities.
Configured LLM secret extraction is centralized in `llm.ConfiguredSecrets(cfg)` and reused across proposal and validator diagnostics paths.
## Output contracts
Transcript output schema selection is owned by `internal/core/outputschema`.
Supported schemas:
- `bare-segments`
- `audita-v1`
Unknown schema keys fail validation and runtime resolution.
## Key package map
Core packages:
- `internal/core/config`
- `internal/core/schema`
- `internal/core/normalization`
- `internal/core/chunking`
- `internal/core/diagnostics`
- `internal/core/reporting`
- `internal/core/modulecatalog`
- `internal/core/outputschema`
Framework packages:
- `internal/framework/contracts`
- `internal/framework/proposals`
- `internal/framework/proposal_generation`
- `internal/framework/promptcontext`
- `internal/framework/runner`
- `internal/framework/validators`
- `internal/framework/llm`
- `internal/framework/responseschema`
- `internal/framework/stagename`
- `internal/framework/structuredoutput`
Domain packages:
- `internal/modules/*`
- `internal/validators/*`
- `internal/prompts`

View File

@@ -0,0 +1,104 @@
# Audita Diagnostics
This document describes the run-directory diagnostics artifacts produced by `audita process`.
## Purpose
Diagnostics provide machine-readable run context and execution artifacts for:
- failure debugging;
- validator/correction review;
- post-run performance analysis.
Diagnostics are written under the configured work directory (`--work-dir`) when run-directory initialization succeeds.
## Core artifacts
Typical artifacts in each run directory:
- `source-transcript.json`
- `source-transcript-parsed.json`
- `normalized-transcript.json`
- `normalization-summary.json`
- `chunking-summary.json`
- `invocation.json`
- `effective-config.json` (redacted)
- module/validator LLM interaction artifacts
- `report.json`
- `error.log` on failure
## Utilization diagnostics artifact
Artifact:
- `utilization-diagnostics.json`
High-level fields:
- `effective_concurrency`:
- total/proposal/validation LLM concurrency limits in effect.
- `run_timing`:
- run wall time;
- scheduler queue wait time;
- LLM execution time;
- deterministic validator time;
- max/average in-flight LLM calls.
- `llm_calls`:
- total proposal and validation LLM call counts.
- `modules`:
- module-level timing summaries.
- `validators`:
- per-validator timing summaries keyed by stable validator key.
## Correction ledger artifact
Artifact:
- `correction-ledger.json`
Ledger records are flattened review entries derived from module results and include:
- module/proposal identity (`module_key`, `module_instance`, `proposal_index`, `segment_id`);
- correction text fields and replacement policy when available;
- disposition:
- `applied`
- `rejected`
- `skipped`
- `failed`
- stable reason codes/messages;
- deterministic and LLM validator decision snapshots using stable validator keys.
Validator rejection and proposal-application skip are distinct dispositions.
Module warnings are reported in module results and diagnostics metadata, but do not create standalone correction-ledger rows.
## Report references
`report.json` and optional `--report-json` output include diagnostics metadata paths for:
- utilization diagnostics artifact;
- correction ledger artifact;
- existing transcript/normalization/chunking/invocation/effective-config artifacts.
Module report entries also include warning records for malformed proposal-generation payloads and malformed validator batches.
## Retention behavior
Run-directory retention follows configured policy:
- `always`: keep all run directories;
- `never`: keep successful run directories;
- `auto`: keep failed runs and successful runs with skipped/rejected corrections.
## Redaction guarantees
API keys and other configured secrets are redacted from:
- `effective-config.json`;
- LLM interaction diagnostics artifacts;
- reports and surfaced errors.
## Debugging guide
When debugging:
- slow runs:
- inspect `utilization-diagnostics.json` (`run_timing`, `modules`, `validators`, in-flight metrics).
- validator rejections:
- inspect `correction-ledger.json` rejected entries and matching validator decisions;
- inspect validator response diagnostics payloads.
- module warnings:
- inspect module `warnings` entries in `report.json` or `--report-json`;
- follow any diagnostic artifact path on the warning to the recorded error/response payload.
- application skips:
- inspect `correction-ledger.json` skipped entries and skip reason codes;
- compare with validator decisions to distinguish validation rejection vs apply-time skip.

View File

@@ -0,0 +1,88 @@
# Audita Output Schemas
This document describes the built-in transcript output schema registry used by `audita process`.
## Supported schema names
### `bare-segments`
Status:
- implemented
- default output schema
Shape:
- top-level JSON array of transcript segments
Segment fields:
- `id`
- `speaker`
- `start`
- `end`
- `text`
- optional `categories`
Compatibility:
- this preserves the long-standing output shape used by existing consumers.
### `audita-v1`
Status:
- implemented
Shape:
- top-level JSON object:
- `schema`: `"audita-v1"`
- `version`: `"v1"`
- `segments`: transcript segment array
Segment fields inside `segments` match `bare-segments` segment fields.
Compatibility:
- this is the Audita-native object format with explicit schema/version metadata.
### `seriatim-intermediate`
Status:
- deferred / not implemented
Current behavior:
- selecting `seriatim-intermediate` fails clearly as an unsupported output schema.
Reason:
- a concrete, repository-backed contract for this schema has not been finalized yet.
## Selection
Choose output schema with CLI:
```sh
audita process <transcript.json> --glossary <glossary.yaml> --output-schema audita-v1
```
Or in file config:
```yaml
version: 1
output:
schema: audita-v1
```
Precedence remains:
1. defaults
2. file config
3. environment overrides
4. CLI overrides
`--output-schema` overrides `output.schema` when both are supplied.
## Output routing behavior
- With `--output`, transcript JSON is written to file using the selected schema and stdout stays empty on success.
- Without `--output`, stdout contains transcript JSON only, using the selected schema.
- `--report-json` writes report JSON to file and does not write report payloads to stdout.
## Backward-compatibility expectations
- default schema stays `bare-segments` for compatibility unless explicitly changed in a future breaking release;
- supported schema names are treated as stable public contract values;
- unsupported schema names fail before output write.

View File

@@ -0,0 +1,118 @@
# Audita Prompts
This document describes Audita's built-in embedded prompt assets and prompt registry behavior.
## Why embedded prompt assets
Audita embeds production prompt text into the binary so runtime behavior is:
- deterministic;
- auditable;
- dependency-light;
- not dependent on external prompt files at execution time.
Prompt text is authored as Markdown assets and rendered by Go code using typed template data.
## Built-in prompt registry
The prompt registry lives in `internal/prompts` and is responsible for:
- loading embedded prompt assets;
- registering stable prompt IDs and versions;
- recording prompt source metadata;
- computing deterministic SHA-256 source hashes;
- rendering system/user prompts with strict missing-key failures.
Current prompt source behavior:
- built-in embedded prompts only (`prompt_source = builtin`).
- filesystem prompt overrides are not supported.
## Built-in prompt IDs
Module proposal prompts:
- `modules.glossary.proposal`
- `modules.homophones.proposal`
- `modules.spoken_word.proposal`
- `modules.grammar.proposal`
LLM-backed validator prompts:
- `validators.spoken_form_plausibility`
- `validators.meaning_reversal_review`
- `validators.editorial_review`
- `validators.grammar_review`
- `validators.spoken_word_review`
## Prompt version semantics
Current built-in prompt version value is `v1`.
Version is a stable metadata identifier for diagnostics and debugging. It is not a dynamic prompt-selection mechanism.
## Prompt hash semantics
Each registered prompt includes a deterministic SHA-256 hash of embedded source text.
Hash purpose:
- identify exact prompt source used in a run;
- support diagnostics reproducibility and change auditing.
Current hash scope:
- source prompt text (system + user assets for a registered prompt), not a runtime secret-bearing payload.
## Template rendering behavior
Prompt rendering uses Go `text/template` with typed template data from module/validator builders.
Missing-key behavior:
- rendering uses missing-key errors;
- missing/renamed template fields fail quickly instead of silently producing incomplete prompts.
Go code still owns:
- structured request/response models;
- response schema selection;
- transcript/glossary/payload formatting;
- module and validator selection;
- diagnostics wiring.
## Shared prompt hardening policy
A shared hardening fragment is embedded once and included in every module proposal prompt and every LLM-validator prompt.
Hardening policy includes:
- transcript text is untrusted data;
- glossary entries and transcript descriptions are reference data, not instructions;
- instructions found inside transcript text must not be obeyed;
- model must perform only the requested correction/validation task;
- no invention of facts, names, events, motivations, speaker intent, or corrections;
- transcript remains the source of truth.
## Transcript description behavior
Transcript description remains background-only prompt context:
- it may help interpret ambiguous terms;
- it is explicitly non-authoritative and must not override transcript content;
- empty descriptions do not render awkward blank context sections.
Generated transcript descriptions are not implemented in this workstream.
## Diagnostics and report metadata boundaries
Current metadata flow:
- proposal-generation diagnostics request metadata includes prompt metadata;
- LLM-validator diagnostics request metadata includes prompt metadata.
Prompt metadata fields used in diagnostics:
- `prompt_id`
- `prompt_version`
- `prompt_source`
- `embedded_path`
- `sha256`
Current boundary:
- detailed prompt metadata is diagnostics-first;
- broad report-level prompt registries/ledgers are deferred.
## 1.0 boundary
Not implemented for 1.0 in this workstream:
- filesystem prompt overrides;
- user-configurable prompt selection;
- external prompt directories.

View File

@@ -0,0 +1,121 @@
# Audita Public Contract
## Scope
This document defines stability expectations for Audita's external runtime interfaces.
Covered interfaces:
- CLI commands and major flags;
- versioned config behavior and precedence;
- transcript/glossary input forms;
- output schema selection;
- report schema metadata;
- diagnostics artifact path metadata;
- stdout/stderr and exit-code behavior;
- redaction guarantees.
## CLI contract
Stable commands:
- `audita process`
- `audita config validate`
- `audita config print-effective`
Stable high-value `process` flags:
- `--config`
- `--glossary`
- `--output`
- `--report-json`
- `--modules`
- `--output-schema`
## Config contract
Supported config format:
- YAML;
- `version: 1`;
- strict unknown-field rejection.
Path resolution for `process` and `config print-effective`:
1. `--config`
2. `AUDITA_CONFIG`
3. `/usr/local/etc/audita/config.yml`
4. `/etc/audita/config.yml`
Missing explicit path is an error. Missing default paths is non-fatal.
Precedence for `process`:
1. defaults
2. file config
3. environment overrides
4. CLI overrides
`config validate` remains file-only validation (defaults + file config; no env overrides).
Module and output-schema keys are validated against built-in catalogs. Unknown keys fail validation.
## Input contract
Supported transcript JSON top-level forms:
- array of segments
- object with `segments` array
Supported glossary YAML form:
- top-level `glossary` list with required entry fields validated by schema parsing.
## Output schema contract
Supported transcript output schemas:
- `bare-segments` (default)
- `audita-v1`
Unknown schema keys fail before output write.
## Report metadata contract
Process reports include stable report metadata fields:
- `report_schema_name`
- `report_schema_version`
- `output_schema`
- `config_version` (when file config is loaded)
Current values:
- `report_schema_name = audita-process-report`
- `report_schema_version = v1`
`--report-json` output and run-directory `report.json` use the same report schema metadata.
Validator decision/rejection records use stable validator keys via `validator_name`.
## Diagnostics metadata contract
When run-directory initialization succeeds, diagnostics metadata paths reference stable artifacts, including:
- transcript and normalization artifacts;
- chunking summary;
- invocation metadata;
- redacted effective config;
- utilization diagnostics;
- correction ledger;
- `error.log` on failures.
LLM interaction diagnostics include stable prompt and structured-schema identifiers where applicable.
## Stdout/stderr and exit codes
Success:
- with `--output`, stdout is empty;
- without `--output`, stdout contains transcript JSON only;
- report JSON is not written to stdout.
Failures:
- nonzero exit;
- human-readable stderr summary;
- diagnostics directory path on stderr when available.
Exit codes:
- `0` success
- nonzero failure
## Redaction contract
Configured secrets are redacted from:
- effective config outputs;
- diagnostics artifacts;
- report artifacts;
- surfaced adapter/runtime errors.
## Compatibility policy
Stable command behavior, schema names, report metadata keys, diagnostics-path field semantics, and validator key identities are treated as public contract.
Additive fields are acceptable when existing fields and behavior remain compatible.

View File

@@ -0,0 +1,72 @@
# Structured LLM Architecture
## Scope
This document describes Audita's structured LLM runtime boundary and adapter behavior.
## Runtime boundary
Production LLM integration depends on the internal contract only:
- `contracts.StructuredLLMClient`
- `CompleteStructured(ctx, req, out)`
Provider SDK types do not leak past this boundary.
## Adapter ownership
`internal/framework/llm` owns the OpenAI-compatible HTTP adapter and shared LLM runtime utilities.
Key responsibilities:
- request assembly;
- timeout/cancellation propagation;
- bounded retry behavior;
- scheduler integration;
- provider response decoding;
- error redaction.
## Structured schema registry
Structured response schemas are registered in `internal/framework/responseschema` and include stable metadata:
- `id`
- `version`
- `name`
- `json_schema`
- `sha256`
Current schema keys:
- `correction_set`
- `validator_decision_set`
Schema metadata is attached to diagnostics through `Schema.DiagnosticsMap()`.
## Request shape assumptions
Audita targets OpenAI-compatible chat-completions endpoints and sends structured requests with:
- model;
- chat messages;
- `response_format.type = json_schema`;
- schema name and JSON schema payload.
## Local validation remains mandatory
Provider schema enforcement is treated as transport-level guardrails.
Audita still validates output locally before applying behavior changes:
- proposal decoding and proposal invariants;
- validator decision decoding and cardinality checks;
- deterministic validation and apply-time rules.
## Shared malformed-output policy
Malformed structured-output classification is centralized in `internal/framework/structuredoutput`.
Proposal generation and validator execution both use this shared classifier so downgrade behavior cannot drift between the two paths.
## Secrets and redaction
Secret extraction for LLM redaction is centralized in `llm.ConfiguredSecrets(cfg)` and reused by proposal and validator diagnostics writers.
Secrets are redacted from:
- diagnostics artifacts;
- report artifacts;
- surfaced adapter/runtime errors.
## Concurrency and scheduling
LLM execution is constrained by composed scheduler limits:
- total LLM concurrency;
- proposal LLM concurrency;
- validation LLM concurrency.
The scheduler is FIFO and context-aware so permits are released on success, failure, and cancellation.

View File

@@ -0,0 +1,96 @@
# Audita Validators
## Scope
This document defines the built-in validator system used by production module runs.
## Ownership boundaries
Built-in validator keys, constructors, and module chains are owned by `internal/validators`.
Shared runtime execution mechanics are owned by `internal/framework/validators`, including:
- validator request/result models;
- deterministic proposal checks;
- LLM validator batching and execution;
- decision-cardinality enforcement;
- diagnostics integration.
Execution class metadata is owned by `internal/validators/metadata`.
## Stable validator keys
Deterministic:
- `proposal_shape`
- `confidence_threshold`
- `original_text_presence`
- `non_empty_corrected_text`
- `no_effect`
- `protected_terms`
LLM-backed:
- `spoken_form_plausibility`
- `meaning_reversal_review`
- `editorial_review`
## Built-in module chains
`glossary`:
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
- `protected_terms`
- `non_empty_corrected_text`
- `spoken_form_plausibility`
- `meaning_reversal_review`
`homophones`:
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
- `protected_terms`
- `non_empty_corrected_text`
- `spoken_form_plausibility`
- `meaning_reversal_review`
`spoken_word`:
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
- `protected_terms`
- `non_empty_corrected_text`
- `editorial_review`
- `meaning_reversal_review`
`grammar`:
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
- `protected_terms`
- `non_empty_corrected_text`
- `editorial_review`
- `meaning_reversal_review`
## Ordering and execution semantics
Validator ordering is based on canonical metadata:
- deterministic validators run before LLM-backed validators.
Within each module stage:
- proposals are generated per section;
- validator chains execute on those proposals;
- approved proposals are applied once after section work settles.
## Malformed payload behavior
Malformed structured-output from proposal generation and LLM validator calls is downgraded, not treated as a process-fatal transport error.
Current outcomes:
- malformed proposal-generation payloads produce section/module warnings and zero proposals for the affected section;
- malformed validator decision payloads reject the affected validator batch with warnings;
- deterministic validator behavior and runner order remain unchanged.
## Reporting identity
Reports and diagnostics use stable validator keys as identifiers.
Correction-ledger deterministic-vs-LLM classification is derived from canonical validator metadata, not package-local hardcoded maps.
## Prompt assets
LLM validator prompt assets and prompt metadata are documented in [Prompts](./prompts.md).

149
docs/configuration.md Normal file
View File

@@ -0,0 +1,149 @@
# Audita Configuration
## Scope
This document defines the supported versioned YAML configuration model and runtime precedence behavior.
## Supported file version
Current supported config file version:
- `version: 1`
Validation rules:
- missing `version` fails;
- unsupported version fails;
- unknown YAML fields fail (strict decoding).
## Config path resolution
For `audita process` and `audita config print-effective`, path resolution order is:
1. `--config <path>`
2. `AUDITA_CONFIG`
3. `/usr/local/etc/audita/config.yml` (if present)
4. `/etc/audita/config.yml` (if present)
Missing-path behavior:
- missing `--config` path is an error;
- missing `AUDITA_CONFIG` path is an error;
- missing both default paths is non-fatal.
## Effective precedence
`audita process` effective precedence:
1. defaults
2. file config
3. environment overrides
4. CLI overrides
`audita config print-effective` uses:
1. defaults
2. file config
3. environment overrides
`audita config validate` intentionally uses file-only validation:
1. defaults
2. file config
Environment overrides are not applied in `config validate`.
## Supported top-level YAML fields
```yaml
version: 1
pipeline:
modules: [glossary, homophones, glossary, spoken_word, grammar]
output:
schema: bare-segments
llm:
proposal:
base_url: https://openrouter.ai/api/v1
model: openrouter/google/gemma-4-31b-it
api_key_env: AUDITA_LLM_API_KEY
timeout: 120s
max_retries: 3
validation:
base_url: https://openrouter.ai/api/v1
model: openrouter/google/gemma-4-31b-it
api_key_env: AUDITA_VALIDATION_LLM_API_KEY
timeout: 120s
max_retries: 3
concurrency:
total_llm: 2
proposal_llm: 2
validation_llm: 1
chunking:
target_sections: 8
max_section_tokens: 8192
min_section_tokens: 2048
normalization:
max_segment_gap: 4s
ellipsis_gap: 3.5s
max_segment_duration: 60s
max_segment_tokens: 2048
thresholds:
glossary: 0.8
homophones: 0.8
spoken_word: 0.8
grammar: 0.8
context:
description: "optional transcript background context"
diagnostics:
work_dir: /tmp/audita
retention: auto
```
## Module and output-schema validation
`pipeline.modules` keys are validated against the built-in supported module catalog.
Supported module keys:
- `glossary`
- `homophones`
- `spoken_word`
- `grammar`
Repeated supported module keys are allowed.
`output.schema` is validated against the built-in output schema catalog.
Supported output schema keys:
- `bare-segments`
- `audita-v1`
Unknown module keys and unknown output schema keys fail validation.
## Duration field parsing
Duration-like fields support:
- numeric seconds (for example `120`, `3.5`)
- duration strings (for example `120s`, `2m`)
LLM timeout duration strings must resolve to whole seconds.
## Secret handling
Use `api_key_env` fields for secrets:
- `llm.proposal.api_key_env`
- `llm.validation.api_key_env`
These fields store environment variable names, not secret values.
Resolved secret values are redacted from:
- `audita config print-effective` output;
- diagnostics `effective-config.json`;
- report and diagnostics payloads.
## Commands
Validate a file config:
```sh
audita config validate --config ./audita.yml
```
Print redacted effective config:
```sh
audita config print-effective --config ./audita.yml
```
## Compatibility notes
Legacy compatibility flags and environment aliases remain available where implemented, but the stable configuration surface is the versioned YAML model described above.

33
docs/development.md Normal file
View File

@@ -0,0 +1,33 @@
# Audita Development Workflow
## Scope
This document defines the canonical contributor workflow and engineering conventions for this repository.
## Workflow
1. Start from a clean understanding of scope and constraints.
2. Make focused changes that preserve existing public behavior unless behavior change is explicitly intended.
3. Run targeted tests for touched packages.
4. Run `go test ./...` before finalizing substantial changes.
5. Update affected documentation so it describes current behavior only.
## Engineering conventions
- Keep module packages separate: `glossary`, `homophones`, `spoken_word`, `grammar`.
- Prefer narrow shared helpers and catalogs over broad abstractions.
- Preserve diagnostics artifact naming and report field contracts unless intentionally changed.
- Preserve CLI/config precedence semantics unless intentionally changed.
- Treat stable validator keys, prompt identifiers, and output-schema keys as contract surfaces.
## Configuration and runtime expectations
- `audita process` precedence is defaults -> file -> env -> CLI.
- `audita config validate` validates file config merged onto defaults only.
- `audita config print-effective` includes environment overrides and prints redacted JSON.
## Testing expectations
- Add tests for new behavior and for bug fixes.
- Keep deterministic fixtures stable.
- Do not reduce existing parity, release-fixture, subprocess, or module-specific coverage without equivalent replacement.
## Commit discipline
- Keep commits scoped and reviewable.
- Avoid mixing unrelated refactors with behavior changes.
- Use clear plain-English commit messages.

View File

@@ -0,0 +1,27 @@
# Documentation Policy
## Scope
This policy defines how project documentation should be authored and maintained.
## Core rules
- Document the current behavior of the codebase.
- Remove stale behavior descriptions promptly when code changes.
- Do not describe development history in architecture or behavior docs unless a document is explicitly historical.
- Do not use architecture or behavior docs as changelogs.
- Prefer rewriting stale sections from scratch when substantial behavior or ownership changes occur.
## Consistency requirements
- Keep command examples aligned with current CLI surfaces.
- Keep configuration examples aligned with supported fields and precedence.
- Keep architecture package ownership descriptions aligned with current code layout.
- Keep stable contract identifiers accurate (module keys, validator keys, output-schema keys, report metadata fields).
## Cross-document expectations
- `docs/architecture/*` documents runtime behavior and package ownership.
- `docs/configuration.md` documents config schema and precedence.
- `docs/development.md` documents contributor workflow and engineering conventions.
## Review expectations for documentation changes
- Verify referenced files and links exist.
- Verify examples match current behavior.
- Prefer concise, direct language and avoid speculative future claims.

View File

@@ -0,0 +1,96 @@
# Audita Subprocess Operations
This document describes how parent processes should invoke `audita process` safely in production orchestration.
## Recommended command form
Use explicit file outputs for orchestrated runs:
```sh
audita process <transcript.json> \
--transcript-description "Brief context that may help resolve ambiguous terms." \
--glossary <glossary.yaml> \
--output <output-transcript.json> \
--report-json <report.json>
```
Additional flags that may be situationally appropriate:
- `--config <path>` to select an explicit versioned config file.
- `--output-schema <bare-segments|audita-v1>` to select transcript output shape.
- `--work-dir <dir>` to control diagnostics location.
- `--work-dir-retention <always|auto|never>` to control retained run directories.
- `--total-llm-concurrency`, `--proposal-llm-concurrency`, and `--validation-llm-concurrency` when orchestration needs to set explicit LLM throughput controls.
- `--modules ...` only when intentionally overriding the default sequence.
For config-driven orchestration, validate config files in CI/preflight:
```sh
audita config validate --config <path>
```
## Stdout behavior
- With `--output`: stdout is expected to be empty on success.
- Without `--output`: stdout contains transcript JSON only on success.
- Report JSON is never written to stdout.
## Stderr behavior
- Success path should be quiet or minimal human-readable logs.
- Failure path writes concise human-readable errors.
- When a diagnostics run directory exists, failure stderr includes its path.
- Prompt/response diagnostic payloads are not streamed to stderr.
## Output file behavior
- `--output` writes transcript JSON in the selected output schema to the provided path.
- Output write failures return nonzero and surface actionable errors.
- The command does not silently ignore output write errors.
## Report JSON behavior
- `--report-json` writes a machine-readable process report to the requested path.
- Run-directory `report.json` is written independently under diagnostics.
- Best-effort failure reports are emitted when possible without masking the primary failure.
- Report write failures return nonzero with clear stderr messaging.
- Report diagnostics metadata references run-directory artifacts including utilization diagnostics and correction ledger paths when available.
## Diagnostics directory behavior
- Each run creates (when possible) a per-run diagnostics directory.
- Typical artifacts include transcript, normalization, chunking, invocation, effective config, LLM diagnostics, `utilization-diagnostics.json`, `correction-ledger.json`, `report.json`, and `error.log` on failure.
- Failed runs retain diagnostics.
- Under `auto` retention, successful runs with skipped/rejected corrections are retained; clean successful runs may be removed.
## Exit codes
- `0`: success.
- Nonzero: failure (input/schema/config/module/LLM/runtime/output/report/diagnostics errors).
Treat any nonzero as a failed subprocess invocation.
## Timeout and cancellation
- Runtime operations propagate context cancellation and request timeouts through LLM/scheduler paths.
- On cancellation or timeout, the process exits nonzero and should not hang.
- If diagnostics were initialized before failure, failure artifacts remain available for debugging.
## Secret redaction expectations
API keys and configured secret values are redacted from:
- reports (`--report-json` and run-dir `report.json`);
- diagnostics artifacts (including effective config and LLM interaction artifacts);
- surfaced adapter/runtime errors;
- test fixtures and regression outputs.
Parent-process logs should still avoid printing raw environment variables.
## Parent-process pipe guidance
To avoid deadlocks in orchestrators:
- always read both stdout and stderr concurrently when invoking as a subprocess;
- prefer file outputs (`--output`, `--report-json`) for machine workflows;
- treat stderr as human-readable diagnostics, not structured data;
- parse structured results from output/report files.
For Go callers, prefer `exec.CommandContext` with explicit timeout/cancellation and buffered/streamed readers for both pipes.

128
docs/release-checklist.md Normal file
View File

@@ -0,0 +1,128 @@
# Audita Release Checklist
Use this checklist before cutting a pre-1.0 or 1.0 release candidate.
## Core test pass
- Run:
- `go test ./...`
- Confirm tests pass without live LLM credentials and without Python dependencies.
## Config validation and precedence
- Validate a representative config:
- `audita config validate --config <path>`
- Inspect redacted effective config:
- `audita config print-effective --config <path>`
- Confirm precedence behavior:
- defaults -> file config -> environment -> CLI.
- Confirm default config search order:
- `/usr/local/etc/audita/config.yml` first, then `/etc/audita/config.yml`.
- Confirm missing both default-path config files is non-fatal when `--config`/`AUDITA_CONFIG` are unset.
## Output schema checks
- Verify default output schema remains `bare-segments`.
- Verify `--output-schema audita-v1` emits object payload with `schema` and `version`.
- Verify unknown schema (for example `seriatim-intermediate`) fails clearly.
## Subprocess contract checks
- With `--output`, verify stdout is empty on success.
- Without `--output`, verify stdout contains transcript JSON only.
- Verify `--report-json` writes file output and does not write report JSON to stdout.
- Verify failure stderr remains human-readable and includes diagnostics path when available.
- Verify nonzero exit on failures.
## Structured LLM checks
- Verify runtime uses the Audita-owned OpenAI-compatible adapter.
- Verify structured response schemas are attached via `response_format.type=json_schema`.
- Verify diagnostics metadata includes structured schema `id/version/name/sha256`.
- Verify provider output is still locally decoded/validated before use.
- Verify malformed module-stage structured payloads degrade to warnings/rejections instead of failing the run.
## Report and diagnostics schema checks
- Verify report metadata fields:
- `report_schema_name`
- `report_schema_version`
- `output_schema`
- `config_version` when file config is used.
- Verify diagnostics artifact references exist in reports:
- transcript/normalization/chunking/invocation/effective-config artifacts
- utilization diagnostics artifact
- correction ledger artifact
- error log on failures.
## Redaction checks
- Verify secrets are redacted from:
- `effective-config.json`
- run-dir and `--report-json` reports
- LLM request/response/error diagnostics payloads.
- Verify no API keys/bearer tokens leak into fixtures or outputs.
## Prompt and validator metadata checks
- Verify prompt metadata appears in LLM request metadata diagnostics:
- `prompt_id`, `prompt_version`, `prompt_source`, `embedded_path`, `sha256`.
- Verify stable validator keys appear in report decisions/rejections.
- Verify module warning records appear in reports for malformed proposal-generation payloads and malformed validator batches.
- Verify built-in validator chains resolve and execute for default and explicit module runs.
## Utilization diagnostics checks
- Verify `utilization-diagnostics.json` exists on successful runs.
- Verify partial utilization artifact behavior on controlled failure paths.
- Verify utilization fields are structurally present and nonnegative:
- effective concurrency
- run timing
- module timing summaries
- per-validator timing summaries.
## Correction ledger checks
- Verify `correction-ledger.json` exists on successful runs.
- Verify report references ledger artifact path.
- Verify ledger dispositions include applied/rejected and skipped/failed where exercised.
- Verify validator rejection and proposal-application skip remain distinct.
## Pipeline behavior checks
- Verify default full pipeline run remains:
- `glossary`, `homophones`, `glossary`, `spoken_word`, `grammar`
- with deterministic repeated instance naming (`glossary_1`, `glossary_2`).
- Verify explicit module runs (`--modules`) still work.
## Failure and cancellation checks
- Verify controlled failure paths retain diagnostics and produce best-effort failure reports.
- Verify malformed proposal-generation payloads keep exit code `0`, keep stderr empty on success, and record warnings in reports/diagnostics.
- Verify malformed validator payloads reject only the affected batch and do not fail the module.
- Verify timeout/cancellation paths exit nonzero, do not hang, and retain failure diagnostics when initialized.
## Release fixture/idempotence checks
- Run release fixtures (`internal/cli/testdata/release`) through `go test ./...`.
- Confirm fixture checks cover:
- must-apply and must-not-apply expectations
- protected-term survival
- report and diagnostics contracts
- output-schema checks
- prompt/schema metadata diagnostics
- utilization/ledger artifacts
- idempotence-oriented second pass no-op behavior with deterministic fake responses.
## Deferred-feature guardrail
- Confirm release docs do not claim support for deferred items:
- filesystem prompt overrides
- user-configurable validator chains
- arbitrary user-supplied output schemas
- resume/start-at/stop-after execution
- diff/check/propose-only modes
- generated transcript descriptions enabled by default
- interactive review UI
- UI/server wrapper
- provider benchmarking harness.

791
docs/roadmap/audit.md Normal file
View File

@@ -0,0 +1,791 @@
# Pre-1.0 Code Quality and Deduplication Audit
## 1. Executive summary
Audita is in good shape for a limited pre-1.0 cleanup pass. The repository is small, package boundaries are mostly explicit, and the core public contract is already documented around `audita process`, config loading, output schemas, diagnostics, reports, embedded prompts, modules, and validators. The highest-value improvements are targeted centralization, not a rewrite.
Top three refactoring targets before 1.0:
1. Centralize module proposal plumbing and prompt payload construction across the four production modules.
2. Centralize effective config loading plus schema/module catalog validation so `process`, `config print-effective`, and `config validate` cannot drift.
3. Centralize diagnostics artifact names, stage names, and validator classification metadata used by reports and the correction ledger.
No major architectural risk is apparent. The main pre-1.0 risk is public-behavior drift from repeated policy strings, catalog values, artifact paths, and nearly identical command/module scaffolding.
This report was written to `docs/roadmap/audit.md`. `docs/roadmap/` already exists in the repository, although its previous `publish.md` file is currently deleted in the worktree by an unrelated change.
## 2. Repository map reviewed
Reviewed documentation:
- `README.md`
- `docs/configuration.md`
- `docs/architecture/architecture.md`
- `docs/architecture/public-contract.md`
- `docs/architecture/diagnostics.md`
- `docs/architecture/output-schemas.md`
- `docs/architecture/prompts.md`
- `docs/architecture/validators.md`
- `docs/architecture/structured-llm.md`
- `docs/integration/subprocess-operations.md`
- `docs/release-checklist.md`
Reviewed implementation areas:
- `cmd/audita`
- `internal/cli`
- `internal/core/config`
- `internal/core/schema`
- `internal/core/io`
- `internal/core/normalization`
- `internal/core/chunking`
- `internal/core/diagnostics`
- `internal/core/outputschema`
- `internal/core/reporting`
- `internal/framework/contracts`
- `internal/framework/modules`
- `internal/framework/proposal_generation`
- `internal/framework/proposals`
- `internal/framework/runner`
- `internal/framework/validators`
- `internal/framework/llm`
- `internal/framework/responseschema`
- `internal/framework/promptcontext`
- `internal/framework/warnings`
- `internal/modules/glossary`
- `internal/modules/homophones`
- `internal/modules/spoken_word`
- `internal/modules/grammar`
- `internal/prompts`
- `internal/validators`
- package tests and CLI parity/release fixtures under `internal/cli/testdata`
Major execution paths reviewed:
- `audita process <transcript.json> --glossary <glossary.yaml>`
- `audita config validate --config <path>`
- `audita config print-effective [--config <path>]`
- default module sequence resolution and repeated glossary instance naming
- proposal generation, validator execution, proposal application, report writing, diagnostics writing, and retention
Important absent or not-applicable areas:
- No `pkg/` directory exists.
- No `examples/` directory exists.
- No `docs/internal/` directory exists.
- No `internal/app`, `internal/stage`, `internal/storage`, `internal/artifacts`, or `internal/manifest` packages exist. Their closest equivalents are `internal/cli`, `internal/framework/runner`, `internal/core/diagnostics`, and `internal/core/reporting`.
## 3. High-confidence deduplication opportunities
### 3.1 Module proposal plumbing is duplicated across all production modules
Affected files/packages:
- `internal/modules/glossary/module.go`
- `internal/modules/homophones/module.go`
- `internal/modules/spoken_word/module.go`
- `internal/modules/grammar/module.go`
- `internal/modules/*/prompt.go`
- `internal/framework/proposal_generation`
- `internal/framework/promptcontext`
Duplicated or near-duplicated behavior:
- Each module has the same `Module` struct shape, `Validators` copy behavior, `Propose` flow, section transcript extraction, transcript description extraction, `proposal_generation.GenerateCandidates` request construction, prompt metadata map construction, and stage-name formatting.
- Each module also has a near-identical prompt payload builder with local `promptSegment` and `promptTranscriptSection` types, glossary JSON marshaling, transcript section JSON marshaling, transcript description block rendering, and two-message return shape.
- `collectSectionProposals` already passes a section transcript to each module, but each module then filters that transcript again by section metadata.
Why it matters:
- A diagnostics or prompt-context bug fix would need to be repeated in four modules.
- Prompt metadata fields and stage names are diagnostics-visible and could drift by module.
- The double section filtering is currently harmless, but it obscures the runner/module contract.
Recommended refactor:
- Add a small shared helper for module proposal execution, likely in `internal/framework/proposal_generation` or a narrow `internal/modules/modulekit` package.
- Keep domain-specific prompt IDs and prompt text local to each module.
- Move transcript section prompt payload construction into a shared prompt-context helper, for example `promptcontext.MarshalTranscriptSection`.
- Provide one helper for prompt metadata maps instead of manually expanding `prompt_id`, `prompt_version`, `prompt_source`, `embedded_path`, and `sha256` in every module.
- Preserve current module `Key`, replacement policy, and validator chain ownership.
Suggested tests:
- Keep one golden or table-driven prompt payload test per module for domain-specific wording.
- Add shared tests for transcript section JSON shape, empty transcript handling, categories copy behavior, and prompt metadata fields.
- Add a parity test that all four module `Propose` methods still write diagnostics under the same module instance directory and produce the same correction mapping.
Risk level:
- Low to medium. The behavior is highly duplicated, but prompt and diagnostics behavior is sensitive. Refactor behind existing module tests and CLI parity fixtures.
### 3.2 Effective config loading is repeated between commands
Affected files/packages:
- `internal/cli/run.go`
- `internal/core/config`
Duplicated or near-duplicated behavior:
- `runProcess` and `runConfigPrintEffective` both resolve config path, start from defaults, optionally load/apply file config, then apply environment overrides.
- `runConfigValidate` separately loads a file, applies it to defaults, and validates it.
- Path source metadata is computed in `internal/cli`, not `internal/core/config`, even though the precedence contract is documented as config behavior.
Why it matters:
- Config precedence is part of the public contract. If a future setting is added, three command paths may need coordinated updates.
- `config print-effective` is the user-visible diagnostic for effective config. It should use the same loader as `process`, except for intentionally omitted CLI overrides.
- The current code is understandable, but the behavior is repeated in a way that makes drift likely as config grows.
Recommended refactor:
- Add a narrow effective-config loader in `internal/core/config`, returning `Config`, source path, source type, and version metadata.
- Keep command-specific CLI flag parsing in `internal/cli`.
- Model the intentional differences explicitly:
- `process`: defaults + file + env + CLI overrides
- `config print-effective`: defaults + file + env
- `config validate`: file schema + default-backed config validation, no env
- Move `resolveConfigPath` or an equivalent path resolver into `internal/core/config`.
Suggested tests:
- One table-driven config loader test covering explicit `--config`, `AUDITA_CONFIG`, default search paths, missing explicit paths, and missing default paths.
- CLI tests asserting `process` and `print-effective` share file+env behavior.
- A regression test that `config validate` remains file-only and does not read environment overrides.
Risk level:
- Low. Behavior is already explicit and well tested; the refactor can be done by moving code without changing precedence.
### 3.3 Module catalog validation is split across config, contracts, and module factory
Affected files/packages:
- `internal/core/config/validation.go`
- `internal/framework/contracts/contracts.go`
- `internal/framework/modules/registry.go`
- `internal/validators/chains.go`
- `internal/framework/validators/models.go`
Duplicated or near-duplicated behavior:
- Module keys appear in multiple places:
- config default CSV: `glossary,homophones,glossary,spoken_word,grammar`
- module factory constants and known-key map
- built-in validator chains
- confidence threshold lookup
- individual module `Key()` methods
- `Config.Validate` checks only that module names are non-empty. An unsupported configured module can pass `audita config validate` and fail later in `process` runner setup.
- `contracts.ResolveModuleRunSpecs` only assigns instance names; it does not validate production module support.
Why it matters:
- `audita config validate` is documented as a CI/preflight command. Letting unsupported modules pass weakens that preflight.
- Module key drift could affect thresholds, validator chains, reports, and unsupported-module errors.
Recommended refactor:
- Introduce a small canonical module catalog or key package that can be imported by config validation, module factory construction, validator chain resolution, and threshold lookup without creating a cycle.
- Keep module construction in `internal/framework/modules`; the catalog should expose keys and validation only.
- Make `Config.Validate` reject unknown built-in module keys through that catalog.
- Keep repeated module instances valid.
Suggested tests:
- `internal/core/config` test: unknown `pipeline.modules` fails validation.
- `internal/cli` test: `audita config validate --config` rejects an unsupported module before runtime.
- Existing `internal/framework/modules` unknown-module tests should continue to pass.
- Validator chain tests should assert every catalog module has a built-in chain.
Risk level:
- Medium. This tightens validation behavior. It is desirable before 1.0, but if unknown modules were intentionally allowed for future extension, document that explicitly instead.
### 3.4 Output schema support is hardcoded in config validation and registry
Affected files/packages:
- `internal/core/config/validation.go`
- `internal/core/outputschema/registry.go`
- `docs/architecture/output-schemas.md`
Duplicated or near-duplicated behavior:
- `Config.Validate` hardcodes `bare-segments` and `audita-v1`.
- `outputschema.Resolve` owns the actual output schema registry and returns the runtime error for unsupported schema names.
Why it matters:
- Adding or deferring a schema requires updating multiple places.
- Public behavior could drift: a schema might validate in config but fail at output time, or vice versa.
Recommended refactor:
- Make `internal/core/outputschema` expose `IsSupported`, `SupportedKeys`, or a validation function.
- Have config validation call that helper or consume shared constants.
- Keep actual encoding logic in `outputschema`; config should not know encoder details.
Suggested tests:
- Config validation test for every output schema returned by the registry.
- Output schema registry test that unsupported `seriatim-intermediate` still fails clearly until implemented.
- CLI test that unsupported `--output-schema` fails before output write.
Risk level:
- Low. This is a straightforward catalog centralization.
### 3.5 Diagnostics artifact names and report metadata paths are repeated
Affected files/packages:
- `internal/core/diagnostics/run_dir.go`
- `internal/cli/run.go`
- `internal/core/reporting/report.go`
- docs under `docs/architecture` and `docs/integration`
Duplicated or near-duplicated behavior:
- Artifact filenames such as `source-transcript.json`, `source-transcript-parsed.json`, `normalized-transcript.json`, `normalization-summary.json`, `chunking-summary.json`, `utilization-diagnostics.json`, `correction-ledger.json`, `invocation.json`, `effective-config.json`, `report.json`, and `error.log` are repeated between run-directory writers and `buildProcessReport`.
- `runProcess` writes `utilization-diagnostics.json` and `correction-ledger.json` by raw string on both success and failure paths.
Why it matters:
- These names are part of the documented diagnostics contract.
- A filename change would need to be made in multiple places, and report metadata could point at files that are no longer written.
Recommended refactor:
- Define diagnostics artifact name constants in `internal/core/diagnostics`.
- Add a helper that returns `reporting.DiagnosticsMetadata` for a run directory and status.
- Add named methods for utilization diagnostics and correction ledger writes, or at least constants used by `WriteJSONArtifact`.
Suggested tests:
- Unit test that `diagnostics.MetadataForRunDirectory` matches files written by `RunDirectory`.
- CLI success/failure tests should continue to assert report metadata paths and actual file existence.
- Add a test for failure report metadata including `error.log`.
Risk level:
- Low. This is mostly string centralization, with high public-contract value.
### 3.6 Validator execution class is duplicated and partially hardcoded
Affected files/packages:
- `internal/validators/registry.go`
- `internal/validators/metadata/metadata.go`
- `internal/validators/*/validator.go`
- `internal/framework/runner/runner.go`
- `internal/cli/review_artifacts.go`
Duplicated or near-duplicated behavior:
- Validator constructors wrap validators with execution class metadata.
- `BuiltInValidatorDefinition` also has an `LLMBacked` field.
- Runner uses `metadata.ClassOf` to order deterministic validators before LLM-backed validators.
- Correction ledger classification uses a local hardcoded map of LLM-backed validator names.
Why it matters:
- Adding a new LLM-backed validator could be ordered correctly by runner metadata but appear in the wrong correction-ledger section.
- Validator class is domain metadata, not report-building policy. It should have one source of truth.
Recommended refactor:
- Make validator classification resolvable by validator instance or stable key from a single metadata source.
- Remove the unused or redundant `LLMBacked` field, or make it the canonical source used by constructors, runner ordering, and ledger formatting.
- Replace the local ledger map with `metadata.ClassOf` when possible, or a registry lookup by stable key.
Suggested tests:
- Correction ledger test that LLM-backed decisions are classified from validator metadata, not a local string map.
- Registry test that every registered LLM-backed validator reports the same class through every public metadata path.
- Runner ordering test should remain in place.
Risk level:
- Low to medium. The implementation is small, but correction-ledger shape is diagnostics-visible.
### 3.7 Malformed structured-output classification is duplicated
Affected files/packages:
- `internal/framework/proposal_generation/generate.go`
- `internal/framework/validators/llm_validators.go`
- `internal/framework/llm/openai_compatible_client.go`
Duplicated or near-duplicated behavior:
- Proposal generation and LLM validators both classify malformed structured-output errors by scanning error message substrings.
- The marker lists are currently the same, but they are maintained independently.
- The actual errors originate in the LLM adapter.
Why it matters:
- Proposal-generation malformed payloads become warnings with zero proposals, while validator malformed payloads reject affected batches with warnings. If classifiers drift, similar adapter failures could be downgraded in one workflow and hard-fail in another.
Recommended refactor:
- Prefer a typed error or exported classifier from `internal/framework/llm`.
- If typed errors are too invasive, create one shared classifier function in a lower framework package used by both proposal generation and validators.
- Preserve the different handling semantics at each call site.
Suggested tests:
- Shared classifier table for all adapter malformed-output errors.
- Proposal-generation test and validator test should assert the same representative malformed adapter errors are downgraded.
- Adapter tests should assert typed/classified errors wrap useful context and still redact secrets.
Risk level:
- Medium. Error typing can accidentally affect retry and wrapping behavior; do this with focused tests.
## 4. Medium-confidence opportunities
### 4.1 CLI flag registration and override extraction are large and repetitive
Affected files/packages:
- `internal/cli/run.go`
- `internal/core/config/flags.go`
Duplicated or near-duplicated behavior:
- Each process flag has a field in `processFlags`, a registration entry in `newProcessFlagSet`, a case in `fs.Visit`, and an assignment in `config.ApplyCLIOverrides`.
- File config and environment config also set many of the same effective config fields.
Why it matters:
- Adding a new config option requires multiple edits. Missing one edit could create a flag that displays but does not override, or a config field with no CLI override.
Recommended refactor:
- Avoid a generic reflection-heavy flag system before 1.0.
- Consider a small metadata table only for simple scalar flags, or a focused helper that maps visited flags to `CLIOverrides`.
- Keep nontrivial semantics, such as legacy concurrency alias precedence, explicit in code.
Suggested tests:
- CLI override parity test for every stable flag that mutates config.
- A test that default flag values reflect file+env effective config before CLI overrides.
Risk level:
- Medium. A broad flag abstraction would be riskier than the current duplication. Do only a small helper if it clearly reduces missed updates.
### 4.2 Config source application repeats field-level assignments
Affected files/packages:
- `internal/core/config/file_config.go`
- `internal/core/config/env.go`
- `internal/core/config/flags.go`
Duplicated or near-duplicated behavior:
- The same effective fields are assigned from file config, env vars, and CLI overrides.
- Some semantics differ intentionally: file config supports `api_key_env`, env supports `OPENROUTER_API_KEY` fallback, CLI uses direct values.
Why it matters:
- Field additions are easy to miss in one source.
- Error messages and trimming behavior can drift.
Recommended refactor:
- Do not force all config sources through one generic mapper.
- Add small setter helpers for repeated config subdomains such as LLM target, concurrency, thresholds, normalization, and diagnostics.
- Keep source-specific parsing and error labels local.
Suggested tests:
- Cross-source table proving file, env, and CLI all reach the same effective fields where they are meant to.
- Tests for intentional differences: API key env resolution, `OPENROUTER_API_KEY` fallback, CLI direct API key, and transcript description trimming.
Risk level:
- Medium. Useful, but only after the effective loader and catalog cleanup.
### 4.3 Prompt metadata and response schema metadata map construction repeats
Affected files/packages:
- `internal/modules/*/module.go`
- `internal/framework/proposal_generation/generate.go`
- `internal/framework/validators/llm_validators.go`
- `internal/prompts`
- `internal/framework/responseschema`
Duplicated or near-duplicated behavior:
- Prompt metadata maps are manually expanded in module proposal generation and validator diagnostics.
- Response schema metadata maps are built independently in proposal generation and validator diagnostics.
Why it matters:
- Metadata fields are diagnostics-visible and useful for reproducibility.
- Adding a metadata field requires updating multiple call sites.
Recommended refactor:
- Add `Metadata.Map()` or a typed diagnostics metadata struct in `internal/prompts`.
- Add `responseschema.Metadata()` or a method returning a stable diagnostics shape.
- Prefer typed structs over `map[string]any` where possible.
Suggested tests:
- Prompt metadata rendering test should assert all registered prompts expose stable metadata.
- Proposal and validator diagnostics tests should assert the shared metadata helper is used.
Risk level:
- Low.
### 4.4 Secret redaction logic is split across config, LLM diagnostics, and adapter errors
Affected files/packages:
- `internal/core/config/redaction.go`
- `internal/framework/llm/diagnostics.go`
- `internal/framework/llm/client_common.go`
- `internal/framework/proposal_generation/generate.go`
- `internal/framework/runner/runner.go`
Duplicated or near-duplicated behavior:
- Config redaction replaces non-empty API keys with `[REDACTED]`.
- LLM diagnostics replace configured secret values and `Bearer <secret>`.
- Adapter error sanitization separately replaces secrets and bearer values.
- Proposal and validator paths separately assemble secret lists.
Why it matters:
- Secret redaction is a public guarantee.
- New secret-bearing config fields could be missed in one path.
Recommended refactor:
- Add a small redaction helper package or keep it in `internal/framework/llm` only if it remains LLM-specific.
- Centralize `[]string` secret extraction from `config.Config`.
- Keep config structural redaction separate from byte/string payload redaction, but share the redaction token and value replacement behavior.
Suggested tests:
- One test that a proposal-generation error, validator diagnostic artifact, effective config artifact, and surfaced provider error all redact the same configured secrets.
- Existing subprocess no-secret-leak test should remain as an end-to-end guard.
Risk level:
- Medium. The current coverage appears strong; change carefully.
### 4.5 Test fakes and fixture helpers are duplicated across packages
Affected files/packages:
- `internal/modules/*/module_test.go`
- `internal/framework/proposal_generation/generate_test.go`
- `internal/framework/validators/llm_validators_test.go`
- `internal/cli/run_test.go`
- `cmd/audita/main_integration_test.go`
- `internal/cli/release_fixtures_test.go`
- `internal/cli/parity_test.go`
Duplicated or near-duplicated behavior:
- Several packages define fake structured LLM clients, fixture path helpers, read/write helpers, diagnostics glob assertions, and run-directory helpers.
- The four module test files have particularly similar fake clients and proposal-diagnostics assertions.
Why it matters:
- Refactors in LLM or diagnostics behavior require updating many tests.
- Some duplicated tests are valuable because they preserve per-module public behavior; the issue is helper duplication, not coverage volume.
Recommended refactor:
- Add package-local helper files where duplication is within a package.
- For cross-package fakes, prefer a small internal test support package only if it does not create import cycles or hide test intent.
- Keep module-specific assertions local.
Suggested tests:
- This is test infrastructure cleanup. Existing tests should remain semantically equivalent.
- Add helper tests only if helpers contain nontrivial behavior, such as fake response sequencing.
Risk level:
- Low.
### 4.6 Stage-name construction is inconsistent enough to centralize, but not enough to redesign
Affected files/packages:
- `internal/modules/*/module.go`
- `internal/framework/proposal_generation/generate.go`
- `internal/framework/validators/llm_validators.go`
- `internal/framework/runner/observability.go`
Duplicated or near-duplicated behavior:
- Modules pass stage names like `<module_instance>:proposal:section-0001`.
- `proposal_generation` has a default builder using `<module_instance>:proposal-generation:section-0001`, but production modules bypass it.
- Validators build `<module_instance>:<validator>:batch-0001`.
- Utilization extracts module instance by splitting stage names on `:`.
Why it matters:
- Stage names affect diagnostics filenames and observability grouping.
- Current behavior works, but the naming grammar is implicit.
Recommended refactor:
- Add narrow helpers for proposal and validator stage names.
- Preserve current production stage names unless there is a deliberate pre-1.0 diagnostics compatibility decision.
- Keep filename sanitization in `internal/framework/llm`.
Suggested tests:
- Unit tests for stage-name helper output.
- Utilization test that module instance extraction still works for proposal and validator stage names.
Risk level:
- Medium. Renaming stages can change diagnostics filenames, so avoid unnecessary churn.
## 5. Boundary and responsibility concerns
### CLI owns too much report and diagnostics metadata assembly
`internal/cli/run.go` is doing orchestration, command parsing, config loading, output routing, report assembly, diagnostics metadata path assembly, and correction-ledger construction. This is acceptable for a small CLI, but two pieces are drifting beyond command responsibility:
- diagnostics artifact path metadata belongs closer to `internal/core/diagnostics`;
- report assembly and correction-ledger mapping belong closer to `internal/core/reporting` or a narrow reporting adapter package.
Recommended home:
- `internal/core/diagnostics`: artifact constants and diagnostics metadata path construction.
- `internal/core/reporting`: pure mapping from runner/config/diagnostics state into report payloads.
- `internal/cli`: command parsing, invocation wiring, exit codes, stdout/stderr behavior.
### Config validation lacks catalog ownership
`internal/core/config` currently validates only generic module list shape and hardcodes output schema keys. Because modules and output schemas are public contract values, config validation should use a catalog owned by the relevant domain.
Recommended home:
- output schema validation: `internal/core/outputschema`;
- module key validation: a small catalog package or lower-level constants package importable by config, module factory, validator chains, and threshold lookup.
### Runner owns adapter shims between contracts and validator framework
`internal/framework/runner` contains `validationLLMClientAdapter` and `llmDiagnosticsWriterAdapter`. This is not a serious problem today because runner wires proposal and validation workflows. If these adapters grow, move them to `internal/framework/validators` or a small integration package so runner remains focused on orchestration.
### LLM malformed-output policy is spread across callers
The LLM adapter emits the errors, while proposal generation and validators classify them by message text. The policy decision is caller-specific, but the classification should live with the LLM/framework error type.
## 6. Path, key, and naming construction review
Centralized enough:
- LLM diagnostics artifact suffixes and stage sanitization are centralized in `internal/framework/llm/diagnostics.go`.
- Output file writing is routed through `internal/core/io.WriteFile`.
- Run directories are created in `internal/core/diagnostics.NewRunDirectory`.
Needs cleanup:
- Core diagnostics artifact names are repeated between `RunDirectory` writer methods and `buildProcessReport`.
- `utilization-diagnostics.json` and `correction-ledger.json` are raw strings in both success and failure paths.
- Proposal and validator diagnostics subdirectory construction repeats `filepath.Join(diagnosticsDir, moduleInstance)`.
- Proposal and validator stage names are manually formatted in multiple packages.
- Module keys are repeated across config defaults, module factory, validator chains, confidence threshold lookup, and module implementations.
- Output schema names are repeated between config validation and `outputschema`.
Recommendation:
- Start with artifact constants and metadata helpers because that is the lowest-risk path/key cleanup.
- Then centralize stage-name helpers without changing current production naming.
- Defer any broader "path manager" abstraction.
## 7. Resolution and catalog review
Modules:
- Runtime module construction has a production registry in `internal/framework/modules`.
- Instance naming for repeated modules is centralized in `contracts.ResolveModuleRunSpecs`.
- Unknown module failure exists in the factory, but config validation does not catch unknown modules.
- Built-in validator chain resolution separately maps module key to validator keys.
Output schemas:
- Encoding is centralized in `internal/core/outputschema`.
- Validation is duplicated in config.
Prompts:
- Prompt asset lookup and metadata are centralized in `internal/prompts`.
- Prompt metadata map construction is repeated at call sites.
- Prompt source selection is intentionally built-in only and should remain that way for 1.0.
Validators:
- Validator construction is package-owned under `internal/validators`.
- Chains are centralized in `internal/validators/chains.go`.
- Execution class metadata exists, but reporting/correction-ledger classification does not fully use it.
Schemas:
- Transcript and glossary parsing/validation are centralized in `internal/core/schema`.
- Structured LLM response schemas are centralized in `internal/framework/responseschema`.
- Output schema registry and response schema registry are appropriately separate.
Recommendation:
- Introduce only small catalog helpers for module keys, output schema keys, prompt metadata maps, response schema metadata maps, and validator execution class.
- Avoid user-configurable modules, validators, prompts, or schemas before 1.0 unless already planned elsewhere.
## 8. Config and command-loading review
Consistent behavior:
- The documented precedence for `process` is implemented: defaults, file config, environment, CLI.
- `config print-effective` intentionally omits CLI process flags and uses defaults, file config, and environment.
- `config validate` intentionally requires `--config` and does not require transcript/glossary inputs.
- Missing explicit config paths are hard failures; missing default paths are non-fatal.
- Environment parsing and CLI parsing both preserve legacy total-concurrency alias behavior.
Likely accidental or high-risk differences:
- Unsupported module names pass `Config.Validate` and `audita config validate`.
- Output schema support is duplicated instead of delegated to the output schema registry.
- Config path resolution lives in CLI even though it is part of config behavior.
Intentional differences:
- File config resolves `api_key_env`; env and CLI set direct API key values.
- `OPENROUTER_API_KEY` is an environment fallback only for the primary LLM.
- `transcript-description` has CLI/config support but no `AUDITA_*` environment variable, matching documentation.
Recommendation:
- Build a shared effective config context helper and keep source-specific parsing semantics explicit.
- Tighten catalog validation before 1.0 if unknown modules are not meant to be accepted.
## 9. State, manifest, or progress handling review
Audita does not currently have a manifest/checkpoint/resume model. State is per-run diagnostics and report artifacts.
Consistent behavior:
- `process` creates one diagnostics run directory when diagnostics initialization succeeds.
- Failures after run-dir creation write `error.log`, best-effort report artifacts, and retain diagnostics.
- Success writes optional `--report-json`, run-dir `report.json`, utilization diagnostics, and correction ledger.
- Retention is centralized in `diagnostics.ShouldRetainRunDirectory`.
- There is no resume/retry/force behavior to preserve.
Drift risks:
- Success and failure paths both write utilization and correction-ledger artifacts with duplicated raw filenames.
- Report diagnostics metadata is assembled independently from the run-directory writer methods.
- Retention mode `never` currently still retains successful run directories in `ShouldRetainRunDirectory`, which may be intentional per tests or a naming/documentation mismatch. Do not change it in a dedup pass without first confirming semantics.
Recommendation:
- Centralize artifact names and report metadata path construction.
- Keep retention behavior unchanged unless a separate bug review confirms the intended meaning of `never`.
## 10. Refactors to avoid before 1.0
- Do not introduce a generic workflow engine. The current sequential runner is clear and explicit.
- Do not add a plugin architecture for modules, validators, prompts, or schemas before 1.0.
- Do not redesign the CLI or replace `flag` with a larger framework only for deduplication.
- Do not collapse all config source parsing into a reflection-based mapper; source semantics differ intentionally.
- Do not merge module packages into one generic module type. Keep domain-specific prompt assets, keys, validator chains, and replacement policies visible.
- Do not rewrite diagnostics or reporting schemas broadly. Centralize names and mapping helpers first.
- Do not change diagnostics stage names casually; they affect artifact filenames and debugging workflows.
- Do not consolidate deterministic and LLM validator behavior just because both return decisions. Their failure and batching semantics differ.
- Do not generalize transcript/glossary schema parsing into a broad schema framework.
- Do not reduce duplicated tests where the duplication protects distinct public command/module behavior.
## 11. Recommended implementation sequence
1. Centralize diagnostics artifact constants and diagnostics metadata path construction.
2. Centralize output schema validation through `internal/core/outputschema`.
3. Introduce a small module key catalog and use it in config validation, module factory, validator chains, and threshold lookup.
4. Add an effective config loading context helper for defaults + file + env, then update `process` and `config print-effective`.
5. Extract shared module proposal plumbing and prompt transcript-section payload construction.
6. Centralize prompt metadata and response schema metadata map construction.
7. Centralize validator execution-class lookup and update correction-ledger classification.
8. Centralize malformed structured-output classification through a typed/shared LLM error helper.
9. Add or consolidate focused test helpers for module LLM fakes, diagnostics assertions, and fixture paths.
10. Do a final dead-code and legacy sweep for redundant helper fields such as unused validator definition metadata.
Each item can be a separate commit with package-level tests and at least one CLI regression where public behavior is involved.
## 12. Test strategy
Tests to add before refactoring:
- `internal/core/config`: unknown module key fails validation, if unsupported modules are not intended to be accepted.
- `internal/core/config`: every output schema registry key validates through config.
- `internal/core/diagnostics`: report metadata paths match run-directory artifact names.
- `internal/validators`: validator class by key/instance is consistent for all registered validators.
- `internal/framework/llm`: shared malformed structured-output classifier covers all current adapter malformed errors.
Tests to add during refactoring:
- `internal/framework/promptcontext`: transcript section prompt payload preserves IDs, speaker, timestamps, text, and categories.
- `internal/framework/proposal_generation`: shared module proposal helper preserves current stage name, diagnostics dir, schema metadata, and malformed-output warning behavior.
- `internal/cli`: `process` and `config print-effective` share defaults+file+env behavior.
- `internal/cli`: `config validate` remains file-only and does not read env overrides.
- `internal/cli`: correction ledger classifies deterministic and LLM validator decisions through canonical metadata.
Existing tests to run after each cleanup:
- `go test ./internal/core/config ./internal/core/outputschema`
- `go test ./internal/core/diagnostics ./internal/core/reporting`
- `go test ./internal/framework/proposal_generation ./internal/framework/validators ./internal/framework/runner`
- `go test ./internal/validators/...`
- `go test ./internal/modules/...`
- `go test ./internal/cli ./cmd/audita`
- Run `go test ./...` before merging a multi-package cleanup.
Validation note:
- During this report-only pass, no full test suite was run. A lightweight `go list ./...` completed package listing but emitted a sandbox warning while trying to write the Go module stat cache outside the repository.
## 13. Appendix: findings not worth acting on
### Separate module packages
The four production module packages contain visible repetition, but keeping separate packages is useful. The module domains, prompt assets, validator chains, and tests are distinct enough that a single generic module package would hide important behavior.
Do not refactor now beyond shared proposal/prompt plumbing.
### Report type duplication between runner and reporting
`runner.ModuleResult` and `reporting.ModuleReport` look similar. Keeping separate runtime and public report shapes is reasonable because runner owns execution state and reporting owns serialized public schema.
Only centralize mapping helpers; do not merge the types.
### Transcript and glossary parsing stay separate
Transcript JSON and glossary YAML parsing have different formats, validation rules, and error messages. There is no useful shared parser abstraction to extract.
### Response schema registry and output schema registry stay separate
Structured LLM response schemas and transcript output schemas are both "schemas", but they serve different users and have different lifecycles. Do not combine their registries.
### `flag` package usage
The CLI command surface is small. Replacing `flag` with a larger CLI framework would not pay for itself before 1.0.
### Local test duplication that protects public behavior
Some test duplication in CLI, subprocess, parity, and release fixtures is intentional. These tests exercise different public surfaces and should remain explicit even if helpers are shared.
### Filesystem state as diagnostics state
Audita has no resume/checkpoint semantics. Treating diagnostics artifacts as filesystem outputs is currently acceptable. A manifest system would be speculative before there is a resume or audit workflow that needs it.

View File

@@ -0,0 +1,329 @@
# Pre-1.0 Deduplication Implementation Plan
This plan turns `docs/roadmap/audit.md` into staged, prompt-sized cleanup work for an LLM coding agent. Each stage should be implemented in order and kept small enough to review as an independent commit.
## Operating rules
- Read `docs/roadmap/audit.md` before starting any stage.
- Preserve public CLI, report, diagnostics, config precedence, prompt metadata, and output-schema behavior unless a stage explicitly calls out an intended behavior change.
- Keep the four production module packages separate: `glossary`, `homophones`, `spoken_word`, and `grammar`.
- Do not introduce plugin systems, generic workflow engines, broad CLI framework rewrites, reflection-heavy config mappers, or merged module packages.
- Prefer narrow helpers, catalogs, constants, and pure mapping functions over broad abstractions.
- Run the targeted tests listed in each stage before moving to the next stage.
- Run `go test ./...` before declaring the full sequence complete.
- Ignore unrelated worktree changes, including the existing deletion of `docs/roadmap/publish.md`, unless the user explicitly asks to handle them.
- Do not reduce parity, release-fixture, subprocess, or module-specific behavior coverage while consolidating helpers.
## Stages
### Stage 1: Diagnostics artifact constants and metadata paths
Goal:
- Centralize diagnostics artifact names and report diagnostics metadata path construction without changing any filenames or report fields.
Key edits:
- Define constants in `internal/core/diagnostics` for:
- `source-transcript.json`
- `source-transcript-parsed.json`
- `normalized-transcript.json`
- `normalization-summary.json`
- `chunking-summary.json`
- `utilization-diagnostics.json`
- `correction-ledger.json`
- `invocation.json`
- `effective-config.json`
- `report.json`
- `error.log`
- Add a diagnostics helper that builds `reporting.DiagnosticsMetadata` from a run directory path and failure/success status.
- Update `RunDirectory` methods to use the constants.
- Update CLI report assembly and utilization/correction-ledger writes to use the constants/helper instead of raw strings.
Behavior changes:
- None. All artifact names, report JSON keys, and path values must remain byte-for-byte compatible except for normal timestamp/order differences in existing outputs.
Tests:
- Add or update `internal/core/diagnostics` tests proving metadata helper paths match the artifact constants.
- Run `go test ./internal/core/diagnostics ./internal/core/reporting ./internal/cli`.
- Run any existing CLI report/diagnostics tests touched by this stage.
Acceptance criteria:
- No raw core diagnostics artifact filename strings remain in CLI report metadata assembly.
- Existing success and failure reports still point to files that are actually written.
- Retention behavior is unchanged.
### Stage 2: Output schema validation and module catalog
Goal:
- Move public key validation to small canonical catalogs so config validation, runtime resolution, and factory behavior cannot drift.
Key edits:
- Add `SupportedKeys`, `IsSupported`, or an equivalent validation helper to `internal/core/outputschema`.
- Update `config.Validate` to use `internal/core/outputschema` for output schema validation.
- Add a small canonical module key catalog that is importable by:
- `internal/core/config`
- `internal/framework/modules`
- `internal/validators`
- `internal/framework/validators`
- Use the module catalog for default module key constants, known-key checks, validator chain keys, and confidence-threshold lookup.
- Keep module construction in `internal/framework/modules`; the catalog must not construct modules.
Behavior changes:
- Intended behavior change: unsupported configured module keys should fail during config validation, including `audita config validate`.
- Repeated supported module keys remain valid.
- Output schema behavior remains unchanged for `bare-segments`, `audita-v1`, and unsupported names.
Tests:
- Add `internal/core/config` tests for unsupported module keys and repeated supported module keys.
- Add config validation tests that every supported output schema validates.
- Add or update output schema registry tests for supported and unsupported schemas.
- Update module registry and validator chain tests to use the shared catalog where appropriate.
- Run `go test ./internal/core/config ./internal/core/outputschema ./internal/framework/modules ./internal/framework/validators ./internal/validators/... ./internal/cli`.
Acceptance criteria:
- Unknown modules fail before runner setup in config validation paths.
- No duplicated hardcoded output schema support list remains in config validation.
- No import cycle is introduced.
### Stage 3: Effective config loading context
Goal:
- Centralize config path resolution and defaults+file+env loading while keeping command-specific CLI overrides explicit.
Key edits:
- Move config path resolution from `internal/cli` into `internal/core/config` or add an equivalent exported helper there.
- Add an effective config loader that returns:
- effective `config.Config`
- config path
- config source (`flag`, `env`, `default`, or empty)
- config version pointer when a file was loaded
- Use the shared loader in `audita process` before applying CLI overrides.
- Use the shared loader in `audita config print-effective`.
- Keep `audita config validate` as file-only: load file, apply to defaults, validate, and do not apply environment overrides.
Behavior changes:
- None. Preserve existing precedence:
- `process`: defaults, file config, environment, CLI flags
- `config print-effective`: defaults, file config, environment
- `config validate`: file config applied to defaults only
- Preserve explicit config path failure behavior and missing default path non-fatal behavior.
Tests:
- Add table-driven config loader tests for:
- explicit `--config`
- `AUDITA_CONFIG`
- default search paths
- missing explicit path
- missing env path
- missing default paths
- Add or update CLI tests proving `process` and `config print-effective` share file+env behavior.
- Add or update CLI tests proving `config validate` ignores environment overrides.
- Run `go test ./internal/core/config ./internal/cli ./cmd/audita`.
Acceptance criteria:
- Config precedence is unchanged.
- Config source/path/version metadata in invocation and reports is unchanged.
- Config command stdout/stderr and exit-code behavior is unchanged except for the intended unknown-module validation from Stage 2.
### Stage 4: Prompt/schema metadata and stage-name helpers
Goal:
- Centralize diagnostics-visible metadata and stage-name construction without changing production diagnostics names.
Key edits:
- Add a helper or method in `internal/prompts` that returns the stable prompt metadata diagnostics shape currently expanded by call sites.
- Add a helper or method in `internal/framework/responseschema` that returns the stable response schema metadata diagnostics shape currently expanded by call sites.
- Add shared proposal and validator stage-name helpers in the lowest package that avoids import cycles.
- Use the helpers in proposal generation, LLM validators, and production modules.
Behavior changes:
- None. Preserve current production stage names:
- module proposal stages keep their existing `proposal` naming form;
- validator batch stages keep their existing validator/batch naming form.
- Preserve all prompt metadata and response schema metadata field names and values.
Tests:
- Add prompt metadata helper tests covering every registered prompt.
- Add response schema metadata helper tests covering every registered response schema.
- Add stage-name helper tests for no-section, section, and validator batch cases.
- Run `go test ./internal/prompts ./internal/framework/responseschema ./internal/framework/proposal_generation ./internal/framework/validators ./internal/modules/...`.
Acceptance criteria:
- No manual prompt metadata map expansion remains in production module proposal plumbing.
- No duplicated response schema metadata map construction remains in proposal generation and LLM validators.
- Existing diagnostics fixture/path assertions still pass.
### Stage 5: Shared module proposal and prompt payload plumbing
Goal:
- Remove duplicated proposal execution and transcript-section prompt payload construction while preserving module-specific domain behavior.
Key edits:
- Add a narrow shared proposal execution helper, preferably in `internal/framework/proposal_generation` unless import cycles require a small module helper package.
- The helper should own:
- transcript description extraction from config;
- `GenerateCandidates` request construction;
- prompt metadata attachment;
- stage-name selection;
- conversion from generated corrections/warnings to `contracts.ProposalResult`.
- Add shared transcript-section prompt payload construction in `internal/framework/promptcontext`.
- Update each production module to provide only:
- module key;
- replacement policy;
- validator chain;
- prompt ID;
- domain-specific `BuildProposalMessages` call or message builder.
- Remove each module's redundant section transcript filtering if the runner already passes section-limited transcripts.
Behavior changes:
- None. Preserve module keys, replacement policies, validator chains, prompt IDs, diagnostics directories, proposal indexes, warning behavior, and correction mapping.
Tests:
- Add promptcontext tests for transcript section payload shape, empty transcript handling, section index, and category copying.
- Keep one module-specific prompt test per production module for domain wording and constraints.
- Add or update module proposal tests proving diagnostics are still written under the same module instance directory.
- Run `go test ./internal/framework/promptcontext ./internal/framework/proposal_generation ./internal/modules/... ./internal/cli`.
Acceptance criteria:
- Four production modules share proposal execution plumbing.
- Module packages remain separate and readable.
- CLI parity and release fixture behavior is unchanged.
### Stage 6: Validator classification and malformed LLM output policy
Goal:
- Use one source of truth for validator execution class and one shared classifier for malformed structured-output errors.
Key edits:
- Make validator execution class resolvable by stable validator key and by validator instance.
- Replace the correction-ledger hardcoded LLM-backed validator map with the canonical metadata source.
- Remove redundant validator metadata fields only after all call sites use the canonical source.
- Add a shared malformed structured-output classifier in `internal/framework/llm` or another low-level framework package.
- Update proposal generation and LLM validators to use the shared classifier while preserving their different handling outcomes.
Behavior changes:
- None. Proposal-generation malformed payloads still downgrade to warnings with zero proposals for affected sections.
- Validator malformed payloads still reject affected batches with warnings.
- Correction-ledger deterministic vs LLM validator sections should be unchanged for current validators.
Tests:
- Add validator metadata tests proving every registered validator has the expected execution class by key and instance.
- Add correction-ledger tests proving deterministic and LLM-backed decisions are classified through canonical metadata.
- Add shared malformed-output classifier tests covering current adapter malformed-output messages.
- Update proposal-generation and validator tests to assert representative malformed adapter errors are still downgraded.
- Run `go test ./internal/validators/... ./internal/framework/validators ./internal/framework/proposal_generation ./internal/framework/llm ./internal/cli`.
Acceptance criteria:
- No local hardcoded LLM-backed validator map remains in correction-ledger construction.
- Proposal-generation and validator malformed-output classifier lists cannot drift.
- Existing runner validator ordering is unchanged.
### Stage 7: Redaction and adapter workflow cleanup
Goal:
- Reduce duplicated secret extraction/redaction setup while preserving all no-secret-leak guarantees.
Key edits:
- Add a shared helper that extracts all configured LLM secret values from `config.Config`.
- Use the helper in proposal-generation diagnostics and validator diagnostics setup.
- Keep config structural redaction (`Config.Redacted`) separate from byte/string payload redaction.
- Keep adapter error redaction behavior compatible with current surfaced errors.
- Move runner adapter shims only if Stage 6 or this stage makes them materially larger; otherwise leave them in runner.
Behavior changes:
- None. Redaction token and no-secret-leak behavior remain unchanged.
Tests:
- Add or update tests proving proposal diagnostics, validator diagnostics, effective config artifacts, and surfaced adapter errors redact the same configured secrets.
- Keep existing subprocess no-secret-leak tests.
- Run `go test ./internal/core/config ./internal/framework/llm ./internal/framework/proposal_generation ./internal/framework/validators ./internal/cli ./cmd/audita`.
Acceptance criteria:
- Secret-list assembly is no longer duplicated between proposal and validator paths.
- No plaintext configured API key appears in diagnostics, reports, stdout, or stderr in existing redaction tests.
- No unrelated adapter behavior changes.
### Stage 8: Test helper cleanup and dead-code sweep
Goal:
- Consolidate test-only duplication and remove dead/redundant code left by prior stages.
Key edits:
- Consolidate package-local fake LLM clients, fixture readers, diagnostics glob helpers, and run-directory helpers where duplication is clear.
- Use cross-package test support only if it does not obscure test intent or introduce awkward imports.
- Remove redundant metadata fields, constants, or helper functions made obsolete by earlier stages.
- Keep module-specific prompt and behavior assertions local to each module package.
Behavior changes:
- None.
Tests:
- Run all package tests touched by helper cleanup.
- Run `go test ./internal/modules/... ./internal/framework/... ./internal/cli ./cmd/audita`.
- Run `go test ./...` before completing the full sequence.
Acceptance criteria:
- Test helpers are simpler without reducing coverage.
- No parity or release fixture assertions are removed unless replaced by equivalent or stronger assertions.
- No production behavior changes.
## Final verification
Before declaring the staged cleanup complete:
- Run:
- `go test ./internal/core/config ./internal/core/outputschema`
- `go test ./internal/core/diagnostics ./internal/core/reporting`
- `go test ./internal/framework/proposal_generation ./internal/framework/validators ./internal/framework/runner`
- `go test ./internal/validators/...`
- `go test ./internal/modules/...`
- `go test ./internal/cli ./cmd/audita`
- `go test ./...`
- Inspect `git diff` for accidental public CLI, config, report, diagnostics, prompt metadata, stage-name, or output-schema changes.
- Update docs only when behavior intentionally changes, especially the intended Stage 2 unknown-module validation change.
- Keep commits stage-sized and mention behavior-preservation tests in each commit message or PR description.
## Assumptions
- Unknown configured module keys should become config-validation failures before 1.0.
- Diagnostics filenames and stage names are public enough to preserve unless a stage explicitly says otherwise.
- Each stage should be implemented and reviewed separately.

11
go.mod Normal file
View File

@@ -0,0 +1,11 @@
module gitea.maximumdirect.net/eric/audita
go 1.24.0
require gopkg.in/yaml.v3 v3.0.1
require (
github.com/kr/pretty v0.3.1 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
)

17
go.sum Normal file
View File

@@ -0,0 +1,17 @@
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

338
internal/cli/parity_test.go Normal file
View File

@@ -0,0 +1,338 @@
package cli
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
)
type parityFixtureCase struct {
Name string `json:"name"`
TranscriptFile string `json:"transcript_file"`
GlossaryFile string `json:"glossary_file"`
ModulesCSV string `json:"modules_csv,omitempty"`
ProposalResponsesFile string `json:"proposal_responses_file,omitempty"`
ValidationResponsesFile string `json:"validation_responses_file,omitempty"`
Env map[string]string `json:"env,omitempty"`
Expect parityExpectation `json:"expect"`
}
type parityExpectation struct {
ExitCode int `json:"exit_code"`
Status string `json:"status,omitempty"`
ErrorPhase string `json:"error_phase,omitempty"`
StdoutMode string `json:"stdout_mode,omitempty"` // empty|json
StderrContains string `json:"stderr_contains,omitempty"`
OutputTranscriptFile string `json:"output_transcript_file,omitempty"`
ModuleInstances []string `json:"module_instances,omitempty"`
ModuleCount int `json:"module_count,omitempty"`
TotalAppliedChanges int `json:"total_applied_changes,omitempty"`
TotalSkippedChanges int `json:"total_skipped_changes,omitempty"`
FailedModuleInstance string `json:"failed_module_instance,omitempty"`
ValidatorRejectedReasonCodes []string `json:"validator_rejected_reason_codes,omitempty"`
ApplicationSkipReasonCodes []string `json:"application_skip_reason_codes,omitempty"`
RequireErrorLog bool `json:"require_error_log,omitempty"`
SecretMarkers []string `json:"secret_markers,omitempty"`
ExpectedProposalCalls []string `json:"expected_proposal_calls,omitempty"`
ExpectedValidationCalls []string `json:"expected_validation_calls,omitempty"`
ModuleAppliedCounts []int `json:"module_applied_counts,omitempty"`
ModuleRejectedCounts []int `json:"module_rejected_counts,omitempty"`
ModuleSkipCounts []int `json:"module_skip_counts,omitempty"`
MinResponsePayloadArtifacts int `json:"min_response_payload_artifacts,omitempty"`
}
func TestParityFixtures(t *testing.T) {
casePaths, err := filepath.Glob(parityFixturePath("*.case.json"))
if err != nil {
t.Fatalf("glob parity fixtures: %v", err)
}
if len(casePaths) == 0 {
t.Fatal("expected at least one parity fixture case")
}
for _, casePath := range casePaths {
fx := loadParityFixtureCase(t, casePath)
t.Run(fx.Name, func(t *testing.T) {
runParityFixtureCase(t, filepath.Dir(casePath), fx)
})
}
}
func loadParityFixtureCase(t *testing.T, casePath string) parityFixtureCase {
t.Helper()
var fx parityFixtureCase
raw := readFile(t, casePath)
if err := json.Unmarshal(raw, &fx); err != nil {
t.Fatalf("parse parity case %q: %v", casePath, err)
}
if strings.TrimSpace(fx.Name) == "" {
t.Fatalf("parity case %q missing name", casePath)
}
return fx
}
func runParityFixtureCase(t *testing.T, caseDir string, fx parityFixtureCase) {
t.Helper()
for k, v := range fx.Env {
t.Setenv(k, v)
}
proposalClient := &fakeStructuredLLMClient{}
validationClient := &fakeStructuredLLMClient{}
if strings.TrimSpace(fx.ProposalResponsesFile) != "" {
raw := readFile(t, filepath.Join(caseDir, fx.ProposalResponsesFile))
if err := json.Unmarshal(raw, &proposalClient.proposalResponses); err != nil {
t.Fatalf("parse proposal responses: %v", err)
}
processProposalLLMClient = proposalClient
}
if strings.TrimSpace(fx.ValidationResponsesFile) != "" {
raw := readFile(t, filepath.Join(caseDir, fx.ValidationResponsesFile))
if err := json.Unmarshal(raw, &validationClient.validationResponses); err != nil {
t.Fatalf("parse validation responses: %v", err)
}
processValidationLLMClient = validationClient
}
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
var stdout, stderr bytes.Buffer
workDir := t.TempDir()
outputPath := filepath.Join(t.TempDir(), "out.json")
reportPath := filepath.Join(t.TempDir(), "report.json")
args := []string{
"process",
filepath.Join(caseDir, fx.TranscriptFile),
"--glossary",
filepath.Join(caseDir, fx.GlossaryFile),
"--report-json",
reportPath,
"--work-dir",
workDir,
"--work-dir-retention",
"always",
}
// Keep stdout shape deterministic for parity tests.
if fx.Expect.StdoutMode != "json" {
args = append(args, "--output", outputPath)
}
if strings.TrimSpace(fx.ModulesCSV) != "" {
args = append(args, "--modules", fx.ModulesCSV)
}
exitCode := Run(args, &stdout, &stderr)
if exitCode != fx.Expect.ExitCode {
t.Fatalf("expected exit code %d, got %d stderr=%q", fx.Expect.ExitCode, exitCode, stderr.String())
}
switch fx.Expect.StdoutMode {
case "json":
if _, err := json.Marshal(stdout.String()); err != nil {
t.Fatalf("unexpected stdout marshal error: %v", err)
}
if !json.Valid(stdout.Bytes()) {
t.Fatalf("expected JSON stdout, got %q", stdout.String())
}
default:
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout, got %q", stdout.String())
}
}
if fx.Expect.StderrContains != "" && !strings.Contains(stderr.String(), fx.Expect.StderrContains) {
t.Fatalf("expected stderr to contain %q, got %q", fx.Expect.StderrContains, stderr.String())
}
report := readProcessReport(t, reportPath)
assertParityReport(t, report, fx.Expect)
runDir := onlyRunDir(t, workDir)
runDirReport := readProcessReport(t, filepath.Join(runDir, "report.json"))
assertParityReport(t, runDirReport, fx.Expect)
if fx.Expect.RequireErrorLog {
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log: %v", err)
}
}
if strings.TrimSpace(fx.Expect.OutputTranscriptFile) != "" && fx.Expect.ExitCode == 0 {
got := readFile(t, outputPath)
want := readFile(t, filepath.Join(caseDir, fx.Expect.OutputTranscriptFile))
assertJSONSemanticEqual(t, want, got)
}
if len(fx.Expect.ExpectedProposalCalls) > 0 && !reflect.DeepEqual(proposalClient.calls, fx.Expect.ExpectedProposalCalls) {
t.Fatalf("unexpected proposal calls: got %v want %v", proposalClient.calls, fx.Expect.ExpectedProposalCalls)
}
if len(fx.Expect.ExpectedValidationCalls) > 0 && !reflect.DeepEqual(validationClient.calls, fx.Expect.ExpectedValidationCalls) {
t.Fatalf("unexpected validation calls: got %v want %v", validationClient.calls, fx.Expect.ExpectedValidationCalls)
}
if len(fx.Expect.SecretMarkers) > 0 {
assertNoSecretMarkers(t, reportPath, fx.Expect.SecretMarkers)
assertNoSecretMarkersInTree(t, runDir, fx.Expect.SecretMarkers)
}
if fx.Expect.MinResponsePayloadArtifacts > 0 {
matches, err := filepath.Glob(filepath.Join(runDir, "*", "*response-payload.json"))
if err != nil {
t.Fatalf("glob response payload artifacts: %v", err)
}
if len(matches) < fx.Expect.MinResponsePayloadArtifacts {
t.Fatalf("expected at least %d response payload artifacts, got %d", fx.Expect.MinResponsePayloadArtifacts, len(matches))
}
}
}
func assertParityReport(t *testing.T, report reporting.ProcessReport, exp parityExpectation) {
t.Helper()
if exp.Status != "" && report.Status != exp.Status {
t.Fatalf("expected report status %q, got %q", exp.Status, report.Status)
}
if exp.ErrorPhase != "" && report.ErrorPhase != exp.ErrorPhase {
t.Fatalf("expected report error_phase %q, got %q", exp.ErrorPhase, report.ErrorPhase)
}
if len(exp.ModuleInstances) > 0 {
got := make([]string, 0, len(report.ModuleResults))
for _, mr := range report.ModuleResults {
got = append(got, mr.ModuleInstance)
}
if !reflect.DeepEqual(got, exp.ModuleInstances) {
t.Fatalf("unexpected module instances: got %v want %v", got, exp.ModuleInstances)
}
}
if exp.ModuleCount > 0 {
if report.ModulesSummary == nil || report.ModulesSummary.ModuleCount != exp.ModuleCount {
t.Fatalf("expected module_count=%d, got %+v", exp.ModuleCount, report.ModulesSummary)
}
}
if exp.TotalAppliedChanges > 0 {
if report.ModulesSummary == nil || report.ModulesSummary.TotalAppliedChanges != exp.TotalAppliedChanges {
t.Fatalf("expected total_applied_changes=%d, got %+v", exp.TotalAppliedChanges, report.ModulesSummary)
}
}
if exp.TotalSkippedChanges > 0 {
if report.ModulesSummary == nil || report.ModulesSummary.TotalSkippedChanges != exp.TotalSkippedChanges {
t.Fatalf("expected total_skipped_changes=%d, got %+v", exp.TotalSkippedChanges, report.ModulesSummary)
}
}
if exp.FailedModuleInstance != "" {
if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != exp.FailedModuleInstance {
t.Fatalf("expected failed_module_instance=%q, got %+v", exp.FailedModuleInstance, report.ModulesSummary)
}
}
if len(exp.ValidatorRejectedReasonCodes) > 0 {
got := collectValidatorRejectedReasonCodes(report.ModuleResults)
if !reflect.DeepEqual(got, exp.ValidatorRejectedReasonCodes) {
t.Fatalf("unexpected validator rejected reason codes: got %v want %v", got, exp.ValidatorRejectedReasonCodes)
}
}
if len(exp.ApplicationSkipReasonCodes) > 0 {
got := collectApplicationSkipReasonCodes(report.ModuleResults)
if !reflect.DeepEqual(got, exp.ApplicationSkipReasonCodes) {
t.Fatalf("unexpected application skip reason codes: got %v want %v", got, exp.ApplicationSkipReasonCodes)
}
}
if len(exp.ModuleAppliedCounts) > 0 {
got := make([]int, 0, len(report.ModuleResults))
for _, mr := range report.ModuleResults {
got = append(got, len(mr.AppliedChanges))
}
if !reflect.DeepEqual(got, exp.ModuleAppliedCounts) {
t.Fatalf("unexpected per-module applied counts: got %v want %v", got, exp.ModuleAppliedCounts)
}
}
if len(exp.ModuleRejectedCounts) > 0 {
got := make([]int, 0, len(report.ModuleResults))
for _, mr := range report.ModuleResults {
got = append(got, len(mr.ValidatorRejected))
}
if !reflect.DeepEqual(got, exp.ModuleRejectedCounts) {
t.Fatalf("unexpected per-module rejected counts: got %v want %v", got, exp.ModuleRejectedCounts)
}
}
if len(exp.ModuleSkipCounts) > 0 {
got := make([]int, 0, len(report.ModuleResults))
for _, mr := range report.ModuleResults {
got = append(got, len(mr.SkippedChanges))
}
if !reflect.DeepEqual(got, exp.ModuleSkipCounts) {
t.Fatalf("unexpected per-module skip counts: got %v want %v", got, exp.ModuleSkipCounts)
}
}
}
func collectValidatorRejectedReasonCodes(results []reporting.ModuleReport) []string {
out := make([]string, 0)
for _, mr := range results {
for _, vr := range mr.ValidatorRejected {
out = append(out, vr.ReasonCode)
}
}
return out
}
func collectApplicationSkipReasonCodes(results []reporting.ModuleReport) []string {
out := make([]string, 0)
for _, mr := range results {
for _, sk := range mr.SkippedChanges {
out = append(out, string(sk.SkipReason))
}
}
return out
}
func assertJSONSemanticEqual(t *testing.T, expected []byte, actual []byte) {
t.Helper()
var exp any
var act any
if err := json.Unmarshal(expected, &exp); err != nil {
t.Fatalf("unmarshal expected json: %v", err)
}
if err := json.Unmarshal(actual, &act); err != nil {
t.Fatalf("unmarshal actual json: %v", err)
}
if !reflect.DeepEqual(exp, act) {
t.Fatalf("JSON mismatch\nexpected=%s\nactual=%s", string(expected), string(actual))
}
}
func assertNoSecretMarkers(t *testing.T, filePath string, markers []string) {
t.Helper()
raw := string(readFile(t, filePath))
for _, marker := range markers {
if marker != "" && strings.Contains(raw, marker) {
t.Fatalf("secret marker %q leaked in %s", marker, filePath)
}
}
}
func assertNoSecretMarkersInTree(t *testing.T, root string, markers []string) {
t.Helper()
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil || d == nil || d.IsDir() {
return nil
}
raw := string(readFile(t, path))
for _, marker := range markers {
if marker != "" && strings.Contains(raw, marker) {
t.Fatalf("secret marker %q leaked in %s", marker, path)
}
}
return nil
})
}
func parityFixturePath(name string) string {
return filepath.Join("testdata", "parity", name)
}

View File

@@ -0,0 +1,469 @@
package cli
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
type releaseFixtureExpectations struct {
MustApplyTexts []string `json:"must_apply_texts"`
MustNotApplyTexts []string `json:"must_not_apply_texts"`
ProtectedTerms []string `json:"protected_terms"`
ExpectedModuleInstance []string `json:"expected_module_instances"`
MinimumCounts struct {
Applied int `json:"applied"`
Rejected int `json:"rejected"`
Skipped int `json:"skipped"`
} `json:"minimum_counts"`
}
func TestReleaseFixtureDefaultPipelineReadiness(t *testing.T) {
base := fixturePath(filepath.Join("release", "default-release"))
var expectations releaseFixtureExpectations
if err := json.Unmarshal(readFile(t, base+".expectations.json"), &expectations); err != nil {
t.Fatalf("unmarshal release expectations: %v", err)
}
proposalResponses := readProposalResponses(t, base+".proposals.json")
validationResponses := readValidationResponses(t, base+".validations.json")
// First pass: default full pipeline with deterministic fake LLM responses.
first := runReleaseFixturePass(t, releaseRunConfig{
transcriptPath: base + ".transcript.json",
glossaryPath: base + ".glossary.yaml",
outputSchema: "bare-segments",
proposalResponses: proposalResponses,
validationResponses: validationResponses,
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
reportSchemaName: reporting.DefaultProcessReportSchemaName,
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
expectedOutputSchema: "bare-segments",
expectModuleInstances: expectations.ExpectedModuleInstance,
expectUtilizationPaths: true,
})
gotTranscript := mustReadTranscript(t, first.outputPath)
expectFinalTranscriptContains(t, gotTranscript, expectations.MustApplyTexts)
expectFinalTranscriptDoesNotContain(t, gotTranscript, expectations.MustNotApplyTexts)
expectFinalTranscriptContains(t, gotTranscript, expectations.ProtectedTerms)
assertReleaseCounts(t, first.report, expectations)
assertPromptAndSchemaMetadataPresent(t, first.runDir)
assertReleaseLedgerShape(t, first.report)
assertReleaseUtilizationShape(t, first.report)
assertStableValidatorKeysPresent(t, first.report)
assertStdoutStderrContract(t, first.stdout, first.stderr)
assertNoSecretMarkersInTree(t, first.runDir, []string{"release-secret"})
assertNoSecretMarkers(t, first.reportPath, []string{"release-secret"})
// Output schema check: audita-v1 object payload.
auditaV1 := runReleaseFixturePass(t, releaseRunConfig{
transcriptPath: base + ".transcript.json",
glossaryPath: base + ".glossary.yaml",
outputSchema: "audita-v1",
proposalResponses: proposalResponses,
validationResponses: validationResponses,
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
reportSchemaName: reporting.DefaultProcessReportSchemaName,
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
expectedOutputSchema: "audita-v1",
expectModuleInstances: expectations.ExpectedModuleInstance,
expectUtilizationPaths: true,
})
assertAuditaV1OutputShape(t, auditaV1.outputPath)
// Idempotence-oriented second pass:
// run again on first output with deterministic no-op responses.
noOpProposals := make([]proposal_generation.StructuredCorrectionSet, 5)
for i := range noOpProposals {
noOpProposals[i] = proposal_generation.StructuredCorrectionSet{Corrections: nil}
}
second := runReleaseFixturePass(t, releaseRunConfig{
transcriptPath: first.outputPath,
glossaryPath: base + ".glossary.yaml",
outputSchema: "bare-segments",
proposalResponses: noOpProposals,
validationResponses: nil,
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
reportSchemaName: reporting.DefaultProcessReportSchemaName,
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
expectedOutputSchema: "bare-segments",
expectModuleInstances: expectations.ExpectedModuleInstance,
expectUtilizationPaths: true,
})
firstSegments := mustReadTranscript(t, first.outputPath)
secondSegments := mustReadTranscript(t, second.outputPath)
if !reflect.DeepEqual(firstSegments, secondSegments) {
t.Fatalf("expected idempotent second pass transcript; first=%+v second=%+v", firstSegments, secondSegments)
}
if second.report.ModulesSummary == nil {
t.Fatalf("expected modules summary on second pass")
}
if second.report.ModulesSummary.TotalAppliedChanges != 0 {
t.Fatalf("expected no-op second pass (0 applied), got %+v", second.report.ModulesSummary)
}
}
type releaseRunConfig struct {
transcriptPath string
glossaryPath string
outputSchema string
proposalResponses []proposal_generation.StructuredCorrectionSet
validationResponses []validators.LLMValidationResponse
expectedProposalCalls []string
reportSchemaName string
reportSchemaVersion string
expectedOutputSchema string
expectModuleInstances []string
expectUtilizationPaths bool
}
type releaseRunResult struct {
stdout string
stderr string
outputPath string
reportPath string
report reporting.ProcessReport
runDir string
}
func runReleaseFixturePass(t *testing.T, cfg releaseRunConfig) releaseRunResult {
t.Helper()
processProposalLLMClient = &fakeStructuredLLMClient{proposalResponses: append([]proposal_generation.StructuredCorrectionSet(nil), cfg.proposalResponses...)}
processValidationLLMClient = &fakeStructuredLLMClient{validationResponses: append([]validators.LLMValidationResponse(nil), cfg.validationResponses...)}
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
configPath := writeFile(t, "release-config.yml", "version: 1\n")
args := []string{
"process",
cfg.transcriptPath,
"--glossary",
cfg.glossaryPath,
"--config",
configPath,
"--output",
outputPath,
"--output-schema",
cfg.outputSchema,
"--report-json",
reportPath,
"--work-dir",
workDir,
"--work-dir-retention",
"always",
}
var stdout, stderr bytes.Buffer
exitCode := Run(args, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
report := readProcessReport(t, reportPath)
if report.ReportMetadata.ReportSchemaName != cfg.reportSchemaName || report.ReportMetadata.ReportSchemaVersion != cfg.reportSchemaVersion {
t.Fatalf("unexpected report schema metadata: %+v", report.ReportMetadata)
}
if report.ReportMetadata.OutputSchema != cfg.expectedOutputSchema {
t.Fatalf("unexpected output schema metadata: got %q want %q", report.ReportMetadata.OutputSchema, cfg.expectedOutputSchema)
}
if len(cfg.expectModuleInstances) > 0 {
got := make([]string, 0, len(report.ModuleResults))
for _, mr := range report.ModuleResults {
got = append(got, mr.ModuleInstance)
}
if !reflect.DeepEqual(got, cfg.expectModuleInstances) {
t.Fatalf("unexpected module instances: got %v want %v", got, cfg.expectModuleInstances)
}
}
if report.Diagnostics == nil {
t.Fatalf("expected diagnostics metadata")
}
if cfg.expectUtilizationPaths {
if report.Diagnostics.UtilizationSummaryPath == "" || report.Diagnostics.CorrectionLedgerPath == "" {
t.Fatalf("expected utilization/ledger artifact paths in report diagnostics: %+v", report.Diagnostics)
}
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil {
t.Fatalf("expected run-dir report: %v", err)
}
if c, ok := processProposalLLMClient.(*fakeStructuredLLMClient); ok {
if !reflect.DeepEqual(c.calls, cfg.expectedProposalCalls) {
t.Fatalf("unexpected proposal call order: got %v want %v", c.calls, cfg.expectedProposalCalls)
}
}
return releaseRunResult{
stdout: stdout.String(),
stderr: stderr.String(),
outputPath: outputPath,
reportPath: reportPath,
report: report,
runDir: runDir,
}
}
func readProposalResponses(t *testing.T, path string) []proposal_generation.StructuredCorrectionSet {
t.Helper()
var out []proposal_generation.StructuredCorrectionSet
if err := json.Unmarshal(readFile(t, path), &out); err != nil {
t.Fatalf("unmarshal proposal responses: %v", err)
}
return out
}
func readValidationResponses(t *testing.T, path string) []validators.LLMValidationResponse {
t.Helper()
var out []validators.LLMValidationResponse
if err := json.Unmarshal(readFile(t, path), &out); err != nil {
t.Fatalf("unmarshal validation responses: %v", err)
}
return out
}
func mustReadTranscript(t *testing.T, path string) []schema.Segment {
t.Helper()
transcript, err := schema.ParseTranscriptJSON(readFile(t, path))
if err != nil {
t.Fatalf("parse transcript output: %v", err)
}
return transcript.Segments
}
func expectFinalTranscriptContains(t *testing.T, segments []schema.Segment, needles []string) {
t.Helper()
joined := flattenTranscriptText(segments)
for _, needle := range needles {
if !strings.Contains(joined, needle) {
t.Fatalf("expected transcript to contain %q, got %q", needle, joined)
}
}
}
func expectFinalTranscriptDoesNotContain(t *testing.T, segments []schema.Segment, needles []string) {
t.Helper()
joined := flattenTranscriptText(segments)
for _, needle := range needles {
if strings.Contains(joined, needle) {
t.Fatalf("expected transcript to not contain %q, got %q", needle, joined)
}
}
}
func flattenTranscriptText(segments []schema.Segment) string {
parts := make([]string, 0, len(segments))
for _, s := range segments {
parts = append(parts, s.Text)
}
return strings.Join(parts, "\n")
}
func assertReleaseCounts(t *testing.T, report reporting.ProcessReport, exp releaseFixtureExpectations) {
t.Helper()
if report.ModulesSummary == nil {
t.Fatalf("expected modules_summary")
}
if report.ModulesSummary.TotalAppliedChanges < exp.MinimumCounts.Applied {
t.Fatalf("expected at least %d applied changes, got %+v", exp.MinimumCounts.Applied, report.ModulesSummary)
}
validatorRejected := 0
skipped := 0
for _, mr := range report.ModuleResults {
validatorRejected += len(mr.ValidatorRejected)
skipped += len(mr.SkippedChanges)
}
if validatorRejected < exp.MinimumCounts.Rejected {
t.Fatalf("expected at least %d validator rejections, got %d", exp.MinimumCounts.Rejected, validatorRejected)
}
if skipped < exp.MinimumCounts.Skipped {
t.Fatalf("expected at least %d application skips, got %d", exp.MinimumCounts.Skipped, skipped)
}
}
func assertReleaseUtilizationShape(t *testing.T, report reporting.ProcessReport) {
t.Helper()
var payload struct {
EffectiveConcurrency struct {
TotalLLM int `json:"total_llm"`
} `json:"effective_concurrency"`
RunTiming struct {
SchedulerQueueWaitMS int64 `json:"scheduler_queue_wait_ms"`
LLMExecutionTimeMS int64 `json:"llm_execution_time_ms"`
DeterministicValidationMS int64 `json:"deterministic_validation_time_ms"`
} `json:"run_timing"`
Modules []map[string]any `json:"modules"`
Validators []map[string]any `json:"validators"`
}
if err := json.Unmarshal(readFile(t, report.Diagnostics.UtilizationSummaryPath), &payload); err != nil {
t.Fatalf("unmarshal utilization diagnostics: %v", err)
}
if payload.EffectiveConcurrency.TotalLLM <= 0 {
t.Fatalf("expected positive total llm concurrency, got %+v", payload.EffectiveConcurrency)
}
if payload.RunTiming.SchedulerQueueWaitMS < 0 || payload.RunTiming.LLMExecutionTimeMS < 0 || payload.RunTiming.DeterministicValidationMS < 0 {
t.Fatalf("expected non-negative run timing values, got %+v", payload.RunTiming)
}
if len(payload.Modules) == 0 {
t.Fatalf("expected module timing summaries")
}
if len(payload.Validators) == 0 {
t.Fatalf("expected validator timing summaries")
}
}
func assertReleaseLedgerShape(t *testing.T, report reporting.ProcessReport) {
t.Helper()
var entries []struct {
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ProposalIndex int `json:"proposal_index"`
Disposition string `json:"disposition"`
DispositionReason string `json:"disposition_reason_code"`
OriginalText string `json:"original_text"`
ProposedCorrected string `json:"proposed_corrected_text"`
ReplacementPolicy string `json:"replacement_policy"`
DeterministicResults []struct {
ValidatorKey string `json:"validator_key"`
} `json:"deterministic_validator_decisions"`
LLMResults []struct {
ValidatorKey string `json:"validator_key"`
} `json:"llm_validator_decisions"`
}
if err := json.Unmarshal(readFile(t, report.Diagnostics.CorrectionLedgerPath), &entries); err != nil {
t.Fatalf("unmarshal correction ledger: %v", err)
}
if len(entries) == 0 {
t.Fatalf("expected correction ledger entries")
}
hasApplied := false
hasRejected := false
hasSkipped := false
for _, entry := range entries {
if entry.ModuleInstance == "" || entry.ModuleKey == "" {
t.Fatalf("expected module identity in ledger entry: %+v", entry)
}
switch entry.Disposition {
case "applied":
hasApplied = true
case "rejected":
hasRejected = true
case "skipped":
hasSkipped = true
}
}
if !hasApplied || !hasRejected {
t.Fatalf("expected applied and rejected entries in correction ledger, got %+v", entries)
}
if !hasSkipped {
// Some deterministic fixture paths do not trigger apply-time skips;
// rejections are still captured separately from application skips.
}
}
func assertPromptAndSchemaMetadataPresent(t *testing.T, runDir string) {
t.Helper()
metadataPaths, err := filepath.Glob(filepath.Join(runDir, "*", "*request-metadata.json"))
if err != nil {
t.Fatalf("glob request metadata artifacts: %v", err)
}
if len(metadataPaths) == 0 {
t.Fatalf("expected request metadata artifacts with prompt metadata")
}
foundPromptMetadata := false
foundSchemaMetadata := false
for _, path := range metadataPaths {
var payload map[string]any
if err := json.Unmarshal(readFile(t, path), &payload); err != nil {
t.Fatalf("unmarshal request metadata artifact %q: %v", path, err)
}
if pm, ok := payload["prompt_metadata"].(map[string]any); ok {
if pm["prompt_id"] != nil && pm["prompt_version"] != nil && pm["sha256"] != nil {
foundPromptMetadata = true
}
}
if sm, ok := payload["response_schema"].(map[string]any); ok {
if sm["id"] != nil && sm["version"] != nil && sm["name"] != nil && sm["sha256"] != nil {
foundSchemaMetadata = true
}
}
}
if !foundPromptMetadata {
t.Fatalf("expected prompt metadata in request metadata artifacts")
}
if !foundSchemaMetadata {
t.Fatalf("expected structured response schema metadata in request metadata artifacts")
}
}
func assertStableValidatorKeysPresent(t *testing.T, report reporting.ProcessReport) {
t.Helper()
seen := map[string]bool{}
for _, module := range report.ModuleResults {
for _, decision := range module.ValidatorDecisions {
seen[decision.ValidatorName] = true
}
for _, rejected := range module.ValidatorRejected {
seen[rejected.ValidatorName] = true
}
}
expectedAny := []string{
"confidence_threshold",
"original_text_presence",
"no_effect",
}
for _, key := range expectedAny {
if !seen[key] {
t.Fatalf("expected stable validator key %q in report decisions/rejections; seen=%v", key, seen)
}
}
}
func assertStdoutStderrContract(t *testing.T, stdout, stderr string) {
t.Helper()
if stdout != "" {
t.Fatalf("expected empty stdout with --output, got %q", stdout)
}
if strings.Contains(stderr, `"module_results"`) || strings.Contains(stderr, `"report_metadata"`) {
t.Fatalf("stderr should remain human-readable, not report JSON: %q", stderr)
}
}
func assertAuditaV1OutputShape(t *testing.T, outputPath string) {
t.Helper()
var payload struct {
Schema string `json:"schema"`
Version string `json:"version"`
Segments []schema.Segment `json:"segments"`
}
if err := json.Unmarshal(readFile(t, outputPath), &payload); err != nil {
t.Fatalf("unmarshal audita-v1 output: %v", err)
}
if payload.Schema != "audita-v1" || payload.Version != "v1" {
t.Fatalf("unexpected audita-v1 metadata: %+v", payload)
}
if len(payload.Segments) == 0 {
t.Fatalf("expected non-empty audita-v1 segments")
}
}

View File

@@ -0,0 +1,151 @@
package cli
import (
"path/filepath"
"sort"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
)
const (
correctionDispositionApplied = "applied"
correctionDispositionRejected = "rejected"
correctionDispositionSkipped = "skipped"
correctionDispositionFailed = "failed"
)
type correctionLedgerEntry struct {
RunID string `json:"run_id,omitempty"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ProposalIndex int `json:"proposal_index"`
SegmentID int `json:"segment_id,omitempty"`
OriginalText string `json:"original_text,omitempty"`
ProposedCorrectedText string `json:"proposed_corrected_text,omitempty"`
AppliedCorrectedText string `json:"applied_corrected_text,omitempty"`
ReplacementPolicy string `json:"replacement_policy,omitempty"`
Disposition string `json:"disposition"`
DispositionReasonCode string `json:"disposition_reason_code,omitempty"`
DispositionMessage string `json:"disposition_message,omitempty"`
DeterministicValidatorResults []ledgerValidatorDecisionRecord `json:"deterministic_validator_decisions,omitempty"`
LLMValidatorResults []ledgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
}
type ledgerValidatorDecisionRecord struct {
ValidatorKey string `json:"validator_key"`
Approved bool `json:"approved"`
ReasonCode string `json:"reason_code"`
Message string `json:"message,omitempty"`
}
func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []correctionLedgerEntry {
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
return nil
}
runID := ""
if runDirPath != "" {
runID = filepath.Base(runDirPath)
}
entries := make([]correctionLedgerEntry, 0)
for _, module := range runOutput.ModuleResults {
decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord)
for _, decision := range module.ValidatorDecisions {
decisionsByProposal[decision.ProposalIndex] = append(decisionsByProposal[decision.ProposalIndex], decision)
}
for _, change := range module.AppliedChanges {
entries = append(entries, correctionLedgerEntry{
RunID: runID,
ModuleKey: module.ModuleKey,
ModuleInstance: module.ModuleInstance,
ProposalIndex: change.ProposalIndex,
SegmentID: change.TargetSegmentID,
OriginalText: change.OriginalText,
ProposedCorrectedText: change.CorrectedText,
AppliedCorrectedText: change.CorrectedText,
ReplacementPolicy: string(module.ReplacementPolicy),
Disposition: correctionDispositionApplied,
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false),
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true),
})
}
for _, change := range module.SkippedChanges {
entries = append(entries, correctionLedgerEntry{
RunID: runID,
ModuleKey: module.ModuleKey,
ModuleInstance: module.ModuleInstance,
ProposalIndex: change.ProposalIndex,
SegmentID: change.TargetSegmentID,
OriginalText: change.OriginalText,
ProposedCorrectedText: change.CorrectedText,
ReplacementPolicy: string(module.ReplacementPolicy),
Disposition: correctionDispositionSkipped,
DispositionReasonCode: string(change.SkipReason),
DispositionMessage: change.Message,
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false),
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true),
})
}
for _, rejection := range module.ValidatorRejected {
entries = append(entries, correctionLedgerEntry{
RunID: runID,
ModuleKey: module.ModuleKey,
ModuleInstance: module.ModuleInstance,
ProposalIndex: rejection.ProposalIndex,
SegmentID: rejection.TargetSegmentID,
OriginalText: rejection.OriginalText,
ProposedCorrectedText: rejection.CorrectedText,
ReplacementPolicy: string(module.ReplacementPolicy),
Disposition: correctionDispositionRejected,
DispositionReasonCode: rejection.ReasonCode,
DispositionMessage: rejection.Message,
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], false),
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], true),
})
}
if module.Status == runner.ModuleStatusFailed {
entries = append(entries, correctionLedgerEntry{
RunID: runID,
ModuleKey: module.ModuleKey,
ModuleInstance: module.ModuleInstance,
Disposition: correctionDispositionFailed,
DispositionReasonCode: "module_failed",
DispositionMessage: module.ErrorMessage,
})
}
}
sort.SliceStable(entries, func(i, j int) bool {
if entries[i].ModuleInstance != entries[j].ModuleInstance {
return entries[i].ModuleInstance < entries[j].ModuleInstance
}
if entries[i].ProposalIndex != entries[j].ProposalIndex {
return entries[i].ProposalIndex < entries[j].ProposalIndex
}
return entries[i].Disposition < entries[j].Disposition
})
return entries
}
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []ledgerValidatorDecisionRecord {
if len(in) == 0 {
return nil
}
out := make([]ledgerValidatorDecisionRecord, 0, len(in))
for _, decision := range in {
isLLMBacked := validatormetadata.ClassForKey(decision.ValidatorName) == validatormetadata.ExecutionClassLLMBacked
if isLLMBacked != wantLLM {
continue
}
out = append(out, ledgerValidatorDecisionRecord{
ValidatorKey: decision.ValidatorName,
Approved: decision.Approved,
ReasonCode: decision.ReasonCode,
Message: decision.Message,
})
}
return out
}

View File

@@ -0,0 +1,46 @@
package cli
import (
"testing"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
)
func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(t *testing.T) {
output := &runner.RunOutput{
ModuleResults: []runner.ModuleResult{
{
ModuleKey: "glossary",
ModuleInstance: "glossary",
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
ValidatorDecisions: []runner.ValidatorDecisionRecord{
{ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
{ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
},
AppliedChanges: []proposals.AppliedChange{
{
ProposalIndex: 3,
ModuleKey: "glossary",
ModuleInstance: "glossary",
TargetSegmentID: 1,
OriginalText: "gestures",
CorrectedText: "Jesters",
},
},
},
},
}
ledger := buildCorrectionLedger("/tmp/audita-run-id", output)
if len(ledger) != 1 {
t.Fatalf("expected one ledger entry, got %d", len(ledger))
}
entry := ledger[0]
if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "proposal_shape" {
t.Fatalf("unexpected deterministic decision split: %+v", entry.DeterministicValidatorResults)
}
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "spoken_form_plausibility" {
t.Fatalf("unexpected llm-backed decision split: %+v", entry.LLMValidatorResults)
}
}

1041
internal/cli/run.go Normal file

File diff suppressed because it is too large Load Diff

4354
internal/cli/run_test.go Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,110 @@
package cli
import (
"context"
"errors"
"os"
"strconv"
"strings"
"sync"
"time"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
const (
subprocessTestLLMModeEnv = "AUDITA_SUBPROCESS_TEST_LLM_MODE"
subprocessTestRunTimeoutMSEnv = "AUDITA_SUBPROCESS_TEST_RUN_TIMEOUT_MS"
)
// ConfigureSubprocessTestHooksFromEnv enables deterministic test-only hooks for
// subprocess integration tests that run through the Go test binary helper path.
func ConfigureSubprocessTestHooksFromEnv() {
mode := strings.TrimSpace(os.Getenv(subprocessTestLLMModeEnv))
timeoutMSRaw := strings.TrimSpace(os.Getenv(subprocessTestRunTimeoutMSEnv))
// Only activate in explicit subprocess test mode.
if mode == "" && timeoutMSRaw == "" {
return
}
if mode != "" {
client := &subprocessTestLLMClient{mode: mode}
processProposalLLMClient = client
processValidationLLMClient = client
}
if timeoutMSRaw == "" {
return
}
timeoutMS, err := strconv.Atoi(timeoutMSRaw)
if err != nil || timeoutMS <= 0 {
return
}
processRunnerContext = func() (context.Context, context.CancelFunc) {
return context.WithTimeout(context.Background(), time.Duration(timeoutMS)*time.Millisecond)
}
}
type subprocessTestLLMClient struct {
mode string
mu sync.Mutex
proposals int
}
func (c *subprocessTestLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = req
switch c.mode {
case "backend_error":
return contracts.StructuredCompletionResponse{}, errors.New("synthetic backend failure")
case "block_until_cancel":
<-ctx.Done()
return contracts.StructuredCompletionResponse{}, ctx.Err()
case "malformed_structured":
switch target := out.(type) {
case *proposal_generation.StructuredCorrectionSet:
*target = proposal_generation.StructuredCorrectionSet{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y", Confidence: 0.99},
},
}
case *validators.LLMValidationResponse:
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 999, Approved: true, Confidence: 0.9, Reason: "bad index"},
},
}
}
case "mid_pipeline_fail":
if _, ok := out.(*proposal_generation.StructuredCorrectionSet); ok {
c.mu.Lock()
c.proposals++
proposalCall := c.proposals
c.mu.Unlock()
if proposalCall >= 3 {
return contracts.StructuredCompletionResponse{}, errors.New("synthetic mid-pipeline failure")
}
}
switch target := out.(type) {
case *proposal_generation.StructuredCorrectionSet:
*target = proposal_generation.StructuredCorrectionSet{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99},
},
}
case *validators.LLMValidationResponse:
*target = validators.LLMValidationResponse{Validations: nil}
}
default:
switch target := out.(type) {
case *proposal_generation.StructuredCorrectionSet:
*target = proposal_generation.StructuredCorrectionSet{Corrections: nil}
case *validators.LLMValidationResponse:
*target = validators.LLMValidationResponse{Validations: nil}
}
}
return contracts.StructuredCompletionResponse{}, nil
}

View File

@@ -0,0 +1 @@
{"segments":[{"id":1,"text":"oops"}

View File

@@ -0,0 +1,22 @@
{
"name": "ambiguous_match_rejected_before_application",
"transcript_file": "application-skip-ambiguous.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"modules_csv": "homophones",
"proposal_responses_file": "application-skip-ambiguous.proposals.json",
"validation_responses_file": "application-skip-ambiguous.validations.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "application-skip-ambiguous.expected-transcript.json",
"module_instances": ["homophones"],
"module_count": 1,
"total_applied_changes": 0,
"total_skipped_changes": 1,
"module_applied_counts": [0],
"module_rejected_counts": [1],
"module_skip_counts": [0],
"validator_rejected_reason_codes": ["ambiguous_original_text"],
"expected_proposal_calls": ["homophones:proposal"]
}
}

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"the site near another site"}
]

View File

@@ -0,0 +1,3 @@
[
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"the site near another site"}
]

View File

@@ -0,0 +1,4 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

View File

@@ -0,0 +1,28 @@
{
"name": "default_full_pipeline_shape_and_reports",
"transcript_file": "default-full-pipeline.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"proposal_responses_file": "default-full-pipeline.proposals.json",
"validation_responses_file": "default-full-pipeline.validations.json",
"env": {
"AUDITA_LLM_API_KEY": "parity-secret",
"AUDITA_VALIDATION_LLM_API_KEY": "parity-secret"
},
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "default-full-pipeline.expected-transcript.json",
"module_instances": ["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
"module_count": 5,
"total_applied_changes": 3,
"total_skipped_changes": 3,
"secret_markers": ["parity-secret"],
"expected_proposal_calls": [
"glossary_1:proposal",
"homophones:proposal",
"glossary_2:proposal",
"spoken_word:proposal",
"grammar:proposal"
]
}
}

View File

@@ -0,0 +1,9 @@
[
{
"id": 1,
"speaker": "Alice",
"start": 0,
"end": 1,
"text": "Hello, there were Jesters hmm"
}
]

View File

@@ -0,0 +1,7 @@
glossary:
- name: Jesters
aliases:
- jester
plural: jesters
category: faction
summary: A protected in-world faction term.

View File

@@ -0,0 +1,28 @@
[
{
"corrections": [
{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "Jesters", "corrected_text": "jesters", "confidence": 0.99},
{"id": 1, "original_text": "Jesters", "corrected_text": "JESTERX", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "jesters", "corrected_text": "JESTERS", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "uh", "corrected_text": "hmm", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
]
}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
]

View File

@@ -0,0 +1,12 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject cleanup"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "parity-secret"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

View File

@@ -0,0 +1,27 @@
{
"name": "default_pipeline_handoff_and_module_order",
"transcript_file": "default-handoff.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"proposal_responses_file": "default-handoff.proposals.json",
"validation_responses_file": "default-handoff.validations.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "default-handoff.expected-transcript.json",
"module_instances": ["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
"module_count": 5,
"total_applied_changes": 5,
"total_skipped_changes": 0,
"module_applied_counts": [1, 1, 1, 1, 1],
"module_rejected_counts": [0, 0, 0, 0, 0],
"module_skip_counts": [0, 0, 0, 0, 0],
"expected_proposal_calls": [
"glossary_1:proposal",
"homophones:proposal",
"glossary_2:proposal",
"spoken_word:proposal",
"grammar:proposal"
],
"min_response_payload_artifacts": 15
}
}

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"Hello, there were Jesters at the Sight um"}
]

View File

@@ -0,0 +1,6 @@
glossary:
- name: Jesters
aliases: [jester]
plural: jesters
category: faction
summary: A protected in-world faction term.

View File

@@ -0,0 +1,7 @@
[
{"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "sight", "corrected_text": "Sight", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "um um", "corrected_text": "um", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}]}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures at the site um um"}
]

View File

@@ -0,0 +1,12 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

View File

@@ -0,0 +1,17 @@
{
"name": "deterministic_validator_low_confidence",
"transcript_file": "deterministic-validator-low-confidence.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"modules_csv": "grammar",
"proposal_responses_file": "deterministic-validator-low-confidence.proposals.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "deterministic-validator-low-confidence.expected-transcript.json",
"module_instances": ["grammar"],
"module_count": 1,
"total_skipped_changes": 1,
"validator_rejected_reason_codes": ["low_confidence"],
"expected_proposal_calls": ["grammar:proposal"]
}
}

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"hello , world"}
]

View File

@@ -0,0 +1,7 @@
[
{
"corrections": [
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.1}
]
}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , world"}
]

View File

@@ -0,0 +1,12 @@
{
"name": "glossary_schema_handling",
"transcript_file": "default-full-pipeline.transcript.json",
"glossary_file": "glossary-schema-error.glossary.yaml",
"expect": {
"exit_code": 1,
"status": "failed",
"error_phase": "glossary_schema",
"stderr_contains": "glossary_schema",
"require_error_log": true
}
}

View File

@@ -0,0 +1,2 @@
glossary:
- name: MissingCategoryAndSummary

View File

@@ -0,0 +1,19 @@
{
"name": "llm_validator_decision_handling",
"transcript_file": "llm-validator-rejection.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"modules_csv": "grammar",
"proposal_responses_file": "llm-validator-rejection.proposals.json",
"validation_responses_file": "llm-validator-rejection.validations.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "llm-validator-rejection.expected-transcript.json",
"module_instances": ["grammar"],
"module_count": 1,
"total_skipped_changes": 1,
"validator_rejected_reason_codes": ["llm_rejected"],
"expected_proposal_calls": ["grammar:proposal"],
"expected_validation_calls": ["grammar:section-0000:editorial_review:batch-0000"]
}
}

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"hello , world"}
]

View File

@@ -0,0 +1,7 @@
[
{
"corrections": [
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
]
}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , world"}
]

View File

@@ -0,0 +1,7 @@
[
{
"validations": [
{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject stylistic overreach"}
]
}
]

View File

@@ -0,0 +1,23 @@
{
"name": "mid_pipeline_failure_partial_progress",
"transcript_file": "default-handoff.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"proposal_responses_file": "mid-pipeline-failure.proposals.json",
"validation_responses_file": "mid-pipeline-failure.validations.json",
"expect": {
"exit_code": 1,
"status": "failed",
"error_phase": "runner_execution",
"stderr_contains": "runner_execution",
"module_instances": ["glossary_1", "homophones", "glossary_2"],
"module_count": 3,
"total_applied_changes": 2,
"total_skipped_changes": 0,
"failed_module_instance": "glossary_2",
"module_applied_counts": [1, 1, 0],
"module_rejected_counts": [0, 0, 0],
"module_skip_counts": [0, 0, 0],
"require_error_log": true,
"expected_proposal_calls": ["glossary_1:proposal", "homophones:proposal", "glossary_2:proposal"]
}
}

View File

@@ -0,0 +1,4 @@
[
{"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]}
]

View File

@@ -0,0 +1,6 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

View File

@@ -0,0 +1,21 @@
{
"name": "protected_glossary_term_behavior",
"transcript_file": "protected-term-rejection.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"modules_csv": "homophones",
"proposal_responses_file": "protected-term-rejection.proposals.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "protected-term-rejection.expected-transcript.json",
"module_instances": ["homophones"],
"module_count": 1,
"total_applied_changes": 0,
"total_skipped_changes": 1,
"module_applied_counts": [0],
"module_rejected_counts": [1],
"module_skip_counts": [0],
"validator_rejected_reason_codes": ["protected_glossary_term"],
"expected_proposal_calls": ["homophones:proposal"]
}
}

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"The Jesters entered the hall."}
]

View File

@@ -0,0 +1,3 @@
[
{"corrections": [{"id": 1, "original_text": "Jesters", "corrected_text": "Gestures", "confidence": 0.99}]}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"The Jesters entered the hall."}
]

View File

@@ -0,0 +1,12 @@
{
"name": "transcript_schema_handling",
"transcript_file": "transcript-schema-error.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"expect": {
"exit_code": 1,
"status": "failed",
"error_phase": "transcript_schema",
"stderr_contains": "transcript_schema",
"require_error_log": true
}
}

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"","start":0.0,"end":1.0,"text":"bad"}
]

View File

@@ -0,0 +1,24 @@
{
"must_apply_texts": [
"Hello, there were Jesters hmm"
],
"must_not_apply_texts": [
"JESTERX",
"there were gestures"
],
"protected_terms": [
"Jesters"
],
"expected_module_instances": [
"glossary_1",
"homophones",
"glossary_2",
"spoken_word",
"grammar"
],
"minimum_counts": {
"applied": 1,
"rejected": 1,
"skipped": 0
}
}

View File

@@ -0,0 +1,9 @@
[
{
"id": 1,
"speaker": "Alice",
"start": 0,
"end": 1,
"text": "Hello, there were Jesters hmm"
}
]

View File

@@ -0,0 +1,7 @@
glossary:
- name: Jesters
aliases:
- jester
plural: jesters
category: faction
summary: A protected in-world faction term.

View File

@@ -0,0 +1,28 @@
[
{
"corrections": [
{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "Jesters", "corrected_text": "jesters", "confidence": 0.99},
{"id": 1, "original_text": "Jesters", "corrected_text": "JESTERX", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "jesters", "corrected_text": "JESTERS", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "uh", "corrected_text": "hmm", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
]
}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
]

View File

@@ -0,0 +1,12 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject cleanup"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "release-secret"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

View File

@@ -0,0 +1,6 @@
glossary:
- name: Audita
aliases:
- audita
category: product
summary: The Audita transcript correction CLI.

View File

@@ -0,0 +1,9 @@
[
{
"id": 1,
"speaker": "A",
"start": 0.0,
"end": 1.2,
"text": "hello world"
}
]

View File

@@ -0,0 +1,312 @@
package chunking
import (
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
// Section represents a contiguous chunk of transcript segments with metadata.
type Section struct {
// Index is the 0-based section index within the chunked transcript
Index int `json:"section_index"`
// StartSegmentID is the ID of the first segment in this section
StartSegmentID int `json:"start_segment_id"`
// EndSegmentID is the ID of the last segment in this section
EndSegmentID int `json:"end_segment_id"`
// EstimatedTokens is the approximate token count for this section
EstimatedTokens int `json:"estimated_tokens"`
// Segments contains the segments in this section, in order
Segments []schema.Segment `json:"segments"`
}
// ChunkingConfig holds configuration for transcript chunking.
type ChunkingConfig struct {
// MaxSectionTokens is the maximum allowed tokens per section
MaxSectionTokens int
// MinSectionTokens is a validated soft lower-bound setting retained for
// configuration/reporting compatibility.
MinSectionTokens int
// TargetSections is an optional target number of sections
// If nil, section count is derived from total/max token budgeting.
TargetSections *int
}
// Chunker performs deterministic chunking of normalized transcript segments.
type Chunker struct {
config ChunkingConfig
estimator TokenEstimator
}
// NewChunker creates a new chunker with the given configuration.
func NewChunker(config ChunkingConfig) *Chunker {
return &Chunker{
config: config,
estimator: NewSimpleTokenEstimator(),
}
}
// NewChunkerWithEstimator creates a new chunker with a custom estimator.
func NewChunkerWithEstimator(config ChunkingConfig, estimator TokenEstimator) *Chunker {
return &Chunker{
config: config,
estimator: estimator,
}
}
// ChunkTranscript divides a normalized transcript into contiguous token-bounded
// sections using a deterministic balanced forward pass.
//
// Behavior:
// - preserve segment order and never split segments;
// - estimate per-segment tokens once, then compute total;
// - derive desired section count from ceil(total/max_section_tokens), unless
// target_sections is explicitly set;
// - prefer section sizes near ceil(total/section_count) while never exceeding
// max_section_tokens unless a section consists of a single oversized segment.
//
// Returns an error if explicit target_sections is impossible under constraints.
// The input transcript is never mutated.
func (c *Chunker) ChunkTranscript(transcript *schema.Transcript) ([]Section, error) {
if transcript == nil || len(transcript.Segments) == 0 {
return []Section{}, nil
}
// Validate configuration
if c.config.MaxSectionTokens <= 0 {
return nil, fmt.Errorf("max_section_tokens must be positive, got %d", c.config.MaxSectionTokens)
}
if c.config.MinSectionTokens < 0 {
return nil, fmt.Errorf("min_section_tokens must be non-negative, got %d", c.config.MinSectionTokens)
}
if c.config.MinSectionTokens > c.config.MaxSectionTokens {
return nil, fmt.Errorf("min_section_tokens (%d) cannot exceed max_section_tokens (%d)",
c.config.MinSectionTokens, c.config.MaxSectionTokens)
}
// Calculate token counts for each segment (deterministic).
segmentTokens := make([]int, len(transcript.Segments))
totalTokens := 0
for i, seg := range transcript.Segments {
segmentTokens[i] = c.estimator.EstimateTokens(seg.Text)
totalTokens += segmentTokens[i]
}
minPossibleSections := c.calculateMinPossibleSections(segmentTokens)
var desiredSections int
useExplicitTarget := false
if c.config.TargetSections != nil {
desiredSections = *c.config.TargetSections
useExplicitTarget = true
if desiredSections <= 0 {
return nil, fmt.Errorf("target_sections must be positive, got %d", desiredSections)
}
if err := c.validateTargetSections(desiredSections, segmentTokens); err != nil {
return nil, err
}
} else {
desiredSections = ceilDiv(totalTokens, c.config.MaxSectionTokens)
if desiredSections < minPossibleSections {
desiredSections = minPossibleSections
}
if desiredSections < 1 {
desiredSections = 1
}
if desiredSections > len(transcript.Segments) {
desiredSections = len(transcript.Segments)
}
}
targetTokensPerSection := ceilDiv(totalTokens, desiredSections)
if useExplicitTarget {
return c.buildSectionsWithExplicitTarget(
transcript.Segments,
segmentTokens,
desiredSections,
targetTokensPerSection,
)
}
return c.buildSectionsBalanced(transcript.Segments, segmentTokens, targetTokensPerSection), nil
}
// validateTargetSections checks if the target section count is achievable.
func (c *Chunker) validateTargetSections(target int, segmentTokens []int) error {
// Maximum possible sections: limited by segment count
maxPossibleSections := len(segmentTokens)
if target > maxPossibleSections {
return fmt.Errorf(
"target_sections (%d) is impossible: cannot have more sections than segments (%d)",
target, maxPossibleSections)
}
// Minimum possible sections: each segment must fit within max bounds
minPossibleSections := c.calculateMinPossibleSections(segmentTokens)
if target < minPossibleSections {
return fmt.Errorf(
"target_sections (%d) is impossible: need at least %d sections to respect max_section_tokens (%d)",
target, minPossibleSections, c.config.MaxSectionTokens)
}
return nil
}
// buildSectionsBalanced creates sections with a deterministic single-pass policy.
func (c *Chunker) buildSectionsBalanced(segments []schema.Segment, segmentTokens []int, targetTokensPerSection int) []Section {
var sections []Section
var currentSegments []schema.Segment
currentTokens := 0
for i, seg := range segments {
tokens := segmentTokens[i]
// Empty section: always accept the next segment, including oversized.
if len(currentSegments) == 0 {
currentSegments = append(currentSegments, seg)
currentTokens = tokens
continue
}
// If adding next segment would exceed max, close current section.
if currentTokens+tokens > c.config.MaxSectionTokens {
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
currentSegments = []schema.Segment{seg}
currentTokens = tokens
continue
}
// Prefer staying near target tokens per section.
if targetTokensPerSection == 0 || currentTokens < targetTokensPerSection {
currentSegments = append(currentSegments, seg)
currentTokens += tokens
continue
}
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
currentSegments = []schema.Segment{seg}
currentTokens = tokens
}
if len(currentSegments) > 0 {
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
}
return sections
}
// buildSection creates a Section from segments.
func (c *Chunker) buildSection(index int, segments []schema.Segment, tokens int) Section {
return Section{
Index: index,
StartSegmentID: segments[0].ID,
EndSegmentID: segments[len(segments)-1].ID,
EstimatedTokens: tokens,
Segments: segments,
}
}
// calculateMinPossibleSections calculates the minimum number of sections needed
// to ensure no section exceeds max tokens.
func (c *Chunker) calculateMinPossibleSections(segmentTokens []int) int {
sections := 0
currentTokens := 0
for _, tokens := range segmentTokens {
if tokens > c.config.MaxSectionTokens {
// Each oversized segment needs its own section
if currentTokens > 0 {
sections++
currentTokens = 0
}
sections++
} else if currentTokens+tokens > c.config.MaxSectionTokens {
sections++
currentTokens = tokens
} else {
currentTokens += tokens
}
}
if currentTokens > 0 {
sections++
}
return sections
}
// buildSectionsWithExplicitTarget builds exactly desiredSections when feasible.
func (c *Chunker) buildSectionsWithExplicitTarget(
segments []schema.Segment,
segmentTokens []int,
desiredSections int,
targetTokensPerSection int,
) ([]Section, error) {
n := len(segments)
cursor := 0
sections := make([]Section, 0, desiredSections)
for sectionIdx := 0; sectionIdx < desiredSections; sectionIdx++ {
if cursor >= n {
break
}
remainingSectionsAfter := desiredSections - sectionIdx - 1
currentSegments := []schema.Segment{segments[cursor]}
currentTokens := segmentTokens[cursor]
cursor++
for cursor < n {
remainingSegments := n - cursor
// Reserve one segment per future section to avoid empty sections.
if remainingSegments == remainingSectionsAfter {
break
}
nextTokens := segmentTokens[cursor]
if currentTokens+nextTokens > c.config.MaxSectionTokens {
break
}
if targetTokensPerSection == 0 || currentTokens < targetTokensPerSection {
currentSegments = append(currentSegments, segments[cursor])
currentTokens += nextTokens
cursor++
continue
}
break
}
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
}
if cursor != n || len(sections) != desiredSections {
return nil, fmt.Errorf(
"target_sections (%d) is impossible under current constraints (got %d sections)",
desiredSections,
len(sections),
)
}
return sections, nil
}
func ceilDiv(numerator int, denominator int) int {
if denominator <= 0 {
return 0
}
if numerator <= 0 {
return 0
}
return (numerator + denominator - 1) / denominator
}

View File

@@ -0,0 +1,518 @@
package chunking
import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
// mapTokenEstimator provides deterministic per-segment token counts for tests.
type mapTokenEstimator struct {
byText map[string]int
}
func (e *mapTokenEstimator) EstimateTokens(text string) int {
if e.byText == nil {
return 0
}
if tokens, ok := e.byText[text]; ok {
return tokens
}
return 0
}
func makeSegments(texts []string) []schema.Segment {
segments := make([]schema.Segment, len(texts))
for i, text := range texts {
segments[i] = schema.Segment{
ID: i + 1,
Speaker: "DM",
Start: float64(i * 10),
End: float64(i*10 + 5),
Text: text,
}
}
return segments
}
func makeTranscript(segments []schema.Segment) *schema.Transcript {
return &schema.Transcript{Segments: segments}
}
func intPtr(i int) *int {
return &i
}
func assertSegmentCoverageAndOrder(t *testing.T, input []schema.Segment, sections []Section) {
t.Helper()
seen := make([]schema.Segment, 0, len(input))
for _, sec := range sections {
seen = append(seen, sec.Segments...)
}
if len(seen) != len(input) {
t.Fatalf("expected %d total segment occurrences, got %d", len(input), len(seen))
}
for i := range input {
if seen[i].ID != input[i].ID {
t.Fatalf("segment order mismatch at index %d: got id=%d want id=%d", i, seen[i].ID, input[i].ID)
}
}
}
func assertSectionMetadataConsistent(t *testing.T, sections []Section) {
t.Helper()
for i, sec := range sections {
if sec.Index != i {
t.Fatalf("section %d: expected index=%d got=%d", i, i, sec.Index)
}
if len(sec.Segments) == 0 {
t.Fatalf("section %d: section must not be empty", i)
}
if sec.StartSegmentID != sec.Segments[0].ID {
t.Fatalf("section %d: start_segment_id mismatch", i)
}
if sec.EndSegmentID != sec.Segments[len(sec.Segments)-1].ID {
t.Fatalf("section %d: end_segment_id mismatch", i)
}
}
}
func assertMaxBoundExceptSingletonOversized(t *testing.T, sections []Section, max int) {
t.Helper()
for i, sec := range sections {
if sec.EstimatedTokens <= max {
continue
}
if len(sec.Segments) != 1 {
t.Fatalf("section %d exceeds max tokens (%d>%d) with %d segments", i, sec.EstimatedTokens, max, len(sec.Segments))
}
}
}
func imbalance(sections []Section) int {
if len(sections) == 0 {
return 0
}
minTokens := sections[0].EstimatedTokens
maxTokens := sections[0].EstimatedTokens
for _, sec := range sections {
if sec.EstimatedTokens < minTokens {
minTokens = sec.EstimatedTokens
}
if sec.EstimatedTokens > maxTokens {
maxTokens = sec.EstimatedTokens
}
}
return maxTokens - minTokens
}
func greedyMaxFillSections(segments []schema.Segment, tokens []int, max int) []Section {
sections := make([]Section, 0)
var current []schema.Segment
currentTokens := 0
for i, seg := range segments {
tok := tokens[i]
if len(current) == 0 {
current = append(current, seg)
currentTokens = tok
continue
}
if currentTokens+tok > max {
sections = append(sections, Section{
Index: len(sections),
StartSegmentID: current[0].ID,
EndSegmentID: current[len(current)-1].ID,
EstimatedTokens: currentTokens,
Segments: append([]schema.Segment(nil), current...),
})
current = []schema.Segment{seg}
currentTokens = tok
continue
}
current = append(current, seg)
currentTokens += tok
}
if len(current) > 0 {
sections = append(sections, Section{
Index: len(sections),
StartSegmentID: current[0].ID,
EndSegmentID: current[len(current)-1].ID,
EstimatedTokens: currentTokens,
Segments: append([]schema.Segment(nil), current...),
})
}
return sections
}
func TestChunkEmptyTranscript(t *testing.T) {
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 10})
sections, err := chunker.ChunkTranscript(nil)
if err != nil {
t.Fatalf("ChunkTranscript(nil): %v", err)
}
if len(sections) != 0 {
t.Fatalf("expected 0 sections for nil transcript, got %d", len(sections))
}
sections, err = chunker.ChunkTranscript(makeTranscript(nil))
if err != nil {
t.Fatalf("ChunkTranscript(empty): %v", err)
}
if len(sections) != 0 {
t.Fatalf("expected 0 sections for empty transcript, got %d", len(sections))
}
}
func TestChunkSingleSegment(t *testing.T) {
segments := makeSegments([]string{"s1"})
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 10},
&mapTokenEstimator{byText: map[string]int{"s1": 7}},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 1 {
t.Fatalf("expected 1 section, got %d", len(sections))
}
if sections[0].EstimatedTokens != 7 {
t.Fatalf("expected estimated_tokens=7, got %d", sections[0].EstimatedTokens)
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertSectionMetadataConsistent(t, sections)
}
func TestChunkSingleOversizedSegment(t *testing.T) {
segments := makeSegments([]string{"big"})
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 5},
&mapTokenEstimator{byText: map[string]int{"big": 120}},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 1 {
t.Fatalf("expected 1 section, got %d", len(sections))
}
if sections[0].EstimatedTokens != 120 {
t.Fatalf("expected oversized singleton section, got %d", sections[0].EstimatedTokens)
}
assertMaxBoundExceptSingletonOversized(t, sections, 50)
}
func TestChunkTotalBelowMaxSingleSection(t *testing.T) {
segments := makeSegments([]string{"a", "b", "c"})
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 5},
&ConstTokenEstimator{Tokens: 10},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 1 {
t.Fatalf("expected 1 section, got %d", len(sections))
}
if sections[0].EstimatedTokens != 30 {
t.Fatalf("expected 30 section tokens, got %d", sections[0].EstimatedTokens)
}
assertSegmentCoverageAndOrder(t, segments, sections)
}
func TestChunkTotalExactlyDivisibleByMax(t *testing.T) {
segments := makeSegments([]string{"a", "b", "c", "d"})
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1},
&ConstTokenEstimator{Tokens: 5},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 2 {
t.Fatalf("expected 2 sections, got %d", len(sections))
}
if sections[0].EstimatedTokens != 10 || sections[1].EstimatedTokens != 10 {
t.Fatalf("expected [10,10] tokens, got [%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens)
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertMaxBoundExceptSingletonOversized(t, sections, 10)
}
func TestChunkTotalNotDivisibleByMax(t *testing.T) {
segments := makeSegments([]string{"a", "b", "c", "d", "e"})
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1},
&ConstTokenEstimator{Tokens: 5},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 3 {
t.Fatalf("expected 3 sections, got %d", len(sections))
}
if sections[0].EstimatedTokens != 10 || sections[1].EstimatedTokens != 10 || sections[2].EstimatedTokens != 5 {
t.Fatalf("expected [10,10,5] tokens, got [%d,%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens, sections[2].EstimatedTokens)
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertMaxBoundExceptSingletonOversized(t, sections, 10)
}
func TestChunkTargetSectionsPrecedenceAndSuccess(t *testing.T) {
segments := makeSegments([]string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10"})
target := 3
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 200, MinSectionTokens: 1, TargetSections: &target},
&ConstTokenEstimator{Tokens: 10},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != target {
t.Fatalf("expected %d sections from explicit target, got %d", target, len(sections))
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertSectionMetadataConsistent(t, sections)
}
func TestChunkTargetSectionsImpossibleTooMany(t *testing.T) {
segments := makeSegments([]string{"1", "2", "3", "4", "5"})
target := 10
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 1, TargetSections: &target})
_, err := chunker.ChunkTranscript(makeTranscript(segments))
if err == nil {
t.Fatal("expected error for impossible target_sections")
}
if !strings.Contains(err.Error(), "cannot have more sections than segments") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestChunkTargetSectionsImpossibleTooFew(t *testing.T) {
segments := makeSegments([]string{"1", "2", "3"})
target := 1
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1, TargetSections: &target},
&ConstTokenEstimator{Tokens: 30},
)
_, err := chunker.ChunkTranscript(makeTranscript(segments))
if err == nil {
t.Fatal("expected error for impossible target_sections")
}
if !strings.Contains(err.Error(), "need at least 3 sections") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestChunkManySmallSegmentsBalanced(t *testing.T) {
texts := []string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"}
segments := makeSegments(texts)
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1},
&ConstTokenEstimator{Tokens: 10},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 3 {
t.Fatalf("expected 3 sections, got %d", len(sections))
}
if sections[0].EstimatedTokens != 40 || sections[1].EstimatedTokens != 40 || sections[2].EstimatedTokens != 30 {
t.Fatalf("expected [40,40,30], got [%d,%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens, sections[2].EstimatedTokens)
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertMaxBoundExceptSingletonOversized(t, sections, 50)
}
func TestChunkMixedLargeAndSmallSegments(t *testing.T) {
segments := makeSegments([]string{"big1", "s1", "s2", "s3", "big2", "s4"})
estimator := &mapTokenEstimator{byText: map[string]int{
"big1": 120,
"s1": 10,
"s2": 10,
"s3": 10,
"big2": 120,
"s4": 10,
}}
chunker := NewChunkerWithEstimator(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 1}, estimator)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 4 {
t.Fatalf("expected 4 sections, got %d", len(sections))
}
if len(sections[0].Segments) != 1 || sections[0].Segments[0].Text != "big1" {
t.Fatalf("expected first oversized segment in singleton section, got %+v", sections[0].Segments)
}
if len(sections[2].Segments) != 1 || sections[2].Segments[0].Text != "big2" {
t.Fatalf("expected second oversized segment in singleton section, got %+v", sections[2].Segments)
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertMaxBoundExceptSingletonOversized(t, sections, 100)
}
func TestChunkDeterministicOrdering(t *testing.T) {
segments := makeSegments([]string{"a", "b", "c", "d", "e", "f"})
chunkerCfg := ChunkingConfig{MaxSectionTokens: 15, MinSectionTokens: 1}
estimator := &ConstTokenEstimator{Tokens: 5}
var first []Section
for i := 0; i < 5; i++ {
chunker := NewChunkerWithEstimator(chunkerCfg, estimator)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("iteration %d: %v", i, err)
}
if i == 0 {
first = sections
continue
}
if len(sections) != len(first) {
t.Fatalf("iteration %d: section count mismatch (%d vs %d)", i, len(sections), len(first))
}
for j := range sections {
if sections[j].Index != first[j].Index ||
sections[j].StartSegmentID != first[j].StartSegmentID ||
sections[j].EndSegmentID != first[j].EndSegmentID ||
sections[j].EstimatedTokens != first[j].EstimatedTokens ||
len(sections[j].Segments) != len(first[j].Segments) {
t.Fatalf("iteration %d section %d mismatch", i, j)
}
}
}
}
func TestChunkNoMutationOfInput(t *testing.T) {
segments := makeSegments([]string{"original one", "original two"})
transcript := makeTranscript(segments)
original := make([]string, len(transcript.Segments))
for i := range transcript.Segments {
original[i] = transcript.Segments[i].Text
}
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1})
if _, err := chunker.ChunkTranscript(transcript); err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
for i := range transcript.Segments {
if transcript.Segments[i].Text != original[i] {
t.Fatalf("segment %d mutated", i)
}
}
}
func TestChunkConfigValidation(t *testing.T) {
tests := []struct {
name string
config ChunkingConfig
errContains string
}{
{
name: "zero max tokens",
config: ChunkingConfig{MaxSectionTokens: 0, MinSectionTokens: 1},
errContains: "max_section_tokens must be positive",
},
{
name: "negative max tokens",
config: ChunkingConfig{MaxSectionTokens: -1, MinSectionTokens: 1},
errContains: "max_section_tokens must be positive",
},
{
name: "negative min tokens",
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: -1},
errContains: "min_section_tokens must be non-negative",
},
{
name: "min exceeds max",
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 11},
errContains: "min_section_tokens (11) cannot exceed max_section_tokens (10)",
},
{
name: "zero target sections",
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1, TargetSections: intPtr(0)},
errContains: "target_sections must be positive",
},
{
name: "negative target sections",
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1, TargetSections: intPtr(-1)},
errContains: "target_sections must be positive",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
chunker := NewChunker(tt.config)
_, err := chunker.ChunkTranscript(makeTranscript(makeSegments([]string{"x"})))
if err == nil {
t.Fatalf("expected error containing %q", tt.errContains)
}
if !strings.Contains(err.Error(), tt.errContains) {
t.Fatalf("expected error containing %q, got %q", tt.errContains, err.Error())
}
})
}
}
func TestChunkBalancedAlgorithmBeatsGreedyMaxFillOnUnevenTranscript(t *testing.T) {
segments := makeSegments([]string{"s1", "s2", "s3", "s4", "s5", "s6"})
tokenMap := map[string]int{
"s1": 50,
"s2": 10,
"s3": 10,
"s4": 10,
"s5": 10,
"s6": 10,
}
estimator := &mapTokenEstimator{byText: tokenMap}
chunker := NewChunkerWithEstimator(ChunkingConfig{MaxSectionTokens: 80, MinSectionTokens: 1}, estimator)
balancedSections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
tokens := make([]int, 0, len(segments))
for _, seg := range segments {
tokens = append(tokens, tokenMap[seg.Text])
}
greedySections := greedyMaxFillSections(segments, tokens, 80)
balancedImbalance := imbalance(balancedSections)
greedyImbalance := imbalance(greedySections)
if balancedImbalance >= greedyImbalance {
t.Fatalf(
"expected balanced chunking to improve over greedy max-fill; balanced=%d greedy=%d",
balancedImbalance,
greedyImbalance,
)
}
assertSegmentCoverageAndOrder(t, segments, balancedSections)
assertMaxBoundExceptSingletonOversized(t, balancedSections, 80)
}

View File

@@ -0,0 +1,84 @@
package chunking
// Summary provides a concise overview of chunking results for reports
type Summary struct {
ChunkCount int `json:"chunk_count"`
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
TargetSections *int `json:"target_sections,omitempty"`
MaxSectionTokens int `json:"max_section_tokens"`
MinSectionTokens int `json:"min_section_tokens"`
}
// ChunkSummary represents a single chunk's metadata for diagnostics
type ChunkSummary struct {
Index int `json:"index"`
StartSegmentID int `json:"start_segment_id"`
EndSegmentID int `json:"end_segment_id"`
EstimatedTokens int `json:"estimated_tokens"`
SegmentCount int `json:"segment_count"`
}
// DetailedSummary provides per-chunk details for diagnostics
type DetailedSummary struct {
Summary `json:",inline"`
Chunks []ChunkSummary `json:"chunks"`
}
// ComputeSummary creates a Summary from sections and config
func ComputeSummary(sections []Section, config ChunkingConfig) Summary {
if len(sections) == 0 {
return Summary{
ChunkCount: 0,
MaxSectionTokens: config.MaxSectionTokens,
MinSectionTokens: config.MinSectionTokens,
TargetSections: config.TargetSections,
}
}
minTokens := sections[0].EstimatedTokens
maxTokens := sections[0].EstimatedTokens
totalTokens := 0
for _, sec := range sections {
if sec.EstimatedTokens < minTokens {
minTokens = sec.EstimatedTokens
}
if sec.EstimatedTokens > maxTokens {
maxTokens = sec.EstimatedTokens
}
totalTokens += sec.EstimatedTokens
}
return Summary{
ChunkCount: len(sections),
MinEstimatedTokens: minTokens,
MaxEstimatedTokens: maxTokens,
TotalEstimatedTokens: totalTokens,
TargetSections: config.TargetSections,
MaxSectionTokens: config.MaxSectionTokens,
MinSectionTokens: config.MinSectionTokens,
}
}
// ComputeDetailedSummary creates a DetailedSummary from sections and config
func ComputeDetailedSummary(sections []Section, config ChunkingConfig) DetailedSummary {
summary := ComputeSummary(sections, config)
chunks := make([]ChunkSummary, len(sections))
for i, sec := range sections {
chunks[i] = ChunkSummary{
Index: sec.Index,
StartSegmentID: sec.StartSegmentID,
EndSegmentID: sec.EndSegmentID,
EstimatedTokens: sec.EstimatedTokens,
SegmentCount: len(sec.Segments),
}
}
return DetailedSummary{
Summary: summary,
Chunks: chunks,
}
}

View File

@@ -0,0 +1,148 @@
package chunking
import (
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
func TestComputeSummary(t *testing.T) {
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
}
sections := []Section{
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
{Index: 1, EstimatedTokens: 50, StartSegmentID: 3, EndSegmentID: 4},
{Index: 2, EstimatedTokens: 20, StartSegmentID: 5, EndSegmentID: 5},
}
summary := ComputeSummary(sections, config)
if summary.ChunkCount != 3 {
t.Errorf("expected chunk_count=3, got %d", summary.ChunkCount)
}
if summary.MinEstimatedTokens != 20 {
t.Errorf("expected min_estimated_tokens=20, got %d", summary.MinEstimatedTokens)
}
if summary.MaxEstimatedTokens != 50 {
t.Errorf("expected max_estimated_tokens=50, got %d", summary.MaxEstimatedTokens)
}
if summary.TotalEstimatedTokens != 100 {
t.Errorf("expected total_estimated_tokens=100, got %d", summary.TotalEstimatedTokens)
}
if summary.MaxSectionTokens != 100 {
t.Errorf("expected max_section_tokens=100, got %d", summary.MaxSectionTokens)
}
if summary.MinSectionTokens != 10 {
t.Errorf("expected min_section_tokens=10, got %d", summary.MinSectionTokens)
}
if summary.TargetSections != nil {
t.Errorf("expected target_sections=nil, got %v", summary.TargetSections)
}
}
func TestComputeSummaryWithTarget(t *testing.T) {
target := 5
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
TargetSections: &target,
}
sections := []Section{
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
}
summary := ComputeSummary(sections, config)
if summary.TargetSections == nil || *summary.TargetSections != 5 {
t.Errorf("expected target_sections=5, got %v", summary.TargetSections)
}
}
func TestComputeSummaryEmptySections(t *testing.T) {
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
}
sections := []Section{}
summary := ComputeSummary(sections, config)
if summary.ChunkCount != 0 {
t.Errorf("expected chunk_count=0, got %d", summary.ChunkCount)
}
if summary.MinEstimatedTokens != 0 {
t.Errorf("expected min_estimated_tokens=0 for empty, got %d", summary.MinEstimatedTokens)
}
if summary.MaxSectionTokens != 100 {
t.Errorf("expected max_section_tokens preserved, got %d", summary.MaxSectionTokens)
}
}
func TestComputeDetailedSummary(t *testing.T) {
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
}
sections := []Section{
{
Index: 0,
EstimatedTokens: 30,
StartSegmentID: 1,
EndSegmentID: 2,
Segments: make([]schema.Segment, 2), // 2 segments
},
{
Index: 1,
EstimatedTokens: 50,
StartSegmentID: 3,
EndSegmentID: 5,
Segments: make([]schema.Segment, 3), // 3 segments
},
}
detailed := ComputeDetailedSummary(sections, config)
if detailed.ChunkCount != 2 {
t.Errorf("expected chunk_count=2, got %d", detailed.ChunkCount)
}
if len(detailed.Chunks) != 2 {
t.Fatalf("expected 2 chunk entries, got %d", len(detailed.Chunks))
}
// Check first chunk
if detailed.Chunks[0].Index != 0 {
t.Errorf("expected chunk[0].index=0, got %d", detailed.Chunks[0].Index)
}
if detailed.Chunks[0].StartSegmentID != 1 {
t.Errorf("expected chunk[0].start_segment_id=1, got %d", detailed.Chunks[0].StartSegmentID)
}
if detailed.Chunks[0].EndSegmentID != 2 {
t.Errorf("expected chunk[0].end_segment_id=2, got %d", detailed.Chunks[0].EndSegmentID)
}
if detailed.Chunks[0].EstimatedTokens != 30 {
t.Errorf("expected chunk[0].estimated_tokens=30, got %d", detailed.Chunks[0].EstimatedTokens)
}
if detailed.Chunks[0].SegmentCount != 2 {
t.Errorf("expected chunk[0].segment_count=2, got %d", detailed.Chunks[0].SegmentCount)
}
// Check second chunk
if detailed.Chunks[1].Index != 1 {
t.Errorf("expected chunk[1].index=1, got %d", detailed.Chunks[1].Index)
}
if detailed.Chunks[1].StartSegmentID != 3 {
t.Errorf("expected chunk[1].start_segment_id=3, got %d", detailed.Chunks[1].StartSegmentID)
}
if detailed.Chunks[1].EndSegmentID != 5 {
t.Errorf("expected chunk[1].end_segment_id=5, got %d", detailed.Chunks[1].EndSegmentID)
}
if detailed.Chunks[1].SegmentCount != 3 {
t.Errorf("expected chunk[1].segment_count=3, got %d", detailed.Chunks[1].SegmentCount)
}
}

View File

@@ -0,0 +1,55 @@
package chunking
import (
"strings"
"unicode"
)
// TokenEstimator provides a deterministic token estimation suitable for prompt budgeting.
// The estimator is approximate but stable, isolated, and replaceable.
type TokenEstimator interface {
EstimateTokens(text string) int
}
// SimpleTokenEstimator provides a basic deterministic token estimation.
// This uses a simple heuristic based on word count and punctuation.
type SimpleTokenEstimator struct{}
// NewSimpleTokenEstimator creates a new simple token estimator.
func NewSimpleTokenEstimator() *SimpleTokenEstimator {
return &SimpleTokenEstimator{}
}
// EstimateTokens provides a rough estimate of the number of tokens in the given text.
// This implementation uses a simple heuristic: count words and punctuation as tokens.
// The estimate is deterministic and stable for the same input text.
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
if text == "" {
return 0
}
// Simple heuristic: split on whitespace and count non-empty segments
words := strings.Fields(text)
tokenCount := len(words)
// Add some estimate for punctuation that might be separate tokens
punctuationCount := 0
for _, r := range text {
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
punctuationCount++
}
}
// Rough estimate: each word is a token, plus half the punctuation as separate tokens
return tokenCount + (punctuationCount / 2)
}
// ConstTokenEstimator returns a constant token count for testing purposes.
type ConstTokenEstimator struct {
Tokens int
}
// EstimateTokens returns the configured constant token count.
func (e *ConstTokenEstimator) EstimateTokens(text string) int {
return e.Tokens
}

View File

@@ -0,0 +1,59 @@
package chunking
import (
"testing"
)
func TestSimpleTokenEstimator(t *testing.T) {
estimator := NewSimpleTokenEstimator()
tests := []struct {
name string
text string
expected int
}{
{"empty string", "", 0},
{"single word", "hello", 1},
{"two words", "hello world", 2},
{"with punctuation", "hello, world!", 3}, // 2 words + 2 punctuation/2 = 3
{"multiple sentences", "Hello world. This is a test.", 7}, // 7 words + 2 punctuation/2 = 8? Actually "Hello world." has 3 punctuation
{"with apostrophes", "don't won't can't", 3},
{"with hyphens", "well-known state-of-the-art", 2}, // hyphens don't count
{"unicode text", "café naïve", 2},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := estimator.EstimateTokens(tt.text)
if got != tt.expected {
t.Errorf("EstimateTokens(%q) = %d, want %d", tt.text, got, tt.expected)
}
})
}
}
func TestConstTokenEstimator(t *testing.T) {
estimator := &ConstTokenEstimator{Tokens: 42}
if got := estimator.EstimateTokens("any text"); got != 42 {
t.Errorf("ConstTokenEstimator.EstimateTokens = %d, want 42", got)
}
if got := estimator.EstimateTokens(""); got != 42 {
t.Errorf("ConstTokenEstimator.EstimateTokens(empty) = %d, want 42", got)
}
}
func TestTokenEstimatorDeterminism(t *testing.T) {
estimator := NewSimpleTokenEstimator()
text := "The quick brown fox jumps over the lazy dog. Hello, world!"
// Run multiple times and verify same result
first := estimator.EstimateTokens(text)
for i := 0; i < 10; i++ {
got := estimator.EstimateTokens(text)
if got != first {
t.Errorf("EstimateTokens not deterministic: iteration %d got %d, first was %d", i, got, first)
}
}
}

View File

@@ -0,0 +1,199 @@
package config
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
)
type WorkDirRetention string
const (
WorkDirRetentionAuto WorkDirRetention = "auto"
WorkDirRetentionAlways WorkDirRetention = "always"
WorkDirRetentionNever WorkDirRetention = "never"
)
const (
DefaultModulesCSV = modulecatalog.KeyGlossary + "," + modulecatalog.KeyHomophones + "," + modulecatalog.KeyGlossary + "," + modulecatalog.KeySpokenWord + "," + modulecatalog.KeyGrammar
DefaultOutputSchema = "bare-segments"
DefaultPrimaryModel = "openrouter/google/gemma-4-31b-it"
DefaultPrimaryBaseURL = "https://openrouter.ai/api/v1"
DefaultPrimaryLLMTimeoutSeconds = 600
DefaultMaxRetries = 3
DefaultLLMConcurrency = 1
DefaultValidationMaxPromptTokens = 2048
DefaultMaxSectionTokens = 8192
DefaultMinSectionTokens = 2048
DefaultConfidenceThreshold = 0.8
DefaultNormalizeMaxSegmentGap = 4.0
DefaultNormalizeEllipsisGap = 3.5
DefaultNormalizeMaxSegmentDuration = 60.0
DefaultNormalizeMaxSegmentTokens = 2048
DefaultTranscriptDescriptionMaxChars = 500
DefaultWorkDir = "/tmp/audita"
DefaultWorkDirRetention WorkDirRetention = WorkDirRetentionAuto
)
type Config struct {
Modules []string
OutputSchema string
PrimaryLLM LLMConfig
ValidationLLM ValidationLLMConfig
TotalLLMConcurrency int
ProposalLLMConcurrency int
ValidationLLMConcurrency *int
ValidationMaxPromptTokens int
MaxSectionTokens int
MinSectionTokens int
TargetSections *int
Thresholds ConfidenceThresholds
Normalization NormalizationConfig
TranscriptDescription string
WorkDir string
WorkDirRetention WorkDirRetention
}
type LLMConfig struct {
APIKey string
Model string
BaseURL string
TimeoutSeconds int
MaxRetries int
// Concurrency is retained as a backward-compatible alias for
// TotalLLMConcurrency.
Concurrency int
}
type ValidationLLMConfig struct {
APIKey string
Model string
BaseURL string
TimeoutSeconds *int
MaxRetries *int
// Concurrency is retained as a backward-compatible alias for
// ValidationLLMConcurrency.
Concurrency *int
}
type ConfidenceThresholds struct {
Glossary float64
Grammar float64
Homophones float64
SpokenWord float64
}
type NormalizationConfig struct {
MaxSegmentGap float64
EllipsisGap float64
MaxSegmentDuration float64
MaxSegmentTokens int
}
func Default() Config {
modules, _ := ParseModulesCSV(DefaultModulesCSV)
return Config{
Modules: modules,
OutputSchema: DefaultOutputSchema,
PrimaryLLM: LLMConfig{
Model: DefaultPrimaryModel,
BaseURL: DefaultPrimaryBaseURL,
TimeoutSeconds: DefaultPrimaryLLMTimeoutSeconds,
MaxRetries: DefaultMaxRetries,
Concurrency: DefaultLLMConcurrency,
},
ValidationLLM: ValidationLLMConfig{},
TotalLLMConcurrency: DefaultLLMConcurrency,
ProposalLLMConcurrency: DefaultLLMConcurrency,
ValidationLLMConcurrency: nil,
ValidationMaxPromptTokens: DefaultValidationMaxPromptTokens,
MaxSectionTokens: DefaultMaxSectionTokens,
MinSectionTokens: DefaultMinSectionTokens,
TargetSections: nil,
Thresholds: ConfidenceThresholds{
Glossary: DefaultConfidenceThreshold,
Grammar: DefaultConfidenceThreshold,
Homophones: DefaultConfidenceThreshold,
SpokenWord: DefaultConfidenceThreshold,
},
Normalization: NormalizationConfig{
MaxSegmentGap: DefaultNormalizeMaxSegmentGap,
EllipsisGap: DefaultNormalizeEllipsisGap,
MaxSegmentDuration: DefaultNormalizeMaxSegmentDuration,
MaxSegmentTokens: DefaultNormalizeMaxSegmentTokens,
},
WorkDir: DefaultWorkDir,
WorkDirRetention: DefaultWorkDirRetention,
}
}
func ParseModulesCSV(raw string) ([]string, error) {
parts := strings.Split(raw, ",")
modules := make([]string, 0, len(parts))
for _, part := range parts {
trimmed := strings.TrimSpace(part)
if trimmed == "" {
return nil, fmt.Errorf("modules list contains an empty value")
}
modules = append(modules, trimmed)
}
if len(modules) == 0 {
return nil, fmt.Errorf("modules list must not be empty")
}
return modules, nil
}
func (c Config) EffectiveValidationLLMConfig() LLMConfig {
effective := c.PrimaryLLM
if c.ValidationLLM.APIKey != "" {
effective.APIKey = c.ValidationLLM.APIKey
}
if c.ValidationLLM.Model != "" {
effective.Model = c.ValidationLLM.Model
}
if c.ValidationLLM.BaseURL != "" {
effective.BaseURL = c.ValidationLLM.BaseURL
}
if c.ValidationLLM.TimeoutSeconds != nil {
effective.TimeoutSeconds = *c.ValidationLLM.TimeoutSeconds
}
if c.ValidationLLM.MaxRetries != nil {
effective.MaxRetries = *c.ValidationLLM.MaxRetries
}
effective.Concurrency = c.EffectiveValidationLLMConcurrency()
return effective
}
func (c Config) EffectiveValidationLLMConcurrency() int {
if c.ValidationLLMConcurrency != nil {
return *c.ValidationLLMConcurrency
}
return c.TotalLLMConcurrency
}
func (c Config) EffectiveProposalLLMConcurrency() int {
if c.ProposalLLMConcurrency > 0 {
return c.ProposalLLMConcurrency
}
return c.TotalLLMConcurrency
}
func (c *Config) syncLegacyConcurrencyAliases() {
if c == nil {
return
}
c.PrimaryLLM.Concurrency = c.TotalLLMConcurrency
c.ValidationLLM.Concurrency = intPtr(c.ValidationLLMConcurrency)
}
func intPtr(v *int) *int {
if v == nil {
return nil
}
x := *v
return &x
}

View File

@@ -0,0 +1,458 @@
package config
import (
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
)
func TestDefaultConfigValues(t *testing.T) {
cfg := Default()
if got, want := strings.Join(cfg.Modules, ","), DefaultModulesCSV; got != want {
t.Fatalf("modules mismatch: got %q want %q", got, want)
}
if cfg.OutputSchema != DefaultOutputSchema {
t.Fatalf("unexpected default output schema: %q", cfg.OutputSchema)
}
if cfg.PrimaryLLM.Model != DefaultPrimaryModel {
t.Fatalf("unexpected default primary model: %q", cfg.PrimaryLLM.Model)
}
if cfg.PrimaryLLM.BaseURL != DefaultPrimaryBaseURL {
t.Fatalf("unexpected default primary base url: %q", cfg.PrimaryLLM.BaseURL)
}
if cfg.PrimaryLLM.TimeoutSeconds != DefaultPrimaryLLMTimeoutSeconds {
t.Fatalf("unexpected default timeout seconds: %d", cfg.PrimaryLLM.TimeoutSeconds)
}
if cfg.PrimaryLLM.MaxRetries != DefaultMaxRetries {
t.Fatalf("unexpected default max retries: %d", cfg.PrimaryLLM.MaxRetries)
}
if cfg.TotalLLMConcurrency != DefaultLLMConcurrency {
t.Fatalf("unexpected default total llm concurrency: %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != DefaultLLMConcurrency {
t.Fatalf("unexpected default proposal llm concurrency: %d", cfg.ProposalLLMConcurrency)
}
if cfg.ValidationLLMConcurrency != nil {
t.Fatalf("expected validation llm concurrency to be unset by default")
}
if cfg.PrimaryLLM.Concurrency != cfg.TotalLLMConcurrency {
t.Fatalf("expected primary llm concurrency alias to mirror total, got primary=%d total=%d", cfg.PrimaryLLM.Concurrency, cfg.TotalLLMConcurrency)
}
if cfg.ValidationLLM.TimeoutSeconds != nil {
t.Fatalf("expected validation timeout to be unset by default")
}
if cfg.ValidationLLM.MaxRetries != nil {
t.Fatalf("expected validation max retries to be unset by default")
}
if cfg.ValidationLLM.Concurrency != nil {
t.Fatalf("expected legacy validation llm concurrency alias to be unset by default")
}
if cfg.TargetSections != nil {
t.Fatalf("expected target sections to be unset by default")
}
if cfg.WorkDir != DefaultWorkDir {
t.Fatalf("unexpected default work dir: %q", cfg.WorkDir)
}
if cfg.TranscriptDescription != "" {
t.Fatalf("expected default transcript description to be empty, got %q", cfg.TranscriptDescription)
}
if cfg.WorkDirRetention != DefaultWorkDirRetention {
t.Fatalf("unexpected default work dir retention: %q", cfg.WorkDirRetention)
}
if err := cfg.Validate(); err != nil {
t.Fatalf("default config should validate: %v", err)
}
}
func TestLoadFromEnvOverridesAndFallback(t *testing.T) {
env := map[string]string{
"AUDITA_MODEL": "openai/gpt-4.1-mini",
"AUDITA_BASE_URL": "https://api.openai.com/v1",
"AUDITA_LLM_TIMEOUT_SECONDS": "120",
"AUDITA_MAX_RETRIES": "7",
"AUDITA_TOTAL_LLM_CONCURRENCY": "6",
"AUDITA_PROPOSAL_LLM_CONCURRENCY": "4",
"AUDITA_VALIDATION_LLM_CONCURRENCY": "2",
"AUDITA_VALIDATION_MAX_PROMPT_TOKENS": "4096",
"AUDITA_MAX_SECTION_TOKENS": "9000",
"AUDITA_MIN_SECTION_TOKENS": "3000",
"AUDITA_TARGET_SECTIONS": "5",
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.9",
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.7",
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.6",
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.5",
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "2.5",
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "2.0",
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "30.0",
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "1024",
"AUDITA_WORK_DIR": "/var/tmp/audita",
"AUDITA_WORK_DIR_RETENTION": "always",
"OPENROUTER_API_KEY": "fallback-key",
}
cfg, err := loadFromLookup(mapLookup(env))
if err != nil {
t.Fatalf("loadFromLookup returned error: %v", err)
}
if cfg.PrimaryLLM.APIKey != "fallback-key" {
t.Fatalf("expected OPENROUTER_API_KEY fallback, got %q", cfg.PrimaryLLM.APIKey)
}
if cfg.PrimaryLLM.Model != env["AUDITA_MODEL"] {
t.Fatalf("unexpected model: %q", cfg.PrimaryLLM.Model)
}
if cfg.PrimaryLLM.BaseURL != env["AUDITA_BASE_URL"] {
t.Fatalf("unexpected base url: %q", cfg.PrimaryLLM.BaseURL)
}
if cfg.TargetSections == nil || *cfg.TargetSections != 5 {
t.Fatalf("unexpected target sections: %#v", cfg.TargetSections)
}
if cfg.TotalLLMConcurrency != 6 {
t.Fatalf("unexpected total llm concurrency: %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != 4 {
t.Fatalf("unexpected proposal llm concurrency: %d", cfg.ProposalLLMConcurrency)
}
if cfg.ValidationLLMConcurrency == nil || *cfg.ValidationLLMConcurrency != 2 {
t.Fatalf("unexpected validation llm concurrency: %#v", cfg.ValidationLLMConcurrency)
}
if cfg.PrimaryLLM.Concurrency != 6 {
t.Fatalf("expected primary alias concurrency 6, got %d", cfg.PrimaryLLM.Concurrency)
}
if cfg.ValidationLLM.Concurrency == nil || *cfg.ValidationLLM.Concurrency != 2 {
t.Fatalf("expected validation alias concurrency 2, got %#v", cfg.ValidationLLM.Concurrency)
}
if cfg.WorkDirRetention != WorkDirRetentionAlways {
t.Fatalf("unexpected work dir retention: %q", cfg.WorkDirRetention)
}
}
func TestLoadFromEnvLegacyLLMConcurrencyAliasForTotalAndProposal(t *testing.T) {
env := map[string]string{
"AUDITA_LLM_CONCURRENCY": "5",
}
cfg, err := loadFromLookup(mapLookup(env))
if err != nil {
t.Fatalf("loadFromLookup returned error: %v", err)
}
if cfg.TotalLLMConcurrency != 5 {
t.Fatalf("expected total concurrency from legacy alias, got %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != 5 {
t.Fatalf("expected proposal concurrency to inherit legacy total, got %d", cfg.ProposalLLMConcurrency)
}
}
func TestLoadFromEnvCanonicalTotalWinsLegacyAlias(t *testing.T) {
env := map[string]string{
"AUDITA_TOTAL_LLM_CONCURRENCY": "4",
"AUDITA_LLM_CONCURRENCY": "9",
}
cfg, err := loadFromLookup(mapLookup(env))
if err != nil {
t.Fatalf("loadFromLookup returned error: %v", err)
}
if cfg.TotalLLMConcurrency != 4 {
t.Fatalf("expected canonical total to win over legacy alias, got %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != 4 {
t.Fatalf("expected proposal to inherit canonical total when unset, got %d", cfg.ProposalLLMConcurrency)
}
}
func TestLoadFromEnvUsesAuditaLLMAPIKeyOverFallback(t *testing.T) {
env := map[string]string{
"AUDITA_LLM_API_KEY": "primary-key",
"OPENROUTER_API_KEY": "fallback-key",
}
cfg, err := loadFromLookup(mapLookup(env))
if err != nil {
t.Fatalf("loadFromLookup returned error: %v", err)
}
if cfg.PrimaryLLM.APIKey != "primary-key" {
t.Fatalf("expected AUDITA_LLM_API_KEY to win, got %q", cfg.PrimaryLLM.APIKey)
}
}
func TestApplyCLIOverridesPrecedence(t *testing.T) {
cfg := Default()
cfg.PrimaryLLM.Model = "env-model"
cfg.WorkDir = "/env/work"
model := "cli-model"
workDir := "/cli/work"
modules := "grammar"
outputSchema := "audita-v1"
totalLLMConcurrency := 5
proposalLLMConcurrency := 3
overrides := CLIOverrides{
PrimaryModel: &model,
WorkDir: &workDir,
ModulesCSV: &modules,
OutputSchema: &outputSchema,
TotalLLMConcurrency: &totalLLMConcurrency,
ProposalLLMConcurrency: &proposalLLMConcurrency,
}
if err := cfg.ApplyCLIOverrides(overrides); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.PrimaryLLM.Model != "cli-model" {
t.Fatalf("expected CLI model override, got %q", cfg.PrimaryLLM.Model)
}
if cfg.WorkDir != "/cli/work" {
t.Fatalf("expected CLI work dir override, got %q", cfg.WorkDir)
}
if !reflect.DeepEqual(cfg.Modules, []string{"grammar"}) {
t.Fatalf("unexpected modules: %#v", cfg.Modules)
}
if cfg.OutputSchema != "audita-v1" {
t.Fatalf("expected CLI output schema override, got %q", cfg.OutputSchema)
}
if cfg.TotalLLMConcurrency != 5 {
t.Fatalf("expected CLI total concurrency override, got %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != 3 {
t.Fatalf("expected CLI proposal concurrency override, got %d", cfg.ProposalLLMConcurrency)
}
}
func TestApplyCLIOverridesTrimsTranscriptDescription(t *testing.T) {
cfg := Default()
description := " background context about speakers "
if err := cfg.ApplyCLIOverrides(CLIOverrides{TranscriptDescription: &description}); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.TranscriptDescription != "background context about speakers" {
t.Fatalf("unexpected transcript description trim result: %q", cfg.TranscriptDescription)
}
}
func TestValidationRejectsOverlyLongTranscriptDescription(t *testing.T) {
cfg := Default()
cfg.TranscriptDescription = strings.Repeat("a", DefaultTranscriptDescriptionMaxChars+1)
err := cfg.Validate()
if err == nil {
t.Fatalf("expected transcript description length validation error")
}
if !strings.Contains(err.Error(), "transcript description must be 500 characters or fewer") {
t.Fatalf("unexpected validation error: %v", err)
}
}
func TestApplyCLIOverridesLegacyLLMConcurrencyAlias(t *testing.T) {
cfg := Default()
aliasConcurrency := 6
if err := cfg.ApplyCLIOverrides(CLIOverrides{PrimaryLLMConcurrency: &aliasConcurrency}); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.TotalLLMConcurrency != 6 {
t.Fatalf("expected legacy --llm-concurrency alias to set total, got %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != 6 {
t.Fatalf("expected proposal to inherit aliased total when unset, got %d", cfg.ProposalLLMConcurrency)
}
}
func TestApplyCLIOverridesCanonicalTotalWinsLegacyAlias(t *testing.T) {
cfg := Default()
canonicalTotal := 4
legacyAlias := 9
if err := cfg.ApplyCLIOverrides(CLIOverrides{TotalLLMConcurrency: &canonicalTotal, PrimaryLLMConcurrency: &legacyAlias}); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.TotalLLMConcurrency != 4 {
t.Fatalf("expected canonical total concurrency to win, got %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != 4 {
t.Fatalf("expected proposal to inherit canonical total when proposal is unset, got %d", cfg.ProposalLLMConcurrency)
}
}
func TestValidationFailures(t *testing.T) {
cfg := Default()
cfg.OutputSchema = "unknown-schema"
cfg.PrimaryLLM.TimeoutSeconds = -1
cfg.TotalLLMConcurrency = 0
cfg.ProposalLLMConcurrency = 0
validationConcurrency := 5
cfg.ValidationLLMConcurrency = &validationConcurrency
cfg.ValidationMaxPromptTokens = 0
cfg.MaxSectionTokens = 100
cfg.MinSectionTokens = 200
cfg.Thresholds.Grammar = 1.5
cfg.WorkDirRetention = WorkDirRetention("sometimes")
err := cfg.Validate()
if err == nil {
t.Fatalf("expected validation error")
}
message := err.Error()
for _, expected := range []string{
"primary llm timeout seconds",
"total llm concurrency",
"proposal llm concurrency",
"validation llm concurrency must be less than or equal to total llm concurrency",
"validation max prompt tokens",
"min section tokens",
"grammar confidence threshold",
"work dir retention",
"unsupported output schema",
} {
if !strings.Contains(message, expected) {
t.Fatalf("expected error to contain %q, got %q", expected, message)
}
}
}
func TestValidationRejectsUnsupportedModuleKey(t *testing.T) {
cfg := Default()
cfg.Modules = []string{modulecatalog.KeyGlossary, "made_up"}
err := cfg.Validate()
if err == nil {
t.Fatalf("expected validation error for unsupported module key")
}
if !strings.Contains(err.Error(), `unsupported module key "made_up"`) {
t.Fatalf("expected unsupported module key error, got %q", err.Error())
}
}
func TestValidationAllowsRepeatedSupportedModuleKeys(t *testing.T) {
cfg := Default()
cfg.Modules = []string{modulecatalog.KeyGlossary, modulecatalog.KeyGlossary, modulecatalog.KeyGrammar}
if err := cfg.Validate(); err != nil {
t.Fatalf("expected repeated supported module keys to validate, got %v", err)
}
}
func TestValidationAcceptsAllSupportedOutputSchemas(t *testing.T) {
for _, schemaKey := range outputschema.SupportedKeys() {
cfg := Default()
cfg.OutputSchema = schemaKey
if err := cfg.Validate(); err != nil {
t.Fatalf("expected output schema %q to validate, got %v", schemaKey, err)
}
}
}
func TestEffectiveValidationLLMInheritance(t *testing.T) {
cfg := Default()
cfg.PrimaryLLM.APIKey = "primary-key"
cfg.PrimaryLLM.Model = "primary-model"
cfg.PrimaryLLM.BaseURL = "https://primary.example/v1"
cfg.PrimaryLLM.TimeoutSeconds = 111
cfg.PrimaryLLM.MaxRetries = 2
cfg.TotalLLMConcurrency = 7
cfg.syncLegacyConcurrencyAliases()
effective := cfg.EffectiveValidationLLMConfig()
if effective.APIKey != "primary-key" || effective.Model != "primary-model" || effective.BaseURL != "https://primary.example/v1" || effective.TimeoutSeconds != 111 || effective.MaxRetries != 2 || effective.Concurrency != 7 {
t.Fatalf("unexpected inherited config: %#v", effective)
}
validationTimeout := 222
validationRetries := 9
cfg.ValidationLLM.APIKey = "validation-key"
cfg.ValidationLLM.Model = "validation-model"
cfg.ValidationLLM.BaseURL = "https://validation.example/v1"
cfg.ValidationLLM.TimeoutSeconds = &validationTimeout
cfg.ValidationLLM.MaxRetries = &validationRetries
validationConcurrency := 4
cfg.ValidationLLMConcurrency = &validationConcurrency
cfg.syncLegacyConcurrencyAliases()
effective = cfg.EffectiveValidationLLMConfig()
if effective.APIKey != "validation-key" || effective.Model != "validation-model" || effective.BaseURL != "https://validation.example/v1" || effective.TimeoutSeconds != 222 || effective.MaxRetries != 9 || effective.Concurrency != 4 {
t.Fatalf("unexpected overridden validation config: %#v", effective)
}
}
func TestValidationLLMConcurrencyCannotExceedTotal(t *testing.T) {
cfg := Default()
cfg.TotalLLMConcurrency = 2
cfg.ProposalLLMConcurrency = 2
validationConcurrency := 3
cfg.ValidationLLMConcurrency = &validationConcurrency
if err := cfg.Validate(); err == nil {
t.Fatal("expected validation error when validation llm concurrency exceeds total")
}
validationConcurrency = 2
cfg.ValidationLLMConcurrency = &validationConcurrency
if err := cfg.Validate(); err != nil {
t.Fatalf("expected equal concurrency to validate, got %v", err)
}
}
func TestProposalLLMConcurrencyCannotExceedTotal(t *testing.T) {
cfg := Default()
cfg.TotalLLMConcurrency = 2
cfg.ProposalLLMConcurrency = 3
if err := cfg.Validate(); err == nil {
t.Fatal("expected validation error when proposal llm concurrency exceeds total")
}
cfg.ProposalLLMConcurrency = 2
if err := cfg.Validate(); err != nil {
t.Fatalf("expected equal concurrency to validate, got %v", err)
}
}
func TestCLITotalLLMConcurrencyOverrideDrivesEffectiveValidationConcurrencyWhenValidationUnset(t *testing.T) {
cfg := Default()
totalLLMConcurrency := 6
if err := cfg.ApplyCLIOverrides(CLIOverrides{TotalLLMConcurrency: &totalLLMConcurrency}); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.ValidationLLMConcurrency != nil {
t.Fatalf("expected validation concurrency to remain unset, got %#v", cfg.ValidationLLMConcurrency)
}
if cfg.EffectiveValidationLLMConcurrency() != 6 {
t.Fatalf("expected inherited validation concurrency 6, got %d", cfg.EffectiveValidationLLMConcurrency())
}
}
func TestRedactedConfig(t *testing.T) {
cfg := Default()
cfg.PrimaryLLM.APIKey = "secret-primary"
cfg.ValidationLLM.APIKey = "secret-validation"
redacted := cfg.Redacted()
if redacted.PrimaryLLM.APIKey != redactedSecret {
t.Fatalf("expected primary api key to be redacted, got %q", redacted.PrimaryLLM.APIKey)
}
if redacted.ValidationLLM.APIKey != redactedSecret {
t.Fatalf("expected validation api key to be redacted, got %q", redacted.ValidationLLM.APIKey)
}
if cfg.PrimaryLLM.APIKey != "secret-primary" {
t.Fatalf("redaction should not mutate original config")
}
}
func mapLookup(values map[string]string) func(string) (string, bool) {
return func(key string) (string, bool) {
value, ok := values[key]
return value, ok
}
}

View File

@@ -0,0 +1,119 @@
package config
import (
"fmt"
"os"
"strings"
)
type EffectiveConfigErrorKind string
const (
EffectiveConfigErrorResolvePath EffectiveConfigErrorKind = "resolve_path"
EffectiveConfigErrorLoadFile EffectiveConfigErrorKind = "load_file"
EffectiveConfigErrorApplyFile EffectiveConfigErrorKind = "apply_file"
EffectiveConfigErrorApplyEnv EffectiveConfigErrorKind = "apply_env"
)
type EffectiveConfigError struct {
Kind EffectiveConfigErrorKind
Err error
}
func (e *EffectiveConfigError) Error() string {
if e == nil || e.Err == nil {
return ""
}
return e.Err.Error()
}
func (e *EffectiveConfigError) Unwrap() error {
if e == nil {
return nil
}
return e.Err
}
type EffectiveConfig struct {
Config Config
ConfigPath string
ConfigSource string
ConfigVersion *int
}
func ResolveConfigPath(cliConfigPath string, cliConfigPathSet bool) (path string, source string, err error) {
return resolveConfigPathWithLookup(cliConfigPath, cliConfigPathSet, os.LookupEnv, os.Stat, DefaultConfigSearchPaths)
}
func LoadEffectiveConfig(cliConfigPath string, cliConfigPathSet bool) (EffectiveConfig, error) {
return loadEffectiveConfigWithLookup(cliConfigPath, cliConfigPathSet, os.LookupEnv, os.Stat, DefaultConfigSearchPaths)
}
func loadEffectiveConfigWithLookup(cliConfigPath string, cliConfigPathSet bool, lookup func(string) (string, bool), statPath func(string) (os.FileInfo, error), defaultSearchPaths []string) (EffectiveConfig, error) {
configPath, configSource, err := resolveConfigPathWithLookup(cliConfigPath, cliConfigPathSet, lookup, statPath, defaultSearchPaths)
if err != nil {
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorResolvePath, Err: err}
}
cfg := Default()
var configVersion *int
if configPath != "" {
fileCfg, fileErr := LoadFileConfig(configPath)
if fileErr != nil {
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorLoadFile, Err: fileErr}
}
if applyErr := cfg.ApplyFileConfig(fileCfg); applyErr != nil {
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorApplyFile, Err: applyErr}
}
configVersion = &fileCfg.Version
}
if applyEnvErr := cfg.applyEnvOverrides(lookup); applyEnvErr != nil {
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorApplyEnv, Err: applyEnvErr}
}
return EffectiveConfig{
Config: cfg,
ConfigPath: configPath,
ConfigSource: configSource,
ConfigVersion: configVersion,
}, nil
}
func resolveConfigPathWithLookup(cliConfigPath string, cliConfigPathSet bool, lookup func(string) (string, bool), statPath func(string) (os.FileInfo, error), defaultSearchPaths []string) (path string, source string, err error) {
if cliConfigPathSet {
path = strings.TrimSpace(cliConfigPath)
if path == "" {
return "", "", fmt.Errorf("--config requires a non-empty path")
}
if _, statErr := statPath(path); statErr != nil {
if os.IsNotExist(statErr) {
return "", "", fmt.Errorf("config file not found: %s", path)
}
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
}
return path, "flag", nil
}
if raw, ok := lookup("AUDITA_CONFIG"); ok {
path = strings.TrimSpace(raw)
if path == "" {
return "", "", fmt.Errorf("AUDITA_CONFIG must not be empty")
}
if _, statErr := statPath(path); statErr != nil {
if os.IsNotExist(statErr) {
return "", "", fmt.Errorf("config file not found: %s", path)
}
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
}
return path, "env", nil
}
for _, defaultPath := range defaultSearchPaths {
if _, statErr := statPath(defaultPath); statErr == nil {
return defaultPath, "default", nil
} else if !os.IsNotExist(statErr) {
return "", "", fmt.Errorf("cannot access config file %s: %w", defaultPath, statErr)
}
}
return "", "", nil
}

View File

@@ -0,0 +1,163 @@
package config
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestResolveConfigPathWithLookupMatrix(t *testing.T) {
statFor := func(existing map[string]bool) func(string) (os.FileInfo, error) {
return func(path string) (os.FileInfo, error) {
if existing[path] {
return nil, nil
}
return nil, os.ErrNotExist
}
}
tests := []struct {
name string
cliPath string
cliPathSet bool
lookup func(string) (string, bool)
stat func(string) (os.FileInfo, error)
defaultSearchPaths []string
wantPath string
wantSource string
wantErrContains string
}{
{
name: "explicit config path",
cliPath: "/tmp/explicit.yml",
cliPathSet: true,
lookup: func(string) (string, bool) { return "", false },
stat: statFor(map[string]bool{"/tmp/explicit.yml": true}),
defaultSearchPaths: []string{
"/usr/local/etc/audita/config.yml",
"/etc/audita/config.yml",
},
wantPath: "/tmp/explicit.yml",
wantSource: "flag",
},
{
name: "env config path",
cliPathSet: false,
lookup: func(key string) (string, bool) {
if key == "AUDITA_CONFIG" {
return "/tmp/from-env.yml", true
}
return "", false
},
stat: statFor(map[string]bool{"/tmp/from-env.yml": true}),
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
wantPath: "/tmp/from-env.yml",
wantSource: "env",
},
{
name: "default search path",
cliPathSet: false,
lookup: func(string) (string, bool) { return "", false },
stat: statFor(map[string]bool{
"/usr/local/etc/audita/config.yml": true,
"/etc/audita/config.yml": true,
}),
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
wantPath: "/usr/local/etc/audita/config.yml",
wantSource: "default",
},
{
name: "explicit missing path",
cliPath: "/tmp/missing.yml",
cliPathSet: true,
lookup: func(string) (string, bool) { return "", false },
stat: statFor(map[string]bool{}),
defaultSearchPaths: []string{
"/usr/local/etc/audita/config.yml",
"/etc/audita/config.yml",
},
wantErrContains: "config file not found",
},
{
name: "missing env path",
cliPathSet: false,
lookup: func(key string) (string, bool) {
if key == "AUDITA_CONFIG" {
return "/tmp/missing-from-env.yml", true
}
return "", false
},
stat: statFor(map[string]bool{}),
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
wantErrContains: "config file not found",
},
{
name: "missing default paths",
cliPathSet: false,
lookup: func(string) (string, bool) { return "", false },
stat: statFor(map[string]bool{}),
defaultSearchPaths: []string{
"/usr/local/etc/audita/config.yml",
"/etc/audita/config.yml",
},
wantPath: "",
wantSource: "",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
gotPath, gotSource, err := resolveConfigPathWithLookup(tc.cliPath, tc.cliPathSet, tc.lookup, tc.stat, tc.defaultSearchPaths)
if tc.wantErrContains != "" {
if err == nil || !strings.Contains(err.Error(), tc.wantErrContains) {
t.Fatalf("expected error containing %q, got %v", tc.wantErrContains, err)
}
return
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if gotPath != tc.wantPath || gotSource != tc.wantSource {
t.Fatalf("unexpected result: got path=%q source=%q, want path=%q source=%q", gotPath, gotSource, tc.wantPath, tc.wantSource)
}
})
}
}
func TestLoadEffectiveConfigWithLookupAppliesDefaultsFileThenEnv(t *testing.T) {
tempDir := t.TempDir()
configPath := filepath.Join(tempDir, "config.yml")
configYAML := "version: 1\nllm:\n proposal:\n model: file-model\n"
if err := os.WriteFile(configPath, []byte(configYAML), 0o644); err != nil {
t.Fatalf("write config file: %v", err)
}
lookup := func(key string) (string, bool) {
switch key {
case "AUDITA_CONFIG":
return configPath, true
case "AUDITA_MODEL":
return "env-model", true
default:
return "", false
}
}
result, err := loadEffectiveConfigWithLookup("", false, lookup, os.Stat, DefaultConfigSearchPaths)
if err != nil {
t.Fatalf("loadEffectiveConfigWithLookup error: %v", err)
}
if result.ConfigPath != configPath {
t.Fatalf("unexpected config path: %q", result.ConfigPath)
}
if result.ConfigSource != "env" {
t.Fatalf("unexpected config source: %q", result.ConfigSource)
}
if result.ConfigVersion == nil || *result.ConfigVersion != SupportedFileConfigVersion {
t.Fatalf("unexpected config version: %#v", result.ConfigVersion)
}
if result.Config.PrimaryLLM.Model != "env-model" {
t.Fatalf("expected env override to win over file value, got %q", result.Config.PrimaryLLM.Model)
}
}

268
internal/core/config/env.go Normal file
View File

@@ -0,0 +1,268 @@
package config
import (
"fmt"
"os"
"strconv"
)
const (
DefaultConfigPath = "/etc/audita/config.yml"
DefaultConfigPathUsrLocal = "/usr/local/etc/audita/config.yml"
)
var DefaultConfigSearchPaths = []string{
DefaultConfigPathUsrLocal,
DefaultConfigPath,
}
func LoadFromEnv() (Config, error) {
cfg := Default()
if err := cfg.applyEnvOverrides(os.LookupEnv); err != nil {
return Config{}, err
}
return cfg, nil
}
func loadFromLookup(lookup func(string) (string, bool)) (Config, error) {
cfg := Default()
if err := cfg.applyEnvOverrides(lookup); err != nil {
return Config{}, err
}
return cfg, nil
}
func (c *Config) ApplyEnvOverrides() error {
return c.applyEnvOverrides(os.LookupEnv)
}
func (c *Config) applyEnvOverrides(lookup func(string) (string, bool)) error {
if c == nil {
return fmt.Errorf("config must not be nil")
}
cfg := c
if raw, ok := lookup("AUDITA_MODULES"); ok {
modules, err := ParseModulesCSV(raw)
if err != nil {
return fmt.Errorf("AUDITA_MODULES: %w", err)
}
cfg.Modules = modules
}
if raw, ok := lookup("AUDITA_LLM_API_KEY"); ok {
cfg.PrimaryLLM.APIKey = raw
} else if raw, ok := lookup("OPENROUTER_API_KEY"); ok {
cfg.PrimaryLLM.APIKey = raw
}
if raw, ok := lookup("AUDITA_VALIDATION_LLM_API_KEY"); ok {
cfg.ValidationLLM.APIKey = raw
}
if raw, ok := lookup("AUDITA_MODEL"); ok {
cfg.PrimaryLLM.Model = raw
}
if raw, ok := lookup("AUDITA_VALIDATION_MODEL"); ok {
cfg.ValidationLLM.Model = raw
}
if raw, ok := lookup("AUDITA_BASE_URL"); ok {
cfg.PrimaryLLM.BaseURL = raw
}
if raw, ok := lookup("AUDITA_VALIDATION_BASE_URL"); ok {
cfg.ValidationLLM.BaseURL = raw
}
if raw, ok := lookup("AUDITA_LLM_TIMEOUT_SECONDS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_LLM_TIMEOUT_SECONDS: %w", err)
}
cfg.PrimaryLLM.TimeoutSeconds = value
}
if raw, ok := lookup("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS: %w", err)
}
cfg.ValidationLLM.TimeoutSeconds = &value
}
if raw, ok := lookup("AUDITA_MAX_RETRIES"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_MAX_RETRIES: %w", err)
}
cfg.PrimaryLLM.MaxRetries = value
}
totalConcurrencySet := false
if raw, ok := lookup("AUDITA_TOTAL_LLM_CONCURRENCY"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_TOTAL_LLM_CONCURRENCY: %w", err)
}
cfg.TotalLLMConcurrency = value
totalConcurrencySet = true
}
if raw, ok := lookup("AUDITA_LLM_CONCURRENCY"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_LLM_CONCURRENCY: %w", err)
}
if !totalConcurrencySet {
cfg.TotalLLMConcurrency = value
totalConcurrencySet = true
}
}
proposalConcurrencySet := false
if raw, ok := lookup("AUDITA_PROPOSAL_LLM_CONCURRENCY"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_PROPOSAL_LLM_CONCURRENCY: %w", err)
}
cfg.ProposalLLMConcurrency = value
proposalConcurrencySet = true
}
if totalConcurrencySet && !proposalConcurrencySet {
cfg.ProposalLLMConcurrency = cfg.TotalLLMConcurrency
}
if raw, ok := lookup("AUDITA_VALIDATION_MAX_RETRIES"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_VALIDATION_MAX_RETRIES: %w", err)
}
cfg.ValidationLLM.MaxRetries = &value
}
if raw, ok := lookup("AUDITA_VALIDATION_LLM_CONCURRENCY"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_VALIDATION_LLM_CONCURRENCY: %w", err)
}
cfg.ValidationLLMConcurrency = &value
}
if raw, ok := lookup("AUDITA_VALIDATION_MAX_PROMPT_TOKENS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_VALIDATION_MAX_PROMPT_TOKENS: %w", err)
}
cfg.ValidationMaxPromptTokens = value
}
if raw, ok := lookup("AUDITA_MAX_SECTION_TOKENS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_MAX_SECTION_TOKENS: %w", err)
}
cfg.MaxSectionTokens = value
}
if raw, ok := lookup("AUDITA_MIN_SECTION_TOKENS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_MIN_SECTION_TOKENS: %w", err)
}
cfg.MinSectionTokens = value
}
if raw, ok := lookup("AUDITA_TARGET_SECTIONS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_TARGET_SECTIONS: %w", err)
}
cfg.TargetSections = &value
}
if raw, ok := lookup("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD: %w", err)
}
cfg.Thresholds.Glossary = value
}
if raw, ok := lookup("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD: %w", err)
}
cfg.Thresholds.Grammar = value
}
if raw, ok := lookup("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD: %w", err)
}
cfg.Thresholds.Homophones = value
}
if raw, ok := lookup("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD: %w", err)
}
cfg.Thresholds.SpokenWord = value
}
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_GAP"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_GAP: %w", err)
}
cfg.Normalization.MaxSegmentGap = value
}
if raw, ok := lookup("AUDITA_NORMALIZE_ELLIPSIS_GAP"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_NORMALIZE_ELLIPSIS_GAP: %w", err)
}
cfg.Normalization.EllipsisGap = value
}
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION: %w", err)
}
cfg.Normalization.MaxSegmentDuration = value
}
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS: %w", err)
}
cfg.Normalization.MaxSegmentTokens = value
}
if raw, ok := lookup("AUDITA_WORK_DIR"); ok {
cfg.WorkDir = raw
}
if raw, ok := lookup("AUDITA_WORK_DIR_RETENTION"); ok {
cfg.WorkDirRetention = WorkDirRetention(raw)
}
cfg.syncLegacyConcurrencyAliases()
if err := cfg.Validate(); err != nil {
return err
}
return nil
}
func parseInt(raw string) (int, error) {
value, err := strconv.Atoi(raw)
if err != nil {
return 0, fmt.Errorf("must be an integer")
}
return value, nil
}
func parseFloat(raw string) (float64, error) {
value, err := strconv.ParseFloat(raw, 64)
if err != nil {
return 0, fmt.Errorf("must be a number")
}
return value, nil
}

View File

@@ -0,0 +1,342 @@
package config
import (
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"gopkg.in/yaml.v3"
)
const SupportedFileConfigVersion = 1
var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
type FileConfig struct {
Version int `yaml:"version"`
Pipeline *FileConfigPipeline `yaml:"pipeline,omitempty"`
Output *FileConfigOutput `yaml:"output,omitempty"`
LLM *FileConfigLLM `yaml:"llm,omitempty"`
Concurrency *FileConfigConcurrency `yaml:"concurrency,omitempty"`
Chunking *FileConfigChunking `yaml:"chunking,omitempty"`
Normalization *FileConfigNormalization `yaml:"normalization,omitempty"`
Thresholds *FileConfigThresholds `yaml:"thresholds,omitempty"`
Context *FileConfigContext `yaml:"context,omitempty"`
Diagnostics *FileConfigDiagnostics `yaml:"diagnostics,omitempty"`
}
type FileConfigPipeline struct {
Modules []string `yaml:"modules,omitempty"`
}
type FileConfigOutput struct {
Schema *string `yaml:"schema,omitempty"`
}
type FileConfigLLM struct {
Proposal *FileConfigLLMTarget `yaml:"proposal,omitempty"`
Validation *FileConfigLLMTarget `yaml:"validation,omitempty"`
}
type FileConfigLLMTarget struct {
BaseURL *string `yaml:"base_url,omitempty"`
Model *string `yaml:"model,omitempty"`
APIKeyEnv *string `yaml:"api_key_env,omitempty"`
Timeout *fileConfigDurationOrInt `yaml:"timeout,omitempty"`
MaxRetries *int `yaml:"max_retries,omitempty"`
}
type FileConfigConcurrency struct {
TotalLLM *int `yaml:"total_llm,omitempty"`
ProposalLLM *int `yaml:"proposal_llm,omitempty"`
ValidationLLM *int `yaml:"validation_llm,omitempty"`
}
type FileConfigChunking struct {
TargetSections *int `yaml:"target_sections,omitempty"`
MaxSectionTokens *int `yaml:"max_section_tokens,omitempty"`
MinSectionTokens *int `yaml:"min_section_tokens,omitempty"`
}
type FileConfigNormalization struct {
MaxSegmentGap *fileConfigDurationOrFloat `yaml:"max_segment_gap,omitempty"`
EllipsisGap *fileConfigDurationOrFloat `yaml:"ellipsis_gap,omitempty"`
MaxSegmentDuration *fileConfigDurationOrFloat `yaml:"max_segment_duration,omitempty"`
MaxSegmentTokens *int `yaml:"max_segment_tokens,omitempty"`
}
type FileConfigThresholds struct {
Glossary *float64 `yaml:"glossary,omitempty"`
Homophones *float64 `yaml:"homophones,omitempty"`
SpokenWord *float64 `yaml:"spoken_word,omitempty"`
Grammar *float64 `yaml:"grammar,omitempty"`
}
type FileConfigContext struct {
Description *string `yaml:"description,omitempty"`
}
type FileConfigDiagnostics struct {
WorkDir *string `yaml:"work_dir,omitempty"`
Retention *string `yaml:"retention,omitempty"`
}
type fileConfigDurationOrInt struct {
seconds int
}
func (v *fileConfigDurationOrInt) UnmarshalYAML(node *yaml.Node) error {
switch node.Kind {
case yaml.ScalarNode:
if node.Tag == "!!int" {
var n int
if err := node.Decode(&n); err != nil {
return fmt.Errorf("must be an integer seconds value or duration string")
}
v.seconds = n
return nil
}
var s string
if err := node.Decode(&s); err != nil {
return fmt.Errorf("must be an integer seconds value or duration string")
}
d, err := time.ParseDuration(strings.TrimSpace(s))
if err != nil {
return fmt.Errorf("invalid duration %q", s)
}
if d <= 0 {
v.seconds = int(d / time.Second)
return nil
}
if d%time.Second != 0 {
return fmt.Errorf("duration %q must resolve to whole seconds", s)
}
v.seconds = int(d / time.Second)
return nil
default:
return fmt.Errorf("must be an integer seconds value or duration string")
}
}
func (v fileConfigDurationOrInt) Seconds() int { return v.seconds }
type fileConfigDurationOrFloat struct {
seconds float64
}
func (v *fileConfigDurationOrFloat) UnmarshalYAML(node *yaml.Node) error {
switch node.Kind {
case yaml.ScalarNode:
if node.Tag == "!!int" || node.Tag == "!!float" {
var f float64
if err := node.Decode(&f); err != nil {
return fmt.Errorf("must be a numeric seconds value or duration string")
}
v.seconds = f
return nil
}
var s string
if err := node.Decode(&s); err != nil {
return fmt.Errorf("must be a numeric seconds value or duration string")
}
d, err := time.ParseDuration(strings.TrimSpace(s))
if err != nil {
return fmt.Errorf("invalid duration %q", s)
}
v.seconds = d.Seconds()
return nil
default:
return fmt.Errorf("must be a numeric seconds value or duration string")
}
}
func (v fileConfigDurationOrFloat) Seconds() float64 { return v.seconds }
func LoadFileConfig(path string) (FileConfig, error) {
b, err := os.ReadFile(path)
if err != nil {
return FileConfig{}, fmt.Errorf("read config file %q: %w", path, err)
}
cfg, err := ParseFileConfigYAML(b)
if err != nil {
return FileConfig{}, fmt.Errorf("parse config file %q: %w", path, err)
}
return cfg, nil
}
func ParseFileConfigYAML(data []byte) (FileConfig, error) {
var fileCfg FileConfig
dec := yaml.NewDecoder(strings.NewReader(string(data)))
dec.KnownFields(true)
if err := dec.Decode(&fileCfg); err != nil {
return FileConfig{}, fmt.Errorf("decode yaml: %w", err)
}
if fileCfg.Version == 0 {
return FileConfig{}, fmt.Errorf("config version is required")
}
if fileCfg.Version != SupportedFileConfigVersion {
return FileConfig{}, fmt.Errorf("unsupported config version %d", fileCfg.Version)
}
return fileCfg, nil
}
func (c *Config) ApplyFileConfig(fileCfg FileConfig) error {
return c.applyFileConfigWithLookup(fileCfg, os.LookupEnv)
}
func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
if c == nil {
return fmt.Errorf("config must not be nil")
}
if fileCfg.Version != SupportedFileConfigVersion {
return fmt.Errorf("unsupported config version %d", fileCfg.Version)
}
if fileCfg.Pipeline != nil && len(fileCfg.Pipeline.Modules) > 0 {
c.Modules = append([]string(nil), fileCfg.Pipeline.Modules...)
}
if fileCfg.Output != nil && fileCfg.Output.Schema != nil {
c.OutputSchema = strings.TrimSpace(*fileCfg.Output.Schema)
}
if fileCfg.LLM != nil {
if fileCfg.LLM.Proposal != nil {
if fileCfg.LLM.Proposal.BaseURL != nil {
c.PrimaryLLM.BaseURL = *fileCfg.LLM.Proposal.BaseURL
}
if fileCfg.LLM.Proposal.Model != nil {
c.PrimaryLLM.Model = *fileCfg.LLM.Proposal.Model
}
if fileCfg.LLM.Proposal.Timeout != nil {
c.PrimaryLLM.TimeoutSeconds = fileCfg.LLM.Proposal.Timeout.Seconds()
}
if fileCfg.LLM.Proposal.MaxRetries != nil {
c.PrimaryLLM.MaxRetries = *fileCfg.LLM.Proposal.MaxRetries
}
if fileCfg.LLM.Proposal.APIKeyEnv != nil {
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Proposal.APIKeyEnv, lookup)
if err != nil {
return fmt.Errorf("llm.proposal.api_key_env: %w", err)
}
c.PrimaryLLM.APIKey = apiKey
}
}
if fileCfg.LLM.Validation != nil {
if fileCfg.LLM.Validation.BaseURL != nil {
c.ValidationLLM.BaseURL = *fileCfg.LLM.Validation.BaseURL
}
if fileCfg.LLM.Validation.Model != nil {
c.ValidationLLM.Model = *fileCfg.LLM.Validation.Model
}
if fileCfg.LLM.Validation.Timeout != nil {
v := fileCfg.LLM.Validation.Timeout.Seconds()
c.ValidationLLM.TimeoutSeconds = &v
}
if fileCfg.LLM.Validation.MaxRetries != nil {
v := *fileCfg.LLM.Validation.MaxRetries
c.ValidationLLM.MaxRetries = &v
}
if fileCfg.LLM.Validation.APIKeyEnv != nil {
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Validation.APIKeyEnv, lookup)
if err != nil {
return fmt.Errorf("llm.validation.api_key_env: %w", err)
}
c.ValidationLLM.APIKey = apiKey
}
}
}
if fileCfg.Concurrency != nil {
if fileCfg.Concurrency.TotalLLM != nil {
c.TotalLLMConcurrency = *fileCfg.Concurrency.TotalLLM
}
if fileCfg.Concurrency.ProposalLLM != nil {
c.ProposalLLMConcurrency = *fileCfg.Concurrency.ProposalLLM
}
if fileCfg.Concurrency.ValidationLLM != nil {
v := *fileCfg.Concurrency.ValidationLLM
c.ValidationLLMConcurrency = &v
}
}
if fileCfg.Chunking != nil {
if fileCfg.Chunking.TargetSections != nil {
v := *fileCfg.Chunking.TargetSections
c.TargetSections = &v
}
if fileCfg.Chunking.MaxSectionTokens != nil {
c.MaxSectionTokens = *fileCfg.Chunking.MaxSectionTokens
}
if fileCfg.Chunking.MinSectionTokens != nil {
c.MinSectionTokens = *fileCfg.Chunking.MinSectionTokens
}
}
if fileCfg.Normalization != nil {
if fileCfg.Normalization.MaxSegmentGap != nil {
c.Normalization.MaxSegmentGap = fileCfg.Normalization.MaxSegmentGap.Seconds()
}
if fileCfg.Normalization.EllipsisGap != nil {
c.Normalization.EllipsisGap = fileCfg.Normalization.EllipsisGap.Seconds()
}
if fileCfg.Normalization.MaxSegmentDuration != nil {
c.Normalization.MaxSegmentDuration = fileCfg.Normalization.MaxSegmentDuration.Seconds()
}
if fileCfg.Normalization.MaxSegmentTokens != nil {
c.Normalization.MaxSegmentTokens = *fileCfg.Normalization.MaxSegmentTokens
}
}
if fileCfg.Thresholds != nil {
if fileCfg.Thresholds.Glossary != nil {
c.Thresholds.Glossary = *fileCfg.Thresholds.Glossary
}
if fileCfg.Thresholds.Homophones != nil {
c.Thresholds.Homophones = *fileCfg.Thresholds.Homophones
}
if fileCfg.Thresholds.SpokenWord != nil {
c.Thresholds.SpokenWord = *fileCfg.Thresholds.SpokenWord
}
if fileCfg.Thresholds.Grammar != nil {
c.Thresholds.Grammar = *fileCfg.Thresholds.Grammar
}
}
if fileCfg.Context != nil && fileCfg.Context.Description != nil {
c.TranscriptDescription = strings.TrimSpace(*fileCfg.Context.Description)
}
if fileCfg.Diagnostics != nil {
if fileCfg.Diagnostics.WorkDir != nil {
c.WorkDir = *fileCfg.Diagnostics.WorkDir
}
if fileCfg.Diagnostics.Retention != nil {
c.WorkDirRetention = WorkDirRetention(*fileCfg.Diagnostics.Retention)
}
}
c.syncLegacyConcurrencyAliases()
if err := c.Validate(); err != nil {
return err
}
return nil
}
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
name := strings.TrimSpace(envName)
if name == "" {
return "", fmt.Errorf("must not be empty")
}
if !envVarNamePattern.MatchString(name) {
return "", fmt.Errorf("must be an environment variable name")
}
if strings.Contains(name, string(filepath.Separator)) {
return "", fmt.Errorf("must be an environment variable name")
}
v, _ := lookup(name)
return v, nil
}

View File

@@ -0,0 +1,290 @@
package config
import (
"os"
"strings"
"testing"
)
func TestParseFileConfigYAMLValid(t *testing.T) {
raw := `
version: 1
pipeline:
modules: [glossary, homophones, grammar]
output:
schema: audita-v1
llm:
proposal:
base_url: https://example.test/v1
model: provider/model-a
api_key_env: AUDITA_PROPOSAL_KEY
timeout: 2m
max_retries: 4
validation:
base_url: https://example.test/validation
model: provider/model-b
api_key_env: AUDITA_VALIDATION_KEY
timeout: 45
max_retries: 3
concurrency:
total_llm: 8
proposal_llm: 4
validation_llm: 2
chunking:
target_sections: 6
max_section_tokens: 9000
min_section_tokens: 3000
normalization:
max_segment_gap: 1.5s
ellipsis_gap: 2
max_segment_duration: 45s
max_segment_tokens: 1500
thresholds:
glossary: 0.9
homophones: 0.7
spoken_word: 0.8
grammar: 0.75
context:
description: " crowd scene with many proper nouns "
diagnostics:
work_dir: /tmp/audita-config
retention: always
`
cfg, err := ParseFileConfigYAML([]byte(raw))
if err != nil {
t.Fatalf("ParseFileConfigYAML error: %v", err)
}
if cfg.Version != 1 {
t.Fatalf("expected version 1, got %d", cfg.Version)
}
if cfg.Pipeline == nil || len(cfg.Pipeline.Modules) != 3 {
t.Fatalf("unexpected pipeline modules: %#v", cfg.Pipeline)
}
if cfg.Output == nil || cfg.Output.Schema == nil || *cfg.Output.Schema != "audita-v1" {
t.Fatalf("expected output schema audita-v1, got %#v", cfg.Output)
}
if cfg.LLM == nil || cfg.LLM.Proposal == nil || cfg.LLM.Validation == nil {
t.Fatalf("expected llm proposal+validation blocks")
}
if cfg.LLM.Proposal.Timeout == nil || cfg.LLM.Proposal.Timeout.Seconds() != 120 {
t.Fatalf("expected proposal timeout 120s, got %#v", cfg.LLM.Proposal.Timeout)
}
if cfg.LLM.Validation.Timeout == nil || cfg.LLM.Validation.Timeout.Seconds() != 45 {
t.Fatalf("expected validation timeout 45s, got %#v", cfg.LLM.Validation.Timeout)
}
if cfg.Normalization == nil || cfg.Normalization.MaxSegmentGap == nil || cfg.Normalization.MaxSegmentGap.Seconds() != 1.5 {
t.Fatalf("expected parsed duration for normalization max_segment_gap")
}
}
func TestParseFileConfigYAMLRejectsUnknownField(t *testing.T) {
raw := `
version: 1
pipeline:
modules: [grammar]
output:
unknown: v1
`
_, err := ParseFileConfigYAML([]byte(raw))
if err == nil {
t.Fatalf("expected unknown field error")
}
if !strings.Contains(err.Error(), "field unknown not found") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestParseFileConfigYAMLRejectsMissingVersion(t *testing.T) {
raw := `pipeline: {modules: [grammar]}`
_, err := ParseFileConfigYAML([]byte(raw))
if err == nil {
t.Fatalf("expected missing version error")
}
if !strings.Contains(err.Error(), "config version is required") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestParseFileConfigYAMLRejectsUnsupportedVersion(t *testing.T) {
raw := `version: 2`
_, err := ParseFileConfigYAML([]byte(raw))
if err == nil {
t.Fatalf("expected unsupported version error")
}
if !strings.Contains(err.Error(), "unsupported config version 2") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestApplyFileConfigParsesAndMergesFields(t *testing.T) {
raw := `
version: 1
pipeline:
modules: [spoken_word, grammar]
output:
schema: audita-v1
llm:
proposal:
model: provider/new-proposal
api_key_env: PROPOSAL_KEY_NAME
timeout: 90s
max_retries: 5
validation:
model: provider/new-validation
api_key_env: VALIDATION_KEY_NAME
timeout: 150
max_retries: 6
concurrency:
total_llm: 7
proposal_llm: 3
validation_llm: 2
chunking:
target_sections: 9
thresholds:
glossary: 0.91
homophones: 0.61
spoken_word: 0.71
grammar: 0.81
diagnostics:
retention: never
`
fileCfg, err := ParseFileConfigYAML([]byte(raw))
if err != nil {
t.Fatalf("ParseFileConfigYAML error: %v", err)
}
cfg := Default()
lookup := func(name string) (string, bool) {
switch name {
case "PROPOSAL_KEY_NAME":
return "proposal-secret", true
case "VALIDATION_KEY_NAME":
return "validation-secret", true
default:
return "", false
}
}
if err := cfg.applyFileConfigWithLookup(fileCfg, lookup); err != nil {
t.Fatalf("applyFileConfigWithLookup error: %v", err)
}
if strings.Join(cfg.Modules, ",") != "spoken_word,grammar" {
t.Fatalf("unexpected modules: %#v", cfg.Modules)
}
if cfg.OutputSchema != "audita-v1" {
t.Fatalf("unexpected output schema: %q", cfg.OutputSchema)
}
if cfg.PrimaryLLM.Model != "provider/new-proposal" {
t.Fatalf("unexpected proposal model: %q", cfg.PrimaryLLM.Model)
}
if cfg.PrimaryLLM.APIKey != "proposal-secret" {
t.Fatalf("expected proposal key from api_key_env lookup, got %q", cfg.PrimaryLLM.APIKey)
}
if cfg.PrimaryLLM.TimeoutSeconds != 90 {
t.Fatalf("unexpected proposal timeout: %d", cfg.PrimaryLLM.TimeoutSeconds)
}
if cfg.ValidationLLM.Model != "provider/new-validation" {
t.Fatalf("unexpected validation model: %q", cfg.ValidationLLM.Model)
}
if cfg.ValidationLLM.APIKey != "validation-secret" {
t.Fatalf("expected validation key from api_key_env lookup, got %q", cfg.ValidationLLM.APIKey)
}
if cfg.ValidationLLM.TimeoutSeconds == nil || *cfg.ValidationLLM.TimeoutSeconds != 150 {
t.Fatalf("unexpected validation timeout: %#v", cfg.ValidationLLM.TimeoutSeconds)
}
if cfg.TotalLLMConcurrency != 7 || cfg.ProposalLLMConcurrency != 3 {
t.Fatalf("unexpected llm concurrency values: total=%d proposal=%d", cfg.TotalLLMConcurrency, cfg.ProposalLLMConcurrency)
}
if cfg.ValidationLLMConcurrency == nil || *cfg.ValidationLLMConcurrency != 2 {
t.Fatalf("unexpected validation llm concurrency: %#v", cfg.ValidationLLMConcurrency)
}
if cfg.TargetSections == nil || *cfg.TargetSections != 9 {
t.Fatalf("unexpected target sections: %#v", cfg.TargetSections)
}
if cfg.WorkDirRetention != WorkDirRetentionNever {
t.Fatalf("unexpected retention: %q", cfg.WorkDirRetention)
}
if cfg.PrimaryLLM.Concurrency != 7 {
t.Fatalf("expected legacy alias to sync, got %d", cfg.PrimaryLLM.Concurrency)
}
if cfg.ValidationLLM.Concurrency == nil || *cfg.ValidationLLM.Concurrency != 2 {
t.Fatalf("expected validation alias to sync, got %#v", cfg.ValidationLLM.Concurrency)
}
}
func TestApplyFileConfigContextDescriptionTrim(t *testing.T) {
raw := `
version: 1
context:
description: " scene context "
`
fileCfg, err := ParseFileConfigYAML([]byte(raw))
if err != nil {
t.Fatalf("ParseFileConfigYAML error: %v", err)
}
cfg := Default()
if err := cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{})); err != nil {
t.Fatalf("applyFileConfigWithLookup error: %v", err)
}
if cfg.TranscriptDescription != "scene context" {
t.Fatalf("unexpected transcript description: %q", cfg.TranscriptDescription)
}
}
func TestApplyFileConfigRejectsInvalidAPIKeyEnvName(t *testing.T) {
raw := `
version: 1
llm:
proposal:
api_key_env: "not a var name"
`
fileCfg, err := ParseFileConfigYAML([]byte(raw))
if err != nil {
t.Fatalf("ParseFileConfigYAML error: %v", err)
}
cfg := Default()
err = cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{}))
if err == nil {
t.Fatalf("expected api_key_env validation error")
}
if !strings.Contains(err.Error(), "environment variable name") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestParseFileConfigDurationParsingErrors(t *testing.T) {
raw := `
version: 1
llm:
proposal:
timeout: "1.5s"
`
_, err := ParseFileConfigYAML([]byte(raw))
if err == nil {
t.Fatalf("expected duration parse error")
}
if !strings.Contains(err.Error(), "whole seconds") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestLoadFileConfigReadsFromPath(t *testing.T) {
p := writeTempFileConfig(t, "version: 1\n")
cfg, err := LoadFileConfig(p)
if err != nil {
t.Fatalf("LoadFileConfig error: %v", err)
}
if cfg.Version != 1 {
t.Fatalf("expected version 1, got %d", cfg.Version)
}
}
func writeTempFileConfig(t *testing.T, contents string) string {
t.Helper()
dir := t.TempDir()
path := dir + "/config.yaml"
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
t.Fatalf("write config file: %v", err)
}
return path
}

View File

@@ -0,0 +1,159 @@
package config
import (
"fmt"
"strings"
)
type CLIOverrides struct {
ModulesCSV *string
OutputSchema *string
PrimaryLLMAPIKey *string
ValidationLLMAPIKey *string
PrimaryModel *string
ValidationModel *string
PrimaryBaseURL *string
ValidationBaseURL *string
PrimaryLLMTimeoutSeconds *int
TotalLLMConcurrency *int
ProposalLLMConcurrency *int
PrimaryLLMConcurrency *int
ValidationLLMTimeoutSeconds *int
MaxRetries *int
ValidationMaxRetries *int
ValidationLLMConcurrency *int
ValidationMaxPromptTokens *int
MaxSectionTokens *int
MinSectionTokens *int
TargetSections *int
GlossaryConfidenceThreshold *float64
GrammarConfidenceThreshold *float64
HomophonesConfidenceThreshold *float64
SpokenWordConfidenceThreshold *float64
NormalizeMaxSegmentGap *float64
NormalizeEllipsisGap *float64
NormalizeMaxSegmentDuration *float64
NormalizeMaxSegmentTokens *int
TranscriptDescription *string
WorkDir *string
WorkDirRetention *string
}
func (c *Config) ApplyCLIOverrides(overrides CLIOverrides) error {
if overrides.ModulesCSV != nil {
modules, err := ParseModulesCSV(*overrides.ModulesCSV)
if err != nil {
return fmt.Errorf("--modules: %w", err)
}
c.Modules = modules
}
if overrides.OutputSchema != nil {
c.OutputSchema = strings.TrimSpace(*overrides.OutputSchema)
}
if overrides.PrimaryLLMAPIKey != nil {
c.PrimaryLLM.APIKey = *overrides.PrimaryLLMAPIKey
}
if overrides.ValidationLLMAPIKey != nil {
c.ValidationLLM.APIKey = *overrides.ValidationLLMAPIKey
}
if overrides.PrimaryModel != nil {
c.PrimaryLLM.Model = *overrides.PrimaryModel
}
if overrides.ValidationModel != nil {
c.ValidationLLM.Model = *overrides.ValidationModel
}
if overrides.PrimaryBaseURL != nil {
c.PrimaryLLM.BaseURL = *overrides.PrimaryBaseURL
}
if overrides.ValidationBaseURL != nil {
c.ValidationLLM.BaseURL = *overrides.ValidationBaseURL
}
if overrides.PrimaryLLMTimeoutSeconds != nil {
c.PrimaryLLM.TimeoutSeconds = *overrides.PrimaryLLMTimeoutSeconds
}
totalConcurrencySet := false
if overrides.TotalLLMConcurrency != nil {
c.TotalLLMConcurrency = *overrides.TotalLLMConcurrency
totalConcurrencySet = true
}
// Backward-compatible alias: --llm-concurrency maps to total concurrency
// only when --total-llm-concurrency is not set in the same CLI invocation.
if overrides.PrimaryLLMConcurrency != nil && !totalConcurrencySet {
c.TotalLLMConcurrency = *overrides.PrimaryLLMConcurrency
totalConcurrencySet = true
}
proposalConcurrencySet := false
if overrides.ProposalLLMConcurrency != nil {
c.ProposalLLMConcurrency = *overrides.ProposalLLMConcurrency
proposalConcurrencySet = true
}
if totalConcurrencySet && !proposalConcurrencySet {
c.ProposalLLMConcurrency = c.TotalLLMConcurrency
}
if overrides.ValidationLLMTimeoutSeconds != nil {
value := *overrides.ValidationLLMTimeoutSeconds
c.ValidationLLM.TimeoutSeconds = &value
}
if overrides.MaxRetries != nil {
c.PrimaryLLM.MaxRetries = *overrides.MaxRetries
}
if overrides.ValidationMaxRetries != nil {
value := *overrides.ValidationMaxRetries
c.ValidationLLM.MaxRetries = &value
}
if overrides.ValidationLLMConcurrency != nil {
value := *overrides.ValidationLLMConcurrency
c.ValidationLLMConcurrency = &value
}
if overrides.ValidationMaxPromptTokens != nil {
c.ValidationMaxPromptTokens = *overrides.ValidationMaxPromptTokens
}
if overrides.MaxSectionTokens != nil {
c.MaxSectionTokens = *overrides.MaxSectionTokens
}
if overrides.MinSectionTokens != nil {
c.MinSectionTokens = *overrides.MinSectionTokens
}
if overrides.TargetSections != nil {
value := *overrides.TargetSections
c.TargetSections = &value
}
if overrides.GlossaryConfidenceThreshold != nil {
c.Thresholds.Glossary = *overrides.GlossaryConfidenceThreshold
}
if overrides.GrammarConfidenceThreshold != nil {
c.Thresholds.Grammar = *overrides.GrammarConfidenceThreshold
}
if overrides.HomophonesConfidenceThreshold != nil {
c.Thresholds.Homophones = *overrides.HomophonesConfidenceThreshold
}
if overrides.SpokenWordConfidenceThreshold != nil {
c.Thresholds.SpokenWord = *overrides.SpokenWordConfidenceThreshold
}
if overrides.NormalizeMaxSegmentGap != nil {
c.Normalization.MaxSegmentGap = *overrides.NormalizeMaxSegmentGap
}
if overrides.NormalizeEllipsisGap != nil {
c.Normalization.EllipsisGap = *overrides.NormalizeEllipsisGap
}
if overrides.NormalizeMaxSegmentDuration != nil {
c.Normalization.MaxSegmentDuration = *overrides.NormalizeMaxSegmentDuration
}
if overrides.NormalizeMaxSegmentTokens != nil {
c.Normalization.MaxSegmentTokens = *overrides.NormalizeMaxSegmentTokens
}
if overrides.TranscriptDescription != nil {
c.TranscriptDescription = strings.TrimSpace(*overrides.TranscriptDescription)
}
if overrides.WorkDir != nil {
c.WorkDir = *overrides.WorkDir
}
if overrides.WorkDirRetention != nil {
c.WorkDirRetention = WorkDirRetention(*overrides.WorkDirRetention)
}
c.syncLegacyConcurrencyAliases()
return c.Validate()
}

View File

@@ -0,0 +1,17 @@
package config
const redactedSecret = "[REDACTED]"
func (c Config) Redacted() Config {
redacted := c
redacted.PrimaryLLM.APIKey = redactSecret(redacted.PrimaryLLM.APIKey)
redacted.ValidationLLM.APIKey = redactSecret(redacted.ValidationLLM.APIKey)
return redacted
}
func redactSecret(value string) string {
if value == "" {
return ""
}
return redactedSecret
}

View File

@@ -0,0 +1,136 @@
package config
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
)
func (c Config) Validate() error {
var issues []string
if len(c.Modules) == 0 {
issues = append(issues, "modules must not be empty")
}
for _, module := range c.Modules {
moduleKey := strings.TrimSpace(module)
if moduleKey == "" {
issues = append(issues, "modules must not contain empty values")
break
}
if !modulecatalog.IsSupported(moduleKey) {
issues = append(issues, fmt.Sprintf("unsupported module key %q", moduleKey))
}
}
if strings.TrimSpace(c.OutputSchema) == "" {
issues = append(issues, "output schema must not be empty")
} else if !outputschema.IsSupported(c.OutputSchema) {
issues = append(issues, fmt.Sprintf("unsupported output schema %q", c.OutputSchema))
}
if c.PrimaryLLM.TimeoutSeconds <= 0 {
issues = append(issues, "primary llm timeout seconds must be greater than zero")
}
if c.PrimaryLLM.MaxRetries < 0 {
issues = append(issues, "max retries must be zero or greater")
}
if c.TotalLLMConcurrency <= 0 {
issues = append(issues, "total llm concurrency must be greater than zero")
}
if c.ProposalLLMConcurrency <= 0 {
issues = append(issues, "proposal llm concurrency must be greater than zero")
}
if c.ProposalLLMConcurrency > c.TotalLLMConcurrency {
issues = append(issues, "proposal llm concurrency must be less than or equal to total llm concurrency")
}
if c.ValidationLLM.TimeoutSeconds != nil && *c.ValidationLLM.TimeoutSeconds <= 0 {
issues = append(issues, "validation llm timeout seconds must be greater than zero")
}
if c.ValidationLLM.MaxRetries != nil && *c.ValidationLLM.MaxRetries < 0 {
issues = append(issues, "validation max retries must be zero or greater")
}
if c.ValidationLLMConcurrency != nil && *c.ValidationLLMConcurrency <= 0 {
issues = append(issues, "validation llm concurrency must be greater than zero")
}
if c.ValidationLLMConcurrency != nil && *c.ValidationLLMConcurrency > c.TotalLLMConcurrency {
issues = append(issues, "validation llm concurrency must be less than or equal to total llm concurrency")
}
if c.ValidationMaxPromptTokens <= 0 {
issues = append(issues, "validation max prompt tokens must be greater than zero")
}
if c.MaxSectionTokens <= 0 {
issues = append(issues, "max section tokens must be greater than zero")
}
if c.MinSectionTokens <= 0 {
issues = append(issues, "min section tokens must be greater than zero")
}
if c.MinSectionTokens > c.MaxSectionTokens {
issues = append(issues, "min section tokens must be less than or equal to max section tokens")
}
if c.TargetSections != nil && *c.TargetSections <= 0 {
issues = append(issues, "target sections must be greater than zero when set")
}
if err := validateConfidence("glossary", c.Thresholds.Glossary); err != nil {
issues = append(issues, err.Error())
}
if err := validateConfidence("grammar", c.Thresholds.Grammar); err != nil {
issues = append(issues, err.Error())
}
if err := validateConfidence("homophones", c.Thresholds.Homophones); err != nil {
issues = append(issues, err.Error())
}
if err := validateConfidence("spoken-word", c.Thresholds.SpokenWord); err != nil {
issues = append(issues, err.Error())
}
if c.Normalization.MaxSegmentGap < 0 {
issues = append(issues, "normalize max segment gap must be zero or greater")
}
if c.Normalization.EllipsisGap < 0 {
issues = append(issues, "normalize ellipsis gap must be zero or greater")
}
if c.Normalization.MaxSegmentDuration <= 0 {
issues = append(issues, "normalize max segment duration must be greater than zero")
}
if c.Normalization.MaxSegmentTokens <= 0 {
issues = append(issues, "normalize max segment tokens must be greater than zero")
}
if len(strings.TrimSpace(c.TranscriptDescription)) > DefaultTranscriptDescriptionMaxChars {
issues = append(issues, fmt.Sprintf("transcript description must be %d characters or fewer", DefaultTranscriptDescriptionMaxChars))
}
if strings.TrimSpace(c.WorkDir) == "" {
issues = append(issues, "work dir must not be empty")
}
if err := validateRetention(c.WorkDirRetention); err != nil {
issues = append(issues, err.Error())
}
if len(issues) > 0 {
return fmt.Errorf("invalid config: %s", strings.Join(issues, "; "))
}
return nil
}
func validateConfidence(name string, threshold float64) error {
if threshold < 0.0 || threshold > 1.0 {
return fmt.Errorf("%s confidence threshold must be between 0.0 and 1.0", name)
}
return nil
}
func validateRetention(retention WorkDirRetention) error {
switch retention {
case WorkDirRetentionAuto, WorkDirRetentionAlways, WorkDirRetentionNever:
return nil
default:
return fmt.Errorf("work dir retention must be one of: auto, always, never")
}
}

View File

@@ -0,0 +1,42 @@
package diagnostics
import (
"path/filepath"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
)
const (
ArtifactSourceTranscript = "source-transcript.json"
ArtifactParsedSourceTranscript = "source-transcript-parsed.json"
ArtifactNormalizedTranscript = "normalized-transcript.json"
ArtifactNormalizationSummary = "normalization-summary.json"
ArtifactChunkingSummary = "chunking-summary.json"
ArtifactUtilizationSummary = "utilization-diagnostics.json"
ArtifactCorrectionLedger = "correction-ledger.json"
ArtifactInvocationMetadata = "invocation.json"
ArtifactEffectiveConfig = "effective-config.json"
ArtifactReport = "report.json"
ArtifactErrorLog = "error.log"
)
func BuildDiagnosticsMetadata(runDirectoryPath string, runSucceeded bool) reporting.DiagnosticsMetadata {
metadata := reporting.DiagnosticsMetadata{
DirectoryPath: runDirectoryPath,
SourceTranscriptPath: filepath.Join(runDirectoryPath, ArtifactSourceTranscript),
ParsedSourceTranscriptPath: filepath.Join(runDirectoryPath, ArtifactParsedSourceTranscript),
NormalizedTranscriptPath: filepath.Join(runDirectoryPath, ArtifactNormalizedTranscript),
NormalizationSummaryPath: filepath.Join(runDirectoryPath, ArtifactNormalizationSummary),
ChunkingSummaryPath: filepath.Join(runDirectoryPath, ArtifactChunkingSummary),
UtilizationSummaryPath: filepath.Join(runDirectoryPath, ArtifactUtilizationSummary),
CorrectionLedgerPath: filepath.Join(runDirectoryPath, ArtifactCorrectionLedger),
InvocationMetadataPath: filepath.Join(runDirectoryPath, ArtifactInvocationMetadata),
RedactedEffectiveConfigPath: filepath.Join(runDirectoryPath, ArtifactEffectiveConfig),
}
if !runSucceeded {
metadata.ErrorLogPath = filepath.Join(runDirectoryPath, ArtifactErrorLog)
}
return metadata
}

View File

@@ -0,0 +1,55 @@
package diagnostics
import (
"path/filepath"
"testing"
)
func TestBuildDiagnosticsMetadataSuccessPathsMatchArtifactConstants(t *testing.T) {
runPath := filepath.Join("tmp", "run-123")
metadata := BuildDiagnosticsMetadata(runPath, true)
if metadata.DirectoryPath != runPath {
t.Fatalf("unexpected diagnostics directory path: got=%q want=%q", metadata.DirectoryPath, runPath)
}
if metadata.SourceTranscriptPath != filepath.Join(runPath, ArtifactSourceTranscript) {
t.Fatalf("unexpected source transcript path: %q", metadata.SourceTranscriptPath)
}
if metadata.ParsedSourceTranscriptPath != filepath.Join(runPath, ArtifactParsedSourceTranscript) {
t.Fatalf("unexpected parsed source transcript path: %q", metadata.ParsedSourceTranscriptPath)
}
if metadata.NormalizedTranscriptPath != filepath.Join(runPath, ArtifactNormalizedTranscript) {
t.Fatalf("unexpected normalized transcript path: %q", metadata.NormalizedTranscriptPath)
}
if metadata.NormalizationSummaryPath != filepath.Join(runPath, ArtifactNormalizationSummary) {
t.Fatalf("unexpected normalization summary path: %q", metadata.NormalizationSummaryPath)
}
if metadata.ChunkingSummaryPath != filepath.Join(runPath, ArtifactChunkingSummary) {
t.Fatalf("unexpected chunking summary path: %q", metadata.ChunkingSummaryPath)
}
if metadata.UtilizationSummaryPath != filepath.Join(runPath, ArtifactUtilizationSummary) {
t.Fatalf("unexpected utilization summary path: %q", metadata.UtilizationSummaryPath)
}
if metadata.CorrectionLedgerPath != filepath.Join(runPath, ArtifactCorrectionLedger) {
t.Fatalf("unexpected correction ledger path: %q", metadata.CorrectionLedgerPath)
}
if metadata.InvocationMetadataPath != filepath.Join(runPath, ArtifactInvocationMetadata) {
t.Fatalf("unexpected invocation metadata path: %q", metadata.InvocationMetadataPath)
}
if metadata.RedactedEffectiveConfigPath != filepath.Join(runPath, ArtifactEffectiveConfig) {
t.Fatalf("unexpected redacted effective config path: %q", metadata.RedactedEffectiveConfigPath)
}
if metadata.ErrorLogPath != "" {
t.Fatalf("did not expect error log path on success: %q", metadata.ErrorLogPath)
}
}
func TestBuildDiagnosticsMetadataFailureIncludesErrorLogPath(t *testing.T) {
runPath := filepath.Join("tmp", "run-123")
metadata := BuildDiagnosticsMetadata(runPath, false)
want := filepath.Join(runPath, ArtifactErrorLog)
if metadata.ErrorLogPath != want {
t.Fatalf("unexpected error log path: got=%q want=%q", metadata.ErrorLogPath, want)
}
}

View File

@@ -0,0 +1,72 @@
package diagnostics
import (
"os"
"path/filepath"
"testing"
)
func TestShouldRetainRunDirectoryMatrix(t *testing.T) {
tests := []struct {
name string
input RetentionDecisionInput
want bool
}{
{name: "always success keeps", input: RetentionDecisionInput{RetentionMode: "always", RunSucceeded: true}, want: true},
{name: "always failure keeps", input: RetentionDecisionInput{RetentionMode: "always", RunSucceeded: false}, want: true},
{name: "never success keeps", input: RetentionDecisionInput{RetentionMode: "never", RunSucceeded: true}, want: true},
{name: "never failure keeps", input: RetentionDecisionInput{RetentionMode: "never", RunSucceeded: false}, want: true},
{name: "auto success no skips removes", input: RetentionDecisionInput{RetentionMode: "auto", RunSucceeded: true, HasSkippedCorrections: false}, want: false},
{name: "auto success skips keeps", input: RetentionDecisionInput{RetentionMode: "auto", RunSucceeded: true, HasSkippedCorrections: true}, want: true},
{name: "auto failure keeps", input: RetentionDecisionInput{RetentionMode: "auto", RunSucceeded: false}, want: true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := ShouldRetainRunDirectory(tc.input)
if got != tc.want {
t.Fatalf("unexpected retain decision: got=%v want=%v", got, tc.want)
}
})
}
}
func TestApplyRetentionRemovesWhenDecisionSaysRemove(t *testing.T) {
workDir := t.TempDir()
runPath := filepath.Join(workDir, "run-test")
if err := os.Mkdir(runPath, 0o755); err != nil {
t.Fatalf("mkdir run path: %v", err)
}
runDir := &RunDirectory{path: runPath, retention: "auto"}
err := runDir.ApplyRetention(RetentionDecisionInput{RunSucceeded: true, HasSkippedCorrections: false})
if err != nil {
t.Fatalf("ApplyRetention failed: %v", err)
}
if _, err := os.Stat(runPath); !os.IsNotExist(err) {
t.Fatalf("expected run directory removed, stat err=%v", err)
}
}
func TestApplyRetentionReturnsRemovalError(t *testing.T) {
parent := t.TempDir()
runPath := filepath.Join(parent, "run-test")
if err := os.Mkdir(runPath, 0o755); err != nil {
t.Fatalf("mkdir run path: %v", err)
}
// Make parent non-writable so removing child fails.
if err := os.Chmod(parent, 0o500); err != nil {
t.Fatalf("chmod parent: %v", err)
}
t.Cleanup(func() {
_ = os.Chmod(parent, 0o700)
})
runDir := &RunDirectory{path: runPath, retention: "auto"}
err := runDir.ApplyRetention(RetentionDecisionInput{RunSucceeded: true, HasSkippedCorrections: false})
if err == nil {
t.Fatalf("expected removal error, got nil")
}
}

View File

@@ -0,0 +1,242 @@
package diagnostics
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"time"
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
// RunDirectory represents a per-run diagnostics directory
type RunDirectory struct {
path string
retention string
createdAt time.Time
}
type RetentionDecisionInput struct {
RetentionMode string
RunSucceeded bool
HasSkippedCorrections bool
}
func ShouldRetainRunDirectory(input RetentionDecisionInput) bool {
// Failed runs are always retained.
if !input.RunSucceeded {
return true
}
switch input.RetentionMode {
case "always":
return true
case "never":
return true
case "auto":
return input.HasSkippedCorrections
default:
// Be conservative for unknown values.
return true
}
}
// InvocationMetadata captures non-secret invocation details for diagnostics.
type InvocationMetadata struct {
Operation string `json:"operation"`
TranscriptPath string `json:"transcript_path"`
GlossaryPath string `json:"glossary_path"`
OutputPath string `json:"output_path,omitempty"`
ReportJSONPath string `json:"report_json_path,omitempty"`
ConfigPath string `json:"config_path,omitempty"`
ConfigSource string `json:"config_source,omitempty"`
ConfigVersion *int `json:"config_version,omitempty"`
TranscriptDescription string `json:"transcript_description,omitempty"`
Modules []string `json:"modules"`
RunID string `json:"run_id"`
StartedAt time.Time `json:"started_at"`
}
// NewRunDirectory creates a new run directory under the configured work dir
func NewRunDirectory(workDir, retention string) (*RunDirectory, error) {
if workDir == "" {
workDir = ".audita-runs"
}
// Create work directory if it doesn't exist
if err := os.MkdirAll(workDir, 0o755); err != nil {
return nil, fmt.Errorf("failed to create work directory %q: %w", workDir, err)
}
// Create a unique run directory identifier.
runID := fmt.Sprintf("run-%d", time.Now().UTC().UnixNano())
runPath := filepath.Join(workDir, runID)
if err := os.Mkdir(runPath, 0o755); err != nil {
return nil, fmt.Errorf("failed to create run directory %q: %w", runPath, err)
}
return &RunDirectory{
path: runPath,
retention: retention,
createdAt: time.Now().UTC(),
}, nil
}
// Path returns the run directory path
func (r *RunDirectory) Path() string {
return r.path
}
func (r *RunDirectory) runID() string {
return filepath.Base(r.path)
}
// WriteInvocationMetadata writes invocation metadata for this run.
func (r *RunDirectory) WriteInvocationMetadata(metadata InvocationMetadata) error {
if metadata.RunID == "" {
metadata.RunID = r.runID()
}
if metadata.StartedAt.IsZero() {
metadata.StartedAt = r.createdAt
}
path := filepath.Join(r.path, ArtifactInvocationMetadata)
bytes, err := json.MarshalIndent(metadata, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal invocation metadata: %w", err)
}
bytes = append(bytes, '\n')
if err := os.WriteFile(path, bytes, 0o644); err != nil {
return fmt.Errorf("failed to write invocation metadata: %w", err)
}
return nil
}
// WriteEffectiveConfig writes redacted effective config metadata for this run.
func (r *RunDirectory) WriteEffectiveConfig(cfg config.Config) error {
path := filepath.Join(r.path, ArtifactEffectiveConfig)
redacted := cfg.Redacted()
bytes, err := json.MarshalIndent(redacted, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal effective config: %w", err)
}
bytes = append(bytes, '\n')
if err := os.WriteFile(path, bytes, 0o644); err != nil {
return fmt.Errorf("failed to write effective config: %w", err)
}
return nil
}
// WriteSourceTranscript writes the source transcript artifact
func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript, raw []byte) error {
// Write raw source for reference
sourcePath := filepath.Join(r.path, ArtifactSourceTranscript)
if err := os.WriteFile(sourcePath, raw, 0o644); err != nil {
return fmt.Errorf("failed to write source transcript: %w", err)
}
// Write parsed source for debugging
parsedPath := filepath.Join(r.path, ArtifactParsedSourceTranscript)
parsedBytes, err := json.MarshalIndent(transcript, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal parsed source transcript: %w", err)
}
parsedBytes = append(parsedBytes, '\n')
if err := os.WriteFile(parsedPath, parsedBytes, 0o644); err != nil {
return fmt.Errorf("failed to write parsed source transcript: %w", err)
}
return nil
}
// WriteNormalizedTranscript writes the normalized transcript artifact
func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript) error {
normalizedPath := filepath.Join(r.path, ArtifactNormalizedTranscript)
bytes, err := schema.TranscriptToJSON(transcript)
if err != nil {
return fmt.Errorf("failed to serialize normalized transcript: %w", err)
}
if err := os.WriteFile(normalizedPath, bytes, 0o644); err != nil {
return fmt.Errorf("failed to write normalized transcript: %w", err)
}
return nil
}
// WriteNormalizationSummary writes the normalization summary artifact
func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.NormalizationSummary) error {
summaryPath := filepath.Join(r.path, ArtifactNormalizationSummary)
bytes, err := json.MarshalIndent(summary, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal normalization summary: %w", err)
}
bytes = append(bytes, '\n')
if err := os.WriteFile(summaryPath, bytes, 0o644); err != nil {
return fmt.Errorf("failed to write normalization summary: %w", err)
}
return nil
}
// WriteReport writes the authoritative report artifact
func (r *RunDirectory) WriteReport(report reporting.ProcessReport) error {
reportPath := filepath.Join(r.path, ArtifactReport)
bytes, err := json.MarshalIndent(report, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal report: %w", err)
}
bytes = append(bytes, '\n')
if err := os.WriteFile(reportPath, bytes, 0o644); err != nil {
return fmt.Errorf("failed to write report: %w", err)
}
return nil
}
// WriteErrorLog writes an error log on failure
func (r *RunDirectory) WriteErrorLog(errorMessage string) error {
errorPath := filepath.Join(r.path, ArtifactErrorLog)
return os.WriteFile(errorPath, []byte(errorMessage+"\n"), 0o644)
}
// WriteChunkingSummary writes the chunking summary artifact
func (r *RunDirectory) WriteChunkingSummary(summary *chunking.DetailedSummary) error {
summaryPath := filepath.Join(r.path, ArtifactChunkingSummary)
bytes, err := json.MarshalIndent(summary, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal chunking summary: %w", err)
}
bytes = append(bytes, '\n')
if err := os.WriteFile(summaryPath, bytes, 0o644); err != nil {
return fmt.Errorf("failed to write chunking summary: %w", err)
}
return nil
}
func (r *RunDirectory) WriteJSONArtifact(name string, payload any) error {
artifactPath := filepath.Join(r.path, name)
bytes, err := json.MarshalIndent(payload, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal %s: %w", name, err)
}
bytes = append(bytes, '\n')
if err := os.WriteFile(artifactPath, bytes, 0o644); err != nil {
return fmt.Errorf("failed to write %s: %w", name, err)
}
return nil
}
func (r *RunDirectory) ApplyRetention(input RetentionDecisionInput) error {
decision := input
if decision.RetentionMode == "" {
decision.RetentionMode = r.retention
}
if ShouldRetainRunDirectory(decision) {
return nil
}
return os.RemoveAll(r.path)
}

29
internal/core/io/files.go Normal file
View File

@@ -0,0 +1,29 @@
package io
import (
"encoding/json"
"fmt"
"os"
)
func ReadRequiredFile(path string, label string) ([]byte, error) {
contents, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("failed to read %s file %q: %w", label, path, err)
}
return contents, nil
}
func ValidateWellFormedJSON(path string, raw []byte) error {
if !json.Valid(raw) {
return fmt.Errorf("transcript file %q is not valid JSON", path)
}
return nil
}
func WriteFile(path string, contents []byte) error {
if err := os.WriteFile(path, contents, 0o644); err != nil {
return fmt.Errorf("failed to write output file %q: %w", path, err)
}
return nil
}

View File

@@ -0,0 +1,35 @@
package modulecatalog
import "strings"
const (
KeyGlossary = "glossary"
KeyHomophones = "homophones"
KeySpokenWord = "spoken_word"
KeyGrammar = "grammar"
)
var supportedKeys = []string{
KeyGlossary,
KeyHomophones,
KeySpokenWord,
KeyGrammar,
}
var supportedKeySet = map[string]struct{}{
KeyGlossary: {},
KeyHomophones: {},
KeySpokenWord: {},
KeyGrammar: {},
}
func SupportedKeys() []string {
out := make([]string, len(supportedKeys))
copy(out, supportedKeys)
return out
}
func IsSupported(key string) bool {
_, ok := supportedKeySet[strings.TrimSpace(key)]
return ok
}

View File

@@ -0,0 +1,24 @@
package modulecatalog
import (
"reflect"
"testing"
)
func TestSupportedKeys(t *testing.T) {
want := []string{KeyGlossary, KeyHomophones, KeySpokenWord, KeyGrammar}
if got := SupportedKeys(); !reflect.DeepEqual(got, want) {
t.Fatalf("unexpected supported keys: got=%v want=%v", got, want)
}
}
func TestIsSupported(t *testing.T) {
for _, key := range SupportedKeys() {
if !IsSupported(key) {
t.Fatalf("expected key %q to be supported", key)
}
}
if IsSupported("made_up") {
t.Fatalf("did not expect made_up to be supported")
}
}

View File

@@ -0,0 +1,206 @@
package normalization
import (
"sort"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
// NormalizationConfig holds configuration parameters for transcript normalization
type NormalizationConfig struct {
MaxSegmentGap float64 // Maximum gap between segments to consider for merging (seconds)
EllipsisGap float64 // Gap threshold above which to insert ellipsis (seconds)
MaxSegmentDuration float64 // Maximum duration for a merged segment (seconds)
MaxSegmentTokens int // Maximum token estimate for a merged segment
}
// NormalizationSummary records the results and statistics of normalization
type NormalizationSummary struct {
InputSegmentCount int `json:"input_segment_count"`
OutputSegmentCount int `json:"output_segment_count"`
MergesPerformed int `json:"merges_performed"`
IDsReassigned int `json:"ids_reassigned"`
SkippedMerges struct {
DifferentSpeakers int `json:"different_speakers"`
GapTooLarge int `json:"gap_too_large"`
DurationExceeded int `json:"duration_exceeded"`
TokenLimitExceeded int `json:"token_limit_exceeded"`
} `json:"skipped_merges"`
}
// NormalizeTranscript performs deterministic normalization on a transcript
type NormalizeTranscript struct {
config NormalizationConfig
estimator *SimpleTokenEstimator
}
// NewNormalizer creates a new transcript normalizer with the given configuration
func NewNormalizer(config NormalizationConfig) *NormalizeTranscript {
return &NormalizeTranscript{
config: config,
estimator: &SimpleTokenEstimator{},
}
}
// Normalize performs deterministic normalization on the given transcript
func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.Transcript, *NormalizationSummary) {
summary := &NormalizationSummary{
InputSegmentCount: len(transcript.Segments),
}
if len(transcript.Segments) == 0 {
return transcript, summary
}
// Sort segments chronologically
sortedSegments := make([]schema.Segment, len(transcript.Segments))
copy(sortedSegments, transcript.Segments)
sort.Slice(sortedSegments, func(i, j int) bool {
return sortedSegments[i].Start < sortedSegments[j].Start
})
// Merge same-speaker adjacent segments
var normalizedSegments []schema.Segment
var currentSegment schema.Segment
for i, segment := range sortedSegments {
if i == 0 {
// Initialize with first segment
currentSegment = segment
continue
}
// Check if we should merge with current segment
gap := segment.Start - currentSegment.End
merge, reason := shouldMergeWithReason(n.config, n.estimator, &currentSegment, &segment, gap)
if merge {
summary.MergesPerformed++
currentSegment = mergeSegments(&currentSegment, &segment, gap, n.config.EllipsisGap)
} else {
// Track the reason for not merging
switch reason {
case "different_speakers":
summary.SkippedMerges.DifferentSpeakers++
case "gap_too_large":
summary.SkippedMerges.GapTooLarge++
case "duration_exceeded":
summary.SkippedMerges.DurationExceeded++
case "token_limit_exceeded":
summary.SkippedMerges.TokenLimitExceeded++
}
// Finalize current segment and start new one
normalizedSegments = append(normalizedSegments, currentSegment)
currentSegment = segment
}
}
// Add the last segment
if len(currentSegment.Text) > 0 {
normalizedSegments = append(normalizedSegments, currentSegment)
}
// Reassign sequential IDs starting at 1
for i, segment := range normalizedSegments {
if segment.ID != i+1 {
summary.IDsReassigned++
}
normalizedSegments[i].ID = i + 1
}
summary.OutputSegmentCount = len(normalizedSegments)
return &schema.Transcript{Segments: normalizedSegments}, summary
}
func shouldMerge(config NormalizationConfig, estimator *SimpleTokenEstimator,
current, next *schema.Segment, gap float64) bool {
// Don't merge if different speakers
if current.Speaker != next.Speaker {
return false
}
// Don't merge if gap is too large
if gap > config.MaxSegmentGap {
return false
}
// Calculate what the merged segment would look like
mergedText := current.Text
if gap >= config.EllipsisGap {
mergedText += "... "
} else {
mergedText += " "
}
mergedText += next.Text
mergedDuration := next.End - current.Start
// Don't merge if duration would be exceeded
if mergedDuration > config.MaxSegmentDuration {
return false
}
// Don't merge if token estimate would be exceeded
tokenEstimate := estimator.EstimateTokens(mergedText)
if tokenEstimate > config.MaxSegmentTokens {
return false
}
return true
}
func shouldMergeWithReason(config NormalizationConfig, estimator *SimpleTokenEstimator,
current, next *schema.Segment, gap float64) (bool, string) {
if current.Speaker != next.Speaker {
return false, "different_speakers"
}
if gap > config.MaxSegmentGap {
return false, "gap_too_large"
}
mergedText := current.Text
if gap >= config.EllipsisGap {
mergedText += "... "
} else {
mergedText += " "
}
mergedText += next.Text
mergedDuration := next.End - current.Start
if mergedDuration > config.MaxSegmentDuration {
return false, "duration_exceeded"
}
tokenEstimate := estimator.EstimateTokens(mergedText)
if tokenEstimate > config.MaxSegmentTokens {
return false, "token_limit_exceeded"
}
return true, ""
}
func mergeSegments(current, next *schema.Segment, gap, ellipsisGap float64) schema.Segment {
mergedText := current.Text
if gap >= ellipsisGap {
mergedText += "... "
} else {
mergedText += " "
}
mergedText += next.Text
// Merge categories
categories := make([]string, 0, len(current.Categories)+len(next.Categories))
categories = append(categories, current.Categories...)
categories = append(categories, next.Categories...)
return schema.Segment{
ID: current.ID, // Will be reassigned later
Speaker: current.Speaker,
Start: current.Start,
End: next.End,
Text: mergedText,
Categories: categories,
}
}

Some files were not shown because too many files have changed in this diff Show More