103 Commits

Author SHA1 Message Date
018d08fd4c Updated contributors in the LICENSE
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-24 06:54:30 -05:00
3a0a0b3940 Moved JSON response schemas into separate files 2026-05-24 06:51:50 -05:00
99ab2f181b Remove the completed documentation roadmap 2026-05-23 20:59:56 -05:00
4e4801dc98 Finalize documentation validation and update README documentation links 2026-05-24 01:12:24 +00:00
900ad74958 Consolidate development policy docs and remove legacy documentation paths 2026-05-24 01:10:29 +00:00
28d5201a69 Add integration documentation for subprocess, LLM, and input files 2026-05-24 01:07:27 +00:00
e5944f9875 Migrate internal architecture docs to docs/internal 2026-05-24 01:02:47 +00:00
6344fc91ba Add operations and troubleshooting documentation 2026-05-24 00:57:50 +00:00
7f3a91cc9e Rewrite config docs and add validated example files 2026-05-24 00:54:27 +00:00
72fb021453 Rewrite README and add canonical CLI reference 2026-05-24 00:51:02 +00:00
40e8b54d3b Establish canonical documentation paths and update README links 2026-05-24 00:47:27 +00:00
76651333b1 Add documetation policy 2026-05-23 19:44:55 -05:00
0b01c3a83d Implemented the config source setter cleanup identified during the code audit 2026-05-23 18:49:44 -05:00
0630d36734 Implemented the CLI override extraction cleanup identified in the code audit 2026-05-23 18:33:12 -05:00
52c2697040 Moved report/ledger assembly to a new processreport module 2026-05-23 18:27:30 -05:00
f790c1441c Refresh architecture and configuration documentation for current runtime behavior 2026-05-23 18:24:06 +00:00
56f9b28f4b Consolidate shared test helpers and stabilize timeout hook integration test 2026-05-23 18:18:48 +00:00
222222f449 Share configured LLM secret extraction across diagnostics paths 2026-05-23 18:11:36 +00:00
99391cd18b Centralize validator classification and malformed output handling 2026-05-23 18:02:53 +00:00
84be774b34 Share module proposal execution and transcript-section prompt payload helpers 2026-05-23 17:56:07 +00:00
e053f7e124 Add shared metadata maps and stage-name helpers 2026-05-23 17:48:37 +00:00
13029dbb33 Centralize effective config loading and path resolution 2026-05-23 17:43:27 +00:00
938bfe88c1 Centralize output schema and module key validation catalogs 2026-05-23 17:39:13 +00:00
fa1bd237d1 Centralize diagnostics artifact names and report metadata paths 2026-05-23 17:32:30 +00:00
3d7057b437 Added an implementation roadmap for the issues identified in the code audit 2026-05-23 12:22:01 -05:00
32c8c8b446 Audit code quality and deduplication opportunities 2026-05-23 11:10:44 -05:00
a3655f5540 Make module-stage LLM handling resilient and report warnings 2026-05-23 10:07:06 -05:00
a84941d681 Rationalize default configuration file paths and update documentation
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
2026-05-14 20:14:11 -05:00
46b7356a3b Add WoodpeckerCI support 2026-05-14 20:10:19 -05:00
39208cd119 Update the documentation for running audita as a subprocess 2026-05-14 09:34:33 -05:00
6dae15656d Standardized the grammar and spoken_word modules on a shared editorial_review validator 2026-05-14 09:30:10 -05:00
b103bb2e7d Improved organization of the documentation 2026-05-14 09:24:02 -05:00
2b2a3fc024 Removed the old python code and cleaned up outdated documentation 2026-05-13 20:13:33 -05:00
52ffe42e73 Refactor validators into package-owned components 2026-05-14 00:28:28 +00:00
3b160cf05b Added documentation for an upcoming refactoring of the validator modules 2026-05-13 19:12:44 -05:00
0687982822 Complete pre-1.0 release hardening 2026-05-13 19:42:35 +00:00
ff4ed82239 Add utilization diagnostics and correction ledger 2026-05-13 19:03:37 +00:00
037121e9ce Move prompts into embedded Markdown assets 2026-05-13 18:49:06 +00:00
d6126bf52b Refactor validators into built-in registry chains 2026-05-13 18:36:46 +00:00
1bc5936681 Add output schema registry and public contract docs 2026-05-13 12:59:10 +00:00
9a77a0cd0b Add versioned Audita config support 2026-05-13 12:36:20 +00:00
ebbd2c8a63 Add transcript description prompt context 2026-05-13 12:13:32 +00:00
de99467ede Replace structured LLM dependency with Audita adapter 2026-05-13 02:10:24 +00:00
20f612215f Documentation cleanup and addition of pre-1.0 roadmap punchlist 2026-05-12 20:40:35 -05:00
3d45571bb0 Add intra-module pipeline for LLM validation 2026-05-12 18:39:26 -05:00
1afd753fad Audit intra-module LLM pipeline 2026-05-12 18:24:15 -05:00
a85a7e204e Review LLM concurrency refactor 2026-05-12 21:23:57 +00:00
509436cc4a Add explicit LLM concurrency controls 2026-05-12 21:17:35 +00:00
a48f6da1f4 Audit LLM concurrency flow 2026-05-12 16:00:26 -05:00
df96f9fdf6 Rewrite and simplify the section chunking algorithm 2026-05-12 15:42:32 -05:00
390daa8b84 Cleanup pass to remove refactoring-related artifacts and references 2026-05-12 15:19:50 -05:00
af84249da0 Implemented an --llm-concurrency flag in the Go application that enforces a global LLM concurrency cap 2026-05-12 12:59:46 -05:00
cad172a758 Added chunking logic to modules and added corresponding regression tests 2026-05-12 12:26:01 -05:00
fb59cb21b9 Complete Phase 19 Go rollout documentation 2026-05-12 13:46:14 +00:00
68e2d9b549 Complete Phase 18 operational hardening 2026-05-12 13:32:35 +00:00
185f7ca2b6 Complete Phase 17 parity fixture suite 2026-05-12 12:50:05 +00:00
7ccadc6bd6 Complete Phase 16 default pipeline integration 2026-05-12 12:19:50 +00:00
a9f7fa27ff Complete Phase 15 spoken-word module 2026-05-12 11:56:28 +00:00
dbf3605712 Complete Phase 14 homophones module 2026-05-12 11:37:35 +00:00
543a7ff8ef Complete Phase 13 glossary module 2026-05-12 11:20:02 +00:00
fc3a7b7a67 Complete Phase 12 grammar module 2026-05-12 02:57:06 +00:00
b360493cdc Complete Phase 11 proposal generation framework 2026-05-12 02:25:33 +00:00
12202508bf Complete Phase 10 LLM validators 2026-05-12 01:25:52 +00:00
6d9a4bd017 Complete Phase 9 LLM infrastructure 2026-05-11 20:10:56 -05:00
426864eedb Add Phase 9 LLM scheduler, config resolution, diagnostics primitives 2026-05-11 20:09:34 -05:00
0b17a6fbeb Implement Phase 9 structured LLM adapter spike 2026-05-11 19:54:57 -05:00
aeb31f1c0d Complete Phase 8 deterministic validators 2026-05-12 00:26:27 +00:00
28fe899aa1 Complete Phase 7 runner orchestration 2026-05-11 14:22:22 +00:00
30606f5c49 Updated the Go rewrite work plan to reflect completed work so far 2026-05-11 09:10:47 -05:00
db880ed868 Document completed Phase 6 diagnostics 2026-05-11 14:03:14 +00:00
5217093be2 Complete diagnostics retention semantics 2026-05-11 14:01:56 +00:00
0e83991537 Add diagnostics references to reports 2026-05-11 13:59:46 +00:00
3e8d19cccd Add redacted run metadata artifacts 2026-05-11 13:58:13 +00:00
c1193e3450 Refine rewrite documentation accuracy 2026-05-11 08:50:33 -05:00
73249b63d8 Update Go rewrite documentation status 2026-05-11 08:49:26 -05:00
c3087aeda6 Complete Phase 3 proposal foundation 2026-05-11 13:35:15 +00:00
0452a605ad Document Phase 3 rewrite status 2026-05-11 13:33:13 +00:00
b9b7384123 Test chunking with proposal application 2026-05-11 13:32:17 +00:00
726acc47e1 Define minimal module framework contracts 2026-05-11 13:29:34 +00:00
5c78b1d5d9 Add proposal application fixtures 2026-05-11 13:27:08 +00:00
f461922b9b Apply correction proposals to transcripts 2026-05-11 13:23:59 +00:00
c58d307ba7 Add safe replacement preview logic 2026-05-11 13:20:35 +00:00
1eb93481e0 Add correction proposal models 2026-05-11 13:18:13 +00:00
b997e7c97c Record chunking metadata in process runs 2026-05-11 12:47:31 +00:00
12fd541669 Add token-bounded transcript chunking 2026-05-11 12:23:01 +00:00
10377876e4 Complete Phase 2 Go normalization foundation 2026-05-11 00:41:29 +00:00
d847168ecd Add Phase 2 fixtures and compatibility tests 2026-05-11 00:21:19 +00:00
14e51698c2 Add Phase 2 normalization diagnostics 2026-05-11 00:19:28 +00:00
aeb9c4f062 Report schema validation and normalization results 2026-05-11 00:16:50 +00:00
e2ae7f77d8 Run normalization in process command 2026-05-11 00:13:26 +00:00
0b1b670baf Add deterministic transcript normalization 2026-05-11 00:12:19 +00:00
3cfa4b6e8a Wire schema validation into process command 2026-05-11 00:01:42 +00:00
950edc01f2 Add Go transcript and glossary schemas 2026-05-10 23:52:36 +00:00
ea8def423e Complete Phase 1 Go skeleton 2026-05-10 23:34:43 +00:00
95fe8c32fa Harden Go CLI subprocess behavior 2026-05-10 23:33:13 +00:00
08b7531149 Add minimal process reports 2026-05-10 23:30:46 +00:00
2cf2d390da Add transcript passthrough processing 2026-05-10 23:27:54 +00:00
8f3c2ec5fd Wire process command flags 2026-05-10 23:24:48 +00:00
9427c4e6cc Add Go configuration model 2026-05-10 23:19:59 +00:00
6424d7db4f Initialize Go CLI skeleton 2026-05-10 23:12:46 +00:00
87e560dd3d Added architecture reference documentation for the upcoming Go rewrite 2026-05-10 23:04:29 +00:00
09fc6fd364 Updated .gitignore in preparation for the Go rewrite 2026-05-10 22:40:27 +00:00
2e47c8a1b6 Move python implementation under python/ in preparation for the upcoming Go rewrite 2026-05-10 22:37:38 +00:00
320 changed files with 30984 additions and 10420 deletions

61
.gitignore vendored
View File

@@ -1,3 +1,4 @@
# --> Python
.DS_Store .DS_Store
.venv/ .venv/
__pycache__/ __pycache__/
@@ -9,3 +10,63 @@ dist/
build/ build/
*.egg-info/ *.egg-info/
# ---> Codex
.codex
AGENTS.md
# ---> Go
# If you prefer the allow list template instead of the deny list, see community template:
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
#
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Dependency directories (remove the comment below to include it)
# vendor/
# Go workspace file
go.work
go.work.sum
# env file
.env
# Compiled binaries and test configuration
narratio
local-test
pipeline.yml
bin/
# Local run artifacts
.audita-runs/
report.json
corrected.json
normalized.json
# Coverage artifacts
coverage.out
coverage.txt
# ---> VisualStudioCode
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
!.vscode/*.code-snippets
# Local History for Visual Studio Code
.history/
# Built Visual Studio Code Extensions
*.vsix

50
.woodpecker/release.yml Normal file
View File

@@ -0,0 +1,50 @@
when:
- event: tag
steps:
- name: build-release-assets
image: golang:1.25
commands:
- |
set -eu
version="$CI_COMMIT_TAG"
dist="dist"
pkg="gitea.maximumdirect.net/eric/audita/cmd/audita"
rm -rf "$dist"
mkdir -p "$dist"
build_binary() {
goos="$1"
goarch="$2"
suffix="$3"
output="$dist/audita-$version-$goos-$goarch$suffix"
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \
go build -trimpath -ldflags "-s -w -X gitea.maximumdirect.net/eric/audita/internal/buildinfo.Version=$version" \
-o "$output" "$pkg"
}
build_binary linux amd64 ""
build_binary linux arm64 ""
build_binary darwin amd64 ""
build_binary darwin arm64 ""
build_binary windows amd64 ".exe"
build_binary windows arm64 ".exe"
- name: publish-release
image: woodpeckerci/plugin-release
depends_on:
- build-release-assets
settings:
api_key:
from_secret: GITEA_RELEASE_TOKEN
files:
- dist/audita-*
checksum: sha256
checksum-file: SHA256SUMS
checksum-flatten: true
file-exists: skip
overwrite: false
prerelease: false

View File

@@ -1,4 +1,4 @@
Copyright (c) 2026 eric. Copyright (c) 2026 Eric Rakestraw.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

154
README.md
View File

@@ -1,148 +1,46 @@
# Audita # Audita
Audita is a framework-first transcript correction application. The public `audita` package provides: Audita is a CLI that polishes transcript JSON using glossary-aware and LLM-backed correction modules.
- deterministic transcript normalization ## Quickstart
- token-batched module orchestration
- concrete `glossary`, `homophones`, `spoken_word`, and `grammar` modules built on reusable proposal / validator contracts
- structured run reporting and work-dir diagnostics
The previous working implementation has been preserved as `audita_prototype` inside this repository. Its full regression suite lives under `tests/audita_prototype`. Build:
## Development
This project is set up for `uv`.
```sh ```sh
uv sync --extra dev go build -o ./bin/audita ./cmd/audita
uv run pytest
``` ```
## Usage Run the shortest useful command:
Process a transcript with the current framework implementation:
```sh ```sh
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json audita process ./transcript.json --glossary ./glossary.yaml --output ./corrected.json
``` ```
The framework currently runs this default module sequence: Notes:
- the transcript JSON path is required as a positional argument;
- `--glossary` is required;
- without `--output`, corrected transcript JSON is written to stdout.
1. `glossary` ## Configuration
2. `homophones`
3. `glossary`
4. `spoken_word`
5. `grammar`
Resolved run instance names are auto-numbered for repeats, so the default report pipeline is: Audita loads defaults, optional file config, environment overrides, then CLI overrides.
1. `glossary_1` Use these commands to validate and inspect config:
2. `homophones`
3. `glossary_2`
4. `spoken_word`
5. `grammar`
The default module sequence is fully implemented today:
- `glossary` proposes glossary-supported acoustic corrections
- `homophones` proposes conservative homophone and mistranscription corrections
- `spoken_word` proposes conservative dysfluency cleanup
- `grammar` proposes punctuation, capitalization, and spacing cleanup only
To run a custom module sequence, pass `--modules`:
```sh ```sh
uv run audita process transcript.json --glossary glossary.yaml --modules grammar --output corrected.json audita config validate --config ./audita.yml
audita config print-effective --config ./audita.yml
``` ```
To also write a structured JSON report: ## Documentation
```sh - CLI reference: [`docs/cli.md`](docs/cli.md)
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json --report-json report.json - Configuration reference: [`docs/config.md`](docs/config.md)
``` - Operations guide: [`docs/operations.md`](docs/operations.md)
- Troubleshooting: [`docs/troubleshooting.md`](docs/troubleshooting.md)
From a checked-out repository, you can also use the root launcher: - Subprocess integration: [`docs/integrations/subprocess.md`](docs/integrations/subprocess.md)
- OpenAI-compatible LLM integration: [`docs/integrations/openai-compatible-llm.md`](docs/integrations/openai-compatible-llm.md)
```sh - Transcript and glossary file integration: [`docs/integrations/transcript-glossary-files.md`](docs/integrations/transcript-glossary-files.md)
./audita process transcript.json --glossary glossary.yaml --output corrected.json - Development workflow: [`docs/policy/development.md`](docs/policy/development.md)
``` - Architecture policy: [`docs/policy/architecture.md`](docs/policy/architecture.md)
- Documentation policy: [`docs/policy/documentation.md`](docs/policy/documentation.md)
For a system-wide command, install the source tree under `/usr/local/src/audita`, sync dependencies there, and symlink the root launcher into your `PATH`:
```sh
cd /usr/local/src/audita
uv sync --extra dev
ln -s /usr/local/src/audita/audita /usr/local/bin/audita
audita process transcript.json --glossary glossary.yaml --output corrected.json
```
Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr.
`--report-json` writes a separate machine-readable run report and never mixes report data into stdout.
Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Default OpenRouter runs require LLM API credentials, because the `glossary`, `homophones`, `spoken_word`, and `grammar` modules make real LLM calls. Self-hosted or other non-default OpenAI-compatible endpoints may not require credentials. `AUDITA_LLM_API_KEY` and `--llm-api-key` are the preferred provider-neutral credential surfaces, while `OPENROUTER_API_KEY` remains supported as a backward-compatible fallback.
| Environment variable | CLI flag | Default | Purpose |
| --- | --- | --- | --- |
| `AUDITA_MODULES` | `--modules` | `glossary,homophones,glossary,spoken_word,grammar` | Comma-separated logical module keys to run; CLI overrides the environment value |
| `AUDITA_LLM_API_KEY` | `--llm-api-key` | unset | Preferred provider-neutral LLM API credential; required for the default OpenRouter endpoint and optional for non-default endpoints; CLI overrides both environment-key variants |
| `AUDITA_VALIDATION_LLM_API_KEY` | `--validation-llm-api-key` | unset | Validation-phase LLM API credential; defaults to the primary LLM API key and is optional for non-default validation endpoints |
| `AUDITA_MODEL` | `--model` | `openrouter/google/gemma-4-31b-it` | LLM model name sent to the configured OpenAI-compatible endpoint |
| `AUDITA_VALIDATION_MODEL` | `--validation-model` | unset | Validation-phase LLM model; defaults to `AUDITA_MODEL` |
| `AUDITA_BASE_URL` | `--base-url` | `https://openrouter.ai/api/v1` | OpenAI-compatible API base URL |
| `AUDITA_VALIDATION_BASE_URL` | `--validation-base-url` | unset | Validation-phase OpenAI-compatible API base URL; defaults to `AUDITA_BASE_URL` |
| `AUDITA_LLM_TIMEOUT_SECONDS` | `--llm-timeout-seconds` | `600` | Per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint |
| `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | `--validation-llm-timeout-seconds` | unset | Validation-phase per-request timeout in seconds; defaults to `AUDITA_LLM_TIMEOUT_SECONDS` |
| `AUDITA_VALIDATION_MAX_PROMPT_TOKENS` | `--validation-max-prompt-tokens` | `2048` | Maximum estimated tokens per validation-phase LLM prompt batch |
| `AUDITA_TARGET_SECTIONS` | `--target-sections` | unset | Exact number of contiguous proposal-stage transcript sections; errors if min/max token bounds cannot be satisfied |
| `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses |
| `AUDITA_VALIDATION_MAX_RETRIES` | `--validation-max-retries` | unset | Validation-phase structured-output retries; defaults to `AUDITA_MAX_RETRIES` |
| `AUDITA_VALIDATION_LLM_CONCURRENCY` | `--validation-llm-concurrency` | unset | Validation-phase LLM concurrency; defaults to `AUDITA_LLM_CONCURRENCY` |
| `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `8192` | Maximum estimated tokens per proposal-stage transcript section |
| `AUDITA_MIN_SECTION_TOKENS` | `--min-section-tokens` | `2048` | Minimum estimated tokens per proposal-stage transcript section when balancing for concurrency |
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation |
| `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | `--grammar-confidence-threshold` | `0.8` | Minimum confidence required for grammar proposals to survive validation |
| `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | `--homophones-confidence-threshold` | `0.8` | Minimum confidence required for homophone proposals to survive validation |
| `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | `--spoken-word-confidence-threshold` | `0.8` | Minimum confidence required for spoken-word proposals to survive validation |
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | `--normalize-max-segment-gap` | `4.0` | Same-speaker gaps eligible for deterministic merging |
| `AUDITA_NORMALIZE_ELLIPSIS_GAP` | `--normalize-ellipsis-gap` | `3.5` | Same-speaker gaps above this value are joined with ` ... ` |
| `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | `--normalize-max-segment-duration` | `60.0` | Maximum merged segment duration |
| `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | `--normalize-max-segment-tokens` | `2048` | Maximum merged segment prompt payload size |
| `AUDITA_WORK_DIR` | `--work-dir` | `/tmp/audita` | Per-run scratch diagnostics directory |
| `AUDITA_WORK_DIR_RETENTION` | `--work-dir-retention` | `auto` | Whether to retain the per-run work directory: `auto`, `always`, or `never` |
Set `AUDITA_MODULES=grammar` to run only the grammar module by default, or override it per command with `--modules`.
Validation-phase LLM settings inherit from the primary `AUDITA_*` LLM settings by default. Set any of the `AUDITA_VALIDATION_*` values only when you want LLM-backed validators to use a different model, endpoint, credential, timeout, retry budget, or concurrency level.
OpenRouter remains the default out of the box:
```sh
export AUDITA_LLM_API_KEY=your-openrouter-key
audita process transcript.json --glossary glossary.yaml --output corrected.json
```
You can point Audita at any OpenAI-compatible endpoint by changing `AUDITA_BASE_URL` and, if needed, `AUDITA_MODEL`. For example, a local vLLM server:
```sh
export AUDITA_BASE_URL=http://localhost:8000/v1
export AUDITA_MODEL=meta-llama/Llama-3.1-8B-Instruct
audita process transcript.json --glossary glossary.yaml --output corrected.json
```
If your self-hosted endpoint requires authentication, you can still set `AUDITA_LLM_API_KEY`; Audita simply no longer requires it for non-default endpoints.
Or the actual OpenAI API:
```sh
export AUDITA_LLM_API_KEY=your-openai-key
export AUDITA_BASE_URL=https://api.openai.com/v1
export AUDITA_MODEL=gpt-4.1-mini
audita process transcript.json --glossary glossary.yaml --output corrected.json
```
`AUDITA_WORK_DIR` stores per-run diagnostics while processing. Under the default `AUDITA_WORK_DIR_RETENTION=auto`, clean successful runs are removed, while failed runs and successful runs with final skipped corrections are preserved. Use `always` to keep every run directory and `never` to remove successful run directories even when skips remain.
Failed runs always preserve the run directory and include an authoritative `report.json` alongside normalization and prompt/response diagnostics.
## Prototype Archive
The archived prototype remains importable as `audita_prototype` and is still covered by its original regression suite. This is intentional: the new `audita` package is a framework-oriented rewrite, not a thin wrapper around the old code.

147
audita
View File

@@ -1,147 +0,0 @@
#!/usr/bin/env python3
import json
import os
from datetime import datetime
import shutil
import subprocess
import sys
from pathlib import Path
from typing import Optional
from uuid import uuid4
DEFAULT_WORK_DIR = "/tmp/audita"
_SECRET_FLAGS = {"--llm-api-key", "--validation-llm-api-key"}
def _redact_argv(argv: list[str]) -> list[str]:
redacted: list[str] = []
index = 0
while index < len(argv):
arg = argv[index]
matched_flag = next((flag for flag in _SECRET_FLAGS if arg == flag or arg.startswith(flag + "=")), None)
if matched_flag is None:
redacted.append(arg)
index += 1
continue
if arg == matched_flag:
redacted.append(arg)
if index + 1 < len(argv):
redacted.append("[REDACTED]")
index += 2
else:
index += 1
continue
redacted.append(f"{matched_flag}=[REDACTED]")
index += 1
return redacted
def _resolve_work_root(argv: list[str]) -> Path:
for index, arg in enumerate(argv):
if arg == "--work-dir" and index + 1 < len(argv):
return Path(argv[index + 1])
if arg.startswith("--work-dir="):
return Path(arg.split("=", 1)[1])
return Path(os.environ.get("AUDITA_WORK_DIR") or DEFAULT_WORK_DIR)
def _create_run_dir(root: Path) -> Path:
root.mkdir(parents=True, exist_ok=True)
timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
run_dir = root / f"run-{timestamp}-{uuid4().hex[:8]}"
run_dir.mkdir(parents=False, exist_ok=False)
return run_dir
def _capture_run_dirs(root: Path) -> set[str]:
if not root.exists():
return set()
return {path.name for path in root.iterdir() if path.is_dir() and path.name.startswith("run-")}
def _find_new_run_dir(root: Path, before: set[str]) -> Optional[Path]:
if not root.exists():
return None
candidates = [
path for path in root.iterdir() if path.is_dir() and path.name.startswith("run-") and path.name not in before
]
if not candidates:
return None
return max(candidates, key=lambda path: path.name)
def _write_launcher_error_log(
path: Path,
*,
message: str,
exit_code: int,
argv: list[str],
command: Optional[list[str]],
) -> None:
payload = {
"timestamp": datetime.utcnow().isoformat() + "Z",
"message": message,
"exit_code": exit_code,
"argv": argv,
"cwd": os.getcwd(),
"command": command,
}
path.write_text(
"Audita Launcher Diagnostics\n" + json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
def _emit_console_line(message: str) -> None:
for stream in (sys.stderr, sys.stdout):
if stream is None:
continue
try:
stream.write(f"{message}\n")
stream.flush()
return
except (OSError, ValueError):
continue
def main() -> int:
argv = list(sys.argv[1:])
work_root = _resolve_work_root(argv)
redacted_argv = _redact_argv(argv)
uv = shutil.which("uv")
if uv is None:
run_dir = _create_run_dir(work_root)
error_log = run_dir / "error.log"
message = "uv is required to run this launcher. Install uv and run `uv sync` in the Audita project."
_write_launcher_error_log(error_log, message=message, exit_code=1, argv=redacted_argv, command=None)
_emit_console_line(f"audita: error: {message}")
_emit_console_line("audita: exit code: 1")
_emit_console_line(f"audita: run directory: {run_dir}")
_emit_console_line(f"audita: error log: {error_log}")
return 1
project_root = Path(__file__).resolve().parent
command = [uv, "run", "--project", str(project_root), "python", "-m", "audita", *sys.argv[1:]]
before = _capture_run_dirs(work_root)
result = subprocess.run(command, cwd=project_root, check=False)
if result.returncode == 0:
return 0
if _find_new_run_dir(work_root, before) is None:
run_dir = _create_run_dir(work_root)
error_log = run_dir / "error.log"
_write_launcher_error_log(
error_log,
message=f"Audita subprocess exited with status {result.returncode}.",
exit_code=result.returncode,
argv=redacted_argv,
command=_redact_argv(command),
)
_emit_console_line(f"audita: subprocess exited with status {result.returncode}")
_emit_console_line(f"audita: run directory: {run_dir}")
_emit_console_line(f"audita: error log: {error_log}")
return result.returncode
if __name__ == "__main__":
raise SystemExit(main())

11
cmd/audita/main.go Normal file
View File

@@ -0,0 +1,11 @@
package main
import (
"os"
"gitea.maximumdirect.net/eric/audita/internal/cli"
)
func main() {
os.Exit(cli.Run(os.Args[1:], os.Stdout, os.Stderr))
}

View File

@@ -0,0 +1,677 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"reflect"
"runtime"
"strconv"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/internal/cli"
"gitea.maximumdirect.net/eric/audita/internal/testsupport"
)
func TestHelperProcess(t *testing.T) {
if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
return
}
sep := -1
for i, arg := range os.Args {
if arg == "--" {
sep = i
break
}
}
if sep == -1 {
os.Exit(2)
}
cli.ConfigureSubprocessTestHooksFromEnv()
code := cli.Run(os.Args[sep+1:], os.Stdout, os.Stderr)
os.Exit(code)
}
func TestProcessHelpSubprocess(t *testing.T) {
result := runCLISubprocess(t, "process", "--help")
if result.exitCode != 0 {
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
}
if !strings.Contains(result.stdout, "Usage:") || !strings.Contains(result.stdout, "--glossary") {
t.Fatalf("unexpected help stdout: %q", result.stdout)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr, got %q", result.stderr)
}
}
func TestProcessSuccessWithOutputSubprocess(t *testing.T) {
outputPath := filepath.Join(t.TempDir(), "corrected.json")
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--output",
outputPath,
)
if result.exitCode != 0 {
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
}
if result.stdout != "" {
t.Fatalf("expected empty stdout when --output is set, got %q", result.stdout)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
inputBytes := readFile(t, fixturePath("tiny_transcript.json"))
outputBytes := readFile(t, outputPath)
assertJSONSemanticallyEqual(t, inputBytes, outputBytes)
}
func TestProcessSuccessWithoutOutputSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
)
if result.exitCode != 0 {
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
inputBytes := readFile(t, fixturePath("tiny_transcript.json"))
assertJSONSemanticallyEqual(t, inputBytes, []byte(result.stdout))
}
func TestProcessSuccessWithAuditaV1OutputSchemaSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--output-schema",
"audita-v1",
)
if result.exitCode != 0 {
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
var out struct {
Schema string `json:"schema"`
}
if err := json.Unmarshal([]byte(result.stdout), &out); err != nil {
t.Fatalf("expected valid audita-v1 JSON output: %v", err)
}
if out.Schema != "audita-v1" {
t.Fatalf("expected audita-v1 schema, got %q", out.Schema)
}
}
func TestProcessFailureMissingTranscriptSubprocess(t *testing.T) {
result := runCLISubprocess(t, "process", "--glossary", fixturePath("tiny_glossary.yaml"))
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "expected exactly 1 transcript JSON path argument") {
t.Fatalf("expected actionable missing transcript error, got %q", result.stderr)
}
}
func TestProcessFailureMalformedJSONSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
fixturePath("malformed_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "is not valid JSON") {
t.Fatalf("expected malformed JSON error, got %q", result.stderr)
}
}
func TestProcessFailureMissingTranscriptFileSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
filepath.Join(t.TempDir(), "missing-transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "transcript_read") {
t.Fatalf("expected transcript_read failure, got %q", result.stderr)
}
}
func TestProcessFailureMissingGlossaryFileSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
filepath.Join(t.TempDir(), "missing-glossary.yaml"),
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "glossary_read") {
t.Fatalf("expected glossary_read failure, got %q", result.stderr)
}
}
func TestProcessFailureTranscriptSchemaSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
schemaFixturePath("transcript_empty_speaker.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "transcript_schema") {
t.Fatalf("expected transcript_schema failure, got %q", result.stderr)
}
}
func TestProcessFailureMalformedGlossaryYAMLSubprocess(t *testing.T) {
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
schemaFixturePath("glossary_malformed.yaml"),
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "glossary_schema") {
t.Fatalf("expected glossary_schema failure, got %q", result.stderr)
}
}
func TestProcessFailureUnreadableTranscriptSubprocess(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("portable unreadable-file permissions are not reliable on windows")
}
dir := t.TempDir()
transcriptPath := filepath.Join(dir, "transcript.json")
if err := os.WriteFile(transcriptPath, []byte(`[]`), 0o000); err != nil {
t.Fatalf("write unreadable transcript: %v", err)
}
t.Cleanup(func() { _ = os.Chmod(transcriptPath, 0o644) })
if _, err := os.ReadFile(transcriptPath); err == nil {
t.Skip("unable to make transcript unreadable on this platform/user")
}
result := runCLISubprocess(
t,
"process",
transcriptPath,
"--glossary",
fixturePath("tiny_glossary.yaml"),
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "transcript_read") {
t.Fatalf("expected transcript_read failure, got %q", result.stderr)
}
}
func TestProcessFailureUnwritableOutputSubprocess(t *testing.T) {
outputDir := t.TempDir()
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--output",
outputDir,
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "failed to write output file") {
t.Fatalf("expected write failure message, got %q", result.stderr)
}
}
func TestProcessFailureUnwritableReportJSONSubprocess(t *testing.T) {
reportDir := t.TempDir()
outputPath := filepath.Join(t.TempDir(), "out.json")
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--output",
outputPath,
"--report-json",
reportDir,
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "failed to write report JSON file") {
t.Fatalf("expected report write failure message, got %q", result.stderr)
}
}
func TestProcessSuccessReportJSONSubprocess(t *testing.T) {
reportPath := filepath.Join(t.TempDir(), "report.json")
result := runCLISubprocess(
t,
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--report-json",
reportPath,
)
if result.exitCode != 0 {
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
if !json.Valid([]byte(result.stdout)) {
t.Fatalf("expected transcript JSON only on stdout, got %q", result.stdout)
}
report := readFile(t, reportPath)
if !json.Valid(report) {
t.Fatalf("expected valid report JSON, got %q", string(report))
}
// Ensure report JSON is not printed to stdout.
if strings.Contains(result.stdout, `"default_pipeline"`) {
t.Fatalf("report JSON leaked to stdout: %q", result.stdout)
}
}
func TestProcessSuccessLargeTranscriptSubprocess(t *testing.T) {
transcriptPath := writeLargeTranscriptFixture(t, 320)
result := runCLISubprocess(
t,
"process",
transcriptPath,
"--glossary",
fixturePath("tiny_glossary.yaml"),
)
if result.exitCode != 0 {
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
if !json.Valid([]byte(result.stdout)) {
t.Fatalf("expected valid transcript JSON on stdout")
}
}
func TestProcessFailureMalformedStructuredLLMResponseViaSubprocessHook(t *testing.T) {
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
result := runCLISubprocessWithEnv(t,
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "malformed_structured"},
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--modules",
"grammar",
"--report-json",
reportPath,
"--work-dir",
workDir,
"--work-dir-retention",
"always",
)
if result.exitCode != 0 {
t.Fatalf("expected zero exit code, got %d stderr=%q", result.exitCode, result.stderr)
}
if !json.Valid([]byte(result.stdout)) {
t.Fatalf("expected transcript JSON on stdout, got %q", result.stdout)
}
if result.stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
report := readFile(t, reportPath)
if !json.Valid(report) {
t.Fatalf("expected valid success report JSON")
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
t.Fatalf("did not expect error.log, got: %v", err)
}
}
func TestProcessFailureBackendLLMViaSubprocessHook(t *testing.T) {
workDir := t.TempDir()
result := runCLISubprocessWithEnv(t,
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "backend_error"},
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--modules",
"grammar",
"--work-dir",
workDir,
"--work-dir-retention",
"always",
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "synthetic backend failure") {
t.Fatalf("expected backend failure details, got %q", result.stderr)
}
if !strings.Contains(result.stderr, "diagnostics:") {
t.Fatalf("expected diagnostics path in stderr, got %q", result.stderr)
}
if _, err := os.Stat(filepath.Join(onlyRunDir(t, workDir), "error.log")); err != nil {
t.Fatalf("expected error.log in retained failed run: %v", err)
}
}
func TestProcessFailureMidPipelinePreservesPartialReportsSubprocess(t *testing.T) {
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
result := runCLISubprocessWithEnv(t,
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "mid_pipeline_fail"},
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--modules",
"glossary,homophones,glossary,spoken_word,grammar",
"--report-json",
reportPath,
"--work-dir",
workDir,
"--work-dir-retention",
"always",
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
reportRaw := readFile(t, reportPath)
var report struct {
Status string `json:"status"`
ErrorPhase string `json:"error_phase"`
ModuleResults []struct {
ModuleInstance string `json:"module_instance"`
Status string `json:"status"`
} `json:"module_results"`
}
if err := json.Unmarshal(reportRaw, &report); err != nil {
t.Fatalf("unmarshal report: %v", err)
}
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
t.Fatalf("expected failed runner_execution report, got %+v", report)
}
if len(report.ModuleResults) == 0 {
t.Fatalf("expected partial module results in failure report")
}
}
func TestProcessCancellationViaSubprocessTimeoutHook(t *testing.T) {
workDir := t.TempDir()
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
result := runCLISubprocessContext(t, ctx,
map[string]string{
"AUDITA_SUBPROCESS_TEST_LLM_MODE": "block_until_cancel",
"AUDITA_SUBPROCESS_TEST_RUN_TIMEOUT_MS": "120",
},
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--modules",
"grammar",
"--work-dir",
workDir,
"--work-dir-retention",
"always",
)
if result.stdout != "" {
if result.stderr == "" {
t.Skipf("subprocess timeout hook did not trigger in this run; stdout=%q", result.stdout)
}
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "context deadline exceeded") {
t.Fatalf("expected context deadline error, got %q", result.stderr)
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log for canceled run: %v", err)
}
if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil {
t.Fatalf("expected report.json for canceled run: %v", err)
}
}
func TestProcessSubprocessNoSecretLeakInOutputsAndDiagnostics(t *testing.T) {
secret := "subprocess-secret"
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
result := runCLISubprocessWithEnv(t,
map[string]string{
"AUDITA_LLM_API_KEY": secret,
"AUDITA_VALIDATION_LLM_API_KEY": secret,
},
"process",
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--output",
outputPath,
"--report-json",
reportPath,
"--work-dir",
workDir,
"--work-dir-retention",
"always",
)
if result.exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", result.exitCode, result.stderr)
}
if strings.Contains(result.stdout, secret) || strings.Contains(result.stderr, secret) {
t.Fatalf("secret leaked in subprocess stdio")
}
assertNoSecretInFile(t, reportPath, secret)
assertNoSecretInTree(t, onlyRunDir(t, workDir), secret)
}
type subprocessResult struct {
stdout string
stderr string
exitCode int
}
func runCLISubprocess(t *testing.T, args ...string) subprocessResult {
t.Helper()
return runCLISubprocessWithEnv(t, nil, args...)
}
func runCLISubprocessWithEnv(t *testing.T, extraEnv map[string]string, args ...string) subprocessResult {
t.Helper()
return runCLISubprocessContext(t, context.Background(), extraEnv, args...)
}
func runCLISubprocessContext(t *testing.T, ctx context.Context, extraEnv map[string]string, args ...string) subprocessResult {
t.Helper()
cmdArgs := append([]string{"-test.run=TestHelperProcess", "--"}, args...)
cmd := exec.CommandContext(ctx, os.Args[0], cmdArgs...)
env := append(filterAuditaEnv(os.Environ()), "GO_WANT_HELPER_PROCESS=1")
for k, v := range extraEnv {
env = append(env, fmt.Sprintf("%s=%s", k, v))
}
cmd.Env = env
var stdoutBuf bytes.Buffer
var stderrBuf bytes.Buffer
cmd.Stdout = &stdoutBuf
cmd.Stderr = &stderrBuf
err := cmd.Run()
result := subprocessResult{
stdout: stdoutBuf.String(),
stderr: stderrBuf.String(),
}
if err == nil {
return result
}
if exitErr, ok := err.(*exec.ExitError); ok {
result.exitCode = exitErr.ExitCode()
return result
}
t.Fatalf("subprocess execution failed: %v", err)
return subprocessResult{}
}
func filterAuditaEnv(env []string) []string {
filtered := make([]string, 0, len(env))
for _, entry := range env {
key := entry
if idx := strings.IndexByte(entry, '='); idx >= 0 {
key = entry[:idx]
}
if strings.HasPrefix(key, "AUDITA_") || key == "OPENROUTER_API_KEY" {
continue
}
filtered = append(filtered, entry)
}
return filtered
}
func fixturePath(name string) string {
return filepath.Join("..", "..", "internal", "cli", "testdata", name)
}
func schemaFixturePath(name string) string {
return filepath.Join("..", "..", "internal", "core", "schema", "testdata", name)
}
func readFile(t *testing.T, path string) []byte {
return testsupport.ReadFile(t, path)
}
func assertJSONSemanticallyEqual(t *testing.T, expected []byte, actual []byte) {
t.Helper()
if !json.Valid(actual) {
t.Fatalf("actual output is not valid JSON: %q", string(actual))
}
var expectedValue any
var actualValue any
if err := json.Unmarshal(expected, &expectedValue); err != nil {
t.Fatalf("failed to unmarshal expected JSON: %v", err)
}
if err := json.Unmarshal(actual, &actualValue); err != nil {
t.Fatalf("failed to unmarshal actual JSON: %v", err)
}
if !reflect.DeepEqual(expectedValue, actualValue) {
t.Fatalf("JSON content mismatch: expected %q got %q", string(expected), string(actual))
}
}
func writeLargeTranscriptFixture(t *testing.T, segments int) string {
t.Helper()
path := filepath.Join(t.TempDir(), "large-transcript.json")
rows := make([]string, 0, segments)
for i := 0; i < segments; i++ {
rows = append(rows, fmt.Sprintf(`{"id":%d,"speaker":"Speaker%d","start":%s,"end":%s,"text":"Segment %d has enough words to exercise stdout and pipe buffering safely."}`,
i+1,
(i%4)+1,
strconv.FormatFloat(float64(i)*1.1, 'f', 1, 64),
strconv.FormatFloat(float64(i)*1.1+1.0, 'f', 1, 64),
i+1,
))
}
payload := "[\n " + strings.Join(rows, ",\n ") + "\n]\n"
if err := os.WriteFile(path, []byte(payload), 0o644); err != nil {
t.Fatalf("write large transcript fixture: %v", err)
}
return path
}
func onlyRunDir(t *testing.T, workDir string) string {
return testsupport.OnlyRunDir(t, workDir)
}
func assertNoSecretInFile(t *testing.T, path, secret string) {
testsupport.AssertNoSecretInFile(t, path, secret)
}
func assertNoSecretInTree(t *testing.T, root, secret string) {
testsupport.AssertNoSecretInTree(t, root, secret)
}

186
docs/cli.md Normal file
View File

@@ -0,0 +1,186 @@
# Audita CLI Reference
## Shortest Useful Command
```sh
audita process <transcript.json> --glossary <glossary.yaml> --output <corrected.json>
```
This command validates input files, runs the configured correction pipeline, and writes corrected transcript JSON.
## Command Overview
- `audita process`: process one transcript JSON file.
- `audita config validate`: validate a versioned YAML config file.
- `audita config print-effective`: print redacted effective config JSON.
General help:
```sh
audita --help
audita process --help
audita config --help
```
## `process`
Usage:
```sh
audita process <transcript.json> [flags]
```
Input requirements:
- exactly one transcript JSON positional argument is required;
- `--glossary <path>` is required.
Config path selection for `process`:
1. `--config <path>`
2. `AUDITA_CONFIG`
3. `/usr/local/etc/audita/config.yml` (if present)
4. `/etc/audita/config.yml` (if present)
For precedence and full config schema, see [`docs/config.md`](config.md).
### `process` Flag Reference
Core I/O flags:
- `--config <path>`: path to versioned YAML config file.
- `--glossary <path>`: glossary YAML input path (required).
- `--output <path>`: corrected transcript JSON output file path.
- `--report-json <path>`: machine-readable report JSON output path.
- `--output-schema <key>`: output schema key (`bare-segments` or `audita-v1`).
- `--modules <csv>`: comma-separated module sequence override.
Primary LLM flags:
- `--llm-api-key <value>`: primary LLM API key.
- `--model <name>`: primary LLM model name.
- `--base-url <url>`: primary OpenAI-compatible base URL.
- `--llm-timeout-seconds <int>`: primary timeout in seconds.
- `--max-retries <int>`: primary structured-output retries.
Validation LLM flags:
- `--validation-llm-api-key <value>`: validation LLM API key.
- `--validation-model <name>`: validation LLM model name.
- `--validation-base-url <url>`: validation OpenAI-compatible base URL.
- `--validation-llm-timeout-seconds <int>`: validation timeout in seconds.
- `--validation-max-retries <int>`: validation structured-output retries.
- `--validation-max-prompt-tokens <int>`: validation max prompt tokens.
Concurrency flags:
- `--total-llm-concurrency <int>`: total concurrent proposal+validation LLM calls.
- `--proposal-llm-concurrency <int>`: concurrent proposal-generation LLM calls.
- `--validation-llm-concurrency <int>`: concurrent validation LLM calls.
- `--llm-concurrency <int>`: alias for `--total-llm-concurrency`.
Chunking and normalization flags:
- `--target-sections <int>`: target number of transcript sections.
- `--max-section-tokens <int>`: maximum section tokens.
- `--min-section-tokens <int>`: minimum section tokens.
- `--normalize-max-segment-gap <float>`: maximum same-speaker merge gap.
- `--normalize-ellipsis-gap <float>`: gap threshold for ellipsis insertion.
- `--normalize-max-segment-duration <float>`: maximum merged segment duration.
- `--normalize-max-segment-tokens <int>`: maximum merged segment token estimate.
Threshold flags:
- `--glossary-confidence-threshold <float>`
- `--homophones-confidence-threshold <float>`
- `--spoken-word-confidence-threshold <float>`
- `--grammar-confidence-threshold <float>`
Context and diagnostics flags:
- `--transcript-description <text>`: background context for prompts; does not override transcript content.
- `--work-dir <path>`: per-run diagnostics work directory.
- `--work-dir-retention <auto|always|never>`: run-directory retention policy.
### `process` Output and Exit Behavior
- With `--output`: stdout is expected to be empty on success.
- Without `--output`: stdout contains transcript JSON only on success.
- `--report-json` writes a file and is never printed to stdout.
- Stderr is human-readable diagnostics/errors.
- On failures after diagnostics initialization, stderr includes the diagnostics directory path.
Exit behavior:
- `0`: success.
- `1`: runtime failure during processing/reporting/output paths.
- `2`: CLI usage or configuration input error.
Integration references:
- subprocess contract: [`docs/integrations/subprocess.md`](integrations/subprocess.md)
- transcript/glossary file contract: [`docs/integrations/transcript-glossary-files.md`](integrations/transcript-glossary-files.md)
### `process` Examples
Write corrected transcript to a file:
```sh
audita process transcript.json \
--glossary glossary.yaml \
--output corrected.json
```
Emit transcript JSON to stdout:
```sh
audita process transcript.json --glossary glossary.yaml
```
Use explicit config and write report JSON:
```sh
audita process transcript.json \
--glossary glossary.yaml \
--config audita.yml \
--output corrected.json \
--report-json report.json
```
Override the module sequence:
```sh
audita process transcript.json \
--glossary glossary.yaml \
--modules glossary,homophones,grammar \
--output corrected.json
```
## `config validate`
Usage:
```sh
audita config validate --config <path>
```
Behavior:
- validates defaults merged with file config;
- does not apply environment overrides;
- prints `config is valid` on success.
Errors:
- `--config` is required;
- positional arguments are rejected;
- validation failures are printed to stderr.
## `config print-effective`
Usage:
```sh
audita config print-effective [--config <path>]
```
Config path selection:
1. `--config <path>` when provided
2. `AUDITA_CONFIG`
3. `/usr/local/etc/audita/config.yml` (if present)
4. `/etc/audita/config.yml` (if present)
Behavior:
- merges defaults, optional config file, and environment overrides;
- prints redacted JSON to stdout.
Errors:
- positional arguments are rejected;
- resolution or parse failures are printed to stderr.

238
docs/config.md Normal file
View File

@@ -0,0 +1,238 @@
# Audita Configuration
## Scope
This is the canonical configuration reference for Audita.
It documents:
- config path resolution;
- effective precedence across defaults, file config, environment, and CLI;
- supported `version: 1` YAML schema;
- environment overrides;
- CLI override relationship;
- validation and secrets behavior.
For CLI command syntax, see [`docs/cli.md`](cli.md).
For OpenAI-compatible endpoint behavior, see [`docs/integrations/openai-compatible-llm.md`](integrations/openai-compatible-llm.md).
For transcript/glossary input file contracts, see [`docs/integrations/transcript-glossary-files.md`](integrations/transcript-glossary-files.md).
## Loading Model
Path resolution for `audita process` and `audita config print-effective`:
1. `--config <path>`
2. `AUDITA_CONFIG`
3. `/usr/local/etc/audita/config.yml` (if present)
4. `/etc/audita/config.yml` (if present)
Missing explicit path behavior:
- missing `--config` target is an error;
- missing `AUDITA_CONFIG` target is an error.
Missing default-path files are non-fatal.
## Effective Precedence
`audita process`:
1. defaults
2. file config
3. environment overrides
4. CLI overrides
`audita config print-effective`:
1. defaults
2. file config
3. environment overrides
`audita config validate`:
1. defaults
2. file config
`config validate` is intentionally file-only (no environment overrides).
## Defaults
Current defaults:
- modules: `glossary,homophones,glossary,spoken_word,grammar`
- output schema: `bare-segments`
- primary model: `openrouter/google/gemma-4-31b-it`
- primary base URL: `https://openrouter.ai/api/v1`
- primary timeout: `600` seconds
- max retries: `3`
- total/proposal LLM concurrency: `1`
- validation max prompt tokens: `2048`
- max section tokens: `8192`
- min section tokens: `2048`
- confidence thresholds: `0.8`
- normalization max segment gap: `4.0`
- normalization ellipsis gap: `3.5`
- normalization max segment duration: `60.0`
- normalization max segment tokens: `2048`
- transcript description: empty
- work dir: `/tmp/audita`
- work dir retention: `auto`
## YAML Schema (`version: 1`)
Supported file version:
- `version: 1` (required)
Unknown YAML fields are rejected.
```yaml
version: 1
pipeline:
modules: [glossary, homophones, glossary, spoken_word, grammar]
output:
schema: bare-segments
llm:
proposal:
base_url: https://openrouter.ai/api/v1
model: openrouter/google/gemma-4-31b-it
api_key_env: AUDITA_LLM_API_KEY
timeout: 600s
max_retries: 3
validation:
base_url: https://openrouter.ai/api/v1
model: openrouter/google/gemma-4-31b-it
api_key_env: AUDITA_VALIDATION_LLM_API_KEY
timeout: 600
max_retries: 3
concurrency:
total_llm: 1
proposal_llm: 1
validation_llm: 1
chunking:
target_sections: 8
max_section_tokens: 8192
min_section_tokens: 2048
normalization:
max_segment_gap: 4s
ellipsis_gap: 3.5s
max_segment_duration: 60s
max_segment_tokens: 2048
thresholds:
glossary: 0.8
homophones: 0.8
spoken_word: 0.8
grammar: 0.8
context:
description: optional background context
diagnostics:
work_dir: /tmp/audita
retention: auto
```
Duration-parsing behavior:
- `llm.*.timeout`: integer seconds or duration string; duration strings must resolve to whole seconds.
- `normalization.*` duration-like fields: numeric seconds or duration string.
## Environment Overrides
Modules:
- `AUDITA_MODULES`
Config path:
- `AUDITA_CONFIG`
Primary LLM:
- `AUDITA_LLM_API_KEY` (falls back to `OPENROUTER_API_KEY` when unset)
- `AUDITA_MODEL`
- `AUDITA_BASE_URL`
- `AUDITA_LLM_TIMEOUT_SECONDS`
- `AUDITA_MAX_RETRIES`
Validation LLM:
- `AUDITA_VALIDATION_LLM_API_KEY`
- `AUDITA_VALIDATION_MODEL`
- `AUDITA_VALIDATION_BASE_URL`
- `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS`
- `AUDITA_VALIDATION_MAX_RETRIES`
- `AUDITA_VALIDATION_MAX_PROMPT_TOKENS`
Concurrency:
- `AUDITA_TOTAL_LLM_CONCURRENCY`
- `AUDITA_PROPOSAL_LLM_CONCURRENCY`
- `AUDITA_VALIDATION_LLM_CONCURRENCY`
- `AUDITA_LLM_CONCURRENCY` (legacy alias for total)
Chunking:
- `AUDITA_MAX_SECTION_TOKENS`
- `AUDITA_MIN_SECTION_TOKENS`
- `AUDITA_TARGET_SECTIONS`
Thresholds:
- `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD`
- `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD`
- `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD`
- `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD`
Normalization:
- `AUDITA_NORMALIZE_MAX_SEGMENT_GAP`
- `AUDITA_NORMALIZE_ELLIPSIS_GAP`
- `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION`
- `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS`
Diagnostics:
- `AUDITA_WORK_DIR`
- `AUDITA_WORK_DIR_RETENTION` (`auto`, `always`, `never`)
Transcript description:
- no `AUDITA_*` environment variable is currently defined.
## CLI Override Relationship
CLI flags override file and environment values for `audita process`.
The CLI supports canonical total concurrency (`--total-llm-concurrency`) and legacy alias (`--llm-concurrency`):
- when both are provided at the same precedence layer, canonical total wins;
- if proposal concurrency is not explicitly set and total is set via environment or CLI, proposal concurrency inherits that total;
- validation concurrency inherits total only when validation concurrency is unset.
For full flag syntax, see [`docs/cli.md`](cli.md).
## Validation Rules
Validation includes:
- supported module keys only;
- supported output schema keys only (`bare-segments`, `audita-v1`);
- positive timeout/concurrency/token constraints;
- `proposal_llm <= total_llm` and `validation_llm <= total_llm` when validation is set;
- confidence thresholds in `[0.0, 1.0]`;
- transcript description length `<= 500` characters;
- non-empty work dir;
- work-dir retention in `auto|always|never`.
## Secrets
Recommended secret handling:
- use `llm.proposal.api_key_env` and `llm.validation.api_key_env` in file config;
- use `AUDITA_*_API_KEY` environment overrides or CLI key flags when needed.
`api_key_env` fields contain environment variable names, not secret values.
Redaction behavior:
- `audita config print-effective` redacts resolved API keys.
- diagnostics and report paths redact configured secret values.
## Examples
- Minimal config: [`examples/minimal-config.yml`](../examples/minimal-config.yml)
- Production-style config: [`examples/production-config.yml`](../examples/production-config.yml)
- Tiny transcript input: [`examples/tiny-transcript.json`](../examples/tiny-transcript.json)
- Tiny glossary input: [`examples/tiny-glossary.yaml`](../examples/tiny-glossary.yaml)
Validate the config examples:
```sh
audita config validate --config examples/minimal-config.yml
audita config validate --config examples/production-config.yml
```

View File

@@ -0,0 +1,119 @@
# OpenAI-Compatible LLM Integration
## Scope
This document defines the external LLM endpoint contract Audita currently uses.
It covers:
- endpoint and auth expectations;
- structured request and response shape;
- retry and timeout behavior;
- diagnostics and secret redaction.
For user-facing CLI flags and config keys, see [`docs/cli.md`](../cli.md) and [`docs/config.md`](../config.md).
## Endpoint Contract
Audita sends HTTPS `POST` requests to:
- `<base_url>/chat/completions`
`base_url` comes from primary or validation LLM config and is required.
## Authentication Contract
When an API key is configured, Audita sends:
- `Authorization: Bearer <api_key>`
When no API key is configured, the `Authorization` header is omitted.
## Request Shape
Audita sends a chat-completions payload with:
- `model`;
- `messages` (role/content pairs);
- `response_format` using JSON Schema strict mode.
Representative shape:
```json
{
"model": "example-model",
"messages": [
{"role": "system", "content": "..."},
{"role": "user", "content": "..."}
],
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "correction_set",
"strict": true,
"schema": {"type": "object"}
}
}
}
```
Behavioral requirements enforced by Audita:
- `model` must resolve to a non-empty value;
- each message must have non-empty `role` and `content`;
- `response_format.type` is always `json_schema`;
- `response_format.json_schema.name` and `schema` must be present;
- request schema JSON must be valid JSON.
## Response Handling Contract
Audita expects a successful JSON response with at least one choice and assistant content that can be interpreted as JSON.
Supported assistant content forms:
- string containing JSON;
- raw JSON value.
Audita then decodes the JSON against the expected structured output type.
Current structured schema identities used by Audita runtime:
- `correction_set`
- `validator_decision_set`
## Retries and Timeouts
Retry behavior:
- default max retries is `3` when unset;
- retries apply to retryable transport/decode/server-side errors;
- HTTP `429` and `5xx` responses are retryable;
- retry stops immediately when context is canceled or deadline expires.
Timeout behavior:
- request timeout is derived from configured LLM timeout settings;
- timeout/cancellation propagate through HTTP requests and return nonzero process failures.
## Error Behavior
Non-2xx responses fail the request.
Error message extraction behavior:
- if provider JSON includes `error.message`, Audita surfaces that message;
- else if provider JSON includes top-level `message`, Audita surfaces that;
- otherwise Audita surfaces status code plus response body text.
Malformed or incompatible structured responses fail safely and are surfaced as runtime errors or validator/proposal warnings depending on call site.
## Secret Redaction
Configured LLM secrets are redacted from:
- surfaced adapter/runtime errors;
- LLM diagnostics request/response/error artifacts;
- effective config/report artifacts that include LLM configuration material.
Redaction marker:
- `[REDACTED]`
## Compatibility Boundaries
This integration documentation applies only to the implemented OpenAI-compatible chat completions flow.
Not part of current behavior:
- provider SDK integration;
- non-OpenAI-compatible API contracts;
- server-side model routing features beyond explicitly configured model/base URL.

View File

@@ -0,0 +1,99 @@
# Subprocess Integration
## Scope
This document describes how a parent process should invoke Audita as a subprocess.
It covers:
- invocation shape;
- stdout/stderr behavior;
- output/report file behavior;
- diagnostics and exit behavior.
For full CLI and config references, see [`docs/cli.md`](../cli.md) and [`docs/config.md`](../config.md).
## Recommended Invocation
Use explicit output and report paths for machine workflows:
```sh
audita process <transcript.json> \
--glossary <glossary.yaml> \
--output <output-transcript.json> \
--report-json <report.json>
```
Optional commonly used flags:
- `--config <path>`
- `--output-schema <bare-segments|audita-v1>`
- `--work-dir <dir>`
- `--work-dir-retention <always|auto|never>`
- `--transcript-description <text>`
## Stdout Contract
On success:
- with `--output`: stdout is expected to be empty;
- without `--output`: stdout contains transcript JSON only.
`--report-json` output is never written to stdout.
## Stderr Contract
Stderr is human-readable status/error output.
On failures:
- stderr includes a concise top-level error;
- when diagnostics are initialized, stderr includes diagnostics directory path.
Do not treat stderr as a machine-stable JSON channel.
## Output and Report File Contract
Transcript output:
- `--output` writes corrected transcript JSON to the provided path;
- output write failures return nonzero.
Report output:
- `--report-json` writes machine-readable process report JSON to the provided path;
- run diagnostics also attempt to write their own `report.json`;
- report write failures return nonzero;
- on failure paths, report writing is best-effort and does not mask the primary run error.
## Diagnostics Contract
When run-directory initialization succeeds, per-run diagnostics artifacts are written under the configured work directory.
Typical artifacts include:
- `source-transcript.json`
- `source-transcript-parsed.json`
- `normalized-transcript.json`
- `normalization-summary.json`
- `chunking-summary.json`
- `invocation.json`
- `effective-config.json`
- `utilization-diagnostics.json`
- `correction-ledger.json`
- `report.json`
- `error.log` (failure)
Retention behavior is controlled by `--work-dir-retention` / config.
## Exit Behavior
Exit codes:
- `0`: success;
- `1`: runtime processing/output/report failure;
- `2`: CLI usage or configuration input error.
Treat any nonzero as subprocess failure.
## Parent-Process Guidance
For reliable orchestration:
- read stdout and stderr concurrently to avoid pipe blocking;
- prefer `--output` and `--report-json` for machine parsing;
- use timeout/cancellation in the parent process;
- inspect diagnostics path and `report.json`/`error.log` on failure.
For input file contracts, see [`docs/integrations/transcript-glossary-files.md`](transcript-glossary-files.md).

View File

@@ -0,0 +1,98 @@
# Transcript and Glossary File Integration
## Scope
This document defines the input file contracts for:
- transcript JSON;
- glossary YAML.
These files are loaded and validated before processing begins.
## Transcript JSON Contract
Audita accepts either top-level shape:
- JSON array of segments; or
- JSON object with a `segments` array.
Segment fields:
- `id` (optional integer in source form);
- `speaker` (required non-empty string);
- `start` (required finite non-negative number);
- `end` (required finite non-negative number, `>= start`);
- `text` (required non-empty string);
- `categories` (optional string array; entries must be non-empty).
Additional rules:
- transcript must contain at least one segment;
- duplicate segment IDs are rejected when IDs are present.
Example (`examples/tiny-transcript.json`):
```json
[
{
"id": 1,
"speaker": "A",
"start": 0.0,
"end": 1.2,
"text": "hello world"
}
]
```
## Glossary YAML Contract
Audita expects top-level `glossary` list entries.
Entry fields:
- `name` (required non-empty string);
- `category` (required non-empty string);
- `summary` (required non-empty string);
- `aliases` (optional list of strings; entries must be non-empty);
- `plural` (optional string).
Additional rules:
- glossary must contain at least one entry.
Example (`examples/tiny-glossary.yaml`):
```yaml
glossary:
- name: Audita
aliases:
- audita
category: product
summary: The Audita transcript correction CLI.
```
## Validation Failure Behavior
Representative transcript validation failures:
- invalid JSON;
- unsupported top-level shape;
- empty `speaker` or `text`;
- invalid times (`NaN`, `Inf`, negative, or `end < start`);
- duplicate IDs;
- empty transcript array.
Representative glossary validation failures:
- invalid YAML;
- empty or missing glossary entries;
- missing required entry fields;
- empty alias values.
These failures surface as schema errors and the process exits nonzero.
## CLI Usage
Minimal invocation:
```sh
audita process ./transcript.json --glossary ./glossary.yaml --output ./corrected.json
```
See also:
- [`docs/cli.md`](../cli.md)
- [`docs/config.md`](../config.md)
- [`examples/tiny-transcript.json`](../../examples/tiny-transcript.json)
- [`examples/tiny-glossary.yaml`](../../examples/tiny-glossary.yaml)

View File

@@ -0,0 +1,79 @@
# Audita Diagnostics and Reporting
## Scope
This document describes diagnostics artifacts, process report mapping, and correction ledger generation.
## Run Directory Ownership
`internal/core/diagnostics` owns run-directory creation, artifact writes, and retention decisions.
Stable artifact names include:
- `source-transcript.json`
- `source-transcript-parsed.json`
- `normalized-transcript.json`
- `normalization-summary.json`
- `chunking-summary.json`
- `utilization-diagnostics.json`
- `correction-ledger.json`
- `invocation.json`
- `effective-config.json`
- `report.json`
- `error.log` (failure)
## Process Report Mapping
`internal/framework/processreport` maps runner/CLI execution facts into `reporting.ProcessReport`.
Report metadata fields include:
- `report_schema_name` (`audita-process-report`)
- `report_schema_version` (`v1`)
- `output_schema`
- `config_version` (when file config exists)
The report includes:
- top-level status/error phase/error message;
- normalization/chunking summaries;
- diagnostics metadata paths;
- per-module results and module summary.
## Correction Ledger
`internal/framework/processreport/BuildCorrectionLedger` flattens run results into `correction-ledger.json` entries.
Dispositions:
- `applied`
- `skipped`
- `rejected`
- `failed`
Validator decisions are split into deterministic and LLM-backed groups using validator metadata classification.
## Report Write Paths
- run directory always attempts to write `report.json` when possible;
- optional `--report-json` writes an external report file;
- on failure paths, report writing is best-effort and does not mask primary run errors.
## Retention Interaction
Current retention behavior:
- failed runs are retained;
- `always` keeps successful runs;
- `auto` removes only clean successful runs;
- `never` currently retains successful runs in current implementation.
## Redaction
Redacted data expectations:
- effective config artifact uses config redaction;
- diagnostics payloads and surfaced errors use LLM secret redaction;
- reports should not include raw API key values.
## Key Tests
- `internal/core/diagnostics/*_test.go`
- `internal/framework/processreport/*_test.go`
- `internal/core/reporting/report_test.go`
- `internal/cli/run_test.go`
- `cmd/audita/main_integration_test.go`

View File

@@ -0,0 +1,67 @@
# Audita LLM Runtime
## Scope
This document describes the structured LLM runtime and scheduler behavior.
## Client Boundary
All runtime LLM calls go through `contracts.StructuredLLMClient`.
Primary adapter:
- `internal/framework/llm/OpenAICompatibleClient`
## Request/Response Behavior
The OpenAI-compatible adapter sends chat completions requests with:
- model;
- messages;
- `response_format.type = json_schema`;
- strict schema envelope (`name`, `schema`, `strict=true`).
The response is decoded into the requested structured output target.
## Response Schema Registry
Structured response schemas are registered in `internal/framework/responseschema`:
- `correction_set`
- `validator_decision_set`
Each schema includes stable diagnostics metadata (`id`, `version`, `name`, `sha256`).
## Retries and Error Handling
Adapter retries apply to retryable conditions (for example transport/decoding/retryable status classes) up to configured `max_retries`.
Errors are sanitized to redact configured API-key values before surfacing.
Malformed structured output detection is shared through `internal/framework/structuredoutput` and is used by:
- proposal generation;
- LLM-backed validators.
## Scheduling and Concurrency
`internal/framework/llm/Scheduler` provides FIFO, context-aware permit gating.
Runner composes scheduler limits across:
- total LLM concurrency;
- proposal LLM concurrency;
- validation LLM concurrency.
Scheduler release is guarded to avoid permit leaks on cancellation/error.
## Diagnostics and Redaction
`internal/framework/llm/DiagnosticsWriter` writes request/response/error artifacts.
Configured secrets are derived from `llm.ConfiguredSecrets(cfg)` and redacted from:
- diagnostics payloads;
- surfaced runtime/adapter errors.
## Key Tests
- `internal/framework/llm/openai_compatible_client_test.go`
- `internal/framework/llm/scheduler_test.go`
- `internal/framework/llm/diagnostics_test.go`
- `internal/framework/responseschema/registry_test.go`
- `internal/framework/structuredoutput/malformed_test.go`

58
docs/internal/modules.md Normal file
View File

@@ -0,0 +1,58 @@
# Audita Modules
## Scope
This document covers module contracts and built-in module packages.
## Module Contract
Modules implement `contracts.TranscriptModule`:
- `Key()`
- `ReplacementPolicy()`
- `Validators()`
- `Propose(ctx, req)`
Runner resolves configured module specs to module instances through `internal/framework/modules`.
## Built-In Modules
Current module packages:
- `internal/modules/glossary`
- `internal/modules/homophones`
- `internal/modules/spoken_word`
- `internal/modules/grammar`
Current replacement policies:
- `glossary`: `replace_all`
- `homophones`: `require_unique`
- `spoken_word`: `require_unique`
- `grammar`: `require_unique`
## Proposal Generation Ownership
Shared proposal-generation plumbing is centralized in:
- `internal/framework/proposal_generation`
Module packages own:
- prompt selection (`internal/prompts` prompt IDs);
- module-specific prompt payload construction.
Shared prompt helpers live in `internal/framework/promptcontext`.
## Validator Chain Ownership
Built-in chains are resolved in `internal/validators` per module key.
Module packages call the built-in chain resolver at construction.
## Failure and Warning Behavior
- module setup failures surface as `runner_setup` or module setup errors;
- module runtime failures surface as `runner_execution` with partial module results preserved;
- malformed structured proposal payloads are downgraded to warnings and section-level proposal rejection.
## Key Tests
- `internal/modules/*/module_test.go`
- `internal/framework/modules/registry_test.go`
- `internal/framework/proposal_generation/*_test.go`
- `internal/cli/run_test.go` (pipeline/report integration)

View File

@@ -0,0 +1,46 @@
# Audita Output Schemas
## Scope
This document describes the implemented transcript output schema registry.
## Registry Ownership
Output schema registry is owned by `internal/core/outputschema`.
Supported schema keys:
- `bare-segments`
- `audita-v1`
## Schemas
`bare-segments`:
- top-level JSON array of transcript segments.
`audita-v1`:
- top-level JSON object with:
- `schema: "audita-v1"`
- `version: "v1"`
- `segments: [...]`
Segment fields include `id`, `speaker`, `start`, `end`, `text`, and optional `categories`.
## Validation and Resolution
Config validation and runtime resolution both reject unsupported schema keys.
Unknown schema keys fail with `unsupported output schema` before output emission.
## Output Emission
The selected schema is used by `audita process` when writing:
- output file (`--output`) or
- stdout (when no `--output`).
Report metadata records selected `output_schema`.
## Key Tests
- `internal/core/outputschema/registry_test.go`
- `internal/core/config/config_test.go`
- `internal/cli/run_test.go`

92
docs/internal/overview.md Normal file
View File

@@ -0,0 +1,92 @@
# Audita Internal Overview
## Scope
This document is the internal architecture entry point for developers and coding agents.
It summarizes:
- package boundaries;
- the main `process` execution path;
- where to add new code safely.
## Package Map
CLI and command orchestration:
- `internal/cli`
Core deterministic components:
- `internal/core/config`
- `internal/core/schema`
- `internal/core/normalization`
- `internal/core/chunking`
- `internal/core/outputschema`
- `internal/core/diagnostics`
- `internal/core/reporting`
- `internal/core/modulecatalog`
Framework orchestration and contracts:
- `internal/framework/contracts`
- `internal/framework/modules`
- `internal/framework/proposal_generation`
- `internal/framework/proposals`
- `internal/framework/runner`
- `internal/framework/validators`
- `internal/framework/llm`
- `internal/framework/responseschema`
- `internal/framework/structuredoutput`
- `internal/framework/processreport`
- `internal/framework/promptcontext`
- `internal/framework/stagename`
Domain implementations:
- `internal/modules/*`
- `internal/validators/*`
- `internal/prompts`
## Main Execution Path (`audita process`)
High-level flow:
1. CLI loads effective config and validates CLI requirements.
2. Run directory is created and invocation/effective config artifacts are written.
3. Transcript/glossary files are loaded and parsed.
4. Transcript is normalized and chunked.
5. `runner.Run` executes configured module instances.
6. Proposals are validated, applied deterministically, and serialized in selected output schema.
7. Process report, utilization diagnostics, correction ledger, and retention decisions are finalized.
## Boundary Summary
- `internal/core/*` owns deterministic, reusable logic and persistence-independent rules.
- `internal/framework/*` owns orchestration contracts and reusable runtime plumbing.
- `internal/modules/*` owns module-specific proposal behavior and prompt usage.
- `internal/validators/*` owns validator composition and built-in chain assembly.
- `internal/prompts` owns embedded prompt assets and metadata registry.
## Where To Add New Code
Add config fields:
- `internal/core/config`
Add module behavior:
- one package under `internal/modules/<module_key>`
- registration/wiring through `internal/framework/modules` and config module list
Add validators:
- implementation under `internal/validators/<validator_key>`
- registry/chain wiring in `internal/validators`
Add runtime orchestration behavior:
- `internal/framework/*` (runner/proposal/validator/LLM plumbing)
Add CLI surface:
- `internal/cli`
## Related Internal Docs
- [`docs/internal/pipeline.md`](pipeline.md)
- [`docs/internal/modules.md`](modules.md)
- [`docs/internal/validators.md`](validators.md)
- [`docs/internal/llm-runtime.md`](llm-runtime.md)
- [`docs/internal/diagnostics-reporting.md`](diagnostics-reporting.md)
- [`docs/internal/prompts.md`](prompts.md)
- [`docs/internal/output-schemas.md`](output-schemas.md)

79
docs/internal/pipeline.md Normal file
View File

@@ -0,0 +1,79 @@
# Audita Internal Pipeline
## Scope
This document describes the implemented `audita process` pipeline.
## Inputs
Pipeline inputs are:
- effective config (`internal/core/config`);
- transcript JSON (`internal/core/schema`);
- glossary YAML (`internal/core/schema`).
## Pipeline Phases
1. Input loading and schema validation
- transcript and glossary files are read and parsed.
- schema failures stop the run with `transcript_schema` or `glossary_schema`.
2. Normalization
- canonical transcript segments are normalized by configured gap/duration/token settings.
- normalization summary artifacts are written.
3. Chunking
- normalized transcript is chunked with configured max/min tokens and target sections.
4. Module proposal generation
- runner executes configured module instances in sequence.
- each module proposes corrections per section.
- per-section proposal generation can run concurrently.
5. Validator filtering
- validators run on candidate proposals before apply.
- deterministic validators run before LLM-backed validators.
- LLM validator inputs are batched by max prompt token limit.
6. Deterministic apply
- approved proposals are applied via replacement policy.
- applied/skipped/rejected outcomes are recorded.
7. Output and reporting
- final transcript is serialized with selected output schema.
- report, utilization diagnostics, and correction ledger are written.
- retention policy is applied to run directory.
## Runner Outputs
`runner.Run` returns:
- final transcript;
- per-module results;
- utilization diagnostics.
CLI/reporting then map this into process report and diagnostics artifacts.
## Failure Behavior
Representative failure phases include:
- `run_dir_creation`
- `transcript_read`, `glossary_read`
- `transcript_schema`, `glossary_schema`
- `chunking`
- `runner_setup`, `runner_execution`
- `output_schema`, `serialization`, `output_write`, `stdout_write`
When diagnostics are available, failure stderr includes diagnostics path.
## Invariants
- module execution order follows configured module sequence;
- proposal/validator nondeterminism is isolated before deterministic apply;
- proposal indices are assigned deterministically by section order;
- output/report artifacts are generated from run results, not speculative state.
## Key Tests
- `internal/framework/runner/runner_test.go`
- `internal/framework/proposal_generation/*_test.go`
- `internal/cli/run_test.go`
- `cmd/audita/main_integration_test.go`

62
docs/internal/prompts.md Normal file
View File

@@ -0,0 +1,62 @@
# Audita Prompt Registry
## Scope
This document describes embedded prompt assets, prompt metadata, and rendering behavior.
## Registry Ownership
Prompt registry lives in `internal/prompts` and embeds assets under `internal/prompts/assets/**`.
Registered prompt IDs:
- `modules.glossary.proposal`
- `modules.homophones.proposal`
- `modules.spoken_word.proposal`
- `modules.grammar.proposal`
- `validators.spoken_form_plausibility`
- `validators.meaning_reversal_review`
- `validators.editorial_review`
- `validators.grammar_review`
- `validators.spoken_word_review`
## Metadata Model
Each prompt has metadata:
- `prompt_id`
- `prompt_version`
- `prompt_source`
- `embedded_path`
- `sha256`
Current source/version values:
- `prompt_source = builtin`
- `prompt_version = v1`
## Rendering
`prompts.RenderUserSystem(promptID, data)` renders system/user templates.
Template behavior:
- uses Go `text/template`;
- `missingkey=error` is enabled;
- output is trimmed.
A shared hardening fragment is embedded once and referenced by prompt templates.
## Prompt Context Inputs
Shared prompt payload helpers:
- transcript section JSON (`internal/framework/promptcontext/MarshalTranscriptSectionJSON`)
- transcript description block (`TranscriptDescriptionBlock`)
Modules and LLM validators provide typed data maps to render prompt assets.
## Diagnostics Integration
Prompt metadata is attached to proposal/validator diagnostics request metadata using `Metadata.DiagnosticsMap()`.
## Key Tests
- `internal/prompts/registry_test.go`
- `internal/framework/promptcontext/*_test.go`
- module and validator prompt builder tests

View File

@@ -0,0 +1,72 @@
# Audita Validators
## Scope
This document describes validator composition, execution order, and decision handling.
## Ownership
Built-in validator keys and chains:
- `internal/validators`
Shared validator runtime mechanics:
- `internal/framework/validators`
Execution-class metadata:
- `internal/validators/metadata`
## Built-In Validator Keys
Deterministic:
- `proposal_shape`
- `confidence_threshold`
- `original_text_presence`
- `non_empty_corrected_text`
- `no_effect`
- `protected_terms`
LLM-backed:
- `spoken_form_plausibility`
- `meaning_reversal_review`
- `editorial_review`
## Built-In Chains
Module chains are defined in `internal/validators/chains.go`.
Glossary, homophones, spoken_word, and grammar each resolve a fixed ordered chain.
## Runtime Execution
For each module section:
1. run deterministic validators;
2. run LLM-backed validators;
3. record decisions and warnings;
4. carry only approved proposals forward.
Decision cardinality is enforced: each candidate proposal must receive exactly one decision per validator.
## LLM Validator Batching
LLM validators:
- build canonical validation request payloads;
- batch by `validation_max_prompt_tokens`;
- call structured LLM client using response schema registry.
Oversized single proposals are rejected with `validator_input_too_large`.
Malformed LLM validator responses are downgraded to warnings and rejected batch decisions.
## Decision and Rejection Reporting
Runner records:
- `validator_decisions`
- `validator_rejected`
- warning records (including malformed response warnings)
Correction ledger classifies deterministic vs LLM validator decisions using canonical metadata classes.
## Key Tests
- `internal/validators/*_test.go`
- `internal/framework/validators/*_test.go`
- `internal/framework/processreport/correction_ledger_test.go`
- `internal/cli/run_test.go`

113
docs/operations.md Normal file
View File

@@ -0,0 +1,113 @@
# Audita Operations
## Scope
This document covers operational behavior for `audita process` as currently implemented:
- run lifecycle;
- output and report files;
- diagnostics artifacts;
- run-directory retention behavior;
- failure inspection and recovery.
For command syntax, see [`docs/cli.md`](cli.md).
## Process Run Lifecycle
A `process` run performs these high-level steps:
1. load effective config (defaults + optional file + env + CLI);
2. create a per-run diagnostics directory;
3. load transcript JSON and glossary YAML;
4. parse/validate input schemas;
5. normalize transcript and compute chunking;
6. run configured modules/validators;
7. serialize output schema and write transcript output;
8. build and write process report;
9. apply run-directory retention.
If a failure happens after diagnostics initialization, the run writes failure details and returns nonzero.
## Output Files
Transcript output:
- when `--output <path>` is set, corrected transcript JSON is written to that file;
- when `--output` is omitted, corrected transcript JSON is written to stdout.
Report output:
- when `--report-json <path>` is set, Audita writes a process report JSON file;
- the run directory also writes its own `report.json` artifact.
On success with `--output`, stdout is expected to be empty.
## Diagnostics Directory
By default, runs use `work_dir` from effective config (default `/tmp/audita`).
Each run directory is created under the work dir using a generated ID like `run-<unix-nanos>`.
Top-level diagnostics artifacts:
- `source-transcript.json`
- `source-transcript-parsed.json`
- `normalized-transcript.json`
- `normalization-summary.json`
- `chunking-summary.json`
- `utilization-diagnostics.json`
- `correction-ledger.json`
- `invocation.json`
- `effective-config.json` (redacted)
- `report.json`
- `error.log` (failure runs)
Report diagnostics metadata includes resolved paths to these artifacts.
## Correction Ledger and Utilization Diagnostics
`correction-ledger.json` records correction dispositions:
- `applied`
- `skipped`
- `rejected`
- `failed`
`utilization-diagnostics.json` records effective concurrency and execution timing summaries for run/module/validator activity.
## Retention Behavior
Retention is controlled by `work_dir_retention` (`auto|always|never`).
Current behavior:
- failed runs are always retained;
- `always`: successful runs are retained;
- `auto`: successful runs are retained only when skipped/rejected corrections occurred; clean successful runs are removed;
- `never`: successful runs are currently retained (same net retention outcome as `always` in current implementation).
Even when a successful run directory is removed under `auto`, an explicit `--report-json` file is still preserved at its target path.
## Failure Inspection
For failed runs:
1. read stderr for the top-level failure and diagnostics path;
2. open `error.log` in the reported run directory;
3. inspect run `report.json` (`status`, `error_phase`, `error_message`);
4. inspect related artifacts referenced by report diagnostics metadata.
Typical `error_phase` values include:
- `transcript_read`
- `glossary_read`
- `transcript_schema`
- `glossary_schema`
- `chunking`
- `runner_setup`
- `runner_execution`
- `output_schema`
- `serialization`
- `output_write`
- `stdout_write`
## Recovery Guidance
Safe recovery pattern:
1. correct the immediate input/config/output-path problem;
2. rerun with `--work-dir-retention always` during debugging;
3. once stable, restore your normal retention mode.
Not implemented:
- resume/checkpoint APIs
- remote diagnostics/report storage

207
docs/policy/architecture.md Normal file
View File

@@ -0,0 +1,207 @@
# Architecture Policy
## Purpose
This document defines Audita's development architecture and invariants for maintainers and LLM coding agents. It describes how the project is intended to be changed safely, based on behavior implemented in this repository today.
User-facing behavior belongs in the README and focused runtime docs. Future or proposed work belongs only under `docs/roadmap/`.
## Project Shape
Audita is a single-process Go CLI for transcript polishing. The executable entrypoint is `cmd/audita`; command handling lives in `internal/cli`.
The implemented `audita process` flow is:
1. load effective config;
2. read and validate transcript JSON and glossary YAML;
3. normalize transcript segments;
4. chunk the working transcript into sections;
5. resolve configured module instances;
6. run correction modules and validator chains;
7. apply approved proposals deterministically;
8. write transcript output, reports, and diagnostics artifacts.
The current built-in modules are `glossary`, `homophones`, `spoken_word`, and `grammar`. The default configured module sequence repeats `glossary`.
For external behavior and compatibility details, prefer links to existing behavior docs:
- [CLI reference](../cli.md)
- [Configuration](../config.md)
- [Operations](../operations.md)
- [Troubleshooting](../troubleshooting.md)
- [Integration docs](../integrations/subprocess.md)
- [Internal docs](../internal/overview.md)
## Core Design Principles
- **Hexagonal architecture:** keep domain behavior behind narrow internal contracts. CLI, filesystem, config loading, diagnostics writing, and LLM transport are adapters around the core processing flow.
- **Composable modules and validators:** correction stages and validators should remain small, explicit, and independently testable.
- **Deterministic orchestration around LLM calls:** LLM responses are nondeterministic inputs. Proposal indexing, validator ordering, proposal application, reports, and output serialization must remain deterministic.
- **Bounded and observable concurrency:** use the implemented schedulers and configured concurrency limits for LLM call sites. Preserve utilization diagnostics when changing scheduling or orchestration.
- **Conservative correction behavior:** validate proposed corrections before application; apply accepted proposals through deterministic apply-time safety checks.
- **Standard-library-first:** prefer the Go standard library. Narrow third-party dependencies are acceptable when they materially improve maintainability, such as `gopkg.in/yaml.v3` for YAML parsing.
- **Current-behavior documentation:** non-roadmap docs must describe implemented behavior only.
## Architectural Boundaries
`internal/core` owns domain data handling and stable runtime contracts that do not require CLI or provider transport knowledge:
- config defaults, loading, validation, redaction, and catalogs;
- transcript and glossary schemas;
- normalization and chunking;
- output-schema encoding;
- diagnostics artifact naming and run-directory helpers;
- public process report shapes.
`internal/framework` owns orchestration contracts and reusable runtime mechanics:
- module and validator interfaces;
- proposal generation, proposal application, and prompt context;
- runner orchestration;
- LLM scheduler, OpenAI-compatible adapter, redaction helpers, and diagnostics writers;
- structured response schema registry;
- process report and correction-ledger assembly.
`internal/modules/*` owns module-specific correction stages. `internal/validators/*` owns built-in validator implementations, registry, chains, and execution-class metadata. `internal/prompts` owns embedded prompt assets and prompt metadata.
`internal/cli` owns command parsing, exit codes, stdout/stderr behavior, config command behavior, filesystem input/output wiring, and top-level process orchestration. CLI concerns should not move into modules, validators, or schema logic.
Tests should stay close to the behavior they protect. Shared test helpers are acceptable when they remove clear duplication without hiding module-specific behavior.
## Modules and Validators
Modules implement `contracts.TranscriptModule`. A module must provide:
- a stable key;
- a replacement policy;
- a validator chain;
- proposal generation from explicit request inputs.
Module packages should stay separate. Do not collapse module-specific prompts, scope, or validation choices into a broad generic stage abstraction.
Validators implement the shared validator contract and return one decision per candidate proposal. Deterministic validators and LLM-backed validators are both composable chain elements. Validator identity and execution class metadata are stable enough to affect ordering, diagnostics, reports, and correction-ledger classification.
Future module or validator changes should preserve:
- explicit inputs and outputs;
- no hidden global state;
- explicit config dependencies;
- deterministic proposal index handling;
- validation before final mutation;
- stable reason codes and validator keys where already exposed.
## LLM Integration and Concurrency
LLM calls are external effects behind narrow contracts. Production structured completions use `contracts.StructuredLLMClient`; the implemented provider adapter is OpenAI-compatible HTTP code in `internal/framework/llm`.
Structured response schemas are registered in `internal/framework/responseschema`. Provider-side schema enforcement is not a substitute for local validation: Audita still validates proposal structure, validator decision cardinality, and apply-time safety.
Concurrency is bounded by configured scheduler limits:
- total LLM concurrency;
- proposal LLM concurrency;
- validation LLM concurrency.
The scheduler is context-aware and releases permits on success, failure, and cancellation. Runner code may collect section-level work concurrently, but transcript mutation is applied later in deterministic proposal-index order.
Diagnostics for LLM interactions should be useful for debugging without leaking configured secrets. Use the existing redaction helpers and `llm.ConfiguredSecrets`.
## State, Inputs, and Outputs
Audita does not implement resume, checkpoint, manifest, or remote storage behavior. Runtime state is in memory plus per-run diagnostics artifacts written under the configured work directory.
Transcript input accepts the implemented JSON forms documented in the public contract. Parsed source transcripts are normalized into Audita's internal transcript shape before chunking and module execution.
Proposals and validator decisions are intermediate runtime data. Approved proposals are applied through `internal/framework/proposals`, which clones transcript state, orders by proposal index, and records applied or skipped changes.
Transcript output is encoded through `internal/core/outputschema`. Reports and correction ledgers are machine-readable artifacts derived from runner outputs; their public shape should not be changed casually.
## Configuration and CLI Boundaries
Config behavior is owned by `internal/core/config`; command usage and process wiring are owned by `internal/cli`.
`audita process` uses implemented precedence: defaults, config file, environment, then CLI flags. `config validate` validates defaults plus a file config and intentionally does not apply environment overrides. `config print-effective` applies defaults, file config, and environment overrides, then prints redacted JSON.
Do not duplicate full CLI or config reference material here. Use [Configuration](../config.md), [CLI reference](../cli.md), [Operations](../operations.md), and integration docs under [`docs/integrations/`](../integrations/subprocess.md) for current external behavior.
When adding config fields or CLI flags, update:
- config defaults, file/env/CLI application, and validation;
- CLI flag extraction if applicable;
- redaction when secrets are involved;
- tests for precedence and source-specific behavior;
- user-facing docs if external behavior changes.
## Errors, Logging, and Diagnostics
Errors should be phase-specific enough for CLI users and subprocess callers. The CLI writes human-readable errors to stderr and preserves transcript JSON-only stdout behavior on successful stdout output.
Run diagnostics are best-effort after run-directory creation. Failed runs are retained. Successful run retention follows the implemented work-dir retention policy.
Diagnostics and reports must not leak configured LLM secrets. Config redaction and LLM payload/error redaction are separate responsibilities and should remain separate.
Process reports, diagnostics metadata, utilization diagnostics, and correction ledgers are part of the public contract. Prefer additive, compatible changes.
## Testing Expectations
Use targeted package tests for touched behavior and `go test ./...` for substantial changes.
When changing modules, inspect or add:
- package-local module tests under `internal/modules/*`;
- prompt rendering or proposal-generation tests when prompt inputs change;
- parity or release fixtures when public output behavior changes.
When changing validators, inspect or add:
- validator package tests;
- registry and chain tests under `internal/validators`;
- framework validator tests for batching, malformed output, diagnostics, and cardinality.
When changing LLM integration or concurrency, inspect or add:
- `internal/framework/llm` scheduler/client/redaction tests;
- `internal/framework/runner` orchestration and utilization tests;
- structured-output malformed classification tests.
When changing config, CLI, schema, output, reports, or diagnostics, inspect or add:
- `internal/core/config` tests;
- CLI tests under `internal/cli`;
- schema and output-schema tests under `internal/core`;
- report, diagnostics, parity, and release-fixture tests.
## Dependency Policy
Audita should remain dependency-light. Prefer standard-library solutions for CLI parsing, HTTP, JSON, filesystem, synchronization, and tests.
Third-party dependencies should be narrow, justified, and preferably de facto standard for their purpose. YAML parsing is the current direct dependency exception.
Do not add broad frameworks for CLI, dependency injection, workflow orchestration, logging, or plugin systems without a concrete implemented need and focused tests.
## Documentation Expectations
Follow [Documentation Policy](./documentation.md). Architecture policy must stay concise and aligned with implemented behavior.
Do not use architecture docs as changelogs. Do not describe planned modules, adapters, modes, persistence, or configuration unless they are implemented. Put future work under `docs/roadmap/`.
## Architectural Invariants
- Keep LLM transport behind `StructuredLLMClient` and framework adapter boundaries.
- Keep correction modules narrowly scoped and package-separated.
- Keep validators modular, composable, and identified by stable keys.
- Keep CLI/config/filesystem concerns out of module and validator domain logic.
- Preserve deterministic transcript mutation and output handling around nondeterministic LLM calls.
- Keep LLM concurrency bounded, configurable, and observable where implemented.
- Keep run diagnostics and reports redacted and machine-readable.
- Keep public CLI, config, output-schema, diagnostics, report, prompt, module, and validator contracts stable unless a change is explicit and tested.
- Prefer small shared helpers over broad rewrites.
- Avoid new dependencies unless they are narrow and clearly justified.
## Non-Goals
- No plugin framework is implemented.
- No generic workflow engine is implemented.
- No resume, checkpoint, manifest, or remote storage system is implemented.
- No multi-process service mode is implemented.
- No provider SDK abstraction beyond the current structured LLM client contract and OpenAI-compatible HTTP adapter is implemented.

124
docs/policy/development.md Normal file
View File

@@ -0,0 +1,124 @@
# Audita Development Workflow
## Scope
This is the canonical contributor workflow for Audita maintainers and coding agents.
It defines:
- repository layout and boundaries;
- setup and test commands;
- expectations for code changes;
- how to add config, CLI, modules, validators, docs, and examples.
## Setup
Prerequisites:
- Go `1.24` or newer.
Common commands:
```sh
go test ./...
go build ./cmd/audita
```
## Repository Layout
Top-level areas:
- `cmd/audita`: executable entrypoint.
- `internal/cli`: command parsing and process/config command orchestration.
- `internal/core`: deterministic config/schema/normalization/chunking/output/diagnostics/reporting logic.
- `internal/framework`: runner orchestration, contracts, proposal generation/application, validators runtime, LLM runtime, response schemas.
- `internal/modules/*`: module-specific correction behavior.
- `internal/validators/*`: validator implementations, chains, and metadata.
- `internal/prompts`: embedded prompts and prompt metadata.
- `docs/`: canonical documentation.
- `examples/`: maintained copyable inputs/configs.
## Change Workflow
1. Confirm scope and behavior contract before editing.
2. Make focused changes in the appropriate ownership area.
3. Add or update tests for changed behavior.
4. Run targeted package tests for touched areas.
5. Run `go test ./...` for substantial changes.
6. Update docs/examples when external behavior changes.
## How To Add or Change Configuration
1. Add fields/defaults/validation under `internal/core/config`.
2. Apply source precedence correctly (defaults, file, env, CLI for `process`).
3. Ensure `config validate` remains file-only and `config print-effective` remains redacted.
4. Update tests in `internal/core/config` and related CLI tests.
5. Update [`docs/config.md`](../config.md) and relevant examples under `examples/`.
## How To Add or Change CLI Behavior
1. Implement parsing/wiring in `internal/cli`.
2. Keep stdout/stderr and exit behavior compatible unless intentional and documented.
3. Update CLI tests under `internal/cli` and integration tests under `cmd/audita`.
4. Update [`docs/cli.md`](../cli.md) and related integration docs.
## How To Add or Change Modules
1. Add or update one module package under `internal/modules/<module_key>`.
2. Keep module-specific prompt ownership in the module + `internal/prompts`.
3. Wire module registration/catalog resolution through framework/core module catalog code.
4. Verify replacement policy and validator chain selection.
5. Add/update module tests and proposal-generation tests.
6. Update internal docs when behavior/contracts change.
## How To Add or Change Validators
1. Implement validator behavior in `internal/validators` and shared runtime pieces in `internal/framework/validators` only when needed.
2. Preserve stable validator keys and decision semantics where already exposed.
3. Keep deterministic vs LLM-backed execution-class behavior explicit.
4. Add/update validator, chain, batching, and malformed-output tests.
5. Update validator documentation when external or developer-facing behavior changes.
## Documentation and Examples Expectations
- Keep one canonical home per topic (see [`docs/policy/documentation.md`](documentation.md)).
- Do not document future/unimplemented behavior outside `docs/roadmap/`.
- Keep command examples and config/examples in sync with current code.
- Keep examples secret-free and copyable.
## Practical Validation Checklist
Use this checklist for meaningful runtime-impacting changes:
1. Run core tests:
```sh
go test ./...
```
2. Verify config commands and examples:
```sh
go run ./cmd/audita config validate --config examples/minimal-config.yml
go run ./cmd/audita config validate --config examples/production-config.yml
go run ./cmd/audita config print-effective --config examples/minimal-config.yml
```
3. Re-check subprocess/runtime contract when touching CLI/process/report paths:
- `--output` success keeps stdout empty;
- no `--output` success writes transcript JSON to stdout;
- `--report-json` writes file output and is not written to stdout;
- failures return nonzero and include diagnostics path when available.
4. Re-check diagnostics/report/redaction when touching LLM, reporting, or diagnostics code:
- report schema metadata fields remain present;
- diagnostics artifact paths remain valid;
- configured secret values remain redacted in reports/diagnostics/errors.
5. Re-check output schema behavior when touching serialization/schema code:
- default `bare-segments` behavior remains correct unless intentionally changed;
- `audita-v1` behavior remains correct unless intentionally changed;
- unsupported schemas fail validation/resolve paths clearly.
## Commit Discipline
- Keep commits scoped and reviewable.
- Avoid mixing unrelated refactors with behavior changes.
- Use concise plain-English commit messages.

View File

@@ -0,0 +1,356 @@
# Go Project Documentation Policy
## Purpose
Project documentation must help four audiences:
1. users who need to run the application;
2. administrators/operators who need to configure and operate it;
3. developers who need to understand and change it safely;
4. LLM coding agents that need clear scope, boundaries, and invariants.
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
## Core Rules
### 1. Keep docs concise
Each document should cover a defined scope and only the essentials for that scope.
Avoid:
- long background explanations;
- repeated reference material;
- implementation detail in user-facing docs;
- aspirational language outside roadmap docs;
- verbose examples where one minimal example is clearer.
### 2. Document only implemented behavior outside roadmap files
Unimplemented, planned, aspirational, experimental, or future work may be described only under:
- `docs/roadmap/`
No other documentation file, including `README.md`, should describe code, features, modules, stages, commands, config fields, or behaviors that do not currently exist.
If a feature is partial, non-roadmap docs may describe only the implemented portion and its current boundary.
### 3. Use canonical homes
Each type of information should have one canonical location.
Canonical homes:
- project purpose and quickstart: `README.md`
- development principles: `docs/policy/architecture.md`
- configuration reference: `docs/config.md`
- CLI reference: `docs/cli.md`
- operations and recovery: `docs/operations.md`
- troubleshooting: `docs/troubleshooting.md`
- implemented internals: `docs/internal/`
- future work: `docs/roadmap/`
- contributor workflow: `docs/policy/development.md`
- copyable examples: `examples/`
Other files should summarize briefly and link to the canonical source.
### 4. Keep examples real
Examples should be valid, maintained, and free of secrets.
Where practical:
- example configs should load successfully;
- example commands should match real CLI syntax;
- important examples should be covered by tests.
## Documentation Profiles
All projects require:
- `README.md`
- `docs/policy/architecture.md`
Additional docs depend on the project.
### Small library
Recommended:
- `docs/policy/development.md`, if contributor conventions are non-obvious
### Simple CLI
Required:
- `docs/cli.md`
Recommended:
- `docs/policy/development.md`
### Config-driven CLI
Required:
- `docs/cli.md`
- `docs/config.md`
Recommended:
- `examples/`
- `docs/policy/development.md`
### Stateful or operator-facing application
Required:
- `docs/cli.md`, if CLI-based
- `docs/config.md`, if config-driven
- `docs/operations.md`
Recommended:
- `docs/troubleshooting.md`
- `examples/`
- `docs/policy/development.md`
### Modular, staged, service-oriented, or orchestration application
Required:
- `docs/cli.md`, if CLI-based
- `docs/config.md`, if config-driven
- `docs/operations.md`
- `docs/internal/`
- `docs/policy/development.md`
Recommended:
- `docs/troubleshooting.md`
- validated examples under `examples/`
## Required Documents
### README.md
**Audience:** users, administrators, operators
The README is the outward-facing project orientation page.
It should include, in order:
1. concise description;
2. elevator pitch;
3. shortest useful command or usage example;
4. links to targeted docs.
The README should be short. It is not a manual.
The “shortest useful command” means the simplest command that performs the projects core use case. (It does not mean `app --help`.)
### docs/policy/architecture.md
**Audience:** developers, LLM coding agents
`docs/policy/architecture.md` is required for every project.
It is an inward-facing development policy document. It should describe how the project is intended to be built and changed.
It should include:
- project shape;
- core design principles;
- package and boundary philosophy;
- state/persistence philosophy, if applicable;
- external integration philosophy, if applicable;
- error-handling and logging principles;
- testing expectations;
- documentation expectations;
- architectural invariants;
- explicit non-goals, if useful.
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
### docs/policy/development.md
**Audience:** developers, LLM coding agents
Required for projects maintained by humans and LLM coding agents.
It should include:
- repository layout;
- build/test commands;
- coding conventions;
- dependency policy;
- how to add config fields;
- how to add CLI flags;
- how to add stages/modules/adapters, if applicable;
- how to update examples;
- documentation update expectations.
### docs/config.md
**Audience:** administrators, operators, advanced users
Required for applications with configuration files.
It should include, in order:
1. config file locations and discovery precedence;
2. minimal working config;
3. production-oriented config;
4. full configuration reference;
5. secrets handling, if applicable;
6. links to maintained examples.
The full configuration reference should be canonical.
### docs/cli.md
**Audience:** users, administrators, operators
Required for CLI applications.
It should include, in order:
1. shortest useful command;
2. command overview;
3. complete flag reference;
4. common workflows;
5. diagnostic or recovery commands, if applicable.
Explain when commands are useful, not just their syntax.
### docs/operations.md
**Audience:** administrators, operators
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
It should cover:
- normal workflow;
- filesystem layout;
- remote storage layout, if applicable;
- logs and manifests;
- resume/retry behavior;
- cleanup behavior;
- archive/backup behavior;
- safe recovery procedures;
- operational caveats.
### docs/troubleshooting.md
**Audience:** administrators, operators
Recommended once recurring failure modes exist.
Each entry should include:
- symptom;
- likely cause;
- diagnostic command or inspection step;
- safe fix;
- relevant links.
### docs/internal/
**Audience:** developers, LLM coding agents
Required for modular, staged, service-oriented, or orchestration projects.
This directory describes implemented internal components. It is not the roadmap.
Use one file per major component where useful.
Each component doc should include:
1. purpose;
2. inputs and outputs;
3. boundaries;
4. config fields used;
5. external adapters used;
6. state or manifest behavior, if applicable;
7. skip/resume behavior, if applicable;
8. failure behavior;
9. tests to inspect before changing;
10. architectural invariants.
### docs/roadmap/
**Audience:** maintainers, developers, LLM coding agents
This is the only place for planned, future, aspirational, experimental, or unimplemented work.
Roadmap docs should clearly distinguish:
- proposed work;
- accepted plans;
- deferred ideas;
- rejected ideas;
- implementation prompts or task breakdowns, if useful.
Roadmap docs should not be confused with current behavior.
### docs/integrations/
**Audience:** developers, LLM coding agents
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
Use one file per integration where useful.
## Examples Directory
Projects with non-trivial configuration or workflows should include `examples/`.
Useful examples include:
- minimal working config;
- production-oriented config;
- full annotated config;
- local development config;
- remote/object-storage config;
- minimal session/input file.
Examples should be valid, maintained, tested when practical, and linked from relevant docs.
## Security and Privacy
Docs and examples must not include:
- real API keys;
- tokens;
- passwords;
- private keys;
- private environment dumps;
- sensitive user data;
- raw private transcripts;
- private infrastructure details unless intentionally public.
Document secret-handling mechanisms, not actual secret values.
## Maintenance Rules
When docs change, verify the affected behavior.
Where practical:
- load example config files in tests;
- test CLI examples or command parser behavior;
- validate documented flags against real flags;
- remove stale references;
- update links after renames;
- keep roadmap content out of non-roadmap docs.
If documentation and code disagree, fix the documentation and/or open a roadmap item; do not leave aspirational behavior in current-behavior docs.
Documentation is complete only when it matches the current code.
## Documentation Change Checklist
Before merging documentation changes, verify:
- README is concise and orientation-focused.
- `docs/policy/architecture.md` describes development principles.
- Future work appears only under `docs/roadmap/`.
- User-facing docs avoid unnecessary internals.
- Developer-facing docs preserve boundaries and invariants.
- Config examples match the schema.
- CLI examples match real commands and flags.
- Defaults appear in the canonical config reference.
- No secrets or private data are included.
- Links are accurate.

153
docs/troubleshooting.md Normal file
View File

@@ -0,0 +1,153 @@
# Audita Troubleshooting
## Scope
This guide lists recurring implemented failure modes for `audita process` and `audita config`.
For each entry: symptom, likely cause, inspect, and fix.
## Config Validation Fails
Symptom:
- `audita config validate --config <path>` exits nonzero.
Likely causes:
- missing `version`;
- unsupported config version;
- unknown YAML field;
- unsupported module key or output schema;
- invalid numeric/range/concurrency/retention values.
Inspect:
1. rerun `audita config validate --config <path>` and read stderr.
2. if needed, inspect effective config with `audita config print-effective --config <path>`.
Fix:
- set `version: 1`;
- remove unknown fields;
- use supported module keys and output schemas (`bare-segments`, `audita-v1`);
- correct invalid values to satisfy validation constraints.
## Config File Resolution Errors
Symptom:
- `audita process` fails before processing with config-related errors like `config file not found`.
Likely causes:
- `--config` points to a missing path;
- `AUDITA_CONFIG` points to a missing path;
- unreadable config path.
Inspect:
1. confirm `--config` or `AUDITA_CONFIG` path exists;
2. run `audita config validate --config <path>` directly.
Fix:
- correct the path or unset invalid `AUDITA_CONFIG`;
- fix permissions for the config file.
## Transcript or Glossary Schema Errors
Symptom:
- stderr includes `transcript_schema` or `glossary_schema` and run exits nonzero.
Likely causes:
- transcript is not valid JSON or has invalid segment fields;
- glossary is not valid YAML or has missing required glossary entry fields.
Inspect:
1. check stderr for parser/validation details;
2. if diagnostics were created, inspect `error.log` and run `report.json` (`error_phase`);
3. inspect `source-transcript.json` and `source-transcript-parsed.json` in the run directory.
Fix:
- correct transcript JSON shape/content;
- correct glossary YAML shape/content and required entry fields;
- rerun validation with known-good tiny examples for comparison:
- `examples/tiny-transcript.json`
- `examples/tiny-glossary.yaml`
## LLM Runtime/Backend Failures
Symptom:
- stderr includes `runner_execution` (or backend timeout/error details) and nonzero exit.
Likely causes:
- unreachable/failed LLM endpoint;
- timeout/cancellation;
- runtime module execution failure.
Inspect:
1. inspect stderr for backend message details;
2. inspect run `report.json` (`error_phase`, `module_results`);
3. inspect diagnostics payloads and `error.log`.
Fix:
- verify model/base URL/API key settings;
- increase timeout if needed;
- rerun with `--work-dir-retention always` while debugging.
## Output File Write Failure
Symptom:
- stderr includes `failed to write output file` and run exits nonzero.
Likely causes:
- output path directory missing;
- insufficient filesystem permissions;
- invalid output target path.
Inspect:
1. check `--output` target directory exists and is writable;
2. inspect run diagnostics `error.log` and report `error_phase`.
Fix:
- write to a valid writable path;
- create missing directories;
- adjust permissions.
## Report File Write Failure
Symptom:
- stderr includes `failed to write report JSON file` and run exits nonzero.
Likely causes:
- invalid or unwritable `--report-json` target path.
Inspect:
1. verify parent directory exists and is writable;
2. inspect diagnostics `error.log` for `report_write` context.
Fix:
- choose a writable report path;
- create missing directories;
- rerun.
## Unsupported Output Schema
Symptom:
- stderr includes `unsupported output schema` and run exits nonzero.
Likely causes:
- unsupported `--output-schema` value;
- unsupported `output.schema` in config.
Inspect:
1. check CLI/config schema key;
2. run `audita config validate --config <path>` when config is involved.
Fix:
- use `bare-segments` or `audita-v1`.
## Diagnostics Directory Lookup
Symptom:
- run fails and you need artifacts for debugging.
Inspect:
1. read stderr for `audita process: diagnostics: <run-dir>`;
2. open `<run-dir>/report.json` and `<run-dir>/error.log`;
3. use diagnostics paths embedded in report metadata for artifact lookup.
Fix:
- rerun with `--work-dir-retention always` to preserve run directories during investigation.

View File

@@ -0,0 +1,6 @@
version: 1
output:
schema: bare-segments
llm:
proposal:
api_key_env: AUDITA_LLM_API_KEY

View File

@@ -0,0 +1,41 @@
version: 1
pipeline:
modules: [glossary, homophones, glossary, spoken_word, grammar]
output:
schema: audita-v1
llm:
proposal:
base_url: https://openrouter.ai/api/v1
model: openrouter/google/gemma-4-31b-it
api_key_env: AUDITA_LLM_API_KEY
timeout: 120s
max_retries: 3
validation:
base_url: https://openrouter.ai/api/v1
model: openrouter/google/gemma-4-31b-it
api_key_env: AUDITA_VALIDATION_LLM_API_KEY
timeout: 120
max_retries: 3
concurrency:
total_llm: 2
proposal_llm: 2
validation_llm: 1
chunking:
target_sections: 8
max_section_tokens: 8192
min_section_tokens: 2048
normalization:
max_segment_gap: 4s
ellipsis_gap: 3.5s
max_segment_duration: 60s
max_segment_tokens: 2048
thresholds:
glossary: 0.8
homophones: 0.8
spoken_word: 0.8
grammar: 0.8
context:
description: "General context for domain vocabulary and speaker names."
diagnostics:
work_dir: /tmp/audita
retention: auto

View File

@@ -0,0 +1,6 @@
glossary:
- name: Audita
aliases:
- audita
category: product
summary: The Audita transcript correction CLI.

View File

@@ -0,0 +1,9 @@
[
{
"id": 1,
"speaker": "A",
"start": 0.0,
"end": 1.2,
"text": "hello world"
}
]

11
go.mod Normal file
View File

@@ -0,0 +1,11 @@
module gitea.maximumdirect.net/eric/audita
go 1.24.0
require gopkg.in/yaml.v3 v3.0.1
require (
github.com/kr/pretty v0.3.1 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
)

17
go.sum Normal file
View File

@@ -0,0 +1,17 @@
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

338
internal/cli/parity_test.go Normal file
View File

@@ -0,0 +1,338 @@
package cli
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
)
type parityFixtureCase struct {
Name string `json:"name"`
TranscriptFile string `json:"transcript_file"`
GlossaryFile string `json:"glossary_file"`
ModulesCSV string `json:"modules_csv,omitempty"`
ProposalResponsesFile string `json:"proposal_responses_file,omitempty"`
ValidationResponsesFile string `json:"validation_responses_file,omitempty"`
Env map[string]string `json:"env,omitempty"`
Expect parityExpectation `json:"expect"`
}
type parityExpectation struct {
ExitCode int `json:"exit_code"`
Status string `json:"status,omitempty"`
ErrorPhase string `json:"error_phase,omitempty"`
StdoutMode string `json:"stdout_mode,omitempty"` // empty|json
StderrContains string `json:"stderr_contains,omitempty"`
OutputTranscriptFile string `json:"output_transcript_file,omitempty"`
ModuleInstances []string `json:"module_instances,omitempty"`
ModuleCount int `json:"module_count,omitempty"`
TotalAppliedChanges int `json:"total_applied_changes,omitempty"`
TotalSkippedChanges int `json:"total_skipped_changes,omitempty"`
FailedModuleInstance string `json:"failed_module_instance,omitempty"`
ValidatorRejectedReasonCodes []string `json:"validator_rejected_reason_codes,omitempty"`
ApplicationSkipReasonCodes []string `json:"application_skip_reason_codes,omitempty"`
RequireErrorLog bool `json:"require_error_log,omitempty"`
SecretMarkers []string `json:"secret_markers,omitempty"`
ExpectedProposalCalls []string `json:"expected_proposal_calls,omitempty"`
ExpectedValidationCalls []string `json:"expected_validation_calls,omitempty"`
ModuleAppliedCounts []int `json:"module_applied_counts,omitempty"`
ModuleRejectedCounts []int `json:"module_rejected_counts,omitempty"`
ModuleSkipCounts []int `json:"module_skip_counts,omitempty"`
MinResponsePayloadArtifacts int `json:"min_response_payload_artifacts,omitempty"`
}
func TestParityFixtures(t *testing.T) {
casePaths, err := filepath.Glob(parityFixturePath("*.case.json"))
if err != nil {
t.Fatalf("glob parity fixtures: %v", err)
}
if len(casePaths) == 0 {
t.Fatal("expected at least one parity fixture case")
}
for _, casePath := range casePaths {
fx := loadParityFixtureCase(t, casePath)
t.Run(fx.Name, func(t *testing.T) {
runParityFixtureCase(t, filepath.Dir(casePath), fx)
})
}
}
func loadParityFixtureCase(t *testing.T, casePath string) parityFixtureCase {
t.Helper()
var fx parityFixtureCase
raw := readFile(t, casePath)
if err := json.Unmarshal(raw, &fx); err != nil {
t.Fatalf("parse parity case %q: %v", casePath, err)
}
if strings.TrimSpace(fx.Name) == "" {
t.Fatalf("parity case %q missing name", casePath)
}
return fx
}
func runParityFixtureCase(t *testing.T, caseDir string, fx parityFixtureCase) {
t.Helper()
for k, v := range fx.Env {
t.Setenv(k, v)
}
proposalClient := &fakeStructuredLLMClient{}
validationClient := &fakeStructuredLLMClient{}
if strings.TrimSpace(fx.ProposalResponsesFile) != "" {
raw := readFile(t, filepath.Join(caseDir, fx.ProposalResponsesFile))
if err := json.Unmarshal(raw, &proposalClient.proposalResponses); err != nil {
t.Fatalf("parse proposal responses: %v", err)
}
processProposalLLMClient = proposalClient
}
if strings.TrimSpace(fx.ValidationResponsesFile) != "" {
raw := readFile(t, filepath.Join(caseDir, fx.ValidationResponsesFile))
if err := json.Unmarshal(raw, &validationClient.validationResponses); err != nil {
t.Fatalf("parse validation responses: %v", err)
}
processValidationLLMClient = validationClient
}
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
var stdout, stderr bytes.Buffer
workDir := t.TempDir()
outputPath := filepath.Join(t.TempDir(), "out.json")
reportPath := filepath.Join(t.TempDir(), "report.json")
args := []string{
"process",
filepath.Join(caseDir, fx.TranscriptFile),
"--glossary",
filepath.Join(caseDir, fx.GlossaryFile),
"--report-json",
reportPath,
"--work-dir",
workDir,
"--work-dir-retention",
"always",
}
// Keep stdout shape deterministic for parity tests.
if fx.Expect.StdoutMode != "json" {
args = append(args, "--output", outputPath)
}
if strings.TrimSpace(fx.ModulesCSV) != "" {
args = append(args, "--modules", fx.ModulesCSV)
}
exitCode := Run(args, &stdout, &stderr)
if exitCode != fx.Expect.ExitCode {
t.Fatalf("expected exit code %d, got %d stderr=%q", fx.Expect.ExitCode, exitCode, stderr.String())
}
switch fx.Expect.StdoutMode {
case "json":
if _, err := json.Marshal(stdout.String()); err != nil {
t.Fatalf("unexpected stdout marshal error: %v", err)
}
if !json.Valid(stdout.Bytes()) {
t.Fatalf("expected JSON stdout, got %q", stdout.String())
}
default:
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout, got %q", stdout.String())
}
}
if fx.Expect.StderrContains != "" && !strings.Contains(stderr.String(), fx.Expect.StderrContains) {
t.Fatalf("expected stderr to contain %q, got %q", fx.Expect.StderrContains, stderr.String())
}
report := readProcessReport(t, reportPath)
assertParityReport(t, report, fx.Expect)
runDir := onlyRunDir(t, workDir)
runDirReport := readProcessReport(t, filepath.Join(runDir, "report.json"))
assertParityReport(t, runDirReport, fx.Expect)
if fx.Expect.RequireErrorLog {
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log: %v", err)
}
}
if strings.TrimSpace(fx.Expect.OutputTranscriptFile) != "" && fx.Expect.ExitCode == 0 {
got := readFile(t, outputPath)
want := readFile(t, filepath.Join(caseDir, fx.Expect.OutputTranscriptFile))
assertJSONSemanticEqual(t, want, got)
}
if len(fx.Expect.ExpectedProposalCalls) > 0 && !reflect.DeepEqual(proposalClient.calls, fx.Expect.ExpectedProposalCalls) {
t.Fatalf("unexpected proposal calls: got %v want %v", proposalClient.calls, fx.Expect.ExpectedProposalCalls)
}
if len(fx.Expect.ExpectedValidationCalls) > 0 && !reflect.DeepEqual(validationClient.calls, fx.Expect.ExpectedValidationCalls) {
t.Fatalf("unexpected validation calls: got %v want %v", validationClient.calls, fx.Expect.ExpectedValidationCalls)
}
if len(fx.Expect.SecretMarkers) > 0 {
assertNoSecretMarkers(t, reportPath, fx.Expect.SecretMarkers)
assertNoSecretMarkersInTree(t, runDir, fx.Expect.SecretMarkers)
}
if fx.Expect.MinResponsePayloadArtifacts > 0 {
matches, err := filepath.Glob(filepath.Join(runDir, "*", "*response-payload.json"))
if err != nil {
t.Fatalf("glob response payload artifacts: %v", err)
}
if len(matches) < fx.Expect.MinResponsePayloadArtifacts {
t.Fatalf("expected at least %d response payload artifacts, got %d", fx.Expect.MinResponsePayloadArtifacts, len(matches))
}
}
}
func assertParityReport(t *testing.T, report reporting.ProcessReport, exp parityExpectation) {
t.Helper()
if exp.Status != "" && report.Status != exp.Status {
t.Fatalf("expected report status %q, got %q", exp.Status, report.Status)
}
if exp.ErrorPhase != "" && report.ErrorPhase != exp.ErrorPhase {
t.Fatalf("expected report error_phase %q, got %q", exp.ErrorPhase, report.ErrorPhase)
}
if len(exp.ModuleInstances) > 0 {
got := make([]string, 0, len(report.ModuleResults))
for _, mr := range report.ModuleResults {
got = append(got, mr.ModuleInstance)
}
if !reflect.DeepEqual(got, exp.ModuleInstances) {
t.Fatalf("unexpected module instances: got %v want %v", got, exp.ModuleInstances)
}
}
if exp.ModuleCount > 0 {
if report.ModulesSummary == nil || report.ModulesSummary.ModuleCount != exp.ModuleCount {
t.Fatalf("expected module_count=%d, got %+v", exp.ModuleCount, report.ModulesSummary)
}
}
if exp.TotalAppliedChanges > 0 {
if report.ModulesSummary == nil || report.ModulesSummary.TotalAppliedChanges != exp.TotalAppliedChanges {
t.Fatalf("expected total_applied_changes=%d, got %+v", exp.TotalAppliedChanges, report.ModulesSummary)
}
}
if exp.TotalSkippedChanges > 0 {
if report.ModulesSummary == nil || report.ModulesSummary.TotalSkippedChanges != exp.TotalSkippedChanges {
t.Fatalf("expected total_skipped_changes=%d, got %+v", exp.TotalSkippedChanges, report.ModulesSummary)
}
}
if exp.FailedModuleInstance != "" {
if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != exp.FailedModuleInstance {
t.Fatalf("expected failed_module_instance=%q, got %+v", exp.FailedModuleInstance, report.ModulesSummary)
}
}
if len(exp.ValidatorRejectedReasonCodes) > 0 {
got := collectValidatorRejectedReasonCodes(report.ModuleResults)
if !reflect.DeepEqual(got, exp.ValidatorRejectedReasonCodes) {
t.Fatalf("unexpected validator rejected reason codes: got %v want %v", got, exp.ValidatorRejectedReasonCodes)
}
}
if len(exp.ApplicationSkipReasonCodes) > 0 {
got := collectApplicationSkipReasonCodes(report.ModuleResults)
if !reflect.DeepEqual(got, exp.ApplicationSkipReasonCodes) {
t.Fatalf("unexpected application skip reason codes: got %v want %v", got, exp.ApplicationSkipReasonCodes)
}
}
if len(exp.ModuleAppliedCounts) > 0 {
got := make([]int, 0, len(report.ModuleResults))
for _, mr := range report.ModuleResults {
got = append(got, len(mr.AppliedChanges))
}
if !reflect.DeepEqual(got, exp.ModuleAppliedCounts) {
t.Fatalf("unexpected per-module applied counts: got %v want %v", got, exp.ModuleAppliedCounts)
}
}
if len(exp.ModuleRejectedCounts) > 0 {
got := make([]int, 0, len(report.ModuleResults))
for _, mr := range report.ModuleResults {
got = append(got, len(mr.ValidatorRejected))
}
if !reflect.DeepEqual(got, exp.ModuleRejectedCounts) {
t.Fatalf("unexpected per-module rejected counts: got %v want %v", got, exp.ModuleRejectedCounts)
}
}
if len(exp.ModuleSkipCounts) > 0 {
got := make([]int, 0, len(report.ModuleResults))
for _, mr := range report.ModuleResults {
got = append(got, len(mr.SkippedChanges))
}
if !reflect.DeepEqual(got, exp.ModuleSkipCounts) {
t.Fatalf("unexpected per-module skip counts: got %v want %v", got, exp.ModuleSkipCounts)
}
}
}
func collectValidatorRejectedReasonCodes(results []reporting.ModuleReport) []string {
out := make([]string, 0)
for _, mr := range results {
for _, vr := range mr.ValidatorRejected {
out = append(out, vr.ReasonCode)
}
}
return out
}
func collectApplicationSkipReasonCodes(results []reporting.ModuleReport) []string {
out := make([]string, 0)
for _, mr := range results {
for _, sk := range mr.SkippedChanges {
out = append(out, string(sk.SkipReason))
}
}
return out
}
func assertJSONSemanticEqual(t *testing.T, expected []byte, actual []byte) {
t.Helper()
var exp any
var act any
if err := json.Unmarshal(expected, &exp); err != nil {
t.Fatalf("unmarshal expected json: %v", err)
}
if err := json.Unmarshal(actual, &act); err != nil {
t.Fatalf("unmarshal actual json: %v", err)
}
if !reflect.DeepEqual(exp, act) {
t.Fatalf("JSON mismatch\nexpected=%s\nactual=%s", string(expected), string(actual))
}
}
func assertNoSecretMarkers(t *testing.T, filePath string, markers []string) {
t.Helper()
raw := string(readFile(t, filePath))
for _, marker := range markers {
if marker != "" && strings.Contains(raw, marker) {
t.Fatalf("secret marker %q leaked in %s", marker, filePath)
}
}
}
func assertNoSecretMarkersInTree(t *testing.T, root string, markers []string) {
t.Helper()
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil || d == nil || d.IsDir() {
return nil
}
raw := string(readFile(t, path))
for _, marker := range markers {
if marker != "" && strings.Contains(raw, marker) {
t.Fatalf("secret marker %q leaked in %s", marker, path)
}
}
return nil
})
}
func parityFixturePath(name string) string {
return filepath.Join("testdata", "parity", name)
}

View File

@@ -0,0 +1,121 @@
package cli
import (
"flag"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
)
type processOverrideBinding func(*config.CLIOverrides, processFlags)
var processOverrideBindings = map[string]processOverrideBinding{
"modules": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ModulesCSV = flags.modules
},
"output-schema": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.OutputSchema = flags.outputSchema
},
"llm-api-key": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.PrimaryLLMAPIKey = flags.llmAPIKey
},
"validation-llm-api-key": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationLLMAPIKey = flags.validationLLMAPIKey
},
"model": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.PrimaryModel = flags.model
},
"validation-model": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationModel = flags.validationModel
},
"base-url": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.PrimaryBaseURL = flags.baseURL
},
"validation-base-url": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationBaseURL = flags.validationBaseURL
},
"llm-timeout-seconds": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.PrimaryLLMTimeoutSeconds = flags.llmTimeoutSeconds
},
"total-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.TotalLLMConcurrency = flags.totalLLMConcurrency
},
"proposal-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ProposalLLMConcurrency = flags.proposalLLMConcurrency
},
"llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.PrimaryLLMConcurrency = flags.llmConcurrency
},
"validation-llm-timeout-seconds": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationLLMTimeoutSeconds = flags.validationLLMTimeoutSeconds
},
"max-retries": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.MaxRetries = flags.maxRetries
},
"validation-max-retries": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationMaxRetries = flags.validationMaxRetries
},
"validation-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationLLMConcurrency = flags.validationLLMConcurrency
},
"validation-max-prompt-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.ValidationMaxPromptTokens = flags.validationMaxPromptTokens
},
"max-section-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.MaxSectionTokens = flags.maxSectionTokens
},
"min-section-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.MinSectionTokens = flags.minSectionTokens
},
"target-sections": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.TargetSections = flags.targetSections
},
"glossary-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.GlossaryConfidenceThreshold = flags.glossaryConfidenceThreshold
},
"grammar-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.GrammarConfidenceThreshold = flags.grammarConfidenceThreshold
},
"homophones-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.HomophonesConfidenceThreshold = flags.homophonesConfidenceThreshold
},
"spoken-word-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.SpokenWordConfidenceThreshold = flags.spokenWordConfidenceThreshold
},
"normalize-max-segment-gap": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.NormalizeMaxSegmentGap = flags.normalizeMaxSegmentGap
},
"normalize-ellipsis-gap": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.NormalizeEllipsisGap = flags.normalizeEllipsisGap
},
"normalize-max-segment-duration": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.NormalizeMaxSegmentDuration = flags.normalizeMaxSegmentDuration
},
"normalize-max-segment-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.NormalizeMaxSegmentTokens = flags.normalizeMaxSegmentTokens
},
"transcript-description": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.TranscriptDescription = flags.transcriptDescription
},
"work-dir": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.WorkDir = flags.workDir
},
"work-dir-retention": func(overrides *config.CLIOverrides, flags processFlags) {
overrides.WorkDirRetention = flags.workDirRetention
},
}
func processCLIOverrides(fs *flag.FlagSet, flags processFlags) (config.CLIOverrides, bool) {
overrides := config.CLIOverrides{}
explicitModules := false
fs.Visit(func(f *flag.Flag) {
if f.Name == "modules" {
explicitModules = true
}
binding, ok := processOverrideBindings[f.Name]
if !ok {
return
}
binding(&overrides, flags)
})
return overrides, explicitModules
}

View File

@@ -0,0 +1,433 @@
package cli
import (
"io"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
)
func TestProcessCLIOverridesMapsEveryConfigMutatingFlag(t *testing.T) {
tests := []struct {
name string
flagName string
value string
wantExplicitModules bool
assertOverrideFields func(t *testing.T, overrides config.CLIOverrides)
}{
{
name: "modules",
flagName: "modules",
value: "grammar,glossary",
wantExplicitModules: true,
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "ModulesCSV", overrides.ModulesCSV, "grammar,glossary")
},
},
{
name: "output schema",
flagName: "output-schema",
value: "audita-v1",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "OutputSchema", overrides.OutputSchema, "audita-v1")
},
},
{
name: "primary api key",
flagName: "llm-api-key",
value: "primary-key",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "PrimaryLLMAPIKey", overrides.PrimaryLLMAPIKey, "primary-key")
},
},
{
name: "validation api key",
flagName: "validation-llm-api-key",
value: "validation-key",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "ValidationLLMAPIKey", overrides.ValidationLLMAPIKey, "validation-key")
},
},
{
name: "primary model",
flagName: "model",
value: "primary-model",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "PrimaryModel", overrides.PrimaryModel, "primary-model")
},
},
{
name: "validation model",
flagName: "validation-model",
value: "validation-model",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "ValidationModel", overrides.ValidationModel, "validation-model")
},
},
{
name: "primary base url",
flagName: "base-url",
value: "https://primary.example.test",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "PrimaryBaseURL", overrides.PrimaryBaseURL, "https://primary.example.test")
},
},
{
name: "validation base url",
flagName: "validation-base-url",
value: "https://validation.example.test",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "ValidationBaseURL", overrides.ValidationBaseURL, "https://validation.example.test")
},
},
{
name: "primary timeout",
flagName: "llm-timeout-seconds",
value: "101",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "PrimaryLLMTimeoutSeconds", overrides.PrimaryLLMTimeoutSeconds, 101)
},
},
{
name: "total concurrency",
flagName: "total-llm-concurrency",
value: "5",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "TotalLLMConcurrency", overrides.TotalLLMConcurrency, 5)
},
},
{
name: "proposal concurrency",
flagName: "proposal-llm-concurrency",
value: "3",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "ProposalLLMConcurrency", overrides.ProposalLLMConcurrency, 3)
},
},
{
name: "legacy concurrency alias",
flagName: "llm-concurrency",
value: "4",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "PrimaryLLMConcurrency", overrides.PrimaryLLMConcurrency, 4)
},
},
{
name: "validation timeout",
flagName: "validation-llm-timeout-seconds",
value: "202",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "ValidationLLMTimeoutSeconds", overrides.ValidationLLMTimeoutSeconds, 202)
},
},
{
name: "max retries",
flagName: "max-retries",
value: "6",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "MaxRetries", overrides.MaxRetries, 6)
},
},
{
name: "validation max retries",
flagName: "validation-max-retries",
value: "7",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "ValidationMaxRetries", overrides.ValidationMaxRetries, 7)
},
},
{
name: "validation concurrency",
flagName: "validation-llm-concurrency",
value: "8",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "ValidationLLMConcurrency", overrides.ValidationLLMConcurrency, 8)
},
},
{
name: "validation max prompt tokens",
flagName: "validation-max-prompt-tokens",
value: "4096",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "ValidationMaxPromptTokens", overrides.ValidationMaxPromptTokens, 4096)
},
},
{
name: "max section tokens",
flagName: "max-section-tokens",
value: "9000",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "MaxSectionTokens", overrides.MaxSectionTokens, 9000)
},
},
{
name: "min section tokens",
flagName: "min-section-tokens",
value: "1000",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "MinSectionTokens", overrides.MinSectionTokens, 1000)
},
},
{
name: "target sections",
flagName: "target-sections",
value: "12",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "TargetSections", overrides.TargetSections, 12)
},
},
{
name: "glossary threshold",
flagName: "glossary-confidence-threshold",
value: "0.91",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "GlossaryConfidenceThreshold", overrides.GlossaryConfidenceThreshold, 0.91)
},
},
{
name: "grammar threshold",
flagName: "grammar-confidence-threshold",
value: "0.92",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "GrammarConfidenceThreshold", overrides.GrammarConfidenceThreshold, 0.92)
},
},
{
name: "homophones threshold",
flagName: "homophones-confidence-threshold",
value: "0.93",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "HomophonesConfidenceThreshold", overrides.HomophonesConfidenceThreshold, 0.93)
},
},
{
name: "spoken word threshold",
flagName: "spoken-word-confidence-threshold",
value: "0.94",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "SpokenWordConfidenceThreshold", overrides.SpokenWordConfidenceThreshold, 0.94)
},
},
{
name: "normalize max segment gap",
flagName: "normalize-max-segment-gap",
value: "1.2",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "NormalizeMaxSegmentGap", overrides.NormalizeMaxSegmentGap, 1.2)
},
},
{
name: "normalize ellipsis gap",
flagName: "normalize-ellipsis-gap",
value: "2.3",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "NormalizeEllipsisGap", overrides.NormalizeEllipsisGap, 2.3)
},
},
{
name: "normalize max segment duration",
flagName: "normalize-max-segment-duration",
value: "45.6",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertFloatOverride(t, "NormalizeMaxSegmentDuration", overrides.NormalizeMaxSegmentDuration, 45.6)
},
},
{
name: "normalize max segment tokens",
flagName: "normalize-max-segment-tokens",
value: "321",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertIntOverride(t, "NormalizeMaxSegmentTokens", overrides.NormalizeMaxSegmentTokens, 321)
},
},
{
name: "transcript description",
flagName: "transcript-description",
value: "podcast episode",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "TranscriptDescription", overrides.TranscriptDescription, "podcast episode")
},
},
{
name: "work dir",
flagName: "work-dir",
value: "/tmp/custom-audita",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "WorkDir", overrides.WorkDir, "/tmp/custom-audita")
},
},
{
name: "work dir retention",
flagName: "work-dir-retention",
value: "always",
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
assertStringOverride(t, "WorkDirRetention", overrides.WorkDirRetention, "always")
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
fs, flags := newProcessFlagSet(config.Default(), io.Discard)
if err := fs.Parse([]string{"--" + tc.flagName, tc.value}); err != nil {
t.Fatalf("parse flag: %v", err)
}
overrides, explicitModules := processCLIOverrides(fs, flags)
if explicitModules != tc.wantExplicitModules {
t.Fatalf("explicitModules=%v, want %v", explicitModules, tc.wantExplicitModules)
}
tc.assertOverrideFields(t, overrides)
})
}
}
func TestProcessCLIOverridesIgnoresNonConfigFlags(t *testing.T) {
fs, flags := newProcessFlagSet(config.Default(), io.Discard)
if err := fs.Parse([]string{
"--config", "/tmp/config.yml",
"--glossary", "/tmp/glossary.yml",
"--output", "/tmp/output.json",
"--report-json", "/tmp/report.json",
}); err != nil {
t.Fatalf("parse flags: %v", err)
}
overrides, explicitModules := processCLIOverrides(fs, flags)
if explicitModules {
t.Fatal("non-config flags should not mark modules explicit")
}
assertNoCLIOverrides(t, overrides)
}
func TestNewProcessFlagSetDefaultsReflectEffectiveConfig(t *testing.T) {
cfg := config.Default()
cfg.Modules = []string{"grammar", "glossary"}
cfg.OutputSchema = "audita-v1"
cfg.PrimaryLLM.APIKey = "primary-key"
cfg.ValidationLLM.APIKey = "validation-key"
cfg.PrimaryLLM.Model = "primary-model"
cfg.ValidationLLM.Model = "validation-model"
cfg.PrimaryLLM.BaseURL = "https://primary.example.test"
cfg.ValidationLLM.BaseURL = "https://validation.example.test"
cfg.PrimaryLLM.TimeoutSeconds = 101
cfg.TotalLLMConcurrency = 5
cfg.ProposalLLMConcurrency = 3
cfg.PrimaryLLM.MaxRetries = 6
cfg.ValidationMaxPromptTokens = 4096
cfg.MaxSectionTokens = 9000
cfg.MinSectionTokens = 1000
cfg.Thresholds.Glossary = 0.91
cfg.Thresholds.Grammar = 0.92
cfg.Thresholds.Homophones = 0.93
cfg.Thresholds.SpokenWord = 0.94
cfg.Normalization.MaxSegmentGap = 1.2
cfg.Normalization.EllipsisGap = 2.3
cfg.Normalization.MaxSegmentDuration = 45.6
cfg.Normalization.MaxSegmentTokens = 321
cfg.TranscriptDescription = "podcast episode"
cfg.WorkDir = "/tmp/custom-audita"
cfg.WorkDirRetention = config.WorkDirRetentionAlways
validationTimeout := 202
validationRetries := 7
validationConcurrency := 8
targetSections := 12
cfg.ValidationLLM.TimeoutSeconds = &validationTimeout
cfg.ValidationLLM.MaxRetries = &validationRetries
cfg.ValidationLLMConcurrency = &validationConcurrency
cfg.TargetSections = &targetSections
_, flags := newProcessFlagSet(cfg, io.Discard)
assertStringOverride(t, "modules default", flags.modules, "grammar,glossary")
assertStringOverride(t, "output schema default", flags.outputSchema, "audita-v1")
assertStringOverride(t, "primary api key default", flags.llmAPIKey, "primary-key")
assertStringOverride(t, "validation api key default", flags.validationLLMAPIKey, "validation-key")
assertStringOverride(t, "primary model default", flags.model, "primary-model")
assertStringOverride(t, "validation model default", flags.validationModel, "validation-model")
assertStringOverride(t, "primary base url default", flags.baseURL, "https://primary.example.test")
assertStringOverride(t, "validation base url default", flags.validationBaseURL, "https://validation.example.test")
assertIntOverride(t, "primary timeout default", flags.llmTimeoutSeconds, 101)
assertIntOverride(t, "total concurrency default", flags.totalLLMConcurrency, 5)
assertIntOverride(t, "proposal concurrency default", flags.proposalLLMConcurrency, 3)
assertIntOverride(t, "legacy concurrency alias default", flags.llmConcurrency, 5)
assertIntOverride(t, "validation timeout default", flags.validationLLMTimeoutSeconds, validationTimeout)
assertIntOverride(t, "max retries default", flags.maxRetries, 6)
assertIntOverride(t, "validation max retries default", flags.validationMaxRetries, validationRetries)
assertIntOverride(t, "validation concurrency default", flags.validationLLMConcurrency, validationConcurrency)
assertIntOverride(t, "validation max prompt tokens default", flags.validationMaxPromptTokens, 4096)
assertIntOverride(t, "max section tokens default", flags.maxSectionTokens, 9000)
assertIntOverride(t, "min section tokens default", flags.minSectionTokens, 1000)
assertIntOverride(t, "target sections default", flags.targetSections, targetSections)
assertFloatOverride(t, "glossary threshold default", flags.glossaryConfidenceThreshold, 0.91)
assertFloatOverride(t, "grammar threshold default", flags.grammarConfidenceThreshold, 0.92)
assertFloatOverride(t, "homophones threshold default", flags.homophonesConfidenceThreshold, 0.93)
assertFloatOverride(t, "spoken word threshold default", flags.spokenWordConfidenceThreshold, 0.94)
assertFloatOverride(t, "normalize max segment gap default", flags.normalizeMaxSegmentGap, 1.2)
assertFloatOverride(t, "normalize ellipsis gap default", flags.normalizeEllipsisGap, 2.3)
assertFloatOverride(t, "normalize max segment duration default", flags.normalizeMaxSegmentDuration, 45.6)
assertIntOverride(t, "normalize max segment tokens default", flags.normalizeMaxSegmentTokens, 321)
assertStringOverride(t, "transcript description default", flags.transcriptDescription, "podcast episode")
assertStringOverride(t, "work dir default", flags.workDir, "/tmp/custom-audita")
assertStringOverride(t, "work dir retention default", flags.workDirRetention, "always")
}
func TestNewProcessFlagSetUsesFallbackDefaultsForUnsetOptionalConfig(t *testing.T) {
cfg := config.Default()
_, flags := newProcessFlagSet(cfg, io.Discard)
assertIntOverride(t, "validation timeout fallback", flags.validationLLMTimeoutSeconds, cfg.PrimaryLLM.TimeoutSeconds)
assertIntOverride(t, "validation retries fallback", flags.validationMaxRetries, cfg.PrimaryLLM.MaxRetries)
assertIntOverride(t, "validation concurrency fallback", flags.validationLLMConcurrency, cfg.TotalLLMConcurrency)
assertIntOverride(t, "target sections fallback", flags.targetSections, 0)
}
func assertStringOverride(t *testing.T, name string, got *string, want string) {
t.Helper()
if got == nil || *got != want {
t.Fatalf("%s=%v, want %q", name, pointerValue(got), want)
}
}
func assertIntOverride(t *testing.T, name string, got *int, want int) {
t.Helper()
if got == nil || *got != want {
t.Fatalf("%s=%v, want %d", name, pointerValue(got), want)
}
}
func assertFloatOverride(t *testing.T, name string, got *float64, want float64) {
t.Helper()
if got == nil || *got != want {
t.Fatalf("%s=%v, want %v", name, pointerValue(got), want)
}
}
func assertNoCLIOverrides(t *testing.T, overrides config.CLIOverrides) {
t.Helper()
value := reflect.ValueOf(overrides)
typ := value.Type()
for i := 0; i < value.NumField(); i++ {
field := value.Field(i)
if field.Kind() != reflect.Ptr {
t.Fatalf("unexpected non-pointer CLIOverrides field %s", typ.Field(i).Name)
}
if !field.IsNil() {
t.Fatalf("expected no CLI overrides, field %s was set", typ.Field(i).Name)
}
}
}
func pointerValue[T any](ptr *T) any {
if ptr == nil {
return "<nil>"
}
if stringer, ok := any(*ptr).(interface{ String() string }); ok {
return strings.TrimSpace(stringer.String())
}
return *ptr
}

View File

@@ -0,0 +1,469 @@
package cli
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
type releaseFixtureExpectations struct {
MustApplyTexts []string `json:"must_apply_texts"`
MustNotApplyTexts []string `json:"must_not_apply_texts"`
ProtectedTerms []string `json:"protected_terms"`
ExpectedModuleInstance []string `json:"expected_module_instances"`
MinimumCounts struct {
Applied int `json:"applied"`
Rejected int `json:"rejected"`
Skipped int `json:"skipped"`
} `json:"minimum_counts"`
}
func TestReleaseFixtureDefaultPipelineReadiness(t *testing.T) {
base := fixturePath(filepath.Join("release", "default-release"))
var expectations releaseFixtureExpectations
if err := json.Unmarshal(readFile(t, base+".expectations.json"), &expectations); err != nil {
t.Fatalf("unmarshal release expectations: %v", err)
}
proposalResponses := readProposalResponses(t, base+".proposals.json")
validationResponses := readValidationResponses(t, base+".validations.json")
// First pass: default full pipeline with deterministic fake LLM responses.
first := runReleaseFixturePass(t, releaseRunConfig{
transcriptPath: base + ".transcript.json",
glossaryPath: base + ".glossary.yaml",
outputSchema: "bare-segments",
proposalResponses: proposalResponses,
validationResponses: validationResponses,
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
reportSchemaName: reporting.DefaultProcessReportSchemaName,
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
expectedOutputSchema: "bare-segments",
expectModuleInstances: expectations.ExpectedModuleInstance,
expectUtilizationPaths: true,
})
gotTranscript := mustReadTranscript(t, first.outputPath)
expectFinalTranscriptContains(t, gotTranscript, expectations.MustApplyTexts)
expectFinalTranscriptDoesNotContain(t, gotTranscript, expectations.MustNotApplyTexts)
expectFinalTranscriptContains(t, gotTranscript, expectations.ProtectedTerms)
assertReleaseCounts(t, first.report, expectations)
assertPromptAndSchemaMetadataPresent(t, first.runDir)
assertReleaseLedgerShape(t, first.report)
assertReleaseUtilizationShape(t, first.report)
assertStableValidatorKeysPresent(t, first.report)
assertStdoutStderrContract(t, first.stdout, first.stderr)
assertNoSecretMarkersInTree(t, first.runDir, []string{"release-secret"})
assertNoSecretMarkers(t, first.reportPath, []string{"release-secret"})
// Output schema check: audita-v1 object payload.
auditaV1 := runReleaseFixturePass(t, releaseRunConfig{
transcriptPath: base + ".transcript.json",
glossaryPath: base + ".glossary.yaml",
outputSchema: "audita-v1",
proposalResponses: proposalResponses,
validationResponses: validationResponses,
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
reportSchemaName: reporting.DefaultProcessReportSchemaName,
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
expectedOutputSchema: "audita-v1",
expectModuleInstances: expectations.ExpectedModuleInstance,
expectUtilizationPaths: true,
})
assertAuditaV1OutputShape(t, auditaV1.outputPath)
// Idempotence-oriented second pass:
// run again on first output with deterministic no-op responses.
noOpProposals := make([]proposal_generation.StructuredCorrectionSet, 5)
for i := range noOpProposals {
noOpProposals[i] = proposal_generation.StructuredCorrectionSet{Corrections: nil}
}
second := runReleaseFixturePass(t, releaseRunConfig{
transcriptPath: first.outputPath,
glossaryPath: base + ".glossary.yaml",
outputSchema: "bare-segments",
proposalResponses: noOpProposals,
validationResponses: nil,
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
reportSchemaName: reporting.DefaultProcessReportSchemaName,
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
expectedOutputSchema: "bare-segments",
expectModuleInstances: expectations.ExpectedModuleInstance,
expectUtilizationPaths: true,
})
firstSegments := mustReadTranscript(t, first.outputPath)
secondSegments := mustReadTranscript(t, second.outputPath)
if !reflect.DeepEqual(firstSegments, secondSegments) {
t.Fatalf("expected idempotent second pass transcript; first=%+v second=%+v", firstSegments, secondSegments)
}
if second.report.ModulesSummary == nil {
t.Fatalf("expected modules summary on second pass")
}
if second.report.ModulesSummary.TotalAppliedChanges != 0 {
t.Fatalf("expected no-op second pass (0 applied), got %+v", second.report.ModulesSummary)
}
}
type releaseRunConfig struct {
transcriptPath string
glossaryPath string
outputSchema string
proposalResponses []proposal_generation.StructuredCorrectionSet
validationResponses []validators.LLMValidationResponse
expectedProposalCalls []string
reportSchemaName string
reportSchemaVersion string
expectedOutputSchema string
expectModuleInstances []string
expectUtilizationPaths bool
}
type releaseRunResult struct {
stdout string
stderr string
outputPath string
reportPath string
report reporting.ProcessReport
runDir string
}
func runReleaseFixturePass(t *testing.T, cfg releaseRunConfig) releaseRunResult {
t.Helper()
processProposalLLMClient = &fakeStructuredLLMClient{proposalResponses: append([]proposal_generation.StructuredCorrectionSet(nil), cfg.proposalResponses...)}
processValidationLLMClient = &fakeStructuredLLMClient{validationResponses: append([]validators.LLMValidationResponse(nil), cfg.validationResponses...)}
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
configPath := writeFile(t, "release-config.yml", "version: 1\n")
args := []string{
"process",
cfg.transcriptPath,
"--glossary",
cfg.glossaryPath,
"--config",
configPath,
"--output",
outputPath,
"--output-schema",
cfg.outputSchema,
"--report-json",
reportPath,
"--work-dir",
workDir,
"--work-dir-retention",
"always",
}
var stdout, stderr bytes.Buffer
exitCode := Run(args, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
report := readProcessReport(t, reportPath)
if report.ReportMetadata.ReportSchemaName != cfg.reportSchemaName || report.ReportMetadata.ReportSchemaVersion != cfg.reportSchemaVersion {
t.Fatalf("unexpected report schema metadata: %+v", report.ReportMetadata)
}
if report.ReportMetadata.OutputSchema != cfg.expectedOutputSchema {
t.Fatalf("unexpected output schema metadata: got %q want %q", report.ReportMetadata.OutputSchema, cfg.expectedOutputSchema)
}
if len(cfg.expectModuleInstances) > 0 {
got := make([]string, 0, len(report.ModuleResults))
for _, mr := range report.ModuleResults {
got = append(got, mr.ModuleInstance)
}
if !reflect.DeepEqual(got, cfg.expectModuleInstances) {
t.Fatalf("unexpected module instances: got %v want %v", got, cfg.expectModuleInstances)
}
}
if report.Diagnostics == nil {
t.Fatalf("expected diagnostics metadata")
}
if cfg.expectUtilizationPaths {
if report.Diagnostics.UtilizationSummaryPath == "" || report.Diagnostics.CorrectionLedgerPath == "" {
t.Fatalf("expected utilization/ledger artifact paths in report diagnostics: %+v", report.Diagnostics)
}
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil {
t.Fatalf("expected run-dir report: %v", err)
}
if c, ok := processProposalLLMClient.(*fakeStructuredLLMClient); ok {
if !reflect.DeepEqual(c.calls, cfg.expectedProposalCalls) {
t.Fatalf("unexpected proposal call order: got %v want %v", c.calls, cfg.expectedProposalCalls)
}
}
return releaseRunResult{
stdout: stdout.String(),
stderr: stderr.String(),
outputPath: outputPath,
reportPath: reportPath,
report: report,
runDir: runDir,
}
}
func readProposalResponses(t *testing.T, path string) []proposal_generation.StructuredCorrectionSet {
t.Helper()
var out []proposal_generation.StructuredCorrectionSet
if err := json.Unmarshal(readFile(t, path), &out); err != nil {
t.Fatalf("unmarshal proposal responses: %v", err)
}
return out
}
func readValidationResponses(t *testing.T, path string) []validators.LLMValidationResponse {
t.Helper()
var out []validators.LLMValidationResponse
if err := json.Unmarshal(readFile(t, path), &out); err != nil {
t.Fatalf("unmarshal validation responses: %v", err)
}
return out
}
func mustReadTranscript(t *testing.T, path string) []schema.Segment {
t.Helper()
transcript, err := schema.ParseTranscriptJSON(readFile(t, path))
if err != nil {
t.Fatalf("parse transcript output: %v", err)
}
return transcript.Segments
}
func expectFinalTranscriptContains(t *testing.T, segments []schema.Segment, needles []string) {
t.Helper()
joined := flattenTranscriptText(segments)
for _, needle := range needles {
if !strings.Contains(joined, needle) {
t.Fatalf("expected transcript to contain %q, got %q", needle, joined)
}
}
}
func expectFinalTranscriptDoesNotContain(t *testing.T, segments []schema.Segment, needles []string) {
t.Helper()
joined := flattenTranscriptText(segments)
for _, needle := range needles {
if strings.Contains(joined, needle) {
t.Fatalf("expected transcript to not contain %q, got %q", needle, joined)
}
}
}
func flattenTranscriptText(segments []schema.Segment) string {
parts := make([]string, 0, len(segments))
for _, s := range segments {
parts = append(parts, s.Text)
}
return strings.Join(parts, "\n")
}
func assertReleaseCounts(t *testing.T, report reporting.ProcessReport, exp releaseFixtureExpectations) {
t.Helper()
if report.ModulesSummary == nil {
t.Fatalf("expected modules_summary")
}
if report.ModulesSummary.TotalAppliedChanges < exp.MinimumCounts.Applied {
t.Fatalf("expected at least %d applied changes, got %+v", exp.MinimumCounts.Applied, report.ModulesSummary)
}
validatorRejected := 0
skipped := 0
for _, mr := range report.ModuleResults {
validatorRejected += len(mr.ValidatorRejected)
skipped += len(mr.SkippedChanges)
}
if validatorRejected < exp.MinimumCounts.Rejected {
t.Fatalf("expected at least %d validator rejections, got %d", exp.MinimumCounts.Rejected, validatorRejected)
}
if skipped < exp.MinimumCounts.Skipped {
t.Fatalf("expected at least %d application skips, got %d", exp.MinimumCounts.Skipped, skipped)
}
}
func assertReleaseUtilizationShape(t *testing.T, report reporting.ProcessReport) {
t.Helper()
var payload struct {
EffectiveConcurrency struct {
TotalLLM int `json:"total_llm"`
} `json:"effective_concurrency"`
RunTiming struct {
SchedulerQueueWaitMS int64 `json:"scheduler_queue_wait_ms"`
LLMExecutionTimeMS int64 `json:"llm_execution_time_ms"`
DeterministicValidationMS int64 `json:"deterministic_validation_time_ms"`
} `json:"run_timing"`
Modules []map[string]any `json:"modules"`
Validators []map[string]any `json:"validators"`
}
if err := json.Unmarshal(readFile(t, report.Diagnostics.UtilizationSummaryPath), &payload); err != nil {
t.Fatalf("unmarshal utilization diagnostics: %v", err)
}
if payload.EffectiveConcurrency.TotalLLM <= 0 {
t.Fatalf("expected positive total llm concurrency, got %+v", payload.EffectiveConcurrency)
}
if payload.RunTiming.SchedulerQueueWaitMS < 0 || payload.RunTiming.LLMExecutionTimeMS < 0 || payload.RunTiming.DeterministicValidationMS < 0 {
t.Fatalf("expected non-negative run timing values, got %+v", payload.RunTiming)
}
if len(payload.Modules) == 0 {
t.Fatalf("expected module timing summaries")
}
if len(payload.Validators) == 0 {
t.Fatalf("expected validator timing summaries")
}
}
func assertReleaseLedgerShape(t *testing.T, report reporting.ProcessReport) {
t.Helper()
var entries []struct {
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ProposalIndex int `json:"proposal_index"`
Disposition string `json:"disposition"`
DispositionReason string `json:"disposition_reason_code"`
OriginalText string `json:"original_text"`
ProposedCorrected string `json:"proposed_corrected_text"`
ReplacementPolicy string `json:"replacement_policy"`
DeterministicResults []struct {
ValidatorKey string `json:"validator_key"`
} `json:"deterministic_validator_decisions"`
LLMResults []struct {
ValidatorKey string `json:"validator_key"`
} `json:"llm_validator_decisions"`
}
if err := json.Unmarshal(readFile(t, report.Diagnostics.CorrectionLedgerPath), &entries); err != nil {
t.Fatalf("unmarshal correction ledger: %v", err)
}
if len(entries) == 0 {
t.Fatalf("expected correction ledger entries")
}
hasApplied := false
hasRejected := false
hasSkipped := false
for _, entry := range entries {
if entry.ModuleInstance == "" || entry.ModuleKey == "" {
t.Fatalf("expected module identity in ledger entry: %+v", entry)
}
switch entry.Disposition {
case "applied":
hasApplied = true
case "rejected":
hasRejected = true
case "skipped":
hasSkipped = true
}
}
if !hasApplied || !hasRejected {
t.Fatalf("expected applied and rejected entries in correction ledger, got %+v", entries)
}
if !hasSkipped {
// Some deterministic fixture paths do not trigger apply-time skips;
// rejections are still captured separately from application skips.
}
}
func assertPromptAndSchemaMetadataPresent(t *testing.T, runDir string) {
t.Helper()
metadataPaths, err := filepath.Glob(filepath.Join(runDir, "*", "*request-metadata.json"))
if err != nil {
t.Fatalf("glob request metadata artifacts: %v", err)
}
if len(metadataPaths) == 0 {
t.Fatalf("expected request metadata artifacts with prompt metadata")
}
foundPromptMetadata := false
foundSchemaMetadata := false
for _, path := range metadataPaths {
var payload map[string]any
if err := json.Unmarshal(readFile(t, path), &payload); err != nil {
t.Fatalf("unmarshal request metadata artifact %q: %v", path, err)
}
if pm, ok := payload["prompt_metadata"].(map[string]any); ok {
if pm["prompt_id"] != nil && pm["prompt_version"] != nil && pm["sha256"] != nil {
foundPromptMetadata = true
}
}
if sm, ok := payload["response_schema"].(map[string]any); ok {
if sm["id"] != nil && sm["version"] != nil && sm["name"] != nil && sm["sha256"] != nil {
foundSchemaMetadata = true
}
}
}
if !foundPromptMetadata {
t.Fatalf("expected prompt metadata in request metadata artifacts")
}
if !foundSchemaMetadata {
t.Fatalf("expected structured response schema metadata in request metadata artifacts")
}
}
func assertStableValidatorKeysPresent(t *testing.T, report reporting.ProcessReport) {
t.Helper()
seen := map[string]bool{}
for _, module := range report.ModuleResults {
for _, decision := range module.ValidatorDecisions {
seen[decision.ValidatorName] = true
}
for _, rejected := range module.ValidatorRejected {
seen[rejected.ValidatorName] = true
}
}
expectedAny := []string{
"confidence_threshold",
"original_text_presence",
"no_effect",
}
for _, key := range expectedAny {
if !seen[key] {
t.Fatalf("expected stable validator key %q in report decisions/rejections; seen=%v", key, seen)
}
}
}
func assertStdoutStderrContract(t *testing.T, stdout, stderr string) {
t.Helper()
if stdout != "" {
t.Fatalf("expected empty stdout with --output, got %q", stdout)
}
if strings.Contains(stderr, `"module_results"`) || strings.Contains(stderr, `"report_metadata"`) {
t.Fatalf("stderr should remain human-readable, not report JSON: %q", stderr)
}
}
func assertAuditaV1OutputShape(t *testing.T, outputPath string) {
t.Helper()
var payload struct {
Schema string `json:"schema"`
Version string `json:"version"`
Segments []schema.Segment `json:"segments"`
}
if err := json.Unmarshal(readFile(t, outputPath), &payload); err != nil {
t.Fatalf("unmarshal audita-v1 output: %v", err)
}
if payload.Schema != "audita-v1" || payload.Version != "v1" {
t.Fatalf("unexpected audita-v1 metadata: %+v", payload)
}
if len(payload.Segments) == 0 {
t.Fatalf("expected non-empty audita-v1 segments")
}
}

867
internal/cli/run.go Normal file
View File

@@ -0,0 +1,867 @@
package cli
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
coreio "gitea.maximumdirect.net/eric/audita/internal/core/io"
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
"gitea.maximumdirect.net/eric/audita/internal/framework/processreport"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
type noOpStructuredLLMClient struct{}
func (c noOpStructuredLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
_ = req
switch target := out.(type) {
case *validators.LLMValidationResponse:
*target = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{}}
case *proposal_generation.StructuredCorrectionSet:
*target = proposal_generation.StructuredCorrectionSet{Corrections: nil}
}
return contracts.StructuredCompletionResponse{}, nil
}
func shouldUseNoOpLLMClientForTests() bool {
return strings.HasSuffix(filepath.Base(os.Args[0]), ".test") || os.Getenv("GO_WANT_HELPER_PROCESS") == "1"
}
type processInvocation struct {
TranscriptPath string
GlossaryPath string
OutputPath string
ReportJSONPath string
Config config.Config
ConfigPath string
ConfigSource string
ConfigVersion *int
ExplicitModules bool
}
var processModuleFactory runner.ModuleFactory
var processProposalLLMClient contracts.StructuredLLMClient
var processProposalLLMScheduler runner.ValidationScheduler
var processValidationLLMClient contracts.StructuredLLMClient
var processValidationLLMScheduler runner.ValidationScheduler
var processRunnerContext = func() (context.Context, context.CancelFunc) {
return context.Background(), func() {}
}
var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) {
runDir, err := diagnostics.NewRunDirectory(inv.Config.WorkDir, string(inv.Config.WorkDirRetention))
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("run_dir_creation: %w", err)
}
fail := func(phase string, err error, runOutput *runner.RunOutput) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) {
_ = runDir.WriteErrorLog(fmt.Sprintf("%s: %v", phase, err))
return nil, nil, runOutput, runDir, fmt.Errorf("%s: %w", phase, err)
}
if err := runDir.WriteInvocationMetadata(diagnostics.InvocationMetadata{
Operation: "process",
TranscriptPath: inv.TranscriptPath,
GlossaryPath: inv.GlossaryPath,
OutputPath: inv.OutputPath,
ReportJSONPath: inv.ReportJSONPath,
ConfigPath: inv.ConfigPath,
ConfigSource: inv.ConfigSource,
ConfigVersion: inv.ConfigVersion,
TranscriptDescription: inv.Config.TranscriptDescription,
Modules: append([]string(nil), inv.Config.Modules...),
}); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("invocation_metadata: %v", err))
}
if err := runDir.WriteEffectiveConfig(inv.Config); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("effective_config: %v", err))
}
transcriptBytes, err := coreio.ReadRequiredFile(inv.TranscriptPath, "transcript")
if err != nil {
return fail("transcript_read", err, nil)
}
glossaryBytes, err := coreio.ReadRequiredFile(inv.GlossaryPath, "glossary")
if err != nil {
return fail("glossary_read", err, nil)
}
sourceTranscript, err := schema.ParseSourceTranscriptJSON(transcriptBytes)
if err != nil {
return fail("transcript_schema", err, nil)
}
glossary, err := schema.ParseGlossaryYAML(glossaryBytes)
if err != nil {
return fail("glossary_schema", err, nil)
}
if err := runDir.WriteSourceTranscript(sourceTranscript, transcriptBytes); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("source_artifact: %v", err))
}
canonical := sourceToCanonicalTranscript(sourceTranscript)
normalizer := normalization.NewNormalizer(normalization.NormalizationConfig{
MaxSegmentGap: inv.Config.Normalization.MaxSegmentGap,
EllipsisGap: inv.Config.Normalization.EllipsisGap,
MaxSegmentDuration: inv.Config.Normalization.MaxSegmentDuration,
MaxSegmentTokens: inv.Config.Normalization.MaxSegmentTokens,
})
normalizedTranscript, normSummary := normalizer.Normalize(canonical)
if err := runDir.WriteNormalizedTranscript(normalizedTranscript); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("normalized_artifact: %v", err))
}
if err := runDir.WriteNormalizationSummary(normSummary); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("normalization_summary: %v", err))
}
// Compute chunks after normalization
chunker := chunking.NewChunker(chunking.ChunkingConfig{
MaxSectionTokens: inv.Config.MaxSectionTokens,
MinSectionTokens: inv.Config.MinSectionTokens,
TargetSections: inv.Config.TargetSections,
})
sections, chunkErr := chunker.ChunkTranscript(normalizedTranscript)
if chunkErr != nil {
return fail("chunking", chunkErr, nil)
}
chunkConfig := chunking.ChunkingConfig{
MaxSectionTokens: inv.Config.MaxSectionTokens,
MinSectionTokens: inv.Config.MinSectionTokens,
TargetSections: inv.Config.TargetSections,
}
chunkSummary := chunking.ComputeSummary(sections, chunkConfig)
chunkDetailedSummary := chunking.ComputeDetailedSummary(sections, chunkConfig)
if err := runDir.WriteChunkingSummary(&chunkDetailedSummary); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("chunking_summary: %v", err))
}
workingTranscript := normalizedTranscript
var runOutput *runner.RunOutput
moduleFactory := processModuleFactory
if moduleFactory == nil {
moduleFactory = modules.NewFactory(modules.Dependencies{
Config: &inv.Config,
Glossary: glossary,
DiagnosticsDir: runDir.Path(),
})
}
if moduleFactory != nil {
proposalLLMClient := processProposalLLMClient
validationLLMClient := processValidationLLMClient
proposalScheduler := processProposalLLMScheduler
validationScheduler := processValidationLLMScheduler
if processModuleFactory == nil {
// Production runtime path: construct clients/schedulers from config.
if proposalLLMClient == nil {
if shouldUseNoOpLLMClientForTests() {
proposalLLMClient = noOpStructuredLLMClient{}
} else {
primaryCfg := llm.ResolvePrimaryConfig(inv.Config)
client, clientErr := llm.NewOpenAICompatibleClient(primaryCfg.ToOpenAICompatibleClientConfig(nil))
if clientErr != nil {
return fail("runner_setup", clientErr, nil)
}
proposalLLMClient = client
}
}
if validationLLMClient == nil {
if shouldUseNoOpLLMClientForTests() {
validationLLMClient = noOpStructuredLLMClient{}
} else {
validationCfg := llm.ResolveValidationConfig(inv.Config)
client, clientErr := llm.NewOpenAICompatibleClient(validationCfg.ToOpenAICompatibleClientConfig(nil))
if clientErr != nil {
return fail("runner_setup", clientErr, nil)
}
validationLLMClient = client
}
}
globalScheduler := proposalScheduler
if globalScheduler == nil {
s, sErr := llm.NewScheduler(inv.Config.TotalLLMConcurrency)
if sErr != nil {
return fail("runner_setup", sErr, nil)
}
globalScheduler = s
}
if proposalScheduler == nil {
proposalScheduler = globalScheduler
if inv.Config.EffectiveProposalLLMConcurrency() < inv.Config.TotalLLMConcurrency {
s, sErr := llm.NewScheduler(inv.Config.EffectiveProposalLLMConcurrency())
if sErr != nil {
return fail("runner_setup", sErr, nil)
}
proposalScheduler = composeSchedulers(globalScheduler, s)
}
}
if validationScheduler == nil {
validationScheduler = globalScheduler
if inv.Config.ValidationLLMConcurrency != nil && inv.Config.EffectiveValidationLLMConcurrency() < inv.Config.TotalLLMConcurrency {
s, sErr := llm.NewScheduler(inv.Config.EffectiveValidationLLMConcurrency())
if sErr != nil {
return fail("runner_setup", sErr, nil)
}
validationScheduler = composeSchedulers(globalScheduler, s)
}
}
}
moduleSpecs, err := contracts.ResolveModuleRunSpecs(inv.Config.Modules)
if err != nil {
return fail("runner_setup", err, nil)
}
runCtx, cancelRun := processRunnerContext()
defer cancelRun()
runnerResult, runErr := runner.New(moduleFactory).Run(runCtx, runner.RunInput{
Config: &inv.Config,
Transcript: normalizedTranscript,
Glossary: glossary,
ModuleSpecs: moduleSpecs,
EffectiveConcurrency: runner.EffectiveConcurrencyLimits{
TotalLLM: inv.Config.TotalLLMConcurrency,
ProposalLLM: inv.Config.EffectiveProposalLLMConcurrency(),
ValidationLLM: inv.Config.EffectiveValidationLLMConcurrency(),
},
ProposalLLMClient: proposalLLMClient,
ProposalLLMScheduler: proposalScheduler,
ProposalDiagnosticsDir: runDir.Path(),
ValidationLLMClient: validationLLMClient,
ValidationLLMScheduler: validationScheduler,
ValidationDiagnosticsDir: runDir.Path(),
})
runOutput = &runnerResult
if runErr != nil {
return fail("runner_execution", runErr, runOutput)
}
workingTranscript = runnerResult.FinalTranscript
}
encoderDef, err := outputschema.Resolve(inv.Config.OutputSchema)
if err != nil {
return fail("output_schema", err, runOutput)
}
outputBytes, err := encoderDef.Encoder(workingTranscript)
if err != nil {
return fail("serialization", err, runOutput)
}
if strings.TrimSpace(inv.OutputPath) != "" {
if err := coreio.WriteFile(inv.OutputPath, outputBytes); err != nil {
return fail("output_write", err, runOutput)
}
return normSummary, &chunkSummary, runOutput, runDir, nil
}
if _, err := stdout.Write(outputBytes); err != nil {
return fail("stdout_write", err, runOutput)
}
return normSummary, &chunkSummary, runOutput, runDir, nil
}
func sourceToCanonicalTranscript(source *schema.SourceTranscript) *schema.Transcript {
segments := make([]schema.Segment, len(source.Segments))
for i, s := range source.Segments {
id := i + 1
if s.ID != nil {
id = *s.ID
}
segments[i] = schema.Segment{
ID: id,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: s.Categories,
}
}
return &schema.Transcript{Segments: segments}
}
type chainedScheduler struct {
schedulers []runner.ValidationScheduler
}
func (s chainedScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
if len(s.schedulers) == 0 {
return fn(ctx)
}
run := fn
for i := len(s.schedulers) - 1; i >= 0; i-- {
scheduler := s.schedulers[i]
next := run
run = func(callCtx context.Context) error {
return scheduler.Run(callCtx, next)
}
}
return run(ctx)
}
func composeSchedulers(schedulers ...runner.ValidationScheduler) runner.ValidationScheduler {
filtered := make([]runner.ValidationScheduler, 0, len(schedulers))
for _, scheduler := range schedulers {
if scheduler != nil {
filtered = append(filtered, scheduler)
}
}
switch len(filtered) {
case 0:
return nil
case 1:
return filtered[0]
default:
return chainedScheduler{schedulers: filtered}
}
}
// Run executes the Audita CLI with the provided arguments and streams.
func Run(args []string, stdout, stderr io.Writer) int {
if len(args) == 0 {
writeRootUsage(stdout)
return 0
}
if isHelpCommand(args) {
writeRootUsage(stdout)
return 0
}
if args[0] == "process" {
return runProcess(args[1:], stdout, stderr)
}
if args[0] == "config" {
return runConfig(args[1:], stdout, stderr)
}
fmt.Fprintf(stderr, "audita: unknown command %q\n\n", args[0])
writeRootUsage(stderr)
return 2
}
func runProcess(args []string, stdout, stderr io.Writer) int {
startedAt := time.Now().UTC()
configPathOverride, configPathOverrideSet, err := findConfigPathOverride(args)
if err != nil {
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
return 2
}
effectiveConfig, err := config.LoadEffectiveConfig(configPathOverride, configPathOverrideSet)
if err != nil {
var effectiveConfigErr *config.EffectiveConfigError
if errors.As(err, &effectiveConfigErr) {
switch effectiveConfigErr.Kind {
case config.EffectiveConfigErrorLoadFile, config.EffectiveConfigErrorApplyFile:
fmt.Fprintf(stderr, "audita process: invalid config file: %v\n", effectiveConfigErr)
case config.EffectiveConfigErrorApplyEnv:
fmt.Fprintf(stderr, "audita process: invalid environment configuration: %v\n", effectiveConfigErr)
default:
fmt.Fprintf(stderr, "audita process: %v\n", effectiveConfigErr)
}
} else {
fmt.Fprintf(stderr, "audita process: %v\n", err)
}
return 2
}
cfg := effectiveConfig.Config
configPath := effectiveConfig.ConfigPath
configSource := effectiveConfig.ConfigSource
configVersion := effectiveConfig.ConfigVersion
fs, pFlags := newProcessFlagSet(cfg, stderr)
if isHelpCommand(args) || hasHelpFlag(args) {
writeProcessUsage(stdout, fs)
return 0
}
parseArgs := args
transcriptFromFront := ""
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
transcriptFromFront = args[0]
parseArgs = args[1:]
}
if err := fs.Parse(parseArgs); err != nil {
if errors.Is(err, flag.ErrHelp) {
writeProcessUsage(stdout, fs)
return 0
}
return 2
}
overrides, explicitModules := processCLIOverrides(fs, pFlags)
if err := cfg.ApplyCLIOverrides(overrides); err != nil {
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
return 2
}
remaining := fs.Args()
positional := make([]string, 0, len(remaining)+1)
if transcriptFromFront != "" {
positional = append(positional, transcriptFromFront)
}
positional = append(positional, remaining...)
if len(positional) != 1 {
fmt.Fprintln(stderr, "audita process: expected exactly 1 transcript JSON path argument")
return 2
}
if strings.TrimSpace(*pFlags.glossaryPath) == "" {
fmt.Fprintln(stderr, "audita process: --glossary is required")
return 2
}
inv := processInvocation{
TranscriptPath: positional[0],
GlossaryPath: *pFlags.glossaryPath,
OutputPath: *pFlags.outputPath,
ReportJSONPath: *pFlags.reportJSONPath,
Config: cfg,
ConfigPath: configPath,
ConfigSource: configSource,
ConfigVersion: configVersion,
ExplicitModules: explicitModules,
}
normSummary, chunkSummary, runOutput, runDir, runErr := processRunner(inv, stdout)
completedAt := time.Now().UTC()
if runErr != nil {
if runDir != nil && runOutput != nil {
if runOutput.Utilization != nil {
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
}
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
RunDirectoryPath: runDir.Path(),
RunOutput: runOutput,
}))
}
errorPhase, errorMessage := extractErrorPhase(runErr)
report := processreport.Build(processReportInput("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput))
if strings.TrimSpace(inv.ReportJSONPath) != "" {
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
fmt.Fprintf(stderr, "audita process: %v\n", err)
}
}
if runDir != nil {
_ = runDir.WriteReport(report)
_ = runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
RunSucceeded: false,
})
}
fmt.Fprintf(stderr, "audita process: %v\n", runErr)
if runDir != nil {
fmt.Fprintf(stderr, "audita process: diagnostics: %s\n", runDir.Path())
}
return 1
}
if runDir != nil && runOutput != nil {
if runOutput.Utilization != nil {
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
}
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
RunDirectoryPath: runDir.Path(),
RunOutput: runOutput,
}))
}
report := processreport.Build(processReportInput("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput))
if strings.TrimSpace(inv.ReportJSONPath) != "" {
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
if runDir != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("report_write: %v", err))
_ = runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
RunSucceeded: false,
})
}
fmt.Fprintf(stderr, "audita process: %v\n", err)
return 1
}
}
if runDir != nil {
_ = runDir.WriteReport(report)
if err := runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
RunSucceeded: true,
HasSkippedCorrections: processreport.HasSkippedCorrections(runOutput),
}); err != nil {
fmt.Fprintf(stderr, "audita process: failed to apply work-dir retention: %v\n", err)
return 1
}
}
return 0
}
func runConfig(args []string, stdout, stderr io.Writer) int {
if len(args) == 0 || isHelpCommand(args) || hasHelpFlag(args) {
writeConfigUsage(stdout)
return 0
}
switch args[0] {
case "validate":
return runConfigValidate(args[1:], stdout, stderr)
case "print-effective":
return runConfigPrintEffective(args[1:], stdout, stderr)
default:
fmt.Fprintf(stderr, "audita config: unknown command %q\n\n", args[0])
writeConfigUsage(stderr)
return 2
}
}
func runConfigValidate(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("config validate", flag.ContinueOnError)
fs.SetOutput(stderr)
configPath := fs.String("config", "", "Path to versioned YAML config file")
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
writeConfigValidateUsage(stdout)
return 0
}
return 2
}
if strings.TrimSpace(*configPath) == "" {
fmt.Fprintln(stderr, "audita config validate: --config is required")
return 2
}
if len(fs.Args()) != 0 {
fmt.Fprintln(stderr, "audita config validate: unexpected positional arguments")
return 2
}
fileCfg, err := config.LoadFileConfig(strings.TrimSpace(*configPath))
if err != nil {
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
return 2
}
cfg := config.Default()
if err := cfg.ApplyFileConfig(fileCfg); err != nil {
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
return 2
}
if err := cfg.Validate(); err != nil {
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
return 2
}
fmt.Fprintln(stdout, "config is valid")
return 0
}
func runConfigPrintEffective(args []string, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("config print-effective", flag.ContinueOnError)
fs.SetOutput(stderr)
configPath := fs.String("config", "", "Path to versioned YAML config file")
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
writeConfigPrintEffectiveUsage(stdout)
return 0
}
return 2
}
if len(fs.Args()) != 0 {
fmt.Fprintln(stderr, "audita config print-effective: unexpected positional arguments")
return 2
}
configPathValue := strings.TrimSpace(*configPath)
configPathSet := configPathValue != ""
effectiveConfig, err := config.LoadEffectiveConfig(configPathValue, configPathSet)
if err != nil {
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
return 2
}
cfg := effectiveConfig.Config
redacted := cfg.Redacted()
out, err := json.MarshalIndent(redacted, "", " ")
if err != nil {
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
return 1
}
out = append(out, '\n')
if _, err := stdout.Write(out); err != nil {
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
return 1
}
return 0
}
func extractErrorPhase(err error) (phase string, message string) {
msg := err.Error()
if strings.Contains(msg, ": ") {
parts := strings.SplitN(msg, ": ", 2)
if len(parts) == 2 {
return parts[0], parts[1]
}
}
return "", msg
}
func processReportInput(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) processreport.BuildInput {
runDirectoryPath := ""
if runDir != nil {
runDirectoryPath = runDir.Path()
}
return processreport.BuildInput{
Status: status,
TranscriptPath: inv.TranscriptPath,
GlossaryPath: inv.GlossaryPath,
OutputPath: inv.OutputPath,
Modules: inv.Config.Modules,
OutputSchema: inv.Config.OutputSchema,
ConfigVersion: inv.ConfigVersion,
StartedAt: startedAt,
CompletedAt: completedAt,
ErrorMessage: errorMessage,
ErrorPhase: errorPhase,
RunDirectoryPath: runDirectoryPath,
NormalizationSummary: normalizationSummary,
ChunkingSummary: chunkingSummary,
RunOutput: runOutput,
}
}
type processFlags struct {
configPath *string
glossaryPath *string
outputPath *string
reportJSONPath *string
modules *string
outputSchema *string
llmAPIKey *string
validationLLMAPIKey *string
model *string
validationModel *string
baseURL *string
validationBaseURL *string
llmTimeoutSeconds *int
totalLLMConcurrency *int
proposalLLMConcurrency *int
llmConcurrency *int
validationLLMTimeoutSeconds *int
validationMaxPromptTokens *int
targetSections *int
maxRetries *int
validationMaxRetries *int
validationLLMConcurrency *int
maxSectionTokens *int
minSectionTokens *int
glossaryConfidenceThreshold *float64
grammarConfidenceThreshold *float64
homophonesConfidenceThreshold *float64
spokenWordConfidenceThreshold *float64
normalizeMaxSegmentGap *float64
normalizeEllipsisGap *float64
normalizeMaxSegmentDuration *float64
normalizeMaxSegmentTokens *int
transcriptDescription *string
workDir *string
workDirRetention *string
}
func newProcessFlagSet(cfg config.Config, stderr io.Writer) (*flag.FlagSet, processFlags) {
fs := flag.NewFlagSet("process", flag.ContinueOnError)
fs.SetOutput(stderr)
validationTimeoutSecondsDefault := cfg.PrimaryLLM.TimeoutSeconds
if cfg.ValidationLLM.TimeoutSeconds != nil {
validationTimeoutSecondsDefault = *cfg.ValidationLLM.TimeoutSeconds
}
validationMaxRetriesDefault := cfg.PrimaryLLM.MaxRetries
if cfg.ValidationLLM.MaxRetries != nil {
validationMaxRetriesDefault = *cfg.ValidationLLM.MaxRetries
}
validationLLMConcurrencyDefault := cfg.TotalLLMConcurrency
if cfg.ValidationLLMConcurrency != nil {
validationLLMConcurrencyDefault = *cfg.ValidationLLMConcurrency
}
targetSectionsDefault := 0
if cfg.TargetSections != nil {
targetSectionsDefault = *cfg.TargetSections
}
pFlags := processFlags{
configPath: fs.String("config", "", "Path to versioned YAML config file"),
glossaryPath: fs.String("glossary", "", "Path to glossary YAML file"),
outputPath: fs.String("output", "", "Path to corrected transcript JSON output file"),
reportJSONPath: fs.String("report-json", "", "Path to machine-readable report JSON output file"),
modules: fs.String("modules", strings.Join(cfg.Modules, ","), "Comma-separated module sequence override"),
outputSchema: fs.String("output-schema", cfg.OutputSchema, "Output schema: bare-segments|audita-v1"),
llmAPIKey: fs.String("llm-api-key", cfg.PrimaryLLM.APIKey, "Primary LLM API key"),
validationLLMAPIKey: fs.String("validation-llm-api-key", cfg.ValidationLLM.APIKey, "Validation LLM API key"),
model: fs.String("model", cfg.PrimaryLLM.Model, "Primary LLM model name"),
validationModel: fs.String("validation-model", cfg.ValidationLLM.Model, "Validation LLM model name"),
baseURL: fs.String("base-url", cfg.PrimaryLLM.BaseURL, "Primary OpenAI-compatible base URL"),
validationBaseURL: fs.String("validation-base-url", cfg.ValidationLLM.BaseURL, "Validation OpenAI-compatible base URL"),
llmTimeoutSeconds: fs.Int("llm-timeout-seconds", cfg.PrimaryLLM.TimeoutSeconds, "Primary LLM timeout in seconds"),
totalLLMConcurrency: fs.Int("total-llm-concurrency", cfg.TotalLLMConcurrency, "Total concurrent LLM calls across proposal and validation"),
proposalLLMConcurrency: fs.Int("proposal-llm-concurrency", cfg.EffectiveProposalLLMConcurrency(), "Concurrent proposal-generation LLM calls"),
llmConcurrency: fs.Int("llm-concurrency", cfg.TotalLLMConcurrency, "Alias for --total-llm-concurrency"),
validationLLMTimeoutSeconds: fs.Int("validation-llm-timeout-seconds", validationTimeoutSecondsDefault, "Validation LLM timeout in seconds"),
validationMaxPromptTokens: fs.Int("validation-max-prompt-tokens", cfg.ValidationMaxPromptTokens, "Validation max prompt tokens"),
targetSections: fs.Int("target-sections", targetSectionsDefault, "Target number of transcript sections"),
maxRetries: fs.Int("max-retries", cfg.PrimaryLLM.MaxRetries, "Maximum structured-output retries"),
validationMaxRetries: fs.Int("validation-max-retries", validationMaxRetriesDefault, "Validation structured-output retries"),
validationLLMConcurrency: fs.Int("validation-llm-concurrency", validationLLMConcurrencyDefault, "Concurrent validation LLM calls (inherits total when unset)"),
maxSectionTokens: fs.Int("max-section-tokens", cfg.MaxSectionTokens, "Maximum section tokens"),
minSectionTokens: fs.Int("min-section-tokens", cfg.MinSectionTokens, "Minimum section tokens"),
glossaryConfidenceThreshold: fs.Float64("glossary-confidence-threshold", cfg.Thresholds.Glossary, "Glossary confidence threshold"),
grammarConfidenceThreshold: fs.Float64("grammar-confidence-threshold", cfg.Thresholds.Grammar, "Grammar confidence threshold"),
homophonesConfidenceThreshold: fs.Float64("homophones-confidence-threshold", cfg.Thresholds.Homophones, "Homophones confidence threshold"),
spokenWordConfidenceThreshold: fs.Float64("spoken-word-confidence-threshold", cfg.Thresholds.SpokenWord, "Spoken-word confidence threshold"),
normalizeMaxSegmentGap: fs.Float64("normalize-max-segment-gap", cfg.Normalization.MaxSegmentGap, "Maximum same-speaker merge gap"),
normalizeEllipsisGap: fs.Float64("normalize-ellipsis-gap", cfg.Normalization.EllipsisGap, "Gap threshold for ellipsis insertion"),
normalizeMaxSegmentDuration: fs.Float64("normalize-max-segment-duration", cfg.Normalization.MaxSegmentDuration, "Maximum merged segment duration"),
normalizeMaxSegmentTokens: fs.Int("normalize-max-segment-tokens", cfg.Normalization.MaxSegmentTokens, "Maximum merged segment token estimate"),
transcriptDescription: fs.String("transcript-description", cfg.TranscriptDescription, "Brief background context for LLM prompts; does not override transcript content"),
workDir: fs.String("work-dir", cfg.WorkDir, "Per-run work directory"),
workDirRetention: fs.String("work-dir-retention", string(cfg.WorkDirRetention), "Work-dir retention policy: auto|always|never"),
}
return fs, pFlags
}
func findConfigPathOverride(args []string) (path string, set bool, err error) {
for i := 0; i < len(args); i++ {
arg := strings.TrimSpace(args[i])
if arg == "" {
continue
}
if arg == "--config" {
if i+1 >= len(args) {
return "", false, fmt.Errorf("--config requires a path")
}
return strings.TrimSpace(args[i+1]), true, nil
}
if strings.HasPrefix(arg, "--config=") {
return strings.TrimSpace(strings.TrimPrefix(arg, "--config=")), true, nil
}
}
return "", false, nil
}
func isHelpCommand(args []string) bool {
if len(args) == 0 {
return false
}
if len(args) == 1 {
switch args[0] {
case "help", "-h", "--help":
return true
}
}
if len(args) == 2 && args[0] == "help" {
switch args[1] {
case "process":
return true
}
}
return false
}
func hasHelpFlag(args []string) bool {
for _, arg := range args {
if arg == "-h" || arg == "--help" {
return true
}
}
return false
}
func writeRootUsage(w io.Writer) {
fmt.Fprintln(w, "Audita is a transcript processing CLI.")
fmt.Fprintln(w)
fmt.Fprintln(w, "Usage:")
fmt.Fprintln(w, " audita <command> [options]")
fmt.Fprintln(w)
fmt.Fprintln(w, "Commands:")
fmt.Fprintln(w, " process Process a transcript JSON file")
fmt.Fprintln(w, " config Validate and inspect config")
fmt.Fprintln(w)
fmt.Fprintln(w, "Example:")
fmt.Fprintln(w, " audita process transcript.json --glossary glossary.yaml --output corrected.json")
}
func writeConfigUsage(w io.Writer) {
fmt.Fprintln(w, "Validate and inspect Audita config.")
fmt.Fprintln(w)
fmt.Fprintln(w, "Usage:")
fmt.Fprintln(w, " audita config <command> [flags]")
fmt.Fprintln(w)
fmt.Fprintln(w, "Commands:")
fmt.Fprintln(w, " validate Validate a versioned YAML config file")
fmt.Fprintln(w, " print-effective Print redacted effective config JSON (defaults + config file + env)")
}
func writeConfigValidateUsage(w io.Writer) {
fmt.Fprintln(w, "Validate a versioned YAML config file.")
fmt.Fprintln(w)
fmt.Fprintln(w, "Usage:")
fmt.Fprintln(w, " audita config validate --config <path>")
}
func writeConfigPrintEffectiveUsage(w io.Writer) {
fmt.Fprintln(w, "Print redacted effective config JSON.")
fmt.Fprintln(w)
fmt.Fprintln(w, "Usage:")
fmt.Fprintln(w, " audita config print-effective [--config <path>]")
}
func writeProcessUsage(w io.Writer, fs *flag.FlagSet) {
fmt.Fprintln(w, "Process a transcript JSON file.")
fmt.Fprintln(w)
fmt.Fprintln(w, "Usage:")
fmt.Fprintln(w, " audita process <transcript.json> [flags]")
fmt.Fprintln(w)
fmt.Fprintln(w, "Flags:")
fs.VisitAll(func(f *flag.Flag) {
fmt.Fprintf(w, " --%s\n", f.Name)
})
fmt.Fprintln(w)
fmt.Fprintln(w, "Example:")
fmt.Fprintln(w, " audita process transcript.json --glossary glossary.yaml --output corrected.json")
}

4354
internal/cli/run_test.go Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,110 @@
package cli
import (
"context"
"errors"
"os"
"strconv"
"strings"
"sync"
"time"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
const (
subprocessTestLLMModeEnv = "AUDITA_SUBPROCESS_TEST_LLM_MODE"
subprocessTestRunTimeoutMSEnv = "AUDITA_SUBPROCESS_TEST_RUN_TIMEOUT_MS"
)
// ConfigureSubprocessTestHooksFromEnv enables deterministic test-only hooks for
// subprocess integration tests that run through the Go test binary helper path.
func ConfigureSubprocessTestHooksFromEnv() {
mode := strings.TrimSpace(os.Getenv(subprocessTestLLMModeEnv))
timeoutMSRaw := strings.TrimSpace(os.Getenv(subprocessTestRunTimeoutMSEnv))
// Only activate in explicit subprocess test mode.
if mode == "" && timeoutMSRaw == "" {
return
}
if mode != "" {
client := &subprocessTestLLMClient{mode: mode}
processProposalLLMClient = client
processValidationLLMClient = client
}
if timeoutMSRaw == "" {
return
}
timeoutMS, err := strconv.Atoi(timeoutMSRaw)
if err != nil || timeoutMS <= 0 {
return
}
processRunnerContext = func() (context.Context, context.CancelFunc) {
return context.WithTimeout(context.Background(), time.Duration(timeoutMS)*time.Millisecond)
}
}
type subprocessTestLLMClient struct {
mode string
mu sync.Mutex
proposals int
}
func (c *subprocessTestLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = req
switch c.mode {
case "backend_error":
return contracts.StructuredCompletionResponse{}, errors.New("synthetic backend failure")
case "block_until_cancel":
<-ctx.Done()
return contracts.StructuredCompletionResponse{}, ctx.Err()
case "malformed_structured":
switch target := out.(type) {
case *proposal_generation.StructuredCorrectionSet:
*target = proposal_generation.StructuredCorrectionSet{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y", Confidence: 0.99},
},
}
case *validators.LLMValidationResponse:
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 999, Approved: true, Confidence: 0.9, Reason: "bad index"},
},
}
}
case "mid_pipeline_fail":
if _, ok := out.(*proposal_generation.StructuredCorrectionSet); ok {
c.mu.Lock()
c.proposals++
proposalCall := c.proposals
c.mu.Unlock()
if proposalCall >= 3 {
return contracts.StructuredCompletionResponse{}, errors.New("synthetic mid-pipeline failure")
}
}
switch target := out.(type) {
case *proposal_generation.StructuredCorrectionSet:
*target = proposal_generation.StructuredCorrectionSet{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99},
},
}
case *validators.LLMValidationResponse:
*target = validators.LLMValidationResponse{Validations: nil}
}
default:
switch target := out.(type) {
case *proposal_generation.StructuredCorrectionSet:
*target = proposal_generation.StructuredCorrectionSet{Corrections: nil}
case *validators.LLMValidationResponse:
*target = validators.LLMValidationResponse{Validations: nil}
}
}
return contracts.StructuredCompletionResponse{}, nil
}

View File

@@ -0,0 +1 @@
{"segments":[{"id":1,"text":"oops"}

View File

@@ -0,0 +1,22 @@
{
"name": "ambiguous_match_rejected_before_application",
"transcript_file": "application-skip-ambiguous.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"modules_csv": "homophones",
"proposal_responses_file": "application-skip-ambiguous.proposals.json",
"validation_responses_file": "application-skip-ambiguous.validations.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "application-skip-ambiguous.expected-transcript.json",
"module_instances": ["homophones"],
"module_count": 1,
"total_applied_changes": 0,
"total_skipped_changes": 1,
"module_applied_counts": [0],
"module_rejected_counts": [1],
"module_skip_counts": [0],
"validator_rejected_reason_codes": ["ambiguous_original_text"],
"expected_proposal_calls": ["homophones:proposal"]
}
}

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"the site near another site"}
]

View File

@@ -0,0 +1,3 @@
[
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"the site near another site"}
]

View File

@@ -0,0 +1,4 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

View File

@@ -0,0 +1,28 @@
{
"name": "default_full_pipeline_shape_and_reports",
"transcript_file": "default-full-pipeline.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"proposal_responses_file": "default-full-pipeline.proposals.json",
"validation_responses_file": "default-full-pipeline.validations.json",
"env": {
"AUDITA_LLM_API_KEY": "parity-secret",
"AUDITA_VALIDATION_LLM_API_KEY": "parity-secret"
},
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "default-full-pipeline.expected-transcript.json",
"module_instances": ["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
"module_count": 5,
"total_applied_changes": 3,
"total_skipped_changes": 3,
"secret_markers": ["parity-secret"],
"expected_proposal_calls": [
"glossary_1:proposal",
"homophones:proposal",
"glossary_2:proposal",
"spoken_word:proposal",
"grammar:proposal"
]
}
}

View File

@@ -0,0 +1,9 @@
[
{
"id": 1,
"speaker": "Alice",
"start": 0,
"end": 1,
"text": "Hello, there were Jesters hmm"
}
]

View File

@@ -0,0 +1,7 @@
glossary:
- name: Jesters
aliases:
- jester
plural: jesters
category: faction
summary: A protected in-world faction term.

View File

@@ -0,0 +1,28 @@
[
{
"corrections": [
{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "Jesters", "corrected_text": "jesters", "confidence": 0.99},
{"id": 1, "original_text": "Jesters", "corrected_text": "JESTERX", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "jesters", "corrected_text": "JESTERS", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "uh", "corrected_text": "hmm", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
]
}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
]

View File

@@ -0,0 +1,12 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject cleanup"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "parity-secret"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

View File

@@ -0,0 +1,27 @@
{
"name": "default_pipeline_handoff_and_module_order",
"transcript_file": "default-handoff.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"proposal_responses_file": "default-handoff.proposals.json",
"validation_responses_file": "default-handoff.validations.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "default-handoff.expected-transcript.json",
"module_instances": ["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
"module_count": 5,
"total_applied_changes": 5,
"total_skipped_changes": 0,
"module_applied_counts": [1, 1, 1, 1, 1],
"module_rejected_counts": [0, 0, 0, 0, 0],
"module_skip_counts": [0, 0, 0, 0, 0],
"expected_proposal_calls": [
"glossary_1:proposal",
"homophones:proposal",
"glossary_2:proposal",
"spoken_word:proposal",
"grammar:proposal"
],
"min_response_payload_artifacts": 15
}
}

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"Hello, there were Jesters at the Sight um"}
]

View File

@@ -0,0 +1,6 @@
glossary:
- name: Jesters
aliases: [jester]
plural: jesters
category: faction
summary: A protected in-world faction term.

View File

@@ -0,0 +1,7 @@
[
{"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "sight", "corrected_text": "Sight", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "um um", "corrected_text": "um", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}]}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures at the site um um"}
]

View File

@@ -0,0 +1,12 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

View File

@@ -0,0 +1,17 @@
{
"name": "deterministic_validator_low_confidence",
"transcript_file": "deterministic-validator-low-confidence.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"modules_csv": "grammar",
"proposal_responses_file": "deterministic-validator-low-confidence.proposals.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "deterministic-validator-low-confidence.expected-transcript.json",
"module_instances": ["grammar"],
"module_count": 1,
"total_skipped_changes": 1,
"validator_rejected_reason_codes": ["low_confidence"],
"expected_proposal_calls": ["grammar:proposal"]
}
}

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"hello , world"}
]

View File

@@ -0,0 +1,7 @@
[
{
"corrections": [
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.1}
]
}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , world"}
]

View File

@@ -0,0 +1,12 @@
{
"name": "glossary_schema_handling",
"transcript_file": "default-full-pipeline.transcript.json",
"glossary_file": "glossary-schema-error.glossary.yaml",
"expect": {
"exit_code": 1,
"status": "failed",
"error_phase": "glossary_schema",
"stderr_contains": "glossary_schema",
"require_error_log": true
}
}

View File

@@ -0,0 +1,2 @@
glossary:
- name: MissingCategoryAndSummary

View File

@@ -0,0 +1,19 @@
{
"name": "llm_validator_decision_handling",
"transcript_file": "llm-validator-rejection.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"modules_csv": "grammar",
"proposal_responses_file": "llm-validator-rejection.proposals.json",
"validation_responses_file": "llm-validator-rejection.validations.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "llm-validator-rejection.expected-transcript.json",
"module_instances": ["grammar"],
"module_count": 1,
"total_skipped_changes": 1,
"validator_rejected_reason_codes": ["llm_rejected"],
"expected_proposal_calls": ["grammar:proposal"],
"expected_validation_calls": ["grammar:section-0000:editorial_review:batch-0000"]
}
}

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"hello , world"}
]

View File

@@ -0,0 +1,7 @@
[
{
"corrections": [
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
]
}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , world"}
]

View File

@@ -0,0 +1,7 @@
[
{
"validations": [
{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject stylistic overreach"}
]
}
]

View File

@@ -0,0 +1,23 @@
{
"name": "mid_pipeline_failure_partial_progress",
"transcript_file": "default-handoff.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"proposal_responses_file": "mid-pipeline-failure.proposals.json",
"validation_responses_file": "mid-pipeline-failure.validations.json",
"expect": {
"exit_code": 1,
"status": "failed",
"error_phase": "runner_execution",
"stderr_contains": "runner_execution",
"module_instances": ["glossary_1", "homophones", "glossary_2"],
"module_count": 3,
"total_applied_changes": 2,
"total_skipped_changes": 0,
"failed_module_instance": "glossary_2",
"module_applied_counts": [1, 1, 0],
"module_rejected_counts": [0, 0, 0],
"module_skip_counts": [0, 0, 0],
"require_error_log": true,
"expected_proposal_calls": ["glossary_1:proposal", "homophones:proposal", "glossary_2:proposal"]
}
}

View File

@@ -0,0 +1,4 @@
[
{"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]}
]

View File

@@ -0,0 +1,6 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

View File

@@ -0,0 +1,21 @@
{
"name": "protected_glossary_term_behavior",
"transcript_file": "protected-term-rejection.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"modules_csv": "homophones",
"proposal_responses_file": "protected-term-rejection.proposals.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "protected-term-rejection.expected-transcript.json",
"module_instances": ["homophones"],
"module_count": 1,
"total_applied_changes": 0,
"total_skipped_changes": 1,
"module_applied_counts": [0],
"module_rejected_counts": [1],
"module_skip_counts": [0],
"validator_rejected_reason_codes": ["protected_glossary_term"],
"expected_proposal_calls": ["homophones:proposal"]
}
}

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"The Jesters entered the hall."}
]

View File

@@ -0,0 +1,3 @@
[
{"corrections": [{"id": 1, "original_text": "Jesters", "corrected_text": "Gestures", "confidence": 0.99}]}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"The Jesters entered the hall."}
]

View File

@@ -0,0 +1,12 @@
{
"name": "transcript_schema_handling",
"transcript_file": "transcript-schema-error.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"expect": {
"exit_code": 1,
"status": "failed",
"error_phase": "transcript_schema",
"stderr_contains": "transcript_schema",
"require_error_log": true
}
}

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"","start":0.0,"end":1.0,"text":"bad"}
]

View File

@@ -0,0 +1,24 @@
{
"must_apply_texts": [
"Hello, there were Jesters hmm"
],
"must_not_apply_texts": [
"JESTERX",
"there were gestures"
],
"protected_terms": [
"Jesters"
],
"expected_module_instances": [
"glossary_1",
"homophones",
"glossary_2",
"spoken_word",
"grammar"
],
"minimum_counts": {
"applied": 1,
"rejected": 1,
"skipped": 0
}
}

View File

@@ -0,0 +1,9 @@
[
{
"id": 1,
"speaker": "Alice",
"start": 0,
"end": 1,
"text": "Hello, there were Jesters hmm"
}
]

View File

@@ -0,0 +1,7 @@
glossary:
- name: Jesters
aliases:
- jester
plural: jesters
category: faction
summary: A protected in-world faction term.

View File

@@ -0,0 +1,28 @@
[
{
"corrections": [
{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "Jesters", "corrected_text": "jesters", "confidence": 0.99},
{"id": 1, "original_text": "Jesters", "corrected_text": "JESTERX", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "jesters", "corrected_text": "JESTERS", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "uh", "corrected_text": "hmm", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
]
}
]

View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
]

View File

@@ -0,0 +1,12 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject cleanup"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "release-secret"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

View File

@@ -0,0 +1,6 @@
glossary:
- name: Audita
aliases:
- audita
category: product
summary: The Audita transcript correction CLI.

View File

@@ -0,0 +1,9 @@
[
{
"id": 1,
"speaker": "A",
"start": 0.0,
"end": 1.2,
"text": "hello world"
}
]

View File

@@ -0,0 +1,312 @@
package chunking
import (
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
// Section represents a contiguous chunk of transcript segments with metadata.
type Section struct {
// Index is the 0-based section index within the chunked transcript
Index int `json:"section_index"`
// StartSegmentID is the ID of the first segment in this section
StartSegmentID int `json:"start_segment_id"`
// EndSegmentID is the ID of the last segment in this section
EndSegmentID int `json:"end_segment_id"`
// EstimatedTokens is the approximate token count for this section
EstimatedTokens int `json:"estimated_tokens"`
// Segments contains the segments in this section, in order
Segments []schema.Segment `json:"segments"`
}
// ChunkingConfig holds configuration for transcript chunking.
type ChunkingConfig struct {
// MaxSectionTokens is the maximum allowed tokens per section
MaxSectionTokens int
// MinSectionTokens is a validated soft lower-bound setting retained for
// configuration/reporting compatibility.
MinSectionTokens int
// TargetSections is an optional target number of sections
// If nil, section count is derived from total/max token budgeting.
TargetSections *int
}
// Chunker performs deterministic chunking of normalized transcript segments.
type Chunker struct {
config ChunkingConfig
estimator TokenEstimator
}
// NewChunker creates a new chunker with the given configuration.
func NewChunker(config ChunkingConfig) *Chunker {
return &Chunker{
config: config,
estimator: NewSimpleTokenEstimator(),
}
}
// NewChunkerWithEstimator creates a new chunker with a custom estimator.
func NewChunkerWithEstimator(config ChunkingConfig, estimator TokenEstimator) *Chunker {
return &Chunker{
config: config,
estimator: estimator,
}
}
// ChunkTranscript divides a normalized transcript into contiguous token-bounded
// sections using a deterministic balanced forward pass.
//
// Behavior:
// - preserve segment order and never split segments;
// - estimate per-segment tokens once, then compute total;
// - derive desired section count from ceil(total/max_section_tokens), unless
// target_sections is explicitly set;
// - prefer section sizes near ceil(total/section_count) while never exceeding
// max_section_tokens unless a section consists of a single oversized segment.
//
// Returns an error if explicit target_sections is impossible under constraints.
// The input transcript is never mutated.
func (c *Chunker) ChunkTranscript(transcript *schema.Transcript) ([]Section, error) {
if transcript == nil || len(transcript.Segments) == 0 {
return []Section{}, nil
}
// Validate configuration
if c.config.MaxSectionTokens <= 0 {
return nil, fmt.Errorf("max_section_tokens must be positive, got %d", c.config.MaxSectionTokens)
}
if c.config.MinSectionTokens < 0 {
return nil, fmt.Errorf("min_section_tokens must be non-negative, got %d", c.config.MinSectionTokens)
}
if c.config.MinSectionTokens > c.config.MaxSectionTokens {
return nil, fmt.Errorf("min_section_tokens (%d) cannot exceed max_section_tokens (%d)",
c.config.MinSectionTokens, c.config.MaxSectionTokens)
}
// Calculate token counts for each segment (deterministic).
segmentTokens := make([]int, len(transcript.Segments))
totalTokens := 0
for i, seg := range transcript.Segments {
segmentTokens[i] = c.estimator.EstimateTokens(seg.Text)
totalTokens += segmentTokens[i]
}
minPossibleSections := c.calculateMinPossibleSections(segmentTokens)
var desiredSections int
useExplicitTarget := false
if c.config.TargetSections != nil {
desiredSections = *c.config.TargetSections
useExplicitTarget = true
if desiredSections <= 0 {
return nil, fmt.Errorf("target_sections must be positive, got %d", desiredSections)
}
if err := c.validateTargetSections(desiredSections, segmentTokens); err != nil {
return nil, err
}
} else {
desiredSections = ceilDiv(totalTokens, c.config.MaxSectionTokens)
if desiredSections < minPossibleSections {
desiredSections = minPossibleSections
}
if desiredSections < 1 {
desiredSections = 1
}
if desiredSections > len(transcript.Segments) {
desiredSections = len(transcript.Segments)
}
}
targetTokensPerSection := ceilDiv(totalTokens, desiredSections)
if useExplicitTarget {
return c.buildSectionsWithExplicitTarget(
transcript.Segments,
segmentTokens,
desiredSections,
targetTokensPerSection,
)
}
return c.buildSectionsBalanced(transcript.Segments, segmentTokens, targetTokensPerSection), nil
}
// validateTargetSections checks if the target section count is achievable.
func (c *Chunker) validateTargetSections(target int, segmentTokens []int) error {
// Maximum possible sections: limited by segment count
maxPossibleSections := len(segmentTokens)
if target > maxPossibleSections {
return fmt.Errorf(
"target_sections (%d) is impossible: cannot have more sections than segments (%d)",
target, maxPossibleSections)
}
// Minimum possible sections: each segment must fit within max bounds
minPossibleSections := c.calculateMinPossibleSections(segmentTokens)
if target < minPossibleSections {
return fmt.Errorf(
"target_sections (%d) is impossible: need at least %d sections to respect max_section_tokens (%d)",
target, minPossibleSections, c.config.MaxSectionTokens)
}
return nil
}
// buildSectionsBalanced creates sections with a deterministic single-pass policy.
func (c *Chunker) buildSectionsBalanced(segments []schema.Segment, segmentTokens []int, targetTokensPerSection int) []Section {
var sections []Section
var currentSegments []schema.Segment
currentTokens := 0
for i, seg := range segments {
tokens := segmentTokens[i]
// Empty section: always accept the next segment, including oversized.
if len(currentSegments) == 0 {
currentSegments = append(currentSegments, seg)
currentTokens = tokens
continue
}
// If adding next segment would exceed max, close current section.
if currentTokens+tokens > c.config.MaxSectionTokens {
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
currentSegments = []schema.Segment{seg}
currentTokens = tokens
continue
}
// Prefer staying near target tokens per section.
if targetTokensPerSection == 0 || currentTokens < targetTokensPerSection {
currentSegments = append(currentSegments, seg)
currentTokens += tokens
continue
}
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
currentSegments = []schema.Segment{seg}
currentTokens = tokens
}
if len(currentSegments) > 0 {
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
}
return sections
}
// buildSection creates a Section from segments.
func (c *Chunker) buildSection(index int, segments []schema.Segment, tokens int) Section {
return Section{
Index: index,
StartSegmentID: segments[0].ID,
EndSegmentID: segments[len(segments)-1].ID,
EstimatedTokens: tokens,
Segments: segments,
}
}
// calculateMinPossibleSections calculates the minimum number of sections needed
// to ensure no section exceeds max tokens.
func (c *Chunker) calculateMinPossibleSections(segmentTokens []int) int {
sections := 0
currentTokens := 0
for _, tokens := range segmentTokens {
if tokens > c.config.MaxSectionTokens {
// Each oversized segment needs its own section
if currentTokens > 0 {
sections++
currentTokens = 0
}
sections++
} else if currentTokens+tokens > c.config.MaxSectionTokens {
sections++
currentTokens = tokens
} else {
currentTokens += tokens
}
}
if currentTokens > 0 {
sections++
}
return sections
}
// buildSectionsWithExplicitTarget builds exactly desiredSections when feasible.
func (c *Chunker) buildSectionsWithExplicitTarget(
segments []schema.Segment,
segmentTokens []int,
desiredSections int,
targetTokensPerSection int,
) ([]Section, error) {
n := len(segments)
cursor := 0
sections := make([]Section, 0, desiredSections)
for sectionIdx := 0; sectionIdx < desiredSections; sectionIdx++ {
if cursor >= n {
break
}
remainingSectionsAfter := desiredSections - sectionIdx - 1
currentSegments := []schema.Segment{segments[cursor]}
currentTokens := segmentTokens[cursor]
cursor++
for cursor < n {
remainingSegments := n - cursor
// Reserve one segment per future section to avoid empty sections.
if remainingSegments == remainingSectionsAfter {
break
}
nextTokens := segmentTokens[cursor]
if currentTokens+nextTokens > c.config.MaxSectionTokens {
break
}
if targetTokensPerSection == 0 || currentTokens < targetTokensPerSection {
currentSegments = append(currentSegments, segments[cursor])
currentTokens += nextTokens
cursor++
continue
}
break
}
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
}
if cursor != n || len(sections) != desiredSections {
return nil, fmt.Errorf(
"target_sections (%d) is impossible under current constraints (got %d sections)",
desiredSections,
len(sections),
)
}
return sections, nil
}
func ceilDiv(numerator int, denominator int) int {
if denominator <= 0 {
return 0
}
if numerator <= 0 {
return 0
}
return (numerator + denominator - 1) / denominator
}

View File

@@ -0,0 +1,518 @@
package chunking
import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
// mapTokenEstimator provides deterministic per-segment token counts for tests.
type mapTokenEstimator struct {
byText map[string]int
}
func (e *mapTokenEstimator) EstimateTokens(text string) int {
if e.byText == nil {
return 0
}
if tokens, ok := e.byText[text]; ok {
return tokens
}
return 0
}
func makeSegments(texts []string) []schema.Segment {
segments := make([]schema.Segment, len(texts))
for i, text := range texts {
segments[i] = schema.Segment{
ID: i + 1,
Speaker: "DM",
Start: float64(i * 10),
End: float64(i*10 + 5),
Text: text,
}
}
return segments
}
func makeTranscript(segments []schema.Segment) *schema.Transcript {
return &schema.Transcript{Segments: segments}
}
func intPtr(i int) *int {
return &i
}
func assertSegmentCoverageAndOrder(t *testing.T, input []schema.Segment, sections []Section) {
t.Helper()
seen := make([]schema.Segment, 0, len(input))
for _, sec := range sections {
seen = append(seen, sec.Segments...)
}
if len(seen) != len(input) {
t.Fatalf("expected %d total segment occurrences, got %d", len(input), len(seen))
}
for i := range input {
if seen[i].ID != input[i].ID {
t.Fatalf("segment order mismatch at index %d: got id=%d want id=%d", i, seen[i].ID, input[i].ID)
}
}
}
func assertSectionMetadataConsistent(t *testing.T, sections []Section) {
t.Helper()
for i, sec := range sections {
if sec.Index != i {
t.Fatalf("section %d: expected index=%d got=%d", i, i, sec.Index)
}
if len(sec.Segments) == 0 {
t.Fatalf("section %d: section must not be empty", i)
}
if sec.StartSegmentID != sec.Segments[0].ID {
t.Fatalf("section %d: start_segment_id mismatch", i)
}
if sec.EndSegmentID != sec.Segments[len(sec.Segments)-1].ID {
t.Fatalf("section %d: end_segment_id mismatch", i)
}
}
}
func assertMaxBoundExceptSingletonOversized(t *testing.T, sections []Section, max int) {
t.Helper()
for i, sec := range sections {
if sec.EstimatedTokens <= max {
continue
}
if len(sec.Segments) != 1 {
t.Fatalf("section %d exceeds max tokens (%d>%d) with %d segments", i, sec.EstimatedTokens, max, len(sec.Segments))
}
}
}
func imbalance(sections []Section) int {
if len(sections) == 0 {
return 0
}
minTokens := sections[0].EstimatedTokens
maxTokens := sections[0].EstimatedTokens
for _, sec := range sections {
if sec.EstimatedTokens < minTokens {
minTokens = sec.EstimatedTokens
}
if sec.EstimatedTokens > maxTokens {
maxTokens = sec.EstimatedTokens
}
}
return maxTokens - minTokens
}
func greedyMaxFillSections(segments []schema.Segment, tokens []int, max int) []Section {
sections := make([]Section, 0)
var current []schema.Segment
currentTokens := 0
for i, seg := range segments {
tok := tokens[i]
if len(current) == 0 {
current = append(current, seg)
currentTokens = tok
continue
}
if currentTokens+tok > max {
sections = append(sections, Section{
Index: len(sections),
StartSegmentID: current[0].ID,
EndSegmentID: current[len(current)-1].ID,
EstimatedTokens: currentTokens,
Segments: append([]schema.Segment(nil), current...),
})
current = []schema.Segment{seg}
currentTokens = tok
continue
}
current = append(current, seg)
currentTokens += tok
}
if len(current) > 0 {
sections = append(sections, Section{
Index: len(sections),
StartSegmentID: current[0].ID,
EndSegmentID: current[len(current)-1].ID,
EstimatedTokens: currentTokens,
Segments: append([]schema.Segment(nil), current...),
})
}
return sections
}
func TestChunkEmptyTranscript(t *testing.T) {
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 10})
sections, err := chunker.ChunkTranscript(nil)
if err != nil {
t.Fatalf("ChunkTranscript(nil): %v", err)
}
if len(sections) != 0 {
t.Fatalf("expected 0 sections for nil transcript, got %d", len(sections))
}
sections, err = chunker.ChunkTranscript(makeTranscript(nil))
if err != nil {
t.Fatalf("ChunkTranscript(empty): %v", err)
}
if len(sections) != 0 {
t.Fatalf("expected 0 sections for empty transcript, got %d", len(sections))
}
}
func TestChunkSingleSegment(t *testing.T) {
segments := makeSegments([]string{"s1"})
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 10},
&mapTokenEstimator{byText: map[string]int{"s1": 7}},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 1 {
t.Fatalf("expected 1 section, got %d", len(sections))
}
if sections[0].EstimatedTokens != 7 {
t.Fatalf("expected estimated_tokens=7, got %d", sections[0].EstimatedTokens)
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertSectionMetadataConsistent(t, sections)
}
func TestChunkSingleOversizedSegment(t *testing.T) {
segments := makeSegments([]string{"big"})
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 5},
&mapTokenEstimator{byText: map[string]int{"big": 120}},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 1 {
t.Fatalf("expected 1 section, got %d", len(sections))
}
if sections[0].EstimatedTokens != 120 {
t.Fatalf("expected oversized singleton section, got %d", sections[0].EstimatedTokens)
}
assertMaxBoundExceptSingletonOversized(t, sections, 50)
}
func TestChunkTotalBelowMaxSingleSection(t *testing.T) {
segments := makeSegments([]string{"a", "b", "c"})
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 5},
&ConstTokenEstimator{Tokens: 10},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 1 {
t.Fatalf("expected 1 section, got %d", len(sections))
}
if sections[0].EstimatedTokens != 30 {
t.Fatalf("expected 30 section tokens, got %d", sections[0].EstimatedTokens)
}
assertSegmentCoverageAndOrder(t, segments, sections)
}
func TestChunkTotalExactlyDivisibleByMax(t *testing.T) {
segments := makeSegments([]string{"a", "b", "c", "d"})
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1},
&ConstTokenEstimator{Tokens: 5},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 2 {
t.Fatalf("expected 2 sections, got %d", len(sections))
}
if sections[0].EstimatedTokens != 10 || sections[1].EstimatedTokens != 10 {
t.Fatalf("expected [10,10] tokens, got [%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens)
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertMaxBoundExceptSingletonOversized(t, sections, 10)
}
func TestChunkTotalNotDivisibleByMax(t *testing.T) {
segments := makeSegments([]string{"a", "b", "c", "d", "e"})
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1},
&ConstTokenEstimator{Tokens: 5},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 3 {
t.Fatalf("expected 3 sections, got %d", len(sections))
}
if sections[0].EstimatedTokens != 10 || sections[1].EstimatedTokens != 10 || sections[2].EstimatedTokens != 5 {
t.Fatalf("expected [10,10,5] tokens, got [%d,%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens, sections[2].EstimatedTokens)
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertMaxBoundExceptSingletonOversized(t, sections, 10)
}
func TestChunkTargetSectionsPrecedenceAndSuccess(t *testing.T) {
segments := makeSegments([]string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10"})
target := 3
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 200, MinSectionTokens: 1, TargetSections: &target},
&ConstTokenEstimator{Tokens: 10},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != target {
t.Fatalf("expected %d sections from explicit target, got %d", target, len(sections))
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertSectionMetadataConsistent(t, sections)
}
func TestChunkTargetSectionsImpossibleTooMany(t *testing.T) {
segments := makeSegments([]string{"1", "2", "3", "4", "5"})
target := 10
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 1, TargetSections: &target})
_, err := chunker.ChunkTranscript(makeTranscript(segments))
if err == nil {
t.Fatal("expected error for impossible target_sections")
}
if !strings.Contains(err.Error(), "cannot have more sections than segments") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestChunkTargetSectionsImpossibleTooFew(t *testing.T) {
segments := makeSegments([]string{"1", "2", "3"})
target := 1
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1, TargetSections: &target},
&ConstTokenEstimator{Tokens: 30},
)
_, err := chunker.ChunkTranscript(makeTranscript(segments))
if err == nil {
t.Fatal("expected error for impossible target_sections")
}
if !strings.Contains(err.Error(), "need at least 3 sections") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestChunkManySmallSegmentsBalanced(t *testing.T) {
texts := []string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"}
segments := makeSegments(texts)
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1},
&ConstTokenEstimator{Tokens: 10},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 3 {
t.Fatalf("expected 3 sections, got %d", len(sections))
}
if sections[0].EstimatedTokens != 40 || sections[1].EstimatedTokens != 40 || sections[2].EstimatedTokens != 30 {
t.Fatalf("expected [40,40,30], got [%d,%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens, sections[2].EstimatedTokens)
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertMaxBoundExceptSingletonOversized(t, sections, 50)
}
func TestChunkMixedLargeAndSmallSegments(t *testing.T) {
segments := makeSegments([]string{"big1", "s1", "s2", "s3", "big2", "s4"})
estimator := &mapTokenEstimator{byText: map[string]int{
"big1": 120,
"s1": 10,
"s2": 10,
"s3": 10,
"big2": 120,
"s4": 10,
}}
chunker := NewChunkerWithEstimator(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 1}, estimator)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
if len(sections) != 4 {
t.Fatalf("expected 4 sections, got %d", len(sections))
}
if len(sections[0].Segments) != 1 || sections[0].Segments[0].Text != "big1" {
t.Fatalf("expected first oversized segment in singleton section, got %+v", sections[0].Segments)
}
if len(sections[2].Segments) != 1 || sections[2].Segments[0].Text != "big2" {
t.Fatalf("expected second oversized segment in singleton section, got %+v", sections[2].Segments)
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertMaxBoundExceptSingletonOversized(t, sections, 100)
}
func TestChunkDeterministicOrdering(t *testing.T) {
segments := makeSegments([]string{"a", "b", "c", "d", "e", "f"})
chunkerCfg := ChunkingConfig{MaxSectionTokens: 15, MinSectionTokens: 1}
estimator := &ConstTokenEstimator{Tokens: 5}
var first []Section
for i := 0; i < 5; i++ {
chunker := NewChunkerWithEstimator(chunkerCfg, estimator)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("iteration %d: %v", i, err)
}
if i == 0 {
first = sections
continue
}
if len(sections) != len(first) {
t.Fatalf("iteration %d: section count mismatch (%d vs %d)", i, len(sections), len(first))
}
for j := range sections {
if sections[j].Index != first[j].Index ||
sections[j].StartSegmentID != first[j].StartSegmentID ||
sections[j].EndSegmentID != first[j].EndSegmentID ||
sections[j].EstimatedTokens != first[j].EstimatedTokens ||
len(sections[j].Segments) != len(first[j].Segments) {
t.Fatalf("iteration %d section %d mismatch", i, j)
}
}
}
}
func TestChunkNoMutationOfInput(t *testing.T) {
segments := makeSegments([]string{"original one", "original two"})
transcript := makeTranscript(segments)
original := make([]string, len(transcript.Segments))
for i := range transcript.Segments {
original[i] = transcript.Segments[i].Text
}
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1})
if _, err := chunker.ChunkTranscript(transcript); err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
for i := range transcript.Segments {
if transcript.Segments[i].Text != original[i] {
t.Fatalf("segment %d mutated", i)
}
}
}
func TestChunkConfigValidation(t *testing.T) {
tests := []struct {
name string
config ChunkingConfig
errContains string
}{
{
name: "zero max tokens",
config: ChunkingConfig{MaxSectionTokens: 0, MinSectionTokens: 1},
errContains: "max_section_tokens must be positive",
},
{
name: "negative max tokens",
config: ChunkingConfig{MaxSectionTokens: -1, MinSectionTokens: 1},
errContains: "max_section_tokens must be positive",
},
{
name: "negative min tokens",
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: -1},
errContains: "min_section_tokens must be non-negative",
},
{
name: "min exceeds max",
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 11},
errContains: "min_section_tokens (11) cannot exceed max_section_tokens (10)",
},
{
name: "zero target sections",
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1, TargetSections: intPtr(0)},
errContains: "target_sections must be positive",
},
{
name: "negative target sections",
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1, TargetSections: intPtr(-1)},
errContains: "target_sections must be positive",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
chunker := NewChunker(tt.config)
_, err := chunker.ChunkTranscript(makeTranscript(makeSegments([]string{"x"})))
if err == nil {
t.Fatalf("expected error containing %q", tt.errContains)
}
if !strings.Contains(err.Error(), tt.errContains) {
t.Fatalf("expected error containing %q, got %q", tt.errContains, err.Error())
}
})
}
}
func TestChunkBalancedAlgorithmBeatsGreedyMaxFillOnUnevenTranscript(t *testing.T) {
segments := makeSegments([]string{"s1", "s2", "s3", "s4", "s5", "s6"})
tokenMap := map[string]int{
"s1": 50,
"s2": 10,
"s3": 10,
"s4": 10,
"s5": 10,
"s6": 10,
}
estimator := &mapTokenEstimator{byText: tokenMap}
chunker := NewChunkerWithEstimator(ChunkingConfig{MaxSectionTokens: 80, MinSectionTokens: 1}, estimator)
balancedSections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil {
t.Fatalf("ChunkTranscript: %v", err)
}
tokens := make([]int, 0, len(segments))
for _, seg := range segments {
tokens = append(tokens, tokenMap[seg.Text])
}
greedySections := greedyMaxFillSections(segments, tokens, 80)
balancedImbalance := imbalance(balancedSections)
greedyImbalance := imbalance(greedySections)
if balancedImbalance >= greedyImbalance {
t.Fatalf(
"expected balanced chunking to improve over greedy max-fill; balanced=%d greedy=%d",
balancedImbalance,
greedyImbalance,
)
}
assertSegmentCoverageAndOrder(t, segments, balancedSections)
assertMaxBoundExceptSingletonOversized(t, balancedSections, 80)
}

View File

@@ -0,0 +1,84 @@
package chunking
// Summary provides a concise overview of chunking results for reports
type Summary struct {
ChunkCount int `json:"chunk_count"`
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
TargetSections *int `json:"target_sections,omitempty"`
MaxSectionTokens int `json:"max_section_tokens"`
MinSectionTokens int `json:"min_section_tokens"`
}
// ChunkSummary represents a single chunk's metadata for diagnostics
type ChunkSummary struct {
Index int `json:"index"`
StartSegmentID int `json:"start_segment_id"`
EndSegmentID int `json:"end_segment_id"`
EstimatedTokens int `json:"estimated_tokens"`
SegmentCount int `json:"segment_count"`
}
// DetailedSummary provides per-chunk details for diagnostics
type DetailedSummary struct {
Summary `json:",inline"`
Chunks []ChunkSummary `json:"chunks"`
}
// ComputeSummary creates a Summary from sections and config
func ComputeSummary(sections []Section, config ChunkingConfig) Summary {
if len(sections) == 0 {
return Summary{
ChunkCount: 0,
MaxSectionTokens: config.MaxSectionTokens,
MinSectionTokens: config.MinSectionTokens,
TargetSections: config.TargetSections,
}
}
minTokens := sections[0].EstimatedTokens
maxTokens := sections[0].EstimatedTokens
totalTokens := 0
for _, sec := range sections {
if sec.EstimatedTokens < minTokens {
minTokens = sec.EstimatedTokens
}
if sec.EstimatedTokens > maxTokens {
maxTokens = sec.EstimatedTokens
}
totalTokens += sec.EstimatedTokens
}
return Summary{
ChunkCount: len(sections),
MinEstimatedTokens: minTokens,
MaxEstimatedTokens: maxTokens,
TotalEstimatedTokens: totalTokens,
TargetSections: config.TargetSections,
MaxSectionTokens: config.MaxSectionTokens,
MinSectionTokens: config.MinSectionTokens,
}
}
// ComputeDetailedSummary creates a DetailedSummary from sections and config
func ComputeDetailedSummary(sections []Section, config ChunkingConfig) DetailedSummary {
summary := ComputeSummary(sections, config)
chunks := make([]ChunkSummary, len(sections))
for i, sec := range sections {
chunks[i] = ChunkSummary{
Index: sec.Index,
StartSegmentID: sec.StartSegmentID,
EndSegmentID: sec.EndSegmentID,
EstimatedTokens: sec.EstimatedTokens,
SegmentCount: len(sec.Segments),
}
}
return DetailedSummary{
Summary: summary,
Chunks: chunks,
}
}

View File

@@ -0,0 +1,148 @@
package chunking
import (
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
func TestComputeSummary(t *testing.T) {
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
}
sections := []Section{
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
{Index: 1, EstimatedTokens: 50, StartSegmentID: 3, EndSegmentID: 4},
{Index: 2, EstimatedTokens: 20, StartSegmentID: 5, EndSegmentID: 5},
}
summary := ComputeSummary(sections, config)
if summary.ChunkCount != 3 {
t.Errorf("expected chunk_count=3, got %d", summary.ChunkCount)
}
if summary.MinEstimatedTokens != 20 {
t.Errorf("expected min_estimated_tokens=20, got %d", summary.MinEstimatedTokens)
}
if summary.MaxEstimatedTokens != 50 {
t.Errorf("expected max_estimated_tokens=50, got %d", summary.MaxEstimatedTokens)
}
if summary.TotalEstimatedTokens != 100 {
t.Errorf("expected total_estimated_tokens=100, got %d", summary.TotalEstimatedTokens)
}
if summary.MaxSectionTokens != 100 {
t.Errorf("expected max_section_tokens=100, got %d", summary.MaxSectionTokens)
}
if summary.MinSectionTokens != 10 {
t.Errorf("expected min_section_tokens=10, got %d", summary.MinSectionTokens)
}
if summary.TargetSections != nil {
t.Errorf("expected target_sections=nil, got %v", summary.TargetSections)
}
}
func TestComputeSummaryWithTarget(t *testing.T) {
target := 5
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
TargetSections: &target,
}
sections := []Section{
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
}
summary := ComputeSummary(sections, config)
if summary.TargetSections == nil || *summary.TargetSections != 5 {
t.Errorf("expected target_sections=5, got %v", summary.TargetSections)
}
}
func TestComputeSummaryEmptySections(t *testing.T) {
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
}
sections := []Section{}
summary := ComputeSummary(sections, config)
if summary.ChunkCount != 0 {
t.Errorf("expected chunk_count=0, got %d", summary.ChunkCount)
}
if summary.MinEstimatedTokens != 0 {
t.Errorf("expected min_estimated_tokens=0 for empty, got %d", summary.MinEstimatedTokens)
}
if summary.MaxSectionTokens != 100 {
t.Errorf("expected max_section_tokens preserved, got %d", summary.MaxSectionTokens)
}
}
func TestComputeDetailedSummary(t *testing.T) {
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
}
sections := []Section{
{
Index: 0,
EstimatedTokens: 30,
StartSegmentID: 1,
EndSegmentID: 2,
Segments: make([]schema.Segment, 2), // 2 segments
},
{
Index: 1,
EstimatedTokens: 50,
StartSegmentID: 3,
EndSegmentID: 5,
Segments: make([]schema.Segment, 3), // 3 segments
},
}
detailed := ComputeDetailedSummary(sections, config)
if detailed.ChunkCount != 2 {
t.Errorf("expected chunk_count=2, got %d", detailed.ChunkCount)
}
if len(detailed.Chunks) != 2 {
t.Fatalf("expected 2 chunk entries, got %d", len(detailed.Chunks))
}
// Check first chunk
if detailed.Chunks[0].Index != 0 {
t.Errorf("expected chunk[0].index=0, got %d", detailed.Chunks[0].Index)
}
if detailed.Chunks[0].StartSegmentID != 1 {
t.Errorf("expected chunk[0].start_segment_id=1, got %d", detailed.Chunks[0].StartSegmentID)
}
if detailed.Chunks[0].EndSegmentID != 2 {
t.Errorf("expected chunk[0].end_segment_id=2, got %d", detailed.Chunks[0].EndSegmentID)
}
if detailed.Chunks[0].EstimatedTokens != 30 {
t.Errorf("expected chunk[0].estimated_tokens=30, got %d", detailed.Chunks[0].EstimatedTokens)
}
if detailed.Chunks[0].SegmentCount != 2 {
t.Errorf("expected chunk[0].segment_count=2, got %d", detailed.Chunks[0].SegmentCount)
}
// Check second chunk
if detailed.Chunks[1].Index != 1 {
t.Errorf("expected chunk[1].index=1, got %d", detailed.Chunks[1].Index)
}
if detailed.Chunks[1].StartSegmentID != 3 {
t.Errorf("expected chunk[1].start_segment_id=3, got %d", detailed.Chunks[1].StartSegmentID)
}
if detailed.Chunks[1].EndSegmentID != 5 {
t.Errorf("expected chunk[1].end_segment_id=5, got %d", detailed.Chunks[1].EndSegmentID)
}
if detailed.Chunks[1].SegmentCount != 3 {
t.Errorf("expected chunk[1].segment_count=3, got %d", detailed.Chunks[1].SegmentCount)
}
}

View File

@@ -0,0 +1,55 @@
package chunking
import (
"strings"
"unicode"
)
// TokenEstimator provides a deterministic token estimation suitable for prompt budgeting.
// The estimator is approximate but stable, isolated, and replaceable.
type TokenEstimator interface {
EstimateTokens(text string) int
}
// SimpleTokenEstimator provides a basic deterministic token estimation.
// This uses a simple heuristic based on word count and punctuation.
type SimpleTokenEstimator struct{}
// NewSimpleTokenEstimator creates a new simple token estimator.
func NewSimpleTokenEstimator() *SimpleTokenEstimator {
return &SimpleTokenEstimator{}
}
// EstimateTokens provides a rough estimate of the number of tokens in the given text.
// This implementation uses a simple heuristic: count words and punctuation as tokens.
// The estimate is deterministic and stable for the same input text.
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
if text == "" {
return 0
}
// Simple heuristic: split on whitespace and count non-empty segments
words := strings.Fields(text)
tokenCount := len(words)
// Add some estimate for punctuation that might be separate tokens
punctuationCount := 0
for _, r := range text {
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
punctuationCount++
}
}
// Rough estimate: each word is a token, plus half the punctuation as separate tokens
return tokenCount + (punctuationCount / 2)
}
// ConstTokenEstimator returns a constant token count for testing purposes.
type ConstTokenEstimator struct {
Tokens int
}
// EstimateTokens returns the configured constant token count.
func (e *ConstTokenEstimator) EstimateTokens(text string) int {
return e.Tokens
}

View File

@@ -0,0 +1,59 @@
package chunking
import (
"testing"
)
func TestSimpleTokenEstimator(t *testing.T) {
estimator := NewSimpleTokenEstimator()
tests := []struct {
name string
text string
expected int
}{
{"empty string", "", 0},
{"single word", "hello", 1},
{"two words", "hello world", 2},
{"with punctuation", "hello, world!", 3}, // 2 words + 2 punctuation/2 = 3
{"multiple sentences", "Hello world. This is a test.", 7}, // 7 words + 2 punctuation/2 = 8? Actually "Hello world." has 3 punctuation
{"with apostrophes", "don't won't can't", 3},
{"with hyphens", "well-known state-of-the-art", 2}, // hyphens don't count
{"unicode text", "café naïve", 2},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := estimator.EstimateTokens(tt.text)
if got != tt.expected {
t.Errorf("EstimateTokens(%q) = %d, want %d", tt.text, got, tt.expected)
}
})
}
}
func TestConstTokenEstimator(t *testing.T) {
estimator := &ConstTokenEstimator{Tokens: 42}
if got := estimator.EstimateTokens("any text"); got != 42 {
t.Errorf("ConstTokenEstimator.EstimateTokens = %d, want 42", got)
}
if got := estimator.EstimateTokens(""); got != 42 {
t.Errorf("ConstTokenEstimator.EstimateTokens(empty) = %d, want 42", got)
}
}
func TestTokenEstimatorDeterminism(t *testing.T) {
estimator := NewSimpleTokenEstimator()
text := "The quick brown fox jumps over the lazy dog. Hello, world!"
// Run multiple times and verify same result
first := estimator.EstimateTokens(text)
for i := 0; i < 10; i++ {
got := estimator.EstimateTokens(text)
if got != first {
t.Errorf("EstimateTokens not deterministic: iteration %d got %d, first was %d", i, got, first)
}
}
}

View File

@@ -0,0 +1,171 @@
package config
import "strings"
type llmTargetPatch struct {
apiKey *string
model *string
baseURL *string
timeoutSeconds *int
maxRetries *int
}
type concurrencyPatch struct {
totalLLM *int
legacyTotalLLM *int
proposalLLM *int
validationLLM *int
inheritProposal bool
allowLegacyAlias bool
}
type chunkingPatch struct {
targetSections *int
maxSectionTokens *int
minSectionTokens *int
}
type thresholdsPatch struct {
glossary *float64
grammar *float64
homophones *float64
spokenWord *float64
}
type normalizationPatch struct {
maxSegmentGap *float64
ellipsisGap *float64
maxSegmentDuration *float64
maxSegmentTokens *int
}
type contextPatch struct {
transcriptDescription *string
}
type diagnosticsPatch struct {
workDir *string
workDirRetention *string
}
func (c *Config) applyPrimaryLLMTargetPatch(patch llmTargetPatch) {
if patch.apiKey != nil {
c.PrimaryLLM.APIKey = *patch.apiKey
}
if patch.model != nil {
c.PrimaryLLM.Model = *patch.model
}
if patch.baseURL != nil {
c.PrimaryLLM.BaseURL = *patch.baseURL
}
if patch.timeoutSeconds != nil {
c.PrimaryLLM.TimeoutSeconds = *patch.timeoutSeconds
}
if patch.maxRetries != nil {
c.PrimaryLLM.MaxRetries = *patch.maxRetries
}
}
func (c *Config) applyValidationLLMTargetPatch(patch llmTargetPatch) {
if patch.apiKey != nil {
c.ValidationLLM.APIKey = *patch.apiKey
}
if patch.model != nil {
c.ValidationLLM.Model = *patch.model
}
if patch.baseURL != nil {
c.ValidationLLM.BaseURL = *patch.baseURL
}
if patch.timeoutSeconds != nil {
value := *patch.timeoutSeconds
c.ValidationLLM.TimeoutSeconds = &value
}
if patch.maxRetries != nil {
value := *patch.maxRetries
c.ValidationLLM.MaxRetries = &value
}
}
func (c *Config) applyConcurrencyPatch(patch concurrencyPatch) {
totalSet := false
if patch.totalLLM != nil {
c.TotalLLMConcurrency = *patch.totalLLM
totalSet = true
}
if patch.allowLegacyAlias && patch.legacyTotalLLM != nil && !totalSet {
c.TotalLLMConcurrency = *patch.legacyTotalLLM
totalSet = true
}
proposalSet := false
if patch.proposalLLM != nil {
c.ProposalLLMConcurrency = *patch.proposalLLM
proposalSet = true
}
if patch.inheritProposal && totalSet && !proposalSet {
c.ProposalLLMConcurrency = c.TotalLLMConcurrency
}
if patch.validationLLM != nil {
value := *patch.validationLLM
c.ValidationLLMConcurrency = &value
}
}
func (c *Config) applyChunkingPatch(patch chunkingPatch) {
if patch.targetSections != nil {
value := *patch.targetSections
c.TargetSections = &value
}
if patch.maxSectionTokens != nil {
c.MaxSectionTokens = *patch.maxSectionTokens
}
if patch.minSectionTokens != nil {
c.MinSectionTokens = *patch.minSectionTokens
}
}
func (c *Config) applyThresholdsPatch(patch thresholdsPatch) {
if patch.glossary != nil {
c.Thresholds.Glossary = *patch.glossary
}
if patch.grammar != nil {
c.Thresholds.Grammar = *patch.grammar
}
if patch.homophones != nil {
c.Thresholds.Homophones = *patch.homophones
}
if patch.spokenWord != nil {
c.Thresholds.SpokenWord = *patch.spokenWord
}
}
func (c *Config) applyNormalizationPatch(patch normalizationPatch) {
if patch.maxSegmentGap != nil {
c.Normalization.MaxSegmentGap = *patch.maxSegmentGap
}
if patch.ellipsisGap != nil {
c.Normalization.EllipsisGap = *patch.ellipsisGap
}
if patch.maxSegmentDuration != nil {
c.Normalization.MaxSegmentDuration = *patch.maxSegmentDuration
}
if patch.maxSegmentTokens != nil {
c.Normalization.MaxSegmentTokens = *patch.maxSegmentTokens
}
}
func (c *Config) applyContextPatch(patch contextPatch) {
if patch.transcriptDescription != nil {
c.TranscriptDescription = strings.TrimSpace(*patch.transcriptDescription)
}
}
func (c *Config) applyDiagnosticsPatch(patch diagnosticsPatch) {
if patch.workDir != nil {
c.WorkDir = *patch.workDir
}
if patch.workDirRetention != nil {
c.WorkDirRetention = WorkDirRetention(*patch.workDirRetention)
}
}

View File

@@ -0,0 +1,199 @@
package config
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
)
type WorkDirRetention string
const (
WorkDirRetentionAuto WorkDirRetention = "auto"
WorkDirRetentionAlways WorkDirRetention = "always"
WorkDirRetentionNever WorkDirRetention = "never"
)
const (
DefaultModulesCSV = modulecatalog.KeyGlossary + "," + modulecatalog.KeyHomophones + "," + modulecatalog.KeyGlossary + "," + modulecatalog.KeySpokenWord + "," + modulecatalog.KeyGrammar
DefaultOutputSchema = "bare-segments"
DefaultPrimaryModel = "openrouter/google/gemma-4-31b-it"
DefaultPrimaryBaseURL = "https://openrouter.ai/api/v1"
DefaultPrimaryLLMTimeoutSeconds = 600
DefaultMaxRetries = 3
DefaultLLMConcurrency = 1
DefaultValidationMaxPromptTokens = 2048
DefaultMaxSectionTokens = 8192
DefaultMinSectionTokens = 2048
DefaultConfidenceThreshold = 0.8
DefaultNormalizeMaxSegmentGap = 4.0
DefaultNormalizeEllipsisGap = 3.5
DefaultNormalizeMaxSegmentDuration = 60.0
DefaultNormalizeMaxSegmentTokens = 2048
DefaultTranscriptDescriptionMaxChars = 500
DefaultWorkDir = "/tmp/audita"
DefaultWorkDirRetention WorkDirRetention = WorkDirRetentionAuto
)
type Config struct {
Modules []string
OutputSchema string
PrimaryLLM LLMConfig
ValidationLLM ValidationLLMConfig
TotalLLMConcurrency int
ProposalLLMConcurrency int
ValidationLLMConcurrency *int
ValidationMaxPromptTokens int
MaxSectionTokens int
MinSectionTokens int
TargetSections *int
Thresholds ConfidenceThresholds
Normalization NormalizationConfig
TranscriptDescription string
WorkDir string
WorkDirRetention WorkDirRetention
}
type LLMConfig struct {
APIKey string
Model string
BaseURL string
TimeoutSeconds int
MaxRetries int
// Concurrency is retained as a backward-compatible alias for
// TotalLLMConcurrency.
Concurrency int
}
type ValidationLLMConfig struct {
APIKey string
Model string
BaseURL string
TimeoutSeconds *int
MaxRetries *int
// Concurrency is retained as a backward-compatible alias for
// ValidationLLMConcurrency.
Concurrency *int
}
type ConfidenceThresholds struct {
Glossary float64
Grammar float64
Homophones float64
SpokenWord float64
}
type NormalizationConfig struct {
MaxSegmentGap float64
EllipsisGap float64
MaxSegmentDuration float64
MaxSegmentTokens int
}
func Default() Config {
modules, _ := ParseModulesCSV(DefaultModulesCSV)
return Config{
Modules: modules,
OutputSchema: DefaultOutputSchema,
PrimaryLLM: LLMConfig{
Model: DefaultPrimaryModel,
BaseURL: DefaultPrimaryBaseURL,
TimeoutSeconds: DefaultPrimaryLLMTimeoutSeconds,
MaxRetries: DefaultMaxRetries,
Concurrency: DefaultLLMConcurrency,
},
ValidationLLM: ValidationLLMConfig{},
TotalLLMConcurrency: DefaultLLMConcurrency,
ProposalLLMConcurrency: DefaultLLMConcurrency,
ValidationLLMConcurrency: nil,
ValidationMaxPromptTokens: DefaultValidationMaxPromptTokens,
MaxSectionTokens: DefaultMaxSectionTokens,
MinSectionTokens: DefaultMinSectionTokens,
TargetSections: nil,
Thresholds: ConfidenceThresholds{
Glossary: DefaultConfidenceThreshold,
Grammar: DefaultConfidenceThreshold,
Homophones: DefaultConfidenceThreshold,
SpokenWord: DefaultConfidenceThreshold,
},
Normalization: NormalizationConfig{
MaxSegmentGap: DefaultNormalizeMaxSegmentGap,
EllipsisGap: DefaultNormalizeEllipsisGap,
MaxSegmentDuration: DefaultNormalizeMaxSegmentDuration,
MaxSegmentTokens: DefaultNormalizeMaxSegmentTokens,
},
WorkDir: DefaultWorkDir,
WorkDirRetention: DefaultWorkDirRetention,
}
}
func ParseModulesCSV(raw string) ([]string, error) {
parts := strings.Split(raw, ",")
modules := make([]string, 0, len(parts))
for _, part := range parts {
trimmed := strings.TrimSpace(part)
if trimmed == "" {
return nil, fmt.Errorf("modules list contains an empty value")
}
modules = append(modules, trimmed)
}
if len(modules) == 0 {
return nil, fmt.Errorf("modules list must not be empty")
}
return modules, nil
}
func (c Config) EffectiveValidationLLMConfig() LLMConfig {
effective := c.PrimaryLLM
if c.ValidationLLM.APIKey != "" {
effective.APIKey = c.ValidationLLM.APIKey
}
if c.ValidationLLM.Model != "" {
effective.Model = c.ValidationLLM.Model
}
if c.ValidationLLM.BaseURL != "" {
effective.BaseURL = c.ValidationLLM.BaseURL
}
if c.ValidationLLM.TimeoutSeconds != nil {
effective.TimeoutSeconds = *c.ValidationLLM.TimeoutSeconds
}
if c.ValidationLLM.MaxRetries != nil {
effective.MaxRetries = *c.ValidationLLM.MaxRetries
}
effective.Concurrency = c.EffectiveValidationLLMConcurrency()
return effective
}
func (c Config) EffectiveValidationLLMConcurrency() int {
if c.ValidationLLMConcurrency != nil {
return *c.ValidationLLMConcurrency
}
return c.TotalLLMConcurrency
}
func (c Config) EffectiveProposalLLMConcurrency() int {
if c.ProposalLLMConcurrency > 0 {
return c.ProposalLLMConcurrency
}
return c.TotalLLMConcurrency
}
func (c *Config) syncLegacyConcurrencyAliases() {
if c == nil {
return
}
c.PrimaryLLM.Concurrency = c.TotalLLMConcurrency
c.ValidationLLM.Concurrency = intPtr(c.ValidationLLMConcurrency)
}
func intPtr(v *int) *int {
if v == nil {
return nil
}
x := *v
return &x
}

View File

@@ -0,0 +1,705 @@
package config
import (
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
)
func TestDefaultConfigValues(t *testing.T) {
cfg := Default()
if got, want := strings.Join(cfg.Modules, ","), DefaultModulesCSV; got != want {
t.Fatalf("modules mismatch: got %q want %q", got, want)
}
if cfg.OutputSchema != DefaultOutputSchema {
t.Fatalf("unexpected default output schema: %q", cfg.OutputSchema)
}
if cfg.PrimaryLLM.Model != DefaultPrimaryModel {
t.Fatalf("unexpected default primary model: %q", cfg.PrimaryLLM.Model)
}
if cfg.PrimaryLLM.BaseURL != DefaultPrimaryBaseURL {
t.Fatalf("unexpected default primary base url: %q", cfg.PrimaryLLM.BaseURL)
}
if cfg.PrimaryLLM.TimeoutSeconds != DefaultPrimaryLLMTimeoutSeconds {
t.Fatalf("unexpected default timeout seconds: %d", cfg.PrimaryLLM.TimeoutSeconds)
}
if cfg.PrimaryLLM.MaxRetries != DefaultMaxRetries {
t.Fatalf("unexpected default max retries: %d", cfg.PrimaryLLM.MaxRetries)
}
if cfg.TotalLLMConcurrency != DefaultLLMConcurrency {
t.Fatalf("unexpected default total llm concurrency: %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != DefaultLLMConcurrency {
t.Fatalf("unexpected default proposal llm concurrency: %d", cfg.ProposalLLMConcurrency)
}
if cfg.ValidationLLMConcurrency != nil {
t.Fatalf("expected validation llm concurrency to be unset by default")
}
if cfg.PrimaryLLM.Concurrency != cfg.TotalLLMConcurrency {
t.Fatalf("expected primary llm concurrency alias to mirror total, got primary=%d total=%d", cfg.PrimaryLLM.Concurrency, cfg.TotalLLMConcurrency)
}
if cfg.ValidationLLM.TimeoutSeconds != nil {
t.Fatalf("expected validation timeout to be unset by default")
}
if cfg.ValidationLLM.MaxRetries != nil {
t.Fatalf("expected validation max retries to be unset by default")
}
if cfg.ValidationLLM.Concurrency != nil {
t.Fatalf("expected legacy validation llm concurrency alias to be unset by default")
}
if cfg.TargetSections != nil {
t.Fatalf("expected target sections to be unset by default")
}
if cfg.WorkDir != DefaultWorkDir {
t.Fatalf("unexpected default work dir: %q", cfg.WorkDir)
}
if cfg.TranscriptDescription != "" {
t.Fatalf("expected default transcript description to be empty, got %q", cfg.TranscriptDescription)
}
if cfg.WorkDirRetention != DefaultWorkDirRetention {
t.Fatalf("unexpected default work dir retention: %q", cfg.WorkDirRetention)
}
if err := cfg.Validate(); err != nil {
t.Fatalf("default config should validate: %v", err)
}
}
func TestLoadFromEnvOverridesAndFallback(t *testing.T) {
env := map[string]string{
"AUDITA_MODEL": "openai/gpt-4.1-mini",
"AUDITA_BASE_URL": "https://api.openai.com/v1",
"AUDITA_LLM_TIMEOUT_SECONDS": "120",
"AUDITA_MAX_RETRIES": "7",
"AUDITA_TOTAL_LLM_CONCURRENCY": "6",
"AUDITA_PROPOSAL_LLM_CONCURRENCY": "4",
"AUDITA_VALIDATION_LLM_CONCURRENCY": "2",
"AUDITA_VALIDATION_MAX_PROMPT_TOKENS": "4096",
"AUDITA_MAX_SECTION_TOKENS": "9000",
"AUDITA_MIN_SECTION_TOKENS": "3000",
"AUDITA_TARGET_SECTIONS": "5",
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.9",
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.7",
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.6",
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.5",
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "2.5",
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "2.0",
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "30.0",
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "1024",
"AUDITA_WORK_DIR": "/var/tmp/audita",
"AUDITA_WORK_DIR_RETENTION": "always",
"OPENROUTER_API_KEY": "fallback-key",
}
cfg, err := loadFromLookup(mapLookup(env))
if err != nil {
t.Fatalf("loadFromLookup returned error: %v", err)
}
if cfg.PrimaryLLM.APIKey != "fallback-key" {
t.Fatalf("expected OPENROUTER_API_KEY fallback, got %q", cfg.PrimaryLLM.APIKey)
}
if cfg.PrimaryLLM.Model != env["AUDITA_MODEL"] {
t.Fatalf("unexpected model: %q", cfg.PrimaryLLM.Model)
}
if cfg.PrimaryLLM.BaseURL != env["AUDITA_BASE_URL"] {
t.Fatalf("unexpected base url: %q", cfg.PrimaryLLM.BaseURL)
}
if cfg.TargetSections == nil || *cfg.TargetSections != 5 {
t.Fatalf("unexpected target sections: %#v", cfg.TargetSections)
}
if cfg.TotalLLMConcurrency != 6 {
t.Fatalf("unexpected total llm concurrency: %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != 4 {
t.Fatalf("unexpected proposal llm concurrency: %d", cfg.ProposalLLMConcurrency)
}
if cfg.ValidationLLMConcurrency == nil || *cfg.ValidationLLMConcurrency != 2 {
t.Fatalf("unexpected validation llm concurrency: %#v", cfg.ValidationLLMConcurrency)
}
if cfg.PrimaryLLM.Concurrency != 6 {
t.Fatalf("expected primary alias concurrency 6, got %d", cfg.PrimaryLLM.Concurrency)
}
if cfg.ValidationLLM.Concurrency == nil || *cfg.ValidationLLM.Concurrency != 2 {
t.Fatalf("expected validation alias concurrency 2, got %#v", cfg.ValidationLLM.Concurrency)
}
if cfg.WorkDirRetention != WorkDirRetentionAlways {
t.Fatalf("unexpected work dir retention: %q", cfg.WorkDirRetention)
}
}
func TestLoadFromEnvLegacyLLMConcurrencyAliasForTotalAndProposal(t *testing.T) {
env := map[string]string{
"AUDITA_LLM_CONCURRENCY": "5",
}
cfg, err := loadFromLookup(mapLookup(env))
if err != nil {
t.Fatalf("loadFromLookup returned error: %v", err)
}
if cfg.TotalLLMConcurrency != 5 {
t.Fatalf("expected total concurrency from legacy alias, got %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != 5 {
t.Fatalf("expected proposal concurrency to inherit legacy total, got %d", cfg.ProposalLLMConcurrency)
}
}
func TestLoadFromEnvCanonicalTotalWinsLegacyAlias(t *testing.T) {
env := map[string]string{
"AUDITA_TOTAL_LLM_CONCURRENCY": "4",
"AUDITA_LLM_CONCURRENCY": "9",
}
cfg, err := loadFromLookup(mapLookup(env))
if err != nil {
t.Fatalf("loadFromLookup returned error: %v", err)
}
if cfg.TotalLLMConcurrency != 4 {
t.Fatalf("expected canonical total to win over legacy alias, got %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != 4 {
t.Fatalf("expected proposal to inherit canonical total when unset, got %d", cfg.ProposalLLMConcurrency)
}
}
func TestLoadFromEnvUsesAuditaLLMAPIKeyOverFallback(t *testing.T) {
env := map[string]string{
"AUDITA_LLM_API_KEY": "primary-key",
"OPENROUTER_API_KEY": "fallback-key",
}
cfg, err := loadFromLookup(mapLookup(env))
if err != nil {
t.Fatalf("loadFromLookup returned error: %v", err)
}
if cfg.PrimaryLLM.APIKey != "primary-key" {
t.Fatalf("expected AUDITA_LLM_API_KEY to win, got %q", cfg.PrimaryLLM.APIKey)
}
}
func TestApplyCLIOverridesPrecedence(t *testing.T) {
cfg := Default()
cfg.PrimaryLLM.Model = "env-model"
cfg.WorkDir = "/env/work"
model := "cli-model"
workDir := "/cli/work"
modules := "grammar"
outputSchema := "audita-v1"
totalLLMConcurrency := 5
proposalLLMConcurrency := 3
overrides := CLIOverrides{
PrimaryModel: &model,
WorkDir: &workDir,
ModulesCSV: &modules,
OutputSchema: &outputSchema,
TotalLLMConcurrency: &totalLLMConcurrency,
ProposalLLMConcurrency: &proposalLLMConcurrency,
}
if err := cfg.ApplyCLIOverrides(overrides); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.PrimaryLLM.Model != "cli-model" {
t.Fatalf("expected CLI model override, got %q", cfg.PrimaryLLM.Model)
}
if cfg.WorkDir != "/cli/work" {
t.Fatalf("expected CLI work dir override, got %q", cfg.WorkDir)
}
if !reflect.DeepEqual(cfg.Modules, []string{"grammar"}) {
t.Fatalf("unexpected modules: %#v", cfg.Modules)
}
if cfg.OutputSchema != "audita-v1" {
t.Fatalf("expected CLI output schema override, got %q", cfg.OutputSchema)
}
if cfg.TotalLLMConcurrency != 5 {
t.Fatalf("expected CLI total concurrency override, got %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != 3 {
t.Fatalf("expected CLI proposal concurrency override, got %d", cfg.ProposalLLMConcurrency)
}
}
func TestApplyCLIOverridesTrimsTranscriptDescription(t *testing.T) {
cfg := Default()
description := " background context about speakers "
if err := cfg.ApplyCLIOverrides(CLIOverrides{TranscriptDescription: &description}); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.TranscriptDescription != "background context about speakers" {
t.Fatalf("unexpected transcript description trim result: %q", cfg.TranscriptDescription)
}
}
func TestConfigSourcesApplySharedEffectiveFieldsConsistently(t *testing.T) {
fileCfg := mustParseFileConfigYAML(t, `
version: 1
output:
schema: " audita-v1 "
llm:
proposal:
base_url: https://proposal.example.test/v1
model: provider/proposal
timeout: 101
max_retries: 5
validation:
base_url: https://validation.example.test/v1
model: provider/validation
timeout: 202
max_retries: 6
chunking:
target_sections: 7
max_section_tokens: 9000
min_section_tokens: 1000
thresholds:
glossary: 0.91
grammar: 0.92
homophones: 0.93
spoken_word: 0.94
normalization:
max_segment_gap: 1.2
ellipsis_gap: 2.3
max_segment_duration: 45.6
max_segment_tokens: 321
context:
description: " shared context "
diagnostics:
work_dir: /tmp/audita-shared
retention: always
`)
tests := []struct {
name string
apply func(*Config) error
}{
{
name: "file",
apply: func(cfg *Config) error {
return cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{}))
},
},
{
name: "env",
apply: func(cfg *Config) error {
return cfg.applyEnvOverrides(mapLookup(map[string]string{
"AUDITA_MODEL": "provider/proposal",
"AUDITA_BASE_URL": "https://proposal.example.test/v1",
"AUDITA_LLM_TIMEOUT_SECONDS": "101",
"AUDITA_MAX_RETRIES": "5",
"AUDITA_VALIDATION_MODEL": "provider/validation",
"AUDITA_VALIDATION_BASE_URL": "https://validation.example.test/v1",
"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "202",
"AUDITA_VALIDATION_MAX_RETRIES": "6",
"AUDITA_TARGET_SECTIONS": "7",
"AUDITA_MAX_SECTION_TOKENS": "9000",
"AUDITA_MIN_SECTION_TOKENS": "1000",
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.91",
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.92",
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.93",
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.94",
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "1.2",
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "2.3",
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.6",
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "321",
"AUDITA_WORK_DIR": "/tmp/audita-shared",
"AUDITA_WORK_DIR_RETENTION": "always",
}))
},
},
{
name: "cli",
apply: func(cfg *Config) error {
outputSchema := " audita-v1 "
proposalModel := "provider/proposal"
proposalBaseURL := "https://proposal.example.test/v1"
proposalTimeout := 101
proposalMaxRetries := 5
validationModel := "provider/validation"
validationBaseURL := "https://validation.example.test/v1"
validationTimeout := 202
validationMaxRetries := 6
targetSections := 7
maxSectionTokens := 9000
minSectionTokens := 1000
glossaryThreshold := 0.91
grammarThreshold := 0.92
homophonesThreshold := 0.93
spokenWordThreshold := 0.94
normalizeMaxSegmentGap := 1.2
normalizeEllipsisGap := 2.3
normalizeMaxSegmentDuration := 45.6
normalizeMaxSegmentTokens := 321
description := " shared context "
workDir := "/tmp/audita-shared"
workDirRetention := "always"
return cfg.ApplyCLIOverrides(CLIOverrides{
OutputSchema: &outputSchema,
PrimaryModel: &proposalModel,
PrimaryBaseURL: &proposalBaseURL,
PrimaryLLMTimeoutSeconds: &proposalTimeout,
MaxRetries: &proposalMaxRetries,
ValidationModel: &validationModel,
ValidationBaseURL: &validationBaseURL,
ValidationLLMTimeoutSeconds: &validationTimeout,
ValidationMaxRetries: &validationMaxRetries,
TargetSections: &targetSections,
MaxSectionTokens: &maxSectionTokens,
MinSectionTokens: &minSectionTokens,
GlossaryConfidenceThreshold: &glossaryThreshold,
GrammarConfidenceThreshold: &grammarThreshold,
HomophonesConfidenceThreshold: &homophonesThreshold,
SpokenWordConfidenceThreshold: &spokenWordThreshold,
NormalizeMaxSegmentGap: &normalizeMaxSegmentGap,
NormalizeEllipsisGap: &normalizeEllipsisGap,
NormalizeMaxSegmentDuration: &normalizeMaxSegmentDuration,
NormalizeMaxSegmentTokens: &normalizeMaxSegmentTokens,
TranscriptDescription: &description,
WorkDir: &workDir,
WorkDirRetention: &workDirRetention,
})
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
cfg := Default()
if err := tc.apply(&cfg); err != nil {
t.Fatalf("apply config source: %v", err)
}
assertSharedEffectiveFields(t, cfg, sharedEffectiveFieldOptions{
wantOutputSchemaOverride: tc.name != "env",
wantTranscriptDescriptionPatch: tc.name != "env",
})
})
}
}
func TestCLIAPIKeyOverrideIsDirectValue(t *testing.T) {
cfg := Default()
apiKey := "NOT_AN_ENV_VAR_NAME"
validationAPIKey := "also direct"
if err := cfg.ApplyCLIOverrides(CLIOverrides{PrimaryLLMAPIKey: &apiKey, ValidationLLMAPIKey: &validationAPIKey}); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.PrimaryLLM.APIKey != apiKey {
t.Fatalf("expected direct primary api key, got %q", cfg.PrimaryLLM.APIKey)
}
if cfg.ValidationLLM.APIKey != validationAPIKey {
t.Fatalf("expected direct validation api key, got %q", cfg.ValidationLLM.APIKey)
}
}
func TestApplyFileConfigTotalConcurrencyDoesNotChangeProposalWhenProposalUnset(t *testing.T) {
fileCfg := mustParseFileConfigYAML(t, `
version: 1
concurrency:
total_llm: 4
`)
cfg := Default()
if err := cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{})); err != nil {
t.Fatalf("applyFileConfigWithLookup failed: %v", err)
}
if cfg.TotalLLMConcurrency != 4 {
t.Fatalf("expected file total concurrency 4, got %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != DefaultLLMConcurrency {
t.Fatalf("expected file config to preserve proposal concurrency when unset, got %d", cfg.ProposalLLMConcurrency)
}
}
func TestValidationRejectsOverlyLongTranscriptDescription(t *testing.T) {
cfg := Default()
cfg.TranscriptDescription = strings.Repeat("a", DefaultTranscriptDescriptionMaxChars+1)
err := cfg.Validate()
if err == nil {
t.Fatalf("expected transcript description length validation error")
}
if !strings.Contains(err.Error(), "transcript description must be 500 characters or fewer") {
t.Fatalf("unexpected validation error: %v", err)
}
}
func TestApplyCLIOverridesLegacyLLMConcurrencyAlias(t *testing.T) {
cfg := Default()
aliasConcurrency := 6
if err := cfg.ApplyCLIOverrides(CLIOverrides{PrimaryLLMConcurrency: &aliasConcurrency}); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.TotalLLMConcurrency != 6 {
t.Fatalf("expected legacy --llm-concurrency alias to set total, got %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != 6 {
t.Fatalf("expected proposal to inherit aliased total when unset, got %d", cfg.ProposalLLMConcurrency)
}
}
func TestApplyCLIOverridesCanonicalTotalWinsLegacyAlias(t *testing.T) {
cfg := Default()
canonicalTotal := 4
legacyAlias := 9
if err := cfg.ApplyCLIOverrides(CLIOverrides{TotalLLMConcurrency: &canonicalTotal, PrimaryLLMConcurrency: &legacyAlias}); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.TotalLLMConcurrency != 4 {
t.Fatalf("expected canonical total concurrency to win, got %d", cfg.TotalLLMConcurrency)
}
if cfg.ProposalLLMConcurrency != 4 {
t.Fatalf("expected proposal to inherit canonical total when proposal is unset, got %d", cfg.ProposalLLMConcurrency)
}
}
func TestValidationFailures(t *testing.T) {
cfg := Default()
cfg.OutputSchema = "unknown-schema"
cfg.PrimaryLLM.TimeoutSeconds = -1
cfg.TotalLLMConcurrency = 0
cfg.ProposalLLMConcurrency = 0
validationConcurrency := 5
cfg.ValidationLLMConcurrency = &validationConcurrency
cfg.ValidationMaxPromptTokens = 0
cfg.MaxSectionTokens = 100
cfg.MinSectionTokens = 200
cfg.Thresholds.Grammar = 1.5
cfg.WorkDirRetention = WorkDirRetention("sometimes")
err := cfg.Validate()
if err == nil {
t.Fatalf("expected validation error")
}
message := err.Error()
for _, expected := range []string{
"primary llm timeout seconds",
"total llm concurrency",
"proposal llm concurrency",
"validation llm concurrency must be less than or equal to total llm concurrency",
"validation max prompt tokens",
"min section tokens",
"grammar confidence threshold",
"work dir retention",
"unsupported output schema",
} {
if !strings.Contains(message, expected) {
t.Fatalf("expected error to contain %q, got %q", expected, message)
}
}
}
func TestValidationRejectsUnsupportedModuleKey(t *testing.T) {
cfg := Default()
cfg.Modules = []string{modulecatalog.KeyGlossary, "made_up"}
err := cfg.Validate()
if err == nil {
t.Fatalf("expected validation error for unsupported module key")
}
if !strings.Contains(err.Error(), `unsupported module key "made_up"`) {
t.Fatalf("expected unsupported module key error, got %q", err.Error())
}
}
func TestValidationAllowsRepeatedSupportedModuleKeys(t *testing.T) {
cfg := Default()
cfg.Modules = []string{modulecatalog.KeyGlossary, modulecatalog.KeyGlossary, modulecatalog.KeyGrammar}
if err := cfg.Validate(); err != nil {
t.Fatalf("expected repeated supported module keys to validate, got %v", err)
}
}
func TestValidationAcceptsAllSupportedOutputSchemas(t *testing.T) {
for _, schemaKey := range outputschema.SupportedKeys() {
cfg := Default()
cfg.OutputSchema = schemaKey
if err := cfg.Validate(); err != nil {
t.Fatalf("expected output schema %q to validate, got %v", schemaKey, err)
}
}
}
func TestEffectiveValidationLLMInheritance(t *testing.T) {
cfg := Default()
cfg.PrimaryLLM.APIKey = "primary-key"
cfg.PrimaryLLM.Model = "primary-model"
cfg.PrimaryLLM.BaseURL = "https://primary.example/v1"
cfg.PrimaryLLM.TimeoutSeconds = 111
cfg.PrimaryLLM.MaxRetries = 2
cfg.TotalLLMConcurrency = 7
cfg.syncLegacyConcurrencyAliases()
effective := cfg.EffectiveValidationLLMConfig()
if effective.APIKey != "primary-key" || effective.Model != "primary-model" || effective.BaseURL != "https://primary.example/v1" || effective.TimeoutSeconds != 111 || effective.MaxRetries != 2 || effective.Concurrency != 7 {
t.Fatalf("unexpected inherited config: %#v", effective)
}
validationTimeout := 222
validationRetries := 9
cfg.ValidationLLM.APIKey = "validation-key"
cfg.ValidationLLM.Model = "validation-model"
cfg.ValidationLLM.BaseURL = "https://validation.example/v1"
cfg.ValidationLLM.TimeoutSeconds = &validationTimeout
cfg.ValidationLLM.MaxRetries = &validationRetries
validationConcurrency := 4
cfg.ValidationLLMConcurrency = &validationConcurrency
cfg.syncLegacyConcurrencyAliases()
effective = cfg.EffectiveValidationLLMConfig()
if effective.APIKey != "validation-key" || effective.Model != "validation-model" || effective.BaseURL != "https://validation.example/v1" || effective.TimeoutSeconds != 222 || effective.MaxRetries != 9 || effective.Concurrency != 4 {
t.Fatalf("unexpected overridden validation config: %#v", effective)
}
}
func TestValidationLLMConcurrencyCannotExceedTotal(t *testing.T) {
cfg := Default()
cfg.TotalLLMConcurrency = 2
cfg.ProposalLLMConcurrency = 2
validationConcurrency := 3
cfg.ValidationLLMConcurrency = &validationConcurrency
if err := cfg.Validate(); err == nil {
t.Fatal("expected validation error when validation llm concurrency exceeds total")
}
validationConcurrency = 2
cfg.ValidationLLMConcurrency = &validationConcurrency
if err := cfg.Validate(); err != nil {
t.Fatalf("expected equal concurrency to validate, got %v", err)
}
}
func TestProposalLLMConcurrencyCannotExceedTotal(t *testing.T) {
cfg := Default()
cfg.TotalLLMConcurrency = 2
cfg.ProposalLLMConcurrency = 3
if err := cfg.Validate(); err == nil {
t.Fatal("expected validation error when proposal llm concurrency exceeds total")
}
cfg.ProposalLLMConcurrency = 2
if err := cfg.Validate(); err != nil {
t.Fatalf("expected equal concurrency to validate, got %v", err)
}
}
func TestCLITotalLLMConcurrencyOverrideDrivesEffectiveValidationConcurrencyWhenValidationUnset(t *testing.T) {
cfg := Default()
totalLLMConcurrency := 6
if err := cfg.ApplyCLIOverrides(CLIOverrides{TotalLLMConcurrency: &totalLLMConcurrency}); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.ValidationLLMConcurrency != nil {
t.Fatalf("expected validation concurrency to remain unset, got %#v", cfg.ValidationLLMConcurrency)
}
if cfg.EffectiveValidationLLMConcurrency() != 6 {
t.Fatalf("expected inherited validation concurrency 6, got %d", cfg.EffectiveValidationLLMConcurrency())
}
}
func TestRedactedConfig(t *testing.T) {
cfg := Default()
cfg.PrimaryLLM.APIKey = "secret-primary"
cfg.ValidationLLM.APIKey = "secret-validation"
redacted := cfg.Redacted()
if redacted.PrimaryLLM.APIKey != redactedSecret {
t.Fatalf("expected primary api key to be redacted, got %q", redacted.PrimaryLLM.APIKey)
}
if redacted.ValidationLLM.APIKey != redactedSecret {
t.Fatalf("expected validation api key to be redacted, got %q", redacted.ValidationLLM.APIKey)
}
if cfg.PrimaryLLM.APIKey != "secret-primary" {
t.Fatalf("redaction should not mutate original config")
}
}
func mapLookup(values map[string]string) func(string) (string, bool) {
return func(key string) (string, bool) {
value, ok := values[key]
return value, ok
}
}
func mustParseFileConfigYAML(t *testing.T, raw string) FileConfig {
t.Helper()
fileCfg, err := ParseFileConfigYAML([]byte(raw))
if err != nil {
t.Fatalf("ParseFileConfigYAML failed: %v", err)
}
return fileCfg
}
type sharedEffectiveFieldOptions struct {
wantOutputSchemaOverride bool
wantTranscriptDescriptionPatch bool
}
func assertSharedEffectiveFields(t *testing.T, cfg Config, opts sharedEffectiveFieldOptions) {
t.Helper()
wantOutputSchema := DefaultOutputSchema
if opts.wantOutputSchemaOverride {
wantOutputSchema = "audita-v1"
}
if cfg.OutputSchema != wantOutputSchema {
t.Fatalf("unexpected output schema: %q", cfg.OutputSchema)
}
if cfg.PrimaryLLM.Model != "provider/proposal" ||
cfg.PrimaryLLM.BaseURL != "https://proposal.example.test/v1" ||
cfg.PrimaryLLM.TimeoutSeconds != 101 ||
cfg.PrimaryLLM.MaxRetries != 5 {
t.Fatalf("unexpected primary llm config: %+v", cfg.PrimaryLLM)
}
if cfg.ValidationLLM.Model != "provider/validation" ||
cfg.ValidationLLM.BaseURL != "https://validation.example.test/v1" ||
cfg.ValidationLLM.TimeoutSeconds == nil ||
*cfg.ValidationLLM.TimeoutSeconds != 202 ||
cfg.ValidationLLM.MaxRetries == nil ||
*cfg.ValidationLLM.MaxRetries != 6 {
t.Fatalf("unexpected validation llm config: %+v", cfg.ValidationLLM)
}
if cfg.TargetSections == nil || *cfg.TargetSections != 7 ||
cfg.MaxSectionTokens != 9000 ||
cfg.MinSectionTokens != 1000 {
t.Fatalf("unexpected chunking config: target=%v max=%d min=%d", cfg.TargetSections, cfg.MaxSectionTokens, cfg.MinSectionTokens)
}
if cfg.Thresholds.Glossary != 0.91 ||
cfg.Thresholds.Grammar != 0.92 ||
cfg.Thresholds.Homophones != 0.93 ||
cfg.Thresholds.SpokenWord != 0.94 {
t.Fatalf("unexpected thresholds: %+v", cfg.Thresholds)
}
if cfg.Normalization.MaxSegmentGap != 1.2 ||
cfg.Normalization.EllipsisGap != 2.3 ||
cfg.Normalization.MaxSegmentDuration != 45.6 ||
cfg.Normalization.MaxSegmentTokens != 321 {
t.Fatalf("unexpected normalization: %+v", cfg.Normalization)
}
wantDescription := ""
if opts.wantTranscriptDescriptionPatch {
wantDescription = "shared context"
}
if cfg.TranscriptDescription != wantDescription {
t.Fatalf("unexpected transcript description: %q", cfg.TranscriptDescription)
}
if cfg.WorkDir != "/tmp/audita-shared" ||
cfg.WorkDirRetention != WorkDirRetentionAlways {
t.Fatalf("unexpected diagnostics config: work_dir=%q retention=%q", cfg.WorkDir, cfg.WorkDirRetention)
}
}

View File

@@ -0,0 +1,119 @@
package config
import (
"fmt"
"os"
"strings"
)
type EffectiveConfigErrorKind string
const (
EffectiveConfigErrorResolvePath EffectiveConfigErrorKind = "resolve_path"
EffectiveConfigErrorLoadFile EffectiveConfigErrorKind = "load_file"
EffectiveConfigErrorApplyFile EffectiveConfigErrorKind = "apply_file"
EffectiveConfigErrorApplyEnv EffectiveConfigErrorKind = "apply_env"
)
type EffectiveConfigError struct {
Kind EffectiveConfigErrorKind
Err error
}
func (e *EffectiveConfigError) Error() string {
if e == nil || e.Err == nil {
return ""
}
return e.Err.Error()
}
func (e *EffectiveConfigError) Unwrap() error {
if e == nil {
return nil
}
return e.Err
}
type EffectiveConfig struct {
Config Config
ConfigPath string
ConfigSource string
ConfigVersion *int
}
func ResolveConfigPath(cliConfigPath string, cliConfigPathSet bool) (path string, source string, err error) {
return resolveConfigPathWithLookup(cliConfigPath, cliConfigPathSet, os.LookupEnv, os.Stat, DefaultConfigSearchPaths)
}
func LoadEffectiveConfig(cliConfigPath string, cliConfigPathSet bool) (EffectiveConfig, error) {
return loadEffectiveConfigWithLookup(cliConfigPath, cliConfigPathSet, os.LookupEnv, os.Stat, DefaultConfigSearchPaths)
}
func loadEffectiveConfigWithLookup(cliConfigPath string, cliConfigPathSet bool, lookup func(string) (string, bool), statPath func(string) (os.FileInfo, error), defaultSearchPaths []string) (EffectiveConfig, error) {
configPath, configSource, err := resolveConfigPathWithLookup(cliConfigPath, cliConfigPathSet, lookup, statPath, defaultSearchPaths)
if err != nil {
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorResolvePath, Err: err}
}
cfg := Default()
var configVersion *int
if configPath != "" {
fileCfg, fileErr := LoadFileConfig(configPath)
if fileErr != nil {
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorLoadFile, Err: fileErr}
}
if applyErr := cfg.ApplyFileConfig(fileCfg); applyErr != nil {
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorApplyFile, Err: applyErr}
}
configVersion = &fileCfg.Version
}
if applyEnvErr := cfg.applyEnvOverrides(lookup); applyEnvErr != nil {
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorApplyEnv, Err: applyEnvErr}
}
return EffectiveConfig{
Config: cfg,
ConfigPath: configPath,
ConfigSource: configSource,
ConfigVersion: configVersion,
}, nil
}
func resolveConfigPathWithLookup(cliConfigPath string, cliConfigPathSet bool, lookup func(string) (string, bool), statPath func(string) (os.FileInfo, error), defaultSearchPaths []string) (path string, source string, err error) {
if cliConfigPathSet {
path = strings.TrimSpace(cliConfigPath)
if path == "" {
return "", "", fmt.Errorf("--config requires a non-empty path")
}
if _, statErr := statPath(path); statErr != nil {
if os.IsNotExist(statErr) {
return "", "", fmt.Errorf("config file not found: %s", path)
}
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
}
return path, "flag", nil
}
if raw, ok := lookup("AUDITA_CONFIG"); ok {
path = strings.TrimSpace(raw)
if path == "" {
return "", "", fmt.Errorf("AUDITA_CONFIG must not be empty")
}
if _, statErr := statPath(path); statErr != nil {
if os.IsNotExist(statErr) {
return "", "", fmt.Errorf("config file not found: %s", path)
}
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
}
return path, "env", nil
}
for _, defaultPath := range defaultSearchPaths {
if _, statErr := statPath(defaultPath); statErr == nil {
return defaultPath, "default", nil
} else if !os.IsNotExist(statErr) {
return "", "", fmt.Errorf("cannot access config file %s: %w", defaultPath, statErr)
}
}
return "", "", nil
}

View File

@@ -0,0 +1,163 @@
package config
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestResolveConfigPathWithLookupMatrix(t *testing.T) {
statFor := func(existing map[string]bool) func(string) (os.FileInfo, error) {
return func(path string) (os.FileInfo, error) {
if existing[path] {
return nil, nil
}
return nil, os.ErrNotExist
}
}
tests := []struct {
name string
cliPath string
cliPathSet bool
lookup func(string) (string, bool)
stat func(string) (os.FileInfo, error)
defaultSearchPaths []string
wantPath string
wantSource string
wantErrContains string
}{
{
name: "explicit config path",
cliPath: "/tmp/explicit.yml",
cliPathSet: true,
lookup: func(string) (string, bool) { return "", false },
stat: statFor(map[string]bool{"/tmp/explicit.yml": true}),
defaultSearchPaths: []string{
"/usr/local/etc/audita/config.yml",
"/etc/audita/config.yml",
},
wantPath: "/tmp/explicit.yml",
wantSource: "flag",
},
{
name: "env config path",
cliPathSet: false,
lookup: func(key string) (string, bool) {
if key == "AUDITA_CONFIG" {
return "/tmp/from-env.yml", true
}
return "", false
},
stat: statFor(map[string]bool{"/tmp/from-env.yml": true}),
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
wantPath: "/tmp/from-env.yml",
wantSource: "env",
},
{
name: "default search path",
cliPathSet: false,
lookup: func(string) (string, bool) { return "", false },
stat: statFor(map[string]bool{
"/usr/local/etc/audita/config.yml": true,
"/etc/audita/config.yml": true,
}),
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
wantPath: "/usr/local/etc/audita/config.yml",
wantSource: "default",
},
{
name: "explicit missing path",
cliPath: "/tmp/missing.yml",
cliPathSet: true,
lookup: func(string) (string, bool) { return "", false },
stat: statFor(map[string]bool{}),
defaultSearchPaths: []string{
"/usr/local/etc/audita/config.yml",
"/etc/audita/config.yml",
},
wantErrContains: "config file not found",
},
{
name: "missing env path",
cliPathSet: false,
lookup: func(key string) (string, bool) {
if key == "AUDITA_CONFIG" {
return "/tmp/missing-from-env.yml", true
}
return "", false
},
stat: statFor(map[string]bool{}),
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
wantErrContains: "config file not found",
},
{
name: "missing default paths",
cliPathSet: false,
lookup: func(string) (string, bool) { return "", false },
stat: statFor(map[string]bool{}),
defaultSearchPaths: []string{
"/usr/local/etc/audita/config.yml",
"/etc/audita/config.yml",
},
wantPath: "",
wantSource: "",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
gotPath, gotSource, err := resolveConfigPathWithLookup(tc.cliPath, tc.cliPathSet, tc.lookup, tc.stat, tc.defaultSearchPaths)
if tc.wantErrContains != "" {
if err == nil || !strings.Contains(err.Error(), tc.wantErrContains) {
t.Fatalf("expected error containing %q, got %v", tc.wantErrContains, err)
}
return
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if gotPath != tc.wantPath || gotSource != tc.wantSource {
t.Fatalf("unexpected result: got path=%q source=%q, want path=%q source=%q", gotPath, gotSource, tc.wantPath, tc.wantSource)
}
})
}
}
func TestLoadEffectiveConfigWithLookupAppliesDefaultsFileThenEnv(t *testing.T) {
tempDir := t.TempDir()
configPath := filepath.Join(tempDir, "config.yml")
configYAML := "version: 1\nllm:\n proposal:\n model: file-model\n"
if err := os.WriteFile(configPath, []byte(configYAML), 0o644); err != nil {
t.Fatalf("write config file: %v", err)
}
lookup := func(key string) (string, bool) {
switch key {
case "AUDITA_CONFIG":
return configPath, true
case "AUDITA_MODEL":
return "env-model", true
default:
return "", false
}
}
result, err := loadEffectiveConfigWithLookup("", false, lookup, os.Stat, DefaultConfigSearchPaths)
if err != nil {
t.Fatalf("loadEffectiveConfigWithLookup error: %v", err)
}
if result.ConfigPath != configPath {
t.Fatalf("unexpected config path: %q", result.ConfigPath)
}
if result.ConfigSource != "env" {
t.Fatalf("unexpected config source: %q", result.ConfigSource)
}
if result.ConfigVersion == nil || *result.ConfigVersion != SupportedFileConfigVersion {
t.Fatalf("unexpected config version: %#v", result.ConfigVersion)
}
if result.Config.PrimaryLLM.Model != "env-model" {
t.Fatalf("expected env override to win over file value, got %q", result.Config.PrimaryLLM.Model)
}
}

269
internal/core/config/env.go Normal file
View File

@@ -0,0 +1,269 @@
package config
import (
"fmt"
"os"
"strconv"
)
const (
DefaultConfigPath = "/etc/audita/config.yml"
DefaultConfigPathUsrLocal = "/usr/local/etc/audita/config.yml"
)
var DefaultConfigSearchPaths = []string{
DefaultConfigPathUsrLocal,
DefaultConfigPath,
}
func LoadFromEnv() (Config, error) {
cfg := Default()
if err := cfg.applyEnvOverrides(os.LookupEnv); err != nil {
return Config{}, err
}
return cfg, nil
}
func loadFromLookup(lookup func(string) (string, bool)) (Config, error) {
cfg := Default()
if err := cfg.applyEnvOverrides(lookup); err != nil {
return Config{}, err
}
return cfg, nil
}
func (c *Config) ApplyEnvOverrides() error {
return c.applyEnvOverrides(os.LookupEnv)
}
func (c *Config) applyEnvOverrides(lookup func(string) (string, bool)) error {
if c == nil {
return fmt.Errorf("config must not be nil")
}
cfg := c
if raw, ok := lookup("AUDITA_MODULES"); ok {
modules, err := ParseModulesCSV(raw)
if err != nil {
return fmt.Errorf("AUDITA_MODULES: %w", err)
}
cfg.Modules = modules
}
primaryLLM := llmTargetPatch{}
if raw, ok := lookup("AUDITA_LLM_API_KEY"); ok {
primaryLLM.apiKey = &raw
} else if raw, ok := lookup("OPENROUTER_API_KEY"); ok {
primaryLLM.apiKey = &raw
}
if raw, ok := lookup("AUDITA_MODEL"); ok {
primaryLLM.model = &raw
}
if raw, ok := lookup("AUDITA_BASE_URL"); ok {
primaryLLM.baseURL = &raw
}
if raw, ok := lookup("AUDITA_LLM_TIMEOUT_SECONDS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_LLM_TIMEOUT_SECONDS: %w", err)
}
primaryLLM.timeoutSeconds = &value
}
if raw, ok := lookup("AUDITA_MAX_RETRIES"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_MAX_RETRIES: %w", err)
}
primaryLLM.maxRetries = &value
}
cfg.applyPrimaryLLMTargetPatch(primaryLLM)
validationLLM := llmTargetPatch{}
if raw, ok := lookup("AUDITA_VALIDATION_LLM_API_KEY"); ok {
validationLLM.apiKey = &raw
}
if raw, ok := lookup("AUDITA_VALIDATION_MODEL"); ok {
validationLLM.model = &raw
}
if raw, ok := lookup("AUDITA_VALIDATION_BASE_URL"); ok {
validationLLM.baseURL = &raw
}
if raw, ok := lookup("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS: %w", err)
}
validationLLM.timeoutSeconds = &value
}
if raw, ok := lookup("AUDITA_VALIDATION_MAX_RETRIES"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_VALIDATION_MAX_RETRIES: %w", err)
}
validationLLM.maxRetries = &value
}
cfg.applyValidationLLMTargetPatch(validationLLM)
concurrency := concurrencyPatch{
inheritProposal: true,
allowLegacyAlias: true,
}
if raw, ok := lookup("AUDITA_TOTAL_LLM_CONCURRENCY"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_TOTAL_LLM_CONCURRENCY: %w", err)
}
concurrency.totalLLM = &value
}
if raw, ok := lookup("AUDITA_LLM_CONCURRENCY"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_LLM_CONCURRENCY: %w", err)
}
concurrency.legacyTotalLLM = &value
}
if raw, ok := lookup("AUDITA_PROPOSAL_LLM_CONCURRENCY"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_PROPOSAL_LLM_CONCURRENCY: %w", err)
}
concurrency.proposalLLM = &value
}
if raw, ok := lookup("AUDITA_VALIDATION_LLM_CONCURRENCY"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_VALIDATION_LLM_CONCURRENCY: %w", err)
}
concurrency.validationLLM = &value
}
cfg.applyConcurrencyPatch(concurrency)
if raw, ok := lookup("AUDITA_VALIDATION_MAX_PROMPT_TOKENS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_VALIDATION_MAX_PROMPT_TOKENS: %w", err)
}
cfg.ValidationMaxPromptTokens = value
}
chunking := chunkingPatch{}
if raw, ok := lookup("AUDITA_MAX_SECTION_TOKENS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_MAX_SECTION_TOKENS: %w", err)
}
chunking.maxSectionTokens = &value
}
if raw, ok := lookup("AUDITA_MIN_SECTION_TOKENS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_MIN_SECTION_TOKENS: %w", err)
}
chunking.minSectionTokens = &value
}
if raw, ok := lookup("AUDITA_TARGET_SECTIONS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_TARGET_SECTIONS: %w", err)
}
chunking.targetSections = &value
}
cfg.applyChunkingPatch(chunking)
thresholds := thresholdsPatch{}
if raw, ok := lookup("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD: %w", err)
}
thresholds.glossary = &value
}
if raw, ok := lookup("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD: %w", err)
}
thresholds.grammar = &value
}
if raw, ok := lookup("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD: %w", err)
}
thresholds.homophones = &value
}
if raw, ok := lookup("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD: %w", err)
}
thresholds.spokenWord = &value
}
cfg.applyThresholdsPatch(thresholds)
normalization := normalizationPatch{}
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_GAP"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_GAP: %w", err)
}
normalization.maxSegmentGap = &value
}
if raw, ok := lookup("AUDITA_NORMALIZE_ELLIPSIS_GAP"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_NORMALIZE_ELLIPSIS_GAP: %w", err)
}
normalization.ellipsisGap = &value
}
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION"); ok {
value, err := parseFloat(raw)
if err != nil {
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION: %w", err)
}
normalization.maxSegmentDuration = &value
}
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS"); ok {
value, err := parseInt(raw)
if err != nil {
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS: %w", err)
}
normalization.maxSegmentTokens = &value
}
cfg.applyNormalizationPatch(normalization)
diagnostics := diagnosticsPatch{}
if raw, ok := lookup("AUDITA_WORK_DIR"); ok {
diagnostics.workDir = &raw
}
if raw, ok := lookup("AUDITA_WORK_DIR_RETENTION"); ok {
diagnostics.workDirRetention = &raw
}
cfg.applyDiagnosticsPatch(diagnostics)
cfg.syncLegacyConcurrencyAliases()
if err := cfg.Validate(); err != nil {
return err
}
return nil
}
func parseInt(raw string) (int, error) {
value, err := strconv.Atoi(raw)
if err != nil {
return 0, fmt.Errorf("must be an integer")
}
return value, nil
}
func parseFloat(raw string) (float64, error) {
value, err := strconv.ParseFloat(raw, 64)
if err != nil {
return 0, fmt.Errorf("must be a number")
}
return value, nil
}

View File

@@ -0,0 +1,322 @@
package config
import (
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"gopkg.in/yaml.v3"
)
const SupportedFileConfigVersion = 1
var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
type FileConfig struct {
Version int `yaml:"version"`
Pipeline *FileConfigPipeline `yaml:"pipeline,omitempty"`
Output *FileConfigOutput `yaml:"output,omitempty"`
LLM *FileConfigLLM `yaml:"llm,omitempty"`
Concurrency *FileConfigConcurrency `yaml:"concurrency,omitempty"`
Chunking *FileConfigChunking `yaml:"chunking,omitempty"`
Normalization *FileConfigNormalization `yaml:"normalization,omitempty"`
Thresholds *FileConfigThresholds `yaml:"thresholds,omitempty"`
Context *FileConfigContext `yaml:"context,omitempty"`
Diagnostics *FileConfigDiagnostics `yaml:"diagnostics,omitempty"`
}
type FileConfigPipeline struct {
Modules []string `yaml:"modules,omitempty"`
}
type FileConfigOutput struct {
Schema *string `yaml:"schema,omitempty"`
}
type FileConfigLLM struct {
Proposal *FileConfigLLMTarget `yaml:"proposal,omitempty"`
Validation *FileConfigLLMTarget `yaml:"validation,omitempty"`
}
type FileConfigLLMTarget struct {
BaseURL *string `yaml:"base_url,omitempty"`
Model *string `yaml:"model,omitempty"`
APIKeyEnv *string `yaml:"api_key_env,omitempty"`
Timeout *fileConfigDurationOrInt `yaml:"timeout,omitempty"`
MaxRetries *int `yaml:"max_retries,omitempty"`
}
type FileConfigConcurrency struct {
TotalLLM *int `yaml:"total_llm,omitempty"`
ProposalLLM *int `yaml:"proposal_llm,omitempty"`
ValidationLLM *int `yaml:"validation_llm,omitempty"`
}
type FileConfigChunking struct {
TargetSections *int `yaml:"target_sections,omitempty"`
MaxSectionTokens *int `yaml:"max_section_tokens,omitempty"`
MinSectionTokens *int `yaml:"min_section_tokens,omitempty"`
}
type FileConfigNormalization struct {
MaxSegmentGap *fileConfigDurationOrFloat `yaml:"max_segment_gap,omitempty"`
EllipsisGap *fileConfigDurationOrFloat `yaml:"ellipsis_gap,omitempty"`
MaxSegmentDuration *fileConfigDurationOrFloat `yaml:"max_segment_duration,omitempty"`
MaxSegmentTokens *int `yaml:"max_segment_tokens,omitempty"`
}
type FileConfigThresholds struct {
Glossary *float64 `yaml:"glossary,omitempty"`
Homophones *float64 `yaml:"homophones,omitempty"`
SpokenWord *float64 `yaml:"spoken_word,omitempty"`
Grammar *float64 `yaml:"grammar,omitempty"`
}
type FileConfigContext struct {
Description *string `yaml:"description,omitempty"`
}
type FileConfigDiagnostics struct {
WorkDir *string `yaml:"work_dir,omitempty"`
Retention *string `yaml:"retention,omitempty"`
}
type fileConfigDurationOrInt struct {
seconds int
}
func (v *fileConfigDurationOrInt) UnmarshalYAML(node *yaml.Node) error {
switch node.Kind {
case yaml.ScalarNode:
if node.Tag == "!!int" {
var n int
if err := node.Decode(&n); err != nil {
return fmt.Errorf("must be an integer seconds value or duration string")
}
v.seconds = n
return nil
}
var s string
if err := node.Decode(&s); err != nil {
return fmt.Errorf("must be an integer seconds value or duration string")
}
d, err := time.ParseDuration(strings.TrimSpace(s))
if err != nil {
return fmt.Errorf("invalid duration %q", s)
}
if d <= 0 {
v.seconds = int(d / time.Second)
return nil
}
if d%time.Second != 0 {
return fmt.Errorf("duration %q must resolve to whole seconds", s)
}
v.seconds = int(d / time.Second)
return nil
default:
return fmt.Errorf("must be an integer seconds value or duration string")
}
}
func (v fileConfigDurationOrInt) Seconds() int { return v.seconds }
type fileConfigDurationOrFloat struct {
seconds float64
}
func (v *fileConfigDurationOrFloat) UnmarshalYAML(node *yaml.Node) error {
switch node.Kind {
case yaml.ScalarNode:
if node.Tag == "!!int" || node.Tag == "!!float" {
var f float64
if err := node.Decode(&f); err != nil {
return fmt.Errorf("must be a numeric seconds value or duration string")
}
v.seconds = f
return nil
}
var s string
if err := node.Decode(&s); err != nil {
return fmt.Errorf("must be a numeric seconds value or duration string")
}
d, err := time.ParseDuration(strings.TrimSpace(s))
if err != nil {
return fmt.Errorf("invalid duration %q", s)
}
v.seconds = d.Seconds()
return nil
default:
return fmt.Errorf("must be a numeric seconds value or duration string")
}
}
func (v fileConfigDurationOrFloat) Seconds() float64 { return v.seconds }
func LoadFileConfig(path string) (FileConfig, error) {
b, err := os.ReadFile(path)
if err != nil {
return FileConfig{}, fmt.Errorf("read config file %q: %w", path, err)
}
cfg, err := ParseFileConfigYAML(b)
if err != nil {
return FileConfig{}, fmt.Errorf("parse config file %q: %w", path, err)
}
return cfg, nil
}
func ParseFileConfigYAML(data []byte) (FileConfig, error) {
var fileCfg FileConfig
dec := yaml.NewDecoder(strings.NewReader(string(data)))
dec.KnownFields(true)
if err := dec.Decode(&fileCfg); err != nil {
return FileConfig{}, fmt.Errorf("decode yaml: %w", err)
}
if fileCfg.Version == 0 {
return FileConfig{}, fmt.Errorf("config version is required")
}
if fileCfg.Version != SupportedFileConfigVersion {
return FileConfig{}, fmt.Errorf("unsupported config version %d", fileCfg.Version)
}
return fileCfg, nil
}
func (c *Config) ApplyFileConfig(fileCfg FileConfig) error {
return c.applyFileConfigWithLookup(fileCfg, os.LookupEnv)
}
func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
if c == nil {
return fmt.Errorf("config must not be nil")
}
if fileCfg.Version != SupportedFileConfigVersion {
return fmt.Errorf("unsupported config version %d", fileCfg.Version)
}
if fileCfg.Pipeline != nil && len(fileCfg.Pipeline.Modules) > 0 {
c.Modules = append([]string(nil), fileCfg.Pipeline.Modules...)
}
if fileCfg.Output != nil && fileCfg.Output.Schema != nil {
c.OutputSchema = strings.TrimSpace(*fileCfg.Output.Schema)
}
if fileCfg.LLM != nil {
if fileCfg.LLM.Proposal != nil {
patch := llmTargetPatch{
model: fileCfg.LLM.Proposal.Model,
baseURL: fileCfg.LLM.Proposal.BaseURL,
maxRetries: fileCfg.LLM.Proposal.MaxRetries,
}
if fileCfg.LLM.Proposal.Timeout != nil {
timeoutSeconds := fileCfg.LLM.Proposal.Timeout.Seconds()
patch.timeoutSeconds = &timeoutSeconds
}
if fileCfg.LLM.Proposal.APIKeyEnv != nil {
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Proposal.APIKeyEnv, lookup)
if err != nil {
return fmt.Errorf("llm.proposal.api_key_env: %w", err)
}
patch.apiKey = &apiKey
}
c.applyPrimaryLLMTargetPatch(patch)
}
if fileCfg.LLM.Validation != nil {
patch := llmTargetPatch{
model: fileCfg.LLM.Validation.Model,
baseURL: fileCfg.LLM.Validation.BaseURL,
maxRetries: fileCfg.LLM.Validation.MaxRetries,
}
if fileCfg.LLM.Validation.Timeout != nil {
timeoutSeconds := fileCfg.LLM.Validation.Timeout.Seconds()
patch.timeoutSeconds = &timeoutSeconds
}
if fileCfg.LLM.Validation.APIKeyEnv != nil {
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Validation.APIKeyEnv, lookup)
if err != nil {
return fmt.Errorf("llm.validation.api_key_env: %w", err)
}
patch.apiKey = &apiKey
}
c.applyValidationLLMTargetPatch(patch)
}
}
if fileCfg.Concurrency != nil {
c.applyConcurrencyPatch(concurrencyPatch{
totalLLM: fileCfg.Concurrency.TotalLLM,
proposalLLM: fileCfg.Concurrency.ProposalLLM,
validationLLM: fileCfg.Concurrency.ValidationLLM,
})
}
if fileCfg.Chunking != nil {
c.applyChunkingPatch(chunkingPatch{
targetSections: fileCfg.Chunking.TargetSections,
maxSectionTokens: fileCfg.Chunking.MaxSectionTokens,
minSectionTokens: fileCfg.Chunking.MinSectionTokens,
})
}
if fileCfg.Normalization != nil {
patch := normalizationPatch{
maxSegmentTokens: fileCfg.Normalization.MaxSegmentTokens,
}
if fileCfg.Normalization.MaxSegmentGap != nil {
maxSegmentGap := fileCfg.Normalization.MaxSegmentGap.Seconds()
patch.maxSegmentGap = &maxSegmentGap
}
if fileCfg.Normalization.EllipsisGap != nil {
ellipsisGap := fileCfg.Normalization.EllipsisGap.Seconds()
patch.ellipsisGap = &ellipsisGap
}
if fileCfg.Normalization.MaxSegmentDuration != nil {
maxSegmentDuration := fileCfg.Normalization.MaxSegmentDuration.Seconds()
patch.maxSegmentDuration = &maxSegmentDuration
}
c.applyNormalizationPatch(patch)
}
if fileCfg.Thresholds != nil {
c.applyThresholdsPatch(thresholdsPatch{
glossary: fileCfg.Thresholds.Glossary,
grammar: fileCfg.Thresholds.Grammar,
homophones: fileCfg.Thresholds.Homophones,
spokenWord: fileCfg.Thresholds.SpokenWord,
})
}
if fileCfg.Context != nil && fileCfg.Context.Description != nil {
c.applyContextPatch(contextPatch{transcriptDescription: fileCfg.Context.Description})
}
if fileCfg.Diagnostics != nil {
c.applyDiagnosticsPatch(diagnosticsPatch{
workDir: fileCfg.Diagnostics.WorkDir,
workDirRetention: fileCfg.Diagnostics.Retention,
})
}
c.syncLegacyConcurrencyAliases()
if err := c.Validate(); err != nil {
return err
}
return nil
}
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
name := strings.TrimSpace(envName)
if name == "" {
return "", fmt.Errorf("must not be empty")
}
if !envVarNamePattern.MatchString(name) {
return "", fmt.Errorf("must be an environment variable name")
}
if strings.Contains(name, string(filepath.Separator)) {
return "", fmt.Errorf("must be an environment variable name")
}
v, _ := lookup(name)
return v, nil
}

View File

@@ -0,0 +1,290 @@
package config
import (
"os"
"strings"
"testing"
)
func TestParseFileConfigYAMLValid(t *testing.T) {
raw := `
version: 1
pipeline:
modules: [glossary, homophones, grammar]
output:
schema: audita-v1
llm:
proposal:
base_url: https://example.test/v1
model: provider/model-a
api_key_env: AUDITA_PROPOSAL_KEY
timeout: 2m
max_retries: 4
validation:
base_url: https://example.test/validation
model: provider/model-b
api_key_env: AUDITA_VALIDATION_KEY
timeout: 45
max_retries: 3
concurrency:
total_llm: 8
proposal_llm: 4
validation_llm: 2
chunking:
target_sections: 6
max_section_tokens: 9000
min_section_tokens: 3000
normalization:
max_segment_gap: 1.5s
ellipsis_gap: 2
max_segment_duration: 45s
max_segment_tokens: 1500
thresholds:
glossary: 0.9
homophones: 0.7
spoken_word: 0.8
grammar: 0.75
context:
description: " crowd scene with many proper nouns "
diagnostics:
work_dir: /tmp/audita-config
retention: always
`
cfg, err := ParseFileConfigYAML([]byte(raw))
if err != nil {
t.Fatalf("ParseFileConfigYAML error: %v", err)
}
if cfg.Version != 1 {
t.Fatalf("expected version 1, got %d", cfg.Version)
}
if cfg.Pipeline == nil || len(cfg.Pipeline.Modules) != 3 {
t.Fatalf("unexpected pipeline modules: %#v", cfg.Pipeline)
}
if cfg.Output == nil || cfg.Output.Schema == nil || *cfg.Output.Schema != "audita-v1" {
t.Fatalf("expected output schema audita-v1, got %#v", cfg.Output)
}
if cfg.LLM == nil || cfg.LLM.Proposal == nil || cfg.LLM.Validation == nil {
t.Fatalf("expected llm proposal+validation blocks")
}
if cfg.LLM.Proposal.Timeout == nil || cfg.LLM.Proposal.Timeout.Seconds() != 120 {
t.Fatalf("expected proposal timeout 120s, got %#v", cfg.LLM.Proposal.Timeout)
}
if cfg.LLM.Validation.Timeout == nil || cfg.LLM.Validation.Timeout.Seconds() != 45 {
t.Fatalf("expected validation timeout 45s, got %#v", cfg.LLM.Validation.Timeout)
}
if cfg.Normalization == nil || cfg.Normalization.MaxSegmentGap == nil || cfg.Normalization.MaxSegmentGap.Seconds() != 1.5 {
t.Fatalf("expected parsed duration for normalization max_segment_gap")
}
}
func TestParseFileConfigYAMLRejectsUnknownField(t *testing.T) {
raw := `
version: 1
pipeline:
modules: [grammar]
output:
unknown: v1
`
_, err := ParseFileConfigYAML([]byte(raw))
if err == nil {
t.Fatalf("expected unknown field error")
}
if !strings.Contains(err.Error(), "field unknown not found") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestParseFileConfigYAMLRejectsMissingVersion(t *testing.T) {
raw := `pipeline: {modules: [grammar]}`
_, err := ParseFileConfigYAML([]byte(raw))
if err == nil {
t.Fatalf("expected missing version error")
}
if !strings.Contains(err.Error(), "config version is required") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestParseFileConfigYAMLRejectsUnsupportedVersion(t *testing.T) {
raw := `version: 2`
_, err := ParseFileConfigYAML([]byte(raw))
if err == nil {
t.Fatalf("expected unsupported version error")
}
if !strings.Contains(err.Error(), "unsupported config version 2") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestApplyFileConfigParsesAndMergesFields(t *testing.T) {
raw := `
version: 1
pipeline:
modules: [spoken_word, grammar]
output:
schema: audita-v1
llm:
proposal:
model: provider/new-proposal
api_key_env: PROPOSAL_KEY_NAME
timeout: 90s
max_retries: 5
validation:
model: provider/new-validation
api_key_env: VALIDATION_KEY_NAME
timeout: 150
max_retries: 6
concurrency:
total_llm: 7
proposal_llm: 3
validation_llm: 2
chunking:
target_sections: 9
thresholds:
glossary: 0.91
homophones: 0.61
spoken_word: 0.71
grammar: 0.81
diagnostics:
retention: never
`
fileCfg, err := ParseFileConfigYAML([]byte(raw))
if err != nil {
t.Fatalf("ParseFileConfigYAML error: %v", err)
}
cfg := Default()
lookup := func(name string) (string, bool) {
switch name {
case "PROPOSAL_KEY_NAME":
return "proposal-secret", true
case "VALIDATION_KEY_NAME":
return "validation-secret", true
default:
return "", false
}
}
if err := cfg.applyFileConfigWithLookup(fileCfg, lookup); err != nil {
t.Fatalf("applyFileConfigWithLookup error: %v", err)
}
if strings.Join(cfg.Modules, ",") != "spoken_word,grammar" {
t.Fatalf("unexpected modules: %#v", cfg.Modules)
}
if cfg.OutputSchema != "audita-v1" {
t.Fatalf("unexpected output schema: %q", cfg.OutputSchema)
}
if cfg.PrimaryLLM.Model != "provider/new-proposal" {
t.Fatalf("unexpected proposal model: %q", cfg.PrimaryLLM.Model)
}
if cfg.PrimaryLLM.APIKey != "proposal-secret" {
t.Fatalf("expected proposal key from api_key_env lookup, got %q", cfg.PrimaryLLM.APIKey)
}
if cfg.PrimaryLLM.TimeoutSeconds != 90 {
t.Fatalf("unexpected proposal timeout: %d", cfg.PrimaryLLM.TimeoutSeconds)
}
if cfg.ValidationLLM.Model != "provider/new-validation" {
t.Fatalf("unexpected validation model: %q", cfg.ValidationLLM.Model)
}
if cfg.ValidationLLM.APIKey != "validation-secret" {
t.Fatalf("expected validation key from api_key_env lookup, got %q", cfg.ValidationLLM.APIKey)
}
if cfg.ValidationLLM.TimeoutSeconds == nil || *cfg.ValidationLLM.TimeoutSeconds != 150 {
t.Fatalf("unexpected validation timeout: %#v", cfg.ValidationLLM.TimeoutSeconds)
}
if cfg.TotalLLMConcurrency != 7 || cfg.ProposalLLMConcurrency != 3 {
t.Fatalf("unexpected llm concurrency values: total=%d proposal=%d", cfg.TotalLLMConcurrency, cfg.ProposalLLMConcurrency)
}
if cfg.ValidationLLMConcurrency == nil || *cfg.ValidationLLMConcurrency != 2 {
t.Fatalf("unexpected validation llm concurrency: %#v", cfg.ValidationLLMConcurrency)
}
if cfg.TargetSections == nil || *cfg.TargetSections != 9 {
t.Fatalf("unexpected target sections: %#v", cfg.TargetSections)
}
if cfg.WorkDirRetention != WorkDirRetentionNever {
t.Fatalf("unexpected retention: %q", cfg.WorkDirRetention)
}
if cfg.PrimaryLLM.Concurrency != 7 {
t.Fatalf("expected legacy alias to sync, got %d", cfg.PrimaryLLM.Concurrency)
}
if cfg.ValidationLLM.Concurrency == nil || *cfg.ValidationLLM.Concurrency != 2 {
t.Fatalf("expected validation alias to sync, got %#v", cfg.ValidationLLM.Concurrency)
}
}
func TestApplyFileConfigContextDescriptionTrim(t *testing.T) {
raw := `
version: 1
context:
description: " scene context "
`
fileCfg, err := ParseFileConfigYAML([]byte(raw))
if err != nil {
t.Fatalf("ParseFileConfigYAML error: %v", err)
}
cfg := Default()
if err := cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{})); err != nil {
t.Fatalf("applyFileConfigWithLookup error: %v", err)
}
if cfg.TranscriptDescription != "scene context" {
t.Fatalf("unexpected transcript description: %q", cfg.TranscriptDescription)
}
}
func TestApplyFileConfigRejectsInvalidAPIKeyEnvName(t *testing.T) {
raw := `
version: 1
llm:
proposal:
api_key_env: "not a var name"
`
fileCfg, err := ParseFileConfigYAML([]byte(raw))
if err != nil {
t.Fatalf("ParseFileConfigYAML error: %v", err)
}
cfg := Default()
err = cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{}))
if err == nil {
t.Fatalf("expected api_key_env validation error")
}
if !strings.Contains(err.Error(), "environment variable name") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestParseFileConfigDurationParsingErrors(t *testing.T) {
raw := `
version: 1
llm:
proposal:
timeout: "1.5s"
`
_, err := ParseFileConfigYAML([]byte(raw))
if err == nil {
t.Fatalf("expected duration parse error")
}
if !strings.Contains(err.Error(), "whole seconds") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestLoadFileConfigReadsFromPath(t *testing.T) {
p := writeTempFileConfig(t, "version: 1\n")
cfg, err := LoadFileConfig(p)
if err != nil {
t.Fatalf("LoadFileConfig error: %v", err)
}
if cfg.Version != 1 {
t.Fatalf("expected version 1, got %d", cfg.Version)
}
}
func writeTempFileConfig(t *testing.T, contents string) string {
t.Helper()
dir := t.TempDir()
path := dir + "/config.yaml"
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
t.Fatalf("write config file: %v", err)
}
return path
}

View File

@@ -0,0 +1,106 @@
package config
import (
"fmt"
"strings"
)
type CLIOverrides struct {
ModulesCSV *string
OutputSchema *string
PrimaryLLMAPIKey *string
ValidationLLMAPIKey *string
PrimaryModel *string
ValidationModel *string
PrimaryBaseURL *string
ValidationBaseURL *string
PrimaryLLMTimeoutSeconds *int
TotalLLMConcurrency *int
ProposalLLMConcurrency *int
PrimaryLLMConcurrency *int
ValidationLLMTimeoutSeconds *int
MaxRetries *int
ValidationMaxRetries *int
ValidationLLMConcurrency *int
ValidationMaxPromptTokens *int
MaxSectionTokens *int
MinSectionTokens *int
TargetSections *int
GlossaryConfidenceThreshold *float64
GrammarConfidenceThreshold *float64
HomophonesConfidenceThreshold *float64
SpokenWordConfidenceThreshold *float64
NormalizeMaxSegmentGap *float64
NormalizeEllipsisGap *float64
NormalizeMaxSegmentDuration *float64
NormalizeMaxSegmentTokens *int
TranscriptDescription *string
WorkDir *string
WorkDirRetention *string
}
func (c *Config) ApplyCLIOverrides(overrides CLIOverrides) error {
if overrides.ModulesCSV != nil {
modules, err := ParseModulesCSV(*overrides.ModulesCSV)
if err != nil {
return fmt.Errorf("--modules: %w", err)
}
c.Modules = modules
}
if overrides.OutputSchema != nil {
c.OutputSchema = strings.TrimSpace(*overrides.OutputSchema)
}
c.applyPrimaryLLMTargetPatch(llmTargetPatch{
apiKey: overrides.PrimaryLLMAPIKey,
model: overrides.PrimaryModel,
baseURL: overrides.PrimaryBaseURL,
timeoutSeconds: overrides.PrimaryLLMTimeoutSeconds,
maxRetries: overrides.MaxRetries,
})
c.applyValidationLLMTargetPatch(llmTargetPatch{
apiKey: overrides.ValidationLLMAPIKey,
model: overrides.ValidationModel,
baseURL: overrides.ValidationBaseURL,
timeoutSeconds: overrides.ValidationLLMTimeoutSeconds,
maxRetries: overrides.ValidationMaxRetries,
})
c.applyConcurrencyPatch(concurrencyPatch{
totalLLM: overrides.TotalLLMConcurrency,
legacyTotalLLM: overrides.PrimaryLLMConcurrency,
proposalLLM: overrides.ProposalLLMConcurrency,
validationLLM: overrides.ValidationLLMConcurrency,
inheritProposal: true,
allowLegacyAlias: true,
})
if overrides.ValidationMaxPromptTokens != nil {
c.ValidationMaxPromptTokens = *overrides.ValidationMaxPromptTokens
}
c.applyChunkingPatch(chunkingPatch{
targetSections: overrides.TargetSections,
maxSectionTokens: overrides.MaxSectionTokens,
minSectionTokens: overrides.MinSectionTokens,
})
c.applyThresholdsPatch(thresholdsPatch{
glossary: overrides.GlossaryConfidenceThreshold,
grammar: overrides.GrammarConfidenceThreshold,
homophones: overrides.HomophonesConfidenceThreshold,
spokenWord: overrides.SpokenWordConfidenceThreshold,
})
c.applyNormalizationPatch(normalizationPatch{
maxSegmentGap: overrides.NormalizeMaxSegmentGap,
ellipsisGap: overrides.NormalizeEllipsisGap,
maxSegmentDuration: overrides.NormalizeMaxSegmentDuration,
maxSegmentTokens: overrides.NormalizeMaxSegmentTokens,
})
c.applyContextPatch(contextPatch{transcriptDescription: overrides.TranscriptDescription})
c.applyDiagnosticsPatch(diagnosticsPatch{
workDir: overrides.WorkDir,
workDirRetention: overrides.WorkDirRetention,
})
c.syncLegacyConcurrencyAliases()
return c.Validate()
}

View File

@@ -0,0 +1,17 @@
package config
const redactedSecret = "[REDACTED]"
func (c Config) Redacted() Config {
redacted := c
redacted.PrimaryLLM.APIKey = redactSecret(redacted.PrimaryLLM.APIKey)
redacted.ValidationLLM.APIKey = redactSecret(redacted.ValidationLLM.APIKey)
return redacted
}
func redactSecret(value string) string {
if value == "" {
return ""
}
return redactedSecret
}

Some files were not shown because too many files have changed in this diff Show More