Compare commits
90 Commits
python-fin
...
0630d36734
| Author | SHA1 | Date | |
|---|---|---|---|
| 0630d36734 | |||
| 52c2697040 | |||
| f790c1441c | |||
| 56f9b28f4b | |||
| 222222f449 | |||
| 99391cd18b | |||
| 84be774b34 | |||
| e053f7e124 | |||
| 13029dbb33 | |||
| 938bfe88c1 | |||
| fa1bd237d1 | |||
| 3d7057b437 | |||
| 32c8c8b446 | |||
| a3655f5540 | |||
| a84941d681 | |||
| 46b7356a3b | |||
| 39208cd119 | |||
| 6dae15656d | |||
| b103bb2e7d | |||
| 2b2a3fc024 | |||
| 52ffe42e73 | |||
| 3b160cf05b | |||
| 0687982822 | |||
| ff4ed82239 | |||
| 037121e9ce | |||
| d6126bf52b | |||
| 1bc5936681 | |||
| 9a77a0cd0b | |||
| ebbd2c8a63 | |||
| de99467ede | |||
| 20f612215f | |||
| 3d45571bb0 | |||
| 1afd753fad | |||
| a85a7e204e | |||
| 509436cc4a | |||
| a48f6da1f4 | |||
| df96f9fdf6 | |||
| 390daa8b84 | |||
| af84249da0 | |||
| cad172a758 | |||
| fb59cb21b9 | |||
| 68e2d9b549 | |||
| 185f7ca2b6 | |||
| 7ccadc6bd6 | |||
| a9f7fa27ff | |||
| dbf3605712 | |||
| 543a7ff8ef | |||
| fc3a7b7a67 | |||
| b360493cdc | |||
| 12202508bf | |||
| 6d9a4bd017 | |||
| 426864eedb | |||
| 0b17a6fbeb | |||
| aeb31f1c0d | |||
| 28fe899aa1 | |||
| 30606f5c49 | |||
| db880ed868 | |||
| 5217093be2 | |||
| 0e83991537 | |||
| 3e8d19cccd | |||
| c1193e3450 | |||
| 73249b63d8 | |||
| c3087aeda6 | |||
| 0452a605ad | |||
| b9b7384123 | |||
| 726acc47e1 | |||
| 5c78b1d5d9 | |||
| f461922b9b | |||
| c58d307ba7 | |||
| 1eb93481e0 | |||
| b997e7c97c | |||
| 12fd541669 | |||
| 10377876e4 | |||
| d847168ecd | |||
| 14e51698c2 | |||
| aeb9c4f062 | |||
| e2ae7f77d8 | |||
| 0b1b670baf | |||
| 3cfa4b6e8a | |||
| 950edc01f2 | |||
| ea8def423e | |||
| 95fe8c32fa | |||
| 08b7531149 | |||
| 2cf2d390da | |||
| 8f3c2ec5fd | |||
| 9427c4e6cc | |||
| 6424d7db4f | |||
| 87e560dd3d | |||
| 09fc6fd364 | |||
| 2e47c8a1b6 |
61
.gitignore
vendored
61
.gitignore
vendored
@@ -1,3 +1,4 @@
|
||||
# --> Python
|
||||
.DS_Store
|
||||
.venv/
|
||||
__pycache__/
|
||||
@@ -9,3 +10,63 @@ dist/
|
||||
build/
|
||||
*.egg-info/
|
||||
|
||||
# ---> Codex
|
||||
.codex
|
||||
AGENTS.md
|
||||
|
||||
# ---> Go
|
||||
# If you prefer the allow list template instead of the deny list, see community template:
|
||||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||
#
|
||||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
|
||||
# Dependency directories (remove the comment below to include it)
|
||||
# vendor/
|
||||
|
||||
# Go workspace file
|
||||
go.work
|
||||
go.work.sum
|
||||
|
||||
# env file
|
||||
.env
|
||||
|
||||
# Compiled binaries and test configuration
|
||||
narratio
|
||||
local-test
|
||||
pipeline.yml
|
||||
bin/
|
||||
|
||||
# Local run artifacts
|
||||
.audita-runs/
|
||||
report.json
|
||||
corrected.json
|
||||
normalized.json
|
||||
|
||||
# Coverage artifacts
|
||||
coverage.out
|
||||
coverage.txt
|
||||
|
||||
# ---> VisualStudioCode
|
||||
.vscode/*
|
||||
!.vscode/settings.json
|
||||
!.vscode/tasks.json
|
||||
!.vscode/launch.json
|
||||
!.vscode/extensions.json
|
||||
!.vscode/*.code-snippets
|
||||
|
||||
# Local History for Visual Studio Code
|
||||
.history/
|
||||
|
||||
# Built Visual Studio Code Extensions
|
||||
*.vsix
|
||||
|
||||
50
.woodpecker/release.yml
Normal file
50
.woodpecker/release.yml
Normal file
@@ -0,0 +1,50 @@
|
||||
when:
|
||||
- event: tag
|
||||
|
||||
steps:
|
||||
- name: build-release-assets
|
||||
image: golang:1.25
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
|
||||
version="$CI_COMMIT_TAG"
|
||||
dist="dist"
|
||||
pkg="gitea.maximumdirect.net/eric/audita/cmd/audita"
|
||||
|
||||
rm -rf "$dist"
|
||||
mkdir -p "$dist"
|
||||
|
||||
build_binary() {
|
||||
goos="$1"
|
||||
goarch="$2"
|
||||
suffix="$3"
|
||||
output="$dist/audita-$version-$goos-$goarch$suffix"
|
||||
|
||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \
|
||||
go build -trimpath -ldflags "-s -w -X gitea.maximumdirect.net/eric/audita/internal/buildinfo.Version=$version" \
|
||||
-o "$output" "$pkg"
|
||||
}
|
||||
|
||||
build_binary linux amd64 ""
|
||||
build_binary linux arm64 ""
|
||||
build_binary darwin amd64 ""
|
||||
build_binary darwin arm64 ""
|
||||
build_binary windows amd64 ".exe"
|
||||
build_binary windows arm64 ".exe"
|
||||
|
||||
- name: publish-release
|
||||
image: woodpeckerci/plugin-release
|
||||
depends_on:
|
||||
- build-release-assets
|
||||
settings:
|
||||
api_key:
|
||||
from_secret: GITEA_RELEASE_TOKEN
|
||||
files:
|
||||
- dist/audita-*
|
||||
checksum: sha256
|
||||
checksum-file: SHA256SUMS
|
||||
checksum-flatten: true
|
||||
file-exists: skip
|
||||
overwrite: false
|
||||
prerelease: false
|
||||
365
README.md
365
README.md
@@ -1,148 +1,305 @@
|
||||
# Audita
|
||||
|
||||
Audita is a framework-first transcript correction application. The public `audita` package provides:
|
||||
Audita is a transcript polishing CLI.
|
||||
|
||||
- deterministic transcript normalization
|
||||
- token-batched module orchestration
|
||||
- concrete `glossary`, `homophones`, `spoken_word`, and `grammar` modules built on reusable proposal / validator contracts
|
||||
- structured run reporting and work-dir diagnostics
|
||||
`audita process` validates transcript/glossary input, normalizes and chunks transcript segments, runs the default correction pipeline, and emits corrected transcript output plus machine-readable diagnostics and reports.
|
||||
|
||||
The previous working implementation has been preserved as `audita_prototype` inside this repository. Its full regression suite lives under `tests/audita_prototype`.
|
||||
## What Audita Does
|
||||
|
||||
## Development
|
||||
Default module sequence:
|
||||
- `glossary`
|
||||
- `homophones`
|
||||
- `glossary`
|
||||
- `spoken_word`
|
||||
- `grammar`
|
||||
|
||||
This project is set up for `uv`.
|
||||
Pipeline behavior includes:
|
||||
- glossary-backed domain/acoustic corrections
|
||||
- conservative homophone and mistranscription corrections
|
||||
- conservative spoken-word dysfluency cleanup with semantic guardrails
|
||||
- grammar/punctuation/capitalization/formatting cleanup
|
||||
- validator-chain enforcement before application
|
||||
- malformed module-stage LLM payloads degrade to warnings/rejections instead of failing the run
|
||||
- run reports and diagnostics artifacts with secret redaction
|
||||
|
||||
## Build and Install
|
||||
|
||||
Build a local binary:
|
||||
|
||||
```sh
|
||||
uv sync --extra dev
|
||||
uv run pytest
|
||||
go build -o ./bin/audita ./cmd/audita
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
Process a transcript with the current framework implementation:
|
||||
Install into your Go bin directory:
|
||||
|
||||
```sh
|
||||
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
go install ./cmd/audita
|
||||
```
|
||||
|
||||
The framework currently runs this default module sequence:
|
||||
|
||||
1. `glossary`
|
||||
2. `homophones`
|
||||
3. `glossary`
|
||||
4. `spoken_word`
|
||||
5. `grammar`
|
||||
|
||||
Resolved run instance names are auto-numbered for repeats, so the default report pipeline is:
|
||||
|
||||
1. `glossary_1`
|
||||
2. `homophones`
|
||||
3. `glossary_2`
|
||||
4. `spoken_word`
|
||||
5. `grammar`
|
||||
|
||||
The default module sequence is fully implemented today:
|
||||
|
||||
- `glossary` proposes glossary-supported acoustic corrections
|
||||
- `homophones` proposes conservative homophone and mistranscription corrections
|
||||
- `spoken_word` proposes conservative dysfluency cleanup
|
||||
- `grammar` proposes punctuation, capitalization, and spacing cleanup only
|
||||
|
||||
To run a custom module sequence, pass `--modules`:
|
||||
CLI help:
|
||||
|
||||
```sh
|
||||
uv run audita process transcript.json --glossary glossary.yaml --modules grammar --output corrected.json
|
||||
audita --help
|
||||
audita process --help
|
||||
audita config --help
|
||||
```
|
||||
|
||||
To also write a structured JSON report:
|
||||
## Test
|
||||
|
||||
Run all tests:
|
||||
|
||||
```sh
|
||||
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json --report-json report.json
|
||||
go test ./...
|
||||
```
|
||||
|
||||
From a checked-out repository, you can also use the root launcher:
|
||||
## Basic Usage
|
||||
|
||||
Required inputs:
|
||||
- transcript JSON path (positional argument)
|
||||
- `--glossary <glossary.yaml>`
|
||||
|
||||
Recommended run:
|
||||
|
||||
```sh
|
||||
./audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
audita process transcript.json \
|
||||
--glossary glossary.yaml \
|
||||
--output corrected.json \
|
||||
--report-json report.json
|
||||
```
|
||||
|
||||
For a system-wide command, install the source tree under `/usr/local/src/audita`, sync dependencies there, and symlink the root launcher into your `PATH`:
|
||||
Select an explicit output schema (default is `bare-segments`):
|
||||
|
||||
```sh
|
||||
cd /usr/local/src/audita
|
||||
uv sync --extra dev
|
||||
ln -s /usr/local/src/audita/audita /usr/local/bin/audita
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
audita process transcript.json \
|
||||
--glossary glossary.yaml \
|
||||
--output-schema audita-v1 \
|
||||
--output corrected.json \
|
||||
--report-json report.json
|
||||
```
|
||||
|
||||
Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr.
|
||||
`--report-json` writes a separate machine-readable run report and never mixes report data into stdout.
|
||||
|
||||
Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Default OpenRouter runs require LLM API credentials, because the `glossary`, `homophones`, `spoken_word`, and `grammar` modules make real LLM calls. Self-hosted or other non-default OpenAI-compatible endpoints may not require credentials. `AUDITA_LLM_API_KEY` and `--llm-api-key` are the preferred provider-neutral credential surfaces, while `OPENROUTER_API_KEY` remains supported as a backward-compatible fallback.
|
||||
|
||||
| Environment variable | CLI flag | Default | Purpose |
|
||||
| --- | --- | --- | --- |
|
||||
| `AUDITA_MODULES` | `--modules` | `glossary,homophones,glossary,spoken_word,grammar` | Comma-separated logical module keys to run; CLI overrides the environment value |
|
||||
| `AUDITA_LLM_API_KEY` | `--llm-api-key` | unset | Preferred provider-neutral LLM API credential; required for the default OpenRouter endpoint and optional for non-default endpoints; CLI overrides both environment-key variants |
|
||||
| `AUDITA_VALIDATION_LLM_API_KEY` | `--validation-llm-api-key` | unset | Validation-phase LLM API credential; defaults to the primary LLM API key and is optional for non-default validation endpoints |
|
||||
| `AUDITA_MODEL` | `--model` | `openrouter/google/gemma-4-31b-it` | LLM model name sent to the configured OpenAI-compatible endpoint |
|
||||
| `AUDITA_VALIDATION_MODEL` | `--validation-model` | unset | Validation-phase LLM model; defaults to `AUDITA_MODEL` |
|
||||
| `AUDITA_BASE_URL` | `--base-url` | `https://openrouter.ai/api/v1` | OpenAI-compatible API base URL |
|
||||
| `AUDITA_VALIDATION_BASE_URL` | `--validation-base-url` | unset | Validation-phase OpenAI-compatible API base URL; defaults to `AUDITA_BASE_URL` |
|
||||
| `AUDITA_LLM_TIMEOUT_SECONDS` | `--llm-timeout-seconds` | `600` | Per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint |
|
||||
| `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | `--validation-llm-timeout-seconds` | unset | Validation-phase per-request timeout in seconds; defaults to `AUDITA_LLM_TIMEOUT_SECONDS` |
|
||||
| `AUDITA_VALIDATION_MAX_PROMPT_TOKENS` | `--validation-max-prompt-tokens` | `2048` | Maximum estimated tokens per validation-phase LLM prompt batch |
|
||||
| `AUDITA_TARGET_SECTIONS` | `--target-sections` | unset | Exact number of contiguous proposal-stage transcript sections; errors if min/max token bounds cannot be satisfied |
|
||||
| `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses |
|
||||
| `AUDITA_VALIDATION_MAX_RETRIES` | `--validation-max-retries` | unset | Validation-phase structured-output retries; defaults to `AUDITA_MAX_RETRIES` |
|
||||
| `AUDITA_VALIDATION_LLM_CONCURRENCY` | `--validation-llm-concurrency` | unset | Validation-phase LLM concurrency; defaults to `AUDITA_LLM_CONCURRENCY` |
|
||||
| `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `8192` | Maximum estimated tokens per proposal-stage transcript section |
|
||||
| `AUDITA_MIN_SECTION_TOKENS` | `--min-section-tokens` | `2048` | Minimum estimated tokens per proposal-stage transcript section when balancing for concurrency |
|
||||
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation |
|
||||
| `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | `--grammar-confidence-threshold` | `0.8` | Minimum confidence required for grammar proposals to survive validation |
|
||||
| `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | `--homophones-confidence-threshold` | `0.8` | Minimum confidence required for homophone proposals to survive validation |
|
||||
| `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | `--spoken-word-confidence-threshold` | `0.8` | Minimum confidence required for spoken-word proposals to survive validation |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | `--normalize-max-segment-gap` | `4.0` | Same-speaker gaps eligible for deterministic merging |
|
||||
| `AUDITA_NORMALIZE_ELLIPSIS_GAP` | `--normalize-ellipsis-gap` | `3.5` | Same-speaker gaps above this value are joined with ` ... ` |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | `--normalize-max-segment-duration` | `60.0` | Maximum merged segment duration |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | `--normalize-max-segment-tokens` | `2048` | Maximum merged segment prompt payload size |
|
||||
| `AUDITA_WORK_DIR` | `--work-dir` | `/tmp/audita` | Per-run scratch diagnostics directory |
|
||||
| `AUDITA_WORK_DIR_RETENTION` | `--work-dir-retention` | `auto` | Whether to retain the per-run work directory: `auto`, `always`, or `never` |
|
||||
|
||||
Set `AUDITA_MODULES=grammar` to run only the grammar module by default, or override it per command with `--modules`.
|
||||
|
||||
Validation-phase LLM settings inherit from the primary `AUDITA_*` LLM settings by default. Set any of the `AUDITA_VALIDATION_*` values only when you want LLM-backed validators to use a different model, endpoint, credential, timeout, retry budget, or concurrency level.
|
||||
|
||||
OpenRouter remains the default out of the box:
|
||||
Recommended config-based run:
|
||||
|
||||
```sh
|
||||
export AUDITA_LLM_API_KEY=your-openrouter-key
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
audita process transcript.json \
|
||||
--glossary glossary.yaml \
|
||||
--config audita.yml \
|
||||
--output corrected.json \
|
||||
--report-json report.json
|
||||
```
|
||||
|
||||
You can point Audita at any OpenAI-compatible endpoint by changing `AUDITA_BASE_URL` and, if needed, `AUDITA_MODEL`. For example, a local vLLM server:
|
||||
Explicit module override:
|
||||
|
||||
```sh
|
||||
export AUDITA_BASE_URL=http://localhost:8000/v1
|
||||
export AUDITA_MODEL=meta-llama/Llama-3.1-8B-Instruct
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
audita process transcript.json \
|
||||
--glossary glossary.yaml \
|
||||
--modules glossary,homophones,grammar \
|
||||
--output corrected.json \
|
||||
--report-json report.json
|
||||
```
|
||||
|
||||
If your self-hosted endpoint requires authentication, you can still set `AUDITA_LLM_API_KEY`; Audita simply no longer requires it for non-default endpoints.
|
||||
|
||||
Or the actual OpenAI API:
|
||||
Optional transcript background context:
|
||||
|
||||
```sh
|
||||
export AUDITA_LLM_API_KEY=your-openai-key
|
||||
export AUDITA_BASE_URL=https://api.openai.com/v1
|
||||
export AUDITA_MODEL=gpt-4.1-mini
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
audita process transcript.json \
|
||||
--glossary glossary.yaml \
|
||||
--transcript-description "Brief context that may help resolve ambiguous terms." \
|
||||
--output corrected.json
|
||||
```
|
||||
|
||||
`AUDITA_WORK_DIR` stores per-run diagnostics while processing. Under the default `AUDITA_WORK_DIR_RETENTION=auto`, clean successful runs are removed, while failed runs and successful runs with final skipped corrections are preserved. Use `always` to keep every run directory and `never` to remove successful run directories even when skips remain.
|
||||
Failed runs always preserve the run directory and include an authoritative `report.json` alongside normalization and prompt/response diagnostics.
|
||||
The transcript description is background context only and does not override transcript content.
|
||||
|
||||
## Prototype Archive
|
||||
Write transcript JSON to stdout (no `--output`):
|
||||
|
||||
The archived prototype remains importable as `audita_prototype` and is still covered by its original regression suite. This is intentional: the new `audita` package is a framework-oriented rewrite, not a thin wrapper around the old code.
|
||||
```sh
|
||||
audita process transcript.json --glossary glossary.yaml
|
||||
```
|
||||
|
||||
Control diagnostics location/retention:
|
||||
|
||||
```sh
|
||||
audita process transcript.json \
|
||||
--glossary glossary.yaml \
|
||||
--work-dir /tmp/audita \
|
||||
--work-dir-retention auto \
|
||||
--output corrected.json \
|
||||
--report-json report.json
|
||||
```
|
||||
|
||||
## Stdout/Stderr Contract
|
||||
|
||||
- With `--output`, stdout is expected to be empty on success.
|
||||
- Without `--output`, stdout contains transcript JSON only on success.
|
||||
- `--report-json` writes a file and is never printed to stdout.
|
||||
- stderr is human-readable diagnostics/errors.
|
||||
- successful runs remain quiet on stderr even when module warnings are recorded in report/diagnostics artifacts.
|
||||
|
||||
For subprocess orchestration guidance, see [`docs/subprocess-operations.md`](docs/subprocess-operations.md).
|
||||
|
||||
## Configuration
|
||||
|
||||
Precedence:
|
||||
1. defaults
|
||||
2. config file (`--config`, `AUDITA_CONFIG`, or default search paths when present: `/usr/local/etc/audita/config.yml`, then `/etc/audita/config.yml`)
|
||||
3. environment (`AUDITA_*`)
|
||||
4. CLI flags
|
||||
|
||||
Config commands:
|
||||
|
||||
```sh
|
||||
audita config validate --config audita.yml
|
||||
audita config print-effective --config audita.yml
|
||||
```
|
||||
|
||||
For full config-file schema and examples, see [`docs/configuration.md`](docs/configuration.md).
|
||||
For output-schema details, see [`docs/architecture/output-schemas.md`](docs/architecture/output-schemas.md).
|
||||
For built-in validator keys and chain definitions, see [`docs/architecture/validators.md`](docs/architecture/validators.md).
|
||||
For embedded prompt assets and prompt metadata behavior, see [`docs/architecture/prompts.md`](docs/architecture/prompts.md).
|
||||
For CLI/process compatibility guarantees, see [`docs/architecture/public-contract.md`](docs/architecture/public-contract.md).
|
||||
|
||||
### Modules
|
||||
|
||||
- `AUDITA_MODULES` (CSV)
|
||||
- CLI: `--modules`
|
||||
|
||||
### Transcript Description
|
||||
|
||||
CLI:
|
||||
- `--transcript-description`
|
||||
|
||||
Behavior:
|
||||
- optional background context for proposal and LLM-validator prompts;
|
||||
- trimmed and length-limited by CLI validation;
|
||||
- does not override transcript content;
|
||||
- no `AUDITA_*` environment variable is currently defined for this setting.
|
||||
|
||||
### Primary LLM
|
||||
|
||||
Environment:
|
||||
- `AUDITA_LLM_API_KEY` (or `OPENROUTER_API_KEY` fallback)
|
||||
- `AUDITA_MODEL`
|
||||
- `AUDITA_BASE_URL`
|
||||
- `AUDITA_LLM_TIMEOUT_SECONDS`
|
||||
- `AUDITA_MAX_RETRIES`
|
||||
|
||||
CLI:
|
||||
- `--llm-api-key`
|
||||
- `--model`
|
||||
- `--base-url`
|
||||
- `--llm-timeout-seconds`
|
||||
- `--max-retries`
|
||||
|
||||
### Validation LLM
|
||||
|
||||
Environment:
|
||||
- `AUDITA_VALIDATION_LLM_API_KEY`
|
||||
- `AUDITA_VALIDATION_MODEL`
|
||||
- `AUDITA_VALIDATION_BASE_URL`
|
||||
- `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS`
|
||||
- `AUDITA_VALIDATION_MAX_RETRIES`
|
||||
- `AUDITA_VALIDATION_LLM_CONCURRENCY`
|
||||
- `AUDITA_VALIDATION_MAX_PROMPT_TOKENS`
|
||||
|
||||
CLI:
|
||||
- `--validation-llm-api-key`
|
||||
- `--validation-model`
|
||||
- `--validation-base-url`
|
||||
- `--validation-llm-timeout-seconds`
|
||||
- `--validation-max-retries`
|
||||
- `--validation-llm-concurrency`
|
||||
- `--validation-max-prompt-tokens`
|
||||
|
||||
### LLM Concurrency
|
||||
|
||||
Environment:
|
||||
- `AUDITA_TOTAL_LLM_CONCURRENCY`
|
||||
- `AUDITA_PROPOSAL_LLM_CONCURRENCY`
|
||||
- `AUDITA_VALIDATION_LLM_CONCURRENCY`
|
||||
- `AUDITA_LLM_CONCURRENCY` (legacy alias for `AUDITA_TOTAL_LLM_CONCURRENCY`)
|
||||
|
||||
CLI:
|
||||
- `--total-llm-concurrency`
|
||||
- `--proposal-llm-concurrency`
|
||||
- `--validation-llm-concurrency`
|
||||
- `--llm-concurrency` (legacy alias for `--total-llm-concurrency`)
|
||||
|
||||
Behavior:
|
||||
- all proposal and validation LLM calls are bounded by total LLM concurrency
|
||||
- proposal LLM calls are additionally bounded by proposal LLM concurrency
|
||||
- when validation concurrency is unset, it inherits total LLM concurrency
|
||||
- when explicitly set, proposal and validation concurrency must each be `<= total-llm-concurrency`
|
||||
- canonical total settings win when both canonical and legacy alias settings are provided at the same precedence layer
|
||||
|
||||
### Confidence Thresholds
|
||||
|
||||
Environment:
|
||||
- `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD`
|
||||
- `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD`
|
||||
- `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD`
|
||||
- `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD`
|
||||
|
||||
CLI:
|
||||
- `--glossary-confidence-threshold`
|
||||
- `--homophones-confidence-threshold`
|
||||
- `--spoken-word-confidence-threshold`
|
||||
- `--grammar-confidence-threshold`
|
||||
|
||||
### Normalization and Chunking
|
||||
|
||||
Environment:
|
||||
- `AUDITA_NORMALIZE_MAX_SEGMENT_GAP`
|
||||
- `AUDITA_NORMALIZE_ELLIPSIS_GAP`
|
||||
- `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION`
|
||||
- `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS`
|
||||
- `AUDITA_MAX_SECTION_TOKENS`
|
||||
- `AUDITA_MIN_SECTION_TOKENS`
|
||||
- `AUDITA_TARGET_SECTIONS`
|
||||
|
||||
CLI:
|
||||
- `--normalize-max-segment-gap`
|
||||
- `--normalize-ellipsis-gap`
|
||||
- `--normalize-max-segment-duration`
|
||||
- `--normalize-max-segment-tokens`
|
||||
- `--max-section-tokens`
|
||||
- `--min-section-tokens`
|
||||
- `--target-sections`
|
||||
|
||||
### Work Directory
|
||||
|
||||
Environment:
|
||||
- `AUDITA_WORK_DIR`
|
||||
- `AUDITA_WORK_DIR_RETENTION` (`auto`, `always`, `never`)
|
||||
|
||||
CLI:
|
||||
- `--work-dir`
|
||||
- `--work-dir-retention`
|
||||
|
||||
Retention behavior:
|
||||
- `always`: keep all run directories
|
||||
- `never`: keep successful run directories
|
||||
- `auto`: keep failed runs and successful runs with skipped/rejected corrections
|
||||
|
||||
## Reports and Diagnostics
|
||||
|
||||
Per-run diagnostics include:
|
||||
- source transcript artifacts
|
||||
- normalized transcript artifact
|
||||
- normalization summary
|
||||
- chunking summary
|
||||
- utilization diagnostics summary
|
||||
- correction ledger
|
||||
- invocation metadata
|
||||
- redacted effective config
|
||||
- module/validator prompt-response diagnostics
|
||||
- `report.json`
|
||||
- `error.log` on failure
|
||||
|
||||
Optional external report output:
|
||||
- `--report-json <path>`
|
||||
|
||||
## Documentation
|
||||
|
||||
- Architecture: [`docs/architecture.md`](docs/architecture.md)
|
||||
- Diagnostics: [`docs/diagnostics.md`](docs/diagnostics.md)
|
||||
- Structured LLM adapter: [`docs/structured-llm.md`](docs/structured-llm.md)
|
||||
- Subprocess operations: [`docs/subprocess-operations.md`](docs/subprocess-operations.md)
|
||||
- Release checklist: [`docs/release-checklist.md`](docs/release-checklist.md)
|
||||
|
||||
147
audita
147
audita
@@ -1,147 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
DEFAULT_WORK_DIR = "/tmp/audita"
|
||||
_SECRET_FLAGS = {"--llm-api-key", "--validation-llm-api-key"}
|
||||
|
||||
|
||||
def _redact_argv(argv: list[str]) -> list[str]:
|
||||
redacted: list[str] = []
|
||||
index = 0
|
||||
while index < len(argv):
|
||||
arg = argv[index]
|
||||
matched_flag = next((flag for flag in _SECRET_FLAGS if arg == flag or arg.startswith(flag + "=")), None)
|
||||
if matched_flag is None:
|
||||
redacted.append(arg)
|
||||
index += 1
|
||||
continue
|
||||
if arg == matched_flag:
|
||||
redacted.append(arg)
|
||||
if index + 1 < len(argv):
|
||||
redacted.append("[REDACTED]")
|
||||
index += 2
|
||||
else:
|
||||
index += 1
|
||||
continue
|
||||
redacted.append(f"{matched_flag}=[REDACTED]")
|
||||
index += 1
|
||||
return redacted
|
||||
|
||||
|
||||
def _resolve_work_root(argv: list[str]) -> Path:
|
||||
for index, arg in enumerate(argv):
|
||||
if arg == "--work-dir" and index + 1 < len(argv):
|
||||
return Path(argv[index + 1])
|
||||
if arg.startswith("--work-dir="):
|
||||
return Path(arg.split("=", 1)[1])
|
||||
return Path(os.environ.get("AUDITA_WORK_DIR") or DEFAULT_WORK_DIR)
|
||||
|
||||
|
||||
def _create_run_dir(root: Path) -> Path:
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
|
||||
run_dir = root / f"run-{timestamp}-{uuid4().hex[:8]}"
|
||||
run_dir.mkdir(parents=False, exist_ok=False)
|
||||
return run_dir
|
||||
|
||||
|
||||
def _capture_run_dirs(root: Path) -> set[str]:
|
||||
if not root.exists():
|
||||
return set()
|
||||
return {path.name for path in root.iterdir() if path.is_dir() and path.name.startswith("run-")}
|
||||
|
||||
|
||||
def _find_new_run_dir(root: Path, before: set[str]) -> Optional[Path]:
|
||||
if not root.exists():
|
||||
return None
|
||||
candidates = [
|
||||
path for path in root.iterdir() if path.is_dir() and path.name.startswith("run-") and path.name not in before
|
||||
]
|
||||
if not candidates:
|
||||
return None
|
||||
return max(candidates, key=lambda path: path.name)
|
||||
|
||||
|
||||
def _write_launcher_error_log(
|
||||
path: Path,
|
||||
*,
|
||||
message: str,
|
||||
exit_code: int,
|
||||
argv: list[str],
|
||||
command: Optional[list[str]],
|
||||
) -> None:
|
||||
payload = {
|
||||
"timestamp": datetime.utcnow().isoformat() + "Z",
|
||||
"message": message,
|
||||
"exit_code": exit_code,
|
||||
"argv": argv,
|
||||
"cwd": os.getcwd(),
|
||||
"command": command,
|
||||
}
|
||||
path.write_text(
|
||||
"Audita Launcher Diagnostics\n" + json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _emit_console_line(message: str) -> None:
|
||||
for stream in (sys.stderr, sys.stdout):
|
||||
if stream is None:
|
||||
continue
|
||||
try:
|
||||
stream.write(f"{message}\n")
|
||||
stream.flush()
|
||||
return
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
|
||||
|
||||
def main() -> int:
|
||||
argv = list(sys.argv[1:])
|
||||
work_root = _resolve_work_root(argv)
|
||||
redacted_argv = _redact_argv(argv)
|
||||
uv = shutil.which("uv")
|
||||
if uv is None:
|
||||
run_dir = _create_run_dir(work_root)
|
||||
error_log = run_dir / "error.log"
|
||||
message = "uv is required to run this launcher. Install uv and run `uv sync` in the Audita project."
|
||||
_write_launcher_error_log(error_log, message=message, exit_code=1, argv=redacted_argv, command=None)
|
||||
_emit_console_line(f"audita: error: {message}")
|
||||
_emit_console_line("audita: exit code: 1")
|
||||
_emit_console_line(f"audita: run directory: {run_dir}")
|
||||
_emit_console_line(f"audita: error log: {error_log}")
|
||||
return 1
|
||||
|
||||
project_root = Path(__file__).resolve().parent
|
||||
command = [uv, "run", "--project", str(project_root), "python", "-m", "audita", *sys.argv[1:]]
|
||||
before = _capture_run_dirs(work_root)
|
||||
result = subprocess.run(command, cwd=project_root, check=False)
|
||||
if result.returncode == 0:
|
||||
return 0
|
||||
if _find_new_run_dir(work_root, before) is None:
|
||||
run_dir = _create_run_dir(work_root)
|
||||
error_log = run_dir / "error.log"
|
||||
_write_launcher_error_log(
|
||||
error_log,
|
||||
message=f"Audita subprocess exited with status {result.returncode}.",
|
||||
exit_code=result.returncode,
|
||||
argv=redacted_argv,
|
||||
command=_redact_argv(command),
|
||||
)
|
||||
_emit_console_line(f"audita: subprocess exited with status {result.returncode}")
|
||||
_emit_console_line(f"audita: run directory: {run_dir}")
|
||||
_emit_console_line(f"audita: error log: {error_log}")
|
||||
return result.returncode
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
11
cmd/audita/main.go
Normal file
11
cmd/audita/main.go
Normal file
@@ -0,0 +1,11 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/cli"
|
||||
)
|
||||
|
||||
func main() {
|
||||
os.Exit(cli.Run(os.Args[1:], os.Stdout, os.Stderr))
|
||||
}
|
||||
677
cmd/audita/main_integration_test.go
Normal file
677
cmd/audita/main_integration_test.go
Normal file
@@ -0,0 +1,677 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/cli"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/testsupport"
|
||||
)
|
||||
|
||||
func TestHelperProcess(t *testing.T) {
|
||||
if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
|
||||
return
|
||||
}
|
||||
|
||||
sep := -1
|
||||
for i, arg := range os.Args {
|
||||
if arg == "--" {
|
||||
sep = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if sep == -1 {
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
cli.ConfigureSubprocessTestHooksFromEnv()
|
||||
code := cli.Run(os.Args[sep+1:], os.Stdout, os.Stderr)
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func TestProcessHelpSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(t, "process", "--help")
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if !strings.Contains(result.stdout, "Usage:") || !strings.Contains(result.stdout, "--glossary") {
|
||||
t.Fatalf("unexpected help stdout: %q", result.stdout)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSuccessWithOutputSubprocess(t *testing.T) {
|
||||
outputPath := filepath.Join(t.TempDir(), "corrected.json")
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output",
|
||||
outputPath,
|
||||
)
|
||||
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout when --output is set, got %q", result.stdout)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
|
||||
inputBytes := readFile(t, fixturePath("tiny_transcript.json"))
|
||||
outputBytes := readFile(t, outputPath)
|
||||
assertJSONSemanticallyEqual(t, inputBytes, outputBytes)
|
||||
}
|
||||
|
||||
func TestProcessSuccessWithoutOutputSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
|
||||
inputBytes := readFile(t, fixturePath("tiny_transcript.json"))
|
||||
assertJSONSemanticallyEqual(t, inputBytes, []byte(result.stdout))
|
||||
}
|
||||
|
||||
func TestProcessSuccessWithAuditaV1OutputSchemaSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output-schema",
|
||||
"audita-v1",
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
var out struct {
|
||||
Schema string `json:"schema"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(result.stdout), &out); err != nil {
|
||||
t.Fatalf("expected valid audita-v1 JSON output: %v", err)
|
||||
}
|
||||
if out.Schema != "audita-v1" {
|
||||
t.Fatalf("expected audita-v1 schema, got %q", out.Schema)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMissingTranscriptSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(t, "process", "--glossary", fixturePath("tiny_glossary.yaml"))
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "expected exactly 1 transcript JSON path argument") {
|
||||
t.Fatalf("expected actionable missing transcript error, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMalformedJSONSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("malformed_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "is not valid JSON") {
|
||||
t.Fatalf("expected malformed JSON error, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMissingTranscriptFileSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
filepath.Join(t.TempDir(), "missing-transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "transcript_read") {
|
||||
t.Fatalf("expected transcript_read failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMissingGlossaryFileSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
filepath.Join(t.TempDir(), "missing-glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "glossary_read") {
|
||||
t.Fatalf("expected glossary_read failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureTranscriptSchemaSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
schemaFixturePath("transcript_empty_speaker.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "transcript_schema") {
|
||||
t.Fatalf("expected transcript_schema failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMalformedGlossaryYAMLSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
schemaFixturePath("glossary_malformed.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "glossary_schema") {
|
||||
t.Fatalf("expected glossary_schema failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureUnreadableTranscriptSubprocess(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("portable unreadable-file permissions are not reliable on windows")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
transcriptPath := filepath.Join(dir, "transcript.json")
|
||||
if err := os.WriteFile(transcriptPath, []byte(`[]`), 0o000); err != nil {
|
||||
t.Fatalf("write unreadable transcript: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.Chmod(transcriptPath, 0o644) })
|
||||
if _, err := os.ReadFile(transcriptPath); err == nil {
|
||||
t.Skip("unable to make transcript unreadable on this platform/user")
|
||||
}
|
||||
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
transcriptPath,
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "transcript_read") {
|
||||
t.Fatalf("expected transcript_read failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureUnwritableOutputSubprocess(t *testing.T) {
|
||||
outputDir := t.TempDir()
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output",
|
||||
outputDir,
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "failed to write output file") {
|
||||
t.Fatalf("expected write failure message, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureUnwritableReportJSONSubprocess(t *testing.T) {
|
||||
reportDir := t.TempDir()
|
||||
outputPath := filepath.Join(t.TempDir(), "out.json")
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output",
|
||||
outputPath,
|
||||
"--report-json",
|
||||
reportDir,
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "failed to write report JSON file") {
|
||||
t.Fatalf("expected report write failure message, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSuccessReportJSONSubprocess(t *testing.T) {
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--report-json",
|
||||
reportPath,
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
if !json.Valid([]byte(result.stdout)) {
|
||||
t.Fatalf("expected transcript JSON only on stdout, got %q", result.stdout)
|
||||
}
|
||||
report := readFile(t, reportPath)
|
||||
if !json.Valid(report) {
|
||||
t.Fatalf("expected valid report JSON, got %q", string(report))
|
||||
}
|
||||
// Ensure report JSON is not printed to stdout.
|
||||
if strings.Contains(result.stdout, `"default_pipeline"`) {
|
||||
t.Fatalf("report JSON leaked to stdout: %q", result.stdout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSuccessLargeTranscriptSubprocess(t *testing.T) {
|
||||
transcriptPath := writeLargeTranscriptFixture(t, 320)
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
transcriptPath,
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
if !json.Valid([]byte(result.stdout)) {
|
||||
t.Fatalf("expected valid transcript JSON on stdout")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMalformedStructuredLLMResponseViaSubprocessHook(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "malformed_structured"},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"grammar",
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected zero exit code, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if !json.Valid([]byte(result.stdout)) {
|
||||
t.Fatalf("expected transcript JSON on stdout, got %q", result.stdout)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
report := readFile(t, reportPath)
|
||||
if !json.Valid(report) {
|
||||
t.Fatalf("expected valid success report JSON")
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
|
||||
t.Fatalf("did not expect error.log, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureBackendLLMViaSubprocessHook(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "backend_error"},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"grammar",
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "synthetic backend failure") {
|
||||
t.Fatalf("expected backend failure details, got %q", result.stderr)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "diagnostics:") {
|
||||
t.Fatalf("expected diagnostics path in stderr, got %q", result.stderr)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(onlyRunDir(t, workDir), "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log in retained failed run: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMidPipelinePreservesPartialReportsSubprocess(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "mid_pipeline_fail"},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"glossary,homophones,glossary,spoken_word,grammar",
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
reportRaw := readFile(t, reportPath)
|
||||
var report struct {
|
||||
Status string `json:"status"`
|
||||
ErrorPhase string `json:"error_phase"`
|
||||
ModuleResults []struct {
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
Status string `json:"status"`
|
||||
} `json:"module_results"`
|
||||
}
|
||||
if err := json.Unmarshal(reportRaw, &report); err != nil {
|
||||
t.Fatalf("unmarshal report: %v", err)
|
||||
}
|
||||
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
|
||||
t.Fatalf("expected failed runner_execution report, got %+v", report)
|
||||
}
|
||||
if len(report.ModuleResults) == 0 {
|
||||
t.Fatalf("expected partial module results in failure report")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessCancellationViaSubprocessTimeoutHook(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
|
||||
result := runCLISubprocessContext(t, ctx,
|
||||
map[string]string{
|
||||
"AUDITA_SUBPROCESS_TEST_LLM_MODE": "block_until_cancel",
|
||||
"AUDITA_SUBPROCESS_TEST_RUN_TIMEOUT_MS": "120",
|
||||
},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"grammar",
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.stdout != "" {
|
||||
if result.stderr == "" {
|
||||
t.Skipf("subprocess timeout hook did not trigger in this run; stdout=%q", result.stdout)
|
||||
}
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "context deadline exceeded") {
|
||||
t.Fatalf("expected context deadline error, got %q", result.stderr)
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log for canceled run: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil {
|
||||
t.Fatalf("expected report.json for canceled run: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSubprocessNoSecretLeakInOutputsAndDiagnostics(t *testing.T) {
|
||||
secret := "subprocess-secret"
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
outputPath := filepath.Join(t.TempDir(), "out.json")
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{
|
||||
"AUDITA_LLM_API_KEY": secret,
|
||||
"AUDITA_VALIDATION_LLM_API_KEY": secret,
|
||||
},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output",
|
||||
outputPath,
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if strings.Contains(result.stdout, secret) || strings.Contains(result.stderr, secret) {
|
||||
t.Fatalf("secret leaked in subprocess stdio")
|
||||
}
|
||||
assertNoSecretInFile(t, reportPath, secret)
|
||||
assertNoSecretInTree(t, onlyRunDir(t, workDir), secret)
|
||||
}
|
||||
|
||||
type subprocessResult struct {
|
||||
stdout string
|
||||
stderr string
|
||||
exitCode int
|
||||
}
|
||||
|
||||
func runCLISubprocess(t *testing.T, args ...string) subprocessResult {
|
||||
t.Helper()
|
||||
return runCLISubprocessWithEnv(t, nil, args...)
|
||||
}
|
||||
|
||||
func runCLISubprocessWithEnv(t *testing.T, extraEnv map[string]string, args ...string) subprocessResult {
|
||||
t.Helper()
|
||||
return runCLISubprocessContext(t, context.Background(), extraEnv, args...)
|
||||
}
|
||||
|
||||
func runCLISubprocessContext(t *testing.T, ctx context.Context, extraEnv map[string]string, args ...string) subprocessResult {
|
||||
t.Helper()
|
||||
cmdArgs := append([]string{"-test.run=TestHelperProcess", "--"}, args...)
|
||||
cmd := exec.CommandContext(ctx, os.Args[0], cmdArgs...)
|
||||
env := append(filterAuditaEnv(os.Environ()), "GO_WANT_HELPER_PROCESS=1")
|
||||
for k, v := range extraEnv {
|
||||
env = append(env, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
cmd.Env = env
|
||||
var stdoutBuf bytes.Buffer
|
||||
var stderrBuf bytes.Buffer
|
||||
cmd.Stdout = &stdoutBuf
|
||||
cmd.Stderr = &stderrBuf
|
||||
|
||||
err := cmd.Run()
|
||||
result := subprocessResult{
|
||||
stdout: stdoutBuf.String(),
|
||||
stderr: stderrBuf.String(),
|
||||
}
|
||||
if err == nil {
|
||||
return result
|
||||
}
|
||||
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
result.exitCode = exitErr.ExitCode()
|
||||
return result
|
||||
}
|
||||
|
||||
t.Fatalf("subprocess execution failed: %v", err)
|
||||
return subprocessResult{}
|
||||
}
|
||||
|
||||
func filterAuditaEnv(env []string) []string {
|
||||
filtered := make([]string, 0, len(env))
|
||||
for _, entry := range env {
|
||||
key := entry
|
||||
if idx := strings.IndexByte(entry, '='); idx >= 0 {
|
||||
key = entry[:idx]
|
||||
}
|
||||
if strings.HasPrefix(key, "AUDITA_") || key == "OPENROUTER_API_KEY" {
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, entry)
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
func fixturePath(name string) string {
|
||||
return filepath.Join("..", "..", "internal", "cli", "testdata", name)
|
||||
}
|
||||
|
||||
func schemaFixturePath(name string) string {
|
||||
return filepath.Join("..", "..", "internal", "core", "schema", "testdata", name)
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) []byte {
|
||||
return testsupport.ReadFile(t, path)
|
||||
}
|
||||
|
||||
func assertJSONSemanticallyEqual(t *testing.T, expected []byte, actual []byte) {
|
||||
t.Helper()
|
||||
if !json.Valid(actual) {
|
||||
t.Fatalf("actual output is not valid JSON: %q", string(actual))
|
||||
}
|
||||
|
||||
var expectedValue any
|
||||
var actualValue any
|
||||
if err := json.Unmarshal(expected, &expectedValue); err != nil {
|
||||
t.Fatalf("failed to unmarshal expected JSON: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(actual, &actualValue); err != nil {
|
||||
t.Fatalf("failed to unmarshal actual JSON: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(expectedValue, actualValue) {
|
||||
t.Fatalf("JSON content mismatch: expected %q got %q", string(expected), string(actual))
|
||||
}
|
||||
}
|
||||
|
||||
func writeLargeTranscriptFixture(t *testing.T, segments int) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "large-transcript.json")
|
||||
rows := make([]string, 0, segments)
|
||||
for i := 0; i < segments; i++ {
|
||||
rows = append(rows, fmt.Sprintf(`{"id":%d,"speaker":"Speaker%d","start":%s,"end":%s,"text":"Segment %d has enough words to exercise stdout and pipe buffering safely."}`,
|
||||
i+1,
|
||||
(i%4)+1,
|
||||
strconv.FormatFloat(float64(i)*1.1, 'f', 1, 64),
|
||||
strconv.FormatFloat(float64(i)*1.1+1.0, 'f', 1, 64),
|
||||
i+1,
|
||||
))
|
||||
}
|
||||
payload := "[\n " + strings.Join(rows, ",\n ") + "\n]\n"
|
||||
if err := os.WriteFile(path, []byte(payload), 0o644); err != nil {
|
||||
t.Fatalf("write large transcript fixture: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func onlyRunDir(t *testing.T, workDir string) string {
|
||||
return testsupport.OnlyRunDir(t, workDir)
|
||||
}
|
||||
|
||||
func assertNoSecretInFile(t *testing.T, path, secret string) {
|
||||
testsupport.AssertNoSecretInFile(t, path, secret)
|
||||
}
|
||||
|
||||
func assertNoSecretInTree(t *testing.T, root, secret string) {
|
||||
testsupport.AssertNoSecretInTree(t, root, secret)
|
||||
}
|
||||
14
docs/architecture.md
Normal file
14
docs/architecture.md
Normal file
@@ -0,0 +1,14 @@
|
||||
# Audita Architecture Index
|
||||
|
||||
This file is the entrypoint for architecture documentation.
|
||||
|
||||
Core architecture overview:
|
||||
- [Architecture Overview](./architecture/architecture.md)
|
||||
|
||||
Focused architecture contracts:
|
||||
- [Public Contract](./architecture/public-contract.md)
|
||||
- [Diagnostics](./architecture/diagnostics.md)
|
||||
- [Structured LLM](./architecture/structured-llm.md)
|
||||
- [Validators](./architecture/validators.md)
|
||||
- [Prompts](./architecture/prompts.md)
|
||||
- [Output Schemas](./architecture/output-schemas.md)
|
||||
153
docs/architecture/architecture.md
Normal file
153
docs/architecture/architecture.md
Normal file
@@ -0,0 +1,153 @@
|
||||
# Audita Architecture
|
||||
|
||||
## Scope
|
||||
This document describes the production architecture implemented in this repository today.
|
||||
|
||||
Audita is a single-process Go CLI that:
|
||||
- loads effective runtime configuration;
|
||||
- reads transcript and glossary inputs;
|
||||
- normalizes and sections transcripts;
|
||||
- runs a built-in module pipeline with validator chains;
|
||||
- writes transcript output and run diagnostics.
|
||||
|
||||
## Runtime entrypoints
|
||||
Primary CLI commands:
|
||||
- `audita process <transcript.json> --glossary <glossary.yaml> [flags]`
|
||||
- `audita config validate --config <config.yml>`
|
||||
- `audita config print-effective [--config <config.yml>]`
|
||||
|
||||
Command ownership lives in `internal/cli/run.go`.
|
||||
|
||||
## Configuration model
|
||||
`internal/core/config` owns defaults, file parsing, environment overrides, CLI overrides, and validation.
|
||||
|
||||
Effective-config loading for `process` and `config print-effective` is centralized in:
|
||||
- `ResolveConfigPath`
|
||||
- `LoadEffectiveConfig`
|
||||
|
||||
Effective precedence for `audita process`:
|
||||
1. defaults
|
||||
2. config file
|
||||
3. environment overrides
|
||||
4. CLI overrides
|
||||
|
||||
`audita config validate` is intentionally file-only validation:
|
||||
- load versioned file;
|
||||
- apply onto defaults;
|
||||
- validate;
|
||||
- do not apply environment overrides.
|
||||
|
||||
Supported module and output-schema keys are validated through shared catalogs:
|
||||
- module keys: `internal/core/modulecatalog`
|
||||
- output schemas: `internal/core/outputschema`
|
||||
|
||||
## Pipeline and module orchestration
|
||||
The built-in module sequence is configured in runtime config and executed by `internal/framework/runner` through resolved module specs.
|
||||
|
||||
Current default sequence:
|
||||
- `glossary`
|
||||
- `homophones`
|
||||
- `glossary`
|
||||
- `spoken_word`
|
||||
- `grammar`
|
||||
|
||||
Execution behavior:
|
||||
- modules execute serially over the working transcript;
|
||||
- section proposal work can run concurrently within a module;
|
||||
- validator execution happens on generated proposals before application;
|
||||
- approved proposals are applied once per module in deterministic proposal-index order.
|
||||
|
||||
Production modules remain separate packages:
|
||||
- `internal/modules/glossary`
|
||||
- `internal/modules/homophones`
|
||||
- `internal/modules/spoken_word`
|
||||
- `internal/modules/grammar`
|
||||
|
||||
## Proposal generation and prompt context
|
||||
Shared proposal plumbing is centralized in `internal/framework/proposal_generation`.
|
||||
|
||||
Module packages provide:
|
||||
- module identity and replacement policy;
|
||||
- module-specific prompt message building;
|
||||
- built-in validator chain selection.
|
||||
|
||||
Shared prompt payload helpers are in `internal/framework/promptcontext`.
|
||||
|
||||
## Validator architecture
|
||||
Built-in validator construction and chain composition live in `internal/validators`.
|
||||
|
||||
Shared validator runtime mechanics live in `internal/framework/validators`.
|
||||
|
||||
Execution class metadata (deterministic vs LLM-backed) is centralized in `internal/validators/metadata` and used for ordering and reporting classification.
|
||||
|
||||
## Structured LLM boundary
|
||||
All production LLM calls go through the internal contract:
|
||||
- `contracts.StructuredLLMClient`
|
||||
- `CompleteStructured(ctx, req, out)`
|
||||
|
||||
The OpenAI-compatible HTTP adapter is implemented in `internal/framework/llm`.
|
||||
|
||||
Structured response schemas are registered in `internal/framework/responseschema` and attached to requests via `response_format` metadata.
|
||||
|
||||
Malformed structured-output detection is centralized in `internal/framework/structuredoutput` and reused by proposal generation and validator execution so downgrade behavior stays consistent.
|
||||
|
||||
## Stage naming and diagnostics metadata
|
||||
Diagnostics stage naming is centralized in `internal/framework/stagename`:
|
||||
- module proposal stage names;
|
||||
- proposal-generation stage names;
|
||||
- validator batch stage names.
|
||||
|
||||
Prompt metadata and response-schema metadata each expose canonical diagnostics maps via:
|
||||
- `prompts.Metadata.DiagnosticsMap()`
|
||||
- `responseschema.Schema.DiagnosticsMap()`
|
||||
|
||||
## Diagnostics and reporting
|
||||
Run-directory artifacts are owned by `internal/core/diagnostics`.
|
||||
|
||||
Stable artifact names are centralized constants (for example transcript artifacts, `invocation.json`, `effective-config.json`, `utilization-diagnostics.json`, `correction-ledger.json`, `report.json`, `error.log`).
|
||||
|
||||
Report diagnostics path metadata is constructed through `BuildDiagnosticsMetadata`, which keeps run-directory artifact references consistent between success and failure reports.
|
||||
|
||||
## Secret redaction
|
||||
Redaction responsibilities are split by concern:
|
||||
- structural config redaction: `config.Config.Redacted()`
|
||||
- byte/string payload redaction for diagnostics and surfaced errors: framework redaction utilities.
|
||||
|
||||
Configured LLM secret extraction is centralized in `llm.ConfiguredSecrets(cfg)` and reused across proposal and validator diagnostics paths.
|
||||
|
||||
## Output contracts
|
||||
Transcript output schema selection is owned by `internal/core/outputschema`.
|
||||
|
||||
Supported schemas:
|
||||
- `bare-segments`
|
||||
- `audita-v1`
|
||||
|
||||
Unknown schema keys fail validation and runtime resolution.
|
||||
|
||||
## Key package map
|
||||
Core packages:
|
||||
- `internal/core/config`
|
||||
- `internal/core/schema`
|
||||
- `internal/core/normalization`
|
||||
- `internal/core/chunking`
|
||||
- `internal/core/diagnostics`
|
||||
- `internal/core/reporting`
|
||||
- `internal/core/modulecatalog`
|
||||
- `internal/core/outputschema`
|
||||
|
||||
Framework packages:
|
||||
- `internal/framework/contracts`
|
||||
- `internal/framework/proposals`
|
||||
- `internal/framework/proposal_generation`
|
||||
- `internal/framework/promptcontext`
|
||||
- `internal/framework/runner`
|
||||
- `internal/framework/validators`
|
||||
- `internal/framework/llm`
|
||||
- `internal/framework/responseschema`
|
||||
- `internal/framework/stagename`
|
||||
- `internal/framework/structuredoutput`
|
||||
|
||||
Domain packages:
|
||||
- `internal/modules/*`
|
||||
- `internal/validators/*`
|
||||
- `internal/prompts`
|
||||
104
docs/architecture/diagnostics.md
Normal file
104
docs/architecture/diagnostics.md
Normal file
@@ -0,0 +1,104 @@
|
||||
# Audita Diagnostics
|
||||
|
||||
This document describes the run-directory diagnostics artifacts produced by `audita process`.
|
||||
|
||||
## Purpose
|
||||
|
||||
Diagnostics provide machine-readable run context and execution artifacts for:
|
||||
- failure debugging;
|
||||
- validator/correction review;
|
||||
- post-run performance analysis.
|
||||
|
||||
Diagnostics are written under the configured work directory (`--work-dir`) when run-directory initialization succeeds.
|
||||
|
||||
## Core artifacts
|
||||
|
||||
Typical artifacts in each run directory:
|
||||
- `source-transcript.json`
|
||||
- `source-transcript-parsed.json`
|
||||
- `normalized-transcript.json`
|
||||
- `normalization-summary.json`
|
||||
- `chunking-summary.json`
|
||||
- `invocation.json`
|
||||
- `effective-config.json` (redacted)
|
||||
- module/validator LLM interaction artifacts
|
||||
- `report.json`
|
||||
- `error.log` on failure
|
||||
|
||||
## Utilization diagnostics artifact
|
||||
|
||||
Artifact:
|
||||
- `utilization-diagnostics.json`
|
||||
|
||||
High-level fields:
|
||||
- `effective_concurrency`:
|
||||
- total/proposal/validation LLM concurrency limits in effect.
|
||||
- `run_timing`:
|
||||
- run wall time;
|
||||
- scheduler queue wait time;
|
||||
- LLM execution time;
|
||||
- deterministic validator time;
|
||||
- max/average in-flight LLM calls.
|
||||
- `llm_calls`:
|
||||
- total proposal and validation LLM call counts.
|
||||
- `modules`:
|
||||
- module-level timing summaries.
|
||||
- `validators`:
|
||||
- per-validator timing summaries keyed by stable validator key.
|
||||
|
||||
## Correction ledger artifact
|
||||
|
||||
Artifact:
|
||||
- `correction-ledger.json`
|
||||
|
||||
Ledger records are flattened review entries derived from module results and include:
|
||||
- module/proposal identity (`module_key`, `module_instance`, `proposal_index`, `segment_id`);
|
||||
- correction text fields and replacement policy when available;
|
||||
- disposition:
|
||||
- `applied`
|
||||
- `rejected`
|
||||
- `skipped`
|
||||
- `failed`
|
||||
- stable reason codes/messages;
|
||||
- deterministic and LLM validator decision snapshots using stable validator keys.
|
||||
|
||||
Validator rejection and proposal-application skip are distinct dispositions.
|
||||
Module warnings are reported in module results and diagnostics metadata, but do not create standalone correction-ledger rows.
|
||||
|
||||
## Report references
|
||||
|
||||
`report.json` and optional `--report-json` output include diagnostics metadata paths for:
|
||||
- utilization diagnostics artifact;
|
||||
- correction ledger artifact;
|
||||
- existing transcript/normalization/chunking/invocation/effective-config artifacts.
|
||||
|
||||
Module report entries also include warning records for malformed proposal-generation payloads and malformed validator batches.
|
||||
|
||||
## Retention behavior
|
||||
|
||||
Run-directory retention follows configured policy:
|
||||
- `always`: keep all run directories;
|
||||
- `never`: keep successful run directories;
|
||||
- `auto`: keep failed runs and successful runs with skipped/rejected corrections.
|
||||
|
||||
## Redaction guarantees
|
||||
|
||||
API keys and other configured secrets are redacted from:
|
||||
- `effective-config.json`;
|
||||
- LLM interaction diagnostics artifacts;
|
||||
- reports and surfaced errors.
|
||||
|
||||
## Debugging guide
|
||||
|
||||
When debugging:
|
||||
- slow runs:
|
||||
- inspect `utilization-diagnostics.json` (`run_timing`, `modules`, `validators`, in-flight metrics).
|
||||
- validator rejections:
|
||||
- inspect `correction-ledger.json` rejected entries and matching validator decisions;
|
||||
- inspect validator response diagnostics payloads.
|
||||
- module warnings:
|
||||
- inspect module `warnings` entries in `report.json` or `--report-json`;
|
||||
- follow any diagnostic artifact path on the warning to the recorded error/response payload.
|
||||
- application skips:
|
||||
- inspect `correction-ledger.json` skipped entries and skip reason codes;
|
||||
- compare with validator decisions to distinguish validation rejection vs apply-time skip.
|
||||
88
docs/architecture/output-schemas.md
Normal file
88
docs/architecture/output-schemas.md
Normal file
@@ -0,0 +1,88 @@
|
||||
# Audita Output Schemas
|
||||
|
||||
This document describes the built-in transcript output schema registry used by `audita process`.
|
||||
|
||||
## Supported schema names
|
||||
|
||||
### `bare-segments`
|
||||
|
||||
Status:
|
||||
- implemented
|
||||
- default output schema
|
||||
|
||||
Shape:
|
||||
- top-level JSON array of transcript segments
|
||||
|
||||
Segment fields:
|
||||
- `id`
|
||||
- `speaker`
|
||||
- `start`
|
||||
- `end`
|
||||
- `text`
|
||||
- optional `categories`
|
||||
|
||||
Compatibility:
|
||||
- this preserves the long-standing output shape used by existing consumers.
|
||||
|
||||
### `audita-v1`
|
||||
|
||||
Status:
|
||||
- implemented
|
||||
|
||||
Shape:
|
||||
- top-level JSON object:
|
||||
- `schema`: `"audita-v1"`
|
||||
- `version`: `"v1"`
|
||||
- `segments`: transcript segment array
|
||||
|
||||
Segment fields inside `segments` match `bare-segments` segment fields.
|
||||
|
||||
Compatibility:
|
||||
- this is the Audita-native object format with explicit schema/version metadata.
|
||||
|
||||
### `seriatim-intermediate`
|
||||
|
||||
Status:
|
||||
- deferred / not implemented
|
||||
|
||||
Current behavior:
|
||||
- selecting `seriatim-intermediate` fails clearly as an unsupported output schema.
|
||||
|
||||
Reason:
|
||||
- a concrete, repository-backed contract for this schema has not been finalized yet.
|
||||
|
||||
## Selection
|
||||
|
||||
Choose output schema with CLI:
|
||||
|
||||
```sh
|
||||
audita process <transcript.json> --glossary <glossary.yaml> --output-schema audita-v1
|
||||
```
|
||||
|
||||
Or in file config:
|
||||
|
||||
```yaml
|
||||
version: 1
|
||||
output:
|
||||
schema: audita-v1
|
||||
```
|
||||
|
||||
Precedence remains:
|
||||
1. defaults
|
||||
2. file config
|
||||
3. environment overrides
|
||||
4. CLI overrides
|
||||
|
||||
`--output-schema` overrides `output.schema` when both are supplied.
|
||||
|
||||
## Output routing behavior
|
||||
|
||||
- With `--output`, transcript JSON is written to file using the selected schema and stdout stays empty on success.
|
||||
- Without `--output`, stdout contains transcript JSON only, using the selected schema.
|
||||
- `--report-json` writes report JSON to file and does not write report payloads to stdout.
|
||||
|
||||
## Backward-compatibility expectations
|
||||
|
||||
- default schema stays `bare-segments` for compatibility unless explicitly changed in a future breaking release;
|
||||
- supported schema names are treated as stable public contract values;
|
||||
- unsupported schema names fail before output write.
|
||||
118
docs/architecture/prompts.md
Normal file
118
docs/architecture/prompts.md
Normal file
@@ -0,0 +1,118 @@
|
||||
# Audita Prompts
|
||||
|
||||
This document describes Audita's built-in embedded prompt assets and prompt registry behavior.
|
||||
|
||||
## Why embedded prompt assets
|
||||
|
||||
Audita embeds production prompt text into the binary so runtime behavior is:
|
||||
- deterministic;
|
||||
- auditable;
|
||||
- dependency-light;
|
||||
- not dependent on external prompt files at execution time.
|
||||
|
||||
Prompt text is authored as Markdown assets and rendered by Go code using typed template data.
|
||||
|
||||
## Built-in prompt registry
|
||||
|
||||
The prompt registry lives in `internal/prompts` and is responsible for:
|
||||
- loading embedded prompt assets;
|
||||
- registering stable prompt IDs and versions;
|
||||
- recording prompt source metadata;
|
||||
- computing deterministic SHA-256 source hashes;
|
||||
- rendering system/user prompts with strict missing-key failures.
|
||||
|
||||
Current prompt source behavior:
|
||||
- built-in embedded prompts only (`prompt_source = builtin`).
|
||||
- filesystem prompt overrides are not supported.
|
||||
|
||||
## Built-in prompt IDs
|
||||
|
||||
Module proposal prompts:
|
||||
- `modules.glossary.proposal`
|
||||
- `modules.homophones.proposal`
|
||||
- `modules.spoken_word.proposal`
|
||||
- `modules.grammar.proposal`
|
||||
|
||||
LLM-backed validator prompts:
|
||||
- `validators.spoken_form_plausibility`
|
||||
- `validators.meaning_reversal_review`
|
||||
- `validators.editorial_review`
|
||||
- `validators.grammar_review`
|
||||
- `validators.spoken_word_review`
|
||||
|
||||
## Prompt version semantics
|
||||
|
||||
Current built-in prompt version value is `v1`.
|
||||
|
||||
Version is a stable metadata identifier for diagnostics and debugging. It is not a dynamic prompt-selection mechanism.
|
||||
|
||||
## Prompt hash semantics
|
||||
|
||||
Each registered prompt includes a deterministic SHA-256 hash of embedded source text.
|
||||
|
||||
Hash purpose:
|
||||
- identify exact prompt source used in a run;
|
||||
- support diagnostics reproducibility and change auditing.
|
||||
|
||||
Current hash scope:
|
||||
- source prompt text (system + user assets for a registered prompt), not a runtime secret-bearing payload.
|
||||
|
||||
## Template rendering behavior
|
||||
|
||||
Prompt rendering uses Go `text/template` with typed template data from module/validator builders.
|
||||
|
||||
Missing-key behavior:
|
||||
- rendering uses missing-key errors;
|
||||
- missing/renamed template fields fail quickly instead of silently producing incomplete prompts.
|
||||
|
||||
Go code still owns:
|
||||
- structured request/response models;
|
||||
- response schema selection;
|
||||
- transcript/glossary/payload formatting;
|
||||
- module and validator selection;
|
||||
- diagnostics wiring.
|
||||
|
||||
## Shared prompt hardening policy
|
||||
|
||||
A shared hardening fragment is embedded once and included in every module proposal prompt and every LLM-validator prompt.
|
||||
|
||||
Hardening policy includes:
|
||||
- transcript text is untrusted data;
|
||||
- glossary entries and transcript descriptions are reference data, not instructions;
|
||||
- instructions found inside transcript text must not be obeyed;
|
||||
- model must perform only the requested correction/validation task;
|
||||
- no invention of facts, names, events, motivations, speaker intent, or corrections;
|
||||
- transcript remains the source of truth.
|
||||
|
||||
## Transcript description behavior
|
||||
|
||||
Transcript description remains background-only prompt context:
|
||||
- it may help interpret ambiguous terms;
|
||||
- it is explicitly non-authoritative and must not override transcript content;
|
||||
- empty descriptions do not render awkward blank context sections.
|
||||
|
||||
Generated transcript descriptions are not implemented in this workstream.
|
||||
|
||||
## Diagnostics and report metadata boundaries
|
||||
|
||||
Current metadata flow:
|
||||
- proposal-generation diagnostics request metadata includes prompt metadata;
|
||||
- LLM-validator diagnostics request metadata includes prompt metadata.
|
||||
|
||||
Prompt metadata fields used in diagnostics:
|
||||
- `prompt_id`
|
||||
- `prompt_version`
|
||||
- `prompt_source`
|
||||
- `embedded_path`
|
||||
- `sha256`
|
||||
|
||||
Current boundary:
|
||||
- detailed prompt metadata is diagnostics-first;
|
||||
- broad report-level prompt registries/ledgers are deferred.
|
||||
|
||||
## 1.0 boundary
|
||||
|
||||
Not implemented for 1.0 in this workstream:
|
||||
- filesystem prompt overrides;
|
||||
- user-configurable prompt selection;
|
||||
- external prompt directories.
|
||||
121
docs/architecture/public-contract.md
Normal file
121
docs/architecture/public-contract.md
Normal file
@@ -0,0 +1,121 @@
|
||||
# Audita Public Contract
|
||||
|
||||
## Scope
|
||||
This document defines stability expectations for Audita's external runtime interfaces.
|
||||
|
||||
Covered interfaces:
|
||||
- CLI commands and major flags;
|
||||
- versioned config behavior and precedence;
|
||||
- transcript/glossary input forms;
|
||||
- output schema selection;
|
||||
- report schema metadata;
|
||||
- diagnostics artifact path metadata;
|
||||
- stdout/stderr and exit-code behavior;
|
||||
- redaction guarantees.
|
||||
|
||||
## CLI contract
|
||||
Stable commands:
|
||||
- `audita process`
|
||||
- `audita config validate`
|
||||
- `audita config print-effective`
|
||||
|
||||
Stable high-value `process` flags:
|
||||
- `--config`
|
||||
- `--glossary`
|
||||
- `--output`
|
||||
- `--report-json`
|
||||
- `--modules`
|
||||
- `--output-schema`
|
||||
|
||||
## Config contract
|
||||
Supported config format:
|
||||
- YAML;
|
||||
- `version: 1`;
|
||||
- strict unknown-field rejection.
|
||||
|
||||
Path resolution for `process` and `config print-effective`:
|
||||
1. `--config`
|
||||
2. `AUDITA_CONFIG`
|
||||
3. `/usr/local/etc/audita/config.yml`
|
||||
4. `/etc/audita/config.yml`
|
||||
|
||||
Missing explicit path is an error. Missing default paths is non-fatal.
|
||||
|
||||
Precedence for `process`:
|
||||
1. defaults
|
||||
2. file config
|
||||
3. environment overrides
|
||||
4. CLI overrides
|
||||
|
||||
`config validate` remains file-only validation (defaults + file config; no env overrides).
|
||||
|
||||
Module and output-schema keys are validated against built-in catalogs. Unknown keys fail validation.
|
||||
|
||||
## Input contract
|
||||
Supported transcript JSON top-level forms:
|
||||
- array of segments
|
||||
- object with `segments` array
|
||||
|
||||
Supported glossary YAML form:
|
||||
- top-level `glossary` list with required entry fields validated by schema parsing.
|
||||
|
||||
## Output schema contract
|
||||
Supported transcript output schemas:
|
||||
- `bare-segments` (default)
|
||||
- `audita-v1`
|
||||
|
||||
Unknown schema keys fail before output write.
|
||||
|
||||
## Report metadata contract
|
||||
Process reports include stable report metadata fields:
|
||||
- `report_schema_name`
|
||||
- `report_schema_version`
|
||||
- `output_schema`
|
||||
- `config_version` (when file config is loaded)
|
||||
|
||||
Current values:
|
||||
- `report_schema_name = audita-process-report`
|
||||
- `report_schema_version = v1`
|
||||
|
||||
`--report-json` output and run-directory `report.json` use the same report schema metadata.
|
||||
|
||||
Validator decision/rejection records use stable validator keys via `validator_name`.
|
||||
|
||||
## Diagnostics metadata contract
|
||||
When run-directory initialization succeeds, diagnostics metadata paths reference stable artifacts, including:
|
||||
- transcript and normalization artifacts;
|
||||
- chunking summary;
|
||||
- invocation metadata;
|
||||
- redacted effective config;
|
||||
- utilization diagnostics;
|
||||
- correction ledger;
|
||||
- `error.log` on failures.
|
||||
|
||||
LLM interaction diagnostics include stable prompt and structured-schema identifiers where applicable.
|
||||
|
||||
## Stdout/stderr and exit codes
|
||||
Success:
|
||||
- with `--output`, stdout is empty;
|
||||
- without `--output`, stdout contains transcript JSON only;
|
||||
- report JSON is not written to stdout.
|
||||
|
||||
Failures:
|
||||
- nonzero exit;
|
||||
- human-readable stderr summary;
|
||||
- diagnostics directory path on stderr when available.
|
||||
|
||||
Exit codes:
|
||||
- `0` success
|
||||
- nonzero failure
|
||||
|
||||
## Redaction contract
|
||||
Configured secrets are redacted from:
|
||||
- effective config outputs;
|
||||
- diagnostics artifacts;
|
||||
- report artifacts;
|
||||
- surfaced adapter/runtime errors.
|
||||
|
||||
## Compatibility policy
|
||||
Stable command behavior, schema names, report metadata keys, diagnostics-path field semantics, and validator key identities are treated as public contract.
|
||||
|
||||
Additive fields are acceptable when existing fields and behavior remain compatible.
|
||||
72
docs/architecture/structured-llm.md
Normal file
72
docs/architecture/structured-llm.md
Normal file
@@ -0,0 +1,72 @@
|
||||
# Structured LLM Architecture
|
||||
|
||||
## Scope
|
||||
This document describes Audita's structured LLM runtime boundary and adapter behavior.
|
||||
|
||||
## Runtime boundary
|
||||
Production LLM integration depends on the internal contract only:
|
||||
- `contracts.StructuredLLMClient`
|
||||
- `CompleteStructured(ctx, req, out)`
|
||||
|
||||
Provider SDK types do not leak past this boundary.
|
||||
|
||||
## Adapter ownership
|
||||
`internal/framework/llm` owns the OpenAI-compatible HTTP adapter and shared LLM runtime utilities.
|
||||
|
||||
Key responsibilities:
|
||||
- request assembly;
|
||||
- timeout/cancellation propagation;
|
||||
- bounded retry behavior;
|
||||
- scheduler integration;
|
||||
- provider response decoding;
|
||||
- error redaction.
|
||||
|
||||
## Structured schema registry
|
||||
Structured response schemas are registered in `internal/framework/responseschema` and include stable metadata:
|
||||
- `id`
|
||||
- `version`
|
||||
- `name`
|
||||
- `json_schema`
|
||||
- `sha256`
|
||||
|
||||
Current schema keys:
|
||||
- `correction_set`
|
||||
- `validator_decision_set`
|
||||
|
||||
Schema metadata is attached to diagnostics through `Schema.DiagnosticsMap()`.
|
||||
|
||||
## Request shape assumptions
|
||||
Audita targets OpenAI-compatible chat-completions endpoints and sends structured requests with:
|
||||
- model;
|
||||
- chat messages;
|
||||
- `response_format.type = json_schema`;
|
||||
- schema name and JSON schema payload.
|
||||
|
||||
## Local validation remains mandatory
|
||||
Provider schema enforcement is treated as transport-level guardrails.
|
||||
|
||||
Audita still validates output locally before applying behavior changes:
|
||||
- proposal decoding and proposal invariants;
|
||||
- validator decision decoding and cardinality checks;
|
||||
- deterministic validation and apply-time rules.
|
||||
|
||||
## Shared malformed-output policy
|
||||
Malformed structured-output classification is centralized in `internal/framework/structuredoutput`.
|
||||
|
||||
Proposal generation and validator execution both use this shared classifier so downgrade behavior cannot drift between the two paths.
|
||||
|
||||
## Secrets and redaction
|
||||
Secret extraction for LLM redaction is centralized in `llm.ConfiguredSecrets(cfg)` and reused by proposal and validator diagnostics writers.
|
||||
|
||||
Secrets are redacted from:
|
||||
- diagnostics artifacts;
|
||||
- report artifacts;
|
||||
- surfaced adapter/runtime errors.
|
||||
|
||||
## Concurrency and scheduling
|
||||
LLM execution is constrained by composed scheduler limits:
|
||||
- total LLM concurrency;
|
||||
- proposal LLM concurrency;
|
||||
- validation LLM concurrency.
|
||||
|
||||
The scheduler is FIFO and context-aware so permits are released on success, failure, and cancellation.
|
||||
96
docs/architecture/validators.md
Normal file
96
docs/architecture/validators.md
Normal file
@@ -0,0 +1,96 @@
|
||||
# Audita Validators
|
||||
|
||||
## Scope
|
||||
This document defines the built-in validator system used by production module runs.
|
||||
|
||||
## Ownership boundaries
|
||||
Built-in validator keys, constructors, and module chains are owned by `internal/validators`.
|
||||
|
||||
Shared runtime execution mechanics are owned by `internal/framework/validators`, including:
|
||||
- validator request/result models;
|
||||
- deterministic proposal checks;
|
||||
- LLM validator batching and execution;
|
||||
- decision-cardinality enforcement;
|
||||
- diagnostics integration.
|
||||
|
||||
Execution class metadata is owned by `internal/validators/metadata`.
|
||||
|
||||
## Stable validator keys
|
||||
Deterministic:
|
||||
- `proposal_shape`
|
||||
- `confidence_threshold`
|
||||
- `original_text_presence`
|
||||
- `non_empty_corrected_text`
|
||||
- `no_effect`
|
||||
- `protected_terms`
|
||||
|
||||
LLM-backed:
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
- `editorial_review`
|
||||
|
||||
## Built-in module chains
|
||||
`glossary`:
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
|
||||
`homophones`:
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
|
||||
`spoken_word`:
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `editorial_review`
|
||||
- `meaning_reversal_review`
|
||||
|
||||
`grammar`:
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `editorial_review`
|
||||
- `meaning_reversal_review`
|
||||
|
||||
## Ordering and execution semantics
|
||||
Validator ordering is based on canonical metadata:
|
||||
- deterministic validators run before LLM-backed validators.
|
||||
|
||||
Within each module stage:
|
||||
- proposals are generated per section;
|
||||
- validator chains execute on those proposals;
|
||||
- approved proposals are applied once after section work settles.
|
||||
|
||||
## Malformed payload behavior
|
||||
Malformed structured-output from proposal generation and LLM validator calls is downgraded, not treated as a process-fatal transport error.
|
||||
|
||||
Current outcomes:
|
||||
- malformed proposal-generation payloads produce section/module warnings and zero proposals for the affected section;
|
||||
- malformed validator decision payloads reject the affected validator batch with warnings;
|
||||
- deterministic validator behavior and runner order remain unchanged.
|
||||
|
||||
## Reporting identity
|
||||
Reports and diagnostics use stable validator keys as identifiers.
|
||||
|
||||
Correction-ledger deterministic-vs-LLM classification is derived from canonical validator metadata, not package-local hardcoded maps.
|
||||
|
||||
## Prompt assets
|
||||
LLM validator prompt assets and prompt metadata are documented in [Prompts](./prompts.md).
|
||||
149
docs/configuration.md
Normal file
149
docs/configuration.md
Normal file
@@ -0,0 +1,149 @@
|
||||
# Audita Configuration
|
||||
|
||||
## Scope
|
||||
This document defines the supported versioned YAML configuration model and runtime precedence behavior.
|
||||
|
||||
## Supported file version
|
||||
Current supported config file version:
|
||||
- `version: 1`
|
||||
|
||||
Validation rules:
|
||||
- missing `version` fails;
|
||||
- unsupported version fails;
|
||||
- unknown YAML fields fail (strict decoding).
|
||||
|
||||
## Config path resolution
|
||||
For `audita process` and `audita config print-effective`, path resolution order is:
|
||||
1. `--config <path>`
|
||||
2. `AUDITA_CONFIG`
|
||||
3. `/usr/local/etc/audita/config.yml` (if present)
|
||||
4. `/etc/audita/config.yml` (if present)
|
||||
|
||||
Missing-path behavior:
|
||||
- missing `--config` path is an error;
|
||||
- missing `AUDITA_CONFIG` path is an error;
|
||||
- missing both default paths is non-fatal.
|
||||
|
||||
## Effective precedence
|
||||
`audita process` effective precedence:
|
||||
1. defaults
|
||||
2. file config
|
||||
3. environment overrides
|
||||
4. CLI overrides
|
||||
|
||||
`audita config print-effective` uses:
|
||||
1. defaults
|
||||
2. file config
|
||||
3. environment overrides
|
||||
|
||||
`audita config validate` intentionally uses file-only validation:
|
||||
1. defaults
|
||||
2. file config
|
||||
|
||||
Environment overrides are not applied in `config validate`.
|
||||
|
||||
## Supported top-level YAML fields
|
||||
```yaml
|
||||
version: 1
|
||||
|
||||
pipeline:
|
||||
modules: [glossary, homophones, glossary, spoken_word, grammar]
|
||||
|
||||
output:
|
||||
schema: bare-segments
|
||||
|
||||
llm:
|
||||
proposal:
|
||||
base_url: https://openrouter.ai/api/v1
|
||||
model: openrouter/google/gemma-4-31b-it
|
||||
api_key_env: AUDITA_LLM_API_KEY
|
||||
timeout: 120s
|
||||
max_retries: 3
|
||||
validation:
|
||||
base_url: https://openrouter.ai/api/v1
|
||||
model: openrouter/google/gemma-4-31b-it
|
||||
api_key_env: AUDITA_VALIDATION_LLM_API_KEY
|
||||
timeout: 120s
|
||||
max_retries: 3
|
||||
|
||||
concurrency:
|
||||
total_llm: 2
|
||||
proposal_llm: 2
|
||||
validation_llm: 1
|
||||
|
||||
chunking:
|
||||
target_sections: 8
|
||||
max_section_tokens: 8192
|
||||
min_section_tokens: 2048
|
||||
|
||||
normalization:
|
||||
max_segment_gap: 4s
|
||||
ellipsis_gap: 3.5s
|
||||
max_segment_duration: 60s
|
||||
max_segment_tokens: 2048
|
||||
|
||||
thresholds:
|
||||
glossary: 0.8
|
||||
homophones: 0.8
|
||||
spoken_word: 0.8
|
||||
grammar: 0.8
|
||||
|
||||
context:
|
||||
description: "optional transcript background context"
|
||||
|
||||
diagnostics:
|
||||
work_dir: /tmp/audita
|
||||
retention: auto
|
||||
```
|
||||
|
||||
## Module and output-schema validation
|
||||
`pipeline.modules` keys are validated against the built-in supported module catalog.
|
||||
|
||||
Supported module keys:
|
||||
- `glossary`
|
||||
- `homophones`
|
||||
- `spoken_word`
|
||||
- `grammar`
|
||||
|
||||
Repeated supported module keys are allowed.
|
||||
|
||||
`output.schema` is validated against the built-in output schema catalog.
|
||||
|
||||
Supported output schema keys:
|
||||
- `bare-segments`
|
||||
- `audita-v1`
|
||||
|
||||
Unknown module keys and unknown output schema keys fail validation.
|
||||
|
||||
## Duration field parsing
|
||||
Duration-like fields support:
|
||||
- numeric seconds (for example `120`, `3.5`)
|
||||
- duration strings (for example `120s`, `2m`)
|
||||
|
||||
LLM timeout duration strings must resolve to whole seconds.
|
||||
|
||||
## Secret handling
|
||||
Use `api_key_env` fields for secrets:
|
||||
- `llm.proposal.api_key_env`
|
||||
- `llm.validation.api_key_env`
|
||||
|
||||
These fields store environment variable names, not secret values.
|
||||
|
||||
Resolved secret values are redacted from:
|
||||
- `audita config print-effective` output;
|
||||
- diagnostics `effective-config.json`;
|
||||
- report and diagnostics payloads.
|
||||
|
||||
## Commands
|
||||
Validate a file config:
|
||||
```sh
|
||||
audita config validate --config ./audita.yml
|
||||
```
|
||||
|
||||
Print redacted effective config:
|
||||
```sh
|
||||
audita config print-effective --config ./audita.yml
|
||||
```
|
||||
|
||||
## Compatibility notes
|
||||
Legacy compatibility flags and environment aliases remain available where implemented, but the stable configuration surface is the versioned YAML model described above.
|
||||
33
docs/development.md
Normal file
33
docs/development.md
Normal file
@@ -0,0 +1,33 @@
|
||||
# Audita Development Workflow
|
||||
|
||||
## Scope
|
||||
This document defines the canonical contributor workflow and engineering conventions for this repository.
|
||||
|
||||
## Workflow
|
||||
1. Start from a clean understanding of scope and constraints.
|
||||
2. Make focused changes that preserve existing public behavior unless behavior change is explicitly intended.
|
||||
3. Run targeted tests for touched packages.
|
||||
4. Run `go test ./...` before finalizing substantial changes.
|
||||
5. Update affected documentation so it describes current behavior only.
|
||||
|
||||
## Engineering conventions
|
||||
- Keep module packages separate: `glossary`, `homophones`, `spoken_word`, `grammar`.
|
||||
- Prefer narrow shared helpers and catalogs over broad abstractions.
|
||||
- Preserve diagnostics artifact naming and report field contracts unless intentionally changed.
|
||||
- Preserve CLI/config precedence semantics unless intentionally changed.
|
||||
- Treat stable validator keys, prompt identifiers, and output-schema keys as contract surfaces.
|
||||
|
||||
## Configuration and runtime expectations
|
||||
- `audita process` precedence is defaults -> file -> env -> CLI.
|
||||
- `audita config validate` validates file config merged onto defaults only.
|
||||
- `audita config print-effective` includes environment overrides and prints redacted JSON.
|
||||
|
||||
## Testing expectations
|
||||
- Add tests for new behavior and for bug fixes.
|
||||
- Keep deterministic fixtures stable.
|
||||
- Do not reduce existing parity, release-fixture, subprocess, or module-specific coverage without equivalent replacement.
|
||||
|
||||
## Commit discipline
|
||||
- Keep commits scoped and reviewable.
|
||||
- Avoid mixing unrelated refactors with behavior changes.
|
||||
- Use clear plain-English commit messages.
|
||||
27
docs/documentation/policy.md
Normal file
27
docs/documentation/policy.md
Normal file
@@ -0,0 +1,27 @@
|
||||
# Documentation Policy
|
||||
|
||||
## Scope
|
||||
This policy defines how project documentation should be authored and maintained.
|
||||
|
||||
## Core rules
|
||||
- Document the current behavior of the codebase.
|
||||
- Remove stale behavior descriptions promptly when code changes.
|
||||
- Do not describe development history in architecture or behavior docs unless a document is explicitly historical.
|
||||
- Do not use architecture or behavior docs as changelogs.
|
||||
- Prefer rewriting stale sections from scratch when substantial behavior or ownership changes occur.
|
||||
|
||||
## Consistency requirements
|
||||
- Keep command examples aligned with current CLI surfaces.
|
||||
- Keep configuration examples aligned with supported fields and precedence.
|
||||
- Keep architecture package ownership descriptions aligned with current code layout.
|
||||
- Keep stable contract identifiers accurate (module keys, validator keys, output-schema keys, report metadata fields).
|
||||
|
||||
## Cross-document expectations
|
||||
- `docs/architecture/*` documents runtime behavior and package ownership.
|
||||
- `docs/configuration.md` documents config schema and precedence.
|
||||
- `docs/development.md` documents contributor workflow and engineering conventions.
|
||||
|
||||
## Review expectations for documentation changes
|
||||
- Verify referenced files and links exist.
|
||||
- Verify examples match current behavior.
|
||||
- Prefer concise, direct language and avoid speculative future claims.
|
||||
96
docs/integration/subprocess-operations.md
Normal file
96
docs/integration/subprocess-operations.md
Normal file
@@ -0,0 +1,96 @@
|
||||
# Audita Subprocess Operations
|
||||
|
||||
This document describes how parent processes should invoke `audita process` safely in production orchestration.
|
||||
|
||||
## Recommended command form
|
||||
|
||||
Use explicit file outputs for orchestrated runs:
|
||||
|
||||
```sh
|
||||
audita process <transcript.json> \
|
||||
--transcript-description "Brief context that may help resolve ambiguous terms." \
|
||||
--glossary <glossary.yaml> \
|
||||
--output <output-transcript.json> \
|
||||
--report-json <report.json>
|
||||
```
|
||||
|
||||
Additional flags that may be situationally appropriate:
|
||||
- `--config <path>` to select an explicit versioned config file.
|
||||
- `--output-schema <bare-segments|audita-v1>` to select transcript output shape.
|
||||
- `--work-dir <dir>` to control diagnostics location.
|
||||
- `--work-dir-retention <always|auto|never>` to control retained run directories.
|
||||
- `--total-llm-concurrency`, `--proposal-llm-concurrency`, and `--validation-llm-concurrency` when orchestration needs to set explicit LLM throughput controls.
|
||||
- `--modules ...` only when intentionally overriding the default sequence.
|
||||
|
||||
For config-driven orchestration, validate config files in CI/preflight:
|
||||
|
||||
```sh
|
||||
audita config validate --config <path>
|
||||
```
|
||||
|
||||
## Stdout behavior
|
||||
|
||||
- With `--output`: stdout is expected to be empty on success.
|
||||
- Without `--output`: stdout contains transcript JSON only on success.
|
||||
- Report JSON is never written to stdout.
|
||||
|
||||
## Stderr behavior
|
||||
|
||||
- Success path should be quiet or minimal human-readable logs.
|
||||
- Failure path writes concise human-readable errors.
|
||||
- When a diagnostics run directory exists, failure stderr includes its path.
|
||||
- Prompt/response diagnostic payloads are not streamed to stderr.
|
||||
|
||||
## Output file behavior
|
||||
|
||||
- `--output` writes transcript JSON in the selected output schema to the provided path.
|
||||
- Output write failures return nonzero and surface actionable errors.
|
||||
- The command does not silently ignore output write errors.
|
||||
|
||||
## Report JSON behavior
|
||||
|
||||
- `--report-json` writes a machine-readable process report to the requested path.
|
||||
- Run-directory `report.json` is written independently under diagnostics.
|
||||
- Best-effort failure reports are emitted when possible without masking the primary failure.
|
||||
- Report write failures return nonzero with clear stderr messaging.
|
||||
- Report diagnostics metadata references run-directory artifacts including utilization diagnostics and correction ledger paths when available.
|
||||
|
||||
## Diagnostics directory behavior
|
||||
|
||||
- Each run creates (when possible) a per-run diagnostics directory.
|
||||
- Typical artifacts include transcript, normalization, chunking, invocation, effective config, LLM diagnostics, `utilization-diagnostics.json`, `correction-ledger.json`, `report.json`, and `error.log` on failure.
|
||||
- Failed runs retain diagnostics.
|
||||
- Under `auto` retention, successful runs with skipped/rejected corrections are retained; clean successful runs may be removed.
|
||||
|
||||
## Exit codes
|
||||
|
||||
- `0`: success.
|
||||
- Nonzero: failure (input/schema/config/module/LLM/runtime/output/report/diagnostics errors).
|
||||
|
||||
Treat any nonzero as a failed subprocess invocation.
|
||||
|
||||
## Timeout and cancellation
|
||||
|
||||
- Runtime operations propagate context cancellation and request timeouts through LLM/scheduler paths.
|
||||
- On cancellation or timeout, the process exits nonzero and should not hang.
|
||||
- If diagnostics were initialized before failure, failure artifacts remain available for debugging.
|
||||
|
||||
## Secret redaction expectations
|
||||
|
||||
API keys and configured secret values are redacted from:
|
||||
- reports (`--report-json` and run-dir `report.json`);
|
||||
- diagnostics artifacts (including effective config and LLM interaction artifacts);
|
||||
- surfaced adapter/runtime errors;
|
||||
- test fixtures and regression outputs.
|
||||
|
||||
Parent-process logs should still avoid printing raw environment variables.
|
||||
|
||||
## Parent-process pipe guidance
|
||||
|
||||
To avoid deadlocks in orchestrators:
|
||||
- always read both stdout and stderr concurrently when invoking as a subprocess;
|
||||
- prefer file outputs (`--output`, `--report-json`) for machine workflows;
|
||||
- treat stderr as human-readable diagnostics, not structured data;
|
||||
- parse structured results from output/report files.
|
||||
|
||||
For Go callers, prefer `exec.CommandContext` with explicit timeout/cancellation and buffered/streamed readers for both pipes.
|
||||
128
docs/release-checklist.md
Normal file
128
docs/release-checklist.md
Normal file
@@ -0,0 +1,128 @@
|
||||
# Audita Release Checklist
|
||||
|
||||
Use this checklist before cutting a pre-1.0 or 1.0 release candidate.
|
||||
|
||||
## Core test pass
|
||||
|
||||
- Run:
|
||||
- `go test ./...`
|
||||
- Confirm tests pass without live LLM credentials and without Python dependencies.
|
||||
|
||||
## Config validation and precedence
|
||||
|
||||
- Validate a representative config:
|
||||
- `audita config validate --config <path>`
|
||||
- Inspect redacted effective config:
|
||||
- `audita config print-effective --config <path>`
|
||||
- Confirm precedence behavior:
|
||||
- defaults -> file config -> environment -> CLI.
|
||||
- Confirm default config search order:
|
||||
- `/usr/local/etc/audita/config.yml` first, then `/etc/audita/config.yml`.
|
||||
- Confirm missing both default-path config files is non-fatal when `--config`/`AUDITA_CONFIG` are unset.
|
||||
|
||||
## Output schema checks
|
||||
|
||||
- Verify default output schema remains `bare-segments`.
|
||||
- Verify `--output-schema audita-v1` emits object payload with `schema` and `version`.
|
||||
- Verify unknown schema (for example `seriatim-intermediate`) fails clearly.
|
||||
|
||||
## Subprocess contract checks
|
||||
|
||||
- With `--output`, verify stdout is empty on success.
|
||||
- Without `--output`, verify stdout contains transcript JSON only.
|
||||
- Verify `--report-json` writes file output and does not write report JSON to stdout.
|
||||
- Verify failure stderr remains human-readable and includes diagnostics path when available.
|
||||
- Verify nonzero exit on failures.
|
||||
|
||||
## Structured LLM checks
|
||||
|
||||
- Verify runtime uses the Audita-owned OpenAI-compatible adapter.
|
||||
- Verify structured response schemas are attached via `response_format.type=json_schema`.
|
||||
- Verify diagnostics metadata includes structured schema `id/version/name/sha256`.
|
||||
- Verify provider output is still locally decoded/validated before use.
|
||||
- Verify malformed module-stage structured payloads degrade to warnings/rejections instead of failing the run.
|
||||
|
||||
## Report and diagnostics schema checks
|
||||
|
||||
- Verify report metadata fields:
|
||||
- `report_schema_name`
|
||||
- `report_schema_version`
|
||||
- `output_schema`
|
||||
- `config_version` when file config is used.
|
||||
- Verify diagnostics artifact references exist in reports:
|
||||
- transcript/normalization/chunking/invocation/effective-config artifacts
|
||||
- utilization diagnostics artifact
|
||||
- correction ledger artifact
|
||||
- error log on failures.
|
||||
|
||||
## Redaction checks
|
||||
|
||||
- Verify secrets are redacted from:
|
||||
- `effective-config.json`
|
||||
- run-dir and `--report-json` reports
|
||||
- LLM request/response/error diagnostics payloads.
|
||||
- Verify no API keys/bearer tokens leak into fixtures or outputs.
|
||||
|
||||
## Prompt and validator metadata checks
|
||||
|
||||
- Verify prompt metadata appears in LLM request metadata diagnostics:
|
||||
- `prompt_id`, `prompt_version`, `prompt_source`, `embedded_path`, `sha256`.
|
||||
- Verify stable validator keys appear in report decisions/rejections.
|
||||
- Verify module warning records appear in reports for malformed proposal-generation payloads and malformed validator batches.
|
||||
- Verify built-in validator chains resolve and execute for default and explicit module runs.
|
||||
|
||||
## Utilization diagnostics checks
|
||||
|
||||
- Verify `utilization-diagnostics.json` exists on successful runs.
|
||||
- Verify partial utilization artifact behavior on controlled failure paths.
|
||||
- Verify utilization fields are structurally present and nonnegative:
|
||||
- effective concurrency
|
||||
- run timing
|
||||
- module timing summaries
|
||||
- per-validator timing summaries.
|
||||
|
||||
## Correction ledger checks
|
||||
|
||||
- Verify `correction-ledger.json` exists on successful runs.
|
||||
- Verify report references ledger artifact path.
|
||||
- Verify ledger dispositions include applied/rejected and skipped/failed where exercised.
|
||||
- Verify validator rejection and proposal-application skip remain distinct.
|
||||
|
||||
## Pipeline behavior checks
|
||||
|
||||
- Verify default full pipeline run remains:
|
||||
- `glossary`, `homophones`, `glossary`, `spoken_word`, `grammar`
|
||||
- with deterministic repeated instance naming (`glossary_1`, `glossary_2`).
|
||||
- Verify explicit module runs (`--modules`) still work.
|
||||
|
||||
## Failure and cancellation checks
|
||||
|
||||
- Verify controlled failure paths retain diagnostics and produce best-effort failure reports.
|
||||
- Verify malformed proposal-generation payloads keep exit code `0`, keep stderr empty on success, and record warnings in reports/diagnostics.
|
||||
- Verify malformed validator payloads reject only the affected batch and do not fail the module.
|
||||
- Verify timeout/cancellation paths exit nonzero, do not hang, and retain failure diagnostics when initialized.
|
||||
|
||||
## Release fixture/idempotence checks
|
||||
|
||||
- Run release fixtures (`internal/cli/testdata/release`) through `go test ./...`.
|
||||
- Confirm fixture checks cover:
|
||||
- must-apply and must-not-apply expectations
|
||||
- protected-term survival
|
||||
- report and diagnostics contracts
|
||||
- output-schema checks
|
||||
- prompt/schema metadata diagnostics
|
||||
- utilization/ledger artifacts
|
||||
- idempotence-oriented second pass no-op behavior with deterministic fake responses.
|
||||
|
||||
## Deferred-feature guardrail
|
||||
|
||||
- Confirm release docs do not claim support for deferred items:
|
||||
- filesystem prompt overrides
|
||||
- user-configurable validator chains
|
||||
- arbitrary user-supplied output schemas
|
||||
- resume/start-at/stop-after execution
|
||||
- diff/check/propose-only modes
|
||||
- generated transcript descriptions enabled by default
|
||||
- interactive review UI
|
||||
- UI/server wrapper
|
||||
- provider benchmarking harness.
|
||||
791
docs/roadmap/audit.md
Normal file
791
docs/roadmap/audit.md
Normal file
@@ -0,0 +1,791 @@
|
||||
# Pre-1.0 Code Quality and Deduplication Audit
|
||||
|
||||
## 1. Executive summary
|
||||
|
||||
Audita is in good shape for a limited pre-1.0 cleanup pass. The repository is small, package boundaries are mostly explicit, and the core public contract is already documented around `audita process`, config loading, output schemas, diagnostics, reports, embedded prompts, modules, and validators. The highest-value improvements are targeted centralization, not a rewrite.
|
||||
|
||||
Top three refactoring targets before 1.0:
|
||||
|
||||
1. Centralize module proposal plumbing and prompt payload construction across the four production modules.
|
||||
2. Centralize effective config loading plus schema/module catalog validation so `process`, `config print-effective`, and `config validate` cannot drift.
|
||||
3. Centralize diagnostics artifact names, stage names, and validator classification metadata used by reports and the correction ledger.
|
||||
|
||||
No major architectural risk is apparent. The main pre-1.0 risk is public-behavior drift from repeated policy strings, catalog values, artifact paths, and nearly identical command/module scaffolding.
|
||||
|
||||
This report was written to `docs/roadmap/audit.md`. `docs/roadmap/` already exists in the repository, although its previous `publish.md` file is currently deleted in the worktree by an unrelated change.
|
||||
|
||||
## 2. Repository map reviewed
|
||||
|
||||
Reviewed documentation:
|
||||
|
||||
- `README.md`
|
||||
- `docs/configuration.md`
|
||||
- `docs/architecture/architecture.md`
|
||||
- `docs/architecture/public-contract.md`
|
||||
- `docs/architecture/diagnostics.md`
|
||||
- `docs/architecture/output-schemas.md`
|
||||
- `docs/architecture/prompts.md`
|
||||
- `docs/architecture/validators.md`
|
||||
- `docs/architecture/structured-llm.md`
|
||||
- `docs/integration/subprocess-operations.md`
|
||||
- `docs/release-checklist.md`
|
||||
|
||||
Reviewed implementation areas:
|
||||
|
||||
- `cmd/audita`
|
||||
- `internal/cli`
|
||||
- `internal/core/config`
|
||||
- `internal/core/schema`
|
||||
- `internal/core/io`
|
||||
- `internal/core/normalization`
|
||||
- `internal/core/chunking`
|
||||
- `internal/core/diagnostics`
|
||||
- `internal/core/outputschema`
|
||||
- `internal/core/reporting`
|
||||
- `internal/framework/contracts`
|
||||
- `internal/framework/modules`
|
||||
- `internal/framework/proposal_generation`
|
||||
- `internal/framework/proposals`
|
||||
- `internal/framework/runner`
|
||||
- `internal/framework/validators`
|
||||
- `internal/framework/llm`
|
||||
- `internal/framework/responseschema`
|
||||
- `internal/framework/promptcontext`
|
||||
- `internal/framework/warnings`
|
||||
- `internal/modules/glossary`
|
||||
- `internal/modules/homophones`
|
||||
- `internal/modules/spoken_word`
|
||||
- `internal/modules/grammar`
|
||||
- `internal/prompts`
|
||||
- `internal/validators`
|
||||
- package tests and CLI parity/release fixtures under `internal/cli/testdata`
|
||||
|
||||
Major execution paths reviewed:
|
||||
|
||||
- `audita process <transcript.json> --glossary <glossary.yaml>`
|
||||
- `audita config validate --config <path>`
|
||||
- `audita config print-effective [--config <path>]`
|
||||
- default module sequence resolution and repeated glossary instance naming
|
||||
- proposal generation, validator execution, proposal application, report writing, diagnostics writing, and retention
|
||||
|
||||
Important absent or not-applicable areas:
|
||||
|
||||
- No `pkg/` directory exists.
|
||||
- No `examples/` directory exists.
|
||||
- No `docs/internal/` directory exists.
|
||||
- No `internal/app`, `internal/stage`, `internal/storage`, `internal/artifacts`, or `internal/manifest` packages exist. Their closest equivalents are `internal/cli`, `internal/framework/runner`, `internal/core/diagnostics`, and `internal/core/reporting`.
|
||||
|
||||
## 3. High-confidence deduplication opportunities
|
||||
|
||||
### 3.1 Module proposal plumbing is duplicated across all production modules
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/modules/glossary/module.go`
|
||||
- `internal/modules/homophones/module.go`
|
||||
- `internal/modules/spoken_word/module.go`
|
||||
- `internal/modules/grammar/module.go`
|
||||
- `internal/modules/*/prompt.go`
|
||||
- `internal/framework/proposal_generation`
|
||||
- `internal/framework/promptcontext`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Each module has the same `Module` struct shape, `Validators` copy behavior, `Propose` flow, section transcript extraction, transcript description extraction, `proposal_generation.GenerateCandidates` request construction, prompt metadata map construction, and stage-name formatting.
|
||||
- Each module also has a near-identical prompt payload builder with local `promptSegment` and `promptTranscriptSection` types, glossary JSON marshaling, transcript section JSON marshaling, transcript description block rendering, and two-message return shape.
|
||||
- `collectSectionProposals` already passes a section transcript to each module, but each module then filters that transcript again by section metadata.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- A diagnostics or prompt-context bug fix would need to be repeated in four modules.
|
||||
- Prompt metadata fields and stage names are diagnostics-visible and could drift by module.
|
||||
- The double section filtering is currently harmless, but it obscures the runner/module contract.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Add a small shared helper for module proposal execution, likely in `internal/framework/proposal_generation` or a narrow `internal/modules/modulekit` package.
|
||||
- Keep domain-specific prompt IDs and prompt text local to each module.
|
||||
- Move transcript section prompt payload construction into a shared prompt-context helper, for example `promptcontext.MarshalTranscriptSection`.
|
||||
- Provide one helper for prompt metadata maps instead of manually expanding `prompt_id`, `prompt_version`, `prompt_source`, `embedded_path`, and `sha256` in every module.
|
||||
- Preserve current module `Key`, replacement policy, and validator chain ownership.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Keep one golden or table-driven prompt payload test per module for domain-specific wording.
|
||||
- Add shared tests for transcript section JSON shape, empty transcript handling, categories copy behavior, and prompt metadata fields.
|
||||
- Add a parity test that all four module `Propose` methods still write diagnostics under the same module instance directory and produce the same correction mapping.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Low to medium. The behavior is highly duplicated, but prompt and diagnostics behavior is sensitive. Refactor behind existing module tests and CLI parity fixtures.
|
||||
|
||||
### 3.2 Effective config loading is repeated between commands
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/cli/run.go`
|
||||
- `internal/core/config`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- `runProcess` and `runConfigPrintEffective` both resolve config path, start from defaults, optionally load/apply file config, then apply environment overrides.
|
||||
- `runConfigValidate` separately loads a file, applies it to defaults, and validates it.
|
||||
- Path source metadata is computed in `internal/cli`, not `internal/core/config`, even though the precedence contract is documented as config behavior.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Config precedence is part of the public contract. If a future setting is added, three command paths may need coordinated updates.
|
||||
- `config print-effective` is the user-visible diagnostic for effective config. It should use the same loader as `process`, except for intentionally omitted CLI overrides.
|
||||
- The current code is understandable, but the behavior is repeated in a way that makes drift likely as config grows.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Add a narrow effective-config loader in `internal/core/config`, returning `Config`, source path, source type, and version metadata.
|
||||
- Keep command-specific CLI flag parsing in `internal/cli`.
|
||||
- Model the intentional differences explicitly:
|
||||
- `process`: defaults + file + env + CLI overrides
|
||||
- `config print-effective`: defaults + file + env
|
||||
- `config validate`: file schema + default-backed config validation, no env
|
||||
- Move `resolveConfigPath` or an equivalent path resolver into `internal/core/config`.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- One table-driven config loader test covering explicit `--config`, `AUDITA_CONFIG`, default search paths, missing explicit paths, and missing default paths.
|
||||
- CLI tests asserting `process` and `print-effective` share file+env behavior.
|
||||
- A regression test that `config validate` remains file-only and does not read environment overrides.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Low. Behavior is already explicit and well tested; the refactor can be done by moving code without changing precedence.
|
||||
|
||||
### 3.3 Module catalog validation is split across config, contracts, and module factory
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/core/config/validation.go`
|
||||
- `internal/framework/contracts/contracts.go`
|
||||
- `internal/framework/modules/registry.go`
|
||||
- `internal/validators/chains.go`
|
||||
- `internal/framework/validators/models.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Module keys appear in multiple places:
|
||||
- config default CSV: `glossary,homophones,glossary,spoken_word,grammar`
|
||||
- module factory constants and known-key map
|
||||
- built-in validator chains
|
||||
- confidence threshold lookup
|
||||
- individual module `Key()` methods
|
||||
- `Config.Validate` checks only that module names are non-empty. An unsupported configured module can pass `audita config validate` and fail later in `process` runner setup.
|
||||
- `contracts.ResolveModuleRunSpecs` only assigns instance names; it does not validate production module support.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- `audita config validate` is documented as a CI/preflight command. Letting unsupported modules pass weakens that preflight.
|
||||
- Module key drift could affect thresholds, validator chains, reports, and unsupported-module errors.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Introduce a small canonical module catalog or key package that can be imported by config validation, module factory construction, validator chain resolution, and threshold lookup without creating a cycle.
|
||||
- Keep module construction in `internal/framework/modules`; the catalog should expose keys and validation only.
|
||||
- Make `Config.Validate` reject unknown built-in module keys through that catalog.
|
||||
- Keep repeated module instances valid.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- `internal/core/config` test: unknown `pipeline.modules` fails validation.
|
||||
- `internal/cli` test: `audita config validate --config` rejects an unsupported module before runtime.
|
||||
- Existing `internal/framework/modules` unknown-module tests should continue to pass.
|
||||
- Validator chain tests should assert every catalog module has a built-in chain.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Medium. This tightens validation behavior. It is desirable before 1.0, but if unknown modules were intentionally allowed for future extension, document that explicitly instead.
|
||||
|
||||
### 3.4 Output schema support is hardcoded in config validation and registry
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/core/config/validation.go`
|
||||
- `internal/core/outputschema/registry.go`
|
||||
- `docs/architecture/output-schemas.md`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- `Config.Validate` hardcodes `bare-segments` and `audita-v1`.
|
||||
- `outputschema.Resolve` owns the actual output schema registry and returns the runtime error for unsupported schema names.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Adding or deferring a schema requires updating multiple places.
|
||||
- Public behavior could drift: a schema might validate in config but fail at output time, or vice versa.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Make `internal/core/outputschema` expose `IsSupported`, `SupportedKeys`, or a validation function.
|
||||
- Have config validation call that helper or consume shared constants.
|
||||
- Keep actual encoding logic in `outputschema`; config should not know encoder details.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Config validation test for every output schema returned by the registry.
|
||||
- Output schema registry test that unsupported `seriatim-intermediate` still fails clearly until implemented.
|
||||
- CLI test that unsupported `--output-schema` fails before output write.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Low. This is a straightforward catalog centralization.
|
||||
|
||||
### 3.5 Diagnostics artifact names and report metadata paths are repeated
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/core/diagnostics/run_dir.go`
|
||||
- `internal/cli/run.go`
|
||||
- `internal/core/reporting/report.go`
|
||||
- docs under `docs/architecture` and `docs/integration`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Artifact filenames such as `source-transcript.json`, `source-transcript-parsed.json`, `normalized-transcript.json`, `normalization-summary.json`, `chunking-summary.json`, `utilization-diagnostics.json`, `correction-ledger.json`, `invocation.json`, `effective-config.json`, `report.json`, and `error.log` are repeated between run-directory writers and `buildProcessReport`.
|
||||
- `runProcess` writes `utilization-diagnostics.json` and `correction-ledger.json` by raw string on both success and failure paths.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- These names are part of the documented diagnostics contract.
|
||||
- A filename change would need to be made in multiple places, and report metadata could point at files that are no longer written.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Define diagnostics artifact name constants in `internal/core/diagnostics`.
|
||||
- Add a helper that returns `reporting.DiagnosticsMetadata` for a run directory and status.
|
||||
- Add named methods for utilization diagnostics and correction ledger writes, or at least constants used by `WriteJSONArtifact`.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Unit test that `diagnostics.MetadataForRunDirectory` matches files written by `RunDirectory`.
|
||||
- CLI success/failure tests should continue to assert report metadata paths and actual file existence.
|
||||
- Add a test for failure report metadata including `error.log`.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Low. This is mostly string centralization, with high public-contract value.
|
||||
|
||||
### 3.6 Validator execution class is duplicated and partially hardcoded
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/validators/registry.go`
|
||||
- `internal/validators/metadata/metadata.go`
|
||||
- `internal/validators/*/validator.go`
|
||||
- `internal/framework/runner/runner.go`
|
||||
- `internal/cli/review_artifacts.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Validator constructors wrap validators with execution class metadata.
|
||||
- `BuiltInValidatorDefinition` also has an `LLMBacked` field.
|
||||
- Runner uses `metadata.ClassOf` to order deterministic validators before LLM-backed validators.
|
||||
- Correction ledger classification uses a local hardcoded map of LLM-backed validator names.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Adding a new LLM-backed validator could be ordered correctly by runner metadata but appear in the wrong correction-ledger section.
|
||||
- Validator class is domain metadata, not report-building policy. It should have one source of truth.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Make validator classification resolvable by validator instance or stable key from a single metadata source.
|
||||
- Remove the unused or redundant `LLMBacked` field, or make it the canonical source used by constructors, runner ordering, and ledger formatting.
|
||||
- Replace the local ledger map with `metadata.ClassOf` when possible, or a registry lookup by stable key.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Correction ledger test that LLM-backed decisions are classified from validator metadata, not a local string map.
|
||||
- Registry test that every registered LLM-backed validator reports the same class through every public metadata path.
|
||||
- Runner ordering test should remain in place.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Low to medium. The implementation is small, but correction-ledger shape is diagnostics-visible.
|
||||
|
||||
### 3.7 Malformed structured-output classification is duplicated
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/framework/proposal_generation/generate.go`
|
||||
- `internal/framework/validators/llm_validators.go`
|
||||
- `internal/framework/llm/openai_compatible_client.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Proposal generation and LLM validators both classify malformed structured-output errors by scanning error message substrings.
|
||||
- The marker lists are currently the same, but they are maintained independently.
|
||||
- The actual errors originate in the LLM adapter.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Proposal-generation malformed payloads become warnings with zero proposals, while validator malformed payloads reject affected batches with warnings. If classifiers drift, similar adapter failures could be downgraded in one workflow and hard-fail in another.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Prefer a typed error or exported classifier from `internal/framework/llm`.
|
||||
- If typed errors are too invasive, create one shared classifier function in a lower framework package used by both proposal generation and validators.
|
||||
- Preserve the different handling semantics at each call site.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Shared classifier table for all adapter malformed-output errors.
|
||||
- Proposal-generation test and validator test should assert the same representative malformed adapter errors are downgraded.
|
||||
- Adapter tests should assert typed/classified errors wrap useful context and still redact secrets.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Medium. Error typing can accidentally affect retry and wrapping behavior; do this with focused tests.
|
||||
|
||||
## 4. Medium-confidence opportunities
|
||||
|
||||
### 4.1 CLI flag registration and override extraction are large and repetitive
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/cli/run.go`
|
||||
- `internal/core/config/flags.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Each process flag has a field in `processFlags`, a registration entry in `newProcessFlagSet`, a case in `fs.Visit`, and an assignment in `config.ApplyCLIOverrides`.
|
||||
- File config and environment config also set many of the same effective config fields.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Adding a new config option requires multiple edits. Missing one edit could create a flag that displays but does not override, or a config field with no CLI override.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Avoid a generic reflection-heavy flag system before 1.0.
|
||||
- Consider a small metadata table only for simple scalar flags, or a focused helper that maps visited flags to `CLIOverrides`.
|
||||
- Keep nontrivial semantics, such as legacy concurrency alias precedence, explicit in code.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- CLI override parity test for every stable flag that mutates config.
|
||||
- A test that default flag values reflect file+env effective config before CLI overrides.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Medium. A broad flag abstraction would be riskier than the current duplication. Do only a small helper if it clearly reduces missed updates.
|
||||
|
||||
### 4.2 Config source application repeats field-level assignments
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/core/config/file_config.go`
|
||||
- `internal/core/config/env.go`
|
||||
- `internal/core/config/flags.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- The same effective fields are assigned from file config, env vars, and CLI overrides.
|
||||
- Some semantics differ intentionally: file config supports `api_key_env`, env supports `OPENROUTER_API_KEY` fallback, CLI uses direct values.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Field additions are easy to miss in one source.
|
||||
- Error messages and trimming behavior can drift.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Do not force all config sources through one generic mapper.
|
||||
- Add small setter helpers for repeated config subdomains such as LLM target, concurrency, thresholds, normalization, and diagnostics.
|
||||
- Keep source-specific parsing and error labels local.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Cross-source table proving file, env, and CLI all reach the same effective fields where they are meant to.
|
||||
- Tests for intentional differences: API key env resolution, `OPENROUTER_API_KEY` fallback, CLI direct API key, and transcript description trimming.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Medium. Useful, but only after the effective loader and catalog cleanup.
|
||||
|
||||
### 4.3 Prompt metadata and response schema metadata map construction repeats
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/modules/*/module.go`
|
||||
- `internal/framework/proposal_generation/generate.go`
|
||||
- `internal/framework/validators/llm_validators.go`
|
||||
- `internal/prompts`
|
||||
- `internal/framework/responseschema`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Prompt metadata maps are manually expanded in module proposal generation and validator diagnostics.
|
||||
- Response schema metadata maps are built independently in proposal generation and validator diagnostics.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Metadata fields are diagnostics-visible and useful for reproducibility.
|
||||
- Adding a metadata field requires updating multiple call sites.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Add `Metadata.Map()` or a typed diagnostics metadata struct in `internal/prompts`.
|
||||
- Add `responseschema.Metadata()` or a method returning a stable diagnostics shape.
|
||||
- Prefer typed structs over `map[string]any` where possible.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Prompt metadata rendering test should assert all registered prompts expose stable metadata.
|
||||
- Proposal and validator diagnostics tests should assert the shared metadata helper is used.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Low.
|
||||
|
||||
### 4.4 Secret redaction logic is split across config, LLM diagnostics, and adapter errors
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/core/config/redaction.go`
|
||||
- `internal/framework/llm/diagnostics.go`
|
||||
- `internal/framework/llm/client_common.go`
|
||||
- `internal/framework/proposal_generation/generate.go`
|
||||
- `internal/framework/runner/runner.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Config redaction replaces non-empty API keys with `[REDACTED]`.
|
||||
- LLM diagnostics replace configured secret values and `Bearer <secret>`.
|
||||
- Adapter error sanitization separately replaces secrets and bearer values.
|
||||
- Proposal and validator paths separately assemble secret lists.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Secret redaction is a public guarantee.
|
||||
- New secret-bearing config fields could be missed in one path.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Add a small redaction helper package or keep it in `internal/framework/llm` only if it remains LLM-specific.
|
||||
- Centralize `[]string` secret extraction from `config.Config`.
|
||||
- Keep config structural redaction separate from byte/string payload redaction, but share the redaction token and value replacement behavior.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- One test that a proposal-generation error, validator diagnostic artifact, effective config artifact, and surfaced provider error all redact the same configured secrets.
|
||||
- Existing subprocess no-secret-leak test should remain as an end-to-end guard.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Medium. The current coverage appears strong; change carefully.
|
||||
|
||||
### 4.5 Test fakes and fixture helpers are duplicated across packages
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/modules/*/module_test.go`
|
||||
- `internal/framework/proposal_generation/generate_test.go`
|
||||
- `internal/framework/validators/llm_validators_test.go`
|
||||
- `internal/cli/run_test.go`
|
||||
- `cmd/audita/main_integration_test.go`
|
||||
- `internal/cli/release_fixtures_test.go`
|
||||
- `internal/cli/parity_test.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Several packages define fake structured LLM clients, fixture path helpers, read/write helpers, diagnostics glob assertions, and run-directory helpers.
|
||||
- The four module test files have particularly similar fake clients and proposal-diagnostics assertions.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Refactors in LLM or diagnostics behavior require updating many tests.
|
||||
- Some duplicated tests are valuable because they preserve per-module public behavior; the issue is helper duplication, not coverage volume.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Add package-local helper files where duplication is within a package.
|
||||
- For cross-package fakes, prefer a small internal test support package only if it does not create import cycles or hide test intent.
|
||||
- Keep module-specific assertions local.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- This is test infrastructure cleanup. Existing tests should remain semantically equivalent.
|
||||
- Add helper tests only if helpers contain nontrivial behavior, such as fake response sequencing.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Low.
|
||||
|
||||
### 4.6 Stage-name construction is inconsistent enough to centralize, but not enough to redesign
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/modules/*/module.go`
|
||||
- `internal/framework/proposal_generation/generate.go`
|
||||
- `internal/framework/validators/llm_validators.go`
|
||||
- `internal/framework/runner/observability.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Modules pass stage names like `<module_instance>:proposal:section-0001`.
|
||||
- `proposal_generation` has a default builder using `<module_instance>:proposal-generation:section-0001`, but production modules bypass it.
|
||||
- Validators build `<module_instance>:<validator>:batch-0001`.
|
||||
- Utilization extracts module instance by splitting stage names on `:`.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Stage names affect diagnostics filenames and observability grouping.
|
||||
- Current behavior works, but the naming grammar is implicit.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Add narrow helpers for proposal and validator stage names.
|
||||
- Preserve current production stage names unless there is a deliberate pre-1.0 diagnostics compatibility decision.
|
||||
- Keep filename sanitization in `internal/framework/llm`.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Unit tests for stage-name helper output.
|
||||
- Utilization test that module instance extraction still works for proposal and validator stage names.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Medium. Renaming stages can change diagnostics filenames, so avoid unnecessary churn.
|
||||
|
||||
## 5. Boundary and responsibility concerns
|
||||
|
||||
### CLI owns too much report and diagnostics metadata assembly
|
||||
|
||||
`internal/cli/run.go` is doing orchestration, command parsing, config loading, output routing, report assembly, diagnostics metadata path assembly, and correction-ledger construction. This is acceptable for a small CLI, but two pieces are drifting beyond command responsibility:
|
||||
|
||||
- diagnostics artifact path metadata belongs closer to `internal/core/diagnostics`;
|
||||
- report assembly and correction-ledger mapping belong closer to `internal/core/reporting` or a narrow reporting adapter package.
|
||||
|
||||
Recommended home:
|
||||
|
||||
- `internal/core/diagnostics`: artifact constants and diagnostics metadata path construction.
|
||||
- `internal/core/reporting`: pure mapping from runner/config/diagnostics state into report payloads.
|
||||
- `internal/cli`: command parsing, invocation wiring, exit codes, stdout/stderr behavior.
|
||||
|
||||
### Config validation lacks catalog ownership
|
||||
|
||||
`internal/core/config` currently validates only generic module list shape and hardcodes output schema keys. Because modules and output schemas are public contract values, config validation should use a catalog owned by the relevant domain.
|
||||
|
||||
Recommended home:
|
||||
|
||||
- output schema validation: `internal/core/outputschema`;
|
||||
- module key validation: a small catalog package or lower-level constants package importable by config, module factory, validator chains, and threshold lookup.
|
||||
|
||||
### Runner owns adapter shims between contracts and validator framework
|
||||
|
||||
`internal/framework/runner` contains `validationLLMClientAdapter` and `llmDiagnosticsWriterAdapter`. This is not a serious problem today because runner wires proposal and validation workflows. If these adapters grow, move them to `internal/framework/validators` or a small integration package so runner remains focused on orchestration.
|
||||
|
||||
### LLM malformed-output policy is spread across callers
|
||||
|
||||
The LLM adapter emits the errors, while proposal generation and validators classify them by message text. The policy decision is caller-specific, but the classification should live with the LLM/framework error type.
|
||||
|
||||
## 6. Path, key, and naming construction review
|
||||
|
||||
Centralized enough:
|
||||
|
||||
- LLM diagnostics artifact suffixes and stage sanitization are centralized in `internal/framework/llm/diagnostics.go`.
|
||||
- Output file writing is routed through `internal/core/io.WriteFile`.
|
||||
- Run directories are created in `internal/core/diagnostics.NewRunDirectory`.
|
||||
|
||||
Needs cleanup:
|
||||
|
||||
- Core diagnostics artifact names are repeated between `RunDirectory` writer methods and `buildProcessReport`.
|
||||
- `utilization-diagnostics.json` and `correction-ledger.json` are raw strings in both success and failure paths.
|
||||
- Proposal and validator diagnostics subdirectory construction repeats `filepath.Join(diagnosticsDir, moduleInstance)`.
|
||||
- Proposal and validator stage names are manually formatted in multiple packages.
|
||||
- Module keys are repeated across config defaults, module factory, validator chains, confidence threshold lookup, and module implementations.
|
||||
- Output schema names are repeated between config validation and `outputschema`.
|
||||
|
||||
Recommendation:
|
||||
|
||||
- Start with artifact constants and metadata helpers because that is the lowest-risk path/key cleanup.
|
||||
- Then centralize stage-name helpers without changing current production naming.
|
||||
- Defer any broader "path manager" abstraction.
|
||||
|
||||
## 7. Resolution and catalog review
|
||||
|
||||
Modules:
|
||||
|
||||
- Runtime module construction has a production registry in `internal/framework/modules`.
|
||||
- Instance naming for repeated modules is centralized in `contracts.ResolveModuleRunSpecs`.
|
||||
- Unknown module failure exists in the factory, but config validation does not catch unknown modules.
|
||||
- Built-in validator chain resolution separately maps module key to validator keys.
|
||||
|
||||
Output schemas:
|
||||
|
||||
- Encoding is centralized in `internal/core/outputschema`.
|
||||
- Validation is duplicated in config.
|
||||
|
||||
Prompts:
|
||||
|
||||
- Prompt asset lookup and metadata are centralized in `internal/prompts`.
|
||||
- Prompt metadata map construction is repeated at call sites.
|
||||
- Prompt source selection is intentionally built-in only and should remain that way for 1.0.
|
||||
|
||||
Validators:
|
||||
|
||||
- Validator construction is package-owned under `internal/validators`.
|
||||
- Chains are centralized in `internal/validators/chains.go`.
|
||||
- Execution class metadata exists, but reporting/correction-ledger classification does not fully use it.
|
||||
|
||||
Schemas:
|
||||
|
||||
- Transcript and glossary parsing/validation are centralized in `internal/core/schema`.
|
||||
- Structured LLM response schemas are centralized in `internal/framework/responseschema`.
|
||||
- Output schema registry and response schema registry are appropriately separate.
|
||||
|
||||
Recommendation:
|
||||
|
||||
- Introduce only small catalog helpers for module keys, output schema keys, prompt metadata maps, response schema metadata maps, and validator execution class.
|
||||
- Avoid user-configurable modules, validators, prompts, or schemas before 1.0 unless already planned elsewhere.
|
||||
|
||||
## 8. Config and command-loading review
|
||||
|
||||
Consistent behavior:
|
||||
|
||||
- The documented precedence for `process` is implemented: defaults, file config, environment, CLI.
|
||||
- `config print-effective` intentionally omits CLI process flags and uses defaults, file config, and environment.
|
||||
- `config validate` intentionally requires `--config` and does not require transcript/glossary inputs.
|
||||
- Missing explicit config paths are hard failures; missing default paths are non-fatal.
|
||||
- Environment parsing and CLI parsing both preserve legacy total-concurrency alias behavior.
|
||||
|
||||
Likely accidental or high-risk differences:
|
||||
|
||||
- Unsupported module names pass `Config.Validate` and `audita config validate`.
|
||||
- Output schema support is duplicated instead of delegated to the output schema registry.
|
||||
- Config path resolution lives in CLI even though it is part of config behavior.
|
||||
|
||||
Intentional differences:
|
||||
|
||||
- File config resolves `api_key_env`; env and CLI set direct API key values.
|
||||
- `OPENROUTER_API_KEY` is an environment fallback only for the primary LLM.
|
||||
- `transcript-description` has CLI/config support but no `AUDITA_*` environment variable, matching documentation.
|
||||
|
||||
Recommendation:
|
||||
|
||||
- Build a shared effective config context helper and keep source-specific parsing semantics explicit.
|
||||
- Tighten catalog validation before 1.0 if unknown modules are not meant to be accepted.
|
||||
|
||||
## 9. State, manifest, or progress handling review
|
||||
|
||||
Audita does not currently have a manifest/checkpoint/resume model. State is per-run diagnostics and report artifacts.
|
||||
|
||||
Consistent behavior:
|
||||
|
||||
- `process` creates one diagnostics run directory when diagnostics initialization succeeds.
|
||||
- Failures after run-dir creation write `error.log`, best-effort report artifacts, and retain diagnostics.
|
||||
- Success writes optional `--report-json`, run-dir `report.json`, utilization diagnostics, and correction ledger.
|
||||
- Retention is centralized in `diagnostics.ShouldRetainRunDirectory`.
|
||||
- There is no resume/retry/force behavior to preserve.
|
||||
|
||||
Drift risks:
|
||||
|
||||
- Success and failure paths both write utilization and correction-ledger artifacts with duplicated raw filenames.
|
||||
- Report diagnostics metadata is assembled independently from the run-directory writer methods.
|
||||
- Retention mode `never` currently still retains successful run directories in `ShouldRetainRunDirectory`, which may be intentional per tests or a naming/documentation mismatch. Do not change it in a dedup pass without first confirming semantics.
|
||||
|
||||
Recommendation:
|
||||
|
||||
- Centralize artifact names and report metadata path construction.
|
||||
- Keep retention behavior unchanged unless a separate bug review confirms the intended meaning of `never`.
|
||||
|
||||
## 10. Refactors to avoid before 1.0
|
||||
|
||||
- Do not introduce a generic workflow engine. The current sequential runner is clear and explicit.
|
||||
- Do not add a plugin architecture for modules, validators, prompts, or schemas before 1.0.
|
||||
- Do not redesign the CLI or replace `flag` with a larger framework only for deduplication.
|
||||
- Do not collapse all config source parsing into a reflection-based mapper; source semantics differ intentionally.
|
||||
- Do not merge module packages into one generic module type. Keep domain-specific prompt assets, keys, validator chains, and replacement policies visible.
|
||||
- Do not rewrite diagnostics or reporting schemas broadly. Centralize names and mapping helpers first.
|
||||
- Do not change diagnostics stage names casually; they affect artifact filenames and debugging workflows.
|
||||
- Do not consolidate deterministic and LLM validator behavior just because both return decisions. Their failure and batching semantics differ.
|
||||
- Do not generalize transcript/glossary schema parsing into a broad schema framework.
|
||||
- Do not reduce duplicated tests where the duplication protects distinct public command/module behavior.
|
||||
|
||||
## 11. Recommended implementation sequence
|
||||
|
||||
1. Centralize diagnostics artifact constants and diagnostics metadata path construction.
|
||||
2. Centralize output schema validation through `internal/core/outputschema`.
|
||||
3. Introduce a small module key catalog and use it in config validation, module factory, validator chains, and threshold lookup.
|
||||
4. Add an effective config loading context helper for defaults + file + env, then update `process` and `config print-effective`.
|
||||
5. Extract shared module proposal plumbing and prompt transcript-section payload construction.
|
||||
6. Centralize prompt metadata and response schema metadata map construction.
|
||||
7. Centralize validator execution-class lookup and update correction-ledger classification.
|
||||
8. Centralize malformed structured-output classification through a typed/shared LLM error helper.
|
||||
9. Add or consolidate focused test helpers for module LLM fakes, diagnostics assertions, and fixture paths.
|
||||
10. Do a final dead-code and legacy sweep for redundant helper fields such as unused validator definition metadata.
|
||||
|
||||
Each item can be a separate commit with package-level tests and at least one CLI regression where public behavior is involved.
|
||||
|
||||
## 12. Test strategy
|
||||
|
||||
Tests to add before refactoring:
|
||||
|
||||
- `internal/core/config`: unknown module key fails validation, if unsupported modules are not intended to be accepted.
|
||||
- `internal/core/config`: every output schema registry key validates through config.
|
||||
- `internal/core/diagnostics`: report metadata paths match run-directory artifact names.
|
||||
- `internal/validators`: validator class by key/instance is consistent for all registered validators.
|
||||
- `internal/framework/llm`: shared malformed structured-output classifier covers all current adapter malformed errors.
|
||||
|
||||
Tests to add during refactoring:
|
||||
|
||||
- `internal/framework/promptcontext`: transcript section prompt payload preserves IDs, speaker, timestamps, text, and categories.
|
||||
- `internal/framework/proposal_generation`: shared module proposal helper preserves current stage name, diagnostics dir, schema metadata, and malformed-output warning behavior.
|
||||
- `internal/cli`: `process` and `config print-effective` share defaults+file+env behavior.
|
||||
- `internal/cli`: `config validate` remains file-only and does not read env overrides.
|
||||
- `internal/cli`: correction ledger classifies deterministic and LLM validator decisions through canonical metadata.
|
||||
|
||||
Existing tests to run after each cleanup:
|
||||
|
||||
- `go test ./internal/core/config ./internal/core/outputschema`
|
||||
- `go test ./internal/core/diagnostics ./internal/core/reporting`
|
||||
- `go test ./internal/framework/proposal_generation ./internal/framework/validators ./internal/framework/runner`
|
||||
- `go test ./internal/validators/...`
|
||||
- `go test ./internal/modules/...`
|
||||
- `go test ./internal/cli ./cmd/audita`
|
||||
- Run `go test ./...` before merging a multi-package cleanup.
|
||||
|
||||
Validation note:
|
||||
|
||||
- During this report-only pass, no full test suite was run. A lightweight `go list ./...` completed package listing but emitted a sandbox warning while trying to write the Go module stat cache outside the repository.
|
||||
|
||||
## 13. Appendix: findings not worth acting on
|
||||
|
||||
### Separate module packages
|
||||
|
||||
The four production module packages contain visible repetition, but keeping separate packages is useful. The module domains, prompt assets, validator chains, and tests are distinct enough that a single generic module package would hide important behavior.
|
||||
|
||||
Do not refactor now beyond shared proposal/prompt plumbing.
|
||||
|
||||
### Report type duplication between runner and reporting
|
||||
|
||||
`runner.ModuleResult` and `reporting.ModuleReport` look similar. Keeping separate runtime and public report shapes is reasonable because runner owns execution state and reporting owns serialized public schema.
|
||||
|
||||
Only centralize mapping helpers; do not merge the types.
|
||||
|
||||
### Transcript and glossary parsing stay separate
|
||||
|
||||
Transcript JSON and glossary YAML parsing have different formats, validation rules, and error messages. There is no useful shared parser abstraction to extract.
|
||||
|
||||
### Response schema registry and output schema registry stay separate
|
||||
|
||||
Structured LLM response schemas and transcript output schemas are both "schemas", but they serve different users and have different lifecycles. Do not combine their registries.
|
||||
|
||||
### `flag` package usage
|
||||
|
||||
The CLI command surface is small. Replacing `flag` with a larger CLI framework would not pay for itself before 1.0.
|
||||
|
||||
### Local test duplication that protects public behavior
|
||||
|
||||
Some test duplication in CLI, subprocess, parity, and release fixtures is intentional. These tests exercise different public surfaces and should remain explicit even if helpers are shared.
|
||||
|
||||
### Filesystem state as diagnostics state
|
||||
|
||||
Audita has no resume/checkpoint semantics. Treating diagnostics artifacts as filesystem outputs is currently acceptable. A manifest system would be speculative before there is a resume or audit workflow that needs it.
|
||||
329
docs/roadmap/implementation.md
Normal file
329
docs/roadmap/implementation.md
Normal file
@@ -0,0 +1,329 @@
|
||||
# Pre-1.0 Deduplication Implementation Plan
|
||||
|
||||
This plan turns `docs/roadmap/audit.md` into staged, prompt-sized cleanup work for an LLM coding agent. Each stage should be implemented in order and kept small enough to review as an independent commit.
|
||||
|
||||
## Operating rules
|
||||
|
||||
- Read `docs/roadmap/audit.md` before starting any stage.
|
||||
- Preserve public CLI, report, diagnostics, config precedence, prompt metadata, and output-schema behavior unless a stage explicitly calls out an intended behavior change.
|
||||
- Keep the four production module packages separate: `glossary`, `homophones`, `spoken_word`, and `grammar`.
|
||||
- Do not introduce plugin systems, generic workflow engines, broad CLI framework rewrites, reflection-heavy config mappers, or merged module packages.
|
||||
- Prefer narrow helpers, catalogs, constants, and pure mapping functions over broad abstractions.
|
||||
- Run the targeted tests listed in each stage before moving to the next stage.
|
||||
- Run `go test ./...` before declaring the full sequence complete.
|
||||
- Ignore unrelated worktree changes, including the existing deletion of `docs/roadmap/publish.md`, unless the user explicitly asks to handle them.
|
||||
- Do not reduce parity, release-fixture, subprocess, or module-specific behavior coverage while consolidating helpers.
|
||||
|
||||
## Stages
|
||||
|
||||
### Stage 1: Diagnostics artifact constants and metadata paths
|
||||
|
||||
Goal:
|
||||
|
||||
- Centralize diagnostics artifact names and report diagnostics metadata path construction without changing any filenames or report fields.
|
||||
|
||||
Key edits:
|
||||
|
||||
- Define constants in `internal/core/diagnostics` for:
|
||||
- `source-transcript.json`
|
||||
- `source-transcript-parsed.json`
|
||||
- `normalized-transcript.json`
|
||||
- `normalization-summary.json`
|
||||
- `chunking-summary.json`
|
||||
- `utilization-diagnostics.json`
|
||||
- `correction-ledger.json`
|
||||
- `invocation.json`
|
||||
- `effective-config.json`
|
||||
- `report.json`
|
||||
- `error.log`
|
||||
- Add a diagnostics helper that builds `reporting.DiagnosticsMetadata` from a run directory path and failure/success status.
|
||||
- Update `RunDirectory` methods to use the constants.
|
||||
- Update CLI report assembly and utilization/correction-ledger writes to use the constants/helper instead of raw strings.
|
||||
|
||||
Behavior changes:
|
||||
|
||||
- None. All artifact names, report JSON keys, and path values must remain byte-for-byte compatible except for normal timestamp/order differences in existing outputs.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add or update `internal/core/diagnostics` tests proving metadata helper paths match the artifact constants.
|
||||
- Run `go test ./internal/core/diagnostics ./internal/core/reporting ./internal/cli`.
|
||||
- Run any existing CLI report/diagnostics tests touched by this stage.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- No raw core diagnostics artifact filename strings remain in CLI report metadata assembly.
|
||||
- Existing success and failure reports still point to files that are actually written.
|
||||
- Retention behavior is unchanged.
|
||||
|
||||
### Stage 2: Output schema validation and module catalog
|
||||
|
||||
Goal:
|
||||
|
||||
- Move public key validation to small canonical catalogs so config validation, runtime resolution, and factory behavior cannot drift.
|
||||
|
||||
Key edits:
|
||||
|
||||
- Add `SupportedKeys`, `IsSupported`, or an equivalent validation helper to `internal/core/outputschema`.
|
||||
- Update `config.Validate` to use `internal/core/outputschema` for output schema validation.
|
||||
- Add a small canonical module key catalog that is importable by:
|
||||
- `internal/core/config`
|
||||
- `internal/framework/modules`
|
||||
- `internal/validators`
|
||||
- `internal/framework/validators`
|
||||
- Use the module catalog for default module key constants, known-key checks, validator chain keys, and confidence-threshold lookup.
|
||||
- Keep module construction in `internal/framework/modules`; the catalog must not construct modules.
|
||||
|
||||
Behavior changes:
|
||||
|
||||
- Intended behavior change: unsupported configured module keys should fail during config validation, including `audita config validate`.
|
||||
- Repeated supported module keys remain valid.
|
||||
- Output schema behavior remains unchanged for `bare-segments`, `audita-v1`, and unsupported names.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add `internal/core/config` tests for unsupported module keys and repeated supported module keys.
|
||||
- Add config validation tests that every supported output schema validates.
|
||||
- Add or update output schema registry tests for supported and unsupported schemas.
|
||||
- Update module registry and validator chain tests to use the shared catalog where appropriate.
|
||||
- Run `go test ./internal/core/config ./internal/core/outputschema ./internal/framework/modules ./internal/framework/validators ./internal/validators/... ./internal/cli`.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- Unknown modules fail before runner setup in config validation paths.
|
||||
- No duplicated hardcoded output schema support list remains in config validation.
|
||||
- No import cycle is introduced.
|
||||
|
||||
### Stage 3: Effective config loading context
|
||||
|
||||
Goal:
|
||||
|
||||
- Centralize config path resolution and defaults+file+env loading while keeping command-specific CLI overrides explicit.
|
||||
|
||||
Key edits:
|
||||
|
||||
- Move config path resolution from `internal/cli` into `internal/core/config` or add an equivalent exported helper there.
|
||||
- Add an effective config loader that returns:
|
||||
- effective `config.Config`
|
||||
- config path
|
||||
- config source (`flag`, `env`, `default`, or empty)
|
||||
- config version pointer when a file was loaded
|
||||
- Use the shared loader in `audita process` before applying CLI overrides.
|
||||
- Use the shared loader in `audita config print-effective`.
|
||||
- Keep `audita config validate` as file-only: load file, apply to defaults, validate, and do not apply environment overrides.
|
||||
|
||||
Behavior changes:
|
||||
|
||||
- None. Preserve existing precedence:
|
||||
- `process`: defaults, file config, environment, CLI flags
|
||||
- `config print-effective`: defaults, file config, environment
|
||||
- `config validate`: file config applied to defaults only
|
||||
- Preserve explicit config path failure behavior and missing default path non-fatal behavior.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add table-driven config loader tests for:
|
||||
- explicit `--config`
|
||||
- `AUDITA_CONFIG`
|
||||
- default search paths
|
||||
- missing explicit path
|
||||
- missing env path
|
||||
- missing default paths
|
||||
- Add or update CLI tests proving `process` and `config print-effective` share file+env behavior.
|
||||
- Add or update CLI tests proving `config validate` ignores environment overrides.
|
||||
- Run `go test ./internal/core/config ./internal/cli ./cmd/audita`.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- Config precedence is unchanged.
|
||||
- Config source/path/version metadata in invocation and reports is unchanged.
|
||||
- Config command stdout/stderr and exit-code behavior is unchanged except for the intended unknown-module validation from Stage 2.
|
||||
|
||||
### Stage 4: Prompt/schema metadata and stage-name helpers
|
||||
|
||||
Goal:
|
||||
|
||||
- Centralize diagnostics-visible metadata and stage-name construction without changing production diagnostics names.
|
||||
|
||||
Key edits:
|
||||
|
||||
- Add a helper or method in `internal/prompts` that returns the stable prompt metadata diagnostics shape currently expanded by call sites.
|
||||
- Add a helper or method in `internal/framework/responseschema` that returns the stable response schema metadata diagnostics shape currently expanded by call sites.
|
||||
- Add shared proposal and validator stage-name helpers in the lowest package that avoids import cycles.
|
||||
- Use the helpers in proposal generation, LLM validators, and production modules.
|
||||
|
||||
Behavior changes:
|
||||
|
||||
- None. Preserve current production stage names:
|
||||
- module proposal stages keep their existing `proposal` naming form;
|
||||
- validator batch stages keep their existing validator/batch naming form.
|
||||
- Preserve all prompt metadata and response schema metadata field names and values.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add prompt metadata helper tests covering every registered prompt.
|
||||
- Add response schema metadata helper tests covering every registered response schema.
|
||||
- Add stage-name helper tests for no-section, section, and validator batch cases.
|
||||
- Run `go test ./internal/prompts ./internal/framework/responseschema ./internal/framework/proposal_generation ./internal/framework/validators ./internal/modules/...`.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- No manual prompt metadata map expansion remains in production module proposal plumbing.
|
||||
- No duplicated response schema metadata map construction remains in proposal generation and LLM validators.
|
||||
- Existing diagnostics fixture/path assertions still pass.
|
||||
|
||||
### Stage 5: Shared module proposal and prompt payload plumbing
|
||||
|
||||
Goal:
|
||||
|
||||
- Remove duplicated proposal execution and transcript-section prompt payload construction while preserving module-specific domain behavior.
|
||||
|
||||
Key edits:
|
||||
|
||||
- Add a narrow shared proposal execution helper, preferably in `internal/framework/proposal_generation` unless import cycles require a small module helper package.
|
||||
- The helper should own:
|
||||
- transcript description extraction from config;
|
||||
- `GenerateCandidates` request construction;
|
||||
- prompt metadata attachment;
|
||||
- stage-name selection;
|
||||
- conversion from generated corrections/warnings to `contracts.ProposalResult`.
|
||||
- Add shared transcript-section prompt payload construction in `internal/framework/promptcontext`.
|
||||
- Update each production module to provide only:
|
||||
- module key;
|
||||
- replacement policy;
|
||||
- validator chain;
|
||||
- prompt ID;
|
||||
- domain-specific `BuildProposalMessages` call or message builder.
|
||||
- Remove each module's redundant section transcript filtering if the runner already passes section-limited transcripts.
|
||||
|
||||
Behavior changes:
|
||||
|
||||
- None. Preserve module keys, replacement policies, validator chains, prompt IDs, diagnostics directories, proposal indexes, warning behavior, and correction mapping.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add promptcontext tests for transcript section payload shape, empty transcript handling, section index, and category copying.
|
||||
- Keep one module-specific prompt test per production module for domain wording and constraints.
|
||||
- Add or update module proposal tests proving diagnostics are still written under the same module instance directory.
|
||||
- Run `go test ./internal/framework/promptcontext ./internal/framework/proposal_generation ./internal/modules/... ./internal/cli`.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- Four production modules share proposal execution plumbing.
|
||||
- Module packages remain separate and readable.
|
||||
- CLI parity and release fixture behavior is unchanged.
|
||||
|
||||
### Stage 6: Validator classification and malformed LLM output policy
|
||||
|
||||
Goal:
|
||||
|
||||
- Use one source of truth for validator execution class and one shared classifier for malformed structured-output errors.
|
||||
|
||||
Key edits:
|
||||
|
||||
- Make validator execution class resolvable by stable validator key and by validator instance.
|
||||
- Replace the correction-ledger hardcoded LLM-backed validator map with the canonical metadata source.
|
||||
- Remove redundant validator metadata fields only after all call sites use the canonical source.
|
||||
- Add a shared malformed structured-output classifier in `internal/framework/llm` or another low-level framework package.
|
||||
- Update proposal generation and LLM validators to use the shared classifier while preserving their different handling outcomes.
|
||||
|
||||
Behavior changes:
|
||||
|
||||
- None. Proposal-generation malformed payloads still downgrade to warnings with zero proposals for affected sections.
|
||||
- Validator malformed payloads still reject affected batches with warnings.
|
||||
- Correction-ledger deterministic vs LLM validator sections should be unchanged for current validators.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add validator metadata tests proving every registered validator has the expected execution class by key and instance.
|
||||
- Add correction-ledger tests proving deterministic and LLM-backed decisions are classified through canonical metadata.
|
||||
- Add shared malformed-output classifier tests covering current adapter malformed-output messages.
|
||||
- Update proposal-generation and validator tests to assert representative malformed adapter errors are still downgraded.
|
||||
- Run `go test ./internal/validators/... ./internal/framework/validators ./internal/framework/proposal_generation ./internal/framework/llm ./internal/cli`.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- No local hardcoded LLM-backed validator map remains in correction-ledger construction.
|
||||
- Proposal-generation and validator malformed-output classifier lists cannot drift.
|
||||
- Existing runner validator ordering is unchanged.
|
||||
|
||||
### Stage 7: Redaction and adapter workflow cleanup
|
||||
|
||||
Goal:
|
||||
|
||||
- Reduce duplicated secret extraction/redaction setup while preserving all no-secret-leak guarantees.
|
||||
|
||||
Key edits:
|
||||
|
||||
- Add a shared helper that extracts all configured LLM secret values from `config.Config`.
|
||||
- Use the helper in proposal-generation diagnostics and validator diagnostics setup.
|
||||
- Keep config structural redaction (`Config.Redacted`) separate from byte/string payload redaction.
|
||||
- Keep adapter error redaction behavior compatible with current surfaced errors.
|
||||
- Move runner adapter shims only if Stage 6 or this stage makes them materially larger; otherwise leave them in runner.
|
||||
|
||||
Behavior changes:
|
||||
|
||||
- None. Redaction token and no-secret-leak behavior remain unchanged.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add or update tests proving proposal diagnostics, validator diagnostics, effective config artifacts, and surfaced adapter errors redact the same configured secrets.
|
||||
- Keep existing subprocess no-secret-leak tests.
|
||||
- Run `go test ./internal/core/config ./internal/framework/llm ./internal/framework/proposal_generation ./internal/framework/validators ./internal/cli ./cmd/audita`.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- Secret-list assembly is no longer duplicated between proposal and validator paths.
|
||||
- No plaintext configured API key appears in diagnostics, reports, stdout, or stderr in existing redaction tests.
|
||||
- No unrelated adapter behavior changes.
|
||||
|
||||
### Stage 8: Test helper cleanup and dead-code sweep
|
||||
|
||||
Goal:
|
||||
|
||||
- Consolidate test-only duplication and remove dead/redundant code left by prior stages.
|
||||
|
||||
Key edits:
|
||||
|
||||
- Consolidate package-local fake LLM clients, fixture readers, diagnostics glob helpers, and run-directory helpers where duplication is clear.
|
||||
- Use cross-package test support only if it does not obscure test intent or introduce awkward imports.
|
||||
- Remove redundant metadata fields, constants, or helper functions made obsolete by earlier stages.
|
||||
- Keep module-specific prompt and behavior assertions local to each module package.
|
||||
|
||||
Behavior changes:
|
||||
|
||||
- None.
|
||||
|
||||
Tests:
|
||||
|
||||
- Run all package tests touched by helper cleanup.
|
||||
- Run `go test ./internal/modules/... ./internal/framework/... ./internal/cli ./cmd/audita`.
|
||||
- Run `go test ./...` before completing the full sequence.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- Test helpers are simpler without reducing coverage.
|
||||
- No parity or release fixture assertions are removed unless replaced by equivalent or stronger assertions.
|
||||
- No production behavior changes.
|
||||
|
||||
## Final verification
|
||||
|
||||
Before declaring the staged cleanup complete:
|
||||
|
||||
- Run:
|
||||
- `go test ./internal/core/config ./internal/core/outputschema`
|
||||
- `go test ./internal/core/diagnostics ./internal/core/reporting`
|
||||
- `go test ./internal/framework/proposal_generation ./internal/framework/validators ./internal/framework/runner`
|
||||
- `go test ./internal/validators/...`
|
||||
- `go test ./internal/modules/...`
|
||||
- `go test ./internal/cli ./cmd/audita`
|
||||
- `go test ./...`
|
||||
- Inspect `git diff` for accidental public CLI, config, report, diagnostics, prompt metadata, stage-name, or output-schema changes.
|
||||
- Update docs only when behavior intentionally changes, especially the intended Stage 2 unknown-module validation change.
|
||||
- Keep commits stage-sized and mention behavior-preservation tests in each commit message or PR description.
|
||||
|
||||
## Assumptions
|
||||
|
||||
- Unknown configured module keys should become config-validation failures before 1.0.
|
||||
- Diagnostics filenames and stage names are public enough to preserve unless a stage explicitly says otherwise.
|
||||
- Each stage should be implemented and reviewed separately.
|
||||
11
go.mod
Normal file
11
go.mod
Normal file
@@ -0,0 +1,11 @@
|
||||
module gitea.maximumdirect.net/eric/audita
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require gopkg.in/yaml.v3 v3.0.1
|
||||
|
||||
require (
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
)
|
||||
17
go.sum
Normal file
17
go.sum
Normal file
@@ -0,0 +1,17 @@
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
338
internal/cli/parity_test.go
Normal file
338
internal/cli/parity_test.go
Normal file
@@ -0,0 +1,338 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
)
|
||||
|
||||
type parityFixtureCase struct {
|
||||
Name string `json:"name"`
|
||||
TranscriptFile string `json:"transcript_file"`
|
||||
GlossaryFile string `json:"glossary_file"`
|
||||
ModulesCSV string `json:"modules_csv,omitempty"`
|
||||
ProposalResponsesFile string `json:"proposal_responses_file,omitempty"`
|
||||
ValidationResponsesFile string `json:"validation_responses_file,omitempty"`
|
||||
Env map[string]string `json:"env,omitempty"`
|
||||
Expect parityExpectation `json:"expect"`
|
||||
}
|
||||
|
||||
type parityExpectation struct {
|
||||
ExitCode int `json:"exit_code"`
|
||||
Status string `json:"status,omitempty"`
|
||||
ErrorPhase string `json:"error_phase,omitempty"`
|
||||
StdoutMode string `json:"stdout_mode,omitempty"` // empty|json
|
||||
StderrContains string `json:"stderr_contains,omitempty"`
|
||||
OutputTranscriptFile string `json:"output_transcript_file,omitempty"`
|
||||
ModuleInstances []string `json:"module_instances,omitempty"`
|
||||
ModuleCount int `json:"module_count,omitempty"`
|
||||
TotalAppliedChanges int `json:"total_applied_changes,omitempty"`
|
||||
TotalSkippedChanges int `json:"total_skipped_changes,omitempty"`
|
||||
FailedModuleInstance string `json:"failed_module_instance,omitempty"`
|
||||
ValidatorRejectedReasonCodes []string `json:"validator_rejected_reason_codes,omitempty"`
|
||||
ApplicationSkipReasonCodes []string `json:"application_skip_reason_codes,omitempty"`
|
||||
RequireErrorLog bool `json:"require_error_log,omitempty"`
|
||||
SecretMarkers []string `json:"secret_markers,omitempty"`
|
||||
ExpectedProposalCalls []string `json:"expected_proposal_calls,omitempty"`
|
||||
ExpectedValidationCalls []string `json:"expected_validation_calls,omitempty"`
|
||||
ModuleAppliedCounts []int `json:"module_applied_counts,omitempty"`
|
||||
ModuleRejectedCounts []int `json:"module_rejected_counts,omitempty"`
|
||||
ModuleSkipCounts []int `json:"module_skip_counts,omitempty"`
|
||||
MinResponsePayloadArtifacts int `json:"min_response_payload_artifacts,omitempty"`
|
||||
}
|
||||
|
||||
func TestParityFixtures(t *testing.T) {
|
||||
casePaths, err := filepath.Glob(parityFixturePath("*.case.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("glob parity fixtures: %v", err)
|
||||
}
|
||||
if len(casePaths) == 0 {
|
||||
t.Fatal("expected at least one parity fixture case")
|
||||
}
|
||||
|
||||
for _, casePath := range casePaths {
|
||||
fx := loadParityFixtureCase(t, casePath)
|
||||
t.Run(fx.Name, func(t *testing.T) {
|
||||
runParityFixtureCase(t, filepath.Dir(casePath), fx)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func loadParityFixtureCase(t *testing.T, casePath string) parityFixtureCase {
|
||||
t.Helper()
|
||||
var fx parityFixtureCase
|
||||
raw := readFile(t, casePath)
|
||||
if err := json.Unmarshal(raw, &fx); err != nil {
|
||||
t.Fatalf("parse parity case %q: %v", casePath, err)
|
||||
}
|
||||
if strings.TrimSpace(fx.Name) == "" {
|
||||
t.Fatalf("parity case %q missing name", casePath)
|
||||
}
|
||||
return fx
|
||||
}
|
||||
|
||||
func runParityFixtureCase(t *testing.T, caseDir string, fx parityFixtureCase) {
|
||||
t.Helper()
|
||||
for k, v := range fx.Env {
|
||||
t.Setenv(k, v)
|
||||
}
|
||||
|
||||
proposalClient := &fakeStructuredLLMClient{}
|
||||
validationClient := &fakeStructuredLLMClient{}
|
||||
|
||||
if strings.TrimSpace(fx.ProposalResponsesFile) != "" {
|
||||
raw := readFile(t, filepath.Join(caseDir, fx.ProposalResponsesFile))
|
||||
if err := json.Unmarshal(raw, &proposalClient.proposalResponses); err != nil {
|
||||
t.Fatalf("parse proposal responses: %v", err)
|
||||
}
|
||||
processProposalLLMClient = proposalClient
|
||||
}
|
||||
if strings.TrimSpace(fx.ValidationResponsesFile) != "" {
|
||||
raw := readFile(t, filepath.Join(caseDir, fx.ValidationResponsesFile))
|
||||
if err := json.Unmarshal(raw, &validationClient.validationResponses); err != nil {
|
||||
t.Fatalf("parse validation responses: %v", err)
|
||||
}
|
||||
processValidationLLMClient = validationClient
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
processProposalLLMClient = nil
|
||||
processValidationLLMClient = nil
|
||||
})
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
workDir := t.TempDir()
|
||||
outputPath := filepath.Join(t.TempDir(), "out.json")
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
args := []string{
|
||||
"process",
|
||||
filepath.Join(caseDir, fx.TranscriptFile),
|
||||
"--glossary",
|
||||
filepath.Join(caseDir, fx.GlossaryFile),
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
}
|
||||
// Keep stdout shape deterministic for parity tests.
|
||||
if fx.Expect.StdoutMode != "json" {
|
||||
args = append(args, "--output", outputPath)
|
||||
}
|
||||
if strings.TrimSpace(fx.ModulesCSV) != "" {
|
||||
args = append(args, "--modules", fx.ModulesCSV)
|
||||
}
|
||||
|
||||
exitCode := Run(args, &stdout, &stderr)
|
||||
if exitCode != fx.Expect.ExitCode {
|
||||
t.Fatalf("expected exit code %d, got %d stderr=%q", fx.Expect.ExitCode, exitCode, stderr.String())
|
||||
}
|
||||
|
||||
switch fx.Expect.StdoutMode {
|
||||
case "json":
|
||||
if _, err := json.Marshal(stdout.String()); err != nil {
|
||||
t.Fatalf("unexpected stdout marshal error: %v", err)
|
||||
}
|
||||
if !json.Valid(stdout.Bytes()) {
|
||||
t.Fatalf("expected JSON stdout, got %q", stdout.String())
|
||||
}
|
||||
default:
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("expected empty stdout, got %q", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
if fx.Expect.StderrContains != "" && !strings.Contains(stderr.String(), fx.Expect.StderrContains) {
|
||||
t.Fatalf("expected stderr to contain %q, got %q", fx.Expect.StderrContains, stderr.String())
|
||||
}
|
||||
|
||||
report := readProcessReport(t, reportPath)
|
||||
assertParityReport(t, report, fx.Expect)
|
||||
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
runDirReport := readProcessReport(t, filepath.Join(runDir, "report.json"))
|
||||
assertParityReport(t, runDirReport, fx.Expect)
|
||||
|
||||
if fx.Expect.RequireErrorLog {
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if strings.TrimSpace(fx.Expect.OutputTranscriptFile) != "" && fx.Expect.ExitCode == 0 {
|
||||
got := readFile(t, outputPath)
|
||||
want := readFile(t, filepath.Join(caseDir, fx.Expect.OutputTranscriptFile))
|
||||
assertJSONSemanticEqual(t, want, got)
|
||||
}
|
||||
|
||||
if len(fx.Expect.ExpectedProposalCalls) > 0 && !reflect.DeepEqual(proposalClient.calls, fx.Expect.ExpectedProposalCalls) {
|
||||
t.Fatalf("unexpected proposal calls: got %v want %v", proposalClient.calls, fx.Expect.ExpectedProposalCalls)
|
||||
}
|
||||
if len(fx.Expect.ExpectedValidationCalls) > 0 && !reflect.DeepEqual(validationClient.calls, fx.Expect.ExpectedValidationCalls) {
|
||||
t.Fatalf("unexpected validation calls: got %v want %v", validationClient.calls, fx.Expect.ExpectedValidationCalls)
|
||||
}
|
||||
|
||||
if len(fx.Expect.SecretMarkers) > 0 {
|
||||
assertNoSecretMarkers(t, reportPath, fx.Expect.SecretMarkers)
|
||||
assertNoSecretMarkersInTree(t, runDir, fx.Expect.SecretMarkers)
|
||||
}
|
||||
if fx.Expect.MinResponsePayloadArtifacts > 0 {
|
||||
matches, err := filepath.Glob(filepath.Join(runDir, "*", "*response-payload.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("glob response payload artifacts: %v", err)
|
||||
}
|
||||
if len(matches) < fx.Expect.MinResponsePayloadArtifacts {
|
||||
t.Fatalf("expected at least %d response payload artifacts, got %d", fx.Expect.MinResponsePayloadArtifacts, len(matches))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertParityReport(t *testing.T, report reporting.ProcessReport, exp parityExpectation) {
|
||||
t.Helper()
|
||||
if exp.Status != "" && report.Status != exp.Status {
|
||||
t.Fatalf("expected report status %q, got %q", exp.Status, report.Status)
|
||||
}
|
||||
if exp.ErrorPhase != "" && report.ErrorPhase != exp.ErrorPhase {
|
||||
t.Fatalf("expected report error_phase %q, got %q", exp.ErrorPhase, report.ErrorPhase)
|
||||
}
|
||||
if len(exp.ModuleInstances) > 0 {
|
||||
got := make([]string, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, mr.ModuleInstance)
|
||||
}
|
||||
if !reflect.DeepEqual(got, exp.ModuleInstances) {
|
||||
t.Fatalf("unexpected module instances: got %v want %v", got, exp.ModuleInstances)
|
||||
}
|
||||
}
|
||||
if exp.ModuleCount > 0 {
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.ModuleCount != exp.ModuleCount {
|
||||
t.Fatalf("expected module_count=%d, got %+v", exp.ModuleCount, report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
if exp.TotalAppliedChanges > 0 {
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.TotalAppliedChanges != exp.TotalAppliedChanges {
|
||||
t.Fatalf("expected total_applied_changes=%d, got %+v", exp.TotalAppliedChanges, report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
if exp.TotalSkippedChanges > 0 {
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.TotalSkippedChanges != exp.TotalSkippedChanges {
|
||||
t.Fatalf("expected total_skipped_changes=%d, got %+v", exp.TotalSkippedChanges, report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
if exp.FailedModuleInstance != "" {
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != exp.FailedModuleInstance {
|
||||
t.Fatalf("expected failed_module_instance=%q, got %+v", exp.FailedModuleInstance, report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
|
||||
if len(exp.ValidatorRejectedReasonCodes) > 0 {
|
||||
got := collectValidatorRejectedReasonCodes(report.ModuleResults)
|
||||
if !reflect.DeepEqual(got, exp.ValidatorRejectedReasonCodes) {
|
||||
t.Fatalf("unexpected validator rejected reason codes: got %v want %v", got, exp.ValidatorRejectedReasonCodes)
|
||||
}
|
||||
}
|
||||
if len(exp.ApplicationSkipReasonCodes) > 0 {
|
||||
got := collectApplicationSkipReasonCodes(report.ModuleResults)
|
||||
if !reflect.DeepEqual(got, exp.ApplicationSkipReasonCodes) {
|
||||
t.Fatalf("unexpected application skip reason codes: got %v want %v", got, exp.ApplicationSkipReasonCodes)
|
||||
}
|
||||
}
|
||||
if len(exp.ModuleAppliedCounts) > 0 {
|
||||
got := make([]int, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, len(mr.AppliedChanges))
|
||||
}
|
||||
if !reflect.DeepEqual(got, exp.ModuleAppliedCounts) {
|
||||
t.Fatalf("unexpected per-module applied counts: got %v want %v", got, exp.ModuleAppliedCounts)
|
||||
}
|
||||
}
|
||||
if len(exp.ModuleRejectedCounts) > 0 {
|
||||
got := make([]int, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, len(mr.ValidatorRejected))
|
||||
}
|
||||
if !reflect.DeepEqual(got, exp.ModuleRejectedCounts) {
|
||||
t.Fatalf("unexpected per-module rejected counts: got %v want %v", got, exp.ModuleRejectedCounts)
|
||||
}
|
||||
}
|
||||
if len(exp.ModuleSkipCounts) > 0 {
|
||||
got := make([]int, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, len(mr.SkippedChanges))
|
||||
}
|
||||
if !reflect.DeepEqual(got, exp.ModuleSkipCounts) {
|
||||
t.Fatalf("unexpected per-module skip counts: got %v want %v", got, exp.ModuleSkipCounts)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func collectValidatorRejectedReasonCodes(results []reporting.ModuleReport) []string {
|
||||
out := make([]string, 0)
|
||||
for _, mr := range results {
|
||||
for _, vr := range mr.ValidatorRejected {
|
||||
out = append(out, vr.ReasonCode)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func collectApplicationSkipReasonCodes(results []reporting.ModuleReport) []string {
|
||||
out := make([]string, 0)
|
||||
for _, mr := range results {
|
||||
for _, sk := range mr.SkippedChanges {
|
||||
out = append(out, string(sk.SkipReason))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func assertJSONSemanticEqual(t *testing.T, expected []byte, actual []byte) {
|
||||
t.Helper()
|
||||
var exp any
|
||||
var act any
|
||||
if err := json.Unmarshal(expected, &exp); err != nil {
|
||||
t.Fatalf("unmarshal expected json: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(actual, &act); err != nil {
|
||||
t.Fatalf("unmarshal actual json: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(exp, act) {
|
||||
t.Fatalf("JSON mismatch\nexpected=%s\nactual=%s", string(expected), string(actual))
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoSecretMarkers(t *testing.T, filePath string, markers []string) {
|
||||
t.Helper()
|
||||
raw := string(readFile(t, filePath))
|
||||
for _, marker := range markers {
|
||||
if marker != "" && strings.Contains(raw, marker) {
|
||||
t.Fatalf("secret marker %q leaked in %s", marker, filePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoSecretMarkersInTree(t *testing.T, root string, markers []string) {
|
||||
t.Helper()
|
||||
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil || d == nil || d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
raw := string(readFile(t, path))
|
||||
for _, marker := range markers {
|
||||
if marker != "" && strings.Contains(raw, marker) {
|
||||
t.Fatalf("secret marker %q leaked in %s", marker, path)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func parityFixturePath(name string) string {
|
||||
return filepath.Join("testdata", "parity", name)
|
||||
}
|
||||
121
internal/cli/process_flags.go
Normal file
121
internal/cli/process_flags.go
Normal file
@@ -0,0 +1,121 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"flag"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
)
|
||||
|
||||
type processOverrideBinding func(*config.CLIOverrides, processFlags)
|
||||
|
||||
var processOverrideBindings = map[string]processOverrideBinding{
|
||||
"modules": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ModulesCSV = flags.modules
|
||||
},
|
||||
"output-schema": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.OutputSchema = flags.outputSchema
|
||||
},
|
||||
"llm-api-key": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.PrimaryLLMAPIKey = flags.llmAPIKey
|
||||
},
|
||||
"validation-llm-api-key": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationLLMAPIKey = flags.validationLLMAPIKey
|
||||
},
|
||||
"model": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.PrimaryModel = flags.model
|
||||
},
|
||||
"validation-model": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationModel = flags.validationModel
|
||||
},
|
||||
"base-url": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.PrimaryBaseURL = flags.baseURL
|
||||
},
|
||||
"validation-base-url": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationBaseURL = flags.validationBaseURL
|
||||
},
|
||||
"llm-timeout-seconds": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.PrimaryLLMTimeoutSeconds = flags.llmTimeoutSeconds
|
||||
},
|
||||
"total-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.TotalLLMConcurrency = flags.totalLLMConcurrency
|
||||
},
|
||||
"proposal-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ProposalLLMConcurrency = flags.proposalLLMConcurrency
|
||||
},
|
||||
"llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.PrimaryLLMConcurrency = flags.llmConcurrency
|
||||
},
|
||||
"validation-llm-timeout-seconds": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationLLMTimeoutSeconds = flags.validationLLMTimeoutSeconds
|
||||
},
|
||||
"max-retries": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.MaxRetries = flags.maxRetries
|
||||
},
|
||||
"validation-max-retries": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationMaxRetries = flags.validationMaxRetries
|
||||
},
|
||||
"validation-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationLLMConcurrency = flags.validationLLMConcurrency
|
||||
},
|
||||
"validation-max-prompt-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationMaxPromptTokens = flags.validationMaxPromptTokens
|
||||
},
|
||||
"max-section-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.MaxSectionTokens = flags.maxSectionTokens
|
||||
},
|
||||
"min-section-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.MinSectionTokens = flags.minSectionTokens
|
||||
},
|
||||
"target-sections": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.TargetSections = flags.targetSections
|
||||
},
|
||||
"glossary-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.GlossaryConfidenceThreshold = flags.glossaryConfidenceThreshold
|
||||
},
|
||||
"grammar-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.GrammarConfidenceThreshold = flags.grammarConfidenceThreshold
|
||||
},
|
||||
"homophones-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.HomophonesConfidenceThreshold = flags.homophonesConfidenceThreshold
|
||||
},
|
||||
"spoken-word-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.SpokenWordConfidenceThreshold = flags.spokenWordConfidenceThreshold
|
||||
},
|
||||
"normalize-max-segment-gap": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.NormalizeMaxSegmentGap = flags.normalizeMaxSegmentGap
|
||||
},
|
||||
"normalize-ellipsis-gap": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.NormalizeEllipsisGap = flags.normalizeEllipsisGap
|
||||
},
|
||||
"normalize-max-segment-duration": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.NormalizeMaxSegmentDuration = flags.normalizeMaxSegmentDuration
|
||||
},
|
||||
"normalize-max-segment-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.NormalizeMaxSegmentTokens = flags.normalizeMaxSegmentTokens
|
||||
},
|
||||
"transcript-description": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.TranscriptDescription = flags.transcriptDescription
|
||||
},
|
||||
"work-dir": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.WorkDir = flags.workDir
|
||||
},
|
||||
"work-dir-retention": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.WorkDirRetention = flags.workDirRetention
|
||||
},
|
||||
}
|
||||
|
||||
func processCLIOverrides(fs *flag.FlagSet, flags processFlags) (config.CLIOverrides, bool) {
|
||||
overrides := config.CLIOverrides{}
|
||||
explicitModules := false
|
||||
fs.Visit(func(f *flag.Flag) {
|
||||
if f.Name == "modules" {
|
||||
explicitModules = true
|
||||
}
|
||||
binding, ok := processOverrideBindings[f.Name]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
binding(&overrides, flags)
|
||||
})
|
||||
return overrides, explicitModules
|
||||
}
|
||||
433
internal/cli/process_flags_test.go
Normal file
433
internal/cli/process_flags_test.go
Normal file
@@ -0,0 +1,433 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"io"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
)
|
||||
|
||||
func TestProcessCLIOverridesMapsEveryConfigMutatingFlag(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
flagName string
|
||||
value string
|
||||
wantExplicitModules bool
|
||||
assertOverrideFields func(t *testing.T, overrides config.CLIOverrides)
|
||||
}{
|
||||
{
|
||||
name: "modules",
|
||||
flagName: "modules",
|
||||
value: "grammar,glossary",
|
||||
wantExplicitModules: true,
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "ModulesCSV", overrides.ModulesCSV, "grammar,glossary")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "output schema",
|
||||
flagName: "output-schema",
|
||||
value: "audita-v1",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "OutputSchema", overrides.OutputSchema, "audita-v1")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "primary api key",
|
||||
flagName: "llm-api-key",
|
||||
value: "primary-key",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "PrimaryLLMAPIKey", overrides.PrimaryLLMAPIKey, "primary-key")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation api key",
|
||||
flagName: "validation-llm-api-key",
|
||||
value: "validation-key",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "ValidationLLMAPIKey", overrides.ValidationLLMAPIKey, "validation-key")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "primary model",
|
||||
flagName: "model",
|
||||
value: "primary-model",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "PrimaryModel", overrides.PrimaryModel, "primary-model")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation model",
|
||||
flagName: "validation-model",
|
||||
value: "validation-model",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "ValidationModel", overrides.ValidationModel, "validation-model")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "primary base url",
|
||||
flagName: "base-url",
|
||||
value: "https://primary.example.test",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "PrimaryBaseURL", overrides.PrimaryBaseURL, "https://primary.example.test")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation base url",
|
||||
flagName: "validation-base-url",
|
||||
value: "https://validation.example.test",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "ValidationBaseURL", overrides.ValidationBaseURL, "https://validation.example.test")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "primary timeout",
|
||||
flagName: "llm-timeout-seconds",
|
||||
value: "101",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "PrimaryLLMTimeoutSeconds", overrides.PrimaryLLMTimeoutSeconds, 101)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "total concurrency",
|
||||
flagName: "total-llm-concurrency",
|
||||
value: "5",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "TotalLLMConcurrency", overrides.TotalLLMConcurrency, 5)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "proposal concurrency",
|
||||
flagName: "proposal-llm-concurrency",
|
||||
value: "3",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "ProposalLLMConcurrency", overrides.ProposalLLMConcurrency, 3)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "legacy concurrency alias",
|
||||
flagName: "llm-concurrency",
|
||||
value: "4",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "PrimaryLLMConcurrency", overrides.PrimaryLLMConcurrency, 4)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation timeout",
|
||||
flagName: "validation-llm-timeout-seconds",
|
||||
value: "202",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "ValidationLLMTimeoutSeconds", overrides.ValidationLLMTimeoutSeconds, 202)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "max retries",
|
||||
flagName: "max-retries",
|
||||
value: "6",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "MaxRetries", overrides.MaxRetries, 6)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation max retries",
|
||||
flagName: "validation-max-retries",
|
||||
value: "7",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "ValidationMaxRetries", overrides.ValidationMaxRetries, 7)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation concurrency",
|
||||
flagName: "validation-llm-concurrency",
|
||||
value: "8",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "ValidationLLMConcurrency", overrides.ValidationLLMConcurrency, 8)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation max prompt tokens",
|
||||
flagName: "validation-max-prompt-tokens",
|
||||
value: "4096",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "ValidationMaxPromptTokens", overrides.ValidationMaxPromptTokens, 4096)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "max section tokens",
|
||||
flagName: "max-section-tokens",
|
||||
value: "9000",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "MaxSectionTokens", overrides.MaxSectionTokens, 9000)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "min section tokens",
|
||||
flagName: "min-section-tokens",
|
||||
value: "1000",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "MinSectionTokens", overrides.MinSectionTokens, 1000)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "target sections",
|
||||
flagName: "target-sections",
|
||||
value: "12",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "TargetSections", overrides.TargetSections, 12)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "glossary threshold",
|
||||
flagName: "glossary-confidence-threshold",
|
||||
value: "0.91",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "GlossaryConfidenceThreshold", overrides.GlossaryConfidenceThreshold, 0.91)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "grammar threshold",
|
||||
flagName: "grammar-confidence-threshold",
|
||||
value: "0.92",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "GrammarConfidenceThreshold", overrides.GrammarConfidenceThreshold, 0.92)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "homophones threshold",
|
||||
flagName: "homophones-confidence-threshold",
|
||||
value: "0.93",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "HomophonesConfidenceThreshold", overrides.HomophonesConfidenceThreshold, 0.93)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "spoken word threshold",
|
||||
flagName: "spoken-word-confidence-threshold",
|
||||
value: "0.94",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "SpokenWordConfidenceThreshold", overrides.SpokenWordConfidenceThreshold, 0.94)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "normalize max segment gap",
|
||||
flagName: "normalize-max-segment-gap",
|
||||
value: "1.2",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "NormalizeMaxSegmentGap", overrides.NormalizeMaxSegmentGap, 1.2)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "normalize ellipsis gap",
|
||||
flagName: "normalize-ellipsis-gap",
|
||||
value: "2.3",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "NormalizeEllipsisGap", overrides.NormalizeEllipsisGap, 2.3)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "normalize max segment duration",
|
||||
flagName: "normalize-max-segment-duration",
|
||||
value: "45.6",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "NormalizeMaxSegmentDuration", overrides.NormalizeMaxSegmentDuration, 45.6)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "normalize max segment tokens",
|
||||
flagName: "normalize-max-segment-tokens",
|
||||
value: "321",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "NormalizeMaxSegmentTokens", overrides.NormalizeMaxSegmentTokens, 321)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "transcript description",
|
||||
flagName: "transcript-description",
|
||||
value: "podcast episode",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "TranscriptDescription", overrides.TranscriptDescription, "podcast episode")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "work dir",
|
||||
flagName: "work-dir",
|
||||
value: "/tmp/custom-audita",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "WorkDir", overrides.WorkDir, "/tmp/custom-audita")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "work dir retention",
|
||||
flagName: "work-dir-retention",
|
||||
value: "always",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "WorkDirRetention", overrides.WorkDirRetention, "always")
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fs, flags := newProcessFlagSet(config.Default(), io.Discard)
|
||||
if err := fs.Parse([]string{"--" + tc.flagName, tc.value}); err != nil {
|
||||
t.Fatalf("parse flag: %v", err)
|
||||
}
|
||||
|
||||
overrides, explicitModules := processCLIOverrides(fs, flags)
|
||||
if explicitModules != tc.wantExplicitModules {
|
||||
t.Fatalf("explicitModules=%v, want %v", explicitModules, tc.wantExplicitModules)
|
||||
}
|
||||
tc.assertOverrideFields(t, overrides)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessCLIOverridesIgnoresNonConfigFlags(t *testing.T) {
|
||||
fs, flags := newProcessFlagSet(config.Default(), io.Discard)
|
||||
if err := fs.Parse([]string{
|
||||
"--config", "/tmp/config.yml",
|
||||
"--glossary", "/tmp/glossary.yml",
|
||||
"--output", "/tmp/output.json",
|
||||
"--report-json", "/tmp/report.json",
|
||||
}); err != nil {
|
||||
t.Fatalf("parse flags: %v", err)
|
||||
}
|
||||
|
||||
overrides, explicitModules := processCLIOverrides(fs, flags)
|
||||
if explicitModules {
|
||||
t.Fatal("non-config flags should not mark modules explicit")
|
||||
}
|
||||
assertNoCLIOverrides(t, overrides)
|
||||
}
|
||||
|
||||
func TestNewProcessFlagSetDefaultsReflectEffectiveConfig(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
cfg.Modules = []string{"grammar", "glossary"}
|
||||
cfg.OutputSchema = "audita-v1"
|
||||
cfg.PrimaryLLM.APIKey = "primary-key"
|
||||
cfg.ValidationLLM.APIKey = "validation-key"
|
||||
cfg.PrimaryLLM.Model = "primary-model"
|
||||
cfg.ValidationLLM.Model = "validation-model"
|
||||
cfg.PrimaryLLM.BaseURL = "https://primary.example.test"
|
||||
cfg.ValidationLLM.BaseURL = "https://validation.example.test"
|
||||
cfg.PrimaryLLM.TimeoutSeconds = 101
|
||||
cfg.TotalLLMConcurrency = 5
|
||||
cfg.ProposalLLMConcurrency = 3
|
||||
cfg.PrimaryLLM.MaxRetries = 6
|
||||
cfg.ValidationMaxPromptTokens = 4096
|
||||
cfg.MaxSectionTokens = 9000
|
||||
cfg.MinSectionTokens = 1000
|
||||
cfg.Thresholds.Glossary = 0.91
|
||||
cfg.Thresholds.Grammar = 0.92
|
||||
cfg.Thresholds.Homophones = 0.93
|
||||
cfg.Thresholds.SpokenWord = 0.94
|
||||
cfg.Normalization.MaxSegmentGap = 1.2
|
||||
cfg.Normalization.EllipsisGap = 2.3
|
||||
cfg.Normalization.MaxSegmentDuration = 45.6
|
||||
cfg.Normalization.MaxSegmentTokens = 321
|
||||
cfg.TranscriptDescription = "podcast episode"
|
||||
cfg.WorkDir = "/tmp/custom-audita"
|
||||
cfg.WorkDirRetention = config.WorkDirRetentionAlways
|
||||
|
||||
validationTimeout := 202
|
||||
validationRetries := 7
|
||||
validationConcurrency := 8
|
||||
targetSections := 12
|
||||
cfg.ValidationLLM.TimeoutSeconds = &validationTimeout
|
||||
cfg.ValidationLLM.MaxRetries = &validationRetries
|
||||
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||
cfg.TargetSections = &targetSections
|
||||
|
||||
_, flags := newProcessFlagSet(cfg, io.Discard)
|
||||
|
||||
assertStringOverride(t, "modules default", flags.modules, "grammar,glossary")
|
||||
assertStringOverride(t, "output schema default", flags.outputSchema, "audita-v1")
|
||||
assertStringOverride(t, "primary api key default", flags.llmAPIKey, "primary-key")
|
||||
assertStringOverride(t, "validation api key default", flags.validationLLMAPIKey, "validation-key")
|
||||
assertStringOverride(t, "primary model default", flags.model, "primary-model")
|
||||
assertStringOverride(t, "validation model default", flags.validationModel, "validation-model")
|
||||
assertStringOverride(t, "primary base url default", flags.baseURL, "https://primary.example.test")
|
||||
assertStringOverride(t, "validation base url default", flags.validationBaseURL, "https://validation.example.test")
|
||||
assertIntOverride(t, "primary timeout default", flags.llmTimeoutSeconds, 101)
|
||||
assertIntOverride(t, "total concurrency default", flags.totalLLMConcurrency, 5)
|
||||
assertIntOverride(t, "proposal concurrency default", flags.proposalLLMConcurrency, 3)
|
||||
assertIntOverride(t, "legacy concurrency alias default", flags.llmConcurrency, 5)
|
||||
assertIntOverride(t, "validation timeout default", flags.validationLLMTimeoutSeconds, validationTimeout)
|
||||
assertIntOverride(t, "max retries default", flags.maxRetries, 6)
|
||||
assertIntOverride(t, "validation max retries default", flags.validationMaxRetries, validationRetries)
|
||||
assertIntOverride(t, "validation concurrency default", flags.validationLLMConcurrency, validationConcurrency)
|
||||
assertIntOverride(t, "validation max prompt tokens default", flags.validationMaxPromptTokens, 4096)
|
||||
assertIntOverride(t, "max section tokens default", flags.maxSectionTokens, 9000)
|
||||
assertIntOverride(t, "min section tokens default", flags.minSectionTokens, 1000)
|
||||
assertIntOverride(t, "target sections default", flags.targetSections, targetSections)
|
||||
assertFloatOverride(t, "glossary threshold default", flags.glossaryConfidenceThreshold, 0.91)
|
||||
assertFloatOverride(t, "grammar threshold default", flags.grammarConfidenceThreshold, 0.92)
|
||||
assertFloatOverride(t, "homophones threshold default", flags.homophonesConfidenceThreshold, 0.93)
|
||||
assertFloatOverride(t, "spoken word threshold default", flags.spokenWordConfidenceThreshold, 0.94)
|
||||
assertFloatOverride(t, "normalize max segment gap default", flags.normalizeMaxSegmentGap, 1.2)
|
||||
assertFloatOverride(t, "normalize ellipsis gap default", flags.normalizeEllipsisGap, 2.3)
|
||||
assertFloatOverride(t, "normalize max segment duration default", flags.normalizeMaxSegmentDuration, 45.6)
|
||||
assertIntOverride(t, "normalize max segment tokens default", flags.normalizeMaxSegmentTokens, 321)
|
||||
assertStringOverride(t, "transcript description default", flags.transcriptDescription, "podcast episode")
|
||||
assertStringOverride(t, "work dir default", flags.workDir, "/tmp/custom-audita")
|
||||
assertStringOverride(t, "work dir retention default", flags.workDirRetention, "always")
|
||||
}
|
||||
|
||||
func TestNewProcessFlagSetUsesFallbackDefaultsForUnsetOptionalConfig(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
|
||||
_, flags := newProcessFlagSet(cfg, io.Discard)
|
||||
|
||||
assertIntOverride(t, "validation timeout fallback", flags.validationLLMTimeoutSeconds, cfg.PrimaryLLM.TimeoutSeconds)
|
||||
assertIntOverride(t, "validation retries fallback", flags.validationMaxRetries, cfg.PrimaryLLM.MaxRetries)
|
||||
assertIntOverride(t, "validation concurrency fallback", flags.validationLLMConcurrency, cfg.TotalLLMConcurrency)
|
||||
assertIntOverride(t, "target sections fallback", flags.targetSections, 0)
|
||||
}
|
||||
|
||||
func assertStringOverride(t *testing.T, name string, got *string, want string) {
|
||||
t.Helper()
|
||||
if got == nil || *got != want {
|
||||
t.Fatalf("%s=%v, want %q", name, pointerValue(got), want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertIntOverride(t *testing.T, name string, got *int, want int) {
|
||||
t.Helper()
|
||||
if got == nil || *got != want {
|
||||
t.Fatalf("%s=%v, want %d", name, pointerValue(got), want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertFloatOverride(t *testing.T, name string, got *float64, want float64) {
|
||||
t.Helper()
|
||||
if got == nil || *got != want {
|
||||
t.Fatalf("%s=%v, want %v", name, pointerValue(got), want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoCLIOverrides(t *testing.T, overrides config.CLIOverrides) {
|
||||
t.Helper()
|
||||
value := reflect.ValueOf(overrides)
|
||||
typ := value.Type()
|
||||
for i := 0; i < value.NumField(); i++ {
|
||||
field := value.Field(i)
|
||||
if field.Kind() != reflect.Ptr {
|
||||
t.Fatalf("unexpected non-pointer CLIOverrides field %s", typ.Field(i).Name)
|
||||
}
|
||||
if !field.IsNil() {
|
||||
t.Fatalf("expected no CLI overrides, field %s was set", typ.Field(i).Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func pointerValue[T any](ptr *T) any {
|
||||
if ptr == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
if stringer, ok := any(*ptr).(interface{ String() string }); ok {
|
||||
return strings.TrimSpace(stringer.String())
|
||||
}
|
||||
return *ptr
|
||||
}
|
||||
469
internal/cli/release_fixtures_test.go
Normal file
469
internal/cli/release_fixtures_test.go
Normal file
@@ -0,0 +1,469 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
)
|
||||
|
||||
type releaseFixtureExpectations struct {
|
||||
MustApplyTexts []string `json:"must_apply_texts"`
|
||||
MustNotApplyTexts []string `json:"must_not_apply_texts"`
|
||||
ProtectedTerms []string `json:"protected_terms"`
|
||||
ExpectedModuleInstance []string `json:"expected_module_instances"`
|
||||
MinimumCounts struct {
|
||||
Applied int `json:"applied"`
|
||||
Rejected int `json:"rejected"`
|
||||
Skipped int `json:"skipped"`
|
||||
} `json:"minimum_counts"`
|
||||
}
|
||||
|
||||
func TestReleaseFixtureDefaultPipelineReadiness(t *testing.T) {
|
||||
base := fixturePath(filepath.Join("release", "default-release"))
|
||||
|
||||
var expectations releaseFixtureExpectations
|
||||
if err := json.Unmarshal(readFile(t, base+".expectations.json"), &expectations); err != nil {
|
||||
t.Fatalf("unmarshal release expectations: %v", err)
|
||||
}
|
||||
|
||||
proposalResponses := readProposalResponses(t, base+".proposals.json")
|
||||
validationResponses := readValidationResponses(t, base+".validations.json")
|
||||
|
||||
// First pass: default full pipeline with deterministic fake LLM responses.
|
||||
first := runReleaseFixturePass(t, releaseRunConfig{
|
||||
transcriptPath: base + ".transcript.json",
|
||||
glossaryPath: base + ".glossary.yaml",
|
||||
outputSchema: "bare-segments",
|
||||
proposalResponses: proposalResponses,
|
||||
validationResponses: validationResponses,
|
||||
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
|
||||
reportSchemaName: reporting.DefaultProcessReportSchemaName,
|
||||
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
||||
expectedOutputSchema: "bare-segments",
|
||||
expectModuleInstances: expectations.ExpectedModuleInstance,
|
||||
expectUtilizationPaths: true,
|
||||
})
|
||||
|
||||
gotTranscript := mustReadTranscript(t, first.outputPath)
|
||||
expectFinalTranscriptContains(t, gotTranscript, expectations.MustApplyTexts)
|
||||
expectFinalTranscriptDoesNotContain(t, gotTranscript, expectations.MustNotApplyTexts)
|
||||
expectFinalTranscriptContains(t, gotTranscript, expectations.ProtectedTerms)
|
||||
|
||||
assertReleaseCounts(t, first.report, expectations)
|
||||
assertPromptAndSchemaMetadataPresent(t, first.runDir)
|
||||
assertReleaseLedgerShape(t, first.report)
|
||||
assertReleaseUtilizationShape(t, first.report)
|
||||
assertStableValidatorKeysPresent(t, first.report)
|
||||
assertStdoutStderrContract(t, first.stdout, first.stderr)
|
||||
assertNoSecretMarkersInTree(t, first.runDir, []string{"release-secret"})
|
||||
assertNoSecretMarkers(t, first.reportPath, []string{"release-secret"})
|
||||
|
||||
// Output schema check: audita-v1 object payload.
|
||||
auditaV1 := runReleaseFixturePass(t, releaseRunConfig{
|
||||
transcriptPath: base + ".transcript.json",
|
||||
glossaryPath: base + ".glossary.yaml",
|
||||
outputSchema: "audita-v1",
|
||||
proposalResponses: proposalResponses,
|
||||
validationResponses: validationResponses,
|
||||
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
|
||||
reportSchemaName: reporting.DefaultProcessReportSchemaName,
|
||||
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
||||
expectedOutputSchema: "audita-v1",
|
||||
expectModuleInstances: expectations.ExpectedModuleInstance,
|
||||
expectUtilizationPaths: true,
|
||||
})
|
||||
assertAuditaV1OutputShape(t, auditaV1.outputPath)
|
||||
|
||||
// Idempotence-oriented second pass:
|
||||
// run again on first output with deterministic no-op responses.
|
||||
noOpProposals := make([]proposal_generation.StructuredCorrectionSet, 5)
|
||||
for i := range noOpProposals {
|
||||
noOpProposals[i] = proposal_generation.StructuredCorrectionSet{Corrections: nil}
|
||||
}
|
||||
second := runReleaseFixturePass(t, releaseRunConfig{
|
||||
transcriptPath: first.outputPath,
|
||||
glossaryPath: base + ".glossary.yaml",
|
||||
outputSchema: "bare-segments",
|
||||
proposalResponses: noOpProposals,
|
||||
validationResponses: nil,
|
||||
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
|
||||
reportSchemaName: reporting.DefaultProcessReportSchemaName,
|
||||
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
||||
expectedOutputSchema: "bare-segments",
|
||||
expectModuleInstances: expectations.ExpectedModuleInstance,
|
||||
expectUtilizationPaths: true,
|
||||
})
|
||||
firstSegments := mustReadTranscript(t, first.outputPath)
|
||||
secondSegments := mustReadTranscript(t, second.outputPath)
|
||||
if !reflect.DeepEqual(firstSegments, secondSegments) {
|
||||
t.Fatalf("expected idempotent second pass transcript; first=%+v second=%+v", firstSegments, secondSegments)
|
||||
}
|
||||
if second.report.ModulesSummary == nil {
|
||||
t.Fatalf("expected modules summary on second pass")
|
||||
}
|
||||
if second.report.ModulesSummary.TotalAppliedChanges != 0 {
|
||||
t.Fatalf("expected no-op second pass (0 applied), got %+v", second.report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
|
||||
type releaseRunConfig struct {
|
||||
transcriptPath string
|
||||
glossaryPath string
|
||||
outputSchema string
|
||||
proposalResponses []proposal_generation.StructuredCorrectionSet
|
||||
validationResponses []validators.LLMValidationResponse
|
||||
expectedProposalCalls []string
|
||||
reportSchemaName string
|
||||
reportSchemaVersion string
|
||||
expectedOutputSchema string
|
||||
expectModuleInstances []string
|
||||
expectUtilizationPaths bool
|
||||
}
|
||||
|
||||
type releaseRunResult struct {
|
||||
stdout string
|
||||
stderr string
|
||||
outputPath string
|
||||
reportPath string
|
||||
report reporting.ProcessReport
|
||||
runDir string
|
||||
}
|
||||
|
||||
func runReleaseFixturePass(t *testing.T, cfg releaseRunConfig) releaseRunResult {
|
||||
t.Helper()
|
||||
|
||||
processProposalLLMClient = &fakeStructuredLLMClient{proposalResponses: append([]proposal_generation.StructuredCorrectionSet(nil), cfg.proposalResponses...)}
|
||||
processValidationLLMClient = &fakeStructuredLLMClient{validationResponses: append([]validators.LLMValidationResponse(nil), cfg.validationResponses...)}
|
||||
t.Cleanup(func() {
|
||||
processProposalLLMClient = nil
|
||||
processValidationLLMClient = nil
|
||||
})
|
||||
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
outputPath := filepath.Join(t.TempDir(), "out.json")
|
||||
configPath := writeFile(t, "release-config.yml", "version: 1\n")
|
||||
|
||||
args := []string{
|
||||
"process",
|
||||
cfg.transcriptPath,
|
||||
"--glossary",
|
||||
cfg.glossaryPath,
|
||||
"--config",
|
||||
configPath,
|
||||
"--output",
|
||||
outputPath,
|
||||
"--output-schema",
|
||||
cfg.outputSchema,
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
}
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
exitCode := Run(args, &stdout, &stderr)
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||
}
|
||||
|
||||
report := readProcessReport(t, reportPath)
|
||||
if report.ReportMetadata.ReportSchemaName != cfg.reportSchemaName || report.ReportMetadata.ReportSchemaVersion != cfg.reportSchemaVersion {
|
||||
t.Fatalf("unexpected report schema metadata: %+v", report.ReportMetadata)
|
||||
}
|
||||
if report.ReportMetadata.OutputSchema != cfg.expectedOutputSchema {
|
||||
t.Fatalf("unexpected output schema metadata: got %q want %q", report.ReportMetadata.OutputSchema, cfg.expectedOutputSchema)
|
||||
}
|
||||
if len(cfg.expectModuleInstances) > 0 {
|
||||
got := make([]string, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, mr.ModuleInstance)
|
||||
}
|
||||
if !reflect.DeepEqual(got, cfg.expectModuleInstances) {
|
||||
t.Fatalf("unexpected module instances: got %v want %v", got, cfg.expectModuleInstances)
|
||||
}
|
||||
}
|
||||
if report.Diagnostics == nil {
|
||||
t.Fatalf("expected diagnostics metadata")
|
||||
}
|
||||
if cfg.expectUtilizationPaths {
|
||||
if report.Diagnostics.UtilizationSummaryPath == "" || report.Diagnostics.CorrectionLedgerPath == "" {
|
||||
t.Fatalf("expected utilization/ledger artifact paths in report diagnostics: %+v", report.Diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil {
|
||||
t.Fatalf("expected run-dir report: %v", err)
|
||||
}
|
||||
|
||||
if c, ok := processProposalLLMClient.(*fakeStructuredLLMClient); ok {
|
||||
if !reflect.DeepEqual(c.calls, cfg.expectedProposalCalls) {
|
||||
t.Fatalf("unexpected proposal call order: got %v want %v", c.calls, cfg.expectedProposalCalls)
|
||||
}
|
||||
}
|
||||
|
||||
return releaseRunResult{
|
||||
stdout: stdout.String(),
|
||||
stderr: stderr.String(),
|
||||
outputPath: outputPath,
|
||||
reportPath: reportPath,
|
||||
report: report,
|
||||
runDir: runDir,
|
||||
}
|
||||
}
|
||||
|
||||
func readProposalResponses(t *testing.T, path string) []proposal_generation.StructuredCorrectionSet {
|
||||
t.Helper()
|
||||
var out []proposal_generation.StructuredCorrectionSet
|
||||
if err := json.Unmarshal(readFile(t, path), &out); err != nil {
|
||||
t.Fatalf("unmarshal proposal responses: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func readValidationResponses(t *testing.T, path string) []validators.LLMValidationResponse {
|
||||
t.Helper()
|
||||
var out []validators.LLMValidationResponse
|
||||
if err := json.Unmarshal(readFile(t, path), &out); err != nil {
|
||||
t.Fatalf("unmarshal validation responses: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mustReadTranscript(t *testing.T, path string) []schema.Segment {
|
||||
t.Helper()
|
||||
transcript, err := schema.ParseTranscriptJSON(readFile(t, path))
|
||||
if err != nil {
|
||||
t.Fatalf("parse transcript output: %v", err)
|
||||
}
|
||||
return transcript.Segments
|
||||
}
|
||||
|
||||
func expectFinalTranscriptContains(t *testing.T, segments []schema.Segment, needles []string) {
|
||||
t.Helper()
|
||||
joined := flattenTranscriptText(segments)
|
||||
for _, needle := range needles {
|
||||
if !strings.Contains(joined, needle) {
|
||||
t.Fatalf("expected transcript to contain %q, got %q", needle, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func expectFinalTranscriptDoesNotContain(t *testing.T, segments []schema.Segment, needles []string) {
|
||||
t.Helper()
|
||||
joined := flattenTranscriptText(segments)
|
||||
for _, needle := range needles {
|
||||
if strings.Contains(joined, needle) {
|
||||
t.Fatalf("expected transcript to not contain %q, got %q", needle, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func flattenTranscriptText(segments []schema.Segment) string {
|
||||
parts := make([]string, 0, len(segments))
|
||||
for _, s := range segments {
|
||||
parts = append(parts, s.Text)
|
||||
}
|
||||
return strings.Join(parts, "\n")
|
||||
}
|
||||
|
||||
func assertReleaseCounts(t *testing.T, report reporting.ProcessReport, exp releaseFixtureExpectations) {
|
||||
t.Helper()
|
||||
if report.ModulesSummary == nil {
|
||||
t.Fatalf("expected modules_summary")
|
||||
}
|
||||
if report.ModulesSummary.TotalAppliedChanges < exp.MinimumCounts.Applied {
|
||||
t.Fatalf("expected at least %d applied changes, got %+v", exp.MinimumCounts.Applied, report.ModulesSummary)
|
||||
}
|
||||
validatorRejected := 0
|
||||
skipped := 0
|
||||
for _, mr := range report.ModuleResults {
|
||||
validatorRejected += len(mr.ValidatorRejected)
|
||||
skipped += len(mr.SkippedChanges)
|
||||
}
|
||||
if validatorRejected < exp.MinimumCounts.Rejected {
|
||||
t.Fatalf("expected at least %d validator rejections, got %d", exp.MinimumCounts.Rejected, validatorRejected)
|
||||
}
|
||||
if skipped < exp.MinimumCounts.Skipped {
|
||||
t.Fatalf("expected at least %d application skips, got %d", exp.MinimumCounts.Skipped, skipped)
|
||||
}
|
||||
}
|
||||
|
||||
func assertReleaseUtilizationShape(t *testing.T, report reporting.ProcessReport) {
|
||||
t.Helper()
|
||||
var payload struct {
|
||||
EffectiveConcurrency struct {
|
||||
TotalLLM int `json:"total_llm"`
|
||||
} `json:"effective_concurrency"`
|
||||
RunTiming struct {
|
||||
SchedulerQueueWaitMS int64 `json:"scheduler_queue_wait_ms"`
|
||||
LLMExecutionTimeMS int64 `json:"llm_execution_time_ms"`
|
||||
DeterministicValidationMS int64 `json:"deterministic_validation_time_ms"`
|
||||
} `json:"run_timing"`
|
||||
Modules []map[string]any `json:"modules"`
|
||||
Validators []map[string]any `json:"validators"`
|
||||
}
|
||||
if err := json.Unmarshal(readFile(t, report.Diagnostics.UtilizationSummaryPath), &payload); err != nil {
|
||||
t.Fatalf("unmarshal utilization diagnostics: %v", err)
|
||||
}
|
||||
if payload.EffectiveConcurrency.TotalLLM <= 0 {
|
||||
t.Fatalf("expected positive total llm concurrency, got %+v", payload.EffectiveConcurrency)
|
||||
}
|
||||
if payload.RunTiming.SchedulerQueueWaitMS < 0 || payload.RunTiming.LLMExecutionTimeMS < 0 || payload.RunTiming.DeterministicValidationMS < 0 {
|
||||
t.Fatalf("expected non-negative run timing values, got %+v", payload.RunTiming)
|
||||
}
|
||||
if len(payload.Modules) == 0 {
|
||||
t.Fatalf("expected module timing summaries")
|
||||
}
|
||||
if len(payload.Validators) == 0 {
|
||||
t.Fatalf("expected validator timing summaries")
|
||||
}
|
||||
}
|
||||
|
||||
func assertReleaseLedgerShape(t *testing.T, report reporting.ProcessReport) {
|
||||
t.Helper()
|
||||
var entries []struct {
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ProposalIndex int `json:"proposal_index"`
|
||||
Disposition string `json:"disposition"`
|
||||
DispositionReason string `json:"disposition_reason_code"`
|
||||
OriginalText string `json:"original_text"`
|
||||
ProposedCorrected string `json:"proposed_corrected_text"`
|
||||
ReplacementPolicy string `json:"replacement_policy"`
|
||||
DeterministicResults []struct {
|
||||
ValidatorKey string `json:"validator_key"`
|
||||
} `json:"deterministic_validator_decisions"`
|
||||
LLMResults []struct {
|
||||
ValidatorKey string `json:"validator_key"`
|
||||
} `json:"llm_validator_decisions"`
|
||||
}
|
||||
if err := json.Unmarshal(readFile(t, report.Diagnostics.CorrectionLedgerPath), &entries); err != nil {
|
||||
t.Fatalf("unmarshal correction ledger: %v", err)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
t.Fatalf("expected correction ledger entries")
|
||||
}
|
||||
hasApplied := false
|
||||
hasRejected := false
|
||||
hasSkipped := false
|
||||
for _, entry := range entries {
|
||||
if entry.ModuleInstance == "" || entry.ModuleKey == "" {
|
||||
t.Fatalf("expected module identity in ledger entry: %+v", entry)
|
||||
}
|
||||
switch entry.Disposition {
|
||||
case "applied":
|
||||
hasApplied = true
|
||||
case "rejected":
|
||||
hasRejected = true
|
||||
case "skipped":
|
||||
hasSkipped = true
|
||||
}
|
||||
}
|
||||
if !hasApplied || !hasRejected {
|
||||
t.Fatalf("expected applied and rejected entries in correction ledger, got %+v", entries)
|
||||
}
|
||||
if !hasSkipped {
|
||||
// Some deterministic fixture paths do not trigger apply-time skips;
|
||||
// rejections are still captured separately from application skips.
|
||||
}
|
||||
}
|
||||
|
||||
func assertPromptAndSchemaMetadataPresent(t *testing.T, runDir string) {
|
||||
t.Helper()
|
||||
metadataPaths, err := filepath.Glob(filepath.Join(runDir, "*", "*request-metadata.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("glob request metadata artifacts: %v", err)
|
||||
}
|
||||
if len(metadataPaths) == 0 {
|
||||
t.Fatalf("expected request metadata artifacts with prompt metadata")
|
||||
}
|
||||
|
||||
foundPromptMetadata := false
|
||||
foundSchemaMetadata := false
|
||||
for _, path := range metadataPaths {
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(readFile(t, path), &payload); err != nil {
|
||||
t.Fatalf("unmarshal request metadata artifact %q: %v", path, err)
|
||||
}
|
||||
if pm, ok := payload["prompt_metadata"].(map[string]any); ok {
|
||||
if pm["prompt_id"] != nil && pm["prompt_version"] != nil && pm["sha256"] != nil {
|
||||
foundPromptMetadata = true
|
||||
}
|
||||
}
|
||||
if sm, ok := payload["response_schema"].(map[string]any); ok {
|
||||
if sm["id"] != nil && sm["version"] != nil && sm["name"] != nil && sm["sha256"] != nil {
|
||||
foundSchemaMetadata = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !foundPromptMetadata {
|
||||
t.Fatalf("expected prompt metadata in request metadata artifacts")
|
||||
}
|
||||
if !foundSchemaMetadata {
|
||||
t.Fatalf("expected structured response schema metadata in request metadata artifacts")
|
||||
}
|
||||
}
|
||||
|
||||
func assertStableValidatorKeysPresent(t *testing.T, report reporting.ProcessReport) {
|
||||
t.Helper()
|
||||
seen := map[string]bool{}
|
||||
for _, module := range report.ModuleResults {
|
||||
for _, decision := range module.ValidatorDecisions {
|
||||
seen[decision.ValidatorName] = true
|
||||
}
|
||||
for _, rejected := range module.ValidatorRejected {
|
||||
seen[rejected.ValidatorName] = true
|
||||
}
|
||||
}
|
||||
expectedAny := []string{
|
||||
"confidence_threshold",
|
||||
"original_text_presence",
|
||||
"no_effect",
|
||||
}
|
||||
for _, key := range expectedAny {
|
||||
if !seen[key] {
|
||||
t.Fatalf("expected stable validator key %q in report decisions/rejections; seen=%v", key, seen)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertStdoutStderrContract(t *testing.T, stdout, stderr string) {
|
||||
t.Helper()
|
||||
if stdout != "" {
|
||||
t.Fatalf("expected empty stdout with --output, got %q", stdout)
|
||||
}
|
||||
if strings.Contains(stderr, `"module_results"`) || strings.Contains(stderr, `"report_metadata"`) {
|
||||
t.Fatalf("stderr should remain human-readable, not report JSON: %q", stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func assertAuditaV1OutputShape(t *testing.T, outputPath string) {
|
||||
t.Helper()
|
||||
var payload struct {
|
||||
Schema string `json:"schema"`
|
||||
Version string `json:"version"`
|
||||
Segments []schema.Segment `json:"segments"`
|
||||
}
|
||||
if err := json.Unmarshal(readFile(t, outputPath), &payload); err != nil {
|
||||
t.Fatalf("unmarshal audita-v1 output: %v", err)
|
||||
}
|
||||
if payload.Schema != "audita-v1" || payload.Version != "v1" {
|
||||
t.Fatalf("unexpected audita-v1 metadata: %+v", payload)
|
||||
}
|
||||
if len(payload.Segments) == 0 {
|
||||
t.Fatalf("expected non-empty audita-v1 segments")
|
||||
}
|
||||
}
|
||||
867
internal/cli/run.go
Normal file
867
internal/cli/run.go
Normal file
@@ -0,0 +1,867 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
||||
coreio "gitea.maximumdirect.net/eric/audita/internal/core/io"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/processreport"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
)
|
||||
|
||||
type noOpStructuredLLMClient struct{}
|
||||
|
||||
func (c noOpStructuredLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
_ = ctx
|
||||
_ = req
|
||||
switch target := out.(type) {
|
||||
case *validators.LLMValidationResponse:
|
||||
*target = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{}}
|
||||
case *proposal_generation.StructuredCorrectionSet:
|
||||
*target = proposal_generation.StructuredCorrectionSet{Corrections: nil}
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
}
|
||||
|
||||
func shouldUseNoOpLLMClientForTests() bool {
|
||||
return strings.HasSuffix(filepath.Base(os.Args[0]), ".test") || os.Getenv("GO_WANT_HELPER_PROCESS") == "1"
|
||||
}
|
||||
|
||||
type processInvocation struct {
|
||||
TranscriptPath string
|
||||
GlossaryPath string
|
||||
OutputPath string
|
||||
ReportJSONPath string
|
||||
Config config.Config
|
||||
ConfigPath string
|
||||
ConfigSource string
|
||||
ConfigVersion *int
|
||||
ExplicitModules bool
|
||||
}
|
||||
|
||||
var processModuleFactory runner.ModuleFactory
|
||||
var processProposalLLMClient contracts.StructuredLLMClient
|
||||
var processProposalLLMScheduler runner.ValidationScheduler
|
||||
var processValidationLLMClient contracts.StructuredLLMClient
|
||||
var processValidationLLMScheduler runner.ValidationScheduler
|
||||
var processRunnerContext = func() (context.Context, context.CancelFunc) {
|
||||
return context.Background(), func() {}
|
||||
}
|
||||
|
||||
var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) {
|
||||
runDir, err := diagnostics.NewRunDirectory(inv.Config.WorkDir, string(inv.Config.WorkDirRetention))
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, fmt.Errorf("run_dir_creation: %w", err)
|
||||
}
|
||||
|
||||
fail := func(phase string, err error, runOutput *runner.RunOutput) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("%s: %v", phase, err))
|
||||
return nil, nil, runOutput, runDir, fmt.Errorf("%s: %w", phase, err)
|
||||
}
|
||||
|
||||
if err := runDir.WriteInvocationMetadata(diagnostics.InvocationMetadata{
|
||||
Operation: "process",
|
||||
TranscriptPath: inv.TranscriptPath,
|
||||
GlossaryPath: inv.GlossaryPath,
|
||||
OutputPath: inv.OutputPath,
|
||||
ReportJSONPath: inv.ReportJSONPath,
|
||||
ConfigPath: inv.ConfigPath,
|
||||
ConfigSource: inv.ConfigSource,
|
||||
ConfigVersion: inv.ConfigVersion,
|
||||
TranscriptDescription: inv.Config.TranscriptDescription,
|
||||
Modules: append([]string(nil), inv.Config.Modules...),
|
||||
}); err != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("invocation_metadata: %v", err))
|
||||
}
|
||||
if err := runDir.WriteEffectiveConfig(inv.Config); err != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("effective_config: %v", err))
|
||||
}
|
||||
|
||||
transcriptBytes, err := coreio.ReadRequiredFile(inv.TranscriptPath, "transcript")
|
||||
if err != nil {
|
||||
return fail("transcript_read", err, nil)
|
||||
}
|
||||
|
||||
glossaryBytes, err := coreio.ReadRequiredFile(inv.GlossaryPath, "glossary")
|
||||
if err != nil {
|
||||
return fail("glossary_read", err, nil)
|
||||
}
|
||||
|
||||
sourceTranscript, err := schema.ParseSourceTranscriptJSON(transcriptBytes)
|
||||
if err != nil {
|
||||
return fail("transcript_schema", err, nil)
|
||||
}
|
||||
|
||||
glossary, err := schema.ParseGlossaryYAML(glossaryBytes)
|
||||
if err != nil {
|
||||
return fail("glossary_schema", err, nil)
|
||||
}
|
||||
|
||||
if err := runDir.WriteSourceTranscript(sourceTranscript, transcriptBytes); err != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("source_artifact: %v", err))
|
||||
}
|
||||
|
||||
canonical := sourceToCanonicalTranscript(sourceTranscript)
|
||||
normalizer := normalization.NewNormalizer(normalization.NormalizationConfig{
|
||||
MaxSegmentGap: inv.Config.Normalization.MaxSegmentGap,
|
||||
EllipsisGap: inv.Config.Normalization.EllipsisGap,
|
||||
MaxSegmentDuration: inv.Config.Normalization.MaxSegmentDuration,
|
||||
MaxSegmentTokens: inv.Config.Normalization.MaxSegmentTokens,
|
||||
})
|
||||
|
||||
normalizedTranscript, normSummary := normalizer.Normalize(canonical)
|
||||
|
||||
if err := runDir.WriteNormalizedTranscript(normalizedTranscript); err != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("normalized_artifact: %v", err))
|
||||
}
|
||||
if err := runDir.WriteNormalizationSummary(normSummary); err != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("normalization_summary: %v", err))
|
||||
}
|
||||
|
||||
// Compute chunks after normalization
|
||||
chunker := chunking.NewChunker(chunking.ChunkingConfig{
|
||||
MaxSectionTokens: inv.Config.MaxSectionTokens,
|
||||
MinSectionTokens: inv.Config.MinSectionTokens,
|
||||
TargetSections: inv.Config.TargetSections,
|
||||
})
|
||||
|
||||
sections, chunkErr := chunker.ChunkTranscript(normalizedTranscript)
|
||||
if chunkErr != nil {
|
||||
return fail("chunking", chunkErr, nil)
|
||||
}
|
||||
|
||||
chunkConfig := chunking.ChunkingConfig{
|
||||
MaxSectionTokens: inv.Config.MaxSectionTokens,
|
||||
MinSectionTokens: inv.Config.MinSectionTokens,
|
||||
TargetSections: inv.Config.TargetSections,
|
||||
}
|
||||
chunkSummary := chunking.ComputeSummary(sections, chunkConfig)
|
||||
chunkDetailedSummary := chunking.ComputeDetailedSummary(sections, chunkConfig)
|
||||
|
||||
if err := runDir.WriteChunkingSummary(&chunkDetailedSummary); err != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("chunking_summary: %v", err))
|
||||
}
|
||||
|
||||
workingTranscript := normalizedTranscript
|
||||
var runOutput *runner.RunOutput
|
||||
moduleFactory := processModuleFactory
|
||||
if moduleFactory == nil {
|
||||
moduleFactory = modules.NewFactory(modules.Dependencies{
|
||||
Config: &inv.Config,
|
||||
Glossary: glossary,
|
||||
DiagnosticsDir: runDir.Path(),
|
||||
})
|
||||
}
|
||||
if moduleFactory != nil {
|
||||
proposalLLMClient := processProposalLLMClient
|
||||
validationLLMClient := processValidationLLMClient
|
||||
proposalScheduler := processProposalLLMScheduler
|
||||
validationScheduler := processValidationLLMScheduler
|
||||
|
||||
if processModuleFactory == nil {
|
||||
// Production runtime path: construct clients/schedulers from config.
|
||||
if proposalLLMClient == nil {
|
||||
if shouldUseNoOpLLMClientForTests() {
|
||||
proposalLLMClient = noOpStructuredLLMClient{}
|
||||
} else {
|
||||
primaryCfg := llm.ResolvePrimaryConfig(inv.Config)
|
||||
client, clientErr := llm.NewOpenAICompatibleClient(primaryCfg.ToOpenAICompatibleClientConfig(nil))
|
||||
if clientErr != nil {
|
||||
return fail("runner_setup", clientErr, nil)
|
||||
}
|
||||
proposalLLMClient = client
|
||||
}
|
||||
}
|
||||
if validationLLMClient == nil {
|
||||
if shouldUseNoOpLLMClientForTests() {
|
||||
validationLLMClient = noOpStructuredLLMClient{}
|
||||
} else {
|
||||
validationCfg := llm.ResolveValidationConfig(inv.Config)
|
||||
client, clientErr := llm.NewOpenAICompatibleClient(validationCfg.ToOpenAICompatibleClientConfig(nil))
|
||||
if clientErr != nil {
|
||||
return fail("runner_setup", clientErr, nil)
|
||||
}
|
||||
validationLLMClient = client
|
||||
}
|
||||
}
|
||||
globalScheduler := proposalScheduler
|
||||
if globalScheduler == nil {
|
||||
s, sErr := llm.NewScheduler(inv.Config.TotalLLMConcurrency)
|
||||
if sErr != nil {
|
||||
return fail("runner_setup", sErr, nil)
|
||||
}
|
||||
globalScheduler = s
|
||||
}
|
||||
if proposalScheduler == nil {
|
||||
proposalScheduler = globalScheduler
|
||||
if inv.Config.EffectiveProposalLLMConcurrency() < inv.Config.TotalLLMConcurrency {
|
||||
s, sErr := llm.NewScheduler(inv.Config.EffectiveProposalLLMConcurrency())
|
||||
if sErr != nil {
|
||||
return fail("runner_setup", sErr, nil)
|
||||
}
|
||||
proposalScheduler = composeSchedulers(globalScheduler, s)
|
||||
}
|
||||
}
|
||||
if validationScheduler == nil {
|
||||
validationScheduler = globalScheduler
|
||||
if inv.Config.ValidationLLMConcurrency != nil && inv.Config.EffectiveValidationLLMConcurrency() < inv.Config.TotalLLMConcurrency {
|
||||
s, sErr := llm.NewScheduler(inv.Config.EffectiveValidationLLMConcurrency())
|
||||
if sErr != nil {
|
||||
return fail("runner_setup", sErr, nil)
|
||||
}
|
||||
validationScheduler = composeSchedulers(globalScheduler, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
moduleSpecs, err := contracts.ResolveModuleRunSpecs(inv.Config.Modules)
|
||||
if err != nil {
|
||||
return fail("runner_setup", err, nil)
|
||||
}
|
||||
|
||||
runCtx, cancelRun := processRunnerContext()
|
||||
defer cancelRun()
|
||||
runnerResult, runErr := runner.New(moduleFactory).Run(runCtx, runner.RunInput{
|
||||
Config: &inv.Config,
|
||||
Transcript: normalizedTranscript,
|
||||
Glossary: glossary,
|
||||
ModuleSpecs: moduleSpecs,
|
||||
EffectiveConcurrency: runner.EffectiveConcurrencyLimits{
|
||||
TotalLLM: inv.Config.TotalLLMConcurrency,
|
||||
ProposalLLM: inv.Config.EffectiveProposalLLMConcurrency(),
|
||||
ValidationLLM: inv.Config.EffectiveValidationLLMConcurrency(),
|
||||
},
|
||||
ProposalLLMClient: proposalLLMClient,
|
||||
ProposalLLMScheduler: proposalScheduler,
|
||||
ProposalDiagnosticsDir: runDir.Path(),
|
||||
ValidationLLMClient: validationLLMClient,
|
||||
ValidationLLMScheduler: validationScheduler,
|
||||
ValidationDiagnosticsDir: runDir.Path(),
|
||||
})
|
||||
runOutput = &runnerResult
|
||||
if runErr != nil {
|
||||
return fail("runner_execution", runErr, runOutput)
|
||||
}
|
||||
workingTranscript = runnerResult.FinalTranscript
|
||||
}
|
||||
|
||||
encoderDef, err := outputschema.Resolve(inv.Config.OutputSchema)
|
||||
if err != nil {
|
||||
return fail("output_schema", err, runOutput)
|
||||
}
|
||||
outputBytes, err := encoderDef.Encoder(workingTranscript)
|
||||
if err != nil {
|
||||
return fail("serialization", err, runOutput)
|
||||
}
|
||||
|
||||
if strings.TrimSpace(inv.OutputPath) != "" {
|
||||
if err := coreio.WriteFile(inv.OutputPath, outputBytes); err != nil {
|
||||
return fail("output_write", err, runOutput)
|
||||
}
|
||||
return normSummary, &chunkSummary, runOutput, runDir, nil
|
||||
}
|
||||
|
||||
if _, err := stdout.Write(outputBytes); err != nil {
|
||||
return fail("stdout_write", err, runOutput)
|
||||
}
|
||||
|
||||
return normSummary, &chunkSummary, runOutput, runDir, nil
|
||||
}
|
||||
|
||||
func sourceToCanonicalTranscript(source *schema.SourceTranscript) *schema.Transcript {
|
||||
segments := make([]schema.Segment, len(source.Segments))
|
||||
for i, s := range source.Segments {
|
||||
id := i + 1
|
||||
if s.ID != nil {
|
||||
id = *s.ID
|
||||
}
|
||||
segments[i] = schema.Segment{
|
||||
ID: id,
|
||||
Speaker: s.Speaker,
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
Text: s.Text,
|
||||
Categories: s.Categories,
|
||||
}
|
||||
}
|
||||
return &schema.Transcript{Segments: segments}
|
||||
}
|
||||
|
||||
type chainedScheduler struct {
|
||||
schedulers []runner.ValidationScheduler
|
||||
}
|
||||
|
||||
func (s chainedScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
|
||||
if len(s.schedulers) == 0 {
|
||||
return fn(ctx)
|
||||
}
|
||||
run := fn
|
||||
for i := len(s.schedulers) - 1; i >= 0; i-- {
|
||||
scheduler := s.schedulers[i]
|
||||
next := run
|
||||
run = func(callCtx context.Context) error {
|
||||
return scheduler.Run(callCtx, next)
|
||||
}
|
||||
}
|
||||
return run(ctx)
|
||||
}
|
||||
|
||||
func composeSchedulers(schedulers ...runner.ValidationScheduler) runner.ValidationScheduler {
|
||||
filtered := make([]runner.ValidationScheduler, 0, len(schedulers))
|
||||
for _, scheduler := range schedulers {
|
||||
if scheduler != nil {
|
||||
filtered = append(filtered, scheduler)
|
||||
}
|
||||
}
|
||||
switch len(filtered) {
|
||||
case 0:
|
||||
return nil
|
||||
case 1:
|
||||
return filtered[0]
|
||||
default:
|
||||
return chainedScheduler{schedulers: filtered}
|
||||
}
|
||||
}
|
||||
|
||||
// Run executes the Audita CLI with the provided arguments and streams.
|
||||
func Run(args []string, stdout, stderr io.Writer) int {
|
||||
if len(args) == 0 {
|
||||
writeRootUsage(stdout)
|
||||
return 0
|
||||
}
|
||||
|
||||
if isHelpCommand(args) {
|
||||
writeRootUsage(stdout)
|
||||
return 0
|
||||
}
|
||||
|
||||
if args[0] == "process" {
|
||||
return runProcess(args[1:], stdout, stderr)
|
||||
}
|
||||
if args[0] == "config" {
|
||||
return runConfig(args[1:], stdout, stderr)
|
||||
}
|
||||
|
||||
fmt.Fprintf(stderr, "audita: unknown command %q\n\n", args[0])
|
||||
writeRootUsage(stderr)
|
||||
return 2
|
||||
}
|
||||
|
||||
func runProcess(args []string, stdout, stderr io.Writer) int {
|
||||
startedAt := time.Now().UTC()
|
||||
|
||||
configPathOverride, configPathOverrideSet, err := findConfigPathOverride(args)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
effectiveConfig, err := config.LoadEffectiveConfig(configPathOverride, configPathOverrideSet)
|
||||
if err != nil {
|
||||
var effectiveConfigErr *config.EffectiveConfigError
|
||||
if errors.As(err, &effectiveConfigErr) {
|
||||
switch effectiveConfigErr.Kind {
|
||||
case config.EffectiveConfigErrorLoadFile, config.EffectiveConfigErrorApplyFile:
|
||||
fmt.Fprintf(stderr, "audita process: invalid config file: %v\n", effectiveConfigErr)
|
||||
case config.EffectiveConfigErrorApplyEnv:
|
||||
fmt.Fprintf(stderr, "audita process: invalid environment configuration: %v\n", effectiveConfigErr)
|
||||
default:
|
||||
fmt.Fprintf(stderr, "audita process: %v\n", effectiveConfigErr)
|
||||
}
|
||||
} else {
|
||||
fmt.Fprintf(stderr, "audita process: %v\n", err)
|
||||
}
|
||||
return 2
|
||||
}
|
||||
|
||||
cfg := effectiveConfig.Config
|
||||
configPath := effectiveConfig.ConfigPath
|
||||
configSource := effectiveConfig.ConfigSource
|
||||
configVersion := effectiveConfig.ConfigVersion
|
||||
|
||||
fs, pFlags := newProcessFlagSet(cfg, stderr)
|
||||
|
||||
if isHelpCommand(args) || hasHelpFlag(args) {
|
||||
writeProcessUsage(stdout, fs)
|
||||
return 0
|
||||
}
|
||||
|
||||
parseArgs := args
|
||||
transcriptFromFront := ""
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
transcriptFromFront = args[0]
|
||||
parseArgs = args[1:]
|
||||
}
|
||||
|
||||
if err := fs.Parse(parseArgs); err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
writeProcessUsage(stdout, fs)
|
||||
return 0
|
||||
}
|
||||
return 2
|
||||
}
|
||||
|
||||
overrides, explicitModules := processCLIOverrides(fs, pFlags)
|
||||
|
||||
if err := cfg.ApplyCLIOverrides(overrides); err != nil {
|
||||
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
|
||||
remaining := fs.Args()
|
||||
positional := make([]string, 0, len(remaining)+1)
|
||||
if transcriptFromFront != "" {
|
||||
positional = append(positional, transcriptFromFront)
|
||||
}
|
||||
positional = append(positional, remaining...)
|
||||
|
||||
if len(positional) != 1 {
|
||||
fmt.Fprintln(stderr, "audita process: expected exactly 1 transcript JSON path argument")
|
||||
return 2
|
||||
}
|
||||
if strings.TrimSpace(*pFlags.glossaryPath) == "" {
|
||||
fmt.Fprintln(stderr, "audita process: --glossary is required")
|
||||
return 2
|
||||
}
|
||||
|
||||
inv := processInvocation{
|
||||
TranscriptPath: positional[0],
|
||||
GlossaryPath: *pFlags.glossaryPath,
|
||||
OutputPath: *pFlags.outputPath,
|
||||
ReportJSONPath: *pFlags.reportJSONPath,
|
||||
Config: cfg,
|
||||
ConfigPath: configPath,
|
||||
ConfigSource: configSource,
|
||||
ConfigVersion: configVersion,
|
||||
ExplicitModules: explicitModules,
|
||||
}
|
||||
|
||||
normSummary, chunkSummary, runOutput, runDir, runErr := processRunner(inv, stdout)
|
||||
completedAt := time.Now().UTC()
|
||||
|
||||
if runErr != nil {
|
||||
if runDir != nil && runOutput != nil {
|
||||
if runOutput.Utilization != nil {
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
||||
}
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
|
||||
RunDirectoryPath: runDir.Path(),
|
||||
RunOutput: runOutput,
|
||||
}))
|
||||
}
|
||||
errorPhase, errorMessage := extractErrorPhase(runErr)
|
||||
report := processreport.Build(processReportInput("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput))
|
||||
|
||||
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
||||
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
||||
fmt.Fprintf(stderr, "audita process: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
if runDir != nil {
|
||||
_ = runDir.WriteReport(report)
|
||||
_ = runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
||||
RunSucceeded: false,
|
||||
})
|
||||
}
|
||||
|
||||
fmt.Fprintf(stderr, "audita process: %v\n", runErr)
|
||||
if runDir != nil {
|
||||
fmt.Fprintf(stderr, "audita process: diagnostics: %s\n", runDir.Path())
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
if runDir != nil && runOutput != nil {
|
||||
if runOutput.Utilization != nil {
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
||||
}
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
|
||||
RunDirectoryPath: runDir.Path(),
|
||||
RunOutput: runOutput,
|
||||
}))
|
||||
}
|
||||
|
||||
report := processreport.Build(processReportInput("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput))
|
||||
|
||||
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
||||
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
||||
if runDir != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("report_write: %v", err))
|
||||
_ = runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
||||
RunSucceeded: false,
|
||||
})
|
||||
}
|
||||
fmt.Fprintf(stderr, "audita process: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
if runDir != nil {
|
||||
_ = runDir.WriteReport(report)
|
||||
if err := runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
||||
RunSucceeded: true,
|
||||
HasSkippedCorrections: processreport.HasSkippedCorrections(runOutput),
|
||||
}); err != nil {
|
||||
fmt.Fprintf(stderr, "audita process: failed to apply work-dir retention: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
func runConfig(args []string, stdout, stderr io.Writer) int {
|
||||
if len(args) == 0 || isHelpCommand(args) || hasHelpFlag(args) {
|
||||
writeConfigUsage(stdout)
|
||||
return 0
|
||||
}
|
||||
|
||||
switch args[0] {
|
||||
case "validate":
|
||||
return runConfigValidate(args[1:], stdout, stderr)
|
||||
case "print-effective":
|
||||
return runConfigPrintEffective(args[1:], stdout, stderr)
|
||||
default:
|
||||
fmt.Fprintf(stderr, "audita config: unknown command %q\n\n", args[0])
|
||||
writeConfigUsage(stderr)
|
||||
return 2
|
||||
}
|
||||
}
|
||||
|
||||
func runConfigValidate(args []string, stdout, stderr io.Writer) int {
|
||||
fs := flag.NewFlagSet("config validate", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
configPath := fs.String("config", "", "Path to versioned YAML config file")
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
writeConfigValidateUsage(stdout)
|
||||
return 0
|
||||
}
|
||||
return 2
|
||||
}
|
||||
if strings.TrimSpace(*configPath) == "" {
|
||||
fmt.Fprintln(stderr, "audita config validate: --config is required")
|
||||
return 2
|
||||
}
|
||||
if len(fs.Args()) != 0 {
|
||||
fmt.Fprintln(stderr, "audita config validate: unexpected positional arguments")
|
||||
return 2
|
||||
}
|
||||
|
||||
fileCfg, err := config.LoadFileConfig(strings.TrimSpace(*configPath))
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
cfg := config.Default()
|
||||
if err := cfg.ApplyFileConfig(fileCfg); err != nil {
|
||||
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
fmt.Fprintln(stdout, "config is valid")
|
||||
return 0
|
||||
}
|
||||
|
||||
func runConfigPrintEffective(args []string, stdout, stderr io.Writer) int {
|
||||
fs := flag.NewFlagSet("config print-effective", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
configPath := fs.String("config", "", "Path to versioned YAML config file")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
writeConfigPrintEffectiveUsage(stdout)
|
||||
return 0
|
||||
}
|
||||
return 2
|
||||
}
|
||||
if len(fs.Args()) != 0 {
|
||||
fmt.Fprintln(stderr, "audita config print-effective: unexpected positional arguments")
|
||||
return 2
|
||||
}
|
||||
|
||||
configPathValue := strings.TrimSpace(*configPath)
|
||||
configPathSet := configPathValue != ""
|
||||
effectiveConfig, err := config.LoadEffectiveConfig(configPathValue, configPathSet)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
|
||||
cfg := effectiveConfig.Config
|
||||
|
||||
redacted := cfg.Redacted()
|
||||
out, err := json.MarshalIndent(redacted, "", " ")
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
out = append(out, '\n')
|
||||
if _, err := stdout.Write(out); err != nil {
|
||||
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func extractErrorPhase(err error) (phase string, message string) {
|
||||
msg := err.Error()
|
||||
if strings.Contains(msg, ": ") {
|
||||
parts := strings.SplitN(msg, ": ", 2)
|
||||
if len(parts) == 2 {
|
||||
return parts[0], parts[1]
|
||||
}
|
||||
}
|
||||
return "", msg
|
||||
}
|
||||
|
||||
func processReportInput(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) processreport.BuildInput {
|
||||
runDirectoryPath := ""
|
||||
if runDir != nil {
|
||||
runDirectoryPath = runDir.Path()
|
||||
}
|
||||
return processreport.BuildInput{
|
||||
Status: status,
|
||||
TranscriptPath: inv.TranscriptPath,
|
||||
GlossaryPath: inv.GlossaryPath,
|
||||
OutputPath: inv.OutputPath,
|
||||
Modules: inv.Config.Modules,
|
||||
OutputSchema: inv.Config.OutputSchema,
|
||||
ConfigVersion: inv.ConfigVersion,
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
ErrorMessage: errorMessage,
|
||||
ErrorPhase: errorPhase,
|
||||
RunDirectoryPath: runDirectoryPath,
|
||||
NormalizationSummary: normalizationSummary,
|
||||
ChunkingSummary: chunkingSummary,
|
||||
RunOutput: runOutput,
|
||||
}
|
||||
}
|
||||
|
||||
type processFlags struct {
|
||||
configPath *string
|
||||
glossaryPath *string
|
||||
outputPath *string
|
||||
reportJSONPath *string
|
||||
modules *string
|
||||
outputSchema *string
|
||||
llmAPIKey *string
|
||||
validationLLMAPIKey *string
|
||||
model *string
|
||||
validationModel *string
|
||||
baseURL *string
|
||||
validationBaseURL *string
|
||||
llmTimeoutSeconds *int
|
||||
totalLLMConcurrency *int
|
||||
proposalLLMConcurrency *int
|
||||
llmConcurrency *int
|
||||
validationLLMTimeoutSeconds *int
|
||||
validationMaxPromptTokens *int
|
||||
targetSections *int
|
||||
maxRetries *int
|
||||
validationMaxRetries *int
|
||||
validationLLMConcurrency *int
|
||||
maxSectionTokens *int
|
||||
minSectionTokens *int
|
||||
glossaryConfidenceThreshold *float64
|
||||
grammarConfidenceThreshold *float64
|
||||
homophonesConfidenceThreshold *float64
|
||||
spokenWordConfidenceThreshold *float64
|
||||
normalizeMaxSegmentGap *float64
|
||||
normalizeEllipsisGap *float64
|
||||
normalizeMaxSegmentDuration *float64
|
||||
normalizeMaxSegmentTokens *int
|
||||
transcriptDescription *string
|
||||
workDir *string
|
||||
workDirRetention *string
|
||||
}
|
||||
|
||||
func newProcessFlagSet(cfg config.Config, stderr io.Writer) (*flag.FlagSet, processFlags) {
|
||||
fs := flag.NewFlagSet("process", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
|
||||
validationTimeoutSecondsDefault := cfg.PrimaryLLM.TimeoutSeconds
|
||||
if cfg.ValidationLLM.TimeoutSeconds != nil {
|
||||
validationTimeoutSecondsDefault = *cfg.ValidationLLM.TimeoutSeconds
|
||||
}
|
||||
|
||||
validationMaxRetriesDefault := cfg.PrimaryLLM.MaxRetries
|
||||
if cfg.ValidationLLM.MaxRetries != nil {
|
||||
validationMaxRetriesDefault = *cfg.ValidationLLM.MaxRetries
|
||||
}
|
||||
|
||||
validationLLMConcurrencyDefault := cfg.TotalLLMConcurrency
|
||||
if cfg.ValidationLLMConcurrency != nil {
|
||||
validationLLMConcurrencyDefault = *cfg.ValidationLLMConcurrency
|
||||
}
|
||||
|
||||
targetSectionsDefault := 0
|
||||
if cfg.TargetSections != nil {
|
||||
targetSectionsDefault = *cfg.TargetSections
|
||||
}
|
||||
|
||||
pFlags := processFlags{
|
||||
configPath: fs.String("config", "", "Path to versioned YAML config file"),
|
||||
glossaryPath: fs.String("glossary", "", "Path to glossary YAML file"),
|
||||
outputPath: fs.String("output", "", "Path to corrected transcript JSON output file"),
|
||||
reportJSONPath: fs.String("report-json", "", "Path to machine-readable report JSON output file"),
|
||||
modules: fs.String("modules", strings.Join(cfg.Modules, ","), "Comma-separated module sequence override"),
|
||||
outputSchema: fs.String("output-schema", cfg.OutputSchema, "Output schema: bare-segments|audita-v1"),
|
||||
llmAPIKey: fs.String("llm-api-key", cfg.PrimaryLLM.APIKey, "Primary LLM API key"),
|
||||
validationLLMAPIKey: fs.String("validation-llm-api-key", cfg.ValidationLLM.APIKey, "Validation LLM API key"),
|
||||
model: fs.String("model", cfg.PrimaryLLM.Model, "Primary LLM model name"),
|
||||
validationModel: fs.String("validation-model", cfg.ValidationLLM.Model, "Validation LLM model name"),
|
||||
baseURL: fs.String("base-url", cfg.PrimaryLLM.BaseURL, "Primary OpenAI-compatible base URL"),
|
||||
validationBaseURL: fs.String("validation-base-url", cfg.ValidationLLM.BaseURL, "Validation OpenAI-compatible base URL"),
|
||||
llmTimeoutSeconds: fs.Int("llm-timeout-seconds", cfg.PrimaryLLM.TimeoutSeconds, "Primary LLM timeout in seconds"),
|
||||
totalLLMConcurrency: fs.Int("total-llm-concurrency", cfg.TotalLLMConcurrency, "Total concurrent LLM calls across proposal and validation"),
|
||||
proposalLLMConcurrency: fs.Int("proposal-llm-concurrency", cfg.EffectiveProposalLLMConcurrency(), "Concurrent proposal-generation LLM calls"),
|
||||
llmConcurrency: fs.Int("llm-concurrency", cfg.TotalLLMConcurrency, "Alias for --total-llm-concurrency"),
|
||||
validationLLMTimeoutSeconds: fs.Int("validation-llm-timeout-seconds", validationTimeoutSecondsDefault, "Validation LLM timeout in seconds"),
|
||||
validationMaxPromptTokens: fs.Int("validation-max-prompt-tokens", cfg.ValidationMaxPromptTokens, "Validation max prompt tokens"),
|
||||
targetSections: fs.Int("target-sections", targetSectionsDefault, "Target number of transcript sections"),
|
||||
maxRetries: fs.Int("max-retries", cfg.PrimaryLLM.MaxRetries, "Maximum structured-output retries"),
|
||||
validationMaxRetries: fs.Int("validation-max-retries", validationMaxRetriesDefault, "Validation structured-output retries"),
|
||||
validationLLMConcurrency: fs.Int("validation-llm-concurrency", validationLLMConcurrencyDefault, "Concurrent validation LLM calls (inherits total when unset)"),
|
||||
maxSectionTokens: fs.Int("max-section-tokens", cfg.MaxSectionTokens, "Maximum section tokens"),
|
||||
minSectionTokens: fs.Int("min-section-tokens", cfg.MinSectionTokens, "Minimum section tokens"),
|
||||
glossaryConfidenceThreshold: fs.Float64("glossary-confidence-threshold", cfg.Thresholds.Glossary, "Glossary confidence threshold"),
|
||||
grammarConfidenceThreshold: fs.Float64("grammar-confidence-threshold", cfg.Thresholds.Grammar, "Grammar confidence threshold"),
|
||||
homophonesConfidenceThreshold: fs.Float64("homophones-confidence-threshold", cfg.Thresholds.Homophones, "Homophones confidence threshold"),
|
||||
spokenWordConfidenceThreshold: fs.Float64("spoken-word-confidence-threshold", cfg.Thresholds.SpokenWord, "Spoken-word confidence threshold"),
|
||||
normalizeMaxSegmentGap: fs.Float64("normalize-max-segment-gap", cfg.Normalization.MaxSegmentGap, "Maximum same-speaker merge gap"),
|
||||
normalizeEllipsisGap: fs.Float64("normalize-ellipsis-gap", cfg.Normalization.EllipsisGap, "Gap threshold for ellipsis insertion"),
|
||||
normalizeMaxSegmentDuration: fs.Float64("normalize-max-segment-duration", cfg.Normalization.MaxSegmentDuration, "Maximum merged segment duration"),
|
||||
normalizeMaxSegmentTokens: fs.Int("normalize-max-segment-tokens", cfg.Normalization.MaxSegmentTokens, "Maximum merged segment token estimate"),
|
||||
transcriptDescription: fs.String("transcript-description", cfg.TranscriptDescription, "Brief background context for LLM prompts; does not override transcript content"),
|
||||
workDir: fs.String("work-dir", cfg.WorkDir, "Per-run work directory"),
|
||||
workDirRetention: fs.String("work-dir-retention", string(cfg.WorkDirRetention), "Work-dir retention policy: auto|always|never"),
|
||||
}
|
||||
|
||||
return fs, pFlags
|
||||
}
|
||||
|
||||
func findConfigPathOverride(args []string) (path string, set bool, err error) {
|
||||
for i := 0; i < len(args); i++ {
|
||||
arg := strings.TrimSpace(args[i])
|
||||
if arg == "" {
|
||||
continue
|
||||
}
|
||||
if arg == "--config" {
|
||||
if i+1 >= len(args) {
|
||||
return "", false, fmt.Errorf("--config requires a path")
|
||||
}
|
||||
return strings.TrimSpace(args[i+1]), true, nil
|
||||
}
|
||||
if strings.HasPrefix(arg, "--config=") {
|
||||
return strings.TrimSpace(strings.TrimPrefix(arg, "--config=")), true, nil
|
||||
}
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
func isHelpCommand(args []string) bool {
|
||||
if len(args) == 0 {
|
||||
return false
|
||||
}
|
||||
if len(args) == 1 {
|
||||
switch args[0] {
|
||||
case "help", "-h", "--help":
|
||||
return true
|
||||
}
|
||||
}
|
||||
if len(args) == 2 && args[0] == "help" {
|
||||
switch args[1] {
|
||||
case "process":
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasHelpFlag(args []string) bool {
|
||||
for _, arg := range args {
|
||||
if arg == "-h" || arg == "--help" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func writeRootUsage(w io.Writer) {
|
||||
fmt.Fprintln(w, "Audita is a transcript processing CLI.")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Usage:")
|
||||
fmt.Fprintln(w, " audita <command> [options]")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Commands:")
|
||||
fmt.Fprintln(w, " process Process a transcript JSON file")
|
||||
fmt.Fprintln(w, " config Validate and inspect config")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Example:")
|
||||
fmt.Fprintln(w, " audita process transcript.json --glossary glossary.yaml --output corrected.json")
|
||||
}
|
||||
|
||||
func writeConfigUsage(w io.Writer) {
|
||||
fmt.Fprintln(w, "Validate and inspect Audita config.")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Usage:")
|
||||
fmt.Fprintln(w, " audita config <command> [flags]")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Commands:")
|
||||
fmt.Fprintln(w, " validate Validate a versioned YAML config file")
|
||||
fmt.Fprintln(w, " print-effective Print redacted effective config JSON (defaults + config file + env)")
|
||||
}
|
||||
|
||||
func writeConfigValidateUsage(w io.Writer) {
|
||||
fmt.Fprintln(w, "Validate a versioned YAML config file.")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Usage:")
|
||||
fmt.Fprintln(w, " audita config validate --config <path>")
|
||||
}
|
||||
|
||||
func writeConfigPrintEffectiveUsage(w io.Writer) {
|
||||
fmt.Fprintln(w, "Print redacted effective config JSON.")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Usage:")
|
||||
fmt.Fprintln(w, " audita config print-effective [--config <path>]")
|
||||
}
|
||||
|
||||
func writeProcessUsage(w io.Writer, fs *flag.FlagSet) {
|
||||
fmt.Fprintln(w, "Process a transcript JSON file.")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Usage:")
|
||||
fmt.Fprintln(w, " audita process <transcript.json> [flags]")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Flags:")
|
||||
fs.VisitAll(func(f *flag.Flag) {
|
||||
fmt.Fprintf(w, " --%s\n", f.Name)
|
||||
})
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Example:")
|
||||
fmt.Fprintln(w, " audita process transcript.json --glossary glossary.yaml --output corrected.json")
|
||||
}
|
||||
4354
internal/cli/run_test.go
Normal file
4354
internal/cli/run_test.go
Normal file
File diff suppressed because it is too large
Load Diff
110
internal/cli/subprocess_test_hooks.go
Normal file
110
internal/cli/subprocess_test_hooks.go
Normal file
@@ -0,0 +1,110 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
)
|
||||
|
||||
const (
|
||||
subprocessTestLLMModeEnv = "AUDITA_SUBPROCESS_TEST_LLM_MODE"
|
||||
subprocessTestRunTimeoutMSEnv = "AUDITA_SUBPROCESS_TEST_RUN_TIMEOUT_MS"
|
||||
)
|
||||
|
||||
// ConfigureSubprocessTestHooksFromEnv enables deterministic test-only hooks for
|
||||
// subprocess integration tests that run through the Go test binary helper path.
|
||||
func ConfigureSubprocessTestHooksFromEnv() {
|
||||
mode := strings.TrimSpace(os.Getenv(subprocessTestLLMModeEnv))
|
||||
timeoutMSRaw := strings.TrimSpace(os.Getenv(subprocessTestRunTimeoutMSEnv))
|
||||
// Only activate in explicit subprocess test mode.
|
||||
if mode == "" && timeoutMSRaw == "" {
|
||||
return
|
||||
}
|
||||
|
||||
if mode != "" {
|
||||
client := &subprocessTestLLMClient{mode: mode}
|
||||
processProposalLLMClient = client
|
||||
processValidationLLMClient = client
|
||||
}
|
||||
|
||||
if timeoutMSRaw == "" {
|
||||
return
|
||||
}
|
||||
timeoutMS, err := strconv.Atoi(timeoutMSRaw)
|
||||
if err != nil || timeoutMS <= 0 {
|
||||
return
|
||||
}
|
||||
processRunnerContext = func() (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.Background(), time.Duration(timeoutMS)*time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
type subprocessTestLLMClient struct {
|
||||
mode string
|
||||
mu sync.Mutex
|
||||
proposals int
|
||||
}
|
||||
|
||||
func (c *subprocessTestLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
_ = req
|
||||
|
||||
switch c.mode {
|
||||
case "backend_error":
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("synthetic backend failure")
|
||||
case "block_until_cancel":
|
||||
<-ctx.Done()
|
||||
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
||||
case "malformed_structured":
|
||||
switch target := out.(type) {
|
||||
case *proposal_generation.StructuredCorrectionSet:
|
||||
*target = proposal_generation.StructuredCorrectionSet{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y", Confidence: 0.99},
|
||||
},
|
||||
}
|
||||
case *validators.LLMValidationResponse:
|
||||
*target = validators.LLMValidationResponse{
|
||||
Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 999, Approved: true, Confidence: 0.9, Reason: "bad index"},
|
||||
},
|
||||
}
|
||||
}
|
||||
case "mid_pipeline_fail":
|
||||
if _, ok := out.(*proposal_generation.StructuredCorrectionSet); ok {
|
||||
c.mu.Lock()
|
||||
c.proposals++
|
||||
proposalCall := c.proposals
|
||||
c.mu.Unlock()
|
||||
|
||||
if proposalCall >= 3 {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("synthetic mid-pipeline failure")
|
||||
}
|
||||
}
|
||||
switch target := out.(type) {
|
||||
case *proposal_generation.StructuredCorrectionSet:
|
||||
*target = proposal_generation.StructuredCorrectionSet{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99},
|
||||
},
|
||||
}
|
||||
case *validators.LLMValidationResponse:
|
||||
*target = validators.LLMValidationResponse{Validations: nil}
|
||||
}
|
||||
default:
|
||||
switch target := out.(type) {
|
||||
case *proposal_generation.StructuredCorrectionSet:
|
||||
*target = proposal_generation.StructuredCorrectionSet{Corrections: nil}
|
||||
case *validators.LLMValidationResponse:
|
||||
*target = validators.LLMValidationResponse{Validations: nil}
|
||||
}
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
}
|
||||
1
internal/cli/testdata/malformed_transcript.json
vendored
Normal file
1
internal/cli/testdata/malformed_transcript.json
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"segments":[{"id":1,"text":"oops"}
|
||||
22
internal/cli/testdata/parity/application-skip-ambiguous.case.json
vendored
Normal file
22
internal/cli/testdata/parity/application-skip-ambiguous.case.json
vendored
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"name": "ambiguous_match_rejected_before_application",
|
||||
"transcript_file": "application-skip-ambiguous.transcript.json",
|
||||
"glossary_file": "default-handoff.glossary.yaml",
|
||||
"modules_csv": "homophones",
|
||||
"proposal_responses_file": "application-skip-ambiguous.proposals.json",
|
||||
"validation_responses_file": "application-skip-ambiguous.validations.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "application-skip-ambiguous.expected-transcript.json",
|
||||
"module_instances": ["homophones"],
|
||||
"module_count": 1,
|
||||
"total_applied_changes": 0,
|
||||
"total_skipped_changes": 1,
|
||||
"module_applied_counts": [0],
|
||||
"module_rejected_counts": [1],
|
||||
"module_skip_counts": [0],
|
||||
"validator_rejected_reason_codes": ["ambiguous_original_text"],
|
||||
"expected_proposal_calls": ["homophones:proposal"]
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/application-skip-ambiguous.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/application-skip-ambiguous.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"the site near another site"}
|
||||
]
|
||||
3
internal/cli/testdata/parity/application-skip-ambiguous.proposals.json
vendored
Normal file
3
internal/cli/testdata/parity/application-skip-ambiguous.proposals.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]}
|
||||
]
|
||||
3
internal/cli/testdata/parity/application-skip-ambiguous.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/application-skip-ambiguous.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"the site near another site"}
|
||||
]
|
||||
4
internal/cli/testdata/parity/application-skip-ambiguous.validations.json
vendored
Normal file
4
internal/cli/testdata/parity/application-skip-ambiguous.validations.json
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
28
internal/cli/testdata/parity/default-full-pipeline.case.json
vendored
Normal file
28
internal/cli/testdata/parity/default-full-pipeline.case.json
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "default_full_pipeline_shape_and_reports",
|
||||
"transcript_file": "default-full-pipeline.transcript.json",
|
||||
"glossary_file": "default-full-pipeline.glossary.yaml",
|
||||
"proposal_responses_file": "default-full-pipeline.proposals.json",
|
||||
"validation_responses_file": "default-full-pipeline.validations.json",
|
||||
"env": {
|
||||
"AUDITA_LLM_API_KEY": "parity-secret",
|
||||
"AUDITA_VALIDATION_LLM_API_KEY": "parity-secret"
|
||||
},
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "default-full-pipeline.expected-transcript.json",
|
||||
"module_instances": ["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
|
||||
"module_count": 5,
|
||||
"total_applied_changes": 3,
|
||||
"total_skipped_changes": 3,
|
||||
"secret_markers": ["parity-secret"],
|
||||
"expected_proposal_calls": [
|
||||
"glossary_1:proposal",
|
||||
"homophones:proposal",
|
||||
"glossary_2:proposal",
|
||||
"spoken_word:proposal",
|
||||
"grammar:proposal"
|
||||
]
|
||||
}
|
||||
}
|
||||
9
internal/cli/testdata/parity/default-full-pipeline.expected-transcript.json
vendored
Normal file
9
internal/cli/testdata/parity/default-full-pipeline.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "Alice",
|
||||
"start": 0,
|
||||
"end": 1,
|
||||
"text": "Hello, there were Jesters hmm"
|
||||
}
|
||||
]
|
||||
7
internal/cli/testdata/parity/default-full-pipeline.glossary.yaml
vendored
Normal file
7
internal/cli/testdata/parity/default-full-pipeline.glossary.yaml
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
glossary:
|
||||
- name: Jesters
|
||||
aliases:
|
||||
- jester
|
||||
plural: jesters
|
||||
category: faction
|
||||
summary: A protected in-world faction term.
|
||||
28
internal/cli/testdata/parity/default-full-pipeline.proposals.json
vendored
Normal file
28
internal/cli/testdata/parity/default-full-pipeline.proposals.json
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
[
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "Jesters", "corrected_text": "jesters", "confidence": 0.99},
|
||||
{"id": 1, "original_text": "Jesters", "corrected_text": "JESTERX", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "jesters", "corrected_text": "JESTERS", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "uh", "corrected_text": "hmm", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
|
||||
]
|
||||
}
|
||||
]
|
||||
3
internal/cli/testdata/parity/default-full-pipeline.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/default-full-pipeline.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
|
||||
]
|
||||
12
internal/cli/testdata/parity/default-full-pipeline.validations.json
vendored
Normal file
12
internal/cli/testdata/parity/default-full-pipeline.validations.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject cleanup"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "parity-secret"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
27
internal/cli/testdata/parity/default-handoff.case.json
vendored
Normal file
27
internal/cli/testdata/parity/default-handoff.case.json
vendored
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"name": "default_pipeline_handoff_and_module_order",
|
||||
"transcript_file": "default-handoff.transcript.json",
|
||||
"glossary_file": "default-handoff.glossary.yaml",
|
||||
"proposal_responses_file": "default-handoff.proposals.json",
|
||||
"validation_responses_file": "default-handoff.validations.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "default-handoff.expected-transcript.json",
|
||||
"module_instances": ["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
|
||||
"module_count": 5,
|
||||
"total_applied_changes": 5,
|
||||
"total_skipped_changes": 0,
|
||||
"module_applied_counts": [1, 1, 1, 1, 1],
|
||||
"module_rejected_counts": [0, 0, 0, 0, 0],
|
||||
"module_skip_counts": [0, 0, 0, 0, 0],
|
||||
"expected_proposal_calls": [
|
||||
"glossary_1:proposal",
|
||||
"homophones:proposal",
|
||||
"glossary_2:proposal",
|
||||
"spoken_word:proposal",
|
||||
"grammar:proposal"
|
||||
],
|
||||
"min_response_payload_artifacts": 15
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/default-handoff.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/default-handoff.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"Hello, there were Jesters at the Sight um"}
|
||||
]
|
||||
6
internal/cli/testdata/parity/default-handoff.glossary.yaml
vendored
Normal file
6
internal/cli/testdata/parity/default-handoff.glossary.yaml
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
glossary:
|
||||
- name: Jesters
|
||||
aliases: [jester]
|
||||
plural: jesters
|
||||
category: faction
|
||||
summary: A protected in-world faction term.
|
||||
7
internal/cli/testdata/parity/default-handoff.proposals.json
vendored
Normal file
7
internal/cli/testdata/parity/default-handoff.proposals.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
[
|
||||
{"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "sight", "corrected_text": "Sight", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "um um", "corrected_text": "um", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}]}
|
||||
]
|
||||
3
internal/cli/testdata/parity/default-handoff.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/default-handoff.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures at the site um um"}
|
||||
]
|
||||
12
internal/cli/testdata/parity/default-handoff.validations.json
vendored
Normal file
12
internal/cli/testdata/parity/default-handoff.validations.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
17
internal/cli/testdata/parity/deterministic-validator-low-confidence.case.json
vendored
Normal file
17
internal/cli/testdata/parity/deterministic-validator-low-confidence.case.json
vendored
Normal file
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"name": "deterministic_validator_low_confidence",
|
||||
"transcript_file": "deterministic-validator-low-confidence.transcript.json",
|
||||
"glossary_file": "default-full-pipeline.glossary.yaml",
|
||||
"modules_csv": "grammar",
|
||||
"proposal_responses_file": "deterministic-validator-low-confidence.proposals.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "deterministic-validator-low-confidence.expected-transcript.json",
|
||||
"module_instances": ["grammar"],
|
||||
"module_count": 1,
|
||||
"total_skipped_changes": 1,
|
||||
"validator_rejected_reason_codes": ["low_confidence"],
|
||||
"expected_proposal_calls": ["grammar:proposal"]
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/deterministic-validator-low-confidence.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/deterministic-validator-low-confidence.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"hello , world"}
|
||||
]
|
||||
7
internal/cli/testdata/parity/deterministic-validator-low-confidence.proposals.json
vendored
Normal file
7
internal/cli/testdata/parity/deterministic-validator-low-confidence.proposals.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
[
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.1}
|
||||
]
|
||||
}
|
||||
]
|
||||
3
internal/cli/testdata/parity/deterministic-validator-low-confidence.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/deterministic-validator-low-confidence.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , world"}
|
||||
]
|
||||
12
internal/cli/testdata/parity/glossary-schema-error.case.json
vendored
Normal file
12
internal/cli/testdata/parity/glossary-schema-error.case.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"name": "glossary_schema_handling",
|
||||
"transcript_file": "default-full-pipeline.transcript.json",
|
||||
"glossary_file": "glossary-schema-error.glossary.yaml",
|
||||
"expect": {
|
||||
"exit_code": 1,
|
||||
"status": "failed",
|
||||
"error_phase": "glossary_schema",
|
||||
"stderr_contains": "glossary_schema",
|
||||
"require_error_log": true
|
||||
}
|
||||
}
|
||||
2
internal/cli/testdata/parity/glossary-schema-error.glossary.yaml
vendored
Normal file
2
internal/cli/testdata/parity/glossary-schema-error.glossary.yaml
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
glossary:
|
||||
- name: MissingCategoryAndSummary
|
||||
19
internal/cli/testdata/parity/llm-validator-rejection.case.json
vendored
Normal file
19
internal/cli/testdata/parity/llm-validator-rejection.case.json
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "llm_validator_decision_handling",
|
||||
"transcript_file": "llm-validator-rejection.transcript.json",
|
||||
"glossary_file": "default-full-pipeline.glossary.yaml",
|
||||
"modules_csv": "grammar",
|
||||
"proposal_responses_file": "llm-validator-rejection.proposals.json",
|
||||
"validation_responses_file": "llm-validator-rejection.validations.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "llm-validator-rejection.expected-transcript.json",
|
||||
"module_instances": ["grammar"],
|
||||
"module_count": 1,
|
||||
"total_skipped_changes": 1,
|
||||
"validator_rejected_reason_codes": ["llm_rejected"],
|
||||
"expected_proposal_calls": ["grammar:proposal"],
|
||||
"expected_validation_calls": ["grammar:section-0000:editorial_review:batch-0000"]
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/llm-validator-rejection.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/llm-validator-rejection.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"hello , world"}
|
||||
]
|
||||
7
internal/cli/testdata/parity/llm-validator-rejection.proposals.json
vendored
Normal file
7
internal/cli/testdata/parity/llm-validator-rejection.proposals.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
[
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
|
||||
]
|
||||
}
|
||||
]
|
||||
3
internal/cli/testdata/parity/llm-validator-rejection.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/llm-validator-rejection.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , world"}
|
||||
]
|
||||
7
internal/cli/testdata/parity/llm-validator-rejection.validations.json
vendored
Normal file
7
internal/cli/testdata/parity/llm-validator-rejection.validations.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
[
|
||||
{
|
||||
"validations": [
|
||||
{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject stylistic overreach"}
|
||||
]
|
||||
}
|
||||
]
|
||||
23
internal/cli/testdata/parity/mid-pipeline-failure.case.json
vendored
Normal file
23
internal/cli/testdata/parity/mid-pipeline-failure.case.json
vendored
Normal file
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "mid_pipeline_failure_partial_progress",
|
||||
"transcript_file": "default-handoff.transcript.json",
|
||||
"glossary_file": "default-handoff.glossary.yaml",
|
||||
"proposal_responses_file": "mid-pipeline-failure.proposals.json",
|
||||
"validation_responses_file": "mid-pipeline-failure.validations.json",
|
||||
"expect": {
|
||||
"exit_code": 1,
|
||||
"status": "failed",
|
||||
"error_phase": "runner_execution",
|
||||
"stderr_contains": "runner_execution",
|
||||
"module_instances": ["glossary_1", "homophones", "glossary_2"],
|
||||
"module_count": 3,
|
||||
"total_applied_changes": 2,
|
||||
"total_skipped_changes": 0,
|
||||
"failed_module_instance": "glossary_2",
|
||||
"module_applied_counts": [1, 1, 0],
|
||||
"module_rejected_counts": [0, 0, 0],
|
||||
"module_skip_counts": [0, 0, 0],
|
||||
"require_error_log": true,
|
||||
"expected_proposal_calls": ["glossary_1:proposal", "homophones:proposal", "glossary_2:proposal"]
|
||||
}
|
||||
}
|
||||
4
internal/cli/testdata/parity/mid-pipeline-failure.proposals.json
vendored
Normal file
4
internal/cli/testdata/parity/mid-pipeline-failure.proposals.json
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
[
|
||||
{"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]}
|
||||
]
|
||||
6
internal/cli/testdata/parity/mid-pipeline-failure.validations.json
vendored
Normal file
6
internal/cli/testdata/parity/mid-pipeline-failure.validations.json
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
21
internal/cli/testdata/parity/protected-term-rejection.case.json
vendored
Normal file
21
internal/cli/testdata/parity/protected-term-rejection.case.json
vendored
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "protected_glossary_term_behavior",
|
||||
"transcript_file": "protected-term-rejection.transcript.json",
|
||||
"glossary_file": "default-handoff.glossary.yaml",
|
||||
"modules_csv": "homophones",
|
||||
"proposal_responses_file": "protected-term-rejection.proposals.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "protected-term-rejection.expected-transcript.json",
|
||||
"module_instances": ["homophones"],
|
||||
"module_count": 1,
|
||||
"total_applied_changes": 0,
|
||||
"total_skipped_changes": 1,
|
||||
"module_applied_counts": [0],
|
||||
"module_rejected_counts": [1],
|
||||
"module_skip_counts": [0],
|
||||
"validator_rejected_reason_codes": ["protected_glossary_term"],
|
||||
"expected_proposal_calls": ["homophones:proposal"]
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/protected-term-rejection.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/protected-term-rejection.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"The Jesters entered the hall."}
|
||||
]
|
||||
3
internal/cli/testdata/parity/protected-term-rejection.proposals.json
vendored
Normal file
3
internal/cli/testdata/parity/protected-term-rejection.proposals.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"corrections": [{"id": 1, "original_text": "Jesters", "corrected_text": "Gestures", "confidence": 0.99}]}
|
||||
]
|
||||
3
internal/cli/testdata/parity/protected-term-rejection.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/protected-term-rejection.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"The Jesters entered the hall."}
|
||||
]
|
||||
12
internal/cli/testdata/parity/transcript-schema-error.case.json
vendored
Normal file
12
internal/cli/testdata/parity/transcript-schema-error.case.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"name": "transcript_schema_handling",
|
||||
"transcript_file": "transcript-schema-error.transcript.json",
|
||||
"glossary_file": "default-full-pipeline.glossary.yaml",
|
||||
"expect": {
|
||||
"exit_code": 1,
|
||||
"status": "failed",
|
||||
"error_phase": "transcript_schema",
|
||||
"stderr_contains": "transcript_schema",
|
||||
"require_error_log": true
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/transcript-schema-error.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/transcript-schema-error.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"","start":0.0,"end":1.0,"text":"bad"}
|
||||
]
|
||||
24
internal/cli/testdata/release/default-release.expectations.json
vendored
Normal file
24
internal/cli/testdata/release/default-release.expectations.json
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"must_apply_texts": [
|
||||
"Hello, there were Jesters hmm"
|
||||
],
|
||||
"must_not_apply_texts": [
|
||||
"JESTERX",
|
||||
"there were gestures"
|
||||
],
|
||||
"protected_terms": [
|
||||
"Jesters"
|
||||
],
|
||||
"expected_module_instances": [
|
||||
"glossary_1",
|
||||
"homophones",
|
||||
"glossary_2",
|
||||
"spoken_word",
|
||||
"grammar"
|
||||
],
|
||||
"minimum_counts": {
|
||||
"applied": 1,
|
||||
"rejected": 1,
|
||||
"skipped": 0
|
||||
}
|
||||
}
|
||||
9
internal/cli/testdata/release/default-release.expected-transcript.json
vendored
Normal file
9
internal/cli/testdata/release/default-release.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "Alice",
|
||||
"start": 0,
|
||||
"end": 1,
|
||||
"text": "Hello, there were Jesters hmm"
|
||||
}
|
||||
]
|
||||
7
internal/cli/testdata/release/default-release.glossary.yaml
vendored
Normal file
7
internal/cli/testdata/release/default-release.glossary.yaml
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
glossary:
|
||||
- name: Jesters
|
||||
aliases:
|
||||
- jester
|
||||
plural: jesters
|
||||
category: faction
|
||||
summary: A protected in-world faction term.
|
||||
28
internal/cli/testdata/release/default-release.proposals.json
vendored
Normal file
28
internal/cli/testdata/release/default-release.proposals.json
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
[
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "Jesters", "corrected_text": "jesters", "confidence": 0.99},
|
||||
{"id": 1, "original_text": "Jesters", "corrected_text": "JESTERX", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "jesters", "corrected_text": "JESTERS", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "uh", "corrected_text": "hmm", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
|
||||
]
|
||||
}
|
||||
]
|
||||
3
internal/cli/testdata/release/default-release.transcript.json
vendored
Normal file
3
internal/cli/testdata/release/default-release.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
|
||||
]
|
||||
12
internal/cli/testdata/release/default-release.validations.json
vendored
Normal file
12
internal/cli/testdata/release/default-release.validations.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject cleanup"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "release-secret"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
6
internal/cli/testdata/tiny_glossary.yaml
vendored
Normal file
6
internal/cli/testdata/tiny_glossary.yaml
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
glossary:
|
||||
- name: Audita
|
||||
aliases:
|
||||
- audita
|
||||
category: product
|
||||
summary: The Audita transcript correction CLI.
|
||||
9
internal/cli/testdata/tiny_transcript.json
vendored
Normal file
9
internal/cli/testdata/tiny_transcript.json
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "A",
|
||||
"start": 0.0,
|
||||
"end": 1.2,
|
||||
"text": "hello world"
|
||||
}
|
||||
]
|
||||
312
internal/core/chunking/sections.go
Normal file
312
internal/core/chunking/sections.go
Normal file
@@ -0,0 +1,312 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
// Section represents a contiguous chunk of transcript segments with metadata.
|
||||
type Section struct {
|
||||
// Index is the 0-based section index within the chunked transcript
|
||||
Index int `json:"section_index"`
|
||||
|
||||
// StartSegmentID is the ID of the first segment in this section
|
||||
StartSegmentID int `json:"start_segment_id"`
|
||||
|
||||
// EndSegmentID is the ID of the last segment in this section
|
||||
EndSegmentID int `json:"end_segment_id"`
|
||||
|
||||
// EstimatedTokens is the approximate token count for this section
|
||||
EstimatedTokens int `json:"estimated_tokens"`
|
||||
|
||||
// Segments contains the segments in this section, in order
|
||||
Segments []schema.Segment `json:"segments"`
|
||||
}
|
||||
|
||||
// ChunkingConfig holds configuration for transcript chunking.
|
||||
type ChunkingConfig struct {
|
||||
// MaxSectionTokens is the maximum allowed tokens per section
|
||||
MaxSectionTokens int
|
||||
|
||||
// MinSectionTokens is a validated soft lower-bound setting retained for
|
||||
// configuration/reporting compatibility.
|
||||
MinSectionTokens int
|
||||
|
||||
// TargetSections is an optional target number of sections
|
||||
// If nil, section count is derived from total/max token budgeting.
|
||||
TargetSections *int
|
||||
}
|
||||
|
||||
// Chunker performs deterministic chunking of normalized transcript segments.
|
||||
type Chunker struct {
|
||||
config ChunkingConfig
|
||||
estimator TokenEstimator
|
||||
}
|
||||
|
||||
// NewChunker creates a new chunker with the given configuration.
|
||||
func NewChunker(config ChunkingConfig) *Chunker {
|
||||
return &Chunker{
|
||||
config: config,
|
||||
estimator: NewSimpleTokenEstimator(),
|
||||
}
|
||||
}
|
||||
|
||||
// NewChunkerWithEstimator creates a new chunker with a custom estimator.
|
||||
func NewChunkerWithEstimator(config ChunkingConfig, estimator TokenEstimator) *Chunker {
|
||||
return &Chunker{
|
||||
config: config,
|
||||
estimator: estimator,
|
||||
}
|
||||
}
|
||||
|
||||
// ChunkTranscript divides a normalized transcript into contiguous token-bounded
|
||||
// sections using a deterministic balanced forward pass.
|
||||
//
|
||||
// Behavior:
|
||||
// - preserve segment order and never split segments;
|
||||
// - estimate per-segment tokens once, then compute total;
|
||||
// - derive desired section count from ceil(total/max_section_tokens), unless
|
||||
// target_sections is explicitly set;
|
||||
// - prefer section sizes near ceil(total/section_count) while never exceeding
|
||||
// max_section_tokens unless a section consists of a single oversized segment.
|
||||
//
|
||||
// Returns an error if explicit target_sections is impossible under constraints.
|
||||
// The input transcript is never mutated.
|
||||
func (c *Chunker) ChunkTranscript(transcript *schema.Transcript) ([]Section, error) {
|
||||
if transcript == nil || len(transcript.Segments) == 0 {
|
||||
return []Section{}, nil
|
||||
}
|
||||
|
||||
// Validate configuration
|
||||
if c.config.MaxSectionTokens <= 0 {
|
||||
return nil, fmt.Errorf("max_section_tokens must be positive, got %d", c.config.MaxSectionTokens)
|
||||
}
|
||||
|
||||
if c.config.MinSectionTokens < 0 {
|
||||
return nil, fmt.Errorf("min_section_tokens must be non-negative, got %d", c.config.MinSectionTokens)
|
||||
}
|
||||
|
||||
if c.config.MinSectionTokens > c.config.MaxSectionTokens {
|
||||
return nil, fmt.Errorf("min_section_tokens (%d) cannot exceed max_section_tokens (%d)",
|
||||
c.config.MinSectionTokens, c.config.MaxSectionTokens)
|
||||
}
|
||||
|
||||
// Calculate token counts for each segment (deterministic).
|
||||
segmentTokens := make([]int, len(transcript.Segments))
|
||||
totalTokens := 0
|
||||
for i, seg := range transcript.Segments {
|
||||
segmentTokens[i] = c.estimator.EstimateTokens(seg.Text)
|
||||
totalTokens += segmentTokens[i]
|
||||
}
|
||||
|
||||
minPossibleSections := c.calculateMinPossibleSections(segmentTokens)
|
||||
|
||||
var desiredSections int
|
||||
useExplicitTarget := false
|
||||
if c.config.TargetSections != nil {
|
||||
desiredSections = *c.config.TargetSections
|
||||
useExplicitTarget = true
|
||||
if desiredSections <= 0 {
|
||||
return nil, fmt.Errorf("target_sections must be positive, got %d", desiredSections)
|
||||
}
|
||||
if err := c.validateTargetSections(desiredSections, segmentTokens); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
desiredSections = ceilDiv(totalTokens, c.config.MaxSectionTokens)
|
||||
if desiredSections < minPossibleSections {
|
||||
desiredSections = minPossibleSections
|
||||
}
|
||||
if desiredSections < 1 {
|
||||
desiredSections = 1
|
||||
}
|
||||
if desiredSections > len(transcript.Segments) {
|
||||
desiredSections = len(transcript.Segments)
|
||||
}
|
||||
}
|
||||
|
||||
targetTokensPerSection := ceilDiv(totalTokens, desiredSections)
|
||||
if useExplicitTarget {
|
||||
return c.buildSectionsWithExplicitTarget(
|
||||
transcript.Segments,
|
||||
segmentTokens,
|
||||
desiredSections,
|
||||
targetTokensPerSection,
|
||||
)
|
||||
}
|
||||
|
||||
return c.buildSectionsBalanced(transcript.Segments, segmentTokens, targetTokensPerSection), nil
|
||||
}
|
||||
|
||||
// validateTargetSections checks if the target section count is achievable.
|
||||
func (c *Chunker) validateTargetSections(target int, segmentTokens []int) error {
|
||||
// Maximum possible sections: limited by segment count
|
||||
maxPossibleSections := len(segmentTokens)
|
||||
|
||||
if target > maxPossibleSections {
|
||||
return fmt.Errorf(
|
||||
"target_sections (%d) is impossible: cannot have more sections than segments (%d)",
|
||||
target, maxPossibleSections)
|
||||
}
|
||||
|
||||
// Minimum possible sections: each segment must fit within max bounds
|
||||
minPossibleSections := c.calculateMinPossibleSections(segmentTokens)
|
||||
|
||||
if target < minPossibleSections {
|
||||
return fmt.Errorf(
|
||||
"target_sections (%d) is impossible: need at least %d sections to respect max_section_tokens (%d)",
|
||||
target, minPossibleSections, c.config.MaxSectionTokens)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildSectionsBalanced creates sections with a deterministic single-pass policy.
|
||||
func (c *Chunker) buildSectionsBalanced(segments []schema.Segment, segmentTokens []int, targetTokensPerSection int) []Section {
|
||||
var sections []Section
|
||||
var currentSegments []schema.Segment
|
||||
currentTokens := 0
|
||||
|
||||
for i, seg := range segments {
|
||||
tokens := segmentTokens[i]
|
||||
|
||||
// Empty section: always accept the next segment, including oversized.
|
||||
if len(currentSegments) == 0 {
|
||||
currentSegments = append(currentSegments, seg)
|
||||
currentTokens = tokens
|
||||
continue
|
||||
}
|
||||
|
||||
// If adding next segment would exceed max, close current section.
|
||||
if currentTokens+tokens > c.config.MaxSectionTokens {
|
||||
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
|
||||
currentSegments = []schema.Segment{seg}
|
||||
currentTokens = tokens
|
||||
continue
|
||||
}
|
||||
|
||||
// Prefer staying near target tokens per section.
|
||||
if targetTokensPerSection == 0 || currentTokens < targetTokensPerSection {
|
||||
currentSegments = append(currentSegments, seg)
|
||||
currentTokens += tokens
|
||||
continue
|
||||
}
|
||||
|
||||
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
|
||||
currentSegments = []schema.Segment{seg}
|
||||
currentTokens = tokens
|
||||
}
|
||||
|
||||
if len(currentSegments) > 0 {
|
||||
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
|
||||
}
|
||||
|
||||
return sections
|
||||
}
|
||||
|
||||
// buildSection creates a Section from segments.
|
||||
func (c *Chunker) buildSection(index int, segments []schema.Segment, tokens int) Section {
|
||||
return Section{
|
||||
Index: index,
|
||||
StartSegmentID: segments[0].ID,
|
||||
EndSegmentID: segments[len(segments)-1].ID,
|
||||
EstimatedTokens: tokens,
|
||||
Segments: segments,
|
||||
}
|
||||
}
|
||||
|
||||
// calculateMinPossibleSections calculates the minimum number of sections needed
|
||||
// to ensure no section exceeds max tokens.
|
||||
func (c *Chunker) calculateMinPossibleSections(segmentTokens []int) int {
|
||||
sections := 0
|
||||
currentTokens := 0
|
||||
|
||||
for _, tokens := range segmentTokens {
|
||||
if tokens > c.config.MaxSectionTokens {
|
||||
// Each oversized segment needs its own section
|
||||
if currentTokens > 0 {
|
||||
sections++
|
||||
currentTokens = 0
|
||||
}
|
||||
sections++
|
||||
} else if currentTokens+tokens > c.config.MaxSectionTokens {
|
||||
sections++
|
||||
currentTokens = tokens
|
||||
} else {
|
||||
currentTokens += tokens
|
||||
}
|
||||
}
|
||||
|
||||
if currentTokens > 0 {
|
||||
sections++
|
||||
}
|
||||
|
||||
return sections
|
||||
}
|
||||
|
||||
// buildSectionsWithExplicitTarget builds exactly desiredSections when feasible.
|
||||
func (c *Chunker) buildSectionsWithExplicitTarget(
|
||||
segments []schema.Segment,
|
||||
segmentTokens []int,
|
||||
desiredSections int,
|
||||
targetTokensPerSection int,
|
||||
) ([]Section, error) {
|
||||
n := len(segments)
|
||||
cursor := 0
|
||||
sections := make([]Section, 0, desiredSections)
|
||||
|
||||
for sectionIdx := 0; sectionIdx < desiredSections; sectionIdx++ {
|
||||
if cursor >= n {
|
||||
break
|
||||
}
|
||||
|
||||
remainingSectionsAfter := desiredSections - sectionIdx - 1
|
||||
currentSegments := []schema.Segment{segments[cursor]}
|
||||
currentTokens := segmentTokens[cursor]
|
||||
cursor++
|
||||
|
||||
for cursor < n {
|
||||
remainingSegments := n - cursor
|
||||
|
||||
// Reserve one segment per future section to avoid empty sections.
|
||||
if remainingSegments == remainingSectionsAfter {
|
||||
break
|
||||
}
|
||||
|
||||
nextTokens := segmentTokens[cursor]
|
||||
if currentTokens+nextTokens > c.config.MaxSectionTokens {
|
||||
break
|
||||
}
|
||||
if targetTokensPerSection == 0 || currentTokens < targetTokensPerSection {
|
||||
currentSegments = append(currentSegments, segments[cursor])
|
||||
currentTokens += nextTokens
|
||||
cursor++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
|
||||
}
|
||||
|
||||
if cursor != n || len(sections) != desiredSections {
|
||||
return nil, fmt.Errorf(
|
||||
"target_sections (%d) is impossible under current constraints (got %d sections)",
|
||||
desiredSections,
|
||||
len(sections),
|
||||
)
|
||||
}
|
||||
|
||||
return sections, nil
|
||||
}
|
||||
|
||||
func ceilDiv(numerator int, denominator int) int {
|
||||
if denominator <= 0 {
|
||||
return 0
|
||||
}
|
||||
if numerator <= 0 {
|
||||
return 0
|
||||
}
|
||||
return (numerator + denominator - 1) / denominator
|
||||
}
|
||||
518
internal/core/chunking/sections_test.go
Normal file
518
internal/core/chunking/sections_test.go
Normal file
@@ -0,0 +1,518 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
// mapTokenEstimator provides deterministic per-segment token counts for tests.
|
||||
type mapTokenEstimator struct {
|
||||
byText map[string]int
|
||||
}
|
||||
|
||||
func (e *mapTokenEstimator) EstimateTokens(text string) int {
|
||||
if e.byText == nil {
|
||||
return 0
|
||||
}
|
||||
if tokens, ok := e.byText[text]; ok {
|
||||
return tokens
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func makeSegments(texts []string) []schema.Segment {
|
||||
segments := make([]schema.Segment, len(texts))
|
||||
for i, text := range texts {
|
||||
segments[i] = schema.Segment{
|
||||
ID: i + 1,
|
||||
Speaker: "DM",
|
||||
Start: float64(i * 10),
|
||||
End: float64(i*10 + 5),
|
||||
Text: text,
|
||||
}
|
||||
}
|
||||
return segments
|
||||
}
|
||||
|
||||
func makeTranscript(segments []schema.Segment) *schema.Transcript {
|
||||
return &schema.Transcript{Segments: segments}
|
||||
}
|
||||
|
||||
func intPtr(i int) *int {
|
||||
return &i
|
||||
}
|
||||
|
||||
func assertSegmentCoverageAndOrder(t *testing.T, input []schema.Segment, sections []Section) {
|
||||
t.Helper()
|
||||
|
||||
seen := make([]schema.Segment, 0, len(input))
|
||||
for _, sec := range sections {
|
||||
seen = append(seen, sec.Segments...)
|
||||
}
|
||||
|
||||
if len(seen) != len(input) {
|
||||
t.Fatalf("expected %d total segment occurrences, got %d", len(input), len(seen))
|
||||
}
|
||||
|
||||
for i := range input {
|
||||
if seen[i].ID != input[i].ID {
|
||||
t.Fatalf("segment order mismatch at index %d: got id=%d want id=%d", i, seen[i].ID, input[i].ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertSectionMetadataConsistent(t *testing.T, sections []Section) {
|
||||
t.Helper()
|
||||
|
||||
for i, sec := range sections {
|
||||
if sec.Index != i {
|
||||
t.Fatalf("section %d: expected index=%d got=%d", i, i, sec.Index)
|
||||
}
|
||||
if len(sec.Segments) == 0 {
|
||||
t.Fatalf("section %d: section must not be empty", i)
|
||||
}
|
||||
if sec.StartSegmentID != sec.Segments[0].ID {
|
||||
t.Fatalf("section %d: start_segment_id mismatch", i)
|
||||
}
|
||||
if sec.EndSegmentID != sec.Segments[len(sec.Segments)-1].ID {
|
||||
t.Fatalf("section %d: end_segment_id mismatch", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertMaxBoundExceptSingletonOversized(t *testing.T, sections []Section, max int) {
|
||||
t.Helper()
|
||||
|
||||
for i, sec := range sections {
|
||||
if sec.EstimatedTokens <= max {
|
||||
continue
|
||||
}
|
||||
if len(sec.Segments) != 1 {
|
||||
t.Fatalf("section %d exceeds max tokens (%d>%d) with %d segments", i, sec.EstimatedTokens, max, len(sec.Segments))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func imbalance(sections []Section) int {
|
||||
if len(sections) == 0 {
|
||||
return 0
|
||||
}
|
||||
minTokens := sections[0].EstimatedTokens
|
||||
maxTokens := sections[0].EstimatedTokens
|
||||
for _, sec := range sections {
|
||||
if sec.EstimatedTokens < minTokens {
|
||||
minTokens = sec.EstimatedTokens
|
||||
}
|
||||
if sec.EstimatedTokens > maxTokens {
|
||||
maxTokens = sec.EstimatedTokens
|
||||
}
|
||||
}
|
||||
return maxTokens - minTokens
|
||||
}
|
||||
|
||||
func greedyMaxFillSections(segments []schema.Segment, tokens []int, max int) []Section {
|
||||
sections := make([]Section, 0)
|
||||
var current []schema.Segment
|
||||
currentTokens := 0
|
||||
|
||||
for i, seg := range segments {
|
||||
tok := tokens[i]
|
||||
if len(current) == 0 {
|
||||
current = append(current, seg)
|
||||
currentTokens = tok
|
||||
continue
|
||||
}
|
||||
if currentTokens+tok > max {
|
||||
sections = append(sections, Section{
|
||||
Index: len(sections),
|
||||
StartSegmentID: current[0].ID,
|
||||
EndSegmentID: current[len(current)-1].ID,
|
||||
EstimatedTokens: currentTokens,
|
||||
Segments: append([]schema.Segment(nil), current...),
|
||||
})
|
||||
current = []schema.Segment{seg}
|
||||
currentTokens = tok
|
||||
continue
|
||||
}
|
||||
current = append(current, seg)
|
||||
currentTokens += tok
|
||||
}
|
||||
|
||||
if len(current) > 0 {
|
||||
sections = append(sections, Section{
|
||||
Index: len(sections),
|
||||
StartSegmentID: current[0].ID,
|
||||
EndSegmentID: current[len(current)-1].ID,
|
||||
EstimatedTokens: currentTokens,
|
||||
Segments: append([]schema.Segment(nil), current...),
|
||||
})
|
||||
}
|
||||
|
||||
return sections
|
||||
}
|
||||
|
||||
func TestChunkEmptyTranscript(t *testing.T) {
|
||||
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 10})
|
||||
|
||||
sections, err := chunker.ChunkTranscript(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript(nil): %v", err)
|
||||
}
|
||||
if len(sections) != 0 {
|
||||
t.Fatalf("expected 0 sections for nil transcript, got %d", len(sections))
|
||||
}
|
||||
|
||||
sections, err = chunker.ChunkTranscript(makeTranscript(nil))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript(empty): %v", err)
|
||||
}
|
||||
if len(sections) != 0 {
|
||||
t.Fatalf("expected 0 sections for empty transcript, got %d", len(sections))
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkSingleSegment(t *testing.T) {
|
||||
segments := makeSegments([]string{"s1"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 10},
|
||||
&mapTokenEstimator{byText: map[string]int{"s1": 7}},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("expected 1 section, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 7 {
|
||||
t.Fatalf("expected estimated_tokens=7, got %d", sections[0].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertSectionMetadataConsistent(t, sections)
|
||||
}
|
||||
|
||||
func TestChunkSingleOversizedSegment(t *testing.T) {
|
||||
segments := makeSegments([]string{"big"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 5},
|
||||
&mapTokenEstimator{byText: map[string]int{"big": 120}},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("expected 1 section, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 120 {
|
||||
t.Fatalf("expected oversized singleton section, got %d", sections[0].EstimatedTokens)
|
||||
}
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 50)
|
||||
}
|
||||
|
||||
func TestChunkTotalBelowMaxSingleSection(t *testing.T) {
|
||||
segments := makeSegments([]string{"a", "b", "c"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 5},
|
||||
&ConstTokenEstimator{Tokens: 10},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("expected 1 section, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 30 {
|
||||
t.Fatalf("expected 30 section tokens, got %d", sections[0].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
}
|
||||
|
||||
func TestChunkTotalExactlyDivisibleByMax(t *testing.T) {
|
||||
segments := makeSegments([]string{"a", "b", "c", "d"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1},
|
||||
&ConstTokenEstimator{Tokens: 5},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 2 {
|
||||
t.Fatalf("expected 2 sections, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 10 || sections[1].EstimatedTokens != 10 {
|
||||
t.Fatalf("expected [10,10] tokens, got [%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 10)
|
||||
}
|
||||
|
||||
func TestChunkTotalNotDivisibleByMax(t *testing.T) {
|
||||
segments := makeSegments([]string{"a", "b", "c", "d", "e"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1},
|
||||
&ConstTokenEstimator{Tokens: 5},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 3 {
|
||||
t.Fatalf("expected 3 sections, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 10 || sections[1].EstimatedTokens != 10 || sections[2].EstimatedTokens != 5 {
|
||||
t.Fatalf("expected [10,10,5] tokens, got [%d,%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens, sections[2].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 10)
|
||||
}
|
||||
|
||||
func TestChunkTargetSectionsPrecedenceAndSuccess(t *testing.T) {
|
||||
segments := makeSegments([]string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10"})
|
||||
target := 3
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 200, MinSectionTokens: 1, TargetSections: &target},
|
||||
&ConstTokenEstimator{Tokens: 10},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != target {
|
||||
t.Fatalf("expected %d sections from explicit target, got %d", target, len(sections))
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertSectionMetadataConsistent(t, sections)
|
||||
}
|
||||
|
||||
func TestChunkTargetSectionsImpossibleTooMany(t *testing.T) {
|
||||
segments := makeSegments([]string{"1", "2", "3", "4", "5"})
|
||||
target := 10
|
||||
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 1, TargetSections: &target})
|
||||
|
||||
_, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err == nil {
|
||||
t.Fatal("expected error for impossible target_sections")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "cannot have more sections than segments") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkTargetSectionsImpossibleTooFew(t *testing.T) {
|
||||
segments := makeSegments([]string{"1", "2", "3"})
|
||||
target := 1
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1, TargetSections: &target},
|
||||
&ConstTokenEstimator{Tokens: 30},
|
||||
)
|
||||
|
||||
_, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err == nil {
|
||||
t.Fatal("expected error for impossible target_sections")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "need at least 3 sections") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkManySmallSegmentsBalanced(t *testing.T) {
|
||||
texts := []string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"}
|
||||
segments := makeSegments(texts)
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1},
|
||||
&ConstTokenEstimator{Tokens: 10},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 3 {
|
||||
t.Fatalf("expected 3 sections, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 40 || sections[1].EstimatedTokens != 40 || sections[2].EstimatedTokens != 30 {
|
||||
t.Fatalf("expected [40,40,30], got [%d,%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens, sections[2].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 50)
|
||||
}
|
||||
|
||||
func TestChunkMixedLargeAndSmallSegments(t *testing.T) {
|
||||
segments := makeSegments([]string{"big1", "s1", "s2", "s3", "big2", "s4"})
|
||||
estimator := &mapTokenEstimator{byText: map[string]int{
|
||||
"big1": 120,
|
||||
"s1": 10,
|
||||
"s2": 10,
|
||||
"s3": 10,
|
||||
"big2": 120,
|
||||
"s4": 10,
|
||||
}}
|
||||
chunker := NewChunkerWithEstimator(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 1}, estimator)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 4 {
|
||||
t.Fatalf("expected 4 sections, got %d", len(sections))
|
||||
}
|
||||
if len(sections[0].Segments) != 1 || sections[0].Segments[0].Text != "big1" {
|
||||
t.Fatalf("expected first oversized segment in singleton section, got %+v", sections[0].Segments)
|
||||
}
|
||||
if len(sections[2].Segments) != 1 || sections[2].Segments[0].Text != "big2" {
|
||||
t.Fatalf("expected second oversized segment in singleton section, got %+v", sections[2].Segments)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 100)
|
||||
}
|
||||
|
||||
func TestChunkDeterministicOrdering(t *testing.T) {
|
||||
segments := makeSegments([]string{"a", "b", "c", "d", "e", "f"})
|
||||
chunkerCfg := ChunkingConfig{MaxSectionTokens: 15, MinSectionTokens: 1}
|
||||
estimator := &ConstTokenEstimator{Tokens: 5}
|
||||
|
||||
var first []Section
|
||||
for i := 0; i < 5; i++ {
|
||||
chunker := NewChunkerWithEstimator(chunkerCfg, estimator)
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("iteration %d: %v", i, err)
|
||||
}
|
||||
if i == 0 {
|
||||
first = sections
|
||||
continue
|
||||
}
|
||||
if len(sections) != len(first) {
|
||||
t.Fatalf("iteration %d: section count mismatch (%d vs %d)", i, len(sections), len(first))
|
||||
}
|
||||
for j := range sections {
|
||||
if sections[j].Index != first[j].Index ||
|
||||
sections[j].StartSegmentID != first[j].StartSegmentID ||
|
||||
sections[j].EndSegmentID != first[j].EndSegmentID ||
|
||||
sections[j].EstimatedTokens != first[j].EstimatedTokens ||
|
||||
len(sections[j].Segments) != len(first[j].Segments) {
|
||||
t.Fatalf("iteration %d section %d mismatch", i, j)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkNoMutationOfInput(t *testing.T) {
|
||||
segments := makeSegments([]string{"original one", "original two"})
|
||||
transcript := makeTranscript(segments)
|
||||
original := make([]string, len(transcript.Segments))
|
||||
for i := range transcript.Segments {
|
||||
original[i] = transcript.Segments[i].Text
|
||||
}
|
||||
|
||||
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1})
|
||||
if _, err := chunker.ChunkTranscript(transcript); err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
for i := range transcript.Segments {
|
||||
if transcript.Segments[i].Text != original[i] {
|
||||
t.Fatalf("segment %d mutated", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkConfigValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config ChunkingConfig
|
||||
errContains string
|
||||
}{
|
||||
{
|
||||
name: "zero max tokens",
|
||||
config: ChunkingConfig{MaxSectionTokens: 0, MinSectionTokens: 1},
|
||||
errContains: "max_section_tokens must be positive",
|
||||
},
|
||||
{
|
||||
name: "negative max tokens",
|
||||
config: ChunkingConfig{MaxSectionTokens: -1, MinSectionTokens: 1},
|
||||
errContains: "max_section_tokens must be positive",
|
||||
},
|
||||
{
|
||||
name: "negative min tokens",
|
||||
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: -1},
|
||||
errContains: "min_section_tokens must be non-negative",
|
||||
},
|
||||
{
|
||||
name: "min exceeds max",
|
||||
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 11},
|
||||
errContains: "min_section_tokens (11) cannot exceed max_section_tokens (10)",
|
||||
},
|
||||
{
|
||||
name: "zero target sections",
|
||||
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1, TargetSections: intPtr(0)},
|
||||
errContains: "target_sections must be positive",
|
||||
},
|
||||
{
|
||||
name: "negative target sections",
|
||||
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1, TargetSections: intPtr(-1)},
|
||||
errContains: "target_sections must be positive",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
chunker := NewChunker(tt.config)
|
||||
_, err := chunker.ChunkTranscript(makeTranscript(makeSegments([]string{"x"})))
|
||||
if err == nil {
|
||||
t.Fatalf("expected error containing %q", tt.errContains)
|
||||
}
|
||||
if !strings.Contains(err.Error(), tt.errContains) {
|
||||
t.Fatalf("expected error containing %q, got %q", tt.errContains, err.Error())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkBalancedAlgorithmBeatsGreedyMaxFillOnUnevenTranscript(t *testing.T) {
|
||||
segments := makeSegments([]string{"s1", "s2", "s3", "s4", "s5", "s6"})
|
||||
tokenMap := map[string]int{
|
||||
"s1": 50,
|
||||
"s2": 10,
|
||||
"s3": 10,
|
||||
"s4": 10,
|
||||
"s5": 10,
|
||||
"s6": 10,
|
||||
}
|
||||
estimator := &mapTokenEstimator{byText: tokenMap}
|
||||
chunker := NewChunkerWithEstimator(ChunkingConfig{MaxSectionTokens: 80, MinSectionTokens: 1}, estimator)
|
||||
|
||||
balancedSections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
|
||||
tokens := make([]int, 0, len(segments))
|
||||
for _, seg := range segments {
|
||||
tokens = append(tokens, tokenMap[seg.Text])
|
||||
}
|
||||
greedySections := greedyMaxFillSections(segments, tokens, 80)
|
||||
|
||||
balancedImbalance := imbalance(balancedSections)
|
||||
greedyImbalance := imbalance(greedySections)
|
||||
if balancedImbalance >= greedyImbalance {
|
||||
t.Fatalf(
|
||||
"expected balanced chunking to improve over greedy max-fill; balanced=%d greedy=%d",
|
||||
balancedImbalance,
|
||||
greedyImbalance,
|
||||
)
|
||||
}
|
||||
|
||||
assertSegmentCoverageAndOrder(t, segments, balancedSections)
|
||||
assertMaxBoundExceptSingletonOversized(t, balancedSections, 80)
|
||||
}
|
||||
84
internal/core/chunking/summary.go
Normal file
84
internal/core/chunking/summary.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package chunking
|
||||
|
||||
// Summary provides a concise overview of chunking results for reports
|
||||
type Summary struct {
|
||||
ChunkCount int `json:"chunk_count"`
|
||||
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
|
||||
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
|
||||
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
|
||||
TargetSections *int `json:"target_sections,omitempty"`
|
||||
MaxSectionTokens int `json:"max_section_tokens"`
|
||||
MinSectionTokens int `json:"min_section_tokens"`
|
||||
}
|
||||
|
||||
// ChunkSummary represents a single chunk's metadata for diagnostics
|
||||
type ChunkSummary struct {
|
||||
Index int `json:"index"`
|
||||
StartSegmentID int `json:"start_segment_id"`
|
||||
EndSegmentID int `json:"end_segment_id"`
|
||||
EstimatedTokens int `json:"estimated_tokens"`
|
||||
SegmentCount int `json:"segment_count"`
|
||||
}
|
||||
|
||||
// DetailedSummary provides per-chunk details for diagnostics
|
||||
type DetailedSummary struct {
|
||||
Summary `json:",inline"`
|
||||
Chunks []ChunkSummary `json:"chunks"`
|
||||
}
|
||||
|
||||
// ComputeSummary creates a Summary from sections and config
|
||||
func ComputeSummary(sections []Section, config ChunkingConfig) Summary {
|
||||
if len(sections) == 0 {
|
||||
return Summary{
|
||||
ChunkCount: 0,
|
||||
MaxSectionTokens: config.MaxSectionTokens,
|
||||
MinSectionTokens: config.MinSectionTokens,
|
||||
TargetSections: config.TargetSections,
|
||||
}
|
||||
}
|
||||
|
||||
minTokens := sections[0].EstimatedTokens
|
||||
maxTokens := sections[0].EstimatedTokens
|
||||
totalTokens := 0
|
||||
|
||||
for _, sec := range sections {
|
||||
if sec.EstimatedTokens < minTokens {
|
||||
minTokens = sec.EstimatedTokens
|
||||
}
|
||||
if sec.EstimatedTokens > maxTokens {
|
||||
maxTokens = sec.EstimatedTokens
|
||||
}
|
||||
totalTokens += sec.EstimatedTokens
|
||||
}
|
||||
|
||||
return Summary{
|
||||
ChunkCount: len(sections),
|
||||
MinEstimatedTokens: minTokens,
|
||||
MaxEstimatedTokens: maxTokens,
|
||||
TotalEstimatedTokens: totalTokens,
|
||||
TargetSections: config.TargetSections,
|
||||
MaxSectionTokens: config.MaxSectionTokens,
|
||||
MinSectionTokens: config.MinSectionTokens,
|
||||
}
|
||||
}
|
||||
|
||||
// ComputeDetailedSummary creates a DetailedSummary from sections and config
|
||||
func ComputeDetailedSummary(sections []Section, config ChunkingConfig) DetailedSummary {
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
chunks := make([]ChunkSummary, len(sections))
|
||||
for i, sec := range sections {
|
||||
chunks[i] = ChunkSummary{
|
||||
Index: sec.Index,
|
||||
StartSegmentID: sec.StartSegmentID,
|
||||
EndSegmentID: sec.EndSegmentID,
|
||||
EstimatedTokens: sec.EstimatedTokens,
|
||||
SegmentCount: len(sec.Segments),
|
||||
}
|
||||
}
|
||||
|
||||
return DetailedSummary{
|
||||
Summary: summary,
|
||||
Chunks: chunks,
|
||||
}
|
||||
}
|
||||
148
internal/core/chunking/summary_test.go
Normal file
148
internal/core/chunking/summary_test.go
Normal file
@@ -0,0 +1,148 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
func TestComputeSummary(t *testing.T) {
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
}
|
||||
|
||||
sections := []Section{
|
||||
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
|
||||
{Index: 1, EstimatedTokens: 50, StartSegmentID: 3, EndSegmentID: 4},
|
||||
{Index: 2, EstimatedTokens: 20, StartSegmentID: 5, EndSegmentID: 5},
|
||||
}
|
||||
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
if summary.ChunkCount != 3 {
|
||||
t.Errorf("expected chunk_count=3, got %d", summary.ChunkCount)
|
||||
}
|
||||
if summary.MinEstimatedTokens != 20 {
|
||||
t.Errorf("expected min_estimated_tokens=20, got %d", summary.MinEstimatedTokens)
|
||||
}
|
||||
if summary.MaxEstimatedTokens != 50 {
|
||||
t.Errorf("expected max_estimated_tokens=50, got %d", summary.MaxEstimatedTokens)
|
||||
}
|
||||
if summary.TotalEstimatedTokens != 100 {
|
||||
t.Errorf("expected total_estimated_tokens=100, got %d", summary.TotalEstimatedTokens)
|
||||
}
|
||||
if summary.MaxSectionTokens != 100 {
|
||||
t.Errorf("expected max_section_tokens=100, got %d", summary.MaxSectionTokens)
|
||||
}
|
||||
if summary.MinSectionTokens != 10 {
|
||||
t.Errorf("expected min_section_tokens=10, got %d", summary.MinSectionTokens)
|
||||
}
|
||||
if summary.TargetSections != nil {
|
||||
t.Errorf("expected target_sections=nil, got %v", summary.TargetSections)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeSummaryWithTarget(t *testing.T) {
|
||||
target := 5
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
TargetSections: &target,
|
||||
}
|
||||
|
||||
sections := []Section{
|
||||
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
|
||||
}
|
||||
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
if summary.TargetSections == nil || *summary.TargetSections != 5 {
|
||||
t.Errorf("expected target_sections=5, got %v", summary.TargetSections)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeSummaryEmptySections(t *testing.T) {
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
}
|
||||
|
||||
sections := []Section{}
|
||||
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
if summary.ChunkCount != 0 {
|
||||
t.Errorf("expected chunk_count=0, got %d", summary.ChunkCount)
|
||||
}
|
||||
if summary.MinEstimatedTokens != 0 {
|
||||
t.Errorf("expected min_estimated_tokens=0 for empty, got %d", summary.MinEstimatedTokens)
|
||||
}
|
||||
if summary.MaxSectionTokens != 100 {
|
||||
t.Errorf("expected max_section_tokens preserved, got %d", summary.MaxSectionTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeDetailedSummary(t *testing.T) {
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
}
|
||||
|
||||
sections := []Section{
|
||||
{
|
||||
Index: 0,
|
||||
EstimatedTokens: 30,
|
||||
StartSegmentID: 1,
|
||||
EndSegmentID: 2,
|
||||
Segments: make([]schema.Segment, 2), // 2 segments
|
||||
},
|
||||
{
|
||||
Index: 1,
|
||||
EstimatedTokens: 50,
|
||||
StartSegmentID: 3,
|
||||
EndSegmentID: 5,
|
||||
Segments: make([]schema.Segment, 3), // 3 segments
|
||||
},
|
||||
}
|
||||
|
||||
detailed := ComputeDetailedSummary(sections, config)
|
||||
|
||||
if detailed.ChunkCount != 2 {
|
||||
t.Errorf("expected chunk_count=2, got %d", detailed.ChunkCount)
|
||||
}
|
||||
if len(detailed.Chunks) != 2 {
|
||||
t.Fatalf("expected 2 chunk entries, got %d", len(detailed.Chunks))
|
||||
}
|
||||
|
||||
// Check first chunk
|
||||
if detailed.Chunks[0].Index != 0 {
|
||||
t.Errorf("expected chunk[0].index=0, got %d", detailed.Chunks[0].Index)
|
||||
}
|
||||
if detailed.Chunks[0].StartSegmentID != 1 {
|
||||
t.Errorf("expected chunk[0].start_segment_id=1, got %d", detailed.Chunks[0].StartSegmentID)
|
||||
}
|
||||
if detailed.Chunks[0].EndSegmentID != 2 {
|
||||
t.Errorf("expected chunk[0].end_segment_id=2, got %d", detailed.Chunks[0].EndSegmentID)
|
||||
}
|
||||
if detailed.Chunks[0].EstimatedTokens != 30 {
|
||||
t.Errorf("expected chunk[0].estimated_tokens=30, got %d", detailed.Chunks[0].EstimatedTokens)
|
||||
}
|
||||
if detailed.Chunks[0].SegmentCount != 2 {
|
||||
t.Errorf("expected chunk[0].segment_count=2, got %d", detailed.Chunks[0].SegmentCount)
|
||||
}
|
||||
|
||||
// Check second chunk
|
||||
if detailed.Chunks[1].Index != 1 {
|
||||
t.Errorf("expected chunk[1].index=1, got %d", detailed.Chunks[1].Index)
|
||||
}
|
||||
if detailed.Chunks[1].StartSegmentID != 3 {
|
||||
t.Errorf("expected chunk[1].start_segment_id=3, got %d", detailed.Chunks[1].StartSegmentID)
|
||||
}
|
||||
if detailed.Chunks[1].EndSegmentID != 5 {
|
||||
t.Errorf("expected chunk[1].end_segment_id=5, got %d", detailed.Chunks[1].EndSegmentID)
|
||||
}
|
||||
if detailed.Chunks[1].SegmentCount != 3 {
|
||||
t.Errorf("expected chunk[1].segment_count=3, got %d", detailed.Chunks[1].SegmentCount)
|
||||
}
|
||||
}
|
||||
55
internal/core/chunking/tokens.go
Normal file
55
internal/core/chunking/tokens.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// TokenEstimator provides a deterministic token estimation suitable for prompt budgeting.
|
||||
// The estimator is approximate but stable, isolated, and replaceable.
|
||||
type TokenEstimator interface {
|
||||
EstimateTokens(text string) int
|
||||
}
|
||||
|
||||
// SimpleTokenEstimator provides a basic deterministic token estimation.
|
||||
// This uses a simple heuristic based on word count and punctuation.
|
||||
type SimpleTokenEstimator struct{}
|
||||
|
||||
// NewSimpleTokenEstimator creates a new simple token estimator.
|
||||
func NewSimpleTokenEstimator() *SimpleTokenEstimator {
|
||||
return &SimpleTokenEstimator{}
|
||||
}
|
||||
|
||||
// EstimateTokens provides a rough estimate of the number of tokens in the given text.
|
||||
// This implementation uses a simple heuristic: count words and punctuation as tokens.
|
||||
// The estimate is deterministic and stable for the same input text.
|
||||
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
|
||||
if text == "" {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Simple heuristic: split on whitespace and count non-empty segments
|
||||
words := strings.Fields(text)
|
||||
tokenCount := len(words)
|
||||
|
||||
// Add some estimate for punctuation that might be separate tokens
|
||||
punctuationCount := 0
|
||||
for _, r := range text {
|
||||
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
|
||||
punctuationCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Rough estimate: each word is a token, plus half the punctuation as separate tokens
|
||||
return tokenCount + (punctuationCount / 2)
|
||||
}
|
||||
|
||||
// ConstTokenEstimator returns a constant token count for testing purposes.
|
||||
type ConstTokenEstimator struct {
|
||||
Tokens int
|
||||
}
|
||||
|
||||
// EstimateTokens returns the configured constant token count.
|
||||
func (e *ConstTokenEstimator) EstimateTokens(text string) int {
|
||||
return e.Tokens
|
||||
}
|
||||
59
internal/core/chunking/tokens_test.go
Normal file
59
internal/core/chunking/tokens_test.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSimpleTokenEstimator(t *testing.T) {
|
||||
estimator := NewSimpleTokenEstimator()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
text string
|
||||
expected int
|
||||
}{
|
||||
{"empty string", "", 0},
|
||||
{"single word", "hello", 1},
|
||||
{"two words", "hello world", 2},
|
||||
{"with punctuation", "hello, world!", 3}, // 2 words + 2 punctuation/2 = 3
|
||||
{"multiple sentences", "Hello world. This is a test.", 7}, // 7 words + 2 punctuation/2 = 8? Actually "Hello world." has 3 punctuation
|
||||
{"with apostrophes", "don't won't can't", 3},
|
||||
{"with hyphens", "well-known state-of-the-art", 2}, // hyphens don't count
|
||||
{"unicode text", "café naïve", 2},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := estimator.EstimateTokens(tt.text)
|
||||
if got != tt.expected {
|
||||
t.Errorf("EstimateTokens(%q) = %d, want %d", tt.text, got, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConstTokenEstimator(t *testing.T) {
|
||||
estimator := &ConstTokenEstimator{Tokens: 42}
|
||||
|
||||
if got := estimator.EstimateTokens("any text"); got != 42 {
|
||||
t.Errorf("ConstTokenEstimator.EstimateTokens = %d, want 42", got)
|
||||
}
|
||||
|
||||
if got := estimator.EstimateTokens(""); got != 42 {
|
||||
t.Errorf("ConstTokenEstimator.EstimateTokens(empty) = %d, want 42", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokenEstimatorDeterminism(t *testing.T) {
|
||||
estimator := NewSimpleTokenEstimator()
|
||||
text := "The quick brown fox jumps over the lazy dog. Hello, world!"
|
||||
|
||||
// Run multiple times and verify same result
|
||||
first := estimator.EstimateTokens(text)
|
||||
for i := 0; i < 10; i++ {
|
||||
got := estimator.EstimateTokens(text)
|
||||
if got != first {
|
||||
t.Errorf("EstimateTokens not deterministic: iteration %d got %d, first was %d", i, got, first)
|
||||
}
|
||||
}
|
||||
}
|
||||
199
internal/core/config/config.go
Normal file
199
internal/core/config/config.go
Normal file
@@ -0,0 +1,199 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||
)
|
||||
|
||||
type WorkDirRetention string
|
||||
|
||||
const (
|
||||
WorkDirRetentionAuto WorkDirRetention = "auto"
|
||||
WorkDirRetentionAlways WorkDirRetention = "always"
|
||||
WorkDirRetentionNever WorkDirRetention = "never"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultModulesCSV = modulecatalog.KeyGlossary + "," + modulecatalog.KeyHomophones + "," + modulecatalog.KeyGlossary + "," + modulecatalog.KeySpokenWord + "," + modulecatalog.KeyGrammar
|
||||
DefaultOutputSchema = "bare-segments"
|
||||
DefaultPrimaryModel = "openrouter/google/gemma-4-31b-it"
|
||||
DefaultPrimaryBaseURL = "https://openrouter.ai/api/v1"
|
||||
DefaultPrimaryLLMTimeoutSeconds = 600
|
||||
DefaultMaxRetries = 3
|
||||
DefaultLLMConcurrency = 1
|
||||
DefaultValidationMaxPromptTokens = 2048
|
||||
DefaultMaxSectionTokens = 8192
|
||||
DefaultMinSectionTokens = 2048
|
||||
DefaultConfidenceThreshold = 0.8
|
||||
DefaultNormalizeMaxSegmentGap = 4.0
|
||||
DefaultNormalizeEllipsisGap = 3.5
|
||||
DefaultNormalizeMaxSegmentDuration = 60.0
|
||||
DefaultNormalizeMaxSegmentTokens = 2048
|
||||
DefaultTranscriptDescriptionMaxChars = 500
|
||||
DefaultWorkDir = "/tmp/audita"
|
||||
DefaultWorkDirRetention WorkDirRetention = WorkDirRetentionAuto
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Modules []string
|
||||
OutputSchema string
|
||||
PrimaryLLM LLMConfig
|
||||
ValidationLLM ValidationLLMConfig
|
||||
TotalLLMConcurrency int
|
||||
ProposalLLMConcurrency int
|
||||
ValidationLLMConcurrency *int
|
||||
ValidationMaxPromptTokens int
|
||||
MaxSectionTokens int
|
||||
MinSectionTokens int
|
||||
TargetSections *int
|
||||
Thresholds ConfidenceThresholds
|
||||
Normalization NormalizationConfig
|
||||
TranscriptDescription string
|
||||
WorkDir string
|
||||
WorkDirRetention WorkDirRetention
|
||||
}
|
||||
|
||||
type LLMConfig struct {
|
||||
APIKey string
|
||||
Model string
|
||||
BaseURL string
|
||||
TimeoutSeconds int
|
||||
MaxRetries int
|
||||
// Concurrency is retained as a backward-compatible alias for
|
||||
// TotalLLMConcurrency.
|
||||
Concurrency int
|
||||
}
|
||||
|
||||
type ValidationLLMConfig struct {
|
||||
APIKey string
|
||||
Model string
|
||||
BaseURL string
|
||||
TimeoutSeconds *int
|
||||
MaxRetries *int
|
||||
// Concurrency is retained as a backward-compatible alias for
|
||||
// ValidationLLMConcurrency.
|
||||
Concurrency *int
|
||||
}
|
||||
|
||||
type ConfidenceThresholds struct {
|
||||
Glossary float64
|
||||
Grammar float64
|
||||
Homophones float64
|
||||
SpokenWord float64
|
||||
}
|
||||
|
||||
type NormalizationConfig struct {
|
||||
MaxSegmentGap float64
|
||||
EllipsisGap float64
|
||||
MaxSegmentDuration float64
|
||||
MaxSegmentTokens int
|
||||
}
|
||||
|
||||
func Default() Config {
|
||||
modules, _ := ParseModulesCSV(DefaultModulesCSV)
|
||||
|
||||
return Config{
|
||||
Modules: modules,
|
||||
OutputSchema: DefaultOutputSchema,
|
||||
PrimaryLLM: LLMConfig{
|
||||
Model: DefaultPrimaryModel,
|
||||
BaseURL: DefaultPrimaryBaseURL,
|
||||
TimeoutSeconds: DefaultPrimaryLLMTimeoutSeconds,
|
||||
MaxRetries: DefaultMaxRetries,
|
||||
Concurrency: DefaultLLMConcurrency,
|
||||
},
|
||||
ValidationLLM: ValidationLLMConfig{},
|
||||
TotalLLMConcurrency: DefaultLLMConcurrency,
|
||||
ProposalLLMConcurrency: DefaultLLMConcurrency,
|
||||
ValidationLLMConcurrency: nil,
|
||||
ValidationMaxPromptTokens: DefaultValidationMaxPromptTokens,
|
||||
MaxSectionTokens: DefaultMaxSectionTokens,
|
||||
MinSectionTokens: DefaultMinSectionTokens,
|
||||
TargetSections: nil,
|
||||
Thresholds: ConfidenceThresholds{
|
||||
Glossary: DefaultConfidenceThreshold,
|
||||
Grammar: DefaultConfidenceThreshold,
|
||||
Homophones: DefaultConfidenceThreshold,
|
||||
SpokenWord: DefaultConfidenceThreshold,
|
||||
},
|
||||
Normalization: NormalizationConfig{
|
||||
MaxSegmentGap: DefaultNormalizeMaxSegmentGap,
|
||||
EllipsisGap: DefaultNormalizeEllipsisGap,
|
||||
MaxSegmentDuration: DefaultNormalizeMaxSegmentDuration,
|
||||
MaxSegmentTokens: DefaultNormalizeMaxSegmentTokens,
|
||||
},
|
||||
WorkDir: DefaultWorkDir,
|
||||
WorkDirRetention: DefaultWorkDirRetention,
|
||||
}
|
||||
}
|
||||
|
||||
func ParseModulesCSV(raw string) ([]string, error) {
|
||||
parts := strings.Split(raw, ",")
|
||||
modules := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
trimmed := strings.TrimSpace(part)
|
||||
if trimmed == "" {
|
||||
return nil, fmt.Errorf("modules list contains an empty value")
|
||||
}
|
||||
modules = append(modules, trimmed)
|
||||
}
|
||||
if len(modules) == 0 {
|
||||
return nil, fmt.Errorf("modules list must not be empty")
|
||||
}
|
||||
return modules, nil
|
||||
}
|
||||
|
||||
func (c Config) EffectiveValidationLLMConfig() LLMConfig {
|
||||
effective := c.PrimaryLLM
|
||||
|
||||
if c.ValidationLLM.APIKey != "" {
|
||||
effective.APIKey = c.ValidationLLM.APIKey
|
||||
}
|
||||
if c.ValidationLLM.Model != "" {
|
||||
effective.Model = c.ValidationLLM.Model
|
||||
}
|
||||
if c.ValidationLLM.BaseURL != "" {
|
||||
effective.BaseURL = c.ValidationLLM.BaseURL
|
||||
}
|
||||
if c.ValidationLLM.TimeoutSeconds != nil {
|
||||
effective.TimeoutSeconds = *c.ValidationLLM.TimeoutSeconds
|
||||
}
|
||||
if c.ValidationLLM.MaxRetries != nil {
|
||||
effective.MaxRetries = *c.ValidationLLM.MaxRetries
|
||||
}
|
||||
effective.Concurrency = c.EffectiveValidationLLMConcurrency()
|
||||
|
||||
return effective
|
||||
}
|
||||
|
||||
func (c Config) EffectiveValidationLLMConcurrency() int {
|
||||
if c.ValidationLLMConcurrency != nil {
|
||||
return *c.ValidationLLMConcurrency
|
||||
}
|
||||
return c.TotalLLMConcurrency
|
||||
}
|
||||
|
||||
func (c Config) EffectiveProposalLLMConcurrency() int {
|
||||
if c.ProposalLLMConcurrency > 0 {
|
||||
return c.ProposalLLMConcurrency
|
||||
}
|
||||
return c.TotalLLMConcurrency
|
||||
}
|
||||
|
||||
func (c *Config) syncLegacyConcurrencyAliases() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.PrimaryLLM.Concurrency = c.TotalLLMConcurrency
|
||||
c.ValidationLLM.Concurrency = intPtr(c.ValidationLLMConcurrency)
|
||||
}
|
||||
|
||||
func intPtr(v *int) *int {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
x := *v
|
||||
return &x
|
||||
}
|
||||
458
internal/core/config/config_test.go
Normal file
458
internal/core/config/config_test.go
Normal file
@@ -0,0 +1,458 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
|
||||
)
|
||||
|
||||
func TestDefaultConfigValues(t *testing.T) {
|
||||
cfg := Default()
|
||||
|
||||
if got, want := strings.Join(cfg.Modules, ","), DefaultModulesCSV; got != want {
|
||||
t.Fatalf("modules mismatch: got %q want %q", got, want)
|
||||
}
|
||||
if cfg.OutputSchema != DefaultOutputSchema {
|
||||
t.Fatalf("unexpected default output schema: %q", cfg.OutputSchema)
|
||||
}
|
||||
if cfg.PrimaryLLM.Model != DefaultPrimaryModel {
|
||||
t.Fatalf("unexpected default primary model: %q", cfg.PrimaryLLM.Model)
|
||||
}
|
||||
if cfg.PrimaryLLM.BaseURL != DefaultPrimaryBaseURL {
|
||||
t.Fatalf("unexpected default primary base url: %q", cfg.PrimaryLLM.BaseURL)
|
||||
}
|
||||
if cfg.PrimaryLLM.TimeoutSeconds != DefaultPrimaryLLMTimeoutSeconds {
|
||||
t.Fatalf("unexpected default timeout seconds: %d", cfg.PrimaryLLM.TimeoutSeconds)
|
||||
}
|
||||
if cfg.PrimaryLLM.MaxRetries != DefaultMaxRetries {
|
||||
t.Fatalf("unexpected default max retries: %d", cfg.PrimaryLLM.MaxRetries)
|
||||
}
|
||||
if cfg.TotalLLMConcurrency != DefaultLLMConcurrency {
|
||||
t.Fatalf("unexpected default total llm concurrency: %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != DefaultLLMConcurrency {
|
||||
t.Fatalf("unexpected default proposal llm concurrency: %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
if cfg.ValidationLLMConcurrency != nil {
|
||||
t.Fatalf("expected validation llm concurrency to be unset by default")
|
||||
}
|
||||
if cfg.PrimaryLLM.Concurrency != cfg.TotalLLMConcurrency {
|
||||
t.Fatalf("expected primary llm concurrency alias to mirror total, got primary=%d total=%d", cfg.PrimaryLLM.Concurrency, cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ValidationLLM.TimeoutSeconds != nil {
|
||||
t.Fatalf("expected validation timeout to be unset by default")
|
||||
}
|
||||
if cfg.ValidationLLM.MaxRetries != nil {
|
||||
t.Fatalf("expected validation max retries to be unset by default")
|
||||
}
|
||||
if cfg.ValidationLLM.Concurrency != nil {
|
||||
t.Fatalf("expected legacy validation llm concurrency alias to be unset by default")
|
||||
}
|
||||
if cfg.TargetSections != nil {
|
||||
t.Fatalf("expected target sections to be unset by default")
|
||||
}
|
||||
if cfg.WorkDir != DefaultWorkDir {
|
||||
t.Fatalf("unexpected default work dir: %q", cfg.WorkDir)
|
||||
}
|
||||
if cfg.TranscriptDescription != "" {
|
||||
t.Fatalf("expected default transcript description to be empty, got %q", cfg.TranscriptDescription)
|
||||
}
|
||||
if cfg.WorkDirRetention != DefaultWorkDirRetention {
|
||||
t.Fatalf("unexpected default work dir retention: %q", cfg.WorkDirRetention)
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("default config should validate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromEnvOverridesAndFallback(t *testing.T) {
|
||||
env := map[string]string{
|
||||
"AUDITA_MODEL": "openai/gpt-4.1-mini",
|
||||
"AUDITA_BASE_URL": "https://api.openai.com/v1",
|
||||
"AUDITA_LLM_TIMEOUT_SECONDS": "120",
|
||||
"AUDITA_MAX_RETRIES": "7",
|
||||
"AUDITA_TOTAL_LLM_CONCURRENCY": "6",
|
||||
"AUDITA_PROPOSAL_LLM_CONCURRENCY": "4",
|
||||
"AUDITA_VALIDATION_LLM_CONCURRENCY": "2",
|
||||
"AUDITA_VALIDATION_MAX_PROMPT_TOKENS": "4096",
|
||||
"AUDITA_MAX_SECTION_TOKENS": "9000",
|
||||
"AUDITA_MIN_SECTION_TOKENS": "3000",
|
||||
"AUDITA_TARGET_SECTIONS": "5",
|
||||
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.9",
|
||||
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.7",
|
||||
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.6",
|
||||
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.5",
|
||||
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "2.5",
|
||||
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "2.0",
|
||||
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "30.0",
|
||||
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "1024",
|
||||
"AUDITA_WORK_DIR": "/var/tmp/audita",
|
||||
"AUDITA_WORK_DIR_RETENTION": "always",
|
||||
"OPENROUTER_API_KEY": "fallback-key",
|
||||
}
|
||||
|
||||
cfg, err := loadFromLookup(mapLookup(env))
|
||||
if err != nil {
|
||||
t.Fatalf("loadFromLookup returned error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.PrimaryLLM.APIKey != "fallback-key" {
|
||||
t.Fatalf("expected OPENROUTER_API_KEY fallback, got %q", cfg.PrimaryLLM.APIKey)
|
||||
}
|
||||
if cfg.PrimaryLLM.Model != env["AUDITA_MODEL"] {
|
||||
t.Fatalf("unexpected model: %q", cfg.PrimaryLLM.Model)
|
||||
}
|
||||
if cfg.PrimaryLLM.BaseURL != env["AUDITA_BASE_URL"] {
|
||||
t.Fatalf("unexpected base url: %q", cfg.PrimaryLLM.BaseURL)
|
||||
}
|
||||
if cfg.TargetSections == nil || *cfg.TargetSections != 5 {
|
||||
t.Fatalf("unexpected target sections: %#v", cfg.TargetSections)
|
||||
}
|
||||
if cfg.TotalLLMConcurrency != 6 {
|
||||
t.Fatalf("unexpected total llm concurrency: %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 4 {
|
||||
t.Fatalf("unexpected proposal llm concurrency: %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
if cfg.ValidationLLMConcurrency == nil || *cfg.ValidationLLMConcurrency != 2 {
|
||||
t.Fatalf("unexpected validation llm concurrency: %#v", cfg.ValidationLLMConcurrency)
|
||||
}
|
||||
if cfg.PrimaryLLM.Concurrency != 6 {
|
||||
t.Fatalf("expected primary alias concurrency 6, got %d", cfg.PrimaryLLM.Concurrency)
|
||||
}
|
||||
if cfg.ValidationLLM.Concurrency == nil || *cfg.ValidationLLM.Concurrency != 2 {
|
||||
t.Fatalf("expected validation alias concurrency 2, got %#v", cfg.ValidationLLM.Concurrency)
|
||||
}
|
||||
if cfg.WorkDirRetention != WorkDirRetentionAlways {
|
||||
t.Fatalf("unexpected work dir retention: %q", cfg.WorkDirRetention)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromEnvLegacyLLMConcurrencyAliasForTotalAndProposal(t *testing.T) {
|
||||
env := map[string]string{
|
||||
"AUDITA_LLM_CONCURRENCY": "5",
|
||||
}
|
||||
|
||||
cfg, err := loadFromLookup(mapLookup(env))
|
||||
if err != nil {
|
||||
t.Fatalf("loadFromLookup returned error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.TotalLLMConcurrency != 5 {
|
||||
t.Fatalf("expected total concurrency from legacy alias, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 5 {
|
||||
t.Fatalf("expected proposal concurrency to inherit legacy total, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromEnvCanonicalTotalWinsLegacyAlias(t *testing.T) {
|
||||
env := map[string]string{
|
||||
"AUDITA_TOTAL_LLM_CONCURRENCY": "4",
|
||||
"AUDITA_LLM_CONCURRENCY": "9",
|
||||
}
|
||||
|
||||
cfg, err := loadFromLookup(mapLookup(env))
|
||||
if err != nil {
|
||||
t.Fatalf("loadFromLookup returned error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.TotalLLMConcurrency != 4 {
|
||||
t.Fatalf("expected canonical total to win over legacy alias, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 4 {
|
||||
t.Fatalf("expected proposal to inherit canonical total when unset, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromEnvUsesAuditaLLMAPIKeyOverFallback(t *testing.T) {
|
||||
env := map[string]string{
|
||||
"AUDITA_LLM_API_KEY": "primary-key",
|
||||
"OPENROUTER_API_KEY": "fallback-key",
|
||||
}
|
||||
|
||||
cfg, err := loadFromLookup(mapLookup(env))
|
||||
if err != nil {
|
||||
t.Fatalf("loadFromLookup returned error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.PrimaryLLM.APIKey != "primary-key" {
|
||||
t.Fatalf("expected AUDITA_LLM_API_KEY to win, got %q", cfg.PrimaryLLM.APIKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCLIOverridesPrecedence(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.PrimaryLLM.Model = "env-model"
|
||||
cfg.WorkDir = "/env/work"
|
||||
|
||||
model := "cli-model"
|
||||
workDir := "/cli/work"
|
||||
modules := "grammar"
|
||||
outputSchema := "audita-v1"
|
||||
totalLLMConcurrency := 5
|
||||
proposalLLMConcurrency := 3
|
||||
overrides := CLIOverrides{
|
||||
PrimaryModel: &model,
|
||||
WorkDir: &workDir,
|
||||
ModulesCSV: &modules,
|
||||
OutputSchema: &outputSchema,
|
||||
TotalLLMConcurrency: &totalLLMConcurrency,
|
||||
ProposalLLMConcurrency: &proposalLLMConcurrency,
|
||||
}
|
||||
|
||||
if err := cfg.ApplyCLIOverrides(overrides); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
|
||||
if cfg.PrimaryLLM.Model != "cli-model" {
|
||||
t.Fatalf("expected CLI model override, got %q", cfg.PrimaryLLM.Model)
|
||||
}
|
||||
if cfg.WorkDir != "/cli/work" {
|
||||
t.Fatalf("expected CLI work dir override, got %q", cfg.WorkDir)
|
||||
}
|
||||
if !reflect.DeepEqual(cfg.Modules, []string{"grammar"}) {
|
||||
t.Fatalf("unexpected modules: %#v", cfg.Modules)
|
||||
}
|
||||
if cfg.OutputSchema != "audita-v1" {
|
||||
t.Fatalf("expected CLI output schema override, got %q", cfg.OutputSchema)
|
||||
}
|
||||
if cfg.TotalLLMConcurrency != 5 {
|
||||
t.Fatalf("expected CLI total concurrency override, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 3 {
|
||||
t.Fatalf("expected CLI proposal concurrency override, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCLIOverridesTrimsTranscriptDescription(t *testing.T) {
|
||||
cfg := Default()
|
||||
description := " background context about speakers "
|
||||
if err := cfg.ApplyCLIOverrides(CLIOverrides{TranscriptDescription: &description}); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
if cfg.TranscriptDescription != "background context about speakers" {
|
||||
t.Fatalf("unexpected transcript description trim result: %q", cfg.TranscriptDescription)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationRejectsOverlyLongTranscriptDescription(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.TranscriptDescription = strings.Repeat("a", DefaultTranscriptDescriptionMaxChars+1)
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatalf("expected transcript description length validation error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "transcript description must be 500 characters or fewer") {
|
||||
t.Fatalf("unexpected validation error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCLIOverridesLegacyLLMConcurrencyAlias(t *testing.T) {
|
||||
cfg := Default()
|
||||
aliasConcurrency := 6
|
||||
|
||||
if err := cfg.ApplyCLIOverrides(CLIOverrides{PrimaryLLMConcurrency: &aliasConcurrency}); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
|
||||
if cfg.TotalLLMConcurrency != 6 {
|
||||
t.Fatalf("expected legacy --llm-concurrency alias to set total, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 6 {
|
||||
t.Fatalf("expected proposal to inherit aliased total when unset, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCLIOverridesCanonicalTotalWinsLegacyAlias(t *testing.T) {
|
||||
cfg := Default()
|
||||
canonicalTotal := 4
|
||||
legacyAlias := 9
|
||||
|
||||
if err := cfg.ApplyCLIOverrides(CLIOverrides{TotalLLMConcurrency: &canonicalTotal, PrimaryLLMConcurrency: &legacyAlias}); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
|
||||
if cfg.TotalLLMConcurrency != 4 {
|
||||
t.Fatalf("expected canonical total concurrency to win, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 4 {
|
||||
t.Fatalf("expected proposal to inherit canonical total when proposal is unset, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationFailures(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.OutputSchema = "unknown-schema"
|
||||
cfg.PrimaryLLM.TimeoutSeconds = -1
|
||||
cfg.TotalLLMConcurrency = 0
|
||||
cfg.ProposalLLMConcurrency = 0
|
||||
validationConcurrency := 5
|
||||
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||
cfg.ValidationMaxPromptTokens = 0
|
||||
cfg.MaxSectionTokens = 100
|
||||
cfg.MinSectionTokens = 200
|
||||
cfg.Thresholds.Grammar = 1.5
|
||||
cfg.WorkDirRetention = WorkDirRetention("sometimes")
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatalf("expected validation error")
|
||||
}
|
||||
|
||||
message := err.Error()
|
||||
for _, expected := range []string{
|
||||
"primary llm timeout seconds",
|
||||
"total llm concurrency",
|
||||
"proposal llm concurrency",
|
||||
"validation llm concurrency must be less than or equal to total llm concurrency",
|
||||
"validation max prompt tokens",
|
||||
"min section tokens",
|
||||
"grammar confidence threshold",
|
||||
"work dir retention",
|
||||
"unsupported output schema",
|
||||
} {
|
||||
if !strings.Contains(message, expected) {
|
||||
t.Fatalf("expected error to contain %q, got %q", expected, message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationRejectsUnsupportedModuleKey(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.Modules = []string{modulecatalog.KeyGlossary, "made_up"}
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatalf("expected validation error for unsupported module key")
|
||||
}
|
||||
if !strings.Contains(err.Error(), `unsupported module key "made_up"`) {
|
||||
t.Fatalf("expected unsupported module key error, got %q", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationAllowsRepeatedSupportedModuleKeys(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.Modules = []string{modulecatalog.KeyGlossary, modulecatalog.KeyGlossary, modulecatalog.KeyGrammar}
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("expected repeated supported module keys to validate, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationAcceptsAllSupportedOutputSchemas(t *testing.T) {
|
||||
for _, schemaKey := range outputschema.SupportedKeys() {
|
||||
cfg := Default()
|
||||
cfg.OutputSchema = schemaKey
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("expected output schema %q to validate, got %v", schemaKey, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveValidationLLMInheritance(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.PrimaryLLM.APIKey = "primary-key"
|
||||
cfg.PrimaryLLM.Model = "primary-model"
|
||||
cfg.PrimaryLLM.BaseURL = "https://primary.example/v1"
|
||||
cfg.PrimaryLLM.TimeoutSeconds = 111
|
||||
cfg.PrimaryLLM.MaxRetries = 2
|
||||
cfg.TotalLLMConcurrency = 7
|
||||
cfg.syncLegacyConcurrencyAliases()
|
||||
|
||||
effective := cfg.EffectiveValidationLLMConfig()
|
||||
if effective.APIKey != "primary-key" || effective.Model != "primary-model" || effective.BaseURL != "https://primary.example/v1" || effective.TimeoutSeconds != 111 || effective.MaxRetries != 2 || effective.Concurrency != 7 {
|
||||
t.Fatalf("unexpected inherited config: %#v", effective)
|
||||
}
|
||||
|
||||
validationTimeout := 222
|
||||
validationRetries := 9
|
||||
cfg.ValidationLLM.APIKey = "validation-key"
|
||||
cfg.ValidationLLM.Model = "validation-model"
|
||||
cfg.ValidationLLM.BaseURL = "https://validation.example/v1"
|
||||
cfg.ValidationLLM.TimeoutSeconds = &validationTimeout
|
||||
cfg.ValidationLLM.MaxRetries = &validationRetries
|
||||
validationConcurrency := 4
|
||||
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||
cfg.syncLegacyConcurrencyAliases()
|
||||
|
||||
effective = cfg.EffectiveValidationLLMConfig()
|
||||
if effective.APIKey != "validation-key" || effective.Model != "validation-model" || effective.BaseURL != "https://validation.example/v1" || effective.TimeoutSeconds != 222 || effective.MaxRetries != 9 || effective.Concurrency != 4 {
|
||||
t.Fatalf("unexpected overridden validation config: %#v", effective)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationLLMConcurrencyCannotExceedTotal(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.TotalLLMConcurrency = 2
|
||||
cfg.ProposalLLMConcurrency = 2
|
||||
validationConcurrency := 3
|
||||
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Fatal("expected validation error when validation llm concurrency exceeds total")
|
||||
}
|
||||
|
||||
validationConcurrency = 2
|
||||
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("expected equal concurrency to validate, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProposalLLMConcurrencyCannotExceedTotal(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.TotalLLMConcurrency = 2
|
||||
cfg.ProposalLLMConcurrency = 3
|
||||
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Fatal("expected validation error when proposal llm concurrency exceeds total")
|
||||
}
|
||||
|
||||
cfg.ProposalLLMConcurrency = 2
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("expected equal concurrency to validate, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLITotalLLMConcurrencyOverrideDrivesEffectiveValidationConcurrencyWhenValidationUnset(t *testing.T) {
|
||||
cfg := Default()
|
||||
totalLLMConcurrency := 6
|
||||
if err := cfg.ApplyCLIOverrides(CLIOverrides{TotalLLMConcurrency: &totalLLMConcurrency}); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
if cfg.ValidationLLMConcurrency != nil {
|
||||
t.Fatalf("expected validation concurrency to remain unset, got %#v", cfg.ValidationLLMConcurrency)
|
||||
}
|
||||
if cfg.EffectiveValidationLLMConcurrency() != 6 {
|
||||
t.Fatalf("expected inherited validation concurrency 6, got %d", cfg.EffectiveValidationLLMConcurrency())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactedConfig(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.PrimaryLLM.APIKey = "secret-primary"
|
||||
cfg.ValidationLLM.APIKey = "secret-validation"
|
||||
|
||||
redacted := cfg.Redacted()
|
||||
|
||||
if redacted.PrimaryLLM.APIKey != redactedSecret {
|
||||
t.Fatalf("expected primary api key to be redacted, got %q", redacted.PrimaryLLM.APIKey)
|
||||
}
|
||||
if redacted.ValidationLLM.APIKey != redactedSecret {
|
||||
t.Fatalf("expected validation api key to be redacted, got %q", redacted.ValidationLLM.APIKey)
|
||||
}
|
||||
if cfg.PrimaryLLM.APIKey != "secret-primary" {
|
||||
t.Fatalf("redaction should not mutate original config")
|
||||
}
|
||||
}
|
||||
|
||||
func mapLookup(values map[string]string) func(string) (string, bool) {
|
||||
return func(key string) (string, bool) {
|
||||
value, ok := values[key]
|
||||
return value, ok
|
||||
}
|
||||
}
|
||||
119
internal/core/config/effective_config.go
Normal file
119
internal/core/config/effective_config.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type EffectiveConfigErrorKind string
|
||||
|
||||
const (
|
||||
EffectiveConfigErrorResolvePath EffectiveConfigErrorKind = "resolve_path"
|
||||
EffectiveConfigErrorLoadFile EffectiveConfigErrorKind = "load_file"
|
||||
EffectiveConfigErrorApplyFile EffectiveConfigErrorKind = "apply_file"
|
||||
EffectiveConfigErrorApplyEnv EffectiveConfigErrorKind = "apply_env"
|
||||
)
|
||||
|
||||
type EffectiveConfigError struct {
|
||||
Kind EffectiveConfigErrorKind
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *EffectiveConfigError) Error() string {
|
||||
if e == nil || e.Err == nil {
|
||||
return ""
|
||||
}
|
||||
return e.Err.Error()
|
||||
}
|
||||
|
||||
func (e *EffectiveConfigError) Unwrap() error {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return e.Err
|
||||
}
|
||||
|
||||
type EffectiveConfig struct {
|
||||
Config Config
|
||||
ConfigPath string
|
||||
ConfigSource string
|
||||
ConfigVersion *int
|
||||
}
|
||||
|
||||
func ResolveConfigPath(cliConfigPath string, cliConfigPathSet bool) (path string, source string, err error) {
|
||||
return resolveConfigPathWithLookup(cliConfigPath, cliConfigPathSet, os.LookupEnv, os.Stat, DefaultConfigSearchPaths)
|
||||
}
|
||||
|
||||
func LoadEffectiveConfig(cliConfigPath string, cliConfigPathSet bool) (EffectiveConfig, error) {
|
||||
return loadEffectiveConfigWithLookup(cliConfigPath, cliConfigPathSet, os.LookupEnv, os.Stat, DefaultConfigSearchPaths)
|
||||
}
|
||||
|
||||
func loadEffectiveConfigWithLookup(cliConfigPath string, cliConfigPathSet bool, lookup func(string) (string, bool), statPath func(string) (os.FileInfo, error), defaultSearchPaths []string) (EffectiveConfig, error) {
|
||||
configPath, configSource, err := resolveConfigPathWithLookup(cliConfigPath, cliConfigPathSet, lookup, statPath, defaultSearchPaths)
|
||||
if err != nil {
|
||||
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorResolvePath, Err: err}
|
||||
}
|
||||
|
||||
cfg := Default()
|
||||
var configVersion *int
|
||||
if configPath != "" {
|
||||
fileCfg, fileErr := LoadFileConfig(configPath)
|
||||
if fileErr != nil {
|
||||
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorLoadFile, Err: fileErr}
|
||||
}
|
||||
if applyErr := cfg.ApplyFileConfig(fileCfg); applyErr != nil {
|
||||
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorApplyFile, Err: applyErr}
|
||||
}
|
||||
configVersion = &fileCfg.Version
|
||||
}
|
||||
if applyEnvErr := cfg.applyEnvOverrides(lookup); applyEnvErr != nil {
|
||||
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorApplyEnv, Err: applyEnvErr}
|
||||
}
|
||||
|
||||
return EffectiveConfig{
|
||||
Config: cfg,
|
||||
ConfigPath: configPath,
|
||||
ConfigSource: configSource,
|
||||
ConfigVersion: configVersion,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func resolveConfigPathWithLookup(cliConfigPath string, cliConfigPathSet bool, lookup func(string) (string, bool), statPath func(string) (os.FileInfo, error), defaultSearchPaths []string) (path string, source string, err error) {
|
||||
if cliConfigPathSet {
|
||||
path = strings.TrimSpace(cliConfigPath)
|
||||
if path == "" {
|
||||
return "", "", fmt.Errorf("--config requires a non-empty path")
|
||||
}
|
||||
if _, statErr := statPath(path); statErr != nil {
|
||||
if os.IsNotExist(statErr) {
|
||||
return "", "", fmt.Errorf("config file not found: %s", path)
|
||||
}
|
||||
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
|
||||
}
|
||||
return path, "flag", nil
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_CONFIG"); ok {
|
||||
path = strings.TrimSpace(raw)
|
||||
if path == "" {
|
||||
return "", "", fmt.Errorf("AUDITA_CONFIG must not be empty")
|
||||
}
|
||||
if _, statErr := statPath(path); statErr != nil {
|
||||
if os.IsNotExist(statErr) {
|
||||
return "", "", fmt.Errorf("config file not found: %s", path)
|
||||
}
|
||||
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
|
||||
}
|
||||
return path, "env", nil
|
||||
}
|
||||
|
||||
for _, defaultPath := range defaultSearchPaths {
|
||||
if _, statErr := statPath(defaultPath); statErr == nil {
|
||||
return defaultPath, "default", nil
|
||||
} else if !os.IsNotExist(statErr) {
|
||||
return "", "", fmt.Errorf("cannot access config file %s: %w", defaultPath, statErr)
|
||||
}
|
||||
}
|
||||
return "", "", nil
|
||||
}
|
||||
163
internal/core/config/effective_config_test.go
Normal file
163
internal/core/config/effective_config_test.go
Normal file
@@ -0,0 +1,163 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestResolveConfigPathWithLookupMatrix(t *testing.T) {
|
||||
statFor := func(existing map[string]bool) func(string) (os.FileInfo, error) {
|
||||
return func(path string) (os.FileInfo, error) {
|
||||
if existing[path] {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
cliPath string
|
||||
cliPathSet bool
|
||||
lookup func(string) (string, bool)
|
||||
stat func(string) (os.FileInfo, error)
|
||||
defaultSearchPaths []string
|
||||
wantPath string
|
||||
wantSource string
|
||||
wantErrContains string
|
||||
}{
|
||||
{
|
||||
name: "explicit config path",
|
||||
cliPath: "/tmp/explicit.yml",
|
||||
cliPathSet: true,
|
||||
lookup: func(string) (string, bool) { return "", false },
|
||||
stat: statFor(map[string]bool{"/tmp/explicit.yml": true}),
|
||||
defaultSearchPaths: []string{
|
||||
"/usr/local/etc/audita/config.yml",
|
||||
"/etc/audita/config.yml",
|
||||
},
|
||||
wantPath: "/tmp/explicit.yml",
|
||||
wantSource: "flag",
|
||||
},
|
||||
{
|
||||
name: "env config path",
|
||||
cliPathSet: false,
|
||||
lookup: func(key string) (string, bool) {
|
||||
if key == "AUDITA_CONFIG" {
|
||||
return "/tmp/from-env.yml", true
|
||||
}
|
||||
return "", false
|
||||
},
|
||||
stat: statFor(map[string]bool{"/tmp/from-env.yml": true}),
|
||||
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
|
||||
wantPath: "/tmp/from-env.yml",
|
||||
wantSource: "env",
|
||||
},
|
||||
{
|
||||
name: "default search path",
|
||||
cliPathSet: false,
|
||||
lookup: func(string) (string, bool) { return "", false },
|
||||
stat: statFor(map[string]bool{
|
||||
"/usr/local/etc/audita/config.yml": true,
|
||||
"/etc/audita/config.yml": true,
|
||||
}),
|
||||
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
|
||||
wantPath: "/usr/local/etc/audita/config.yml",
|
||||
wantSource: "default",
|
||||
},
|
||||
{
|
||||
name: "explicit missing path",
|
||||
cliPath: "/tmp/missing.yml",
|
||||
cliPathSet: true,
|
||||
lookup: func(string) (string, bool) { return "", false },
|
||||
stat: statFor(map[string]bool{}),
|
||||
defaultSearchPaths: []string{
|
||||
"/usr/local/etc/audita/config.yml",
|
||||
"/etc/audita/config.yml",
|
||||
},
|
||||
wantErrContains: "config file not found",
|
||||
},
|
||||
{
|
||||
name: "missing env path",
|
||||
cliPathSet: false,
|
||||
lookup: func(key string) (string, bool) {
|
||||
if key == "AUDITA_CONFIG" {
|
||||
return "/tmp/missing-from-env.yml", true
|
||||
}
|
||||
return "", false
|
||||
},
|
||||
stat: statFor(map[string]bool{}),
|
||||
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
|
||||
wantErrContains: "config file not found",
|
||||
},
|
||||
{
|
||||
name: "missing default paths",
|
||||
cliPathSet: false,
|
||||
lookup: func(string) (string, bool) { return "", false },
|
||||
stat: statFor(map[string]bool{}),
|
||||
defaultSearchPaths: []string{
|
||||
"/usr/local/etc/audita/config.yml",
|
||||
"/etc/audita/config.yml",
|
||||
},
|
||||
wantPath: "",
|
||||
wantSource: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
gotPath, gotSource, err := resolveConfigPathWithLookup(tc.cliPath, tc.cliPathSet, tc.lookup, tc.stat, tc.defaultSearchPaths)
|
||||
if tc.wantErrContains != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantErrContains) {
|
||||
t.Fatalf("expected error containing %q, got %v", tc.wantErrContains, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if gotPath != tc.wantPath || gotSource != tc.wantSource {
|
||||
t.Fatalf("unexpected result: got path=%q source=%q, want path=%q source=%q", gotPath, gotSource, tc.wantPath, tc.wantSource)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadEffectiveConfigWithLookupAppliesDefaultsFileThenEnv(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
configPath := filepath.Join(tempDir, "config.yml")
|
||||
configYAML := "version: 1\nllm:\n proposal:\n model: file-model\n"
|
||||
if err := os.WriteFile(configPath, []byte(configYAML), 0o644); err != nil {
|
||||
t.Fatalf("write config file: %v", err)
|
||||
}
|
||||
|
||||
lookup := func(key string) (string, bool) {
|
||||
switch key {
|
||||
case "AUDITA_CONFIG":
|
||||
return configPath, true
|
||||
case "AUDITA_MODEL":
|
||||
return "env-model", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
result, err := loadEffectiveConfigWithLookup("", false, lookup, os.Stat, DefaultConfigSearchPaths)
|
||||
if err != nil {
|
||||
t.Fatalf("loadEffectiveConfigWithLookup error: %v", err)
|
||||
}
|
||||
if result.ConfigPath != configPath {
|
||||
t.Fatalf("unexpected config path: %q", result.ConfigPath)
|
||||
}
|
||||
if result.ConfigSource != "env" {
|
||||
t.Fatalf("unexpected config source: %q", result.ConfigSource)
|
||||
}
|
||||
if result.ConfigVersion == nil || *result.ConfigVersion != SupportedFileConfigVersion {
|
||||
t.Fatalf("unexpected config version: %#v", result.ConfigVersion)
|
||||
}
|
||||
if result.Config.PrimaryLLM.Model != "env-model" {
|
||||
t.Fatalf("expected env override to win over file value, got %q", result.Config.PrimaryLLM.Model)
|
||||
}
|
||||
}
|
||||
268
internal/core/config/env.go
Normal file
268
internal/core/config/env.go
Normal file
@@ -0,0 +1,268 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultConfigPath = "/etc/audita/config.yml"
|
||||
DefaultConfigPathUsrLocal = "/usr/local/etc/audita/config.yml"
|
||||
)
|
||||
|
||||
var DefaultConfigSearchPaths = []string{
|
||||
DefaultConfigPathUsrLocal,
|
||||
DefaultConfigPath,
|
||||
}
|
||||
|
||||
func LoadFromEnv() (Config, error) {
|
||||
cfg := Default()
|
||||
if err := cfg.applyEnvOverrides(os.LookupEnv); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func loadFromLookup(lookup func(string) (string, bool)) (Config, error) {
|
||||
cfg := Default()
|
||||
if err := cfg.applyEnvOverrides(lookup); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func (c *Config) ApplyEnvOverrides() error {
|
||||
return c.applyEnvOverrides(os.LookupEnv)
|
||||
}
|
||||
|
||||
func (c *Config) applyEnvOverrides(lookup func(string) (string, bool)) error {
|
||||
if c == nil {
|
||||
return fmt.Errorf("config must not be nil")
|
||||
}
|
||||
|
||||
cfg := c
|
||||
if raw, ok := lookup("AUDITA_MODULES"); ok {
|
||||
modules, err := ParseModulesCSV(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_MODULES: %w", err)
|
||||
}
|
||||
cfg.Modules = modules
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_LLM_API_KEY"); ok {
|
||||
cfg.PrimaryLLM.APIKey = raw
|
||||
} else if raw, ok := lookup("OPENROUTER_API_KEY"); ok {
|
||||
cfg.PrimaryLLM.APIKey = raw
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_LLM_API_KEY"); ok {
|
||||
cfg.ValidationLLM.APIKey = raw
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_MODEL"); ok {
|
||||
cfg.PrimaryLLM.Model = raw
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_MODEL"); ok {
|
||||
cfg.ValidationLLM.Model = raw
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_BASE_URL"); ok {
|
||||
cfg.PrimaryLLM.BaseURL = raw
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_BASE_URL"); ok {
|
||||
cfg.ValidationLLM.BaseURL = raw
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_LLM_TIMEOUT_SECONDS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_LLM_TIMEOUT_SECONDS: %w", err)
|
||||
}
|
||||
cfg.PrimaryLLM.TimeoutSeconds = value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS: %w", err)
|
||||
}
|
||||
cfg.ValidationLLM.TimeoutSeconds = &value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_MAX_RETRIES"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_MAX_RETRIES: %w", err)
|
||||
}
|
||||
cfg.PrimaryLLM.MaxRetries = value
|
||||
}
|
||||
totalConcurrencySet := false
|
||||
if raw, ok := lookup("AUDITA_TOTAL_LLM_CONCURRENCY"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_TOTAL_LLM_CONCURRENCY: %w", err)
|
||||
}
|
||||
cfg.TotalLLMConcurrency = value
|
||||
totalConcurrencySet = true
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_LLM_CONCURRENCY"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_LLM_CONCURRENCY: %w", err)
|
||||
}
|
||||
if !totalConcurrencySet {
|
||||
cfg.TotalLLMConcurrency = value
|
||||
totalConcurrencySet = true
|
||||
}
|
||||
}
|
||||
|
||||
proposalConcurrencySet := false
|
||||
if raw, ok := lookup("AUDITA_PROPOSAL_LLM_CONCURRENCY"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_PROPOSAL_LLM_CONCURRENCY: %w", err)
|
||||
}
|
||||
cfg.ProposalLLMConcurrency = value
|
||||
proposalConcurrencySet = true
|
||||
}
|
||||
if totalConcurrencySet && !proposalConcurrencySet {
|
||||
cfg.ProposalLLMConcurrency = cfg.TotalLLMConcurrency
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_MAX_RETRIES"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_VALIDATION_MAX_RETRIES: %w", err)
|
||||
}
|
||||
cfg.ValidationLLM.MaxRetries = &value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_LLM_CONCURRENCY"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_VALIDATION_LLM_CONCURRENCY: %w", err)
|
||||
}
|
||||
cfg.ValidationLLMConcurrency = &value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_MAX_PROMPT_TOKENS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_VALIDATION_MAX_PROMPT_TOKENS: %w", err)
|
||||
}
|
||||
cfg.ValidationMaxPromptTokens = value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_MAX_SECTION_TOKENS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_MAX_SECTION_TOKENS: %w", err)
|
||||
}
|
||||
cfg.MaxSectionTokens = value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_MIN_SECTION_TOKENS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_MIN_SECTION_TOKENS: %w", err)
|
||||
}
|
||||
cfg.MinSectionTokens = value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_TARGET_SECTIONS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_TARGET_SECTIONS: %w", err)
|
||||
}
|
||||
cfg.TargetSections = &value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD: %w", err)
|
||||
}
|
||||
cfg.Thresholds.Glossary = value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD: %w", err)
|
||||
}
|
||||
cfg.Thresholds.Grammar = value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD: %w", err)
|
||||
}
|
||||
cfg.Thresholds.Homophones = value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD: %w", err)
|
||||
}
|
||||
cfg.Thresholds.SpokenWord = value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_GAP"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_GAP: %w", err)
|
||||
}
|
||||
cfg.Normalization.MaxSegmentGap = value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_NORMALIZE_ELLIPSIS_GAP"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_NORMALIZE_ELLIPSIS_GAP: %w", err)
|
||||
}
|
||||
cfg.Normalization.EllipsisGap = value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION: %w", err)
|
||||
}
|
||||
cfg.Normalization.MaxSegmentDuration = value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS: %w", err)
|
||||
}
|
||||
cfg.Normalization.MaxSegmentTokens = value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_WORK_DIR"); ok {
|
||||
cfg.WorkDir = raw
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_WORK_DIR_RETENTION"); ok {
|
||||
cfg.WorkDirRetention = WorkDirRetention(raw)
|
||||
}
|
||||
|
||||
cfg.syncLegacyConcurrencyAliases()
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseInt(raw string) (int, error) {
|
||||
value, err := strconv.Atoi(raw)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("must be an integer")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func parseFloat(raw string) (float64, error) {
|
||||
value, err := strconv.ParseFloat(raw, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("must be a number")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
342
internal/core/config/file_config.go
Normal file
342
internal/core/config/file_config.go
Normal file
@@ -0,0 +1,342 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const SupportedFileConfigVersion = 1
|
||||
|
||||
var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
type FileConfig struct {
|
||||
Version int `yaml:"version"`
|
||||
Pipeline *FileConfigPipeline `yaml:"pipeline,omitempty"`
|
||||
Output *FileConfigOutput `yaml:"output,omitempty"`
|
||||
LLM *FileConfigLLM `yaml:"llm,omitempty"`
|
||||
Concurrency *FileConfigConcurrency `yaml:"concurrency,omitempty"`
|
||||
Chunking *FileConfigChunking `yaml:"chunking,omitempty"`
|
||||
Normalization *FileConfigNormalization `yaml:"normalization,omitempty"`
|
||||
Thresholds *FileConfigThresholds `yaml:"thresholds,omitempty"`
|
||||
Context *FileConfigContext `yaml:"context,omitempty"`
|
||||
Diagnostics *FileConfigDiagnostics `yaml:"diagnostics,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigPipeline struct {
|
||||
Modules []string `yaml:"modules,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigOutput struct {
|
||||
Schema *string `yaml:"schema,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigLLM struct {
|
||||
Proposal *FileConfigLLMTarget `yaml:"proposal,omitempty"`
|
||||
Validation *FileConfigLLMTarget `yaml:"validation,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigLLMTarget struct {
|
||||
BaseURL *string `yaml:"base_url,omitempty"`
|
||||
Model *string `yaml:"model,omitempty"`
|
||||
APIKeyEnv *string `yaml:"api_key_env,omitempty"`
|
||||
Timeout *fileConfigDurationOrInt `yaml:"timeout,omitempty"`
|
||||
MaxRetries *int `yaml:"max_retries,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigConcurrency struct {
|
||||
TotalLLM *int `yaml:"total_llm,omitempty"`
|
||||
ProposalLLM *int `yaml:"proposal_llm,omitempty"`
|
||||
ValidationLLM *int `yaml:"validation_llm,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigChunking struct {
|
||||
TargetSections *int `yaml:"target_sections,omitempty"`
|
||||
MaxSectionTokens *int `yaml:"max_section_tokens,omitempty"`
|
||||
MinSectionTokens *int `yaml:"min_section_tokens,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigNormalization struct {
|
||||
MaxSegmentGap *fileConfigDurationOrFloat `yaml:"max_segment_gap,omitempty"`
|
||||
EllipsisGap *fileConfigDurationOrFloat `yaml:"ellipsis_gap,omitempty"`
|
||||
MaxSegmentDuration *fileConfigDurationOrFloat `yaml:"max_segment_duration,omitempty"`
|
||||
MaxSegmentTokens *int `yaml:"max_segment_tokens,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigThresholds struct {
|
||||
Glossary *float64 `yaml:"glossary,omitempty"`
|
||||
Homophones *float64 `yaml:"homophones,omitempty"`
|
||||
SpokenWord *float64 `yaml:"spoken_word,omitempty"`
|
||||
Grammar *float64 `yaml:"grammar,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigContext struct {
|
||||
Description *string `yaml:"description,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigDiagnostics struct {
|
||||
WorkDir *string `yaml:"work_dir,omitempty"`
|
||||
Retention *string `yaml:"retention,omitempty"`
|
||||
}
|
||||
|
||||
type fileConfigDurationOrInt struct {
|
||||
seconds int
|
||||
}
|
||||
|
||||
func (v *fileConfigDurationOrInt) UnmarshalYAML(node *yaml.Node) error {
|
||||
switch node.Kind {
|
||||
case yaml.ScalarNode:
|
||||
if node.Tag == "!!int" {
|
||||
var n int
|
||||
if err := node.Decode(&n); err != nil {
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
v.seconds = n
|
||||
return nil
|
||||
}
|
||||
|
||||
var s string
|
||||
if err := node.Decode(&s); err != nil {
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
d, err := time.ParseDuration(strings.TrimSpace(s))
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid duration %q", s)
|
||||
}
|
||||
if d <= 0 {
|
||||
v.seconds = int(d / time.Second)
|
||||
return nil
|
||||
}
|
||||
if d%time.Second != 0 {
|
||||
return fmt.Errorf("duration %q must resolve to whole seconds", s)
|
||||
}
|
||||
v.seconds = int(d / time.Second)
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
}
|
||||
|
||||
func (v fileConfigDurationOrInt) Seconds() int { return v.seconds }
|
||||
|
||||
type fileConfigDurationOrFloat struct {
|
||||
seconds float64
|
||||
}
|
||||
|
||||
func (v *fileConfigDurationOrFloat) UnmarshalYAML(node *yaml.Node) error {
|
||||
switch node.Kind {
|
||||
case yaml.ScalarNode:
|
||||
if node.Tag == "!!int" || node.Tag == "!!float" {
|
||||
var f float64
|
||||
if err := node.Decode(&f); err != nil {
|
||||
return fmt.Errorf("must be a numeric seconds value or duration string")
|
||||
}
|
||||
v.seconds = f
|
||||
return nil
|
||||
}
|
||||
var s string
|
||||
if err := node.Decode(&s); err != nil {
|
||||
return fmt.Errorf("must be a numeric seconds value or duration string")
|
||||
}
|
||||
d, err := time.ParseDuration(strings.TrimSpace(s))
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid duration %q", s)
|
||||
}
|
||||
v.seconds = d.Seconds()
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("must be a numeric seconds value or duration string")
|
||||
}
|
||||
}
|
||||
|
||||
func (v fileConfigDurationOrFloat) Seconds() float64 { return v.seconds }
|
||||
|
||||
func LoadFileConfig(path string) (FileConfig, error) {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return FileConfig{}, fmt.Errorf("read config file %q: %w", path, err)
|
||||
}
|
||||
cfg, err := ParseFileConfigYAML(b)
|
||||
if err != nil {
|
||||
return FileConfig{}, fmt.Errorf("parse config file %q: %w", path, err)
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func ParseFileConfigYAML(data []byte) (FileConfig, error) {
|
||||
var fileCfg FileConfig
|
||||
dec := yaml.NewDecoder(strings.NewReader(string(data)))
|
||||
dec.KnownFields(true)
|
||||
if err := dec.Decode(&fileCfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("decode yaml: %w", err)
|
||||
}
|
||||
if fileCfg.Version == 0 {
|
||||
return FileConfig{}, fmt.Errorf("config version is required")
|
||||
}
|
||||
if fileCfg.Version != SupportedFileConfigVersion {
|
||||
return FileConfig{}, fmt.Errorf("unsupported config version %d", fileCfg.Version)
|
||||
}
|
||||
return fileCfg, nil
|
||||
}
|
||||
|
||||
func (c *Config) ApplyFileConfig(fileCfg FileConfig) error {
|
||||
return c.applyFileConfigWithLookup(fileCfg, os.LookupEnv)
|
||||
}
|
||||
|
||||
func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
|
||||
if c == nil {
|
||||
return fmt.Errorf("config must not be nil")
|
||||
}
|
||||
if fileCfg.Version != SupportedFileConfigVersion {
|
||||
return fmt.Errorf("unsupported config version %d", fileCfg.Version)
|
||||
}
|
||||
|
||||
if fileCfg.Pipeline != nil && len(fileCfg.Pipeline.Modules) > 0 {
|
||||
c.Modules = append([]string(nil), fileCfg.Pipeline.Modules...)
|
||||
}
|
||||
if fileCfg.Output != nil && fileCfg.Output.Schema != nil {
|
||||
c.OutputSchema = strings.TrimSpace(*fileCfg.Output.Schema)
|
||||
}
|
||||
|
||||
if fileCfg.LLM != nil {
|
||||
if fileCfg.LLM.Proposal != nil {
|
||||
if fileCfg.LLM.Proposal.BaseURL != nil {
|
||||
c.PrimaryLLM.BaseURL = *fileCfg.LLM.Proposal.BaseURL
|
||||
}
|
||||
if fileCfg.LLM.Proposal.Model != nil {
|
||||
c.PrimaryLLM.Model = *fileCfg.LLM.Proposal.Model
|
||||
}
|
||||
if fileCfg.LLM.Proposal.Timeout != nil {
|
||||
c.PrimaryLLM.TimeoutSeconds = fileCfg.LLM.Proposal.Timeout.Seconds()
|
||||
}
|
||||
if fileCfg.LLM.Proposal.MaxRetries != nil {
|
||||
c.PrimaryLLM.MaxRetries = *fileCfg.LLM.Proposal.MaxRetries
|
||||
}
|
||||
if fileCfg.LLM.Proposal.APIKeyEnv != nil {
|
||||
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Proposal.APIKeyEnv, lookup)
|
||||
if err != nil {
|
||||
return fmt.Errorf("llm.proposal.api_key_env: %w", err)
|
||||
}
|
||||
c.PrimaryLLM.APIKey = apiKey
|
||||
}
|
||||
}
|
||||
if fileCfg.LLM.Validation != nil {
|
||||
if fileCfg.LLM.Validation.BaseURL != nil {
|
||||
c.ValidationLLM.BaseURL = *fileCfg.LLM.Validation.BaseURL
|
||||
}
|
||||
if fileCfg.LLM.Validation.Model != nil {
|
||||
c.ValidationLLM.Model = *fileCfg.LLM.Validation.Model
|
||||
}
|
||||
if fileCfg.LLM.Validation.Timeout != nil {
|
||||
v := fileCfg.LLM.Validation.Timeout.Seconds()
|
||||
c.ValidationLLM.TimeoutSeconds = &v
|
||||
}
|
||||
if fileCfg.LLM.Validation.MaxRetries != nil {
|
||||
v := *fileCfg.LLM.Validation.MaxRetries
|
||||
c.ValidationLLM.MaxRetries = &v
|
||||
}
|
||||
if fileCfg.LLM.Validation.APIKeyEnv != nil {
|
||||
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Validation.APIKeyEnv, lookup)
|
||||
if err != nil {
|
||||
return fmt.Errorf("llm.validation.api_key_env: %w", err)
|
||||
}
|
||||
c.ValidationLLM.APIKey = apiKey
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if fileCfg.Concurrency != nil {
|
||||
if fileCfg.Concurrency.TotalLLM != nil {
|
||||
c.TotalLLMConcurrency = *fileCfg.Concurrency.TotalLLM
|
||||
}
|
||||
if fileCfg.Concurrency.ProposalLLM != nil {
|
||||
c.ProposalLLMConcurrency = *fileCfg.Concurrency.ProposalLLM
|
||||
}
|
||||
if fileCfg.Concurrency.ValidationLLM != nil {
|
||||
v := *fileCfg.Concurrency.ValidationLLM
|
||||
c.ValidationLLMConcurrency = &v
|
||||
}
|
||||
}
|
||||
|
||||
if fileCfg.Chunking != nil {
|
||||
if fileCfg.Chunking.TargetSections != nil {
|
||||
v := *fileCfg.Chunking.TargetSections
|
||||
c.TargetSections = &v
|
||||
}
|
||||
if fileCfg.Chunking.MaxSectionTokens != nil {
|
||||
c.MaxSectionTokens = *fileCfg.Chunking.MaxSectionTokens
|
||||
}
|
||||
if fileCfg.Chunking.MinSectionTokens != nil {
|
||||
c.MinSectionTokens = *fileCfg.Chunking.MinSectionTokens
|
||||
}
|
||||
}
|
||||
|
||||
if fileCfg.Normalization != nil {
|
||||
if fileCfg.Normalization.MaxSegmentGap != nil {
|
||||
c.Normalization.MaxSegmentGap = fileCfg.Normalization.MaxSegmentGap.Seconds()
|
||||
}
|
||||
if fileCfg.Normalization.EllipsisGap != nil {
|
||||
c.Normalization.EllipsisGap = fileCfg.Normalization.EllipsisGap.Seconds()
|
||||
}
|
||||
if fileCfg.Normalization.MaxSegmentDuration != nil {
|
||||
c.Normalization.MaxSegmentDuration = fileCfg.Normalization.MaxSegmentDuration.Seconds()
|
||||
}
|
||||
if fileCfg.Normalization.MaxSegmentTokens != nil {
|
||||
c.Normalization.MaxSegmentTokens = *fileCfg.Normalization.MaxSegmentTokens
|
||||
}
|
||||
}
|
||||
|
||||
if fileCfg.Thresholds != nil {
|
||||
if fileCfg.Thresholds.Glossary != nil {
|
||||
c.Thresholds.Glossary = *fileCfg.Thresholds.Glossary
|
||||
}
|
||||
if fileCfg.Thresholds.Homophones != nil {
|
||||
c.Thresholds.Homophones = *fileCfg.Thresholds.Homophones
|
||||
}
|
||||
if fileCfg.Thresholds.SpokenWord != nil {
|
||||
c.Thresholds.SpokenWord = *fileCfg.Thresholds.SpokenWord
|
||||
}
|
||||
if fileCfg.Thresholds.Grammar != nil {
|
||||
c.Thresholds.Grammar = *fileCfg.Thresholds.Grammar
|
||||
}
|
||||
}
|
||||
|
||||
if fileCfg.Context != nil && fileCfg.Context.Description != nil {
|
||||
c.TranscriptDescription = strings.TrimSpace(*fileCfg.Context.Description)
|
||||
}
|
||||
|
||||
if fileCfg.Diagnostics != nil {
|
||||
if fileCfg.Diagnostics.WorkDir != nil {
|
||||
c.WorkDir = *fileCfg.Diagnostics.WorkDir
|
||||
}
|
||||
if fileCfg.Diagnostics.Retention != nil {
|
||||
c.WorkDirRetention = WorkDirRetention(*fileCfg.Diagnostics.Retention)
|
||||
}
|
||||
}
|
||||
|
||||
c.syncLegacyConcurrencyAliases()
|
||||
if err := c.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
|
||||
name := strings.TrimSpace(envName)
|
||||
if name == "" {
|
||||
return "", fmt.Errorf("must not be empty")
|
||||
}
|
||||
if !envVarNamePattern.MatchString(name) {
|
||||
return "", fmt.Errorf("must be an environment variable name")
|
||||
}
|
||||
if strings.Contains(name, string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("must be an environment variable name")
|
||||
}
|
||||
v, _ := lookup(name)
|
||||
return v, nil
|
||||
}
|
||||
290
internal/core/config/file_config_test.go
Normal file
290
internal/core/config/file_config_test.go
Normal file
@@ -0,0 +1,290 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseFileConfigYAMLValid(t *testing.T) {
|
||||
raw := `
|
||||
version: 1
|
||||
pipeline:
|
||||
modules: [glossary, homophones, grammar]
|
||||
output:
|
||||
schema: audita-v1
|
||||
llm:
|
||||
proposal:
|
||||
base_url: https://example.test/v1
|
||||
model: provider/model-a
|
||||
api_key_env: AUDITA_PROPOSAL_KEY
|
||||
timeout: 2m
|
||||
max_retries: 4
|
||||
validation:
|
||||
base_url: https://example.test/validation
|
||||
model: provider/model-b
|
||||
api_key_env: AUDITA_VALIDATION_KEY
|
||||
timeout: 45
|
||||
max_retries: 3
|
||||
concurrency:
|
||||
total_llm: 8
|
||||
proposal_llm: 4
|
||||
validation_llm: 2
|
||||
chunking:
|
||||
target_sections: 6
|
||||
max_section_tokens: 9000
|
||||
min_section_tokens: 3000
|
||||
normalization:
|
||||
max_segment_gap: 1.5s
|
||||
ellipsis_gap: 2
|
||||
max_segment_duration: 45s
|
||||
max_segment_tokens: 1500
|
||||
thresholds:
|
||||
glossary: 0.9
|
||||
homophones: 0.7
|
||||
spoken_word: 0.8
|
||||
grammar: 0.75
|
||||
context:
|
||||
description: " crowd scene with many proper nouns "
|
||||
diagnostics:
|
||||
work_dir: /tmp/audita-config
|
||||
retention: always
|
||||
`
|
||||
cfg, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML error: %v", err)
|
||||
}
|
||||
if cfg.Version != 1 {
|
||||
t.Fatalf("expected version 1, got %d", cfg.Version)
|
||||
}
|
||||
if cfg.Pipeline == nil || len(cfg.Pipeline.Modules) != 3 {
|
||||
t.Fatalf("unexpected pipeline modules: %#v", cfg.Pipeline)
|
||||
}
|
||||
if cfg.Output == nil || cfg.Output.Schema == nil || *cfg.Output.Schema != "audita-v1" {
|
||||
t.Fatalf("expected output schema audita-v1, got %#v", cfg.Output)
|
||||
}
|
||||
if cfg.LLM == nil || cfg.LLM.Proposal == nil || cfg.LLM.Validation == nil {
|
||||
t.Fatalf("expected llm proposal+validation blocks")
|
||||
}
|
||||
if cfg.LLM.Proposal.Timeout == nil || cfg.LLM.Proposal.Timeout.Seconds() != 120 {
|
||||
t.Fatalf("expected proposal timeout 120s, got %#v", cfg.LLM.Proposal.Timeout)
|
||||
}
|
||||
if cfg.LLM.Validation.Timeout == nil || cfg.LLM.Validation.Timeout.Seconds() != 45 {
|
||||
t.Fatalf("expected validation timeout 45s, got %#v", cfg.LLM.Validation.Timeout)
|
||||
}
|
||||
if cfg.Normalization == nil || cfg.Normalization.MaxSegmentGap == nil || cfg.Normalization.MaxSegmentGap.Seconds() != 1.5 {
|
||||
t.Fatalf("expected parsed duration for normalization max_segment_gap")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigYAMLRejectsUnknownField(t *testing.T) {
|
||||
raw := `
|
||||
version: 1
|
||||
pipeline:
|
||||
modules: [grammar]
|
||||
output:
|
||||
unknown: v1
|
||||
`
|
||||
_, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err == nil {
|
||||
t.Fatalf("expected unknown field error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "field unknown not found") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigYAMLRejectsMissingVersion(t *testing.T) {
|
||||
raw := `pipeline: {modules: [grammar]}`
|
||||
_, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err == nil {
|
||||
t.Fatalf("expected missing version error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "config version is required") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigYAMLRejectsUnsupportedVersion(t *testing.T) {
|
||||
raw := `version: 2`
|
||||
_, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err == nil {
|
||||
t.Fatalf("expected unsupported version error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "unsupported config version 2") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigParsesAndMergesFields(t *testing.T) {
|
||||
raw := `
|
||||
version: 1
|
||||
pipeline:
|
||||
modules: [spoken_word, grammar]
|
||||
output:
|
||||
schema: audita-v1
|
||||
llm:
|
||||
proposal:
|
||||
model: provider/new-proposal
|
||||
api_key_env: PROPOSAL_KEY_NAME
|
||||
timeout: 90s
|
||||
max_retries: 5
|
||||
validation:
|
||||
model: provider/new-validation
|
||||
api_key_env: VALIDATION_KEY_NAME
|
||||
timeout: 150
|
||||
max_retries: 6
|
||||
concurrency:
|
||||
total_llm: 7
|
||||
proposal_llm: 3
|
||||
validation_llm: 2
|
||||
chunking:
|
||||
target_sections: 9
|
||||
thresholds:
|
||||
glossary: 0.91
|
||||
homophones: 0.61
|
||||
spoken_word: 0.71
|
||||
grammar: 0.81
|
||||
diagnostics:
|
||||
retention: never
|
||||
`
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML error: %v", err)
|
||||
}
|
||||
|
||||
cfg := Default()
|
||||
lookup := func(name string) (string, bool) {
|
||||
switch name {
|
||||
case "PROPOSAL_KEY_NAME":
|
||||
return "proposal-secret", true
|
||||
case "VALIDATION_KEY_NAME":
|
||||
return "validation-secret", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
if err := cfg.applyFileConfigWithLookup(fileCfg, lookup); err != nil {
|
||||
t.Fatalf("applyFileConfigWithLookup error: %v", err)
|
||||
}
|
||||
if strings.Join(cfg.Modules, ",") != "spoken_word,grammar" {
|
||||
t.Fatalf("unexpected modules: %#v", cfg.Modules)
|
||||
}
|
||||
if cfg.OutputSchema != "audita-v1" {
|
||||
t.Fatalf("unexpected output schema: %q", cfg.OutputSchema)
|
||||
}
|
||||
if cfg.PrimaryLLM.Model != "provider/new-proposal" {
|
||||
t.Fatalf("unexpected proposal model: %q", cfg.PrimaryLLM.Model)
|
||||
}
|
||||
if cfg.PrimaryLLM.APIKey != "proposal-secret" {
|
||||
t.Fatalf("expected proposal key from api_key_env lookup, got %q", cfg.PrimaryLLM.APIKey)
|
||||
}
|
||||
if cfg.PrimaryLLM.TimeoutSeconds != 90 {
|
||||
t.Fatalf("unexpected proposal timeout: %d", cfg.PrimaryLLM.TimeoutSeconds)
|
||||
}
|
||||
if cfg.ValidationLLM.Model != "provider/new-validation" {
|
||||
t.Fatalf("unexpected validation model: %q", cfg.ValidationLLM.Model)
|
||||
}
|
||||
if cfg.ValidationLLM.APIKey != "validation-secret" {
|
||||
t.Fatalf("expected validation key from api_key_env lookup, got %q", cfg.ValidationLLM.APIKey)
|
||||
}
|
||||
if cfg.ValidationLLM.TimeoutSeconds == nil || *cfg.ValidationLLM.TimeoutSeconds != 150 {
|
||||
t.Fatalf("unexpected validation timeout: %#v", cfg.ValidationLLM.TimeoutSeconds)
|
||||
}
|
||||
if cfg.TotalLLMConcurrency != 7 || cfg.ProposalLLMConcurrency != 3 {
|
||||
t.Fatalf("unexpected llm concurrency values: total=%d proposal=%d", cfg.TotalLLMConcurrency, cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
if cfg.ValidationLLMConcurrency == nil || *cfg.ValidationLLMConcurrency != 2 {
|
||||
t.Fatalf("unexpected validation llm concurrency: %#v", cfg.ValidationLLMConcurrency)
|
||||
}
|
||||
if cfg.TargetSections == nil || *cfg.TargetSections != 9 {
|
||||
t.Fatalf("unexpected target sections: %#v", cfg.TargetSections)
|
||||
}
|
||||
if cfg.WorkDirRetention != WorkDirRetentionNever {
|
||||
t.Fatalf("unexpected retention: %q", cfg.WorkDirRetention)
|
||||
}
|
||||
if cfg.PrimaryLLM.Concurrency != 7 {
|
||||
t.Fatalf("expected legacy alias to sync, got %d", cfg.PrimaryLLM.Concurrency)
|
||||
}
|
||||
if cfg.ValidationLLM.Concurrency == nil || *cfg.ValidationLLM.Concurrency != 2 {
|
||||
t.Fatalf("expected validation alias to sync, got %#v", cfg.ValidationLLM.Concurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigContextDescriptionTrim(t *testing.T) {
|
||||
raw := `
|
||||
version: 1
|
||||
context:
|
||||
description: " scene context "
|
||||
`
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML error: %v", err)
|
||||
}
|
||||
cfg := Default()
|
||||
if err := cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{})); err != nil {
|
||||
t.Fatalf("applyFileConfigWithLookup error: %v", err)
|
||||
}
|
||||
if cfg.TranscriptDescription != "scene context" {
|
||||
t.Fatalf("unexpected transcript description: %q", cfg.TranscriptDescription)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigRejectsInvalidAPIKeyEnvName(t *testing.T) {
|
||||
raw := `
|
||||
version: 1
|
||||
llm:
|
||||
proposal:
|
||||
api_key_env: "not a var name"
|
||||
`
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML error: %v", err)
|
||||
}
|
||||
cfg := Default()
|
||||
err = cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{}))
|
||||
if err == nil {
|
||||
t.Fatalf("expected api_key_env validation error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "environment variable name") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigDurationParsingErrors(t *testing.T) {
|
||||
raw := `
|
||||
version: 1
|
||||
llm:
|
||||
proposal:
|
||||
timeout: "1.5s"
|
||||
`
|
||||
_, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err == nil {
|
||||
t.Fatalf("expected duration parse error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "whole seconds") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileConfigReadsFromPath(t *testing.T) {
|
||||
p := writeTempFileConfig(t, "version: 1\n")
|
||||
cfg, err := LoadFileConfig(p)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadFileConfig error: %v", err)
|
||||
}
|
||||
if cfg.Version != 1 {
|
||||
t.Fatalf("expected version 1, got %d", cfg.Version)
|
||||
}
|
||||
}
|
||||
|
||||
func writeTempFileConfig(t *testing.T, contents string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
path := dir + "/config.yaml"
|
||||
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
|
||||
t.Fatalf("write config file: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
159
internal/core/config/flags.go
Normal file
159
internal/core/config/flags.go
Normal file
@@ -0,0 +1,159 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type CLIOverrides struct {
|
||||
ModulesCSV *string
|
||||
OutputSchema *string
|
||||
PrimaryLLMAPIKey *string
|
||||
ValidationLLMAPIKey *string
|
||||
PrimaryModel *string
|
||||
ValidationModel *string
|
||||
PrimaryBaseURL *string
|
||||
ValidationBaseURL *string
|
||||
PrimaryLLMTimeoutSeconds *int
|
||||
TotalLLMConcurrency *int
|
||||
ProposalLLMConcurrency *int
|
||||
PrimaryLLMConcurrency *int
|
||||
ValidationLLMTimeoutSeconds *int
|
||||
MaxRetries *int
|
||||
ValidationMaxRetries *int
|
||||
ValidationLLMConcurrency *int
|
||||
ValidationMaxPromptTokens *int
|
||||
MaxSectionTokens *int
|
||||
MinSectionTokens *int
|
||||
TargetSections *int
|
||||
GlossaryConfidenceThreshold *float64
|
||||
GrammarConfidenceThreshold *float64
|
||||
HomophonesConfidenceThreshold *float64
|
||||
SpokenWordConfidenceThreshold *float64
|
||||
NormalizeMaxSegmentGap *float64
|
||||
NormalizeEllipsisGap *float64
|
||||
NormalizeMaxSegmentDuration *float64
|
||||
NormalizeMaxSegmentTokens *int
|
||||
TranscriptDescription *string
|
||||
WorkDir *string
|
||||
WorkDirRetention *string
|
||||
}
|
||||
|
||||
func (c *Config) ApplyCLIOverrides(overrides CLIOverrides) error {
|
||||
if overrides.ModulesCSV != nil {
|
||||
modules, err := ParseModulesCSV(*overrides.ModulesCSV)
|
||||
if err != nil {
|
||||
return fmt.Errorf("--modules: %w", err)
|
||||
}
|
||||
c.Modules = modules
|
||||
}
|
||||
if overrides.OutputSchema != nil {
|
||||
c.OutputSchema = strings.TrimSpace(*overrides.OutputSchema)
|
||||
}
|
||||
|
||||
if overrides.PrimaryLLMAPIKey != nil {
|
||||
c.PrimaryLLM.APIKey = *overrides.PrimaryLLMAPIKey
|
||||
}
|
||||
if overrides.ValidationLLMAPIKey != nil {
|
||||
c.ValidationLLM.APIKey = *overrides.ValidationLLMAPIKey
|
||||
}
|
||||
if overrides.PrimaryModel != nil {
|
||||
c.PrimaryLLM.Model = *overrides.PrimaryModel
|
||||
}
|
||||
if overrides.ValidationModel != nil {
|
||||
c.ValidationLLM.Model = *overrides.ValidationModel
|
||||
}
|
||||
if overrides.PrimaryBaseURL != nil {
|
||||
c.PrimaryLLM.BaseURL = *overrides.PrimaryBaseURL
|
||||
}
|
||||
if overrides.ValidationBaseURL != nil {
|
||||
c.ValidationLLM.BaseURL = *overrides.ValidationBaseURL
|
||||
}
|
||||
if overrides.PrimaryLLMTimeoutSeconds != nil {
|
||||
c.PrimaryLLM.TimeoutSeconds = *overrides.PrimaryLLMTimeoutSeconds
|
||||
}
|
||||
totalConcurrencySet := false
|
||||
if overrides.TotalLLMConcurrency != nil {
|
||||
c.TotalLLMConcurrency = *overrides.TotalLLMConcurrency
|
||||
totalConcurrencySet = true
|
||||
}
|
||||
// Backward-compatible alias: --llm-concurrency maps to total concurrency
|
||||
// only when --total-llm-concurrency is not set in the same CLI invocation.
|
||||
if overrides.PrimaryLLMConcurrency != nil && !totalConcurrencySet {
|
||||
c.TotalLLMConcurrency = *overrides.PrimaryLLMConcurrency
|
||||
totalConcurrencySet = true
|
||||
}
|
||||
proposalConcurrencySet := false
|
||||
if overrides.ProposalLLMConcurrency != nil {
|
||||
c.ProposalLLMConcurrency = *overrides.ProposalLLMConcurrency
|
||||
proposalConcurrencySet = true
|
||||
}
|
||||
if totalConcurrencySet && !proposalConcurrencySet {
|
||||
c.ProposalLLMConcurrency = c.TotalLLMConcurrency
|
||||
}
|
||||
if overrides.ValidationLLMTimeoutSeconds != nil {
|
||||
value := *overrides.ValidationLLMTimeoutSeconds
|
||||
c.ValidationLLM.TimeoutSeconds = &value
|
||||
}
|
||||
if overrides.MaxRetries != nil {
|
||||
c.PrimaryLLM.MaxRetries = *overrides.MaxRetries
|
||||
}
|
||||
if overrides.ValidationMaxRetries != nil {
|
||||
value := *overrides.ValidationMaxRetries
|
||||
c.ValidationLLM.MaxRetries = &value
|
||||
}
|
||||
if overrides.ValidationLLMConcurrency != nil {
|
||||
value := *overrides.ValidationLLMConcurrency
|
||||
c.ValidationLLMConcurrency = &value
|
||||
}
|
||||
if overrides.ValidationMaxPromptTokens != nil {
|
||||
c.ValidationMaxPromptTokens = *overrides.ValidationMaxPromptTokens
|
||||
}
|
||||
if overrides.MaxSectionTokens != nil {
|
||||
c.MaxSectionTokens = *overrides.MaxSectionTokens
|
||||
}
|
||||
if overrides.MinSectionTokens != nil {
|
||||
c.MinSectionTokens = *overrides.MinSectionTokens
|
||||
}
|
||||
if overrides.TargetSections != nil {
|
||||
value := *overrides.TargetSections
|
||||
c.TargetSections = &value
|
||||
}
|
||||
if overrides.GlossaryConfidenceThreshold != nil {
|
||||
c.Thresholds.Glossary = *overrides.GlossaryConfidenceThreshold
|
||||
}
|
||||
if overrides.GrammarConfidenceThreshold != nil {
|
||||
c.Thresholds.Grammar = *overrides.GrammarConfidenceThreshold
|
||||
}
|
||||
if overrides.HomophonesConfidenceThreshold != nil {
|
||||
c.Thresholds.Homophones = *overrides.HomophonesConfidenceThreshold
|
||||
}
|
||||
if overrides.SpokenWordConfidenceThreshold != nil {
|
||||
c.Thresholds.SpokenWord = *overrides.SpokenWordConfidenceThreshold
|
||||
}
|
||||
if overrides.NormalizeMaxSegmentGap != nil {
|
||||
c.Normalization.MaxSegmentGap = *overrides.NormalizeMaxSegmentGap
|
||||
}
|
||||
if overrides.NormalizeEllipsisGap != nil {
|
||||
c.Normalization.EllipsisGap = *overrides.NormalizeEllipsisGap
|
||||
}
|
||||
if overrides.NormalizeMaxSegmentDuration != nil {
|
||||
c.Normalization.MaxSegmentDuration = *overrides.NormalizeMaxSegmentDuration
|
||||
}
|
||||
if overrides.NormalizeMaxSegmentTokens != nil {
|
||||
c.Normalization.MaxSegmentTokens = *overrides.NormalizeMaxSegmentTokens
|
||||
}
|
||||
if overrides.TranscriptDescription != nil {
|
||||
c.TranscriptDescription = strings.TrimSpace(*overrides.TranscriptDescription)
|
||||
}
|
||||
if overrides.WorkDir != nil {
|
||||
c.WorkDir = *overrides.WorkDir
|
||||
}
|
||||
if overrides.WorkDirRetention != nil {
|
||||
c.WorkDirRetention = WorkDirRetention(*overrides.WorkDirRetention)
|
||||
}
|
||||
|
||||
c.syncLegacyConcurrencyAliases()
|
||||
|
||||
return c.Validate()
|
||||
}
|
||||
17
internal/core/config/redaction.go
Normal file
17
internal/core/config/redaction.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package config
|
||||
|
||||
const redactedSecret = "[REDACTED]"
|
||||
|
||||
func (c Config) Redacted() Config {
|
||||
redacted := c
|
||||
redacted.PrimaryLLM.APIKey = redactSecret(redacted.PrimaryLLM.APIKey)
|
||||
redacted.ValidationLLM.APIKey = redactSecret(redacted.ValidationLLM.APIKey)
|
||||
return redacted
|
||||
}
|
||||
|
||||
func redactSecret(value string) string {
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
return redactedSecret
|
||||
}
|
||||
136
internal/core/config/validation.go
Normal file
136
internal/core/config/validation.go
Normal file
@@ -0,0 +1,136 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
|
||||
)
|
||||
|
||||
func (c Config) Validate() error {
|
||||
var issues []string
|
||||
|
||||
if len(c.Modules) == 0 {
|
||||
issues = append(issues, "modules must not be empty")
|
||||
}
|
||||
for _, module := range c.Modules {
|
||||
moduleKey := strings.TrimSpace(module)
|
||||
if moduleKey == "" {
|
||||
issues = append(issues, "modules must not contain empty values")
|
||||
break
|
||||
}
|
||||
if !modulecatalog.IsSupported(moduleKey) {
|
||||
issues = append(issues, fmt.Sprintf("unsupported module key %q", moduleKey))
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(c.OutputSchema) == "" {
|
||||
issues = append(issues, "output schema must not be empty")
|
||||
} else if !outputschema.IsSupported(c.OutputSchema) {
|
||||
issues = append(issues, fmt.Sprintf("unsupported output schema %q", c.OutputSchema))
|
||||
}
|
||||
|
||||
if c.PrimaryLLM.TimeoutSeconds <= 0 {
|
||||
issues = append(issues, "primary llm timeout seconds must be greater than zero")
|
||||
}
|
||||
if c.PrimaryLLM.MaxRetries < 0 {
|
||||
issues = append(issues, "max retries must be zero or greater")
|
||||
}
|
||||
if c.TotalLLMConcurrency <= 0 {
|
||||
issues = append(issues, "total llm concurrency must be greater than zero")
|
||||
}
|
||||
if c.ProposalLLMConcurrency <= 0 {
|
||||
issues = append(issues, "proposal llm concurrency must be greater than zero")
|
||||
}
|
||||
if c.ProposalLLMConcurrency > c.TotalLLMConcurrency {
|
||||
issues = append(issues, "proposal llm concurrency must be less than or equal to total llm concurrency")
|
||||
}
|
||||
|
||||
if c.ValidationLLM.TimeoutSeconds != nil && *c.ValidationLLM.TimeoutSeconds <= 0 {
|
||||
issues = append(issues, "validation llm timeout seconds must be greater than zero")
|
||||
}
|
||||
if c.ValidationLLM.MaxRetries != nil && *c.ValidationLLM.MaxRetries < 0 {
|
||||
issues = append(issues, "validation max retries must be zero or greater")
|
||||
}
|
||||
if c.ValidationLLMConcurrency != nil && *c.ValidationLLMConcurrency <= 0 {
|
||||
issues = append(issues, "validation llm concurrency must be greater than zero")
|
||||
}
|
||||
if c.ValidationLLMConcurrency != nil && *c.ValidationLLMConcurrency > c.TotalLLMConcurrency {
|
||||
issues = append(issues, "validation llm concurrency must be less than or equal to total llm concurrency")
|
||||
}
|
||||
|
||||
if c.ValidationMaxPromptTokens <= 0 {
|
||||
issues = append(issues, "validation max prompt tokens must be greater than zero")
|
||||
}
|
||||
if c.MaxSectionTokens <= 0 {
|
||||
issues = append(issues, "max section tokens must be greater than zero")
|
||||
}
|
||||
if c.MinSectionTokens <= 0 {
|
||||
issues = append(issues, "min section tokens must be greater than zero")
|
||||
}
|
||||
if c.MinSectionTokens > c.MaxSectionTokens {
|
||||
issues = append(issues, "min section tokens must be less than or equal to max section tokens")
|
||||
}
|
||||
if c.TargetSections != nil && *c.TargetSections <= 0 {
|
||||
issues = append(issues, "target sections must be greater than zero when set")
|
||||
}
|
||||
|
||||
if err := validateConfidence("glossary", c.Thresholds.Glossary); err != nil {
|
||||
issues = append(issues, err.Error())
|
||||
}
|
||||
if err := validateConfidence("grammar", c.Thresholds.Grammar); err != nil {
|
||||
issues = append(issues, err.Error())
|
||||
}
|
||||
if err := validateConfidence("homophones", c.Thresholds.Homophones); err != nil {
|
||||
issues = append(issues, err.Error())
|
||||
}
|
||||
if err := validateConfidence("spoken-word", c.Thresholds.SpokenWord); err != nil {
|
||||
issues = append(issues, err.Error())
|
||||
}
|
||||
|
||||
if c.Normalization.MaxSegmentGap < 0 {
|
||||
issues = append(issues, "normalize max segment gap must be zero or greater")
|
||||
}
|
||||
if c.Normalization.EllipsisGap < 0 {
|
||||
issues = append(issues, "normalize ellipsis gap must be zero or greater")
|
||||
}
|
||||
if c.Normalization.MaxSegmentDuration <= 0 {
|
||||
issues = append(issues, "normalize max segment duration must be greater than zero")
|
||||
}
|
||||
if c.Normalization.MaxSegmentTokens <= 0 {
|
||||
issues = append(issues, "normalize max segment tokens must be greater than zero")
|
||||
}
|
||||
if len(strings.TrimSpace(c.TranscriptDescription)) > DefaultTranscriptDescriptionMaxChars {
|
||||
issues = append(issues, fmt.Sprintf("transcript description must be %d characters or fewer", DefaultTranscriptDescriptionMaxChars))
|
||||
}
|
||||
|
||||
if strings.TrimSpace(c.WorkDir) == "" {
|
||||
issues = append(issues, "work dir must not be empty")
|
||||
}
|
||||
|
||||
if err := validateRetention(c.WorkDirRetention); err != nil {
|
||||
issues = append(issues, err.Error())
|
||||
}
|
||||
|
||||
if len(issues) > 0 {
|
||||
return fmt.Errorf("invalid config: %s", strings.Join(issues, "; "))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateConfidence(name string, threshold float64) error {
|
||||
if threshold < 0.0 || threshold > 1.0 {
|
||||
return fmt.Errorf("%s confidence threshold must be between 0.0 and 1.0", name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateRetention(retention WorkDirRetention) error {
|
||||
switch retention {
|
||||
case WorkDirRetentionAuto, WorkDirRetentionAlways, WorkDirRetentionNever:
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("work dir retention must be one of: auto, always, never")
|
||||
}
|
||||
}
|
||||
42
internal/core/diagnostics/artifacts.go
Normal file
42
internal/core/diagnostics/artifacts.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
)
|
||||
|
||||
const (
|
||||
ArtifactSourceTranscript = "source-transcript.json"
|
||||
ArtifactParsedSourceTranscript = "source-transcript-parsed.json"
|
||||
ArtifactNormalizedTranscript = "normalized-transcript.json"
|
||||
ArtifactNormalizationSummary = "normalization-summary.json"
|
||||
ArtifactChunkingSummary = "chunking-summary.json"
|
||||
ArtifactUtilizationSummary = "utilization-diagnostics.json"
|
||||
ArtifactCorrectionLedger = "correction-ledger.json"
|
||||
ArtifactInvocationMetadata = "invocation.json"
|
||||
ArtifactEffectiveConfig = "effective-config.json"
|
||||
ArtifactReport = "report.json"
|
||||
ArtifactErrorLog = "error.log"
|
||||
)
|
||||
|
||||
func BuildDiagnosticsMetadata(runDirectoryPath string, runSucceeded bool) reporting.DiagnosticsMetadata {
|
||||
metadata := reporting.DiagnosticsMetadata{
|
||||
DirectoryPath: runDirectoryPath,
|
||||
SourceTranscriptPath: filepath.Join(runDirectoryPath, ArtifactSourceTranscript),
|
||||
ParsedSourceTranscriptPath: filepath.Join(runDirectoryPath, ArtifactParsedSourceTranscript),
|
||||
NormalizedTranscriptPath: filepath.Join(runDirectoryPath, ArtifactNormalizedTranscript),
|
||||
NormalizationSummaryPath: filepath.Join(runDirectoryPath, ArtifactNormalizationSummary),
|
||||
ChunkingSummaryPath: filepath.Join(runDirectoryPath, ArtifactChunkingSummary),
|
||||
UtilizationSummaryPath: filepath.Join(runDirectoryPath, ArtifactUtilizationSummary),
|
||||
CorrectionLedgerPath: filepath.Join(runDirectoryPath, ArtifactCorrectionLedger),
|
||||
InvocationMetadataPath: filepath.Join(runDirectoryPath, ArtifactInvocationMetadata),
|
||||
RedactedEffectiveConfigPath: filepath.Join(runDirectoryPath, ArtifactEffectiveConfig),
|
||||
}
|
||||
|
||||
if !runSucceeded {
|
||||
metadata.ErrorLogPath = filepath.Join(runDirectoryPath, ArtifactErrorLog)
|
||||
}
|
||||
|
||||
return metadata
|
||||
}
|
||||
55
internal/core/diagnostics/artifacts_test.go
Normal file
55
internal/core/diagnostics/artifacts_test.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBuildDiagnosticsMetadataSuccessPathsMatchArtifactConstants(t *testing.T) {
|
||||
runPath := filepath.Join("tmp", "run-123")
|
||||
metadata := BuildDiagnosticsMetadata(runPath, true)
|
||||
|
||||
if metadata.DirectoryPath != runPath {
|
||||
t.Fatalf("unexpected diagnostics directory path: got=%q want=%q", metadata.DirectoryPath, runPath)
|
||||
}
|
||||
if metadata.SourceTranscriptPath != filepath.Join(runPath, ArtifactSourceTranscript) {
|
||||
t.Fatalf("unexpected source transcript path: %q", metadata.SourceTranscriptPath)
|
||||
}
|
||||
if metadata.ParsedSourceTranscriptPath != filepath.Join(runPath, ArtifactParsedSourceTranscript) {
|
||||
t.Fatalf("unexpected parsed source transcript path: %q", metadata.ParsedSourceTranscriptPath)
|
||||
}
|
||||
if metadata.NormalizedTranscriptPath != filepath.Join(runPath, ArtifactNormalizedTranscript) {
|
||||
t.Fatalf("unexpected normalized transcript path: %q", metadata.NormalizedTranscriptPath)
|
||||
}
|
||||
if metadata.NormalizationSummaryPath != filepath.Join(runPath, ArtifactNormalizationSummary) {
|
||||
t.Fatalf("unexpected normalization summary path: %q", metadata.NormalizationSummaryPath)
|
||||
}
|
||||
if metadata.ChunkingSummaryPath != filepath.Join(runPath, ArtifactChunkingSummary) {
|
||||
t.Fatalf("unexpected chunking summary path: %q", metadata.ChunkingSummaryPath)
|
||||
}
|
||||
if metadata.UtilizationSummaryPath != filepath.Join(runPath, ArtifactUtilizationSummary) {
|
||||
t.Fatalf("unexpected utilization summary path: %q", metadata.UtilizationSummaryPath)
|
||||
}
|
||||
if metadata.CorrectionLedgerPath != filepath.Join(runPath, ArtifactCorrectionLedger) {
|
||||
t.Fatalf("unexpected correction ledger path: %q", metadata.CorrectionLedgerPath)
|
||||
}
|
||||
if metadata.InvocationMetadataPath != filepath.Join(runPath, ArtifactInvocationMetadata) {
|
||||
t.Fatalf("unexpected invocation metadata path: %q", metadata.InvocationMetadataPath)
|
||||
}
|
||||
if metadata.RedactedEffectiveConfigPath != filepath.Join(runPath, ArtifactEffectiveConfig) {
|
||||
t.Fatalf("unexpected redacted effective config path: %q", metadata.RedactedEffectiveConfigPath)
|
||||
}
|
||||
if metadata.ErrorLogPath != "" {
|
||||
t.Fatalf("did not expect error log path on success: %q", metadata.ErrorLogPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDiagnosticsMetadataFailureIncludesErrorLogPath(t *testing.T) {
|
||||
runPath := filepath.Join("tmp", "run-123")
|
||||
metadata := BuildDiagnosticsMetadata(runPath, false)
|
||||
|
||||
want := filepath.Join(runPath, ArtifactErrorLog)
|
||||
if metadata.ErrorLogPath != want {
|
||||
t.Fatalf("unexpected error log path: got=%q want=%q", metadata.ErrorLogPath, want)
|
||||
}
|
||||
}
|
||||
72
internal/core/diagnostics/retention_test.go
Normal file
72
internal/core/diagnostics/retention_test.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestShouldRetainRunDirectoryMatrix(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input RetentionDecisionInput
|
||||
want bool
|
||||
}{
|
||||
{name: "always success keeps", input: RetentionDecisionInput{RetentionMode: "always", RunSucceeded: true}, want: true},
|
||||
{name: "always failure keeps", input: RetentionDecisionInput{RetentionMode: "always", RunSucceeded: false}, want: true},
|
||||
{name: "never success keeps", input: RetentionDecisionInput{RetentionMode: "never", RunSucceeded: true}, want: true},
|
||||
{name: "never failure keeps", input: RetentionDecisionInput{RetentionMode: "never", RunSucceeded: false}, want: true},
|
||||
{name: "auto success no skips removes", input: RetentionDecisionInput{RetentionMode: "auto", RunSucceeded: true, HasSkippedCorrections: false}, want: false},
|
||||
{name: "auto success skips keeps", input: RetentionDecisionInput{RetentionMode: "auto", RunSucceeded: true, HasSkippedCorrections: true}, want: true},
|
||||
{name: "auto failure keeps", input: RetentionDecisionInput{RetentionMode: "auto", RunSucceeded: false}, want: true},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := ShouldRetainRunDirectory(tc.input)
|
||||
if got != tc.want {
|
||||
t.Fatalf("unexpected retain decision: got=%v want=%v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyRetentionRemovesWhenDecisionSaysRemove(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
runPath := filepath.Join(workDir, "run-test")
|
||||
if err := os.Mkdir(runPath, 0o755); err != nil {
|
||||
t.Fatalf("mkdir run path: %v", err)
|
||||
}
|
||||
|
||||
runDir := &RunDirectory{path: runPath, retention: "auto"}
|
||||
err := runDir.ApplyRetention(RetentionDecisionInput{RunSucceeded: true, HasSkippedCorrections: false})
|
||||
if err != nil {
|
||||
t.Fatalf("ApplyRetention failed: %v", err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(runPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected run directory removed, stat err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyRetentionReturnsRemovalError(t *testing.T) {
|
||||
parent := t.TempDir()
|
||||
runPath := filepath.Join(parent, "run-test")
|
||||
if err := os.Mkdir(runPath, 0o755); err != nil {
|
||||
t.Fatalf("mkdir run path: %v", err)
|
||||
}
|
||||
|
||||
// Make parent non-writable so removing child fails.
|
||||
if err := os.Chmod(parent, 0o500); err != nil {
|
||||
t.Fatalf("chmod parent: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = os.Chmod(parent, 0o700)
|
||||
})
|
||||
|
||||
runDir := &RunDirectory{path: runPath, retention: "auto"}
|
||||
err := runDir.ApplyRetention(RetentionDecisionInput{RunSucceeded: true, HasSkippedCorrections: false})
|
||||
if err == nil {
|
||||
t.Fatalf("expected removal error, got nil")
|
||||
}
|
||||
}
|
||||
242
internal/core/diagnostics/run_dir.go
Normal file
242
internal/core/diagnostics/run_dir.go
Normal file
@@ -0,0 +1,242 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
// RunDirectory represents a per-run diagnostics directory
|
||||
type RunDirectory struct {
|
||||
path string
|
||||
retention string
|
||||
createdAt time.Time
|
||||
}
|
||||
|
||||
type RetentionDecisionInput struct {
|
||||
RetentionMode string
|
||||
RunSucceeded bool
|
||||
HasSkippedCorrections bool
|
||||
}
|
||||
|
||||
func ShouldRetainRunDirectory(input RetentionDecisionInput) bool {
|
||||
// Failed runs are always retained.
|
||||
if !input.RunSucceeded {
|
||||
return true
|
||||
}
|
||||
|
||||
switch input.RetentionMode {
|
||||
case "always":
|
||||
return true
|
||||
case "never":
|
||||
return true
|
||||
case "auto":
|
||||
return input.HasSkippedCorrections
|
||||
default:
|
||||
// Be conservative for unknown values.
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// InvocationMetadata captures non-secret invocation details for diagnostics.
|
||||
type InvocationMetadata struct {
|
||||
Operation string `json:"operation"`
|
||||
TranscriptPath string `json:"transcript_path"`
|
||||
GlossaryPath string `json:"glossary_path"`
|
||||
OutputPath string `json:"output_path,omitempty"`
|
||||
ReportJSONPath string `json:"report_json_path,omitempty"`
|
||||
ConfigPath string `json:"config_path,omitempty"`
|
||||
ConfigSource string `json:"config_source,omitempty"`
|
||||
ConfigVersion *int `json:"config_version,omitempty"`
|
||||
TranscriptDescription string `json:"transcript_description,omitempty"`
|
||||
Modules []string `json:"modules"`
|
||||
RunID string `json:"run_id"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
}
|
||||
|
||||
// NewRunDirectory creates a new run directory under the configured work dir
|
||||
func NewRunDirectory(workDir, retention string) (*RunDirectory, error) {
|
||||
if workDir == "" {
|
||||
workDir = ".audita-runs"
|
||||
}
|
||||
|
||||
// Create work directory if it doesn't exist
|
||||
if err := os.MkdirAll(workDir, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create work directory %q: %w", workDir, err)
|
||||
}
|
||||
|
||||
// Create a unique run directory identifier.
|
||||
runID := fmt.Sprintf("run-%d", time.Now().UTC().UnixNano())
|
||||
runPath := filepath.Join(workDir, runID)
|
||||
|
||||
if err := os.Mkdir(runPath, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create run directory %q: %w", runPath, err)
|
||||
}
|
||||
|
||||
return &RunDirectory{
|
||||
path: runPath,
|
||||
retention: retention,
|
||||
createdAt: time.Now().UTC(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Path returns the run directory path
|
||||
func (r *RunDirectory) Path() string {
|
||||
return r.path
|
||||
}
|
||||
|
||||
func (r *RunDirectory) runID() string {
|
||||
return filepath.Base(r.path)
|
||||
}
|
||||
|
||||
// WriteInvocationMetadata writes invocation metadata for this run.
|
||||
func (r *RunDirectory) WriteInvocationMetadata(metadata InvocationMetadata) error {
|
||||
if metadata.RunID == "" {
|
||||
metadata.RunID = r.runID()
|
||||
}
|
||||
if metadata.StartedAt.IsZero() {
|
||||
metadata.StartedAt = r.createdAt
|
||||
}
|
||||
|
||||
path := filepath.Join(r.path, ArtifactInvocationMetadata)
|
||||
bytes, err := json.MarshalIndent(metadata, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal invocation metadata: %w", err)
|
||||
}
|
||||
bytes = append(bytes, '\n')
|
||||
if err := os.WriteFile(path, bytes, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write invocation metadata: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteEffectiveConfig writes redacted effective config metadata for this run.
|
||||
func (r *RunDirectory) WriteEffectiveConfig(cfg config.Config) error {
|
||||
path := filepath.Join(r.path, ArtifactEffectiveConfig)
|
||||
redacted := cfg.Redacted()
|
||||
bytes, err := json.MarshalIndent(redacted, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal effective config: %w", err)
|
||||
}
|
||||
bytes = append(bytes, '\n')
|
||||
if err := os.WriteFile(path, bytes, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write effective config: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteSourceTranscript writes the source transcript artifact
|
||||
func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript, raw []byte) error {
|
||||
// Write raw source for reference
|
||||
sourcePath := filepath.Join(r.path, ArtifactSourceTranscript)
|
||||
if err := os.WriteFile(sourcePath, raw, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write source transcript: %w", err)
|
||||
}
|
||||
|
||||
// Write parsed source for debugging
|
||||
parsedPath := filepath.Join(r.path, ArtifactParsedSourceTranscript)
|
||||
parsedBytes, err := json.MarshalIndent(transcript, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal parsed source transcript: %w", err)
|
||||
}
|
||||
parsedBytes = append(parsedBytes, '\n')
|
||||
if err := os.WriteFile(parsedPath, parsedBytes, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write parsed source transcript: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteNormalizedTranscript writes the normalized transcript artifact
|
||||
func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript) error {
|
||||
normalizedPath := filepath.Join(r.path, ArtifactNormalizedTranscript)
|
||||
bytes, err := schema.TranscriptToJSON(transcript)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to serialize normalized transcript: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(normalizedPath, bytes, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write normalized transcript: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteNormalizationSummary writes the normalization summary artifact
|
||||
func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.NormalizationSummary) error {
|
||||
summaryPath := filepath.Join(r.path, ArtifactNormalizationSummary)
|
||||
bytes, err := json.MarshalIndent(summary, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal normalization summary: %w", err)
|
||||
}
|
||||
bytes = append(bytes, '\n')
|
||||
if err := os.WriteFile(summaryPath, bytes, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write normalization summary: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteReport writes the authoritative report artifact
|
||||
func (r *RunDirectory) WriteReport(report reporting.ProcessReport) error {
|
||||
reportPath := filepath.Join(r.path, ArtifactReport)
|
||||
bytes, err := json.MarshalIndent(report, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal report: %w", err)
|
||||
}
|
||||
bytes = append(bytes, '\n')
|
||||
if err := os.WriteFile(reportPath, bytes, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write report: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteErrorLog writes an error log on failure
|
||||
func (r *RunDirectory) WriteErrorLog(errorMessage string) error {
|
||||
errorPath := filepath.Join(r.path, ArtifactErrorLog)
|
||||
return os.WriteFile(errorPath, []byte(errorMessage+"\n"), 0o644)
|
||||
}
|
||||
|
||||
// WriteChunkingSummary writes the chunking summary artifact
|
||||
func (r *RunDirectory) WriteChunkingSummary(summary *chunking.DetailedSummary) error {
|
||||
summaryPath := filepath.Join(r.path, ArtifactChunkingSummary)
|
||||
bytes, err := json.MarshalIndent(summary, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal chunking summary: %w", err)
|
||||
}
|
||||
bytes = append(bytes, '\n')
|
||||
if err := os.WriteFile(summaryPath, bytes, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write chunking summary: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *RunDirectory) WriteJSONArtifact(name string, payload any) error {
|
||||
artifactPath := filepath.Join(r.path, name)
|
||||
bytes, err := json.MarshalIndent(payload, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal %s: %w", name, err)
|
||||
}
|
||||
bytes = append(bytes, '\n')
|
||||
if err := os.WriteFile(artifactPath, bytes, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write %s: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *RunDirectory) ApplyRetention(input RetentionDecisionInput) error {
|
||||
decision := input
|
||||
if decision.RetentionMode == "" {
|
||||
decision.RetentionMode = r.retention
|
||||
}
|
||||
|
||||
if ShouldRetainRunDirectory(decision) {
|
||||
return nil
|
||||
}
|
||||
return os.RemoveAll(r.path)
|
||||
}
|
||||
29
internal/core/io/files.go
Normal file
29
internal/core/io/files.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package io
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func ReadRequiredFile(path string, label string) ([]byte, error) {
|
||||
contents, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read %s file %q: %w", label, path, err)
|
||||
}
|
||||
return contents, nil
|
||||
}
|
||||
|
||||
func ValidateWellFormedJSON(path string, raw []byte) error {
|
||||
if !json.Valid(raw) {
|
||||
return fmt.Errorf("transcript file %q is not valid JSON", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func WriteFile(path string, contents []byte) error {
|
||||
if err := os.WriteFile(path, contents, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write output file %q: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
35
internal/core/modulecatalog/catalog.go
Normal file
35
internal/core/modulecatalog/catalog.go
Normal file
@@ -0,0 +1,35 @@
|
||||
package modulecatalog
|
||||
|
||||
import "strings"
|
||||
|
||||
const (
|
||||
KeyGlossary = "glossary"
|
||||
KeyHomophones = "homophones"
|
||||
KeySpokenWord = "spoken_word"
|
||||
KeyGrammar = "grammar"
|
||||
)
|
||||
|
||||
var supportedKeys = []string{
|
||||
KeyGlossary,
|
||||
KeyHomophones,
|
||||
KeySpokenWord,
|
||||
KeyGrammar,
|
||||
}
|
||||
|
||||
var supportedKeySet = map[string]struct{}{
|
||||
KeyGlossary: {},
|
||||
KeyHomophones: {},
|
||||
KeySpokenWord: {},
|
||||
KeyGrammar: {},
|
||||
}
|
||||
|
||||
func SupportedKeys() []string {
|
||||
out := make([]string, len(supportedKeys))
|
||||
copy(out, supportedKeys)
|
||||
return out
|
||||
}
|
||||
|
||||
func IsSupported(key string) bool {
|
||||
_, ok := supportedKeySet[strings.TrimSpace(key)]
|
||||
return ok
|
||||
}
|
||||
24
internal/core/modulecatalog/catalog_test.go
Normal file
24
internal/core/modulecatalog/catalog_test.go
Normal file
@@ -0,0 +1,24 @@
|
||||
package modulecatalog
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSupportedKeys(t *testing.T) {
|
||||
want := []string{KeyGlossary, KeyHomophones, KeySpokenWord, KeyGrammar}
|
||||
if got := SupportedKeys(); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("unexpected supported keys: got=%v want=%v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsSupported(t *testing.T) {
|
||||
for _, key := range SupportedKeys() {
|
||||
if !IsSupported(key) {
|
||||
t.Fatalf("expected key %q to be supported", key)
|
||||
}
|
||||
}
|
||||
if IsSupported("made_up") {
|
||||
t.Fatalf("did not expect made_up to be supported")
|
||||
}
|
||||
}
|
||||
206
internal/core/normalization/normalize.go
Normal file
206
internal/core/normalization/normalize.go
Normal file
@@ -0,0 +1,206 @@
|
||||
package normalization
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
// NormalizationConfig holds configuration parameters for transcript normalization
|
||||
type NormalizationConfig struct {
|
||||
MaxSegmentGap float64 // Maximum gap between segments to consider for merging (seconds)
|
||||
EllipsisGap float64 // Gap threshold above which to insert ellipsis (seconds)
|
||||
MaxSegmentDuration float64 // Maximum duration for a merged segment (seconds)
|
||||
MaxSegmentTokens int // Maximum token estimate for a merged segment
|
||||
}
|
||||
|
||||
// NormalizationSummary records the results and statistics of normalization
|
||||
type NormalizationSummary struct {
|
||||
InputSegmentCount int `json:"input_segment_count"`
|
||||
OutputSegmentCount int `json:"output_segment_count"`
|
||||
MergesPerformed int `json:"merges_performed"`
|
||||
IDsReassigned int `json:"ids_reassigned"`
|
||||
SkippedMerges struct {
|
||||
DifferentSpeakers int `json:"different_speakers"`
|
||||
GapTooLarge int `json:"gap_too_large"`
|
||||
DurationExceeded int `json:"duration_exceeded"`
|
||||
TokenLimitExceeded int `json:"token_limit_exceeded"`
|
||||
} `json:"skipped_merges"`
|
||||
}
|
||||
|
||||
// NormalizeTranscript performs deterministic normalization on a transcript
|
||||
type NormalizeTranscript struct {
|
||||
config NormalizationConfig
|
||||
estimator *SimpleTokenEstimator
|
||||
}
|
||||
|
||||
// NewNormalizer creates a new transcript normalizer with the given configuration
|
||||
func NewNormalizer(config NormalizationConfig) *NormalizeTranscript {
|
||||
return &NormalizeTranscript{
|
||||
config: config,
|
||||
estimator: &SimpleTokenEstimator{},
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize performs deterministic normalization on the given transcript
|
||||
func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.Transcript, *NormalizationSummary) {
|
||||
summary := &NormalizationSummary{
|
||||
InputSegmentCount: len(transcript.Segments),
|
||||
}
|
||||
|
||||
if len(transcript.Segments) == 0 {
|
||||
return transcript, summary
|
||||
}
|
||||
|
||||
// Sort segments chronologically
|
||||
sortedSegments := make([]schema.Segment, len(transcript.Segments))
|
||||
copy(sortedSegments, transcript.Segments)
|
||||
sort.Slice(sortedSegments, func(i, j int) bool {
|
||||
return sortedSegments[i].Start < sortedSegments[j].Start
|
||||
})
|
||||
|
||||
// Merge same-speaker adjacent segments
|
||||
var normalizedSegments []schema.Segment
|
||||
var currentSegment schema.Segment
|
||||
|
||||
for i, segment := range sortedSegments {
|
||||
if i == 0 {
|
||||
// Initialize with first segment
|
||||
currentSegment = segment
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if we should merge with current segment
|
||||
gap := segment.Start - currentSegment.End
|
||||
merge, reason := shouldMergeWithReason(n.config, n.estimator, ¤tSegment, &segment, gap)
|
||||
if merge {
|
||||
summary.MergesPerformed++
|
||||
currentSegment = mergeSegments(¤tSegment, &segment, gap, n.config.EllipsisGap)
|
||||
} else {
|
||||
// Track the reason for not merging
|
||||
switch reason {
|
||||
case "different_speakers":
|
||||
summary.SkippedMerges.DifferentSpeakers++
|
||||
case "gap_too_large":
|
||||
summary.SkippedMerges.GapTooLarge++
|
||||
case "duration_exceeded":
|
||||
summary.SkippedMerges.DurationExceeded++
|
||||
case "token_limit_exceeded":
|
||||
summary.SkippedMerges.TokenLimitExceeded++
|
||||
}
|
||||
// Finalize current segment and start new one
|
||||
normalizedSegments = append(normalizedSegments, currentSegment)
|
||||
currentSegment = segment
|
||||
}
|
||||
}
|
||||
|
||||
// Add the last segment
|
||||
if len(currentSegment.Text) > 0 {
|
||||
normalizedSegments = append(normalizedSegments, currentSegment)
|
||||
}
|
||||
|
||||
// Reassign sequential IDs starting at 1
|
||||
for i, segment := range normalizedSegments {
|
||||
if segment.ID != i+1 {
|
||||
summary.IDsReassigned++
|
||||
}
|
||||
normalizedSegments[i].ID = i + 1
|
||||
}
|
||||
|
||||
summary.OutputSegmentCount = len(normalizedSegments)
|
||||
|
||||
return &schema.Transcript{Segments: normalizedSegments}, summary
|
||||
}
|
||||
|
||||
func shouldMerge(config NormalizationConfig, estimator *SimpleTokenEstimator,
|
||||
current, next *schema.Segment, gap float64) bool {
|
||||
|
||||
// Don't merge if different speakers
|
||||
if current.Speaker != next.Speaker {
|
||||
return false
|
||||
}
|
||||
|
||||
// Don't merge if gap is too large
|
||||
if gap > config.MaxSegmentGap {
|
||||
return false
|
||||
}
|
||||
|
||||
// Calculate what the merged segment would look like
|
||||
mergedText := current.Text
|
||||
if gap >= config.EllipsisGap {
|
||||
mergedText += "... "
|
||||
} else {
|
||||
mergedText += " "
|
||||
}
|
||||
mergedText += next.Text
|
||||
|
||||
mergedDuration := next.End - current.Start
|
||||
|
||||
// Don't merge if duration would be exceeded
|
||||
if mergedDuration > config.MaxSegmentDuration {
|
||||
return false
|
||||
}
|
||||
|
||||
// Don't merge if token estimate would be exceeded
|
||||
tokenEstimate := estimator.EstimateTokens(mergedText)
|
||||
if tokenEstimate > config.MaxSegmentTokens {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func shouldMergeWithReason(config NormalizationConfig, estimator *SimpleTokenEstimator,
|
||||
current, next *schema.Segment, gap float64) (bool, string) {
|
||||
|
||||
if current.Speaker != next.Speaker {
|
||||
return false, "different_speakers"
|
||||
}
|
||||
if gap > config.MaxSegmentGap {
|
||||
return false, "gap_too_large"
|
||||
}
|
||||
|
||||
mergedText := current.Text
|
||||
if gap >= config.EllipsisGap {
|
||||
mergedText += "... "
|
||||
} else {
|
||||
mergedText += " "
|
||||
}
|
||||
mergedText += next.Text
|
||||
|
||||
mergedDuration := next.End - current.Start
|
||||
if mergedDuration > config.MaxSegmentDuration {
|
||||
return false, "duration_exceeded"
|
||||
}
|
||||
|
||||
tokenEstimate := estimator.EstimateTokens(mergedText)
|
||||
if tokenEstimate > config.MaxSegmentTokens {
|
||||
return false, "token_limit_exceeded"
|
||||
}
|
||||
|
||||
return true, ""
|
||||
}
|
||||
|
||||
func mergeSegments(current, next *schema.Segment, gap, ellipsisGap float64) schema.Segment {
|
||||
mergedText := current.Text
|
||||
if gap >= ellipsisGap {
|
||||
mergedText += "... "
|
||||
} else {
|
||||
mergedText += " "
|
||||
}
|
||||
mergedText += next.Text
|
||||
|
||||
// Merge categories
|
||||
categories := make([]string, 0, len(current.Categories)+len(next.Categories))
|
||||
categories = append(categories, current.Categories...)
|
||||
categories = append(categories, next.Categories...)
|
||||
|
||||
return schema.Segment{
|
||||
ID: current.ID, // Will be reassigned later
|
||||
Speaker: current.Speaker,
|
||||
Start: current.Start,
|
||||
End: next.End,
|
||||
Text: mergedText,
|
||||
Categories: categories,
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user