Compare commits
76 Commits
python-fin
...
v0.9.1
| Author | SHA1 | Date | |
|---|---|---|---|
| a84941d681 | |||
| 46b7356a3b | |||
| 39208cd119 | |||
| 6dae15656d | |||
| b103bb2e7d | |||
| 2b2a3fc024 | |||
| 52ffe42e73 | |||
| 3b160cf05b | |||
| 0687982822 | |||
| ff4ed82239 | |||
| 037121e9ce | |||
| d6126bf52b | |||
| 1bc5936681 | |||
| 9a77a0cd0b | |||
| ebbd2c8a63 | |||
| de99467ede | |||
| 20f612215f | |||
| 3d45571bb0 | |||
| 1afd753fad | |||
| a85a7e204e | |||
| 509436cc4a | |||
| a48f6da1f4 | |||
| df96f9fdf6 | |||
| 390daa8b84 | |||
| af84249da0 | |||
| cad172a758 | |||
| fb59cb21b9 | |||
| 68e2d9b549 | |||
| 185f7ca2b6 | |||
| 7ccadc6bd6 | |||
| a9f7fa27ff | |||
| dbf3605712 | |||
| 543a7ff8ef | |||
| fc3a7b7a67 | |||
| b360493cdc | |||
| 12202508bf | |||
| 6d9a4bd017 | |||
| 426864eedb | |||
| 0b17a6fbeb | |||
| aeb31f1c0d | |||
| 28fe899aa1 | |||
| 30606f5c49 | |||
| db880ed868 | |||
| 5217093be2 | |||
| 0e83991537 | |||
| 3e8d19cccd | |||
| c1193e3450 | |||
| 73249b63d8 | |||
| c3087aeda6 | |||
| 0452a605ad | |||
| b9b7384123 | |||
| 726acc47e1 | |||
| 5c78b1d5d9 | |||
| f461922b9b | |||
| c58d307ba7 | |||
| 1eb93481e0 | |||
| b997e7c97c | |||
| 12fd541669 | |||
| 10377876e4 | |||
| d847168ecd | |||
| 14e51698c2 | |||
| aeb9c4f062 | |||
| e2ae7f77d8 | |||
| 0b1b670baf | |||
| 3cfa4b6e8a | |||
| 950edc01f2 | |||
| ea8def423e | |||
| 95fe8c32fa | |||
| 08b7531149 | |||
| 2cf2d390da | |||
| 8f3c2ec5fd | |||
| 9427c4e6cc | |||
| 6424d7db4f | |||
| 87e560dd3d | |||
| 09fc6fd364 | |||
| 2e47c8a1b6 |
61
.gitignore
vendored
61
.gitignore
vendored
@@ -1,3 +1,4 @@
|
||||
# --> Python
|
||||
.DS_Store
|
||||
.venv/
|
||||
__pycache__/
|
||||
@@ -9,3 +10,63 @@ dist/
|
||||
build/
|
||||
*.egg-info/
|
||||
|
||||
# ---> Codex
|
||||
.codex
|
||||
AGENTS.md
|
||||
|
||||
# ---> Go
|
||||
# If you prefer the allow list template instead of the deny list, see community template:
|
||||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||
#
|
||||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
|
||||
# Dependency directories (remove the comment below to include it)
|
||||
# vendor/
|
||||
|
||||
# Go workspace file
|
||||
go.work
|
||||
go.work.sum
|
||||
|
||||
# env file
|
||||
.env
|
||||
|
||||
# Compiled binaries and test configuration
|
||||
narratio
|
||||
local-test
|
||||
pipeline.yml
|
||||
bin/
|
||||
|
||||
# Local run artifacts
|
||||
.audita-runs/
|
||||
report.json
|
||||
corrected.json
|
||||
normalized.json
|
||||
|
||||
# Coverage artifacts
|
||||
coverage.out
|
||||
coverage.txt
|
||||
|
||||
# ---> VisualStudioCode
|
||||
.vscode/*
|
||||
!.vscode/settings.json
|
||||
!.vscode/tasks.json
|
||||
!.vscode/launch.json
|
||||
!.vscode/extensions.json
|
||||
!.vscode/*.code-snippets
|
||||
|
||||
# Local History for Visual Studio Code
|
||||
.history/
|
||||
|
||||
# Built Visual Studio Code Extensions
|
||||
*.vsix
|
||||
|
||||
50
.woodpecker/release.yml
Normal file
50
.woodpecker/release.yml
Normal file
@@ -0,0 +1,50 @@
|
||||
when:
|
||||
- event: tag
|
||||
|
||||
steps:
|
||||
- name: build-release-assets
|
||||
image: golang:1.25
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
|
||||
version="$CI_COMMIT_TAG"
|
||||
dist="dist"
|
||||
pkg="gitea.maximumdirect.net/eric/audita/cmd/audita"
|
||||
|
||||
rm -rf "$dist"
|
||||
mkdir -p "$dist"
|
||||
|
||||
build_binary() {
|
||||
goos="$1"
|
||||
goarch="$2"
|
||||
suffix="$3"
|
||||
output="$dist/audita-$version-$goos-$goarch$suffix"
|
||||
|
||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \
|
||||
go build -trimpath -ldflags "-s -w -X gitea.maximumdirect.net/eric/audita/internal/buildinfo.Version=$version" \
|
||||
-o "$output" "$pkg"
|
||||
}
|
||||
|
||||
build_binary linux amd64 ""
|
||||
build_binary linux arm64 ""
|
||||
build_binary darwin amd64 ""
|
||||
build_binary darwin arm64 ""
|
||||
build_binary windows amd64 ".exe"
|
||||
build_binary windows arm64 ".exe"
|
||||
|
||||
- name: publish-release
|
||||
image: woodpeckerci/plugin-release
|
||||
depends_on:
|
||||
- build-release-assets
|
||||
settings:
|
||||
api_key:
|
||||
from_secret: GITEA_RELEASE_TOKEN
|
||||
files:
|
||||
- dist/audita-*
|
||||
checksum: sha256
|
||||
checksum-file: SHA256SUMS
|
||||
checksum-flatten: true
|
||||
file-exists: skip
|
||||
overwrite: false
|
||||
prerelease: false
|
||||
363
README.md
363
README.md
@@ -1,148 +1,303 @@
|
||||
# Audita
|
||||
|
||||
Audita is a framework-first transcript correction application. The public `audita` package provides:
|
||||
Audita is a transcript polishing CLI.
|
||||
|
||||
- deterministic transcript normalization
|
||||
- token-batched module orchestration
|
||||
- concrete `glossary`, `homophones`, `spoken_word`, and `grammar` modules built on reusable proposal / validator contracts
|
||||
- structured run reporting and work-dir diagnostics
|
||||
`audita process` validates transcript/glossary input, normalizes and chunks transcript segments, runs the default correction pipeline, and emits corrected transcript output plus machine-readable diagnostics and reports.
|
||||
|
||||
The previous working implementation has been preserved as `audita_prototype` inside this repository. Its full regression suite lives under `tests/audita_prototype`.
|
||||
## What Audita Does
|
||||
|
||||
## Development
|
||||
Default module sequence:
|
||||
- `glossary`
|
||||
- `homophones`
|
||||
- `glossary`
|
||||
- `spoken_word`
|
||||
- `grammar`
|
||||
|
||||
This project is set up for `uv`.
|
||||
Pipeline behavior includes:
|
||||
- glossary-backed domain/acoustic corrections
|
||||
- conservative homophone and mistranscription corrections
|
||||
- conservative spoken-word dysfluency cleanup with semantic guardrails
|
||||
- grammar/punctuation/capitalization/formatting cleanup
|
||||
- validator-chain enforcement before application
|
||||
- run reports and diagnostics artifacts with secret redaction
|
||||
|
||||
## Build and Install
|
||||
|
||||
Build a local binary:
|
||||
|
||||
```sh
|
||||
uv sync --extra dev
|
||||
uv run pytest
|
||||
go build -o ./bin/audita ./cmd/audita
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
Process a transcript with the current framework implementation:
|
||||
Install into your Go bin directory:
|
||||
|
||||
```sh
|
||||
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
go install ./cmd/audita
|
||||
```
|
||||
|
||||
The framework currently runs this default module sequence:
|
||||
|
||||
1. `glossary`
|
||||
2. `homophones`
|
||||
3. `glossary`
|
||||
4. `spoken_word`
|
||||
5. `grammar`
|
||||
|
||||
Resolved run instance names are auto-numbered for repeats, so the default report pipeline is:
|
||||
|
||||
1. `glossary_1`
|
||||
2. `homophones`
|
||||
3. `glossary_2`
|
||||
4. `spoken_word`
|
||||
5. `grammar`
|
||||
|
||||
The default module sequence is fully implemented today:
|
||||
|
||||
- `glossary` proposes glossary-supported acoustic corrections
|
||||
- `homophones` proposes conservative homophone and mistranscription corrections
|
||||
- `spoken_word` proposes conservative dysfluency cleanup
|
||||
- `grammar` proposes punctuation, capitalization, and spacing cleanup only
|
||||
|
||||
To run a custom module sequence, pass `--modules`:
|
||||
CLI help:
|
||||
|
||||
```sh
|
||||
uv run audita process transcript.json --glossary glossary.yaml --modules grammar --output corrected.json
|
||||
audita --help
|
||||
audita process --help
|
||||
audita config --help
|
||||
```
|
||||
|
||||
To also write a structured JSON report:
|
||||
## Test
|
||||
|
||||
Run all tests:
|
||||
|
||||
```sh
|
||||
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json --report-json report.json
|
||||
go test ./...
|
||||
```
|
||||
|
||||
From a checked-out repository, you can also use the root launcher:
|
||||
## Basic Usage
|
||||
|
||||
Required inputs:
|
||||
- transcript JSON path (positional argument)
|
||||
- `--glossary <glossary.yaml>`
|
||||
|
||||
Recommended run:
|
||||
|
||||
```sh
|
||||
./audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
audita process transcript.json \
|
||||
--glossary glossary.yaml \
|
||||
--output corrected.json \
|
||||
--report-json report.json
|
||||
```
|
||||
|
||||
For a system-wide command, install the source tree under `/usr/local/src/audita`, sync dependencies there, and symlink the root launcher into your `PATH`:
|
||||
Select an explicit output schema (default is `bare-segments`):
|
||||
|
||||
```sh
|
||||
cd /usr/local/src/audita
|
||||
uv sync --extra dev
|
||||
ln -s /usr/local/src/audita/audita /usr/local/bin/audita
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
audita process transcript.json \
|
||||
--glossary glossary.yaml \
|
||||
--output-schema audita-v1 \
|
||||
--output corrected.json \
|
||||
--report-json report.json
|
||||
```
|
||||
|
||||
Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr.
|
||||
`--report-json` writes a separate machine-readable run report and never mixes report data into stdout.
|
||||
|
||||
Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Default OpenRouter runs require LLM API credentials, because the `glossary`, `homophones`, `spoken_word`, and `grammar` modules make real LLM calls. Self-hosted or other non-default OpenAI-compatible endpoints may not require credentials. `AUDITA_LLM_API_KEY` and `--llm-api-key` are the preferred provider-neutral credential surfaces, while `OPENROUTER_API_KEY` remains supported as a backward-compatible fallback.
|
||||
|
||||
| Environment variable | CLI flag | Default | Purpose |
|
||||
| --- | --- | --- | --- |
|
||||
| `AUDITA_MODULES` | `--modules` | `glossary,homophones,glossary,spoken_word,grammar` | Comma-separated logical module keys to run; CLI overrides the environment value |
|
||||
| `AUDITA_LLM_API_KEY` | `--llm-api-key` | unset | Preferred provider-neutral LLM API credential; required for the default OpenRouter endpoint and optional for non-default endpoints; CLI overrides both environment-key variants |
|
||||
| `AUDITA_VALIDATION_LLM_API_KEY` | `--validation-llm-api-key` | unset | Validation-phase LLM API credential; defaults to the primary LLM API key and is optional for non-default validation endpoints |
|
||||
| `AUDITA_MODEL` | `--model` | `openrouter/google/gemma-4-31b-it` | LLM model name sent to the configured OpenAI-compatible endpoint |
|
||||
| `AUDITA_VALIDATION_MODEL` | `--validation-model` | unset | Validation-phase LLM model; defaults to `AUDITA_MODEL` |
|
||||
| `AUDITA_BASE_URL` | `--base-url` | `https://openrouter.ai/api/v1` | OpenAI-compatible API base URL |
|
||||
| `AUDITA_VALIDATION_BASE_URL` | `--validation-base-url` | unset | Validation-phase OpenAI-compatible API base URL; defaults to `AUDITA_BASE_URL` |
|
||||
| `AUDITA_LLM_TIMEOUT_SECONDS` | `--llm-timeout-seconds` | `600` | Per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint |
|
||||
| `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | `--validation-llm-timeout-seconds` | unset | Validation-phase per-request timeout in seconds; defaults to `AUDITA_LLM_TIMEOUT_SECONDS` |
|
||||
| `AUDITA_VALIDATION_MAX_PROMPT_TOKENS` | `--validation-max-prompt-tokens` | `2048` | Maximum estimated tokens per validation-phase LLM prompt batch |
|
||||
| `AUDITA_TARGET_SECTIONS` | `--target-sections` | unset | Exact number of contiguous proposal-stage transcript sections; errors if min/max token bounds cannot be satisfied |
|
||||
| `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses |
|
||||
| `AUDITA_VALIDATION_MAX_RETRIES` | `--validation-max-retries` | unset | Validation-phase structured-output retries; defaults to `AUDITA_MAX_RETRIES` |
|
||||
| `AUDITA_VALIDATION_LLM_CONCURRENCY` | `--validation-llm-concurrency` | unset | Validation-phase LLM concurrency; defaults to `AUDITA_LLM_CONCURRENCY` |
|
||||
| `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `8192` | Maximum estimated tokens per proposal-stage transcript section |
|
||||
| `AUDITA_MIN_SECTION_TOKENS` | `--min-section-tokens` | `2048` | Minimum estimated tokens per proposal-stage transcript section when balancing for concurrency |
|
||||
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation |
|
||||
| `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | `--grammar-confidence-threshold` | `0.8` | Minimum confidence required for grammar proposals to survive validation |
|
||||
| `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | `--homophones-confidence-threshold` | `0.8` | Minimum confidence required for homophone proposals to survive validation |
|
||||
| `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | `--spoken-word-confidence-threshold` | `0.8` | Minimum confidence required for spoken-word proposals to survive validation |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | `--normalize-max-segment-gap` | `4.0` | Same-speaker gaps eligible for deterministic merging |
|
||||
| `AUDITA_NORMALIZE_ELLIPSIS_GAP` | `--normalize-ellipsis-gap` | `3.5` | Same-speaker gaps above this value are joined with ` ... ` |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | `--normalize-max-segment-duration` | `60.0` | Maximum merged segment duration |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | `--normalize-max-segment-tokens` | `2048` | Maximum merged segment prompt payload size |
|
||||
| `AUDITA_WORK_DIR` | `--work-dir` | `/tmp/audita` | Per-run scratch diagnostics directory |
|
||||
| `AUDITA_WORK_DIR_RETENTION` | `--work-dir-retention` | `auto` | Whether to retain the per-run work directory: `auto`, `always`, or `never` |
|
||||
|
||||
Set `AUDITA_MODULES=grammar` to run only the grammar module by default, or override it per command with `--modules`.
|
||||
|
||||
Validation-phase LLM settings inherit from the primary `AUDITA_*` LLM settings by default. Set any of the `AUDITA_VALIDATION_*` values only when you want LLM-backed validators to use a different model, endpoint, credential, timeout, retry budget, or concurrency level.
|
||||
|
||||
OpenRouter remains the default out of the box:
|
||||
Recommended config-based run:
|
||||
|
||||
```sh
|
||||
export AUDITA_LLM_API_KEY=your-openrouter-key
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
audita process transcript.json \
|
||||
--glossary glossary.yaml \
|
||||
--config audita.yml \
|
||||
--output corrected.json \
|
||||
--report-json report.json
|
||||
```
|
||||
|
||||
You can point Audita at any OpenAI-compatible endpoint by changing `AUDITA_BASE_URL` and, if needed, `AUDITA_MODEL`. For example, a local vLLM server:
|
||||
Explicit module override:
|
||||
|
||||
```sh
|
||||
export AUDITA_BASE_URL=http://localhost:8000/v1
|
||||
export AUDITA_MODEL=meta-llama/Llama-3.1-8B-Instruct
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
audita process transcript.json \
|
||||
--glossary glossary.yaml \
|
||||
--modules glossary,homophones,grammar \
|
||||
--output corrected.json \
|
||||
--report-json report.json
|
||||
```
|
||||
|
||||
If your self-hosted endpoint requires authentication, you can still set `AUDITA_LLM_API_KEY`; Audita simply no longer requires it for non-default endpoints.
|
||||
|
||||
Or the actual OpenAI API:
|
||||
Optional transcript background context:
|
||||
|
||||
```sh
|
||||
export AUDITA_LLM_API_KEY=your-openai-key
|
||||
export AUDITA_BASE_URL=https://api.openai.com/v1
|
||||
export AUDITA_MODEL=gpt-4.1-mini
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
audita process transcript.json \
|
||||
--glossary glossary.yaml \
|
||||
--transcript-description "Brief context that may help resolve ambiguous terms." \
|
||||
--output corrected.json
|
||||
```
|
||||
|
||||
`AUDITA_WORK_DIR` stores per-run diagnostics while processing. Under the default `AUDITA_WORK_DIR_RETENTION=auto`, clean successful runs are removed, while failed runs and successful runs with final skipped corrections are preserved. Use `always` to keep every run directory and `never` to remove successful run directories even when skips remain.
|
||||
Failed runs always preserve the run directory and include an authoritative `report.json` alongside normalization and prompt/response diagnostics.
|
||||
The transcript description is background context only and does not override transcript content.
|
||||
|
||||
## Prototype Archive
|
||||
Write transcript JSON to stdout (no `--output`):
|
||||
|
||||
The archived prototype remains importable as `audita_prototype` and is still covered by its original regression suite. This is intentional: the new `audita` package is a framework-oriented rewrite, not a thin wrapper around the old code.
|
||||
```sh
|
||||
audita process transcript.json --glossary glossary.yaml
|
||||
```
|
||||
|
||||
Control diagnostics location/retention:
|
||||
|
||||
```sh
|
||||
audita process transcript.json \
|
||||
--glossary glossary.yaml \
|
||||
--work-dir /tmp/audita \
|
||||
--work-dir-retention auto \
|
||||
--output corrected.json \
|
||||
--report-json report.json
|
||||
```
|
||||
|
||||
## Stdout/Stderr Contract
|
||||
|
||||
- With `--output`, stdout is expected to be empty on success.
|
||||
- Without `--output`, stdout contains transcript JSON only on success.
|
||||
- `--report-json` writes a file and is never printed to stdout.
|
||||
- stderr is human-readable diagnostics/errors.
|
||||
|
||||
For subprocess orchestration guidance, see [`docs/subprocess-operations.md`](docs/subprocess-operations.md).
|
||||
|
||||
## Configuration
|
||||
|
||||
Precedence:
|
||||
1. defaults
|
||||
2. config file (`--config`, `AUDITA_CONFIG`, or default search paths when present: `/usr/local/etc/audita/config.yml`, then `/etc/audita/config.yml`)
|
||||
3. environment (`AUDITA_*`)
|
||||
4. CLI flags
|
||||
|
||||
Config commands:
|
||||
|
||||
```sh
|
||||
audita config validate --config audita.yml
|
||||
audita config print-effective --config audita.yml
|
||||
```
|
||||
|
||||
For full config-file schema and examples, see [`docs/configuration.md`](docs/configuration.md).
|
||||
For output-schema details, see [`docs/output-schemas.md`](docs/output-schemas.md).
|
||||
For built-in validator keys and chain definitions, see [`docs/validators.md`](docs/validators.md).
|
||||
For embedded prompt assets and prompt metadata behavior, see [`docs/prompts.md`](docs/prompts.md).
|
||||
For CLI/process compatibility guarantees, see [`docs/public-contract.md`](docs/public-contract.md).
|
||||
|
||||
### Modules
|
||||
|
||||
- `AUDITA_MODULES` (CSV)
|
||||
- CLI: `--modules`
|
||||
|
||||
### Transcript Description
|
||||
|
||||
CLI:
|
||||
- `--transcript-description`
|
||||
|
||||
Behavior:
|
||||
- optional background context for proposal and LLM-validator prompts;
|
||||
- trimmed and length-limited by CLI validation;
|
||||
- does not override transcript content;
|
||||
- no `AUDITA_*` environment variable is currently defined for this setting.
|
||||
|
||||
### Primary LLM
|
||||
|
||||
Environment:
|
||||
- `AUDITA_LLM_API_KEY` (or `OPENROUTER_API_KEY` fallback)
|
||||
- `AUDITA_MODEL`
|
||||
- `AUDITA_BASE_URL`
|
||||
- `AUDITA_LLM_TIMEOUT_SECONDS`
|
||||
- `AUDITA_MAX_RETRIES`
|
||||
|
||||
CLI:
|
||||
- `--llm-api-key`
|
||||
- `--model`
|
||||
- `--base-url`
|
||||
- `--llm-timeout-seconds`
|
||||
- `--max-retries`
|
||||
|
||||
### Validation LLM
|
||||
|
||||
Environment:
|
||||
- `AUDITA_VALIDATION_LLM_API_KEY`
|
||||
- `AUDITA_VALIDATION_MODEL`
|
||||
- `AUDITA_VALIDATION_BASE_URL`
|
||||
- `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS`
|
||||
- `AUDITA_VALIDATION_MAX_RETRIES`
|
||||
- `AUDITA_VALIDATION_LLM_CONCURRENCY`
|
||||
- `AUDITA_VALIDATION_MAX_PROMPT_TOKENS`
|
||||
|
||||
CLI:
|
||||
- `--validation-llm-api-key`
|
||||
- `--validation-model`
|
||||
- `--validation-base-url`
|
||||
- `--validation-llm-timeout-seconds`
|
||||
- `--validation-max-retries`
|
||||
- `--validation-llm-concurrency`
|
||||
- `--validation-max-prompt-tokens`
|
||||
|
||||
### LLM Concurrency
|
||||
|
||||
Environment:
|
||||
- `AUDITA_TOTAL_LLM_CONCURRENCY`
|
||||
- `AUDITA_PROPOSAL_LLM_CONCURRENCY`
|
||||
- `AUDITA_VALIDATION_LLM_CONCURRENCY`
|
||||
- `AUDITA_LLM_CONCURRENCY` (legacy alias for `AUDITA_TOTAL_LLM_CONCURRENCY`)
|
||||
|
||||
CLI:
|
||||
- `--total-llm-concurrency`
|
||||
- `--proposal-llm-concurrency`
|
||||
- `--validation-llm-concurrency`
|
||||
- `--llm-concurrency` (legacy alias for `--total-llm-concurrency`)
|
||||
|
||||
Behavior:
|
||||
- all proposal and validation LLM calls are bounded by total LLM concurrency
|
||||
- proposal LLM calls are additionally bounded by proposal LLM concurrency
|
||||
- when validation concurrency is unset, it inherits total LLM concurrency
|
||||
- when explicitly set, proposal and validation concurrency must each be `<= total-llm-concurrency`
|
||||
- canonical total settings win when both canonical and legacy alias settings are provided at the same precedence layer
|
||||
|
||||
### Confidence Thresholds
|
||||
|
||||
Environment:
|
||||
- `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD`
|
||||
- `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD`
|
||||
- `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD`
|
||||
- `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD`
|
||||
|
||||
CLI:
|
||||
- `--glossary-confidence-threshold`
|
||||
- `--homophones-confidence-threshold`
|
||||
- `--spoken-word-confidence-threshold`
|
||||
- `--grammar-confidence-threshold`
|
||||
|
||||
### Normalization and Chunking
|
||||
|
||||
Environment:
|
||||
- `AUDITA_NORMALIZE_MAX_SEGMENT_GAP`
|
||||
- `AUDITA_NORMALIZE_ELLIPSIS_GAP`
|
||||
- `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION`
|
||||
- `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS`
|
||||
- `AUDITA_MAX_SECTION_TOKENS`
|
||||
- `AUDITA_MIN_SECTION_TOKENS`
|
||||
- `AUDITA_TARGET_SECTIONS`
|
||||
|
||||
CLI:
|
||||
- `--normalize-max-segment-gap`
|
||||
- `--normalize-ellipsis-gap`
|
||||
- `--normalize-max-segment-duration`
|
||||
- `--normalize-max-segment-tokens`
|
||||
- `--max-section-tokens`
|
||||
- `--min-section-tokens`
|
||||
- `--target-sections`
|
||||
|
||||
### Work Directory
|
||||
|
||||
Environment:
|
||||
- `AUDITA_WORK_DIR`
|
||||
- `AUDITA_WORK_DIR_RETENTION` (`auto`, `always`, `never`)
|
||||
|
||||
CLI:
|
||||
- `--work-dir`
|
||||
- `--work-dir-retention`
|
||||
|
||||
Retention behavior:
|
||||
- `always`: keep all run directories
|
||||
- `never`: keep successful run directories
|
||||
- `auto`: keep failed runs and successful runs with skipped/rejected corrections
|
||||
|
||||
## Reports and Diagnostics
|
||||
|
||||
Per-run diagnostics include:
|
||||
- source transcript artifacts
|
||||
- normalized transcript artifact
|
||||
- normalization summary
|
||||
- chunking summary
|
||||
- utilization diagnostics summary
|
||||
- correction ledger
|
||||
- invocation metadata
|
||||
- redacted effective config
|
||||
- module/validator prompt-response diagnostics
|
||||
- `report.json`
|
||||
- `error.log` on failure
|
||||
|
||||
Optional external report output:
|
||||
- `--report-json <path>`
|
||||
|
||||
## Documentation
|
||||
|
||||
- Architecture: [`docs/architecture.md`](docs/architecture.md)
|
||||
- Diagnostics: [`docs/diagnostics.md`](docs/diagnostics.md)
|
||||
- Structured LLM adapter: [`docs/structured-llm.md`](docs/structured-llm.md)
|
||||
- Subprocess operations: [`docs/subprocess-operations.md`](docs/subprocess-operations.md)
|
||||
- Release checklist: [`docs/release-checklist.md`](docs/release-checklist.md)
|
||||
|
||||
147
audita
147
audita
@@ -1,147 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
DEFAULT_WORK_DIR = "/tmp/audita"
|
||||
_SECRET_FLAGS = {"--llm-api-key", "--validation-llm-api-key"}
|
||||
|
||||
|
||||
def _redact_argv(argv: list[str]) -> list[str]:
|
||||
redacted: list[str] = []
|
||||
index = 0
|
||||
while index < len(argv):
|
||||
arg = argv[index]
|
||||
matched_flag = next((flag for flag in _SECRET_FLAGS if arg == flag or arg.startswith(flag + "=")), None)
|
||||
if matched_flag is None:
|
||||
redacted.append(arg)
|
||||
index += 1
|
||||
continue
|
||||
if arg == matched_flag:
|
||||
redacted.append(arg)
|
||||
if index + 1 < len(argv):
|
||||
redacted.append("[REDACTED]")
|
||||
index += 2
|
||||
else:
|
||||
index += 1
|
||||
continue
|
||||
redacted.append(f"{matched_flag}=[REDACTED]")
|
||||
index += 1
|
||||
return redacted
|
||||
|
||||
|
||||
def _resolve_work_root(argv: list[str]) -> Path:
|
||||
for index, arg in enumerate(argv):
|
||||
if arg == "--work-dir" and index + 1 < len(argv):
|
||||
return Path(argv[index + 1])
|
||||
if arg.startswith("--work-dir="):
|
||||
return Path(arg.split("=", 1)[1])
|
||||
return Path(os.environ.get("AUDITA_WORK_DIR") or DEFAULT_WORK_DIR)
|
||||
|
||||
|
||||
def _create_run_dir(root: Path) -> Path:
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
|
||||
run_dir = root / f"run-{timestamp}-{uuid4().hex[:8]}"
|
||||
run_dir.mkdir(parents=False, exist_ok=False)
|
||||
return run_dir
|
||||
|
||||
|
||||
def _capture_run_dirs(root: Path) -> set[str]:
|
||||
if not root.exists():
|
||||
return set()
|
||||
return {path.name for path in root.iterdir() if path.is_dir() and path.name.startswith("run-")}
|
||||
|
||||
|
||||
def _find_new_run_dir(root: Path, before: set[str]) -> Optional[Path]:
|
||||
if not root.exists():
|
||||
return None
|
||||
candidates = [
|
||||
path for path in root.iterdir() if path.is_dir() and path.name.startswith("run-") and path.name not in before
|
||||
]
|
||||
if not candidates:
|
||||
return None
|
||||
return max(candidates, key=lambda path: path.name)
|
||||
|
||||
|
||||
def _write_launcher_error_log(
|
||||
path: Path,
|
||||
*,
|
||||
message: str,
|
||||
exit_code: int,
|
||||
argv: list[str],
|
||||
command: Optional[list[str]],
|
||||
) -> None:
|
||||
payload = {
|
||||
"timestamp": datetime.utcnow().isoformat() + "Z",
|
||||
"message": message,
|
||||
"exit_code": exit_code,
|
||||
"argv": argv,
|
||||
"cwd": os.getcwd(),
|
||||
"command": command,
|
||||
}
|
||||
path.write_text(
|
||||
"Audita Launcher Diagnostics\n" + json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _emit_console_line(message: str) -> None:
|
||||
for stream in (sys.stderr, sys.stdout):
|
||||
if stream is None:
|
||||
continue
|
||||
try:
|
||||
stream.write(f"{message}\n")
|
||||
stream.flush()
|
||||
return
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
|
||||
|
||||
def main() -> int:
|
||||
argv = list(sys.argv[1:])
|
||||
work_root = _resolve_work_root(argv)
|
||||
redacted_argv = _redact_argv(argv)
|
||||
uv = shutil.which("uv")
|
||||
if uv is None:
|
||||
run_dir = _create_run_dir(work_root)
|
||||
error_log = run_dir / "error.log"
|
||||
message = "uv is required to run this launcher. Install uv and run `uv sync` in the Audita project."
|
||||
_write_launcher_error_log(error_log, message=message, exit_code=1, argv=redacted_argv, command=None)
|
||||
_emit_console_line(f"audita: error: {message}")
|
||||
_emit_console_line("audita: exit code: 1")
|
||||
_emit_console_line(f"audita: run directory: {run_dir}")
|
||||
_emit_console_line(f"audita: error log: {error_log}")
|
||||
return 1
|
||||
|
||||
project_root = Path(__file__).resolve().parent
|
||||
command = [uv, "run", "--project", str(project_root), "python", "-m", "audita", *sys.argv[1:]]
|
||||
before = _capture_run_dirs(work_root)
|
||||
result = subprocess.run(command, cwd=project_root, check=False)
|
||||
if result.returncode == 0:
|
||||
return 0
|
||||
if _find_new_run_dir(work_root, before) is None:
|
||||
run_dir = _create_run_dir(work_root)
|
||||
error_log = run_dir / "error.log"
|
||||
_write_launcher_error_log(
|
||||
error_log,
|
||||
message=f"Audita subprocess exited with status {result.returncode}.",
|
||||
exit_code=result.returncode,
|
||||
argv=redacted_argv,
|
||||
command=_redact_argv(command),
|
||||
)
|
||||
_emit_console_line(f"audita: subprocess exited with status {result.returncode}")
|
||||
_emit_console_line(f"audita: run directory: {run_dir}")
|
||||
_emit_console_line(f"audita: error log: {error_log}")
|
||||
return result.returncode
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
11
cmd/audita/main.go
Normal file
11
cmd/audita/main.go
Normal file
@@ -0,0 +1,11 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/cli"
|
||||
)
|
||||
|
||||
func main() {
|
||||
os.Exit(cli.Run(os.Args[1:], os.Stdout, os.Stderr))
|
||||
}
|
||||
709
cmd/audita/main_integration_test.go
Normal file
709
cmd/audita/main_integration_test.go
Normal file
@@ -0,0 +1,709 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/cli"
|
||||
)
|
||||
|
||||
func TestHelperProcess(t *testing.T) {
|
||||
if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
|
||||
return
|
||||
}
|
||||
|
||||
sep := -1
|
||||
for i, arg := range os.Args {
|
||||
if arg == "--" {
|
||||
sep = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if sep == -1 {
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
cli.ConfigureSubprocessTestHooksFromEnv()
|
||||
code := cli.Run(os.Args[sep+1:], os.Stdout, os.Stderr)
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func TestProcessHelpSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(t, "process", "--help")
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if !strings.Contains(result.stdout, "Usage:") || !strings.Contains(result.stdout, "--glossary") {
|
||||
t.Fatalf("unexpected help stdout: %q", result.stdout)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSuccessWithOutputSubprocess(t *testing.T) {
|
||||
outputPath := filepath.Join(t.TempDir(), "corrected.json")
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output",
|
||||
outputPath,
|
||||
)
|
||||
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout when --output is set, got %q", result.stdout)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
|
||||
inputBytes := readFile(t, fixturePath("tiny_transcript.json"))
|
||||
outputBytes := readFile(t, outputPath)
|
||||
assertJSONSemanticallyEqual(t, inputBytes, outputBytes)
|
||||
}
|
||||
|
||||
func TestProcessSuccessWithoutOutputSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
|
||||
inputBytes := readFile(t, fixturePath("tiny_transcript.json"))
|
||||
assertJSONSemanticallyEqual(t, inputBytes, []byte(result.stdout))
|
||||
}
|
||||
|
||||
func TestProcessSuccessWithAuditaV1OutputSchemaSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output-schema",
|
||||
"audita-v1",
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
var out struct {
|
||||
Schema string `json:"schema"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(result.stdout), &out); err != nil {
|
||||
t.Fatalf("expected valid audita-v1 JSON output: %v", err)
|
||||
}
|
||||
if out.Schema != "audita-v1" {
|
||||
t.Fatalf("expected audita-v1 schema, got %q", out.Schema)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMissingTranscriptSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(t, "process", "--glossary", fixturePath("tiny_glossary.yaml"))
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "expected exactly 1 transcript JSON path argument") {
|
||||
t.Fatalf("expected actionable missing transcript error, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMalformedJSONSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("malformed_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "is not valid JSON") {
|
||||
t.Fatalf("expected malformed JSON error, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMissingTranscriptFileSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
filepath.Join(t.TempDir(), "missing-transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "transcript_read") {
|
||||
t.Fatalf("expected transcript_read failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMissingGlossaryFileSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
filepath.Join(t.TempDir(), "missing-glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "glossary_read") {
|
||||
t.Fatalf("expected glossary_read failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureTranscriptSchemaSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
schemaFixturePath("transcript_empty_speaker.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "transcript_schema") {
|
||||
t.Fatalf("expected transcript_schema failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMalformedGlossaryYAMLSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
schemaFixturePath("glossary_malformed.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "glossary_schema") {
|
||||
t.Fatalf("expected glossary_schema failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureUnreadableTranscriptSubprocess(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("portable unreadable-file permissions are not reliable on windows")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
transcriptPath := filepath.Join(dir, "transcript.json")
|
||||
if err := os.WriteFile(transcriptPath, []byte(`[]`), 0o000); err != nil {
|
||||
t.Fatalf("write unreadable transcript: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.Chmod(transcriptPath, 0o644) })
|
||||
if _, err := os.ReadFile(transcriptPath); err == nil {
|
||||
t.Skip("unable to make transcript unreadable on this platform/user")
|
||||
}
|
||||
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
transcriptPath,
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "transcript_read") {
|
||||
t.Fatalf("expected transcript_read failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureUnwritableOutputSubprocess(t *testing.T) {
|
||||
outputDir := t.TempDir()
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output",
|
||||
outputDir,
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "failed to write output file") {
|
||||
t.Fatalf("expected write failure message, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureUnwritableReportJSONSubprocess(t *testing.T) {
|
||||
reportDir := t.TempDir()
|
||||
outputPath := filepath.Join(t.TempDir(), "out.json")
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output",
|
||||
outputPath,
|
||||
"--report-json",
|
||||
reportDir,
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "failed to write report JSON file") {
|
||||
t.Fatalf("expected report write failure message, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSuccessReportJSONSubprocess(t *testing.T) {
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--report-json",
|
||||
reportPath,
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
if !json.Valid([]byte(result.stdout)) {
|
||||
t.Fatalf("expected transcript JSON only on stdout, got %q", result.stdout)
|
||||
}
|
||||
report := readFile(t, reportPath)
|
||||
if !json.Valid(report) {
|
||||
t.Fatalf("expected valid report JSON, got %q", string(report))
|
||||
}
|
||||
// Ensure report JSON is not printed to stdout.
|
||||
if strings.Contains(result.stdout, `"default_pipeline"`) {
|
||||
t.Fatalf("report JSON leaked to stdout: %q", result.stdout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSuccessLargeTranscriptSubprocess(t *testing.T) {
|
||||
transcriptPath := writeLargeTranscriptFixture(t, 320)
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
transcriptPath,
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
if !json.Valid([]byte(result.stdout)) {
|
||||
t.Fatalf("expected valid transcript JSON on stdout")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMalformedStructuredLLMResponseViaSubprocessHook(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "malformed_structured"},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"grammar",
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "runner_execution") {
|
||||
t.Fatalf("expected runner_execution failure, got %q", result.stderr)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "diagnostics:") {
|
||||
t.Fatalf("expected diagnostics path in stderr, got %q", result.stderr)
|
||||
}
|
||||
report := readFile(t, reportPath)
|
||||
if !json.Valid(report) {
|
||||
t.Fatalf("expected valid failure report JSON")
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureBackendLLMViaSubprocessHook(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "backend_error"},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"grammar",
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "synthetic backend failure") {
|
||||
t.Fatalf("expected backend failure details, got %q", result.stderr)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "diagnostics:") {
|
||||
t.Fatalf("expected diagnostics path in stderr, got %q", result.stderr)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(onlyRunDir(t, workDir), "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log in retained failed run: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMidPipelinePreservesPartialReportsSubprocess(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "mid_pipeline_fail"},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"glossary,homophones,glossary,spoken_word,grammar",
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
reportRaw := readFile(t, reportPath)
|
||||
var report struct {
|
||||
Status string `json:"status"`
|
||||
ErrorPhase string `json:"error_phase"`
|
||||
ModuleResults []struct {
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
Status string `json:"status"`
|
||||
} `json:"module_results"`
|
||||
}
|
||||
if err := json.Unmarshal(reportRaw, &report); err != nil {
|
||||
t.Fatalf("unmarshal report: %v", err)
|
||||
}
|
||||
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
|
||||
t.Fatalf("expected failed runner_execution report, got %+v", report)
|
||||
}
|
||||
if len(report.ModuleResults) == 0 {
|
||||
t.Fatalf("expected partial module results in failure report")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessCancellationViaSubprocessTimeoutHook(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
|
||||
result := runCLISubprocessContext(t, ctx,
|
||||
map[string]string{
|
||||
"AUDITA_SUBPROCESS_TEST_LLM_MODE": "block_until_cancel",
|
||||
"AUDITA_SUBPROCESS_TEST_RUN_TIMEOUT_MS": "120",
|
||||
},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"grammar",
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "context deadline exceeded") {
|
||||
t.Fatalf("expected context deadline error, got %q", result.stderr)
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log for canceled run: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil {
|
||||
t.Fatalf("expected report.json for canceled run: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSubprocessNoSecretLeakInOutputsAndDiagnostics(t *testing.T) {
|
||||
secret := "subprocess-secret"
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
outputPath := filepath.Join(t.TempDir(), "out.json")
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{
|
||||
"AUDITA_LLM_API_KEY": secret,
|
||||
"AUDITA_VALIDATION_LLM_API_KEY": secret,
|
||||
},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output",
|
||||
outputPath,
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if strings.Contains(result.stdout, secret) || strings.Contains(result.stderr, secret) {
|
||||
t.Fatalf("secret leaked in subprocess stdio")
|
||||
}
|
||||
assertNoSecretInFile(t, reportPath, secret)
|
||||
assertNoSecretInTree(t, onlyRunDir(t, workDir), secret)
|
||||
}
|
||||
|
||||
type subprocessResult struct {
|
||||
stdout string
|
||||
stderr string
|
||||
exitCode int
|
||||
}
|
||||
|
||||
func runCLISubprocess(t *testing.T, args ...string) subprocessResult {
|
||||
t.Helper()
|
||||
return runCLISubprocessWithEnv(t, nil, args...)
|
||||
}
|
||||
|
||||
func runCLISubprocessWithEnv(t *testing.T, extraEnv map[string]string, args ...string) subprocessResult {
|
||||
t.Helper()
|
||||
return runCLISubprocessContext(t, context.Background(), extraEnv, args...)
|
||||
}
|
||||
|
||||
func runCLISubprocessContext(t *testing.T, ctx context.Context, extraEnv map[string]string, args ...string) subprocessResult {
|
||||
t.Helper()
|
||||
cmdArgs := append([]string{"-test.run=TestHelperProcess", "--"}, args...)
|
||||
cmd := exec.CommandContext(ctx, os.Args[0], cmdArgs...)
|
||||
env := append(filterAuditaEnv(os.Environ()), "GO_WANT_HELPER_PROCESS=1")
|
||||
for k, v := range extraEnv {
|
||||
env = append(env, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
cmd.Env = env
|
||||
var stdoutBuf bytes.Buffer
|
||||
var stderrBuf bytes.Buffer
|
||||
cmd.Stdout = &stdoutBuf
|
||||
cmd.Stderr = &stderrBuf
|
||||
|
||||
err := cmd.Run()
|
||||
result := subprocessResult{
|
||||
stdout: stdoutBuf.String(),
|
||||
stderr: stderrBuf.String(),
|
||||
}
|
||||
if err == nil {
|
||||
return result
|
||||
}
|
||||
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
result.exitCode = exitErr.ExitCode()
|
||||
return result
|
||||
}
|
||||
|
||||
t.Fatalf("subprocess execution failed: %v", err)
|
||||
return subprocessResult{}
|
||||
}
|
||||
|
||||
func filterAuditaEnv(env []string) []string {
|
||||
filtered := make([]string, 0, len(env))
|
||||
for _, entry := range env {
|
||||
key := entry
|
||||
if idx := strings.IndexByte(entry, '='); idx >= 0 {
|
||||
key = entry[:idx]
|
||||
}
|
||||
if strings.HasPrefix(key, "AUDITA_") || key == "OPENROUTER_API_KEY" {
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, entry)
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
func fixturePath(name string) string {
|
||||
return filepath.Join("..", "..", "internal", "cli", "testdata", name)
|
||||
}
|
||||
|
||||
func schemaFixturePath(name string) string {
|
||||
return filepath.Join("..", "..", "internal", "core", "schema", "testdata", name)
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read file %q: %v", path, err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func assertJSONSemanticallyEqual(t *testing.T, expected []byte, actual []byte) {
|
||||
t.Helper()
|
||||
if !json.Valid(actual) {
|
||||
t.Fatalf("actual output is not valid JSON: %q", string(actual))
|
||||
}
|
||||
|
||||
var expectedValue any
|
||||
var actualValue any
|
||||
if err := json.Unmarshal(expected, &expectedValue); err != nil {
|
||||
t.Fatalf("failed to unmarshal expected JSON: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(actual, &actualValue); err != nil {
|
||||
t.Fatalf("failed to unmarshal actual JSON: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(expectedValue, actualValue) {
|
||||
t.Fatalf("JSON content mismatch: expected %q got %q", string(expected), string(actual))
|
||||
}
|
||||
}
|
||||
|
||||
func writeLargeTranscriptFixture(t *testing.T, segments int) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "large-transcript.json")
|
||||
rows := make([]string, 0, segments)
|
||||
for i := 0; i < segments; i++ {
|
||||
rows = append(rows, fmt.Sprintf(`{"id":%d,"speaker":"Speaker%d","start":%s,"end":%s,"text":"Segment %d has enough words to exercise stdout and pipe buffering safely."}`,
|
||||
i+1,
|
||||
(i%4)+1,
|
||||
strconv.FormatFloat(float64(i)*1.1, 'f', 1, 64),
|
||||
strconv.FormatFloat(float64(i)*1.1+1.0, 'f', 1, 64),
|
||||
i+1,
|
||||
))
|
||||
}
|
||||
payload := "[\n " + strings.Join(rows, ",\n ") + "\n]\n"
|
||||
if err := os.WriteFile(path, []byte(payload), 0o644); err != nil {
|
||||
t.Fatalf("write large transcript fixture: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func onlyRunDir(t *testing.T, workDir string) string {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(workDir)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read work dir %q: %v", workDir, err)
|
||||
}
|
||||
dirs := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
dirs = append(dirs, filepath.Join(workDir, e.Name()))
|
||||
}
|
||||
}
|
||||
if len(dirs) != 1 {
|
||||
t.Fatalf("expected exactly one run dir in %q, found %d", workDir, len(dirs))
|
||||
}
|
||||
return dirs[0]
|
||||
}
|
||||
|
||||
func assertNoSecretInFile(t *testing.T, path, secret string) {
|
||||
t.Helper()
|
||||
raw := string(readFile(t, path))
|
||||
if strings.Contains(raw, secret) {
|
||||
t.Fatalf("secret leaked in %s", path)
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoSecretInTree(t *testing.T, root, secret string) {
|
||||
t.Helper()
|
||||
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil || d == nil || d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
raw, readErr := os.ReadFile(path)
|
||||
if readErr == nil && strings.Contains(string(raw), secret) {
|
||||
t.Fatalf("secret leaked in %s", path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
849
docs/architecture/architecture.md
Normal file
849
docs/architecture/architecture.md
Normal file
@@ -0,0 +1,849 @@
|
||||
# Audita Architecture
|
||||
|
||||
## Scope and intent
|
||||
This document describes:
|
||||
- the architecture used in production today.
|
||||
|
||||
Historical rewrite details live in `docs/rewrite-notes.md`.
|
||||
|
||||
## Current implementation status
|
||||
Implemented today:
|
||||
- Go CLI entrypoint and `audita process` wiring.
|
||||
- Config defaults, env loading, CLI override precedence, and validation.
|
||||
- Transcript and glossary parsing/validation.
|
||||
- Deterministic transcript normalization.
|
||||
- Deterministic token estimation and transcript chunking.
|
||||
- Per-run diagnostics directory creation plus process-level artifacts.
|
||||
- Process report JSON output with diagnostics artifact references.
|
||||
- Framework foundation packages for contracts and proposal application.
|
||||
- Production runner orchestration package with deterministic sequential module execution.
|
||||
- Module-level report structures with applied/skipped change records.
|
||||
- Runtime validator models and deterministic validators.
|
||||
- Deterministic validator-chain execution in the runner with cardinality enforcement.
|
||||
- Module-level validator decision/rejection reporting.
|
||||
- Internal structured LLM client contract plus an Audita-owned OpenAI-compatible structured LLM adapter package.
|
||||
- Bounded FIFO LLM scheduler infrastructure with context-aware permit handling.
|
||||
- Runtime primary/validation LLM effective-config resolution helpers with validation inheritance.
|
||||
- Generic JSON prompt/response diagnostics writer primitives with secret redaction.
|
||||
- LLM-backed validator models, prompt builders, batching, and runtime execution.
|
||||
- Runner wiring for LLM validators via the internal structured LLM abstraction and scheduler hooks.
|
||||
- LLM validator diagnostics artifacts and report-level decision metadata paths.
|
||||
- Shared LLM proposal-generation helper with structured correction-set parsing.
|
||||
- Deterministic proposal-index assignment and enriched proposal mapping for shared generation.
|
||||
- Proposal-generation diagnostics artifacts with secret redaction.
|
||||
- Production module registry with known-key recognition and explicit unsupported-module errors.
|
||||
- Production `grammar` module implementation in `internal/modules/grammar`.
|
||||
- Production `glossary` module implementation in `internal/modules/glossary`.
|
||||
- Production `homophones` module implementation in `internal/modules/homophones`.
|
||||
- Production `spoken_word` module implementation in `internal/modules/spoken_word`.
|
||||
- Explicit runtime support for `--modules grammar` through the production runner path.
|
||||
- Explicit runtime support for `--modules glossary`, including repeated stages such as `--modules glossary,glossary`.
|
||||
- Explicit runtime support for `--modules homophones` through the production runner path.
|
||||
- Explicit runtime support for `--modules spoken_word` through the production runner path.
|
||||
|
||||
Current reality:
|
||||
- all production modules exist and are wired into the default runtime path.
|
||||
- a normal `audita process` run without `--modules` now executes the full sequence:
|
||||
- `glossary`
|
||||
- `homophones`
|
||||
- `glossary`
|
||||
- `spoken_word`
|
||||
- `grammar`
|
||||
- repeated glossary stages are deterministic and reported distinctly as `glossary_1` and `glossary_2`.
|
||||
|
||||
## Actual Go package layout
|
||||
|
||||
```text
|
||||
cmd/audita/
|
||||
main.go
|
||||
|
||||
internal/cli/
|
||||
run.go
|
||||
|
||||
internal/core/config/
|
||||
config.go
|
||||
env.go
|
||||
flags.go
|
||||
redaction.go
|
||||
validation.go
|
||||
|
||||
internal/core/schema/
|
||||
transcript.go
|
||||
glossary.go
|
||||
errors.go
|
||||
|
||||
internal/core/io/
|
||||
files.go
|
||||
|
||||
internal/core/normalization/
|
||||
normalize.go
|
||||
tokens.go
|
||||
|
||||
internal/core/chunking/
|
||||
sections.go
|
||||
summary.go
|
||||
tokens.go
|
||||
|
||||
internal/core/diagnostics/
|
||||
run_dir.go
|
||||
|
||||
internal/core/reporting/
|
||||
report.go
|
||||
|
||||
internal/framework/contracts/
|
||||
contracts.go
|
||||
|
||||
internal/framework/proposals/
|
||||
proposal.go
|
||||
policy.go
|
||||
preview.go
|
||||
apply.go
|
||||
|
||||
internal/framework/runner/
|
||||
observability.go
|
||||
runner.go
|
||||
|
||||
internal/framework/proposal_generation/
|
||||
generate.go
|
||||
|
||||
internal/framework/modules/
|
||||
registry.go
|
||||
|
||||
internal/modules/grammar/
|
||||
module.go
|
||||
prompt.go
|
||||
|
||||
internal/modules/glossary/
|
||||
module.go
|
||||
prompt.go
|
||||
|
||||
internal/modules/homophones/
|
||||
module.go
|
||||
prompt.go
|
||||
|
||||
internal/modules/spoken_word/
|
||||
module.go
|
||||
prompt.go
|
||||
|
||||
internal/framework/validators/
|
||||
models.go
|
||||
deterministic.go
|
||||
llm_models.go
|
||||
llm_prompt_builders.go
|
||||
llm_batching.go
|
||||
llm_validators.go
|
||||
|
||||
internal/validators/
|
||||
metadata/
|
||||
metadata.go
|
||||
registry.go
|
||||
chains.go
|
||||
confidence_threshold/
|
||||
validator.go
|
||||
original_text_presence/
|
||||
validator.go
|
||||
non_empty_corrected_text/
|
||||
validator.go
|
||||
no_effect/
|
||||
validator.go
|
||||
protected_terms/
|
||||
validator.go
|
||||
spoken_form_plausibility/
|
||||
validator.go
|
||||
meaning_reversal_review/
|
||||
validator.go
|
||||
editorial_review/
|
||||
validator.go
|
||||
grammar_review/
|
||||
validator.go
|
||||
spoken_word_review/
|
||||
validator.go
|
||||
|
||||
internal/prompts/
|
||||
registry.go
|
||||
render.go
|
||||
assets/
|
||||
shared/
|
||||
modules/
|
||||
validators/
|
||||
|
||||
internal/framework/llm/
|
||||
openai_compatible_client.go
|
||||
scheduler.go
|
||||
effective_config.go
|
||||
diagnostics.go
|
||||
|
||||
internal/framework/responseschema/
|
||||
registry.go
|
||||
registry_test.go
|
||||
|
||||
internal/cli/
|
||||
review_artifacts.go
|
||||
parity_test.go
|
||||
release_fixtures_test.go
|
||||
testdata/
|
||||
parity/
|
||||
release/
|
||||
```
|
||||
|
||||
## Current CLI behavior
|
||||
Primary commands:
|
||||
|
||||
```sh
|
||||
audita process <transcript.json> --glossary <glossary.yaml> [flags]
|
||||
audita config validate --config <config.yml>
|
||||
audita config print-effective [--config <config.yml>]
|
||||
```
|
||||
|
||||
Current runtime flow (`internal/cli/run.go`):
|
||||
1. Build runtime config from:
|
||||
- defaults;
|
||||
- file config source (`--config`, `AUDITA_CONFIG`, or default search paths when present: `/usr/local/etc/audita/config.yml`, then `/etc/audita/config.yml`);
|
||||
- environment overrides;
|
||||
- CLI overrides.
|
||||
2. Parse flags and apply CLI overrides.
|
||||
3. Validate transcript positional argument and required `--glossary`.
|
||||
4. Create per-run diagnostics directory.
|
||||
5. Read transcript and glossary files.
|
||||
6. Parse/validate transcript and glossary.
|
||||
7. Write source transcript artifacts.
|
||||
8. Normalize transcript.
|
||||
9. Write normalized transcript and normalization summary artifacts.
|
||||
10. Chunk normalized transcript and compute chunk summaries.
|
||||
11. Write chunking summary artifact.
|
||||
12. Execute runner modules sequentially:
|
||||
- default run path uses configured default sequence (`glossary,homophones,glossary,spoken_word,grammar`);
|
||||
- explicit `--modules` overrides the default sequence;
|
||||
- test/injected module factory path remains available for deterministic runtime tests.
|
||||
- each module recomputes chunks from the current working transcript, runs chunk proposal work concurrently, aggregates deterministically, validates, and applies approved proposals once.
|
||||
13. Output working transcript to `--output` file or stdout.
|
||||
14. Build process report metadata.
|
||||
15. Optionally write `--report-json`; always write run-dir `report.json`.
|
||||
16. Apply work-dir retention.
|
||||
|
||||
Config command behavior (`internal/cli/run.go`):
|
||||
- `audita config validate --config <path>`:
|
||||
- loads and validates a versioned YAML config file;
|
||||
- does not require transcript or glossary inputs.
|
||||
- `audita config print-effective [--config <path>]`:
|
||||
- builds effective config from defaults + file config + env overrides;
|
||||
- prints redacted JSON to stdout;
|
||||
- does not require transcript or glossary inputs.
|
||||
|
||||
Parity fixture status:
|
||||
- representative Python-parity fixture coverage exists under `internal/cli/testdata/parity`;
|
||||
- parity tests use fake structured LLM responses for deterministic behavior, including default full-pipeline shape assertions;
|
||||
- parity comparisons intentionally ignore nondeterministic metadata (timestamps, run IDs, temp paths, token usage) and remain strict for deterministic contract fields (transcript content, module order/instance naming, applied/skipped/rejected counts, and status).
|
||||
- intentional Python-vs-Go differences and open parity gaps are documented in `docs/python-parity.md`.
|
||||
|
||||
Important behavior details:
|
||||
- Glossary is validated and is used for explicit glossary/grammar/homophones/spoken_word module correction paths.
|
||||
- Default production CLI behavior now executes the full production module sequence unless `--modules` override is supplied.
|
||||
- Explicit `--modules grammar`, `--modules glossary`, `--modules homophones`, and `--modules spoken_word` continue to run production module paths with LLM-backed proposal generation and validator-chain execution.
|
||||
- Default runs (without explicit module selection) perform LLM calls through production module and validator paths.
|
||||
- Success path is generally quiet on stderr.
|
||||
- Source IDs are preserved into a canonical transcript before normalization; normalization then reassigns output IDs sequentially from `1`.
|
||||
|
||||
## Implemented data contracts
|
||||
|
||||
### Transcript input
|
||||
Accepted top-level forms:
|
||||
- bare JSON array of segments
|
||||
- object with `segments` array
|
||||
|
||||
Source segment contract:
|
||||
- `id` optional integer
|
||||
- `speaker` non-empty string
|
||||
- `start` finite non-negative number
|
||||
- `end` finite non-negative number with `end >= start`
|
||||
- `text` non-empty string
|
||||
- `categories` optional array of non-empty strings
|
||||
|
||||
Additional checks:
|
||||
- duplicate explicit source IDs are rejected.
|
||||
|
||||
### Transcript output
|
||||
Transcript output is selected through an output schema registry (`internal/core/outputschema`).
|
||||
|
||||
Supported output schemas:
|
||||
- `bare-segments` (default):
|
||||
- top-level JSON array of normalized segments;
|
||||
- each segment includes `id`, `speaker`, `start`, `end`, `text`, optional `categories`.
|
||||
- `audita-v1`:
|
||||
- top-level object with:
|
||||
- `schema: "audita-v1"`
|
||||
- `version: "v1"`
|
||||
- `segments: [...]` (same normalized segment payload).
|
||||
|
||||
Current status:
|
||||
- `seriatim-intermediate` is not implemented yet; selecting it fails clearly as an unsupported output schema.
|
||||
|
||||
Selection behavior:
|
||||
- CLI: `--output-schema <name>`
|
||||
- file config: `output.schema: <name>`
|
||||
- precedence remains runtime-wide defaults -> file config -> env -> CLI.
|
||||
|
||||
Both stdout transcript output and `--output` file output use the same selected output encoder.
|
||||
|
||||
### Glossary input
|
||||
YAML with `glossary` entries. Required fields per entry:
|
||||
- `name`, `category`, `summary`
|
||||
|
||||
Optional:
|
||||
- `aliases`, `plural`
|
||||
|
||||
## Implemented config/env/flag behavior
|
||||
Precedence for `audita process`:
|
||||
1. defaults (`config.Default()`)
|
||||
2. config file (if resolved from `--config`, `AUDITA_CONFIG`, or default path)
|
||||
3. environment overrides
|
||||
4. CLI flags (`ApplyCLIOverrides`)
|
||||
|
||||
File-config source behavior:
|
||||
- explicit `--config <path>`:
|
||||
- required to exist, otherwise process fails clearly.
|
||||
- `AUDITA_CONFIG` (when `--config` is not provided):
|
||||
- required to exist, otherwise process fails clearly.
|
||||
- default paths `/usr/local/etc/audita/config.yml`, then `/etc/audita/config.yml` (when neither explicit source is provided):
|
||||
- first existing path in that order is used;
|
||||
- both missing is silently ignored.
|
||||
|
||||
Versioned file-config behavior (`internal/core/config/file_config.go`):
|
||||
- supported version: `version: 1`;
|
||||
- missing version fails;
|
||||
- unsupported version fails;
|
||||
- strict unknown-field rejection is enabled.
|
||||
|
||||
`api_key_env` behavior:
|
||||
- file config can declare API key environment variable names for proposal/validation LLM settings;
|
||||
- runtime resolves those names from the process environment during config application;
|
||||
- no direct API-key value field is supported in file config.
|
||||
|
||||
Redaction behavior:
|
||||
- effective config artifacts and `audita config print-effective` both use the same redaction path (`Config.Redacted()`), so API keys are not emitted in plaintext.
|
||||
|
||||
Implemented config surfaces include:
|
||||
- module list
|
||||
- primary and validation LLM settings
|
||||
- total/proposal/validation LLM concurrency controls
|
||||
- transcript description context (`--transcript-description`)
|
||||
- section token controls and target sections
|
||||
- confidence thresholds
|
||||
- normalization controls
|
||||
- work-dir and retention mode
|
||||
|
||||
Current caveat:
|
||||
- LLM/module-related settings are active for default and explicit module-run paths.
|
||||
- compatibility environment variables and lower-level CLI tuning flags remain available while the preferred config-driven surface is adopted.
|
||||
|
||||
Transcript description behavior:
|
||||
- `--transcript-description` is a process-flag input for optional user-supplied background context.
|
||||
- runtime config stores this value in `Config.TranscriptDescription` after CLI trimming and length validation.
|
||||
- default value is empty; empty values produce no prompt context section.
|
||||
- this value is intentionally non-secret and appears in effective config and invocation metadata artifacts.
|
||||
|
||||
## Implemented transcript description prompt context
|
||||
Transcript description context is wired through production prompt paths:
|
||||
- proposal prompts for `glossary`, `homophones`, `spoken_word`, and `grammar`;
|
||||
- LLM-backed validator prompts for spoken-form plausibility, meaning reversal, editorial review, grammar review, and spoken-word review.
|
||||
|
||||
Prompt guardrail semantics are consistent across modules and validators:
|
||||
- transcript description is labeled as "background context only";
|
||||
- it may help interpret ambiguous terms;
|
||||
- it must not override transcript content;
|
||||
- the model must not invent corrections, facts, names, events, motivations, or speaker intent from this description.
|
||||
|
||||
Generated transcript descriptions remain deferred and are not implemented in the current runtime.
|
||||
|
||||
## Implemented embedded prompt assets
|
||||
Prompt assets are now built-in embedded Markdown files under `internal/prompts/assets`:
|
||||
- `assets/modules/*` for production module proposal prompts;
|
||||
- `assets/validators/*` for LLM-backed validator prompts;
|
||||
- `assets/shared/prompt_hardening.md` for shared prompt-injection hardening text.
|
||||
|
||||
Prompt source behavior:
|
||||
- built-in embedded prompts are the only supported source in current runtime;
|
||||
- filesystem prompt overrides and prompt-source selection flags are not implemented.
|
||||
|
||||
`internal/prompts` registry responsibilities:
|
||||
- register stable prompt IDs;
|
||||
- register prompt version and source metadata;
|
||||
- load embedded assets;
|
||||
- compute deterministic SHA-256 prompt source hashes;
|
||||
- render system/user prompts with `text/template` using missing-key errors.
|
||||
|
||||
Prompt metadata fields:
|
||||
- `prompt_id`
|
||||
- `prompt_version`
|
||||
- `prompt_source` (`builtin`)
|
||||
- `embedded_path`
|
||||
- `sha256`
|
||||
|
||||
Prompt rendering flow:
|
||||
- module proposal builders construct typed template data (section JSON, glossary JSON, transcript-description block) and render via `internal/prompts`;
|
||||
- validator prompt builders construct typed template data (validation payload JSON, transcript-description block) and render via `internal/prompts`.
|
||||
|
||||
Shared prompt hardening:
|
||||
- the same centralized hardening fragment is included in every proposal and LLM-validator prompt;
|
||||
- hardening text enforces untrusted transcript handling, no instruction-following from transcript content, and no invented facts/corrections.
|
||||
|
||||
Prompt metadata diagnostics flow:
|
||||
- proposal-generation diagnostics request metadata includes prompt metadata;
|
||||
- LLM-validator diagnostics request metadata includes prompt metadata;
|
||||
- detailed prompt metadata is diagnostics-scoped today and is not yet expanded into broad report-level prompt registries.
|
||||
|
||||
## Implemented structured LLM infrastructure
|
||||
`internal/framework/contracts` now defines a typed structured-completion contract:
|
||||
- `StructuredLLMClient.CompleteStructured(ctx, req, out)`
|
||||
- caller-owned typed decode target via `out` pointer.
|
||||
- caller-selected structured response schema metadata via `StructuredCompletionRequest.ResponseSchema`.
|
||||
|
||||
`internal/framework/llm` provides `OpenAICompatibleClient`, a direct `net/http` adapter over OpenAI-compatible chat completions:
|
||||
- configurable `base_url`, model, optional API key, retries, HTTP client, and request timeout;
|
||||
- OpenAI-compatible endpoint behavior (for example OpenAI/OpenRouter/local-compatible base URLs);
|
||||
- request message translation from `contracts.LLMMessage` to chat-completions messages;
|
||||
- strict `response_format.type = json_schema` with registered structured response schemas (`strict: true`, schema name, and schema body);
|
||||
- response metadata mapping (provider/model/token usage) into Audita-owned response types;
|
||||
- API-key redaction in adapter-returned errors;
|
||||
- context cancellation and timeout propagation through request contexts and HTTP client timeouts;
|
||||
- bounded retry behavior for transient request failures and malformed retryable structured responses.
|
||||
|
||||
Structured response schemas are owned by Audita in `internal/framework/responseschema` and currently include:
|
||||
- key `correction_set`:
|
||||
- id `audita.correction_set`
|
||||
- version `v1`
|
||||
- name `audita_correction_set_v1`
|
||||
- sha256 `05f8ff3fa04f68115c0cb1859d2656f51aa5c0bae8ff2470b2d4f6f531953195`
|
||||
- key `validator_decision_set`:
|
||||
- id `audita.validator_decision_set`
|
||||
- version `v1`
|
||||
- name `audita_validator_decision_set_v1`
|
||||
- sha256 `b73f4790b98fbb955f0aec5496dd8ce9a8fe14aa2f35c700b4b4e5634f106fd5`
|
||||
|
||||
Provider-level structured output is treated as a guardrail, not a trust boundary:
|
||||
- the adapter decodes assistant message content into caller-owned structs;
|
||||
- proposal-generation and validator layers continue local validation (shape, cardinality, confidence bounds, and proposal-index semantics) before changes can be applied.
|
||||
|
||||
Current runtime boundary:
|
||||
- the default CLI runtime path (without explicit module selection) instantiates the full production module sequence.
|
||||
- LLM calls are exercised in production in both default full-pipeline runs and explicit `--modules` runs, and in tests when fake/injected clients are used.
|
||||
- normal `go test ./...` does not require real LLM credentials or Python dependencies.
|
||||
|
||||
`internal/framework/llm` also provides:
|
||||
- a bounded FIFO `Scheduler` for controlled concurrent LLM calls with reliable permit release on success, error, and cancellation;
|
||||
- primary/validation effective-config resolution helpers, including validation inheritance fallback to total LLM concurrency settings;
|
||||
- generic interaction diagnostics primitives that write machine-readable JSON artifacts for request metadata, request payload, response payload, and optional error payload with secret redaction.
|
||||
|
||||
Structured LLM diagnostics behavior:
|
||||
- proposal-generation and validator diagnostics include structured response schema metadata (`id`, `version`, `name`, `sha256`) when schema-driven calls are made;
|
||||
- API keys and bearer tokens are redacted from request/response/error diagnostics artifacts and surfaced errors.
|
||||
|
||||
Dependency posture:
|
||||
- the runtime no longer depends on `instructor-go`;
|
||||
- structured LLM behavior is implemented through Audita-owned code paths behind `StructuredLLMClient`.
|
||||
|
||||
LLM concurrency runtime behavior:
|
||||
- `total` concurrency bounds all proposal and validation LLM calls.
|
||||
- `proposal` concurrency adds a proposal-only sub-cap, composed with total.
|
||||
- `validation` concurrency adds a validation-only sub-cap, composed with total.
|
||||
- legacy `llm-concurrency` inputs remain compatibility aliases for total concurrency.
|
||||
- modules execute serially, chunk proposals run concurrently within each module, and approved proposals are applied once per module in deterministic order.
|
||||
|
||||
## Implemented normalization behavior
|
||||
Normalization (`internal/core/normalization`) currently:
|
||||
- sorts by segment start time;
|
||||
- merges adjacent same-speaker segments when constraints pass;
|
||||
- uses gap-based joiners:
|
||||
- gap `< ellipsis_gap` -> single space join
|
||||
- gap `>= ellipsis_gap` -> `... ` join
|
||||
- enforces merged duration and token-limit constraints;
|
||||
- reassigns output IDs sequentially from `1`;
|
||||
- returns `NormalizationSummary` with merge and skip counters.
|
||||
|
||||
Note: merged categories are concatenated (not deduplicated).
|
||||
|
||||
## Implemented chunking behavior
|
||||
Chunking (`internal/core/chunking`) currently provides:
|
||||
- deterministic heuristic token estimation;
|
||||
- contiguous sectioning with section metadata;
|
||||
- max/min section token validation;
|
||||
- optional `target_sections` override for section-count planning;
|
||||
- summary and detailed summary generation.
|
||||
|
||||
Current behavior details:
|
||||
- if a single segment exceeds max tokens, it is emitted as its own section (not hard-failed);
|
||||
- default section count is planned from `ceil(total_tokens / max_section_tokens)`;
|
||||
- section sizing targets `ceil(total_tokens / section_count)` with a deterministic forward pass;
|
||||
- sections remain contiguous and ordered, and segments are never split.
|
||||
|
||||
## Implemented proposal/replacement infrastructure
|
||||
`internal/framework/proposals` provides deterministic proposal composition logic:
|
||||
- `CorrectionProposal` and `EnrichedCorrectionProposal` models;
|
||||
- replacement policies: `require_unique`, `replace_all`;
|
||||
- safe preview (`PreviewProposalForSegment`) with stable skip reasons;
|
||||
- deterministic apply (`ApplyProposals`) in ascending `proposal_index` order;
|
||||
- applied/skipped change records suitable for reporting.
|
||||
|
||||
`internal/framework/contracts` provides interfaces and run-spec metadata scaffolding, including deterministic repeated module instance naming (`ResolveModuleRunSpecs`).
|
||||
|
||||
These primitives are wired into the production runner and report model. The grammar, glossary, homophones, and spoken_word modules are implemented.
|
||||
|
||||
## Implemented validator runtime infrastructure
|
||||
`internal/framework/validators` provides deterministic validator infrastructure:
|
||||
- runtime validation request/result models;
|
||||
- stable validator reason codes;
|
||||
- cardinality enforcement for validator decisions:
|
||||
- missing proposal indexes fail
|
||||
- duplicate proposal indexes fail
|
||||
- unknown proposal indexes fail
|
||||
- deterministic validators:
|
||||
- confidence threshold by module key/config threshold
|
||||
- original-text presence against current working transcript
|
||||
- non-empty corrected text
|
||||
- identical/no-effect rejection
|
||||
- conservative protected glossary-term guard for non-glossary modules
|
||||
|
||||
`internal/framework/runner` executes module pipelines with deterministic boundaries:
|
||||
- modules still execute serially over the working transcript;
|
||||
- section proposal work is launched promptly and can run concurrently;
|
||||
- section-level validator-chain work starts as section proposals become available (deterministic validators before LLM-backed validators);
|
||||
- proposal-generation and LLM-validator calls can overlap under composed scheduler limits;
|
||||
- approved proposals are still applied once per module after section work settles.
|
||||
|
||||
Validator rejections are reported distinctly from proposal-application skips.
|
||||
|
||||
Validator composition is now explicit and registry-backed through `internal/validators`:
|
||||
- built-in validator registry with stable keys and lookup/build failure for unknown keys;
|
||||
- built-in chain definitions per production module key;
|
||||
- production modules resolve validator chains from those built-in definitions.
|
||||
|
||||
Package ownership boundary:
|
||||
- `internal/validators/<validator_key>` owns built-in validator construction and stable key identity.
|
||||
- `internal/framework/validators` remains shared runtime machinery:
|
||||
- request/result models;
|
||||
- decision cardinality enforcement;
|
||||
- protected-vocabulary helpers;
|
||||
- generic LLM-backed validator runtime, batching, and diagnostics glue.
|
||||
|
||||
Validator execution classification metadata:
|
||||
- `internal/validators/metadata` defines execution class markers:
|
||||
- `deterministic`
|
||||
- `llm_backed`
|
||||
- runner ordering uses this metadata interface rather than concrete framework validator type assertions.
|
||||
- validators without classification metadata default to deterministic ordering.
|
||||
|
||||
`protected_terms` construction ownership:
|
||||
- `internal/validators/protected_terms.New()` builds the general (non-glossary-stage) variant.
|
||||
- `internal/validators/protected_terms.NewGlossaryStage()` builds the glossary-stage variant used by glossary chains.
|
||||
- both variants preserve the stable key `protected_terms`.
|
||||
|
||||
Stable built-in validator keys:
|
||||
- deterministic:
|
||||
- `confidence_threshold`
|
||||
- `original_text_presence`
|
||||
- `non_empty_corrected_text`
|
||||
- `no_effect`
|
||||
- `protected_terms`
|
||||
- LLM-backed:
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
- `editorial_review`
|
||||
- `grammar_review`
|
||||
- `spoken_word_review`
|
||||
|
||||
Built-in module chains:
|
||||
- `glossary`:
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
- `homophones`:
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
- `spoken_word`:
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `spoken_word_review`
|
||||
- `meaning_reversal_review`
|
||||
- `grammar`:
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `grammar_review`
|
||||
- `meaning_reversal_review`
|
||||
|
||||
1.0 boundary:
|
||||
- validator chains are built-in and not user-configurable from config/CLI.
|
||||
- existing threshold and batching knobs remain configurable.
|
||||
|
||||
## Implemented LLM-backed validator infrastructure
|
||||
`internal/framework/validators` now includes LLM-backed validator support:
|
||||
- typed request/response models for structured LLM validation;
|
||||
- prompt builders for:
|
||||
- spoken-form plausibility
|
||||
- meaning reversal detection
|
||||
- editorial review
|
||||
- grammar review
|
||||
- spoken-word review
|
||||
- deterministic batching by `validation_max_prompt_tokens`;
|
||||
- strict cardinality validation of structured LLM decisions (missing/duplicate/unknown indexes fail);
|
||||
- safe failure behavior for malformed/invalid structured responses.
|
||||
|
||||
`internal/framework/runner` wires LLM validators into existing validator chains using:
|
||||
- the internal structured LLM client abstraction (`contracts.StructuredLLMClient`);
|
||||
- bounded scheduler hooks for validator call execution;
|
||||
- diagnostics writer hooks for machine-readable prompt/response artifacts with secret redaction.
|
||||
|
||||
## Implemented shared proposal-generation infrastructure
|
||||
`internal/framework/proposal_generation` provides a reusable, prompt-agnostic helper for future real modules:
|
||||
- structured request model including module key/instance, replacement policy, working transcript context, optional section metadata, glossary, config, and diagnostics context;
|
||||
- structured correction-set response model (`corrections`) mapped into existing `proposals.CorrectionProposal` and `proposals.EnrichedCorrectionProposal` models;
|
||||
- deterministic proposal-index assignment through a caller-provided `start_index`;
|
||||
- structured LLM calls through `contracts.StructuredLLMClient` only (no direct provider calls);
|
||||
- optional bounded execution through scheduler hooks (`contracts.LLMScheduler`);
|
||||
- prompt/response diagnostics artifact writing via the generic `internal/framework/llm` diagnostics primitives with redaction of API keys/secrets.
|
||||
|
||||
This helper only produces candidate proposals; validator-chain execution and proposal application remain runner responsibilities.
|
||||
|
||||
## Implemented production module-registry scaffolding
|
||||
`internal/framework/modules` now provides a production registry scaffold:
|
||||
- recognizes intended module keys:
|
||||
- `glossary`
|
||||
- `homophones`
|
||||
- `spoken_word`
|
||||
- `grammar`
|
||||
- supports explicit constructor registration with dependency injection for:
|
||||
- run spec
|
||||
- config
|
||||
- glossary
|
||||
- proposal/validation structured LLM clients
|
||||
- proposal/validation schedulers
|
||||
- diagnostics directory context
|
||||
- returns explicit errors for unknown keys (`unsupported_module`).
|
||||
|
||||
The `grammar`, `glossary`, `homophones`, and `spoken_word` module keys are now registered and constructible.
|
||||
|
||||
## Implemented grammar production module
|
||||
`internal/modules/grammar` now provides the first production module:
|
||||
- prompt builder faithfully constrained to punctuation/capitalization/spacing/article cleanup;
|
||||
- explicit guardrails against meaning-changing rewrites, style rewrites, summarization, and invention;
|
||||
- proposal generation through `internal/framework/proposal_generation` and `contracts.StructuredLLMClient`;
|
||||
- scheduler-aware proposal calls through existing `contracts.LLMScheduler` hooks;
|
||||
- replacement policy `require_unique` (current runtime policy);
|
||||
- validator chain integration using existing deterministic + LLM-backed validators;
|
||||
- grammar confidence threshold enforcement through existing validator/config infrastructure;
|
||||
- module-level reporting and diagnostics capture through existing runner/reporting paths.
|
||||
|
||||
## Implemented glossary production module
|
||||
`internal/modules/glossary` now provides the second production module:
|
||||
- prompt builder aligned to Python glossary-module intent, constrained to glossary-backed domain/acoustic corrections;
|
||||
- prompt context includes glossary names, aliases, categories, summaries, and plural forms where available;
|
||||
- guardrails against broad style rewriting and against replacing unrelated terms simply because they appear in glossary entries;
|
||||
- proposal generation through `internal/framework/proposal_generation` and `contracts.StructuredLLMClient`;
|
||||
- scheduler-aware proposal calls through existing `contracts.LLMScheduler` hooks;
|
||||
- replacement policy `replace_all` (matching Python glossary behavior);
|
||||
- validator chain integration using existing deterministic + LLM-backed validators;
|
||||
- glossary confidence threshold enforcement through existing validator/config infrastructure;
|
||||
- module-level reporting and diagnostics capture through existing runner/reporting paths;
|
||||
- explicit support for repeated glossary stages with deterministic instance names (`glossary_1`, `glossary_2`, ...), where later stages see prior-stage working transcript changes.
|
||||
|
||||
## Implemented protected-term behavior
|
||||
`internal/framework/validators/protected_terms.go` provides deterministic glossary-derived protected vocabulary:
|
||||
- extracts protected terms from glossary names and aliases;
|
||||
- includes explicit plural fields and synthetic plural forms where safe;
|
||||
- deduplicates and returns stable ordering for repeatable behavior/tests.
|
||||
|
||||
This vocabulary is used by deterministic validators for both glossary-stage and non-glossary-stage protection checks, keeping protected-term guardrails active across modules.
|
||||
|
||||
## Implemented homophones production module
|
||||
`internal/modules/homophones` now provides the third production module:
|
||||
- prompt builder aligned to Python homophones-module intent, constrained to conservative homophone/near-homophone/mistranscription corrections;
|
||||
- prompt context includes protected glossary names/aliases/plurals to avoid damaging known terms;
|
||||
- explicit guardrails against punctuation cleanup, grammar cleanup, style rewriting, summarization, and content invention;
|
||||
- proposal generation through `internal/framework/proposal_generation` and `contracts.StructuredLLMClient`;
|
||||
- scheduler-aware proposal calls through existing `contracts.LLMScheduler` hooks;
|
||||
- replacement policy `require_unique` (matching Python homophones behavior);
|
||||
- validator chain integration using existing deterministic + LLM-backed validators;
|
||||
- homophones confidence threshold enforcement through existing validator/config infrastructure;
|
||||
- protected-term guardrails for non-glossary modules remain active and are exercised through the homophones path;
|
||||
- module-level reporting and diagnostics capture through existing runner/reporting paths.
|
||||
|
||||
## Implemented spoken_word production module
|
||||
`internal/modules/spoken_word` now provides the fourth production module:
|
||||
- prompt builder aligned to Python spoken_word-module intent, constrained to conservative dysfluency cleanup;
|
||||
- strong prompt guardrails preserving meaning/intent/voice/named entities/domain terms and substantive content;
|
||||
- explicit guardrails against summarization, style rewriting, grammar-only cleanup, punctuation-only cleanup, invention, and meaning-changing rewrites;
|
||||
- proposal generation through `internal/framework/proposal_generation` and `contracts.StructuredLLMClient`;
|
||||
- scheduler-aware proposal calls through existing `contracts.LLMScheduler` hooks;
|
||||
- replacement policy `require_unique` (matching Python spoken_word behavior);
|
||||
- validator chain integration using existing deterministic + LLM-backed validators, including strong semantic guardrails (`spoken_word_review`, `meaning_reversal_review`);
|
||||
- spoken_word confidence threshold enforcement through existing validator/config infrastructure;
|
||||
- protected-term guardrails for non-glossary modules remain active and are exercised through the spoken_word path;
|
||||
- module-level reporting and diagnostics capture through existing runner/reporting paths.
|
||||
|
||||
## Reports and diagnostics (implemented)
|
||||
Current per-run artifacts include:
|
||||
- `source-transcript.json`
|
||||
- `source-transcript-parsed.json`
|
||||
- `normalized-transcript.json`
|
||||
- `normalization-summary.json`
|
||||
- `chunking-summary.json`
|
||||
- `utilization-diagnostics.json`
|
||||
- `correction-ledger.json`
|
||||
- `invocation.json`
|
||||
- `effective-config.json` (redacted credentials)
|
||||
- `report.json`
|
||||
- `error.log` on failure
|
||||
|
||||
`--report-json` writes a separate report file when requested.
|
||||
|
||||
Current process reports include diagnostics metadata references for:
|
||||
- diagnostics directory path;
|
||||
- source transcript artifact path;
|
||||
- parsed source transcript artifact path;
|
||||
- normalized transcript artifact path;
|
||||
- normalization summary artifact path;
|
||||
- chunking summary artifact path;
|
||||
- utilization diagnostics artifact path;
|
||||
- correction ledger artifact path;
|
||||
- invocation metadata artifact path;
|
||||
- redacted effective-config artifact path;
|
||||
- error-log artifact path on failure.
|
||||
|
||||
Current process reports also include:
|
||||
- module-level results (when runner modules execute), including applied/skipped proposal changes;
|
||||
- run-level module summary totals and failed module instance metadata.
|
||||
- module-level validator decisions and validator rejections.
|
||||
- optional decision-level diagnostic artifact paths for validator LLM interactions when available.
|
||||
- stable validator keys in `validator_name` fields for validator decisions/rejections.
|
||||
- explicit report metadata:
|
||||
- report schema name;
|
||||
- report schema version;
|
||||
- selected output schema;
|
||||
- config file version when config file input is used.
|
||||
- review/observability artifacts:
|
||||
- run-level and module-level utilization/timing summaries;
|
||||
- flattened correction ledger entries for applied/rejected/skipped/failed correction dispositions.
|
||||
|
||||
Utilization diagnostics collection:
|
||||
- collection is performed in the runner path via lightweight instrumentation around LLM scheduler and structured-client execution (`internal/framework/runner`);
|
||||
- instrumentation is observational only and does not change scheduler acquisition/release semantics or module execution order;
|
||||
- serialized artifact: `utilization-diagnostics.json`.
|
||||
|
||||
Utilization diagnostics high-level shape:
|
||||
- `effective_concurrency`:
|
||||
- `total_llm`, `proposal_llm`, `validation_llm`;
|
||||
- `run_timing`:
|
||||
- `run_wall_time_ms`;
|
||||
- `scheduler_queue_wait_ms`;
|
||||
- `llm_execution_time_ms`;
|
||||
- `deterministic_validation_time_ms`;
|
||||
- `max_in_flight_llm_calls`;
|
||||
- `average_in_flight_llm_calls`;
|
||||
- `llm_calls`:
|
||||
- `total_proposal_calls`;
|
||||
- `total_validation_calls`;
|
||||
- `modules`:
|
||||
- per-module key/instance timing summaries including module wall time and per-module call counts;
|
||||
- `validators`:
|
||||
- per-validator summaries keyed by stable validator key with elapsed time and LLM-backed marker.
|
||||
|
||||
Correction ledger construction:
|
||||
- ledger entries are built from runner module results in the CLI report/diagnostics path (`internal/cli/review_artifacts.go`);
|
||||
- serialized artifact: `correction-ledger.json`;
|
||||
- one flattened record per applied/validator-rejected/application-skipped outcome where data is available, plus module-failed records for failed module instances.
|
||||
|
||||
Correction ledger high-level shape:
|
||||
- run/module/proposal identity:
|
||||
- `run_id`, `module_key`, `module_instance`, `proposal_index`, `segment_id`;
|
||||
- correction payload:
|
||||
- `original_text`, `proposed_corrected_text`, `applied_corrected_text` (when applied), `replacement_policy`;
|
||||
- disposition:
|
||||
- `disposition` in `{applied,rejected,skipped,failed}`;
|
||||
- `disposition_reason_code`, `disposition_message`;
|
||||
- validator decision snapshots:
|
||||
- `deterministic_validator_decisions[]`;
|
||||
- `llm_validator_decisions[]`;
|
||||
- each decision uses stable validator keys and reason codes.
|
||||
|
||||
Identity and metadata boundaries:
|
||||
- stable module keys/instance names and stable validator keys are included directly in ledger records;
|
||||
- prompt metadata and structured response schema metadata remain in LLM interaction diagnostics payloads and are not duplicated into every ledger row;
|
||||
- reports reference artifact paths for utilization and ledger files through diagnostics metadata.
|
||||
|
||||
Redaction and retention:
|
||||
- secret redaction guarantees continue to apply to diagnostics/report artifacts;
|
||||
- utilization and ledger artifacts are emitted within the existing run-directory retention model (`auto|always|never`) and are retained/removed with the run directory.
|
||||
|
||||
Current report schema metadata values:
|
||||
- `report_metadata.report_schema_name = "audita-process-report"`
|
||||
- `report_metadata.report_schema_version = "v1"`
|
||||
|
||||
Retention modes implemented in `ApplyRetention`:
|
||||
- `always`: keep all run directories.
|
||||
- `never`: keep successful run directories.
|
||||
- `auto`: keep failed runs and successful runs with skipped corrections.
|
||||
- failed runs are always retained.
|
||||
|
||||
Current runtime note:
|
||||
- default non-explicit runs usually have no module-level skipped corrections, so `auto` commonly removes clean successful run directories.
|
||||
- explicit grammar/glossary/homophones/spoken_word runs can produce validator rejections and application skips, which are reflected in reports and retention input.
|
||||
|
||||
## Current tests and quality posture
|
||||
Implemented tests currently cover:
|
||||
- CLI argument handling and behavior (`internal/cli/run_test.go`)
|
||||
- subprocess stdout/stderr and exit-code behavior (`cmd/audita/main_integration_test.go`)
|
||||
- config/env/override validation (`internal/core/config/*_test.go`)
|
||||
- transcript and glossary schema validation (`internal/core/schema/*_test.go`)
|
||||
- deterministic normalization (`internal/core/normalization/*_test.go`)
|
||||
- deterministic chunking and summaries (`internal/core/chunking/*_test.go`)
|
||||
- proposal preview/apply semantics (`internal/framework/proposals/*_test.go`)
|
||||
- contracts/foundation composition tests (`internal/framework/contracts/*_test.go`)
|
||||
- runner sequencing and failure behavior with deterministic fake modules (`internal/framework/runner/*_test.go`)
|
||||
- CLI runner integration through injected fake module factories (`internal/cli/run_test.go`)
|
||||
- validator models, cardinality enforcement, and deterministic validators (`internal/framework/validators/*_test.go`)
|
||||
- LLM-backed validator batching, prompt builders, structured-response safety, scheduler hooks, and diagnostics redaction (`internal/framework/validators/*_test.go`, `internal/framework/runner/*_test.go`)
|
||||
- shared proposal-generation request/response parsing, deterministic indexing, scheduler hooks, and diagnostics redaction (`internal/framework/proposal_generation/*_test.go`, `internal/framework/runner/*_test.go`)
|
||||
- production module-registry known-key recognition and unsupported/internal-registry error behavior (`internal/framework/modules/*_test.go`, `internal/cli/run_test.go`)
|
||||
- production grammar module prompt constraints, proposal mapping, validator-chain behavior, confidence-threshold enforcement, diagnostics redaction, and explicit CLI/runtime integration (`internal/modules/grammar/*_test.go`, `internal/cli/run_test.go`, `internal/framework/runner/*_test.go`)
|
||||
- production glossary module prompt constraints, proposal mapping, validator-chain behavior, confidence-threshold enforcement, diagnostics redaction, repeated-stage behavior, and explicit CLI/runtime integration (`internal/modules/glossary/*_test.go`, `internal/cli/run_test.go`, `internal/framework/runner/*_test.go`)
|
||||
- production homophones module prompt constraints, proposal mapping, validator-chain behavior, confidence-threshold enforcement, diagnostics redaction, protected-term behavior, and explicit CLI/runtime integration (`internal/modules/homophones/*_test.go`, `internal/cli/run_test.go`, `internal/framework/runner/*_test.go`)
|
||||
- production spoken_word module prompt constraints, proposal mapping, validator-chain behavior, semantic guardrail behavior, confidence-threshold enforcement, diagnostics redaction, protected-term behavior, and explicit CLI/runtime integration (`internal/modules/spoken_word/*_test.go`, `internal/cli/run_test.go`, `internal/framework/runner/*_test.go`)
|
||||
- glossary-derived protected-term extraction and stable behavior (`internal/framework/validators/protected_terms_test.go`)
|
||||
- default full-pipeline runtime shape and ordering (`internal/cli/run_test.go`, `cmd/audita/main_integration_test.go`, `internal/cli/parity_test.go`)
|
||||
- subprocess operational hardening behavior including large-input, failure-mode, timeout/cancellation, backend-failure, and partial-progress paths (`cmd/audita/main_integration_test.go`)
|
||||
- report/diagnostics redaction and artifact-shape behavior across success and failure paths (`internal/cli/run_test.go`, `cmd/audita/main_integration_test.go`)
|
||||
- curated release-fixture and idempotence-oriented readiness checks using fake structured LLM responses (`internal/cli/release_fixtures_test.go`, `internal/cli/testdata/release`)
|
||||
|
||||
## Operational hardening status
|
||||
The runtime now includes hardened subprocess behavior for parent-process callers:
|
||||
- deterministic success/failure exit codes;
|
||||
- strict stdout/stderr separation suitable for machine orchestration;
|
||||
- failure stderr summaries that include diagnostics location when available;
|
||||
- retained failure diagnostics (`report.json`, `error.log`, and artifacts written before failure);
|
||||
- deterministic timeout/cancellation behavior in tests;
|
||||
- redaction coverage for API keys/secrets across reports, diagnostics artifacts, and surfaced errors.
|
||||
- stable output routing behavior:
|
||||
- with `--output`, stdout remains empty on success;
|
||||
- without `--output`, stdout contains only transcript JSON in the selected output schema;
|
||||
- `--report-json` writes report data to file only (never stdout).
|
||||
|
||||
Operational caller guidance is documented in [`docs/subprocess-operations.md`](docs/subprocess-operations.md).
|
||||
|
||||
## Final status
|
||||
- Audita's default full module-sequence runtime is implemented and tested.
|
||||
- Parity fixtures and operational hardening coverage are in place.
|
||||
- Historical migration context is documented in [`docs/migration-from-python.md`](docs/migration-from-python.md).
|
||||
98
docs/architecture/diagnostics.md
Normal file
98
docs/architecture/diagnostics.md
Normal file
@@ -0,0 +1,98 @@
|
||||
# Audita Diagnostics
|
||||
|
||||
This document describes the run-directory diagnostics artifacts produced by `audita process`.
|
||||
|
||||
## Purpose
|
||||
|
||||
Diagnostics provide machine-readable run context and execution artifacts for:
|
||||
- failure debugging;
|
||||
- validator/correction review;
|
||||
- post-run performance analysis.
|
||||
|
||||
Diagnostics are written under the configured work directory (`--work-dir`) when run-directory initialization succeeds.
|
||||
|
||||
## Core artifacts
|
||||
|
||||
Typical artifacts in each run directory:
|
||||
- `source-transcript.json`
|
||||
- `source-transcript-parsed.json`
|
||||
- `normalized-transcript.json`
|
||||
- `normalization-summary.json`
|
||||
- `chunking-summary.json`
|
||||
- `invocation.json`
|
||||
- `effective-config.json` (redacted)
|
||||
- module/validator LLM interaction artifacts
|
||||
- `report.json`
|
||||
- `error.log` on failure
|
||||
|
||||
## Utilization diagnostics artifact
|
||||
|
||||
Artifact:
|
||||
- `utilization-diagnostics.json`
|
||||
|
||||
High-level fields:
|
||||
- `effective_concurrency`:
|
||||
- total/proposal/validation LLM concurrency limits in effect.
|
||||
- `run_timing`:
|
||||
- run wall time;
|
||||
- scheduler queue wait time;
|
||||
- LLM execution time;
|
||||
- deterministic validator time;
|
||||
- max/average in-flight LLM calls.
|
||||
- `llm_calls`:
|
||||
- total proposal and validation LLM call counts.
|
||||
- `modules`:
|
||||
- module-level timing summaries.
|
||||
- `validators`:
|
||||
- per-validator timing summaries keyed by stable validator key.
|
||||
|
||||
## Correction ledger artifact
|
||||
|
||||
Artifact:
|
||||
- `correction-ledger.json`
|
||||
|
||||
Ledger records are flattened review entries derived from module results and include:
|
||||
- module/proposal identity (`module_key`, `module_instance`, `proposal_index`, `segment_id`);
|
||||
- correction text fields and replacement policy when available;
|
||||
- disposition:
|
||||
- `applied`
|
||||
- `rejected`
|
||||
- `skipped`
|
||||
- `failed`
|
||||
- stable reason codes/messages;
|
||||
- deterministic and LLM validator decision snapshots using stable validator keys.
|
||||
|
||||
Validator rejection and proposal-application skip are distinct dispositions.
|
||||
|
||||
## Report references
|
||||
|
||||
`report.json` and optional `--report-json` output include diagnostics metadata paths for:
|
||||
- utilization diagnostics artifact;
|
||||
- correction ledger artifact;
|
||||
- existing transcript/normalization/chunking/invocation/effective-config artifacts.
|
||||
|
||||
## Retention behavior
|
||||
|
||||
Run-directory retention follows configured policy:
|
||||
- `always`: keep all run directories;
|
||||
- `never`: keep successful run directories;
|
||||
- `auto`: keep failed runs and successful runs with skipped/rejected corrections.
|
||||
|
||||
## Redaction guarantees
|
||||
|
||||
API keys and other configured secrets are redacted from:
|
||||
- `effective-config.json`;
|
||||
- LLM interaction diagnostics artifacts;
|
||||
- reports and surfaced errors.
|
||||
|
||||
## Debugging guide
|
||||
|
||||
When debugging:
|
||||
- slow runs:
|
||||
- inspect `utilization-diagnostics.json` (`run_timing`, `modules`, `validators`, in-flight metrics).
|
||||
- validator rejections:
|
||||
- inspect `correction-ledger.json` rejected entries and matching validator decisions;
|
||||
- inspect validator response diagnostics payloads.
|
||||
- application skips:
|
||||
- inspect `correction-ledger.json` skipped entries and skip reason codes;
|
||||
- compare with validator decisions to distinguish validation rejection vs apply-time skip.
|
||||
88
docs/architecture/output-schemas.md
Normal file
88
docs/architecture/output-schemas.md
Normal file
@@ -0,0 +1,88 @@
|
||||
# Audita Output Schemas
|
||||
|
||||
This document describes the built-in transcript output schema registry used by `audita process`.
|
||||
|
||||
## Supported schema names
|
||||
|
||||
### `bare-segments`
|
||||
|
||||
Status:
|
||||
- implemented
|
||||
- default output schema
|
||||
|
||||
Shape:
|
||||
- top-level JSON array of transcript segments
|
||||
|
||||
Segment fields:
|
||||
- `id`
|
||||
- `speaker`
|
||||
- `start`
|
||||
- `end`
|
||||
- `text`
|
||||
- optional `categories`
|
||||
|
||||
Compatibility:
|
||||
- this preserves the long-standing output shape used by existing consumers.
|
||||
|
||||
### `audita-v1`
|
||||
|
||||
Status:
|
||||
- implemented
|
||||
|
||||
Shape:
|
||||
- top-level JSON object:
|
||||
- `schema`: `"audita-v1"`
|
||||
- `version`: `"v1"`
|
||||
- `segments`: transcript segment array
|
||||
|
||||
Segment fields inside `segments` match `bare-segments` segment fields.
|
||||
|
||||
Compatibility:
|
||||
- this is the Audita-native object format with explicit schema/version metadata.
|
||||
|
||||
### `seriatim-intermediate`
|
||||
|
||||
Status:
|
||||
- deferred / not implemented
|
||||
|
||||
Current behavior:
|
||||
- selecting `seriatim-intermediate` fails clearly as an unsupported output schema.
|
||||
|
||||
Reason:
|
||||
- a concrete, repository-backed contract for this schema has not been finalized yet.
|
||||
|
||||
## Selection
|
||||
|
||||
Choose output schema with CLI:
|
||||
|
||||
```sh
|
||||
audita process <transcript.json> --glossary <glossary.yaml> --output-schema audita-v1
|
||||
```
|
||||
|
||||
Or in file config:
|
||||
|
||||
```yaml
|
||||
version: 1
|
||||
output:
|
||||
schema: audita-v1
|
||||
```
|
||||
|
||||
Precedence remains:
|
||||
1. defaults
|
||||
2. file config
|
||||
3. environment overrides
|
||||
4. CLI overrides
|
||||
|
||||
`--output-schema` overrides `output.schema` when both are supplied.
|
||||
|
||||
## Output routing behavior
|
||||
|
||||
- With `--output`, transcript JSON is written to file using the selected schema and stdout stays empty on success.
|
||||
- Without `--output`, stdout contains transcript JSON only, using the selected schema.
|
||||
- `--report-json` writes report JSON to file and does not write report payloads to stdout.
|
||||
|
||||
## Backward-compatibility expectations
|
||||
|
||||
- default schema stays `bare-segments` for compatibility unless explicitly changed in a future breaking release;
|
||||
- supported schema names are treated as stable public contract values;
|
||||
- unsupported schema names fail before output write.
|
||||
118
docs/architecture/prompts.md
Normal file
118
docs/architecture/prompts.md
Normal file
@@ -0,0 +1,118 @@
|
||||
# Audita Prompts
|
||||
|
||||
This document describes Audita's built-in embedded prompt assets and prompt registry behavior.
|
||||
|
||||
## Why embedded prompt assets
|
||||
|
||||
Audita embeds production prompt text into the binary so runtime behavior is:
|
||||
- deterministic;
|
||||
- auditable;
|
||||
- dependency-light;
|
||||
- not dependent on external prompt files at execution time.
|
||||
|
||||
Prompt text is authored as Markdown assets and rendered by Go code using typed template data.
|
||||
|
||||
## Built-in prompt registry
|
||||
|
||||
The prompt registry lives in `internal/prompts` and is responsible for:
|
||||
- loading embedded prompt assets;
|
||||
- registering stable prompt IDs and versions;
|
||||
- recording prompt source metadata;
|
||||
- computing deterministic SHA-256 source hashes;
|
||||
- rendering system/user prompts with strict missing-key failures.
|
||||
|
||||
Current prompt source behavior:
|
||||
- built-in embedded prompts only (`prompt_source = builtin`).
|
||||
- filesystem prompt overrides are not supported.
|
||||
|
||||
## Built-in prompt IDs
|
||||
|
||||
Module proposal prompts:
|
||||
- `modules.glossary.proposal`
|
||||
- `modules.homophones.proposal`
|
||||
- `modules.spoken_word.proposal`
|
||||
- `modules.grammar.proposal`
|
||||
|
||||
LLM-backed validator prompts:
|
||||
- `validators.spoken_form_plausibility`
|
||||
- `validators.meaning_reversal_review`
|
||||
- `validators.editorial_review`
|
||||
- `validators.grammar_review`
|
||||
- `validators.spoken_word_review`
|
||||
|
||||
## Prompt version semantics
|
||||
|
||||
Current built-in prompt version value is `v1`.
|
||||
|
||||
Version is a stable metadata identifier for diagnostics and debugging. It is not a dynamic prompt-selection mechanism.
|
||||
|
||||
## Prompt hash semantics
|
||||
|
||||
Each registered prompt includes a deterministic SHA-256 hash of embedded source text.
|
||||
|
||||
Hash purpose:
|
||||
- identify exact prompt source used in a run;
|
||||
- support diagnostics reproducibility and change auditing.
|
||||
|
||||
Current hash scope:
|
||||
- source prompt text (system + user assets for a registered prompt), not a runtime secret-bearing payload.
|
||||
|
||||
## Template rendering behavior
|
||||
|
||||
Prompt rendering uses Go `text/template` with typed template data from module/validator builders.
|
||||
|
||||
Missing-key behavior:
|
||||
- rendering uses missing-key errors;
|
||||
- missing/renamed template fields fail quickly instead of silently producing incomplete prompts.
|
||||
|
||||
Go code still owns:
|
||||
- structured request/response models;
|
||||
- response schema selection;
|
||||
- transcript/glossary/payload formatting;
|
||||
- module and validator selection;
|
||||
- diagnostics wiring.
|
||||
|
||||
## Shared prompt hardening policy
|
||||
|
||||
A shared hardening fragment is embedded once and included in every module proposal prompt and every LLM-validator prompt.
|
||||
|
||||
Hardening policy includes:
|
||||
- transcript text is untrusted data;
|
||||
- glossary entries and transcript descriptions are reference data, not instructions;
|
||||
- instructions found inside transcript text must not be obeyed;
|
||||
- model must perform only the requested correction/validation task;
|
||||
- no invention of facts, names, events, motivations, speaker intent, or corrections;
|
||||
- transcript remains the source of truth.
|
||||
|
||||
## Transcript description behavior
|
||||
|
||||
Transcript description remains background-only prompt context:
|
||||
- it may help interpret ambiguous terms;
|
||||
- it is explicitly non-authoritative and must not override transcript content;
|
||||
- empty descriptions do not render awkward blank context sections.
|
||||
|
||||
Generated transcript descriptions are not implemented in this workstream.
|
||||
|
||||
## Diagnostics and report metadata boundaries
|
||||
|
||||
Current metadata flow:
|
||||
- proposal-generation diagnostics request metadata includes prompt metadata;
|
||||
- LLM-validator diagnostics request metadata includes prompt metadata.
|
||||
|
||||
Prompt metadata fields used in diagnostics:
|
||||
- `prompt_id`
|
||||
- `prompt_version`
|
||||
- `prompt_source`
|
||||
- `embedded_path`
|
||||
- `sha256`
|
||||
|
||||
Current boundary:
|
||||
- detailed prompt metadata is diagnostics-first;
|
||||
- broad report-level prompt registries/ledgers are deferred.
|
||||
|
||||
## 1.0 boundary
|
||||
|
||||
Not implemented for 1.0 in this workstream:
|
||||
- filesystem prompt overrides;
|
||||
- user-configurable prompt selection;
|
||||
- external prompt directories.
|
||||
165
docs/architecture/public-contract.md
Normal file
165
docs/architecture/public-contract.md
Normal file
@@ -0,0 +1,165 @@
|
||||
# Audita Public Contract
|
||||
|
||||
This document defines stability expectations for Audita's external process and data interfaces.
|
||||
|
||||
## Scope
|
||||
|
||||
This contract covers:
|
||||
- CLI invocation and behavior
|
||||
- versioned config file behavior
|
||||
- transcript/glossary input forms
|
||||
- transcript output schema selection
|
||||
- process report schema metadata
|
||||
- stable validator key identifiers in report/diagnostics records
|
||||
- prompt metadata identifiers in diagnostics
|
||||
- diagnostics directory behavior
|
||||
- utilization diagnostics and correction-ledger artifact presence/pathing in diagnostics metadata
|
||||
- stdout/stderr and exit-code behavior
|
||||
- secret redaction guarantees
|
||||
- compatibility and deprecation policy
|
||||
|
||||
## CLI stability expectations
|
||||
|
||||
Stable commands:
|
||||
- `audita process`
|
||||
- `audita config validate`
|
||||
- `audita config print-effective`
|
||||
|
||||
For `audita process`, stable high-value flags include:
|
||||
- `--config`
|
||||
- `--glossary`
|
||||
- `--output`
|
||||
- `--report-json`
|
||||
- `--modules`
|
||||
- `--output-schema`
|
||||
|
||||
Compatibility flags and lower-level tuning flags remain available; they may be narrowed over time with explicit compatibility notes.
|
||||
|
||||
## Config file stability expectations
|
||||
|
||||
Supported file format:
|
||||
- YAML
|
||||
- strict unknown-field rejection
|
||||
- explicit `version`
|
||||
|
||||
Supported version:
|
||||
- `version: 1`
|
||||
|
||||
Precedence for `audita process`:
|
||||
1. built-in defaults
|
||||
2. file config
|
||||
3. environment overrides
|
||||
4. CLI overrides
|
||||
|
||||
Config source behavior:
|
||||
- `--config <path>`: missing path is a clear failure
|
||||
- `AUDITA_CONFIG`: missing path is a clear failure
|
||||
- defaults `/usr/local/etc/audita/config.yml`, then `/etc/audita/config.yml`: both missing is non-fatal
|
||||
|
||||
## Supported transcript input forms
|
||||
|
||||
Audita accepts transcript JSON as either:
|
||||
- a top-level array of segments
|
||||
- an object with a `segments` array
|
||||
|
||||
Segments must satisfy the schema and validation rules enforced by `internal/core/schema`.
|
||||
|
||||
## Supported glossary input form
|
||||
|
||||
Audita accepts glossary YAML with a top-level `glossary` entry list and validates required fields per entry.
|
||||
|
||||
## Supported output schema names
|
||||
|
||||
Built-in output schema registry supports:
|
||||
- `bare-segments` (default)
|
||||
- `audita-v1`
|
||||
|
||||
`seriatim-intermediate` is planned but not implemented.
|
||||
|
||||
Unknown output schema names fail clearly.
|
||||
|
||||
## Report schema/versioning expectations
|
||||
|
||||
Process report payloads include `report_metadata` with:
|
||||
- `report_schema_name`
|
||||
- `report_schema_version`
|
||||
- `output_schema`
|
||||
- `config_version` when file config is used
|
||||
|
||||
Current values:
|
||||
- `report_schema_name`: `audita-process-report`
|
||||
- `report_schema_version`: `v1`
|
||||
|
||||
`--report-json` output and diagnostics run-dir `report.json` use the same report schema metadata.
|
||||
|
||||
Validator decision/rejection records in reports use stable validator keys in `validator_name`.
|
||||
Report diagnostics metadata includes artifact-path fields for utilization diagnostics and correction ledger when diagnostics initialization succeeds.
|
||||
|
||||
## Diagnostics directory behavior
|
||||
|
||||
When diagnostics directory creation succeeds, Audita writes run artifacts including:
|
||||
- invocation metadata
|
||||
- redacted effective config
|
||||
- transcript/normalization/chunking artifacts
|
||||
- utilization diagnostics (`utilization-diagnostics.json`)
|
||||
- correction ledger (`correction-ledger.json`)
|
||||
- report and failure error log (when applicable)
|
||||
- module/LLM diagnostics artifacts as available
|
||||
|
||||
Retention behavior is controlled by configured retention mode; failed runs are retained.
|
||||
|
||||
Diagnostics metadata for LLM interactions may include semi-public prompt identifiers:
|
||||
- `prompt_id`
|
||||
- `prompt_version`
|
||||
- `prompt_source`
|
||||
- `embedded_path`
|
||||
- `sha256`
|
||||
|
||||
These are diagnostic identifiers, not user-facing prompt override controls.
|
||||
|
||||
## Stdout/stderr behavior
|
||||
|
||||
Success behavior:
|
||||
- with `--output`, stdout is empty
|
||||
- without `--output`, stdout contains only transcript JSON in selected output schema
|
||||
- report JSON is not written to stdout
|
||||
|
||||
Failure behavior:
|
||||
- stderr contains human-readable error summary
|
||||
- nonzero exit
|
||||
- diagnostics path is printed when available
|
||||
|
||||
## Exit-code behavior
|
||||
|
||||
- `0`: success
|
||||
- nonzero: failure
|
||||
|
||||
Treat any nonzero exit as a failed invocation.
|
||||
|
||||
## Secret redaction guarantees
|
||||
|
||||
Audita redacts API keys and authorization secrets from:
|
||||
- effective config outputs (`audita config print-effective`, diagnostics effective-config artifact)
|
||||
- report artifacts
|
||||
- LLM diagnostics artifacts
|
||||
- surfaced request/response error messages
|
||||
|
||||
Config files should reference secrets via environment variable names (`api_key_env`) rather than embedding secret values.
|
||||
|
||||
## Compatibility and deprecation policy
|
||||
|
||||
- Existing stable schema names, report metadata keys, and top-level command behavior are treated as public contract.
|
||||
- Compatibility inputs (legacy flags/env aliases) may remain during transition windows.
|
||||
- Any planned removal or behavior change should include clear compatibility notes and migration guidance.
|
||||
|
||||
## Breaking changes after 1.0
|
||||
|
||||
After 1.0, breaking changes include, for example:
|
||||
- changing default success/failure exit-code semantics
|
||||
- changing stdout/stderr routing semantics
|
||||
- silently changing default output schema shape
|
||||
- removing supported output schema names without compatibility strategy
|
||||
- changing report schema fields or meanings incompatibly
|
||||
- changing config version semantics incompatibly without version bump
|
||||
|
||||
Additive fields, additive diagnostics, and new optional schema names are generally non-breaking when existing behavior remains intact.
|
||||
90
docs/architecture/structured-llm.md
Normal file
90
docs/architecture/structured-llm.md
Normal file
@@ -0,0 +1,90 @@
|
||||
# Structured LLM Architecture
|
||||
|
||||
## Purpose
|
||||
|
||||
This document describes Audita's structured LLM runtime boundary and adapter behavior.
|
||||
|
||||
## Why Audita owns the adapter
|
||||
|
||||
Audita owns a small structured LLM adapter so that core runtime behavior is controlled inside the repository:
|
||||
- request construction and schema handling are explicit and testable;
|
||||
- retries, timeouts, cancellation, and error redaction are consistent across modules and validators;
|
||||
- provider SDK types are not exposed outside the adapter boundary;
|
||||
- dependency weight and transitive provider-specific behavior are reduced.
|
||||
|
||||
At runtime, the rest of Audita depends only on the internal contract:
|
||||
- `StructuredLLMClient`
|
||||
- `CompleteStructured(ctx, req, out)`
|
||||
|
||||
## OpenAI-compatible request shape
|
||||
|
||||
At a conceptual level, Audita sends chat completion requests with:
|
||||
- `model`
|
||||
- `messages` (role/content pairs)
|
||||
- `response_format`:
|
||||
- `type = "json_schema"`
|
||||
- `json_schema.name` (stable schema name)
|
||||
- `json_schema.strict = true`
|
||||
- `json_schema.schema` (registered JSON Schema payload)
|
||||
|
||||
The adapter uses OpenAI-compatible `POST {base_url}/chat/completions` over `net/http`.
|
||||
|
||||
## Structured response schema registry
|
||||
|
||||
Structured response schemas are registered in `internal/framework/responseschema` with stable metadata:
|
||||
- schema key
|
||||
- schema ID
|
||||
- schema version
|
||||
- schema name (OpenAI-compatible `response_format` name)
|
||||
- raw JSON Schema payload
|
||||
- SHA-256 hash
|
||||
|
||||
Current schemas:
|
||||
- `correction_set`:
|
||||
- id `audita.correction_set`
|
||||
- version `v1`
|
||||
- name `audita_correction_set_v1`
|
||||
- `validator_decision_set`:
|
||||
- id `audita.validator_decision_set`
|
||||
- version `v1`
|
||||
- name `audita_validator_decision_set_v1`
|
||||
|
||||
## Provider compatibility assumptions
|
||||
|
||||
Audita assumes an OpenAI-compatible chat-completions endpoint that:
|
||||
- accepts message arrays with model selection;
|
||||
- accepts `response_format.type = json_schema`;
|
||||
- returns a completion with assistant message content and optional usage metadata.
|
||||
|
||||
Provider-specific differences are expected in strictness and error payload shapes, so the adapter treats provider output as untrusted until locally decoded.
|
||||
|
||||
## Local decode and validation remain mandatory
|
||||
|
||||
Provider-level structured output is a transport guardrail, not final validation.
|
||||
|
||||
After receiving a response, Audita still:
|
||||
- decodes assistant content into typed request-specific structs;
|
||||
- validates proposal and validator payload invariants locally;
|
||||
- enforces deterministic validator/cardinality rules before any transcript application.
|
||||
|
||||
This protects runtime correctness even when provider responses are malformed, partial, or semantically inconsistent.
|
||||
|
||||
## Diagnostics and redaction
|
||||
|
||||
When structured schemas are used, diagnostics metadata records:
|
||||
- schema ID
|
||||
- schema version
|
||||
- schema name
|
||||
- schema hash
|
||||
|
||||
Diagnostics and surfaced errors preserve secret redaction:
|
||||
- API keys and bearer tokens are redacted from request/response/error artifacts;
|
||||
- redaction is applied before diagnostic files are written.
|
||||
|
||||
## Runtime behavior guarantees
|
||||
|
||||
The structured LLM path preserves existing runtime guarantees:
|
||||
- bounded LLM call execution through schedulers;
|
||||
- context-aware cancellation and timeout propagation;
|
||||
- retry behavior for transient failures and retryable malformed structured responses;
|
||||
- deterministic module/chunk/proposal/validator behavior outside provider nondeterminism.
|
||||
148
docs/architecture/validators.md
Normal file
148
docs/architecture/validators.md
Normal file
@@ -0,0 +1,148 @@
|
||||
# Audita Validators
|
||||
|
||||
This document describes Audita's built-in validator registry and module validator chains.
|
||||
|
||||
For LLM-backed validator prompt asset details, see [`docs/prompts.md`](prompts.md).
|
||||
|
||||
## Package ownership
|
||||
|
||||
Built-in validator construction is package-owned under `internal/validators/<validator_key>`:
|
||||
- `internal/validators/confidence_threshold`
|
||||
- `internal/validators/original_text_presence`
|
||||
- `internal/validators/non_empty_corrected_text`
|
||||
- `internal/validators/no_effect`
|
||||
- `internal/validators/protected_terms`
|
||||
- `internal/validators/spoken_form_plausibility`
|
||||
- `internal/validators/meaning_reversal_review`
|
||||
- `internal/validators/editorial_review`
|
||||
|
||||
Registry and chain wiring stay in:
|
||||
- `internal/validators/registry.go`
|
||||
- `internal/validators/chains.go`
|
||||
|
||||
Shared validator runtime mechanics stay in `internal/framework/validators`:
|
||||
- request/result/decision models
|
||||
- decision cardinality helpers
|
||||
- protected vocabulary helpers
|
||||
- shared LLM validator runtime, batching, and diagnostics helpers
|
||||
|
||||
Execution classification metadata is defined in `internal/validators/metadata`:
|
||||
- `deterministic`
|
||||
- `llm_backed`
|
||||
|
||||
Runner ordering uses this metadata so deterministic validators run before LLM-backed validators without concrete framework type assertions.
|
||||
|
||||
## Scope
|
||||
|
||||
Validator chains are built-in runtime behavior.
|
||||
|
||||
Current 1.0 boundary:
|
||||
- built-in validator keys and built-in module chains are stable runtime identifiers;
|
||||
- thresholds and batching knobs remain configurable where already supported;
|
||||
- arbitrary user-defined validator chains are deferred.
|
||||
|
||||
## Built-in validator keys
|
||||
|
||||
### Deterministic validators
|
||||
|
||||
- `confidence_threshold`
|
||||
- checks proposal confidence against module-specific configured threshold.
|
||||
- `original_text_presence`
|
||||
- ensures target segment exists and `original_text` exists in current working segment text.
|
||||
- `non_empty_corrected_text`
|
||||
- rejects blank/whitespace-only `corrected_text`.
|
||||
- `no_effect`
|
||||
- rejects proposals where `original_text == corrected_text`.
|
||||
- `protected_terms`
|
||||
- protects glossary-derived terms from unsafe mutations in non-glossary modules.
|
||||
- glossary stages use glossary-specific protection logic but still report this same stable key.
|
||||
|
||||
### LLM-backed validators
|
||||
|
||||
- `spoken_form_plausibility`
|
||||
- checks whether proposed spoken-form change remains plausible in transcript context.
|
||||
- `meaning_reversal_review`
|
||||
- checks for likely meaning reversal or semantic contradiction.
|
||||
- `editorial_review`
|
||||
- performs conservative editorial safety review.
|
||||
|
||||
## Built-in module chains
|
||||
|
||||
Current built-in chains resolved from `internal/validators/chains.go`:
|
||||
|
||||
- `glossary`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
|
||||
- `homophones`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
|
||||
- `spoken_word`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `editorial_review`
|
||||
- `meaning_reversal_review`
|
||||
|
||||
- `grammar`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `editorial_review`
|
||||
- `meaning_reversal_review`
|
||||
|
||||
## Protected terms construction
|
||||
|
||||
`protected_terms` has explicit constructors:
|
||||
- general constructor used by non-glossary modules through the built-in registry
|
||||
- glossary-stage constructor used by glossary chain resolution
|
||||
|
||||
Both variants preserve existing behavior and report the stable key `protected_terms`.
|
||||
|
||||
## Execution semantics
|
||||
|
||||
- modules execute serially;
|
||||
- section proposal work can run concurrently within a module;
|
||||
- deterministic validators run before LLM-backed validators;
|
||||
- malformed/missing/duplicate/unknown LLM validator decisions fail safely;
|
||||
- approved proposals are applied once per module after section work settles.
|
||||
|
||||
## Validator rejections vs proposal-application skips
|
||||
|
||||
- validator rejection:
|
||||
- proposal is denied by validator-chain review and appears in validator rejection reporting with validator key and reason code.
|
||||
- proposal-application skip:
|
||||
- proposal passed validators but could not be applied under replacement-policy semantics (for example no matching span at apply time).
|
||||
|
||||
These are separate outcomes and are reported separately.
|
||||
|
||||
## Reporting and diagnostics identity
|
||||
|
||||
- report validator decision/rejection entries use stable validator keys in `validator_name`.
|
||||
- validator LLM diagnostics include validator identity in interaction metadata and structured response schema metadata.
|
||||
- correction ledger entries include deterministic and LLM validator decision snapshots keyed by the same stable validator keys, and keep validator rejection distinct from application-level skip.
|
||||
|
||||
Prompt assets are unchanged by the validator package-ownership refactor and remain built-in under `internal/prompts`.
|
||||
|
||||
## Configurable knobs that remain supported
|
||||
|
||||
- per-module confidence thresholds (`thresholds.*` / equivalent env+CLI overrides)
|
||||
- validation batching limits (`validation_max_prompt_tokens` / equivalent env+CLI overrides)
|
||||
- validation LLM model/base URL/timeout/retries/concurrency settings
|
||||
|
||||
These tune validator behavior without exposing arbitrary user-defined chains.
|
||||
190
docs/configuration.md
Normal file
190
docs/configuration.md
Normal file
@@ -0,0 +1,190 @@
|
||||
# Audita Configuration
|
||||
|
||||
This document describes Audita's versioned YAML config support and related commands.
|
||||
|
||||
## Purpose
|
||||
|
||||
Audita's config file provides a stable place for pipeline defaults and runtime tuning that would otherwise require many environment variables or CLI flags.
|
||||
|
||||
Use config files for baseline settings, then use environment variables and CLI flags for deployment and per-run overrides.
|
||||
|
||||
## Supported version
|
||||
|
||||
Current supported config version:
|
||||
|
||||
- `version: 1`
|
||||
|
||||
Rules:
|
||||
|
||||
- missing `version` fails validation;
|
||||
- unknown versions fail validation;
|
||||
- unknown fields fail validation (strict decoding).
|
||||
|
||||
## Config path resolution
|
||||
|
||||
For `audita process`, config path resolution is:
|
||||
|
||||
1. `--config <path>` if provided
|
||||
2. `AUDITA_CONFIG` if set and `--config` is not provided
|
||||
3. default `/usr/local/etc/audita/config.yml` if present
|
||||
4. fallback default `/etc/audita/config.yml` if present
|
||||
|
||||
Missing-file behavior:
|
||||
|
||||
- missing `--config` path: hard failure;
|
||||
- missing `AUDITA_CONFIG` path: hard failure;
|
||||
- missing both default-path files: non-fatal, run continues.
|
||||
|
||||
## Precedence model
|
||||
|
||||
Effective config precedence is:
|
||||
|
||||
1. built-in defaults
|
||||
2. file config
|
||||
3. environment overrides
|
||||
4. CLI overrides
|
||||
|
||||
## Supported YAML fields
|
||||
|
||||
```yaml
|
||||
version: 1
|
||||
|
||||
pipeline:
|
||||
modules: [glossary, homophones, glossary, spoken_word, grammar]
|
||||
|
||||
output:
|
||||
schema: bare-segments
|
||||
|
||||
llm:
|
||||
proposal:
|
||||
base_url: https://openrouter.ai/api/v1
|
||||
model: openrouter/google/gemma-4-31b-it
|
||||
api_key_env: AUDITA_LLM_API_KEY
|
||||
timeout: 120s
|
||||
max_retries: 3
|
||||
|
||||
validation:
|
||||
base_url: https://openrouter.ai/api/v1
|
||||
model: openrouter/google/gemma-4-31b-it
|
||||
api_key_env: AUDITA_VALIDATION_LLM_API_KEY
|
||||
timeout: 120s
|
||||
max_retries: 3
|
||||
|
||||
concurrency:
|
||||
total_llm: 2
|
||||
proposal_llm: 2
|
||||
validation_llm: 1
|
||||
|
||||
chunking:
|
||||
target_sections: 8
|
||||
max_section_tokens: 8192
|
||||
min_section_tokens: 2048
|
||||
|
||||
normalization:
|
||||
max_segment_gap: 4s
|
||||
ellipsis_gap: 3.5s
|
||||
max_segment_duration: 60s
|
||||
max_segment_tokens: 2048
|
||||
|
||||
thresholds:
|
||||
glossary: 0.8
|
||||
homophones: 0.8
|
||||
spoken_word: 0.8
|
||||
grammar: 0.8
|
||||
|
||||
context:
|
||||
description: "optional transcript background context"
|
||||
|
||||
diagnostics:
|
||||
work_dir: /tmp/audita
|
||||
retention: auto
|
||||
```
|
||||
|
||||
`context.description` provides background-only transcript context for prompts.
|
||||
If both config and CLI provide a description, `--transcript-description` takes precedence.
|
||||
|
||||
`output.schema` supports the built-in output schema registry values:
|
||||
- `bare-segments` (default)
|
||||
- `audita-v1`
|
||||
|
||||
Unknown schema names fail clearly before transcript output is written.
|
||||
|
||||
Duration-like fields accept either:
|
||||
|
||||
- numeric seconds (for example `120`, `3.5`), or
|
||||
- duration strings (for example `120s`, `2m`).
|
||||
|
||||
For LLM timeouts, duration strings must resolve to whole seconds.
|
||||
|
||||
## Secret handling
|
||||
|
||||
Use `api_key_env` for secrets:
|
||||
|
||||
- `llm.proposal.api_key_env`
|
||||
- `llm.validation.api_key_env`
|
||||
|
||||
These fields must contain environment variable names, not secret values.
|
||||
|
||||
At runtime, Audita resolves those names from the process environment.
|
||||
|
||||
Redaction behavior:
|
||||
|
||||
- run diagnostics `effective-config.json` is redacted;
|
||||
- `audita config print-effective` output is redacted;
|
||||
- API keys are never emitted in plaintext by those outputs.
|
||||
|
||||
## Config commands
|
||||
|
||||
Validate a config file:
|
||||
|
||||
```sh
|
||||
audita config validate --config ./audita.yml
|
||||
```
|
||||
|
||||
Print redacted effective config:
|
||||
|
||||
```sh
|
||||
audita config print-effective --config ./audita.yml
|
||||
```
|
||||
|
||||
`print-effective` loads defaults, then file config, then environment overrides.
|
||||
|
||||
## Example: local OpenAI-compatible endpoint
|
||||
|
||||
```yaml
|
||||
version: 1
|
||||
|
||||
llm:
|
||||
proposal:
|
||||
base_url: http://localhost:8000/v1
|
||||
model: local/proposal-model
|
||||
api_key_env: AUDITA_LLM_API_KEY
|
||||
timeout: 90s
|
||||
max_retries: 2
|
||||
|
||||
validation:
|
||||
base_url: http://localhost:8000/v1
|
||||
model: local/validation-model
|
||||
api_key_env: AUDITA_VALIDATION_LLM_API_KEY
|
||||
timeout: 90s
|
||||
max_retries: 2
|
||||
|
||||
pipeline:
|
||||
modules: [glossary, homophones, glossary, spoken_word, grammar]
|
||||
|
||||
diagnostics:
|
||||
work_dir: /tmp/audita
|
||||
retention: auto
|
||||
```
|
||||
|
||||
## Compatibility notes
|
||||
|
||||
Existing environment variables and lower-level CLI flags remain available for compatibility.
|
||||
|
||||
Current guidance:
|
||||
|
||||
- prefer file config for baseline behavior;
|
||||
- keep environment variables for secrets/deployment-specific overrides;
|
||||
- use CLI flags for per-run overrides.
|
||||
- validator chains are built-in and are not user-configurable in config.
|
||||
- prompt source selection and filesystem prompt overrides are not config options.
|
||||
96
docs/integration/subprocess-operations.md
Normal file
96
docs/integration/subprocess-operations.md
Normal file
@@ -0,0 +1,96 @@
|
||||
# Audita Subprocess Operations
|
||||
|
||||
This document describes how parent processes should invoke `audita process` safely in production orchestration.
|
||||
|
||||
## Recommended command form
|
||||
|
||||
Use explicit file outputs for orchestrated runs:
|
||||
|
||||
```sh
|
||||
audita process <transcript.json> \
|
||||
--transcript-description "Brief context that may help resolve ambiguous terms." \
|
||||
--glossary <glossary.yaml> \
|
||||
--output <output-transcript.json> \
|
||||
--report-json <report.json>
|
||||
```
|
||||
|
||||
Additional flags that may be situationally appropriate:
|
||||
- `--config <path>` to select an explicit versioned config file.
|
||||
- `--output-schema <bare-segments|audita-v1>` to select transcript output shape.
|
||||
- `--work-dir <dir>` to control diagnostics location.
|
||||
- `--work-dir-retention <always|auto|never>` to control retained run directories.
|
||||
- `--total-llm-concurrency`, `--proposal-llm-concurrency`, and `--validation-llm-concurrency` when orchestration needs to set explicit LLM throughput controls.
|
||||
- `--modules ...` only when intentionally overriding the default sequence.
|
||||
|
||||
For config-driven orchestration, validate config files in CI/preflight:
|
||||
|
||||
```sh
|
||||
audita config validate --config <path>
|
||||
```
|
||||
|
||||
## Stdout behavior
|
||||
|
||||
- With `--output`: stdout is expected to be empty on success.
|
||||
- Without `--output`: stdout contains transcript JSON only on success.
|
||||
- Report JSON is never written to stdout.
|
||||
|
||||
## Stderr behavior
|
||||
|
||||
- Success path should be quiet or minimal human-readable logs.
|
||||
- Failure path writes concise human-readable errors.
|
||||
- When a diagnostics run directory exists, failure stderr includes its path.
|
||||
- Prompt/response diagnostic payloads are not streamed to stderr.
|
||||
|
||||
## Output file behavior
|
||||
|
||||
- `--output` writes transcript JSON in the selected output schema to the provided path.
|
||||
- Output write failures return nonzero and surface actionable errors.
|
||||
- The command does not silently ignore output write errors.
|
||||
|
||||
## Report JSON behavior
|
||||
|
||||
- `--report-json` writes a machine-readable process report to the requested path.
|
||||
- Run-directory `report.json` is written independently under diagnostics.
|
||||
- Best-effort failure reports are emitted when possible without masking the primary failure.
|
||||
- Report write failures return nonzero with clear stderr messaging.
|
||||
- Report diagnostics metadata references run-directory artifacts including utilization diagnostics and correction ledger paths when available.
|
||||
|
||||
## Diagnostics directory behavior
|
||||
|
||||
- Each run creates (when possible) a per-run diagnostics directory.
|
||||
- Typical artifacts include transcript, normalization, chunking, invocation, effective config, LLM diagnostics, `utilization-diagnostics.json`, `correction-ledger.json`, `report.json`, and `error.log` on failure.
|
||||
- Failed runs retain diagnostics.
|
||||
- Under `auto` retention, successful runs with skipped/rejected corrections are retained; clean successful runs may be removed.
|
||||
|
||||
## Exit codes
|
||||
|
||||
- `0`: success.
|
||||
- Nonzero: failure (input/schema/config/module/LLM/runtime/output/report/diagnostics errors).
|
||||
|
||||
Treat any nonzero as a failed subprocess invocation.
|
||||
|
||||
## Timeout and cancellation
|
||||
|
||||
- Runtime operations propagate context cancellation and request timeouts through LLM/scheduler paths.
|
||||
- On cancellation or timeout, the process exits nonzero and should not hang.
|
||||
- If diagnostics were initialized before failure, failure artifacts remain available for debugging.
|
||||
|
||||
## Secret redaction expectations
|
||||
|
||||
API keys and configured secret values are redacted from:
|
||||
- reports (`--report-json` and run-dir `report.json`);
|
||||
- diagnostics artifacts (including effective config and LLM interaction artifacts);
|
||||
- surfaced adapter/runtime errors;
|
||||
- test fixtures and regression outputs.
|
||||
|
||||
Parent-process logs should still avoid printing raw environment variables.
|
||||
|
||||
## Parent-process pipe guidance
|
||||
|
||||
To avoid deadlocks in orchestrators:
|
||||
- always read both stdout and stderr concurrently when invoking as a subprocess;
|
||||
- prefer file outputs (`--output`, `--report-json`) for machine workflows;
|
||||
- treat stderr as human-readable diagnostics, not structured data;
|
||||
- parse structured results from output/report files.
|
||||
|
||||
For Go callers, prefer `exec.CommandContext` with explicit timeout/cancellation and buffered/streamed readers for both pipes.
|
||||
124
docs/release-checklist.md
Normal file
124
docs/release-checklist.md
Normal file
@@ -0,0 +1,124 @@
|
||||
# Audita Release Checklist
|
||||
|
||||
Use this checklist before cutting a pre-1.0 or 1.0 release candidate.
|
||||
|
||||
## Core test pass
|
||||
|
||||
- Run:
|
||||
- `go test ./...`
|
||||
- Confirm tests pass without live LLM credentials and without Python dependencies.
|
||||
|
||||
## Config validation and precedence
|
||||
|
||||
- Validate a representative config:
|
||||
- `audita config validate --config <path>`
|
||||
- Inspect redacted effective config:
|
||||
- `audita config print-effective --config <path>`
|
||||
- Confirm precedence behavior:
|
||||
- defaults -> file config -> environment -> CLI.
|
||||
- Confirm default config search order:
|
||||
- `/usr/local/etc/audita/config.yml` first, then `/etc/audita/config.yml`.
|
||||
- Confirm missing both default-path config files is non-fatal when `--config`/`AUDITA_CONFIG` are unset.
|
||||
|
||||
## Output schema checks
|
||||
|
||||
- Verify default output schema remains `bare-segments`.
|
||||
- Verify `--output-schema audita-v1` emits object payload with `schema` and `version`.
|
||||
- Verify unknown schema (for example `seriatim-intermediate`) fails clearly.
|
||||
|
||||
## Subprocess contract checks
|
||||
|
||||
- With `--output`, verify stdout is empty on success.
|
||||
- Without `--output`, verify stdout contains transcript JSON only.
|
||||
- Verify `--report-json` writes file output and does not write report JSON to stdout.
|
||||
- Verify failure stderr remains human-readable and includes diagnostics path when available.
|
||||
- Verify nonzero exit on failures.
|
||||
|
||||
## Structured LLM checks
|
||||
|
||||
- Verify runtime uses the Audita-owned OpenAI-compatible adapter.
|
||||
- Verify structured response schemas are attached via `response_format.type=json_schema`.
|
||||
- Verify diagnostics metadata includes structured schema `id/version/name/sha256`.
|
||||
- Verify provider output is still locally decoded/validated before use.
|
||||
|
||||
## Report and diagnostics schema checks
|
||||
|
||||
- Verify report metadata fields:
|
||||
- `report_schema_name`
|
||||
- `report_schema_version`
|
||||
- `output_schema`
|
||||
- `config_version` when file config is used.
|
||||
- Verify diagnostics artifact references exist in reports:
|
||||
- transcript/normalization/chunking/invocation/effective-config artifacts
|
||||
- utilization diagnostics artifact
|
||||
- correction ledger artifact
|
||||
- error log on failures.
|
||||
|
||||
## Redaction checks
|
||||
|
||||
- Verify secrets are redacted from:
|
||||
- `effective-config.json`
|
||||
- run-dir and `--report-json` reports
|
||||
- LLM request/response/error diagnostics payloads.
|
||||
- Verify no API keys/bearer tokens leak into fixtures or outputs.
|
||||
|
||||
## Prompt and validator metadata checks
|
||||
|
||||
- Verify prompt metadata appears in LLM request metadata diagnostics:
|
||||
- `prompt_id`, `prompt_version`, `prompt_source`, `embedded_path`, `sha256`.
|
||||
- Verify stable validator keys appear in report decisions/rejections.
|
||||
- Verify built-in validator chains resolve and execute for default and explicit module runs.
|
||||
|
||||
## Utilization diagnostics checks
|
||||
|
||||
- Verify `utilization-diagnostics.json` exists on successful runs.
|
||||
- Verify partial utilization artifact behavior on controlled failure paths.
|
||||
- Verify utilization fields are structurally present and nonnegative:
|
||||
- effective concurrency
|
||||
- run timing
|
||||
- module timing summaries
|
||||
- per-validator timing summaries.
|
||||
|
||||
## Correction ledger checks
|
||||
|
||||
- Verify `correction-ledger.json` exists on successful runs.
|
||||
- Verify report references ledger artifact path.
|
||||
- Verify ledger dispositions include applied/rejected and skipped/failed where exercised.
|
||||
- Verify validator rejection and proposal-application skip remain distinct.
|
||||
|
||||
## Pipeline behavior checks
|
||||
|
||||
- Verify default full pipeline run remains:
|
||||
- `glossary`, `homophones`, `glossary`, `spoken_word`, `grammar`
|
||||
- with deterministic repeated instance naming (`glossary_1`, `glossary_2`).
|
||||
- Verify explicit module runs (`--modules`) still work.
|
||||
|
||||
## Failure and cancellation checks
|
||||
|
||||
- Verify controlled failure paths retain diagnostics and produce best-effort failure reports.
|
||||
- Verify timeout/cancellation paths exit nonzero, do not hang, and retain failure diagnostics when initialized.
|
||||
|
||||
## Release fixture/idempotence checks
|
||||
|
||||
- Run release fixtures (`internal/cli/testdata/release`) through `go test ./...`.
|
||||
- Confirm fixture checks cover:
|
||||
- must-apply and must-not-apply expectations
|
||||
- protected-term survival
|
||||
- report and diagnostics contracts
|
||||
- output-schema checks
|
||||
- prompt/schema metadata diagnostics
|
||||
- utilization/ledger artifacts
|
||||
- idempotence-oriented second pass no-op behavior with deterministic fake responses.
|
||||
|
||||
## Deferred-feature guardrail
|
||||
|
||||
- Confirm release docs do not claim support for deferred items:
|
||||
- filesystem prompt overrides
|
||||
- user-configurable validator chains
|
||||
- arbitrary user-supplied output schemas
|
||||
- resume/start-at/stop-after execution
|
||||
- diff/check/propose-only modes
|
||||
- generated transcript descriptions enabled by default
|
||||
- interactive review UI
|
||||
- UI/server wrapper
|
||||
- provider benchmarking harness.
|
||||
11
go.mod
Normal file
11
go.mod
Normal file
@@ -0,0 +1,11 @@
|
||||
module gitea.maximumdirect.net/eric/audita
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require gopkg.in/yaml.v3 v3.0.1
|
||||
|
||||
require (
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
)
|
||||
17
go.sum
Normal file
17
go.sum
Normal file
@@ -0,0 +1,17 @@
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
338
internal/cli/parity_test.go
Normal file
338
internal/cli/parity_test.go
Normal file
@@ -0,0 +1,338 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
)
|
||||
|
||||
type parityFixtureCase struct {
|
||||
Name string `json:"name"`
|
||||
TranscriptFile string `json:"transcript_file"`
|
||||
GlossaryFile string `json:"glossary_file"`
|
||||
ModulesCSV string `json:"modules_csv,omitempty"`
|
||||
ProposalResponsesFile string `json:"proposal_responses_file,omitempty"`
|
||||
ValidationResponsesFile string `json:"validation_responses_file,omitempty"`
|
||||
Env map[string]string `json:"env,omitempty"`
|
||||
Expect parityExpectation `json:"expect"`
|
||||
}
|
||||
|
||||
type parityExpectation struct {
|
||||
ExitCode int `json:"exit_code"`
|
||||
Status string `json:"status,omitempty"`
|
||||
ErrorPhase string `json:"error_phase,omitempty"`
|
||||
StdoutMode string `json:"stdout_mode,omitempty"` // empty|json
|
||||
StderrContains string `json:"stderr_contains,omitempty"`
|
||||
OutputTranscriptFile string `json:"output_transcript_file,omitempty"`
|
||||
ModuleInstances []string `json:"module_instances,omitempty"`
|
||||
ModuleCount int `json:"module_count,omitempty"`
|
||||
TotalAppliedChanges int `json:"total_applied_changes,omitempty"`
|
||||
TotalSkippedChanges int `json:"total_skipped_changes,omitempty"`
|
||||
FailedModuleInstance string `json:"failed_module_instance,omitempty"`
|
||||
ValidatorRejectedReasonCodes []string `json:"validator_rejected_reason_codes,omitempty"`
|
||||
ApplicationSkipReasonCodes []string `json:"application_skip_reason_codes,omitempty"`
|
||||
RequireErrorLog bool `json:"require_error_log,omitempty"`
|
||||
SecretMarkers []string `json:"secret_markers,omitempty"`
|
||||
ExpectedProposalCalls []string `json:"expected_proposal_calls,omitempty"`
|
||||
ExpectedValidationCalls []string `json:"expected_validation_calls,omitempty"`
|
||||
ModuleAppliedCounts []int `json:"module_applied_counts,omitempty"`
|
||||
ModuleRejectedCounts []int `json:"module_rejected_counts,omitempty"`
|
||||
ModuleSkipCounts []int `json:"module_skip_counts,omitempty"`
|
||||
MinResponsePayloadArtifacts int `json:"min_response_payload_artifacts,omitempty"`
|
||||
}
|
||||
|
||||
func TestParityFixtures(t *testing.T) {
|
||||
casePaths, err := filepath.Glob(parityFixturePath("*.case.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("glob parity fixtures: %v", err)
|
||||
}
|
||||
if len(casePaths) == 0 {
|
||||
t.Fatal("expected at least one parity fixture case")
|
||||
}
|
||||
|
||||
for _, casePath := range casePaths {
|
||||
fx := loadParityFixtureCase(t, casePath)
|
||||
t.Run(fx.Name, func(t *testing.T) {
|
||||
runParityFixtureCase(t, filepath.Dir(casePath), fx)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func loadParityFixtureCase(t *testing.T, casePath string) parityFixtureCase {
|
||||
t.Helper()
|
||||
var fx parityFixtureCase
|
||||
raw := readFile(t, casePath)
|
||||
if err := json.Unmarshal(raw, &fx); err != nil {
|
||||
t.Fatalf("parse parity case %q: %v", casePath, err)
|
||||
}
|
||||
if strings.TrimSpace(fx.Name) == "" {
|
||||
t.Fatalf("parity case %q missing name", casePath)
|
||||
}
|
||||
return fx
|
||||
}
|
||||
|
||||
func runParityFixtureCase(t *testing.T, caseDir string, fx parityFixtureCase) {
|
||||
t.Helper()
|
||||
for k, v := range fx.Env {
|
||||
t.Setenv(k, v)
|
||||
}
|
||||
|
||||
proposalClient := &fakeStructuredLLMClient{}
|
||||
validationClient := &fakeStructuredLLMClient{}
|
||||
|
||||
if strings.TrimSpace(fx.ProposalResponsesFile) != "" {
|
||||
raw := readFile(t, filepath.Join(caseDir, fx.ProposalResponsesFile))
|
||||
if err := json.Unmarshal(raw, &proposalClient.proposalResponses); err != nil {
|
||||
t.Fatalf("parse proposal responses: %v", err)
|
||||
}
|
||||
processProposalLLMClient = proposalClient
|
||||
}
|
||||
if strings.TrimSpace(fx.ValidationResponsesFile) != "" {
|
||||
raw := readFile(t, filepath.Join(caseDir, fx.ValidationResponsesFile))
|
||||
if err := json.Unmarshal(raw, &validationClient.validationResponses); err != nil {
|
||||
t.Fatalf("parse validation responses: %v", err)
|
||||
}
|
||||
processValidationLLMClient = validationClient
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
processProposalLLMClient = nil
|
||||
processValidationLLMClient = nil
|
||||
})
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
workDir := t.TempDir()
|
||||
outputPath := filepath.Join(t.TempDir(), "out.json")
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
args := []string{
|
||||
"process",
|
||||
filepath.Join(caseDir, fx.TranscriptFile),
|
||||
"--glossary",
|
||||
filepath.Join(caseDir, fx.GlossaryFile),
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
}
|
||||
// Keep stdout shape deterministic for parity tests.
|
||||
if fx.Expect.StdoutMode != "json" {
|
||||
args = append(args, "--output", outputPath)
|
||||
}
|
||||
if strings.TrimSpace(fx.ModulesCSV) != "" {
|
||||
args = append(args, "--modules", fx.ModulesCSV)
|
||||
}
|
||||
|
||||
exitCode := Run(args, &stdout, &stderr)
|
||||
if exitCode != fx.Expect.ExitCode {
|
||||
t.Fatalf("expected exit code %d, got %d stderr=%q", fx.Expect.ExitCode, exitCode, stderr.String())
|
||||
}
|
||||
|
||||
switch fx.Expect.StdoutMode {
|
||||
case "json":
|
||||
if _, err := json.Marshal(stdout.String()); err != nil {
|
||||
t.Fatalf("unexpected stdout marshal error: %v", err)
|
||||
}
|
||||
if !json.Valid(stdout.Bytes()) {
|
||||
t.Fatalf("expected JSON stdout, got %q", stdout.String())
|
||||
}
|
||||
default:
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("expected empty stdout, got %q", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
if fx.Expect.StderrContains != "" && !strings.Contains(stderr.String(), fx.Expect.StderrContains) {
|
||||
t.Fatalf("expected stderr to contain %q, got %q", fx.Expect.StderrContains, stderr.String())
|
||||
}
|
||||
|
||||
report := readProcessReport(t, reportPath)
|
||||
assertParityReport(t, report, fx.Expect)
|
||||
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
runDirReport := readProcessReport(t, filepath.Join(runDir, "report.json"))
|
||||
assertParityReport(t, runDirReport, fx.Expect)
|
||||
|
||||
if fx.Expect.RequireErrorLog {
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if strings.TrimSpace(fx.Expect.OutputTranscriptFile) != "" && fx.Expect.ExitCode == 0 {
|
||||
got := readFile(t, outputPath)
|
||||
want := readFile(t, filepath.Join(caseDir, fx.Expect.OutputTranscriptFile))
|
||||
assertJSONSemanticEqual(t, want, got)
|
||||
}
|
||||
|
||||
if len(fx.Expect.ExpectedProposalCalls) > 0 && !reflect.DeepEqual(proposalClient.calls, fx.Expect.ExpectedProposalCalls) {
|
||||
t.Fatalf("unexpected proposal calls: got %v want %v", proposalClient.calls, fx.Expect.ExpectedProposalCalls)
|
||||
}
|
||||
if len(fx.Expect.ExpectedValidationCalls) > 0 && !reflect.DeepEqual(validationClient.calls, fx.Expect.ExpectedValidationCalls) {
|
||||
t.Fatalf("unexpected validation calls: got %v want %v", validationClient.calls, fx.Expect.ExpectedValidationCalls)
|
||||
}
|
||||
|
||||
if len(fx.Expect.SecretMarkers) > 0 {
|
||||
assertNoSecretMarkers(t, reportPath, fx.Expect.SecretMarkers)
|
||||
assertNoSecretMarkersInTree(t, runDir, fx.Expect.SecretMarkers)
|
||||
}
|
||||
if fx.Expect.MinResponsePayloadArtifacts > 0 {
|
||||
matches, err := filepath.Glob(filepath.Join(runDir, "*", "*response-payload.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("glob response payload artifacts: %v", err)
|
||||
}
|
||||
if len(matches) < fx.Expect.MinResponsePayloadArtifacts {
|
||||
t.Fatalf("expected at least %d response payload artifacts, got %d", fx.Expect.MinResponsePayloadArtifacts, len(matches))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertParityReport(t *testing.T, report reporting.ProcessReport, exp parityExpectation) {
|
||||
t.Helper()
|
||||
if exp.Status != "" && report.Status != exp.Status {
|
||||
t.Fatalf("expected report status %q, got %q", exp.Status, report.Status)
|
||||
}
|
||||
if exp.ErrorPhase != "" && report.ErrorPhase != exp.ErrorPhase {
|
||||
t.Fatalf("expected report error_phase %q, got %q", exp.ErrorPhase, report.ErrorPhase)
|
||||
}
|
||||
if len(exp.ModuleInstances) > 0 {
|
||||
got := make([]string, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, mr.ModuleInstance)
|
||||
}
|
||||
if !reflect.DeepEqual(got, exp.ModuleInstances) {
|
||||
t.Fatalf("unexpected module instances: got %v want %v", got, exp.ModuleInstances)
|
||||
}
|
||||
}
|
||||
if exp.ModuleCount > 0 {
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.ModuleCount != exp.ModuleCount {
|
||||
t.Fatalf("expected module_count=%d, got %+v", exp.ModuleCount, report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
if exp.TotalAppliedChanges > 0 {
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.TotalAppliedChanges != exp.TotalAppliedChanges {
|
||||
t.Fatalf("expected total_applied_changes=%d, got %+v", exp.TotalAppliedChanges, report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
if exp.TotalSkippedChanges > 0 {
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.TotalSkippedChanges != exp.TotalSkippedChanges {
|
||||
t.Fatalf("expected total_skipped_changes=%d, got %+v", exp.TotalSkippedChanges, report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
if exp.FailedModuleInstance != "" {
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != exp.FailedModuleInstance {
|
||||
t.Fatalf("expected failed_module_instance=%q, got %+v", exp.FailedModuleInstance, report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
|
||||
if len(exp.ValidatorRejectedReasonCodes) > 0 {
|
||||
got := collectValidatorRejectedReasonCodes(report.ModuleResults)
|
||||
if !reflect.DeepEqual(got, exp.ValidatorRejectedReasonCodes) {
|
||||
t.Fatalf("unexpected validator rejected reason codes: got %v want %v", got, exp.ValidatorRejectedReasonCodes)
|
||||
}
|
||||
}
|
||||
if len(exp.ApplicationSkipReasonCodes) > 0 {
|
||||
got := collectApplicationSkipReasonCodes(report.ModuleResults)
|
||||
if !reflect.DeepEqual(got, exp.ApplicationSkipReasonCodes) {
|
||||
t.Fatalf("unexpected application skip reason codes: got %v want %v", got, exp.ApplicationSkipReasonCodes)
|
||||
}
|
||||
}
|
||||
if len(exp.ModuleAppliedCounts) > 0 {
|
||||
got := make([]int, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, len(mr.AppliedChanges))
|
||||
}
|
||||
if !reflect.DeepEqual(got, exp.ModuleAppliedCounts) {
|
||||
t.Fatalf("unexpected per-module applied counts: got %v want %v", got, exp.ModuleAppliedCounts)
|
||||
}
|
||||
}
|
||||
if len(exp.ModuleRejectedCounts) > 0 {
|
||||
got := make([]int, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, len(mr.ValidatorRejected))
|
||||
}
|
||||
if !reflect.DeepEqual(got, exp.ModuleRejectedCounts) {
|
||||
t.Fatalf("unexpected per-module rejected counts: got %v want %v", got, exp.ModuleRejectedCounts)
|
||||
}
|
||||
}
|
||||
if len(exp.ModuleSkipCounts) > 0 {
|
||||
got := make([]int, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, len(mr.SkippedChanges))
|
||||
}
|
||||
if !reflect.DeepEqual(got, exp.ModuleSkipCounts) {
|
||||
t.Fatalf("unexpected per-module skip counts: got %v want %v", got, exp.ModuleSkipCounts)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func collectValidatorRejectedReasonCodes(results []reporting.ModuleReport) []string {
|
||||
out := make([]string, 0)
|
||||
for _, mr := range results {
|
||||
for _, vr := range mr.ValidatorRejected {
|
||||
out = append(out, vr.ReasonCode)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func collectApplicationSkipReasonCodes(results []reporting.ModuleReport) []string {
|
||||
out := make([]string, 0)
|
||||
for _, mr := range results {
|
||||
for _, sk := range mr.SkippedChanges {
|
||||
out = append(out, string(sk.SkipReason))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func assertJSONSemanticEqual(t *testing.T, expected []byte, actual []byte) {
|
||||
t.Helper()
|
||||
var exp any
|
||||
var act any
|
||||
if err := json.Unmarshal(expected, &exp); err != nil {
|
||||
t.Fatalf("unmarshal expected json: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(actual, &act); err != nil {
|
||||
t.Fatalf("unmarshal actual json: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(exp, act) {
|
||||
t.Fatalf("JSON mismatch\nexpected=%s\nactual=%s", string(expected), string(actual))
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoSecretMarkers(t *testing.T, filePath string, markers []string) {
|
||||
t.Helper()
|
||||
raw := string(readFile(t, filePath))
|
||||
for _, marker := range markers {
|
||||
if marker != "" && strings.Contains(raw, marker) {
|
||||
t.Fatalf("secret marker %q leaked in %s", marker, filePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoSecretMarkersInTree(t *testing.T, root string, markers []string) {
|
||||
t.Helper()
|
||||
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil || d == nil || d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
raw := string(readFile(t, path))
|
||||
for _, marker := range markers {
|
||||
if marker != "" && strings.Contains(raw, marker) {
|
||||
t.Fatalf("secret marker %q leaked in %s", marker, path)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func parityFixturePath(name string) string {
|
||||
return filepath.Join("testdata", "parity", name)
|
||||
}
|
||||
469
internal/cli/release_fixtures_test.go
Normal file
469
internal/cli/release_fixtures_test.go
Normal file
@@ -0,0 +1,469 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
)
|
||||
|
||||
type releaseFixtureExpectations struct {
|
||||
MustApplyTexts []string `json:"must_apply_texts"`
|
||||
MustNotApplyTexts []string `json:"must_not_apply_texts"`
|
||||
ProtectedTerms []string `json:"protected_terms"`
|
||||
ExpectedModuleInstance []string `json:"expected_module_instances"`
|
||||
MinimumCounts struct {
|
||||
Applied int `json:"applied"`
|
||||
Rejected int `json:"rejected"`
|
||||
Skipped int `json:"skipped"`
|
||||
} `json:"minimum_counts"`
|
||||
}
|
||||
|
||||
func TestReleaseFixtureDefaultPipelineReadiness(t *testing.T) {
|
||||
base := fixturePath(filepath.Join("release", "default-release"))
|
||||
|
||||
var expectations releaseFixtureExpectations
|
||||
if err := json.Unmarshal(readFile(t, base+".expectations.json"), &expectations); err != nil {
|
||||
t.Fatalf("unmarshal release expectations: %v", err)
|
||||
}
|
||||
|
||||
proposalResponses := readProposalResponses(t, base+".proposals.json")
|
||||
validationResponses := readValidationResponses(t, base+".validations.json")
|
||||
|
||||
// First pass: default full pipeline with deterministic fake LLM responses.
|
||||
first := runReleaseFixturePass(t, releaseRunConfig{
|
||||
transcriptPath: base + ".transcript.json",
|
||||
glossaryPath: base + ".glossary.yaml",
|
||||
outputSchema: "bare-segments",
|
||||
proposalResponses: proposalResponses,
|
||||
validationResponses: validationResponses,
|
||||
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
|
||||
reportSchemaName: reporting.DefaultProcessReportSchemaName,
|
||||
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
||||
expectedOutputSchema: "bare-segments",
|
||||
expectModuleInstances: expectations.ExpectedModuleInstance,
|
||||
expectUtilizationPaths: true,
|
||||
})
|
||||
|
||||
gotTranscript := mustReadTranscript(t, first.outputPath)
|
||||
expectFinalTranscriptContains(t, gotTranscript, expectations.MustApplyTexts)
|
||||
expectFinalTranscriptDoesNotContain(t, gotTranscript, expectations.MustNotApplyTexts)
|
||||
expectFinalTranscriptContains(t, gotTranscript, expectations.ProtectedTerms)
|
||||
|
||||
assertReleaseCounts(t, first.report, expectations)
|
||||
assertPromptAndSchemaMetadataPresent(t, first.runDir)
|
||||
assertReleaseLedgerShape(t, first.report)
|
||||
assertReleaseUtilizationShape(t, first.report)
|
||||
assertStableValidatorKeysPresent(t, first.report)
|
||||
assertStdoutStderrContract(t, first.stdout, first.stderr)
|
||||
assertNoSecretMarkersInTree(t, first.runDir, []string{"release-secret"})
|
||||
assertNoSecretMarkers(t, first.reportPath, []string{"release-secret"})
|
||||
|
||||
// Output schema check: audita-v1 object payload.
|
||||
auditaV1 := runReleaseFixturePass(t, releaseRunConfig{
|
||||
transcriptPath: base + ".transcript.json",
|
||||
glossaryPath: base + ".glossary.yaml",
|
||||
outputSchema: "audita-v1",
|
||||
proposalResponses: proposalResponses,
|
||||
validationResponses: validationResponses,
|
||||
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
|
||||
reportSchemaName: reporting.DefaultProcessReportSchemaName,
|
||||
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
||||
expectedOutputSchema: "audita-v1",
|
||||
expectModuleInstances: expectations.ExpectedModuleInstance,
|
||||
expectUtilizationPaths: true,
|
||||
})
|
||||
assertAuditaV1OutputShape(t, auditaV1.outputPath)
|
||||
|
||||
// Idempotence-oriented second pass:
|
||||
// run again on first output with deterministic no-op responses.
|
||||
noOpProposals := make([]proposal_generation.StructuredCorrectionSet, 5)
|
||||
for i := range noOpProposals {
|
||||
noOpProposals[i] = proposal_generation.StructuredCorrectionSet{Corrections: nil}
|
||||
}
|
||||
second := runReleaseFixturePass(t, releaseRunConfig{
|
||||
transcriptPath: first.outputPath,
|
||||
glossaryPath: base + ".glossary.yaml",
|
||||
outputSchema: "bare-segments",
|
||||
proposalResponses: noOpProposals,
|
||||
validationResponses: nil,
|
||||
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
|
||||
reportSchemaName: reporting.DefaultProcessReportSchemaName,
|
||||
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
||||
expectedOutputSchema: "bare-segments",
|
||||
expectModuleInstances: expectations.ExpectedModuleInstance,
|
||||
expectUtilizationPaths: true,
|
||||
})
|
||||
firstSegments := mustReadTranscript(t, first.outputPath)
|
||||
secondSegments := mustReadTranscript(t, second.outputPath)
|
||||
if !reflect.DeepEqual(firstSegments, secondSegments) {
|
||||
t.Fatalf("expected idempotent second pass transcript; first=%+v second=%+v", firstSegments, secondSegments)
|
||||
}
|
||||
if second.report.ModulesSummary == nil {
|
||||
t.Fatalf("expected modules summary on second pass")
|
||||
}
|
||||
if second.report.ModulesSummary.TotalAppliedChanges != 0 {
|
||||
t.Fatalf("expected no-op second pass (0 applied), got %+v", second.report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
|
||||
type releaseRunConfig struct {
|
||||
transcriptPath string
|
||||
glossaryPath string
|
||||
outputSchema string
|
||||
proposalResponses []proposal_generation.StructuredCorrectionSet
|
||||
validationResponses []validators.LLMValidationResponse
|
||||
expectedProposalCalls []string
|
||||
reportSchemaName string
|
||||
reportSchemaVersion string
|
||||
expectedOutputSchema string
|
||||
expectModuleInstances []string
|
||||
expectUtilizationPaths bool
|
||||
}
|
||||
|
||||
type releaseRunResult struct {
|
||||
stdout string
|
||||
stderr string
|
||||
outputPath string
|
||||
reportPath string
|
||||
report reporting.ProcessReport
|
||||
runDir string
|
||||
}
|
||||
|
||||
func runReleaseFixturePass(t *testing.T, cfg releaseRunConfig) releaseRunResult {
|
||||
t.Helper()
|
||||
|
||||
processProposalLLMClient = &fakeStructuredLLMClient{proposalResponses: append([]proposal_generation.StructuredCorrectionSet(nil), cfg.proposalResponses...)}
|
||||
processValidationLLMClient = &fakeStructuredLLMClient{validationResponses: append([]validators.LLMValidationResponse(nil), cfg.validationResponses...)}
|
||||
t.Cleanup(func() {
|
||||
processProposalLLMClient = nil
|
||||
processValidationLLMClient = nil
|
||||
})
|
||||
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
outputPath := filepath.Join(t.TempDir(), "out.json")
|
||||
configPath := writeFile(t, "release-config.yml", "version: 1\n")
|
||||
|
||||
args := []string{
|
||||
"process",
|
||||
cfg.transcriptPath,
|
||||
"--glossary",
|
||||
cfg.glossaryPath,
|
||||
"--config",
|
||||
configPath,
|
||||
"--output",
|
||||
outputPath,
|
||||
"--output-schema",
|
||||
cfg.outputSchema,
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
}
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
exitCode := Run(args, &stdout, &stderr)
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||
}
|
||||
|
||||
report := readProcessReport(t, reportPath)
|
||||
if report.ReportMetadata.ReportSchemaName != cfg.reportSchemaName || report.ReportMetadata.ReportSchemaVersion != cfg.reportSchemaVersion {
|
||||
t.Fatalf("unexpected report schema metadata: %+v", report.ReportMetadata)
|
||||
}
|
||||
if report.ReportMetadata.OutputSchema != cfg.expectedOutputSchema {
|
||||
t.Fatalf("unexpected output schema metadata: got %q want %q", report.ReportMetadata.OutputSchema, cfg.expectedOutputSchema)
|
||||
}
|
||||
if len(cfg.expectModuleInstances) > 0 {
|
||||
got := make([]string, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, mr.ModuleInstance)
|
||||
}
|
||||
if !reflect.DeepEqual(got, cfg.expectModuleInstances) {
|
||||
t.Fatalf("unexpected module instances: got %v want %v", got, cfg.expectModuleInstances)
|
||||
}
|
||||
}
|
||||
if report.Diagnostics == nil {
|
||||
t.Fatalf("expected diagnostics metadata")
|
||||
}
|
||||
if cfg.expectUtilizationPaths {
|
||||
if report.Diagnostics.UtilizationSummaryPath == "" || report.Diagnostics.CorrectionLedgerPath == "" {
|
||||
t.Fatalf("expected utilization/ledger artifact paths in report diagnostics: %+v", report.Diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil {
|
||||
t.Fatalf("expected run-dir report: %v", err)
|
||||
}
|
||||
|
||||
if c, ok := processProposalLLMClient.(*fakeStructuredLLMClient); ok {
|
||||
if !reflect.DeepEqual(c.calls, cfg.expectedProposalCalls) {
|
||||
t.Fatalf("unexpected proposal call order: got %v want %v", c.calls, cfg.expectedProposalCalls)
|
||||
}
|
||||
}
|
||||
|
||||
return releaseRunResult{
|
||||
stdout: stdout.String(),
|
||||
stderr: stderr.String(),
|
||||
outputPath: outputPath,
|
||||
reportPath: reportPath,
|
||||
report: report,
|
||||
runDir: runDir,
|
||||
}
|
||||
}
|
||||
|
||||
func readProposalResponses(t *testing.T, path string) []proposal_generation.StructuredCorrectionSet {
|
||||
t.Helper()
|
||||
var out []proposal_generation.StructuredCorrectionSet
|
||||
if err := json.Unmarshal(readFile(t, path), &out); err != nil {
|
||||
t.Fatalf("unmarshal proposal responses: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func readValidationResponses(t *testing.T, path string) []validators.LLMValidationResponse {
|
||||
t.Helper()
|
||||
var out []validators.LLMValidationResponse
|
||||
if err := json.Unmarshal(readFile(t, path), &out); err != nil {
|
||||
t.Fatalf("unmarshal validation responses: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mustReadTranscript(t *testing.T, path string) []schema.Segment {
|
||||
t.Helper()
|
||||
transcript, err := schema.ParseTranscriptJSON(readFile(t, path))
|
||||
if err != nil {
|
||||
t.Fatalf("parse transcript output: %v", err)
|
||||
}
|
||||
return transcript.Segments
|
||||
}
|
||||
|
||||
func expectFinalTranscriptContains(t *testing.T, segments []schema.Segment, needles []string) {
|
||||
t.Helper()
|
||||
joined := flattenTranscriptText(segments)
|
||||
for _, needle := range needles {
|
||||
if !strings.Contains(joined, needle) {
|
||||
t.Fatalf("expected transcript to contain %q, got %q", needle, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func expectFinalTranscriptDoesNotContain(t *testing.T, segments []schema.Segment, needles []string) {
|
||||
t.Helper()
|
||||
joined := flattenTranscriptText(segments)
|
||||
for _, needle := range needles {
|
||||
if strings.Contains(joined, needle) {
|
||||
t.Fatalf("expected transcript to not contain %q, got %q", needle, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func flattenTranscriptText(segments []schema.Segment) string {
|
||||
parts := make([]string, 0, len(segments))
|
||||
for _, s := range segments {
|
||||
parts = append(parts, s.Text)
|
||||
}
|
||||
return strings.Join(parts, "\n")
|
||||
}
|
||||
|
||||
func assertReleaseCounts(t *testing.T, report reporting.ProcessReport, exp releaseFixtureExpectations) {
|
||||
t.Helper()
|
||||
if report.ModulesSummary == nil {
|
||||
t.Fatalf("expected modules_summary")
|
||||
}
|
||||
if report.ModulesSummary.TotalAppliedChanges < exp.MinimumCounts.Applied {
|
||||
t.Fatalf("expected at least %d applied changes, got %+v", exp.MinimumCounts.Applied, report.ModulesSummary)
|
||||
}
|
||||
validatorRejected := 0
|
||||
skipped := 0
|
||||
for _, mr := range report.ModuleResults {
|
||||
validatorRejected += len(mr.ValidatorRejected)
|
||||
skipped += len(mr.SkippedChanges)
|
||||
}
|
||||
if validatorRejected < exp.MinimumCounts.Rejected {
|
||||
t.Fatalf("expected at least %d validator rejections, got %d", exp.MinimumCounts.Rejected, validatorRejected)
|
||||
}
|
||||
if skipped < exp.MinimumCounts.Skipped {
|
||||
t.Fatalf("expected at least %d application skips, got %d", exp.MinimumCounts.Skipped, skipped)
|
||||
}
|
||||
}
|
||||
|
||||
func assertReleaseUtilizationShape(t *testing.T, report reporting.ProcessReport) {
|
||||
t.Helper()
|
||||
var payload struct {
|
||||
EffectiveConcurrency struct {
|
||||
TotalLLM int `json:"total_llm"`
|
||||
} `json:"effective_concurrency"`
|
||||
RunTiming struct {
|
||||
SchedulerQueueWaitMS int64 `json:"scheduler_queue_wait_ms"`
|
||||
LLMExecutionTimeMS int64 `json:"llm_execution_time_ms"`
|
||||
DeterministicValidationMS int64 `json:"deterministic_validation_time_ms"`
|
||||
} `json:"run_timing"`
|
||||
Modules []map[string]any `json:"modules"`
|
||||
Validators []map[string]any `json:"validators"`
|
||||
}
|
||||
if err := json.Unmarshal(readFile(t, report.Diagnostics.UtilizationSummaryPath), &payload); err != nil {
|
||||
t.Fatalf("unmarshal utilization diagnostics: %v", err)
|
||||
}
|
||||
if payload.EffectiveConcurrency.TotalLLM <= 0 {
|
||||
t.Fatalf("expected positive total llm concurrency, got %+v", payload.EffectiveConcurrency)
|
||||
}
|
||||
if payload.RunTiming.SchedulerQueueWaitMS < 0 || payload.RunTiming.LLMExecutionTimeMS < 0 || payload.RunTiming.DeterministicValidationMS < 0 {
|
||||
t.Fatalf("expected non-negative run timing values, got %+v", payload.RunTiming)
|
||||
}
|
||||
if len(payload.Modules) == 0 {
|
||||
t.Fatalf("expected module timing summaries")
|
||||
}
|
||||
if len(payload.Validators) == 0 {
|
||||
t.Fatalf("expected validator timing summaries")
|
||||
}
|
||||
}
|
||||
|
||||
func assertReleaseLedgerShape(t *testing.T, report reporting.ProcessReport) {
|
||||
t.Helper()
|
||||
var entries []struct {
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ProposalIndex int `json:"proposal_index"`
|
||||
Disposition string `json:"disposition"`
|
||||
DispositionReason string `json:"disposition_reason_code"`
|
||||
OriginalText string `json:"original_text"`
|
||||
ProposedCorrected string `json:"proposed_corrected_text"`
|
||||
ReplacementPolicy string `json:"replacement_policy"`
|
||||
DeterministicResults []struct {
|
||||
ValidatorKey string `json:"validator_key"`
|
||||
} `json:"deterministic_validator_decisions"`
|
||||
LLMResults []struct {
|
||||
ValidatorKey string `json:"validator_key"`
|
||||
} `json:"llm_validator_decisions"`
|
||||
}
|
||||
if err := json.Unmarshal(readFile(t, report.Diagnostics.CorrectionLedgerPath), &entries); err != nil {
|
||||
t.Fatalf("unmarshal correction ledger: %v", err)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
t.Fatalf("expected correction ledger entries")
|
||||
}
|
||||
hasApplied := false
|
||||
hasRejected := false
|
||||
hasSkipped := false
|
||||
for _, entry := range entries {
|
||||
if entry.ModuleInstance == "" || entry.ModuleKey == "" {
|
||||
t.Fatalf("expected module identity in ledger entry: %+v", entry)
|
||||
}
|
||||
switch entry.Disposition {
|
||||
case "applied":
|
||||
hasApplied = true
|
||||
case "rejected":
|
||||
hasRejected = true
|
||||
case "skipped":
|
||||
hasSkipped = true
|
||||
}
|
||||
}
|
||||
if !hasApplied || !hasRejected {
|
||||
t.Fatalf("expected applied and rejected entries in correction ledger, got %+v", entries)
|
||||
}
|
||||
if !hasSkipped {
|
||||
// Some deterministic fixture paths do not trigger apply-time skips;
|
||||
// rejections are still captured separately from application skips.
|
||||
}
|
||||
}
|
||||
|
||||
func assertPromptAndSchemaMetadataPresent(t *testing.T, runDir string) {
|
||||
t.Helper()
|
||||
metadataPaths, err := filepath.Glob(filepath.Join(runDir, "*", "*request-metadata.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("glob request metadata artifacts: %v", err)
|
||||
}
|
||||
if len(metadataPaths) == 0 {
|
||||
t.Fatalf("expected request metadata artifacts with prompt metadata")
|
||||
}
|
||||
|
||||
foundPromptMetadata := false
|
||||
foundSchemaMetadata := false
|
||||
for _, path := range metadataPaths {
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(readFile(t, path), &payload); err != nil {
|
||||
t.Fatalf("unmarshal request metadata artifact %q: %v", path, err)
|
||||
}
|
||||
if pm, ok := payload["prompt_metadata"].(map[string]any); ok {
|
||||
if pm["prompt_id"] != nil && pm["prompt_version"] != nil && pm["sha256"] != nil {
|
||||
foundPromptMetadata = true
|
||||
}
|
||||
}
|
||||
if sm, ok := payload["response_schema"].(map[string]any); ok {
|
||||
if sm["id"] != nil && sm["version"] != nil && sm["name"] != nil && sm["sha256"] != nil {
|
||||
foundSchemaMetadata = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !foundPromptMetadata {
|
||||
t.Fatalf("expected prompt metadata in request metadata artifacts")
|
||||
}
|
||||
if !foundSchemaMetadata {
|
||||
t.Fatalf("expected structured response schema metadata in request metadata artifacts")
|
||||
}
|
||||
}
|
||||
|
||||
func assertStableValidatorKeysPresent(t *testing.T, report reporting.ProcessReport) {
|
||||
t.Helper()
|
||||
seen := map[string]bool{}
|
||||
for _, module := range report.ModuleResults {
|
||||
for _, decision := range module.ValidatorDecisions {
|
||||
seen[decision.ValidatorName] = true
|
||||
}
|
||||
for _, rejected := range module.ValidatorRejected {
|
||||
seen[rejected.ValidatorName] = true
|
||||
}
|
||||
}
|
||||
expectedAny := []string{
|
||||
"confidence_threshold",
|
||||
"original_text_presence",
|
||||
"no_effect",
|
||||
}
|
||||
for _, key := range expectedAny {
|
||||
if !seen[key] {
|
||||
t.Fatalf("expected stable validator key %q in report decisions/rejections; seen=%v", key, seen)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertStdoutStderrContract(t *testing.T, stdout, stderr string) {
|
||||
t.Helper()
|
||||
if stdout != "" {
|
||||
t.Fatalf("expected empty stdout with --output, got %q", stdout)
|
||||
}
|
||||
if strings.Contains(stderr, `"module_results"`) || strings.Contains(stderr, `"report_metadata"`) {
|
||||
t.Fatalf("stderr should remain human-readable, not report JSON: %q", stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func assertAuditaV1OutputShape(t *testing.T, outputPath string) {
|
||||
t.Helper()
|
||||
var payload struct {
|
||||
Schema string `json:"schema"`
|
||||
Version string `json:"version"`
|
||||
Segments []schema.Segment `json:"segments"`
|
||||
}
|
||||
if err := json.Unmarshal(readFile(t, outputPath), &payload); err != nil {
|
||||
t.Fatalf("unmarshal audita-v1 output: %v", err)
|
||||
}
|
||||
if payload.Schema != "audita-v1" || payload.Version != "v1" {
|
||||
t.Fatalf("unexpected audita-v1 metadata: %+v", payload)
|
||||
}
|
||||
if len(payload.Segments) == 0 {
|
||||
t.Fatalf("expected non-empty audita-v1 segments")
|
||||
}
|
||||
}
|
||||
155
internal/cli/review_artifacts.go
Normal file
155
internal/cli/review_artifacts.go
Normal file
@@ -0,0 +1,155 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"sort"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
)
|
||||
|
||||
const (
|
||||
correctionDispositionApplied = "applied"
|
||||
correctionDispositionRejected = "rejected"
|
||||
correctionDispositionSkipped = "skipped"
|
||||
correctionDispositionFailed = "failed"
|
||||
)
|
||||
|
||||
type correctionLedgerEntry struct {
|
||||
RunID string `json:"run_id,omitempty"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ProposalIndex int `json:"proposal_index"`
|
||||
SegmentID int `json:"segment_id,omitempty"`
|
||||
OriginalText string `json:"original_text,omitempty"`
|
||||
ProposedCorrectedText string `json:"proposed_corrected_text,omitempty"`
|
||||
AppliedCorrectedText string `json:"applied_corrected_text,omitempty"`
|
||||
ReplacementPolicy string `json:"replacement_policy,omitempty"`
|
||||
Disposition string `json:"disposition"`
|
||||
DispositionReasonCode string `json:"disposition_reason_code,omitempty"`
|
||||
DispositionMessage string `json:"disposition_message,omitempty"`
|
||||
DeterministicValidatorResults []ledgerValidatorDecisionRecord `json:"deterministic_validator_decisions,omitempty"`
|
||||
LLMValidatorResults []ledgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
|
||||
}
|
||||
|
||||
type ledgerValidatorDecisionRecord struct {
|
||||
ValidatorKey string `json:"validator_key"`
|
||||
Approved bool `json:"approved"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []correctionLedgerEntry {
|
||||
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
||||
return nil
|
||||
}
|
||||
runID := ""
|
||||
if runDirPath != "" {
|
||||
runID = filepath.Base(runDirPath)
|
||||
}
|
||||
|
||||
entries := make([]correctionLedgerEntry, 0)
|
||||
llmBacked := map[string]bool{
|
||||
"spoken_form_plausibility": true,
|
||||
"meaning_reversal_review": true,
|
||||
"editorial_review": true,
|
||||
}
|
||||
|
||||
for _, module := range runOutput.ModuleResults {
|
||||
decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord)
|
||||
for _, decision := range module.ValidatorDecisions {
|
||||
decisionsByProposal[decision.ProposalIndex] = append(decisionsByProposal[decision.ProposalIndex], decision)
|
||||
}
|
||||
|
||||
for _, change := range module.AppliedChanges {
|
||||
entries = append(entries, correctionLedgerEntry{
|
||||
RunID: runID,
|
||||
ModuleKey: module.ModuleKey,
|
||||
ModuleInstance: module.ModuleInstance,
|
||||
ProposalIndex: change.ProposalIndex,
|
||||
SegmentID: change.TargetSegmentID,
|
||||
OriginalText: change.OriginalText,
|
||||
ProposedCorrectedText: change.CorrectedText,
|
||||
AppliedCorrectedText: change.CorrectedText,
|
||||
ReplacementPolicy: string(module.ReplacementPolicy),
|
||||
Disposition: correctionDispositionApplied,
|
||||
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false, llmBacked),
|
||||
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true, llmBacked),
|
||||
})
|
||||
}
|
||||
for _, change := range module.SkippedChanges {
|
||||
entries = append(entries, correctionLedgerEntry{
|
||||
RunID: runID,
|
||||
ModuleKey: module.ModuleKey,
|
||||
ModuleInstance: module.ModuleInstance,
|
||||
ProposalIndex: change.ProposalIndex,
|
||||
SegmentID: change.TargetSegmentID,
|
||||
OriginalText: change.OriginalText,
|
||||
ProposedCorrectedText: change.CorrectedText,
|
||||
ReplacementPolicy: string(module.ReplacementPolicy),
|
||||
Disposition: correctionDispositionSkipped,
|
||||
DispositionReasonCode: string(change.SkipReason),
|
||||
DispositionMessage: change.Message,
|
||||
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false, llmBacked),
|
||||
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true, llmBacked),
|
||||
})
|
||||
}
|
||||
for _, rejection := range module.ValidatorRejected {
|
||||
entries = append(entries, correctionLedgerEntry{
|
||||
RunID: runID,
|
||||
ModuleKey: module.ModuleKey,
|
||||
ModuleInstance: module.ModuleInstance,
|
||||
ProposalIndex: rejection.ProposalIndex,
|
||||
SegmentID: rejection.TargetSegmentID,
|
||||
OriginalText: rejection.OriginalText,
|
||||
ProposedCorrectedText: rejection.CorrectedText,
|
||||
ReplacementPolicy: string(module.ReplacementPolicy),
|
||||
Disposition: correctionDispositionRejected,
|
||||
DispositionReasonCode: rejection.ReasonCode,
|
||||
DispositionMessage: rejection.Message,
|
||||
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], false, llmBacked),
|
||||
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], true, llmBacked),
|
||||
})
|
||||
}
|
||||
if module.Status == runner.ModuleStatusFailed {
|
||||
entries = append(entries, correctionLedgerEntry{
|
||||
RunID: runID,
|
||||
ModuleKey: module.ModuleKey,
|
||||
ModuleInstance: module.ModuleInstance,
|
||||
Disposition: correctionDispositionFailed,
|
||||
DispositionReasonCode: "module_failed",
|
||||
DispositionMessage: module.ErrorMessage,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
sort.SliceStable(entries, func(i, j int) bool {
|
||||
if entries[i].ModuleInstance != entries[j].ModuleInstance {
|
||||
return entries[i].ModuleInstance < entries[j].ModuleInstance
|
||||
}
|
||||
if entries[i].ProposalIndex != entries[j].ProposalIndex {
|
||||
return entries[i].ProposalIndex < entries[j].ProposalIndex
|
||||
}
|
||||
return entries[i].Disposition < entries[j].Disposition
|
||||
})
|
||||
|
||||
return entries
|
||||
}
|
||||
|
||||
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool, llmBacked map[string]bool) []ledgerValidatorDecisionRecord {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]ledgerValidatorDecisionRecord, 0, len(in))
|
||||
for _, decision := range in {
|
||||
if llmBacked[decision.ValidatorName] != wantLLM {
|
||||
continue
|
||||
}
|
||||
out = append(out, ledgerValidatorDecisionRecord{
|
||||
ValidatorKey: decision.ValidatorName,
|
||||
Approved: decision.Approved,
|
||||
ReasonCode: decision.ReasonCode,
|
||||
Message: decision.Message,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
1106
internal/cli/run.go
Normal file
1106
internal/cli/run.go
Normal file
File diff suppressed because it is too large
Load Diff
4359
internal/cli/run_test.go
Normal file
4359
internal/cli/run_test.go
Normal file
File diff suppressed because it is too large
Load Diff
113
internal/cli/subprocess_test_hooks.go
Normal file
113
internal/cli/subprocess_test_hooks.go
Normal file
@@ -0,0 +1,113 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
)
|
||||
|
||||
const (
|
||||
subprocessTestLLMModeEnv = "AUDITA_SUBPROCESS_TEST_LLM_MODE"
|
||||
subprocessTestRunTimeoutMSEnv = "AUDITA_SUBPROCESS_TEST_RUN_TIMEOUT_MS"
|
||||
)
|
||||
|
||||
// ConfigureSubprocessTestHooksFromEnv enables deterministic test-only hooks for
|
||||
// subprocess integration tests that run through the Go test binary helper path.
|
||||
func ConfigureSubprocessTestHooksFromEnv() {
|
||||
mode := strings.TrimSpace(os.Getenv(subprocessTestLLMModeEnv))
|
||||
timeoutMSRaw := strings.TrimSpace(os.Getenv(subprocessTestRunTimeoutMSEnv))
|
||||
// Only activate in explicit subprocess test mode.
|
||||
if mode == "" && timeoutMSRaw == "" {
|
||||
return
|
||||
}
|
||||
|
||||
if mode != "" {
|
||||
client := &subprocessTestLLMClient{mode: mode}
|
||||
processProposalLLMClient = client
|
||||
processValidationLLMClient = client
|
||||
}
|
||||
|
||||
if timeoutMSRaw == "" {
|
||||
return
|
||||
}
|
||||
timeoutMS, err := strconv.Atoi(timeoutMSRaw)
|
||||
if err != nil || timeoutMS <= 0 {
|
||||
return
|
||||
}
|
||||
processRunnerContext = func() (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.Background(), time.Duration(timeoutMS)*time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
type subprocessTestLLMClient struct {
|
||||
mode string
|
||||
mu sync.Mutex
|
||||
proposals int
|
||||
}
|
||||
|
||||
func (c *subprocessTestLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
_ = req
|
||||
|
||||
switch c.mode {
|
||||
case "backend_error":
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("synthetic backend failure")
|
||||
case "block_until_cancel":
|
||||
<-ctx.Done()
|
||||
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
||||
case "malformed_structured":
|
||||
switch target := out.(type) {
|
||||
case *proposal_generation.StructuredCorrectionSet:
|
||||
*target = proposal_generation.StructuredCorrectionSet{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y", Confidence: 0.99},
|
||||
},
|
||||
}
|
||||
case *validators.LLMValidationResponse:
|
||||
*target = validators.LLMValidationResponse{
|
||||
Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 999, Approved: true, Confidence: 0.9, Reason: "bad index"},
|
||||
},
|
||||
}
|
||||
}
|
||||
case "mid_pipeline_fail":
|
||||
switch target := out.(type) {
|
||||
case *proposal_generation.StructuredCorrectionSet:
|
||||
c.mu.Lock()
|
||||
c.proposals++
|
||||
proposalCall := c.proposals
|
||||
c.mu.Unlock()
|
||||
|
||||
if proposalCall >= 3 {
|
||||
*target = proposal_generation.StructuredCorrectionSet{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y", Confidence: 0.99},
|
||||
},
|
||||
}
|
||||
} else {
|
||||
*target = proposal_generation.StructuredCorrectionSet{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99},
|
||||
},
|
||||
}
|
||||
}
|
||||
case *validators.LLMValidationResponse:
|
||||
*target = validators.LLMValidationResponse{Validations: nil}
|
||||
}
|
||||
default:
|
||||
switch target := out.(type) {
|
||||
case *proposal_generation.StructuredCorrectionSet:
|
||||
*target = proposal_generation.StructuredCorrectionSet{Corrections: nil}
|
||||
case *validators.LLMValidationResponse:
|
||||
*target = validators.LLMValidationResponse{Validations: nil}
|
||||
}
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
}
|
||||
1
internal/cli/testdata/malformed_transcript.json
vendored
Normal file
1
internal/cli/testdata/malformed_transcript.json
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"segments":[{"id":1,"text":"oops"}
|
||||
22
internal/cli/testdata/parity/application-skip-ambiguous.case.json
vendored
Normal file
22
internal/cli/testdata/parity/application-skip-ambiguous.case.json
vendored
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"name": "ambiguous_match_rejected_before_application",
|
||||
"transcript_file": "application-skip-ambiguous.transcript.json",
|
||||
"glossary_file": "default-handoff.glossary.yaml",
|
||||
"modules_csv": "homophones",
|
||||
"proposal_responses_file": "application-skip-ambiguous.proposals.json",
|
||||
"validation_responses_file": "application-skip-ambiguous.validations.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "application-skip-ambiguous.expected-transcript.json",
|
||||
"module_instances": ["homophones"],
|
||||
"module_count": 1,
|
||||
"total_applied_changes": 0,
|
||||
"total_skipped_changes": 1,
|
||||
"module_applied_counts": [0],
|
||||
"module_rejected_counts": [1],
|
||||
"module_skip_counts": [0],
|
||||
"validator_rejected_reason_codes": ["ambiguous_original_text"],
|
||||
"expected_proposal_calls": ["homophones:proposal"]
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/application-skip-ambiguous.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/application-skip-ambiguous.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"the site near another site"}
|
||||
]
|
||||
3
internal/cli/testdata/parity/application-skip-ambiguous.proposals.json
vendored
Normal file
3
internal/cli/testdata/parity/application-skip-ambiguous.proposals.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]}
|
||||
]
|
||||
3
internal/cli/testdata/parity/application-skip-ambiguous.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/application-skip-ambiguous.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"the site near another site"}
|
||||
]
|
||||
4
internal/cli/testdata/parity/application-skip-ambiguous.validations.json
vendored
Normal file
4
internal/cli/testdata/parity/application-skip-ambiguous.validations.json
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
28
internal/cli/testdata/parity/default-full-pipeline.case.json
vendored
Normal file
28
internal/cli/testdata/parity/default-full-pipeline.case.json
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "default_full_pipeline_shape_and_reports",
|
||||
"transcript_file": "default-full-pipeline.transcript.json",
|
||||
"glossary_file": "default-full-pipeline.glossary.yaml",
|
||||
"proposal_responses_file": "default-full-pipeline.proposals.json",
|
||||
"validation_responses_file": "default-full-pipeline.validations.json",
|
||||
"env": {
|
||||
"AUDITA_LLM_API_KEY": "parity-secret",
|
||||
"AUDITA_VALIDATION_LLM_API_KEY": "parity-secret"
|
||||
},
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "default-full-pipeline.expected-transcript.json",
|
||||
"module_instances": ["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
|
||||
"module_count": 5,
|
||||
"total_applied_changes": 3,
|
||||
"total_skipped_changes": 3,
|
||||
"secret_markers": ["parity-secret"],
|
||||
"expected_proposal_calls": [
|
||||
"glossary_1:proposal",
|
||||
"homophones:proposal",
|
||||
"glossary_2:proposal",
|
||||
"spoken_word:proposal",
|
||||
"grammar:proposal"
|
||||
]
|
||||
}
|
||||
}
|
||||
9
internal/cli/testdata/parity/default-full-pipeline.expected-transcript.json
vendored
Normal file
9
internal/cli/testdata/parity/default-full-pipeline.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "Alice",
|
||||
"start": 0,
|
||||
"end": 1,
|
||||
"text": "Hello, there were Jesters hmm"
|
||||
}
|
||||
]
|
||||
7
internal/cli/testdata/parity/default-full-pipeline.glossary.yaml
vendored
Normal file
7
internal/cli/testdata/parity/default-full-pipeline.glossary.yaml
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
glossary:
|
||||
- name: Jesters
|
||||
aliases:
|
||||
- jester
|
||||
plural: jesters
|
||||
category: faction
|
||||
summary: A protected in-world faction term.
|
||||
28
internal/cli/testdata/parity/default-full-pipeline.proposals.json
vendored
Normal file
28
internal/cli/testdata/parity/default-full-pipeline.proposals.json
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
[
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "Jesters", "corrected_text": "jesters", "confidence": 0.99},
|
||||
{"id": 1, "original_text": "Jesters", "corrected_text": "JESTERX", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "jesters", "corrected_text": "JESTERS", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "uh", "corrected_text": "hmm", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
|
||||
]
|
||||
}
|
||||
]
|
||||
3
internal/cli/testdata/parity/default-full-pipeline.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/default-full-pipeline.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
|
||||
]
|
||||
12
internal/cli/testdata/parity/default-full-pipeline.validations.json
vendored
Normal file
12
internal/cli/testdata/parity/default-full-pipeline.validations.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject cleanup"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "parity-secret"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
27
internal/cli/testdata/parity/default-handoff.case.json
vendored
Normal file
27
internal/cli/testdata/parity/default-handoff.case.json
vendored
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"name": "default_pipeline_handoff_and_module_order",
|
||||
"transcript_file": "default-handoff.transcript.json",
|
||||
"glossary_file": "default-handoff.glossary.yaml",
|
||||
"proposal_responses_file": "default-handoff.proposals.json",
|
||||
"validation_responses_file": "default-handoff.validations.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "default-handoff.expected-transcript.json",
|
||||
"module_instances": ["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
|
||||
"module_count": 5,
|
||||
"total_applied_changes": 5,
|
||||
"total_skipped_changes": 0,
|
||||
"module_applied_counts": [1, 1, 1, 1, 1],
|
||||
"module_rejected_counts": [0, 0, 0, 0, 0],
|
||||
"module_skip_counts": [0, 0, 0, 0, 0],
|
||||
"expected_proposal_calls": [
|
||||
"glossary_1:proposal",
|
||||
"homophones:proposal",
|
||||
"glossary_2:proposal",
|
||||
"spoken_word:proposal",
|
||||
"grammar:proposal"
|
||||
],
|
||||
"min_response_payload_artifacts": 15
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/default-handoff.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/default-handoff.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"Hello, there were Jesters at the Sight um"}
|
||||
]
|
||||
6
internal/cli/testdata/parity/default-handoff.glossary.yaml
vendored
Normal file
6
internal/cli/testdata/parity/default-handoff.glossary.yaml
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
glossary:
|
||||
- name: Jesters
|
||||
aliases: [jester]
|
||||
plural: jesters
|
||||
category: faction
|
||||
summary: A protected in-world faction term.
|
||||
7
internal/cli/testdata/parity/default-handoff.proposals.json
vendored
Normal file
7
internal/cli/testdata/parity/default-handoff.proposals.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
[
|
||||
{"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "sight", "corrected_text": "Sight", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "um um", "corrected_text": "um", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}]}
|
||||
]
|
||||
3
internal/cli/testdata/parity/default-handoff.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/default-handoff.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures at the site um um"}
|
||||
]
|
||||
12
internal/cli/testdata/parity/default-handoff.validations.json
vendored
Normal file
12
internal/cli/testdata/parity/default-handoff.validations.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
17
internal/cli/testdata/parity/deterministic-validator-low-confidence.case.json
vendored
Normal file
17
internal/cli/testdata/parity/deterministic-validator-low-confidence.case.json
vendored
Normal file
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"name": "deterministic_validator_low_confidence",
|
||||
"transcript_file": "deterministic-validator-low-confidence.transcript.json",
|
||||
"glossary_file": "default-full-pipeline.glossary.yaml",
|
||||
"modules_csv": "grammar",
|
||||
"proposal_responses_file": "deterministic-validator-low-confidence.proposals.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "deterministic-validator-low-confidence.expected-transcript.json",
|
||||
"module_instances": ["grammar"],
|
||||
"module_count": 1,
|
||||
"total_skipped_changes": 1,
|
||||
"validator_rejected_reason_codes": ["low_confidence"],
|
||||
"expected_proposal_calls": ["grammar:proposal"]
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/deterministic-validator-low-confidence.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/deterministic-validator-low-confidence.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"hello , world"}
|
||||
]
|
||||
7
internal/cli/testdata/parity/deterministic-validator-low-confidence.proposals.json
vendored
Normal file
7
internal/cli/testdata/parity/deterministic-validator-low-confidence.proposals.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
[
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.1}
|
||||
]
|
||||
}
|
||||
]
|
||||
3
internal/cli/testdata/parity/deterministic-validator-low-confidence.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/deterministic-validator-low-confidence.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , world"}
|
||||
]
|
||||
12
internal/cli/testdata/parity/glossary-schema-error.case.json
vendored
Normal file
12
internal/cli/testdata/parity/glossary-schema-error.case.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"name": "glossary_schema_handling",
|
||||
"transcript_file": "default-full-pipeline.transcript.json",
|
||||
"glossary_file": "glossary-schema-error.glossary.yaml",
|
||||
"expect": {
|
||||
"exit_code": 1,
|
||||
"status": "failed",
|
||||
"error_phase": "glossary_schema",
|
||||
"stderr_contains": "glossary_schema",
|
||||
"require_error_log": true
|
||||
}
|
||||
}
|
||||
2
internal/cli/testdata/parity/glossary-schema-error.glossary.yaml
vendored
Normal file
2
internal/cli/testdata/parity/glossary-schema-error.glossary.yaml
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
glossary:
|
||||
- name: MissingCategoryAndSummary
|
||||
19
internal/cli/testdata/parity/llm-validator-rejection.case.json
vendored
Normal file
19
internal/cli/testdata/parity/llm-validator-rejection.case.json
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "llm_validator_decision_handling",
|
||||
"transcript_file": "llm-validator-rejection.transcript.json",
|
||||
"glossary_file": "default-full-pipeline.glossary.yaml",
|
||||
"modules_csv": "grammar",
|
||||
"proposal_responses_file": "llm-validator-rejection.proposals.json",
|
||||
"validation_responses_file": "llm-validator-rejection.validations.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "llm-validator-rejection.expected-transcript.json",
|
||||
"module_instances": ["grammar"],
|
||||
"module_count": 1,
|
||||
"total_skipped_changes": 1,
|
||||
"validator_rejected_reason_codes": ["llm_rejected"],
|
||||
"expected_proposal_calls": ["grammar:proposal"],
|
||||
"expected_validation_calls": ["grammar:section-0000:editorial_review:batch-0000"]
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/llm-validator-rejection.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/llm-validator-rejection.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"hello , world"}
|
||||
]
|
||||
7
internal/cli/testdata/parity/llm-validator-rejection.proposals.json
vendored
Normal file
7
internal/cli/testdata/parity/llm-validator-rejection.proposals.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
[
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
|
||||
]
|
||||
}
|
||||
]
|
||||
3
internal/cli/testdata/parity/llm-validator-rejection.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/llm-validator-rejection.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , world"}
|
||||
]
|
||||
7
internal/cli/testdata/parity/llm-validator-rejection.validations.json
vendored
Normal file
7
internal/cli/testdata/parity/llm-validator-rejection.validations.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
[
|
||||
{
|
||||
"validations": [
|
||||
{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject stylistic overreach"}
|
||||
]
|
||||
}
|
||||
]
|
||||
23
internal/cli/testdata/parity/mid-pipeline-failure.case.json
vendored
Normal file
23
internal/cli/testdata/parity/mid-pipeline-failure.case.json
vendored
Normal file
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "mid_pipeline_failure_partial_progress",
|
||||
"transcript_file": "default-handoff.transcript.json",
|
||||
"glossary_file": "default-handoff.glossary.yaml",
|
||||
"proposal_responses_file": "mid-pipeline-failure.proposals.json",
|
||||
"validation_responses_file": "mid-pipeline-failure.validations.json",
|
||||
"expect": {
|
||||
"exit_code": 1,
|
||||
"status": "failed",
|
||||
"error_phase": "runner_execution",
|
||||
"stderr_contains": "runner_execution",
|
||||
"module_instances": ["glossary_1", "homophones", "glossary_2"],
|
||||
"module_count": 3,
|
||||
"total_applied_changes": 2,
|
||||
"total_skipped_changes": 0,
|
||||
"failed_module_instance": "glossary_2",
|
||||
"module_applied_counts": [1, 1, 0],
|
||||
"module_rejected_counts": [0, 0, 0],
|
||||
"module_skip_counts": [0, 0, 0],
|
||||
"require_error_log": true,
|
||||
"expected_proposal_calls": ["glossary_1:proposal", "homophones:proposal", "glossary_2:proposal"]
|
||||
}
|
||||
}
|
||||
4
internal/cli/testdata/parity/mid-pipeline-failure.proposals.json
vendored
Normal file
4
internal/cli/testdata/parity/mid-pipeline-failure.proposals.json
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
[
|
||||
{"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]}
|
||||
]
|
||||
6
internal/cli/testdata/parity/mid-pipeline-failure.validations.json
vendored
Normal file
6
internal/cli/testdata/parity/mid-pipeline-failure.validations.json
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
21
internal/cli/testdata/parity/protected-term-rejection.case.json
vendored
Normal file
21
internal/cli/testdata/parity/protected-term-rejection.case.json
vendored
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "protected_glossary_term_behavior",
|
||||
"transcript_file": "protected-term-rejection.transcript.json",
|
||||
"glossary_file": "default-handoff.glossary.yaml",
|
||||
"modules_csv": "homophones",
|
||||
"proposal_responses_file": "protected-term-rejection.proposals.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "protected-term-rejection.expected-transcript.json",
|
||||
"module_instances": ["homophones"],
|
||||
"module_count": 1,
|
||||
"total_applied_changes": 0,
|
||||
"total_skipped_changes": 1,
|
||||
"module_applied_counts": [0],
|
||||
"module_rejected_counts": [1],
|
||||
"module_skip_counts": [0],
|
||||
"validator_rejected_reason_codes": ["protected_glossary_term"],
|
||||
"expected_proposal_calls": ["homophones:proposal"]
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/protected-term-rejection.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/protected-term-rejection.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"The Jesters entered the hall."}
|
||||
]
|
||||
3
internal/cli/testdata/parity/protected-term-rejection.proposals.json
vendored
Normal file
3
internal/cli/testdata/parity/protected-term-rejection.proposals.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"corrections": [{"id": 1, "original_text": "Jesters", "corrected_text": "Gestures", "confidence": 0.99}]}
|
||||
]
|
||||
3
internal/cli/testdata/parity/protected-term-rejection.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/protected-term-rejection.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"The Jesters entered the hall."}
|
||||
]
|
||||
12
internal/cli/testdata/parity/transcript-schema-error.case.json
vendored
Normal file
12
internal/cli/testdata/parity/transcript-schema-error.case.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"name": "transcript_schema_handling",
|
||||
"transcript_file": "transcript-schema-error.transcript.json",
|
||||
"glossary_file": "default-full-pipeline.glossary.yaml",
|
||||
"expect": {
|
||||
"exit_code": 1,
|
||||
"status": "failed",
|
||||
"error_phase": "transcript_schema",
|
||||
"stderr_contains": "transcript_schema",
|
||||
"require_error_log": true
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/transcript-schema-error.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/transcript-schema-error.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"","start":0.0,"end":1.0,"text":"bad"}
|
||||
]
|
||||
24
internal/cli/testdata/release/default-release.expectations.json
vendored
Normal file
24
internal/cli/testdata/release/default-release.expectations.json
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"must_apply_texts": [
|
||||
"Hello, there were Jesters hmm"
|
||||
],
|
||||
"must_not_apply_texts": [
|
||||
"JESTERX",
|
||||
"there were gestures"
|
||||
],
|
||||
"protected_terms": [
|
||||
"Jesters"
|
||||
],
|
||||
"expected_module_instances": [
|
||||
"glossary_1",
|
||||
"homophones",
|
||||
"glossary_2",
|
||||
"spoken_word",
|
||||
"grammar"
|
||||
],
|
||||
"minimum_counts": {
|
||||
"applied": 1,
|
||||
"rejected": 1,
|
||||
"skipped": 0
|
||||
}
|
||||
}
|
||||
9
internal/cli/testdata/release/default-release.expected-transcript.json
vendored
Normal file
9
internal/cli/testdata/release/default-release.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "Alice",
|
||||
"start": 0,
|
||||
"end": 1,
|
||||
"text": "Hello, there were Jesters hmm"
|
||||
}
|
||||
]
|
||||
7
internal/cli/testdata/release/default-release.glossary.yaml
vendored
Normal file
7
internal/cli/testdata/release/default-release.glossary.yaml
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
glossary:
|
||||
- name: Jesters
|
||||
aliases:
|
||||
- jester
|
||||
plural: jesters
|
||||
category: faction
|
||||
summary: A protected in-world faction term.
|
||||
28
internal/cli/testdata/release/default-release.proposals.json
vendored
Normal file
28
internal/cli/testdata/release/default-release.proposals.json
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
[
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "Jesters", "corrected_text": "jesters", "confidence": 0.99},
|
||||
{"id": 1, "original_text": "Jesters", "corrected_text": "JESTERX", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "jesters", "corrected_text": "JESTERS", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "uh", "corrected_text": "hmm", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
|
||||
]
|
||||
}
|
||||
]
|
||||
3
internal/cli/testdata/release/default-release.transcript.json
vendored
Normal file
3
internal/cli/testdata/release/default-release.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
|
||||
]
|
||||
12
internal/cli/testdata/release/default-release.validations.json
vendored
Normal file
12
internal/cli/testdata/release/default-release.validations.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject cleanup"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "release-secret"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
6
internal/cli/testdata/tiny_glossary.yaml
vendored
Normal file
6
internal/cli/testdata/tiny_glossary.yaml
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
glossary:
|
||||
- name: Audita
|
||||
aliases:
|
||||
- audita
|
||||
category: product
|
||||
summary: The Audita transcript correction CLI.
|
||||
9
internal/cli/testdata/tiny_transcript.json
vendored
Normal file
9
internal/cli/testdata/tiny_transcript.json
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "A",
|
||||
"start": 0.0,
|
||||
"end": 1.2,
|
||||
"text": "hello world"
|
||||
}
|
||||
]
|
||||
312
internal/core/chunking/sections.go
Normal file
312
internal/core/chunking/sections.go
Normal file
@@ -0,0 +1,312 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
// Section represents a contiguous chunk of transcript segments with metadata.
|
||||
type Section struct {
|
||||
// Index is the 0-based section index within the chunked transcript
|
||||
Index int `json:"section_index"`
|
||||
|
||||
// StartSegmentID is the ID of the first segment in this section
|
||||
StartSegmentID int `json:"start_segment_id"`
|
||||
|
||||
// EndSegmentID is the ID of the last segment in this section
|
||||
EndSegmentID int `json:"end_segment_id"`
|
||||
|
||||
// EstimatedTokens is the approximate token count for this section
|
||||
EstimatedTokens int `json:"estimated_tokens"`
|
||||
|
||||
// Segments contains the segments in this section, in order
|
||||
Segments []schema.Segment `json:"segments"`
|
||||
}
|
||||
|
||||
// ChunkingConfig holds configuration for transcript chunking.
|
||||
type ChunkingConfig struct {
|
||||
// MaxSectionTokens is the maximum allowed tokens per section
|
||||
MaxSectionTokens int
|
||||
|
||||
// MinSectionTokens is a validated soft lower-bound setting retained for
|
||||
// configuration/reporting compatibility.
|
||||
MinSectionTokens int
|
||||
|
||||
// TargetSections is an optional target number of sections
|
||||
// If nil, section count is derived from total/max token budgeting.
|
||||
TargetSections *int
|
||||
}
|
||||
|
||||
// Chunker performs deterministic chunking of normalized transcript segments.
|
||||
type Chunker struct {
|
||||
config ChunkingConfig
|
||||
estimator TokenEstimator
|
||||
}
|
||||
|
||||
// NewChunker creates a new chunker with the given configuration.
|
||||
func NewChunker(config ChunkingConfig) *Chunker {
|
||||
return &Chunker{
|
||||
config: config,
|
||||
estimator: NewSimpleTokenEstimator(),
|
||||
}
|
||||
}
|
||||
|
||||
// NewChunkerWithEstimator creates a new chunker with a custom estimator.
|
||||
func NewChunkerWithEstimator(config ChunkingConfig, estimator TokenEstimator) *Chunker {
|
||||
return &Chunker{
|
||||
config: config,
|
||||
estimator: estimator,
|
||||
}
|
||||
}
|
||||
|
||||
// ChunkTranscript divides a normalized transcript into contiguous token-bounded
|
||||
// sections using a deterministic balanced forward pass.
|
||||
//
|
||||
// Behavior:
|
||||
// - preserve segment order and never split segments;
|
||||
// - estimate per-segment tokens once, then compute total;
|
||||
// - derive desired section count from ceil(total/max_section_tokens), unless
|
||||
// target_sections is explicitly set;
|
||||
// - prefer section sizes near ceil(total/section_count) while never exceeding
|
||||
// max_section_tokens unless a section consists of a single oversized segment.
|
||||
//
|
||||
// Returns an error if explicit target_sections is impossible under constraints.
|
||||
// The input transcript is never mutated.
|
||||
func (c *Chunker) ChunkTranscript(transcript *schema.Transcript) ([]Section, error) {
|
||||
if transcript == nil || len(transcript.Segments) == 0 {
|
||||
return []Section{}, nil
|
||||
}
|
||||
|
||||
// Validate configuration
|
||||
if c.config.MaxSectionTokens <= 0 {
|
||||
return nil, fmt.Errorf("max_section_tokens must be positive, got %d", c.config.MaxSectionTokens)
|
||||
}
|
||||
|
||||
if c.config.MinSectionTokens < 0 {
|
||||
return nil, fmt.Errorf("min_section_tokens must be non-negative, got %d", c.config.MinSectionTokens)
|
||||
}
|
||||
|
||||
if c.config.MinSectionTokens > c.config.MaxSectionTokens {
|
||||
return nil, fmt.Errorf("min_section_tokens (%d) cannot exceed max_section_tokens (%d)",
|
||||
c.config.MinSectionTokens, c.config.MaxSectionTokens)
|
||||
}
|
||||
|
||||
// Calculate token counts for each segment (deterministic).
|
||||
segmentTokens := make([]int, len(transcript.Segments))
|
||||
totalTokens := 0
|
||||
for i, seg := range transcript.Segments {
|
||||
segmentTokens[i] = c.estimator.EstimateTokens(seg.Text)
|
||||
totalTokens += segmentTokens[i]
|
||||
}
|
||||
|
||||
minPossibleSections := c.calculateMinPossibleSections(segmentTokens)
|
||||
|
||||
var desiredSections int
|
||||
useExplicitTarget := false
|
||||
if c.config.TargetSections != nil {
|
||||
desiredSections = *c.config.TargetSections
|
||||
useExplicitTarget = true
|
||||
if desiredSections <= 0 {
|
||||
return nil, fmt.Errorf("target_sections must be positive, got %d", desiredSections)
|
||||
}
|
||||
if err := c.validateTargetSections(desiredSections, segmentTokens); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
desiredSections = ceilDiv(totalTokens, c.config.MaxSectionTokens)
|
||||
if desiredSections < minPossibleSections {
|
||||
desiredSections = minPossibleSections
|
||||
}
|
||||
if desiredSections < 1 {
|
||||
desiredSections = 1
|
||||
}
|
||||
if desiredSections > len(transcript.Segments) {
|
||||
desiredSections = len(transcript.Segments)
|
||||
}
|
||||
}
|
||||
|
||||
targetTokensPerSection := ceilDiv(totalTokens, desiredSections)
|
||||
if useExplicitTarget {
|
||||
return c.buildSectionsWithExplicitTarget(
|
||||
transcript.Segments,
|
||||
segmentTokens,
|
||||
desiredSections,
|
||||
targetTokensPerSection,
|
||||
)
|
||||
}
|
||||
|
||||
return c.buildSectionsBalanced(transcript.Segments, segmentTokens, targetTokensPerSection), nil
|
||||
}
|
||||
|
||||
// validateTargetSections checks if the target section count is achievable.
|
||||
func (c *Chunker) validateTargetSections(target int, segmentTokens []int) error {
|
||||
// Maximum possible sections: limited by segment count
|
||||
maxPossibleSections := len(segmentTokens)
|
||||
|
||||
if target > maxPossibleSections {
|
||||
return fmt.Errorf(
|
||||
"target_sections (%d) is impossible: cannot have more sections than segments (%d)",
|
||||
target, maxPossibleSections)
|
||||
}
|
||||
|
||||
// Minimum possible sections: each segment must fit within max bounds
|
||||
minPossibleSections := c.calculateMinPossibleSections(segmentTokens)
|
||||
|
||||
if target < minPossibleSections {
|
||||
return fmt.Errorf(
|
||||
"target_sections (%d) is impossible: need at least %d sections to respect max_section_tokens (%d)",
|
||||
target, minPossibleSections, c.config.MaxSectionTokens)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildSectionsBalanced creates sections with a deterministic single-pass policy.
|
||||
func (c *Chunker) buildSectionsBalanced(segments []schema.Segment, segmentTokens []int, targetTokensPerSection int) []Section {
|
||||
var sections []Section
|
||||
var currentSegments []schema.Segment
|
||||
currentTokens := 0
|
||||
|
||||
for i, seg := range segments {
|
||||
tokens := segmentTokens[i]
|
||||
|
||||
// Empty section: always accept the next segment, including oversized.
|
||||
if len(currentSegments) == 0 {
|
||||
currentSegments = append(currentSegments, seg)
|
||||
currentTokens = tokens
|
||||
continue
|
||||
}
|
||||
|
||||
// If adding next segment would exceed max, close current section.
|
||||
if currentTokens+tokens > c.config.MaxSectionTokens {
|
||||
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
|
||||
currentSegments = []schema.Segment{seg}
|
||||
currentTokens = tokens
|
||||
continue
|
||||
}
|
||||
|
||||
// Prefer staying near target tokens per section.
|
||||
if targetTokensPerSection == 0 || currentTokens < targetTokensPerSection {
|
||||
currentSegments = append(currentSegments, seg)
|
||||
currentTokens += tokens
|
||||
continue
|
||||
}
|
||||
|
||||
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
|
||||
currentSegments = []schema.Segment{seg}
|
||||
currentTokens = tokens
|
||||
}
|
||||
|
||||
if len(currentSegments) > 0 {
|
||||
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
|
||||
}
|
||||
|
||||
return sections
|
||||
}
|
||||
|
||||
// buildSection creates a Section from segments.
|
||||
func (c *Chunker) buildSection(index int, segments []schema.Segment, tokens int) Section {
|
||||
return Section{
|
||||
Index: index,
|
||||
StartSegmentID: segments[0].ID,
|
||||
EndSegmentID: segments[len(segments)-1].ID,
|
||||
EstimatedTokens: tokens,
|
||||
Segments: segments,
|
||||
}
|
||||
}
|
||||
|
||||
// calculateMinPossibleSections calculates the minimum number of sections needed
|
||||
// to ensure no section exceeds max tokens.
|
||||
func (c *Chunker) calculateMinPossibleSections(segmentTokens []int) int {
|
||||
sections := 0
|
||||
currentTokens := 0
|
||||
|
||||
for _, tokens := range segmentTokens {
|
||||
if tokens > c.config.MaxSectionTokens {
|
||||
// Each oversized segment needs its own section
|
||||
if currentTokens > 0 {
|
||||
sections++
|
||||
currentTokens = 0
|
||||
}
|
||||
sections++
|
||||
} else if currentTokens+tokens > c.config.MaxSectionTokens {
|
||||
sections++
|
||||
currentTokens = tokens
|
||||
} else {
|
||||
currentTokens += tokens
|
||||
}
|
||||
}
|
||||
|
||||
if currentTokens > 0 {
|
||||
sections++
|
||||
}
|
||||
|
||||
return sections
|
||||
}
|
||||
|
||||
// buildSectionsWithExplicitTarget builds exactly desiredSections when feasible.
|
||||
func (c *Chunker) buildSectionsWithExplicitTarget(
|
||||
segments []schema.Segment,
|
||||
segmentTokens []int,
|
||||
desiredSections int,
|
||||
targetTokensPerSection int,
|
||||
) ([]Section, error) {
|
||||
n := len(segments)
|
||||
cursor := 0
|
||||
sections := make([]Section, 0, desiredSections)
|
||||
|
||||
for sectionIdx := 0; sectionIdx < desiredSections; sectionIdx++ {
|
||||
if cursor >= n {
|
||||
break
|
||||
}
|
||||
|
||||
remainingSectionsAfter := desiredSections - sectionIdx - 1
|
||||
currentSegments := []schema.Segment{segments[cursor]}
|
||||
currentTokens := segmentTokens[cursor]
|
||||
cursor++
|
||||
|
||||
for cursor < n {
|
||||
remainingSegments := n - cursor
|
||||
|
||||
// Reserve one segment per future section to avoid empty sections.
|
||||
if remainingSegments == remainingSectionsAfter {
|
||||
break
|
||||
}
|
||||
|
||||
nextTokens := segmentTokens[cursor]
|
||||
if currentTokens+nextTokens > c.config.MaxSectionTokens {
|
||||
break
|
||||
}
|
||||
if targetTokensPerSection == 0 || currentTokens < targetTokensPerSection {
|
||||
currentSegments = append(currentSegments, segments[cursor])
|
||||
currentTokens += nextTokens
|
||||
cursor++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
|
||||
}
|
||||
|
||||
if cursor != n || len(sections) != desiredSections {
|
||||
return nil, fmt.Errorf(
|
||||
"target_sections (%d) is impossible under current constraints (got %d sections)",
|
||||
desiredSections,
|
||||
len(sections),
|
||||
)
|
||||
}
|
||||
|
||||
return sections, nil
|
||||
}
|
||||
|
||||
func ceilDiv(numerator int, denominator int) int {
|
||||
if denominator <= 0 {
|
||||
return 0
|
||||
}
|
||||
if numerator <= 0 {
|
||||
return 0
|
||||
}
|
||||
return (numerator + denominator - 1) / denominator
|
||||
}
|
||||
518
internal/core/chunking/sections_test.go
Normal file
518
internal/core/chunking/sections_test.go
Normal file
@@ -0,0 +1,518 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
// mapTokenEstimator provides deterministic per-segment token counts for tests.
|
||||
type mapTokenEstimator struct {
|
||||
byText map[string]int
|
||||
}
|
||||
|
||||
func (e *mapTokenEstimator) EstimateTokens(text string) int {
|
||||
if e.byText == nil {
|
||||
return 0
|
||||
}
|
||||
if tokens, ok := e.byText[text]; ok {
|
||||
return tokens
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func makeSegments(texts []string) []schema.Segment {
|
||||
segments := make([]schema.Segment, len(texts))
|
||||
for i, text := range texts {
|
||||
segments[i] = schema.Segment{
|
||||
ID: i + 1,
|
||||
Speaker: "DM",
|
||||
Start: float64(i * 10),
|
||||
End: float64(i*10 + 5),
|
||||
Text: text,
|
||||
}
|
||||
}
|
||||
return segments
|
||||
}
|
||||
|
||||
func makeTranscript(segments []schema.Segment) *schema.Transcript {
|
||||
return &schema.Transcript{Segments: segments}
|
||||
}
|
||||
|
||||
func intPtr(i int) *int {
|
||||
return &i
|
||||
}
|
||||
|
||||
func assertSegmentCoverageAndOrder(t *testing.T, input []schema.Segment, sections []Section) {
|
||||
t.Helper()
|
||||
|
||||
seen := make([]schema.Segment, 0, len(input))
|
||||
for _, sec := range sections {
|
||||
seen = append(seen, sec.Segments...)
|
||||
}
|
||||
|
||||
if len(seen) != len(input) {
|
||||
t.Fatalf("expected %d total segment occurrences, got %d", len(input), len(seen))
|
||||
}
|
||||
|
||||
for i := range input {
|
||||
if seen[i].ID != input[i].ID {
|
||||
t.Fatalf("segment order mismatch at index %d: got id=%d want id=%d", i, seen[i].ID, input[i].ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertSectionMetadataConsistent(t *testing.T, sections []Section) {
|
||||
t.Helper()
|
||||
|
||||
for i, sec := range sections {
|
||||
if sec.Index != i {
|
||||
t.Fatalf("section %d: expected index=%d got=%d", i, i, sec.Index)
|
||||
}
|
||||
if len(sec.Segments) == 0 {
|
||||
t.Fatalf("section %d: section must not be empty", i)
|
||||
}
|
||||
if sec.StartSegmentID != sec.Segments[0].ID {
|
||||
t.Fatalf("section %d: start_segment_id mismatch", i)
|
||||
}
|
||||
if sec.EndSegmentID != sec.Segments[len(sec.Segments)-1].ID {
|
||||
t.Fatalf("section %d: end_segment_id mismatch", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertMaxBoundExceptSingletonOversized(t *testing.T, sections []Section, max int) {
|
||||
t.Helper()
|
||||
|
||||
for i, sec := range sections {
|
||||
if sec.EstimatedTokens <= max {
|
||||
continue
|
||||
}
|
||||
if len(sec.Segments) != 1 {
|
||||
t.Fatalf("section %d exceeds max tokens (%d>%d) with %d segments", i, sec.EstimatedTokens, max, len(sec.Segments))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func imbalance(sections []Section) int {
|
||||
if len(sections) == 0 {
|
||||
return 0
|
||||
}
|
||||
minTokens := sections[0].EstimatedTokens
|
||||
maxTokens := sections[0].EstimatedTokens
|
||||
for _, sec := range sections {
|
||||
if sec.EstimatedTokens < minTokens {
|
||||
minTokens = sec.EstimatedTokens
|
||||
}
|
||||
if sec.EstimatedTokens > maxTokens {
|
||||
maxTokens = sec.EstimatedTokens
|
||||
}
|
||||
}
|
||||
return maxTokens - minTokens
|
||||
}
|
||||
|
||||
func greedyMaxFillSections(segments []schema.Segment, tokens []int, max int) []Section {
|
||||
sections := make([]Section, 0)
|
||||
var current []schema.Segment
|
||||
currentTokens := 0
|
||||
|
||||
for i, seg := range segments {
|
||||
tok := tokens[i]
|
||||
if len(current) == 0 {
|
||||
current = append(current, seg)
|
||||
currentTokens = tok
|
||||
continue
|
||||
}
|
||||
if currentTokens+tok > max {
|
||||
sections = append(sections, Section{
|
||||
Index: len(sections),
|
||||
StartSegmentID: current[0].ID,
|
||||
EndSegmentID: current[len(current)-1].ID,
|
||||
EstimatedTokens: currentTokens,
|
||||
Segments: append([]schema.Segment(nil), current...),
|
||||
})
|
||||
current = []schema.Segment{seg}
|
||||
currentTokens = tok
|
||||
continue
|
||||
}
|
||||
current = append(current, seg)
|
||||
currentTokens += tok
|
||||
}
|
||||
|
||||
if len(current) > 0 {
|
||||
sections = append(sections, Section{
|
||||
Index: len(sections),
|
||||
StartSegmentID: current[0].ID,
|
||||
EndSegmentID: current[len(current)-1].ID,
|
||||
EstimatedTokens: currentTokens,
|
||||
Segments: append([]schema.Segment(nil), current...),
|
||||
})
|
||||
}
|
||||
|
||||
return sections
|
||||
}
|
||||
|
||||
func TestChunkEmptyTranscript(t *testing.T) {
|
||||
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 10})
|
||||
|
||||
sections, err := chunker.ChunkTranscript(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript(nil): %v", err)
|
||||
}
|
||||
if len(sections) != 0 {
|
||||
t.Fatalf("expected 0 sections for nil transcript, got %d", len(sections))
|
||||
}
|
||||
|
||||
sections, err = chunker.ChunkTranscript(makeTranscript(nil))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript(empty): %v", err)
|
||||
}
|
||||
if len(sections) != 0 {
|
||||
t.Fatalf("expected 0 sections for empty transcript, got %d", len(sections))
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkSingleSegment(t *testing.T) {
|
||||
segments := makeSegments([]string{"s1"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 10},
|
||||
&mapTokenEstimator{byText: map[string]int{"s1": 7}},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("expected 1 section, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 7 {
|
||||
t.Fatalf("expected estimated_tokens=7, got %d", sections[0].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertSectionMetadataConsistent(t, sections)
|
||||
}
|
||||
|
||||
func TestChunkSingleOversizedSegment(t *testing.T) {
|
||||
segments := makeSegments([]string{"big"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 5},
|
||||
&mapTokenEstimator{byText: map[string]int{"big": 120}},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("expected 1 section, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 120 {
|
||||
t.Fatalf("expected oversized singleton section, got %d", sections[0].EstimatedTokens)
|
||||
}
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 50)
|
||||
}
|
||||
|
||||
func TestChunkTotalBelowMaxSingleSection(t *testing.T) {
|
||||
segments := makeSegments([]string{"a", "b", "c"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 5},
|
||||
&ConstTokenEstimator{Tokens: 10},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("expected 1 section, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 30 {
|
||||
t.Fatalf("expected 30 section tokens, got %d", sections[0].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
}
|
||||
|
||||
func TestChunkTotalExactlyDivisibleByMax(t *testing.T) {
|
||||
segments := makeSegments([]string{"a", "b", "c", "d"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1},
|
||||
&ConstTokenEstimator{Tokens: 5},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 2 {
|
||||
t.Fatalf("expected 2 sections, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 10 || sections[1].EstimatedTokens != 10 {
|
||||
t.Fatalf("expected [10,10] tokens, got [%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 10)
|
||||
}
|
||||
|
||||
func TestChunkTotalNotDivisibleByMax(t *testing.T) {
|
||||
segments := makeSegments([]string{"a", "b", "c", "d", "e"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1},
|
||||
&ConstTokenEstimator{Tokens: 5},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 3 {
|
||||
t.Fatalf("expected 3 sections, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 10 || sections[1].EstimatedTokens != 10 || sections[2].EstimatedTokens != 5 {
|
||||
t.Fatalf("expected [10,10,5] tokens, got [%d,%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens, sections[2].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 10)
|
||||
}
|
||||
|
||||
func TestChunkTargetSectionsPrecedenceAndSuccess(t *testing.T) {
|
||||
segments := makeSegments([]string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10"})
|
||||
target := 3
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 200, MinSectionTokens: 1, TargetSections: &target},
|
||||
&ConstTokenEstimator{Tokens: 10},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != target {
|
||||
t.Fatalf("expected %d sections from explicit target, got %d", target, len(sections))
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertSectionMetadataConsistent(t, sections)
|
||||
}
|
||||
|
||||
func TestChunkTargetSectionsImpossibleTooMany(t *testing.T) {
|
||||
segments := makeSegments([]string{"1", "2", "3", "4", "5"})
|
||||
target := 10
|
||||
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 1, TargetSections: &target})
|
||||
|
||||
_, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err == nil {
|
||||
t.Fatal("expected error for impossible target_sections")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "cannot have more sections than segments") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkTargetSectionsImpossibleTooFew(t *testing.T) {
|
||||
segments := makeSegments([]string{"1", "2", "3"})
|
||||
target := 1
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1, TargetSections: &target},
|
||||
&ConstTokenEstimator{Tokens: 30},
|
||||
)
|
||||
|
||||
_, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err == nil {
|
||||
t.Fatal("expected error for impossible target_sections")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "need at least 3 sections") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkManySmallSegmentsBalanced(t *testing.T) {
|
||||
texts := []string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"}
|
||||
segments := makeSegments(texts)
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1},
|
||||
&ConstTokenEstimator{Tokens: 10},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 3 {
|
||||
t.Fatalf("expected 3 sections, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 40 || sections[1].EstimatedTokens != 40 || sections[2].EstimatedTokens != 30 {
|
||||
t.Fatalf("expected [40,40,30], got [%d,%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens, sections[2].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 50)
|
||||
}
|
||||
|
||||
func TestChunkMixedLargeAndSmallSegments(t *testing.T) {
|
||||
segments := makeSegments([]string{"big1", "s1", "s2", "s3", "big2", "s4"})
|
||||
estimator := &mapTokenEstimator{byText: map[string]int{
|
||||
"big1": 120,
|
||||
"s1": 10,
|
||||
"s2": 10,
|
||||
"s3": 10,
|
||||
"big2": 120,
|
||||
"s4": 10,
|
||||
}}
|
||||
chunker := NewChunkerWithEstimator(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 1}, estimator)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 4 {
|
||||
t.Fatalf("expected 4 sections, got %d", len(sections))
|
||||
}
|
||||
if len(sections[0].Segments) != 1 || sections[0].Segments[0].Text != "big1" {
|
||||
t.Fatalf("expected first oversized segment in singleton section, got %+v", sections[0].Segments)
|
||||
}
|
||||
if len(sections[2].Segments) != 1 || sections[2].Segments[0].Text != "big2" {
|
||||
t.Fatalf("expected second oversized segment in singleton section, got %+v", sections[2].Segments)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 100)
|
||||
}
|
||||
|
||||
func TestChunkDeterministicOrdering(t *testing.T) {
|
||||
segments := makeSegments([]string{"a", "b", "c", "d", "e", "f"})
|
||||
chunkerCfg := ChunkingConfig{MaxSectionTokens: 15, MinSectionTokens: 1}
|
||||
estimator := &ConstTokenEstimator{Tokens: 5}
|
||||
|
||||
var first []Section
|
||||
for i := 0; i < 5; i++ {
|
||||
chunker := NewChunkerWithEstimator(chunkerCfg, estimator)
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("iteration %d: %v", i, err)
|
||||
}
|
||||
if i == 0 {
|
||||
first = sections
|
||||
continue
|
||||
}
|
||||
if len(sections) != len(first) {
|
||||
t.Fatalf("iteration %d: section count mismatch (%d vs %d)", i, len(sections), len(first))
|
||||
}
|
||||
for j := range sections {
|
||||
if sections[j].Index != first[j].Index ||
|
||||
sections[j].StartSegmentID != first[j].StartSegmentID ||
|
||||
sections[j].EndSegmentID != first[j].EndSegmentID ||
|
||||
sections[j].EstimatedTokens != first[j].EstimatedTokens ||
|
||||
len(sections[j].Segments) != len(first[j].Segments) {
|
||||
t.Fatalf("iteration %d section %d mismatch", i, j)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkNoMutationOfInput(t *testing.T) {
|
||||
segments := makeSegments([]string{"original one", "original two"})
|
||||
transcript := makeTranscript(segments)
|
||||
original := make([]string, len(transcript.Segments))
|
||||
for i := range transcript.Segments {
|
||||
original[i] = transcript.Segments[i].Text
|
||||
}
|
||||
|
||||
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1})
|
||||
if _, err := chunker.ChunkTranscript(transcript); err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
for i := range transcript.Segments {
|
||||
if transcript.Segments[i].Text != original[i] {
|
||||
t.Fatalf("segment %d mutated", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkConfigValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config ChunkingConfig
|
||||
errContains string
|
||||
}{
|
||||
{
|
||||
name: "zero max tokens",
|
||||
config: ChunkingConfig{MaxSectionTokens: 0, MinSectionTokens: 1},
|
||||
errContains: "max_section_tokens must be positive",
|
||||
},
|
||||
{
|
||||
name: "negative max tokens",
|
||||
config: ChunkingConfig{MaxSectionTokens: -1, MinSectionTokens: 1},
|
||||
errContains: "max_section_tokens must be positive",
|
||||
},
|
||||
{
|
||||
name: "negative min tokens",
|
||||
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: -1},
|
||||
errContains: "min_section_tokens must be non-negative",
|
||||
},
|
||||
{
|
||||
name: "min exceeds max",
|
||||
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 11},
|
||||
errContains: "min_section_tokens (11) cannot exceed max_section_tokens (10)",
|
||||
},
|
||||
{
|
||||
name: "zero target sections",
|
||||
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1, TargetSections: intPtr(0)},
|
||||
errContains: "target_sections must be positive",
|
||||
},
|
||||
{
|
||||
name: "negative target sections",
|
||||
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1, TargetSections: intPtr(-1)},
|
||||
errContains: "target_sections must be positive",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
chunker := NewChunker(tt.config)
|
||||
_, err := chunker.ChunkTranscript(makeTranscript(makeSegments([]string{"x"})))
|
||||
if err == nil {
|
||||
t.Fatalf("expected error containing %q", tt.errContains)
|
||||
}
|
||||
if !strings.Contains(err.Error(), tt.errContains) {
|
||||
t.Fatalf("expected error containing %q, got %q", tt.errContains, err.Error())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkBalancedAlgorithmBeatsGreedyMaxFillOnUnevenTranscript(t *testing.T) {
|
||||
segments := makeSegments([]string{"s1", "s2", "s3", "s4", "s5", "s6"})
|
||||
tokenMap := map[string]int{
|
||||
"s1": 50,
|
||||
"s2": 10,
|
||||
"s3": 10,
|
||||
"s4": 10,
|
||||
"s5": 10,
|
||||
"s6": 10,
|
||||
}
|
||||
estimator := &mapTokenEstimator{byText: tokenMap}
|
||||
chunker := NewChunkerWithEstimator(ChunkingConfig{MaxSectionTokens: 80, MinSectionTokens: 1}, estimator)
|
||||
|
||||
balancedSections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
|
||||
tokens := make([]int, 0, len(segments))
|
||||
for _, seg := range segments {
|
||||
tokens = append(tokens, tokenMap[seg.Text])
|
||||
}
|
||||
greedySections := greedyMaxFillSections(segments, tokens, 80)
|
||||
|
||||
balancedImbalance := imbalance(balancedSections)
|
||||
greedyImbalance := imbalance(greedySections)
|
||||
if balancedImbalance >= greedyImbalance {
|
||||
t.Fatalf(
|
||||
"expected balanced chunking to improve over greedy max-fill; balanced=%d greedy=%d",
|
||||
balancedImbalance,
|
||||
greedyImbalance,
|
||||
)
|
||||
}
|
||||
|
||||
assertSegmentCoverageAndOrder(t, segments, balancedSections)
|
||||
assertMaxBoundExceptSingletonOversized(t, balancedSections, 80)
|
||||
}
|
||||
84
internal/core/chunking/summary.go
Normal file
84
internal/core/chunking/summary.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package chunking
|
||||
|
||||
// Summary provides a concise overview of chunking results for reports
|
||||
type Summary struct {
|
||||
ChunkCount int `json:"chunk_count"`
|
||||
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
|
||||
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
|
||||
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
|
||||
TargetSections *int `json:"target_sections,omitempty"`
|
||||
MaxSectionTokens int `json:"max_section_tokens"`
|
||||
MinSectionTokens int `json:"min_section_tokens"`
|
||||
}
|
||||
|
||||
// ChunkSummary represents a single chunk's metadata for diagnostics
|
||||
type ChunkSummary struct {
|
||||
Index int `json:"index"`
|
||||
StartSegmentID int `json:"start_segment_id"`
|
||||
EndSegmentID int `json:"end_segment_id"`
|
||||
EstimatedTokens int `json:"estimated_tokens"`
|
||||
SegmentCount int `json:"segment_count"`
|
||||
}
|
||||
|
||||
// DetailedSummary provides per-chunk details for diagnostics
|
||||
type DetailedSummary struct {
|
||||
Summary `json:",inline"`
|
||||
Chunks []ChunkSummary `json:"chunks"`
|
||||
}
|
||||
|
||||
// ComputeSummary creates a Summary from sections and config
|
||||
func ComputeSummary(sections []Section, config ChunkingConfig) Summary {
|
||||
if len(sections) == 0 {
|
||||
return Summary{
|
||||
ChunkCount: 0,
|
||||
MaxSectionTokens: config.MaxSectionTokens,
|
||||
MinSectionTokens: config.MinSectionTokens,
|
||||
TargetSections: config.TargetSections,
|
||||
}
|
||||
}
|
||||
|
||||
minTokens := sections[0].EstimatedTokens
|
||||
maxTokens := sections[0].EstimatedTokens
|
||||
totalTokens := 0
|
||||
|
||||
for _, sec := range sections {
|
||||
if sec.EstimatedTokens < minTokens {
|
||||
minTokens = sec.EstimatedTokens
|
||||
}
|
||||
if sec.EstimatedTokens > maxTokens {
|
||||
maxTokens = sec.EstimatedTokens
|
||||
}
|
||||
totalTokens += sec.EstimatedTokens
|
||||
}
|
||||
|
||||
return Summary{
|
||||
ChunkCount: len(sections),
|
||||
MinEstimatedTokens: minTokens,
|
||||
MaxEstimatedTokens: maxTokens,
|
||||
TotalEstimatedTokens: totalTokens,
|
||||
TargetSections: config.TargetSections,
|
||||
MaxSectionTokens: config.MaxSectionTokens,
|
||||
MinSectionTokens: config.MinSectionTokens,
|
||||
}
|
||||
}
|
||||
|
||||
// ComputeDetailedSummary creates a DetailedSummary from sections and config
|
||||
func ComputeDetailedSummary(sections []Section, config ChunkingConfig) DetailedSummary {
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
chunks := make([]ChunkSummary, len(sections))
|
||||
for i, sec := range sections {
|
||||
chunks[i] = ChunkSummary{
|
||||
Index: sec.Index,
|
||||
StartSegmentID: sec.StartSegmentID,
|
||||
EndSegmentID: sec.EndSegmentID,
|
||||
EstimatedTokens: sec.EstimatedTokens,
|
||||
SegmentCount: len(sec.Segments),
|
||||
}
|
||||
}
|
||||
|
||||
return DetailedSummary{
|
||||
Summary: summary,
|
||||
Chunks: chunks,
|
||||
}
|
||||
}
|
||||
148
internal/core/chunking/summary_test.go
Normal file
148
internal/core/chunking/summary_test.go
Normal file
@@ -0,0 +1,148 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
func TestComputeSummary(t *testing.T) {
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
}
|
||||
|
||||
sections := []Section{
|
||||
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
|
||||
{Index: 1, EstimatedTokens: 50, StartSegmentID: 3, EndSegmentID: 4},
|
||||
{Index: 2, EstimatedTokens: 20, StartSegmentID: 5, EndSegmentID: 5},
|
||||
}
|
||||
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
if summary.ChunkCount != 3 {
|
||||
t.Errorf("expected chunk_count=3, got %d", summary.ChunkCount)
|
||||
}
|
||||
if summary.MinEstimatedTokens != 20 {
|
||||
t.Errorf("expected min_estimated_tokens=20, got %d", summary.MinEstimatedTokens)
|
||||
}
|
||||
if summary.MaxEstimatedTokens != 50 {
|
||||
t.Errorf("expected max_estimated_tokens=50, got %d", summary.MaxEstimatedTokens)
|
||||
}
|
||||
if summary.TotalEstimatedTokens != 100 {
|
||||
t.Errorf("expected total_estimated_tokens=100, got %d", summary.TotalEstimatedTokens)
|
||||
}
|
||||
if summary.MaxSectionTokens != 100 {
|
||||
t.Errorf("expected max_section_tokens=100, got %d", summary.MaxSectionTokens)
|
||||
}
|
||||
if summary.MinSectionTokens != 10 {
|
||||
t.Errorf("expected min_section_tokens=10, got %d", summary.MinSectionTokens)
|
||||
}
|
||||
if summary.TargetSections != nil {
|
||||
t.Errorf("expected target_sections=nil, got %v", summary.TargetSections)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeSummaryWithTarget(t *testing.T) {
|
||||
target := 5
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
TargetSections: &target,
|
||||
}
|
||||
|
||||
sections := []Section{
|
||||
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
|
||||
}
|
||||
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
if summary.TargetSections == nil || *summary.TargetSections != 5 {
|
||||
t.Errorf("expected target_sections=5, got %v", summary.TargetSections)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeSummaryEmptySections(t *testing.T) {
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
}
|
||||
|
||||
sections := []Section{}
|
||||
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
if summary.ChunkCount != 0 {
|
||||
t.Errorf("expected chunk_count=0, got %d", summary.ChunkCount)
|
||||
}
|
||||
if summary.MinEstimatedTokens != 0 {
|
||||
t.Errorf("expected min_estimated_tokens=0 for empty, got %d", summary.MinEstimatedTokens)
|
||||
}
|
||||
if summary.MaxSectionTokens != 100 {
|
||||
t.Errorf("expected max_section_tokens preserved, got %d", summary.MaxSectionTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeDetailedSummary(t *testing.T) {
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
}
|
||||
|
||||
sections := []Section{
|
||||
{
|
||||
Index: 0,
|
||||
EstimatedTokens: 30,
|
||||
StartSegmentID: 1,
|
||||
EndSegmentID: 2,
|
||||
Segments: make([]schema.Segment, 2), // 2 segments
|
||||
},
|
||||
{
|
||||
Index: 1,
|
||||
EstimatedTokens: 50,
|
||||
StartSegmentID: 3,
|
||||
EndSegmentID: 5,
|
||||
Segments: make([]schema.Segment, 3), // 3 segments
|
||||
},
|
||||
}
|
||||
|
||||
detailed := ComputeDetailedSummary(sections, config)
|
||||
|
||||
if detailed.ChunkCount != 2 {
|
||||
t.Errorf("expected chunk_count=2, got %d", detailed.ChunkCount)
|
||||
}
|
||||
if len(detailed.Chunks) != 2 {
|
||||
t.Fatalf("expected 2 chunk entries, got %d", len(detailed.Chunks))
|
||||
}
|
||||
|
||||
// Check first chunk
|
||||
if detailed.Chunks[0].Index != 0 {
|
||||
t.Errorf("expected chunk[0].index=0, got %d", detailed.Chunks[0].Index)
|
||||
}
|
||||
if detailed.Chunks[0].StartSegmentID != 1 {
|
||||
t.Errorf("expected chunk[0].start_segment_id=1, got %d", detailed.Chunks[0].StartSegmentID)
|
||||
}
|
||||
if detailed.Chunks[0].EndSegmentID != 2 {
|
||||
t.Errorf("expected chunk[0].end_segment_id=2, got %d", detailed.Chunks[0].EndSegmentID)
|
||||
}
|
||||
if detailed.Chunks[0].EstimatedTokens != 30 {
|
||||
t.Errorf("expected chunk[0].estimated_tokens=30, got %d", detailed.Chunks[0].EstimatedTokens)
|
||||
}
|
||||
if detailed.Chunks[0].SegmentCount != 2 {
|
||||
t.Errorf("expected chunk[0].segment_count=2, got %d", detailed.Chunks[0].SegmentCount)
|
||||
}
|
||||
|
||||
// Check second chunk
|
||||
if detailed.Chunks[1].Index != 1 {
|
||||
t.Errorf("expected chunk[1].index=1, got %d", detailed.Chunks[1].Index)
|
||||
}
|
||||
if detailed.Chunks[1].StartSegmentID != 3 {
|
||||
t.Errorf("expected chunk[1].start_segment_id=3, got %d", detailed.Chunks[1].StartSegmentID)
|
||||
}
|
||||
if detailed.Chunks[1].EndSegmentID != 5 {
|
||||
t.Errorf("expected chunk[1].end_segment_id=5, got %d", detailed.Chunks[1].EndSegmentID)
|
||||
}
|
||||
if detailed.Chunks[1].SegmentCount != 3 {
|
||||
t.Errorf("expected chunk[1].segment_count=3, got %d", detailed.Chunks[1].SegmentCount)
|
||||
}
|
||||
}
|
||||
55
internal/core/chunking/tokens.go
Normal file
55
internal/core/chunking/tokens.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// TokenEstimator provides a deterministic token estimation suitable for prompt budgeting.
|
||||
// The estimator is approximate but stable, isolated, and replaceable.
|
||||
type TokenEstimator interface {
|
||||
EstimateTokens(text string) int
|
||||
}
|
||||
|
||||
// SimpleTokenEstimator provides a basic deterministic token estimation.
|
||||
// This uses a simple heuristic based on word count and punctuation.
|
||||
type SimpleTokenEstimator struct{}
|
||||
|
||||
// NewSimpleTokenEstimator creates a new simple token estimator.
|
||||
func NewSimpleTokenEstimator() *SimpleTokenEstimator {
|
||||
return &SimpleTokenEstimator{}
|
||||
}
|
||||
|
||||
// EstimateTokens provides a rough estimate of the number of tokens in the given text.
|
||||
// This implementation uses a simple heuristic: count words and punctuation as tokens.
|
||||
// The estimate is deterministic and stable for the same input text.
|
||||
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
|
||||
if text == "" {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Simple heuristic: split on whitespace and count non-empty segments
|
||||
words := strings.Fields(text)
|
||||
tokenCount := len(words)
|
||||
|
||||
// Add some estimate for punctuation that might be separate tokens
|
||||
punctuationCount := 0
|
||||
for _, r := range text {
|
||||
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
|
||||
punctuationCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Rough estimate: each word is a token, plus half the punctuation as separate tokens
|
||||
return tokenCount + (punctuationCount / 2)
|
||||
}
|
||||
|
||||
// ConstTokenEstimator returns a constant token count for testing purposes.
|
||||
type ConstTokenEstimator struct {
|
||||
Tokens int
|
||||
}
|
||||
|
||||
// EstimateTokens returns the configured constant token count.
|
||||
func (e *ConstTokenEstimator) EstimateTokens(text string) int {
|
||||
return e.Tokens
|
||||
}
|
||||
59
internal/core/chunking/tokens_test.go
Normal file
59
internal/core/chunking/tokens_test.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSimpleTokenEstimator(t *testing.T) {
|
||||
estimator := NewSimpleTokenEstimator()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
text string
|
||||
expected int
|
||||
}{
|
||||
{"empty string", "", 0},
|
||||
{"single word", "hello", 1},
|
||||
{"two words", "hello world", 2},
|
||||
{"with punctuation", "hello, world!", 3}, // 2 words + 2 punctuation/2 = 3
|
||||
{"multiple sentences", "Hello world. This is a test.", 7}, // 7 words + 2 punctuation/2 = 8? Actually "Hello world." has 3 punctuation
|
||||
{"with apostrophes", "don't won't can't", 3},
|
||||
{"with hyphens", "well-known state-of-the-art", 2}, // hyphens don't count
|
||||
{"unicode text", "café naïve", 2},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := estimator.EstimateTokens(tt.text)
|
||||
if got != tt.expected {
|
||||
t.Errorf("EstimateTokens(%q) = %d, want %d", tt.text, got, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConstTokenEstimator(t *testing.T) {
|
||||
estimator := &ConstTokenEstimator{Tokens: 42}
|
||||
|
||||
if got := estimator.EstimateTokens("any text"); got != 42 {
|
||||
t.Errorf("ConstTokenEstimator.EstimateTokens = %d, want 42", got)
|
||||
}
|
||||
|
||||
if got := estimator.EstimateTokens(""); got != 42 {
|
||||
t.Errorf("ConstTokenEstimator.EstimateTokens(empty) = %d, want 42", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokenEstimatorDeterminism(t *testing.T) {
|
||||
estimator := NewSimpleTokenEstimator()
|
||||
text := "The quick brown fox jumps over the lazy dog. Hello, world!"
|
||||
|
||||
// Run multiple times and verify same result
|
||||
first := estimator.EstimateTokens(text)
|
||||
for i := 0; i < 10; i++ {
|
||||
got := estimator.EstimateTokens(text)
|
||||
if got != first {
|
||||
t.Errorf("EstimateTokens not deterministic: iteration %d got %d, first was %d", i, got, first)
|
||||
}
|
||||
}
|
||||
}
|
||||
197
internal/core/config/config.go
Normal file
197
internal/core/config/config.go
Normal file
@@ -0,0 +1,197 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type WorkDirRetention string
|
||||
|
||||
const (
|
||||
WorkDirRetentionAuto WorkDirRetention = "auto"
|
||||
WorkDirRetentionAlways WorkDirRetention = "always"
|
||||
WorkDirRetentionNever WorkDirRetention = "never"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultModulesCSV = "glossary,homophones,glossary,spoken_word,grammar"
|
||||
DefaultOutputSchema = "bare-segments"
|
||||
DefaultPrimaryModel = "openrouter/google/gemma-4-31b-it"
|
||||
DefaultPrimaryBaseURL = "https://openrouter.ai/api/v1"
|
||||
DefaultPrimaryLLMTimeoutSeconds = 600
|
||||
DefaultMaxRetries = 3
|
||||
DefaultLLMConcurrency = 1
|
||||
DefaultValidationMaxPromptTokens = 2048
|
||||
DefaultMaxSectionTokens = 8192
|
||||
DefaultMinSectionTokens = 2048
|
||||
DefaultConfidenceThreshold = 0.8
|
||||
DefaultNormalizeMaxSegmentGap = 4.0
|
||||
DefaultNormalizeEllipsisGap = 3.5
|
||||
DefaultNormalizeMaxSegmentDuration = 60.0
|
||||
DefaultNormalizeMaxSegmentTokens = 2048
|
||||
DefaultTranscriptDescriptionMaxChars = 500
|
||||
DefaultWorkDir = "/tmp/audita"
|
||||
DefaultWorkDirRetention WorkDirRetention = WorkDirRetentionAuto
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Modules []string
|
||||
OutputSchema string
|
||||
PrimaryLLM LLMConfig
|
||||
ValidationLLM ValidationLLMConfig
|
||||
TotalLLMConcurrency int
|
||||
ProposalLLMConcurrency int
|
||||
ValidationLLMConcurrency *int
|
||||
ValidationMaxPromptTokens int
|
||||
MaxSectionTokens int
|
||||
MinSectionTokens int
|
||||
TargetSections *int
|
||||
Thresholds ConfidenceThresholds
|
||||
Normalization NormalizationConfig
|
||||
TranscriptDescription string
|
||||
WorkDir string
|
||||
WorkDirRetention WorkDirRetention
|
||||
}
|
||||
|
||||
type LLMConfig struct {
|
||||
APIKey string
|
||||
Model string
|
||||
BaseURL string
|
||||
TimeoutSeconds int
|
||||
MaxRetries int
|
||||
// Concurrency is retained as a backward-compatible alias for
|
||||
// TotalLLMConcurrency.
|
||||
Concurrency int
|
||||
}
|
||||
|
||||
type ValidationLLMConfig struct {
|
||||
APIKey string
|
||||
Model string
|
||||
BaseURL string
|
||||
TimeoutSeconds *int
|
||||
MaxRetries *int
|
||||
// Concurrency is retained as a backward-compatible alias for
|
||||
// ValidationLLMConcurrency.
|
||||
Concurrency *int
|
||||
}
|
||||
|
||||
type ConfidenceThresholds struct {
|
||||
Glossary float64
|
||||
Grammar float64
|
||||
Homophones float64
|
||||
SpokenWord float64
|
||||
}
|
||||
|
||||
type NormalizationConfig struct {
|
||||
MaxSegmentGap float64
|
||||
EllipsisGap float64
|
||||
MaxSegmentDuration float64
|
||||
MaxSegmentTokens int
|
||||
}
|
||||
|
||||
func Default() Config {
|
||||
modules, _ := ParseModulesCSV(DefaultModulesCSV)
|
||||
|
||||
return Config{
|
||||
Modules: modules,
|
||||
OutputSchema: DefaultOutputSchema,
|
||||
PrimaryLLM: LLMConfig{
|
||||
Model: DefaultPrimaryModel,
|
||||
BaseURL: DefaultPrimaryBaseURL,
|
||||
TimeoutSeconds: DefaultPrimaryLLMTimeoutSeconds,
|
||||
MaxRetries: DefaultMaxRetries,
|
||||
Concurrency: DefaultLLMConcurrency,
|
||||
},
|
||||
ValidationLLM: ValidationLLMConfig{},
|
||||
TotalLLMConcurrency: DefaultLLMConcurrency,
|
||||
ProposalLLMConcurrency: DefaultLLMConcurrency,
|
||||
ValidationLLMConcurrency: nil,
|
||||
ValidationMaxPromptTokens: DefaultValidationMaxPromptTokens,
|
||||
MaxSectionTokens: DefaultMaxSectionTokens,
|
||||
MinSectionTokens: DefaultMinSectionTokens,
|
||||
TargetSections: nil,
|
||||
Thresholds: ConfidenceThresholds{
|
||||
Glossary: DefaultConfidenceThreshold,
|
||||
Grammar: DefaultConfidenceThreshold,
|
||||
Homophones: DefaultConfidenceThreshold,
|
||||
SpokenWord: DefaultConfidenceThreshold,
|
||||
},
|
||||
Normalization: NormalizationConfig{
|
||||
MaxSegmentGap: DefaultNormalizeMaxSegmentGap,
|
||||
EllipsisGap: DefaultNormalizeEllipsisGap,
|
||||
MaxSegmentDuration: DefaultNormalizeMaxSegmentDuration,
|
||||
MaxSegmentTokens: DefaultNormalizeMaxSegmentTokens,
|
||||
},
|
||||
WorkDir: DefaultWorkDir,
|
||||
WorkDirRetention: DefaultWorkDirRetention,
|
||||
}
|
||||
}
|
||||
|
||||
func ParseModulesCSV(raw string) ([]string, error) {
|
||||
parts := strings.Split(raw, ",")
|
||||
modules := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
trimmed := strings.TrimSpace(part)
|
||||
if trimmed == "" {
|
||||
return nil, fmt.Errorf("modules list contains an empty value")
|
||||
}
|
||||
modules = append(modules, trimmed)
|
||||
}
|
||||
if len(modules) == 0 {
|
||||
return nil, fmt.Errorf("modules list must not be empty")
|
||||
}
|
||||
return modules, nil
|
||||
}
|
||||
|
||||
func (c Config) EffectiveValidationLLMConfig() LLMConfig {
|
||||
effective := c.PrimaryLLM
|
||||
|
||||
if c.ValidationLLM.APIKey != "" {
|
||||
effective.APIKey = c.ValidationLLM.APIKey
|
||||
}
|
||||
if c.ValidationLLM.Model != "" {
|
||||
effective.Model = c.ValidationLLM.Model
|
||||
}
|
||||
if c.ValidationLLM.BaseURL != "" {
|
||||
effective.BaseURL = c.ValidationLLM.BaseURL
|
||||
}
|
||||
if c.ValidationLLM.TimeoutSeconds != nil {
|
||||
effective.TimeoutSeconds = *c.ValidationLLM.TimeoutSeconds
|
||||
}
|
||||
if c.ValidationLLM.MaxRetries != nil {
|
||||
effective.MaxRetries = *c.ValidationLLM.MaxRetries
|
||||
}
|
||||
effective.Concurrency = c.EffectiveValidationLLMConcurrency()
|
||||
|
||||
return effective
|
||||
}
|
||||
|
||||
func (c Config) EffectiveValidationLLMConcurrency() int {
|
||||
if c.ValidationLLMConcurrency != nil {
|
||||
return *c.ValidationLLMConcurrency
|
||||
}
|
||||
return c.TotalLLMConcurrency
|
||||
}
|
||||
|
||||
func (c Config) EffectiveProposalLLMConcurrency() int {
|
||||
if c.ProposalLLMConcurrency > 0 {
|
||||
return c.ProposalLLMConcurrency
|
||||
}
|
||||
return c.TotalLLMConcurrency
|
||||
}
|
||||
|
||||
func (c *Config) syncLegacyConcurrencyAliases() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.PrimaryLLM.Concurrency = c.TotalLLMConcurrency
|
||||
c.ValidationLLM.Concurrency = intPtr(c.ValidationLLMConcurrency)
|
||||
}
|
||||
|
||||
func intPtr(v *int) *int {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
x := *v
|
||||
return &x
|
||||
}
|
||||
423
internal/core/config/config_test.go
Normal file
423
internal/core/config/config_test.go
Normal file
@@ -0,0 +1,423 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDefaultConfigValues(t *testing.T) {
|
||||
cfg := Default()
|
||||
|
||||
if got, want := strings.Join(cfg.Modules, ","), DefaultModulesCSV; got != want {
|
||||
t.Fatalf("modules mismatch: got %q want %q", got, want)
|
||||
}
|
||||
if cfg.OutputSchema != DefaultOutputSchema {
|
||||
t.Fatalf("unexpected default output schema: %q", cfg.OutputSchema)
|
||||
}
|
||||
if cfg.PrimaryLLM.Model != DefaultPrimaryModel {
|
||||
t.Fatalf("unexpected default primary model: %q", cfg.PrimaryLLM.Model)
|
||||
}
|
||||
if cfg.PrimaryLLM.BaseURL != DefaultPrimaryBaseURL {
|
||||
t.Fatalf("unexpected default primary base url: %q", cfg.PrimaryLLM.BaseURL)
|
||||
}
|
||||
if cfg.PrimaryLLM.TimeoutSeconds != DefaultPrimaryLLMTimeoutSeconds {
|
||||
t.Fatalf("unexpected default timeout seconds: %d", cfg.PrimaryLLM.TimeoutSeconds)
|
||||
}
|
||||
if cfg.PrimaryLLM.MaxRetries != DefaultMaxRetries {
|
||||
t.Fatalf("unexpected default max retries: %d", cfg.PrimaryLLM.MaxRetries)
|
||||
}
|
||||
if cfg.TotalLLMConcurrency != DefaultLLMConcurrency {
|
||||
t.Fatalf("unexpected default total llm concurrency: %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != DefaultLLMConcurrency {
|
||||
t.Fatalf("unexpected default proposal llm concurrency: %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
if cfg.ValidationLLMConcurrency != nil {
|
||||
t.Fatalf("expected validation llm concurrency to be unset by default")
|
||||
}
|
||||
if cfg.PrimaryLLM.Concurrency != cfg.TotalLLMConcurrency {
|
||||
t.Fatalf("expected primary llm concurrency alias to mirror total, got primary=%d total=%d", cfg.PrimaryLLM.Concurrency, cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ValidationLLM.TimeoutSeconds != nil {
|
||||
t.Fatalf("expected validation timeout to be unset by default")
|
||||
}
|
||||
if cfg.ValidationLLM.MaxRetries != nil {
|
||||
t.Fatalf("expected validation max retries to be unset by default")
|
||||
}
|
||||
if cfg.ValidationLLM.Concurrency != nil {
|
||||
t.Fatalf("expected legacy validation llm concurrency alias to be unset by default")
|
||||
}
|
||||
if cfg.TargetSections != nil {
|
||||
t.Fatalf("expected target sections to be unset by default")
|
||||
}
|
||||
if cfg.WorkDir != DefaultWorkDir {
|
||||
t.Fatalf("unexpected default work dir: %q", cfg.WorkDir)
|
||||
}
|
||||
if cfg.TranscriptDescription != "" {
|
||||
t.Fatalf("expected default transcript description to be empty, got %q", cfg.TranscriptDescription)
|
||||
}
|
||||
if cfg.WorkDirRetention != DefaultWorkDirRetention {
|
||||
t.Fatalf("unexpected default work dir retention: %q", cfg.WorkDirRetention)
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("default config should validate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromEnvOverridesAndFallback(t *testing.T) {
|
||||
env := map[string]string{
|
||||
"AUDITA_MODEL": "openai/gpt-4.1-mini",
|
||||
"AUDITA_BASE_URL": "https://api.openai.com/v1",
|
||||
"AUDITA_LLM_TIMEOUT_SECONDS": "120",
|
||||
"AUDITA_MAX_RETRIES": "7",
|
||||
"AUDITA_TOTAL_LLM_CONCURRENCY": "6",
|
||||
"AUDITA_PROPOSAL_LLM_CONCURRENCY": "4",
|
||||
"AUDITA_VALIDATION_LLM_CONCURRENCY": "2",
|
||||
"AUDITA_VALIDATION_MAX_PROMPT_TOKENS": "4096",
|
||||
"AUDITA_MAX_SECTION_TOKENS": "9000",
|
||||
"AUDITA_MIN_SECTION_TOKENS": "3000",
|
||||
"AUDITA_TARGET_SECTIONS": "5",
|
||||
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.9",
|
||||
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.7",
|
||||
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.6",
|
||||
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.5",
|
||||
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "2.5",
|
||||
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "2.0",
|
||||
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "30.0",
|
||||
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "1024",
|
||||
"AUDITA_WORK_DIR": "/var/tmp/audita",
|
||||
"AUDITA_WORK_DIR_RETENTION": "always",
|
||||
"OPENROUTER_API_KEY": "fallback-key",
|
||||
}
|
||||
|
||||
cfg, err := loadFromLookup(mapLookup(env))
|
||||
if err != nil {
|
||||
t.Fatalf("loadFromLookup returned error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.PrimaryLLM.APIKey != "fallback-key" {
|
||||
t.Fatalf("expected OPENROUTER_API_KEY fallback, got %q", cfg.PrimaryLLM.APIKey)
|
||||
}
|
||||
if cfg.PrimaryLLM.Model != env["AUDITA_MODEL"] {
|
||||
t.Fatalf("unexpected model: %q", cfg.PrimaryLLM.Model)
|
||||
}
|
||||
if cfg.PrimaryLLM.BaseURL != env["AUDITA_BASE_URL"] {
|
||||
t.Fatalf("unexpected base url: %q", cfg.PrimaryLLM.BaseURL)
|
||||
}
|
||||
if cfg.TargetSections == nil || *cfg.TargetSections != 5 {
|
||||
t.Fatalf("unexpected target sections: %#v", cfg.TargetSections)
|
||||
}
|
||||
if cfg.TotalLLMConcurrency != 6 {
|
||||
t.Fatalf("unexpected total llm concurrency: %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 4 {
|
||||
t.Fatalf("unexpected proposal llm concurrency: %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
if cfg.ValidationLLMConcurrency == nil || *cfg.ValidationLLMConcurrency != 2 {
|
||||
t.Fatalf("unexpected validation llm concurrency: %#v", cfg.ValidationLLMConcurrency)
|
||||
}
|
||||
if cfg.PrimaryLLM.Concurrency != 6 {
|
||||
t.Fatalf("expected primary alias concurrency 6, got %d", cfg.PrimaryLLM.Concurrency)
|
||||
}
|
||||
if cfg.ValidationLLM.Concurrency == nil || *cfg.ValidationLLM.Concurrency != 2 {
|
||||
t.Fatalf("expected validation alias concurrency 2, got %#v", cfg.ValidationLLM.Concurrency)
|
||||
}
|
||||
if cfg.WorkDirRetention != WorkDirRetentionAlways {
|
||||
t.Fatalf("unexpected work dir retention: %q", cfg.WorkDirRetention)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromEnvLegacyLLMConcurrencyAliasForTotalAndProposal(t *testing.T) {
|
||||
env := map[string]string{
|
||||
"AUDITA_LLM_CONCURRENCY": "5",
|
||||
}
|
||||
|
||||
cfg, err := loadFromLookup(mapLookup(env))
|
||||
if err != nil {
|
||||
t.Fatalf("loadFromLookup returned error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.TotalLLMConcurrency != 5 {
|
||||
t.Fatalf("expected total concurrency from legacy alias, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 5 {
|
||||
t.Fatalf("expected proposal concurrency to inherit legacy total, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromEnvCanonicalTotalWinsLegacyAlias(t *testing.T) {
|
||||
env := map[string]string{
|
||||
"AUDITA_TOTAL_LLM_CONCURRENCY": "4",
|
||||
"AUDITA_LLM_CONCURRENCY": "9",
|
||||
}
|
||||
|
||||
cfg, err := loadFromLookup(mapLookup(env))
|
||||
if err != nil {
|
||||
t.Fatalf("loadFromLookup returned error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.TotalLLMConcurrency != 4 {
|
||||
t.Fatalf("expected canonical total to win over legacy alias, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 4 {
|
||||
t.Fatalf("expected proposal to inherit canonical total when unset, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromEnvUsesAuditaLLMAPIKeyOverFallback(t *testing.T) {
|
||||
env := map[string]string{
|
||||
"AUDITA_LLM_API_KEY": "primary-key",
|
||||
"OPENROUTER_API_KEY": "fallback-key",
|
||||
}
|
||||
|
||||
cfg, err := loadFromLookup(mapLookup(env))
|
||||
if err != nil {
|
||||
t.Fatalf("loadFromLookup returned error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.PrimaryLLM.APIKey != "primary-key" {
|
||||
t.Fatalf("expected AUDITA_LLM_API_KEY to win, got %q", cfg.PrimaryLLM.APIKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCLIOverridesPrecedence(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.PrimaryLLM.Model = "env-model"
|
||||
cfg.WorkDir = "/env/work"
|
||||
|
||||
model := "cli-model"
|
||||
workDir := "/cli/work"
|
||||
modules := "grammar"
|
||||
outputSchema := "audita-v1"
|
||||
totalLLMConcurrency := 5
|
||||
proposalLLMConcurrency := 3
|
||||
overrides := CLIOverrides{
|
||||
PrimaryModel: &model,
|
||||
WorkDir: &workDir,
|
||||
ModulesCSV: &modules,
|
||||
OutputSchema: &outputSchema,
|
||||
TotalLLMConcurrency: &totalLLMConcurrency,
|
||||
ProposalLLMConcurrency: &proposalLLMConcurrency,
|
||||
}
|
||||
|
||||
if err := cfg.ApplyCLIOverrides(overrides); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
|
||||
if cfg.PrimaryLLM.Model != "cli-model" {
|
||||
t.Fatalf("expected CLI model override, got %q", cfg.PrimaryLLM.Model)
|
||||
}
|
||||
if cfg.WorkDir != "/cli/work" {
|
||||
t.Fatalf("expected CLI work dir override, got %q", cfg.WorkDir)
|
||||
}
|
||||
if !reflect.DeepEqual(cfg.Modules, []string{"grammar"}) {
|
||||
t.Fatalf("unexpected modules: %#v", cfg.Modules)
|
||||
}
|
||||
if cfg.OutputSchema != "audita-v1" {
|
||||
t.Fatalf("expected CLI output schema override, got %q", cfg.OutputSchema)
|
||||
}
|
||||
if cfg.TotalLLMConcurrency != 5 {
|
||||
t.Fatalf("expected CLI total concurrency override, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 3 {
|
||||
t.Fatalf("expected CLI proposal concurrency override, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCLIOverridesTrimsTranscriptDescription(t *testing.T) {
|
||||
cfg := Default()
|
||||
description := " background context about speakers "
|
||||
if err := cfg.ApplyCLIOverrides(CLIOverrides{TranscriptDescription: &description}); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
if cfg.TranscriptDescription != "background context about speakers" {
|
||||
t.Fatalf("unexpected transcript description trim result: %q", cfg.TranscriptDescription)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationRejectsOverlyLongTranscriptDescription(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.TranscriptDescription = strings.Repeat("a", DefaultTranscriptDescriptionMaxChars+1)
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatalf("expected transcript description length validation error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "transcript description must be 500 characters or fewer") {
|
||||
t.Fatalf("unexpected validation error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCLIOverridesLegacyLLMConcurrencyAlias(t *testing.T) {
|
||||
cfg := Default()
|
||||
aliasConcurrency := 6
|
||||
|
||||
if err := cfg.ApplyCLIOverrides(CLIOverrides{PrimaryLLMConcurrency: &aliasConcurrency}); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
|
||||
if cfg.TotalLLMConcurrency != 6 {
|
||||
t.Fatalf("expected legacy --llm-concurrency alias to set total, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 6 {
|
||||
t.Fatalf("expected proposal to inherit aliased total when unset, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCLIOverridesCanonicalTotalWinsLegacyAlias(t *testing.T) {
|
||||
cfg := Default()
|
||||
canonicalTotal := 4
|
||||
legacyAlias := 9
|
||||
|
||||
if err := cfg.ApplyCLIOverrides(CLIOverrides{TotalLLMConcurrency: &canonicalTotal, PrimaryLLMConcurrency: &legacyAlias}); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
|
||||
if cfg.TotalLLMConcurrency != 4 {
|
||||
t.Fatalf("expected canonical total concurrency to win, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 4 {
|
||||
t.Fatalf("expected proposal to inherit canonical total when proposal is unset, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationFailures(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.OutputSchema = "unknown-schema"
|
||||
cfg.PrimaryLLM.TimeoutSeconds = -1
|
||||
cfg.TotalLLMConcurrency = 0
|
||||
cfg.ProposalLLMConcurrency = 0
|
||||
validationConcurrency := 5
|
||||
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||
cfg.ValidationMaxPromptTokens = 0
|
||||
cfg.MaxSectionTokens = 100
|
||||
cfg.MinSectionTokens = 200
|
||||
cfg.Thresholds.Grammar = 1.5
|
||||
cfg.WorkDirRetention = WorkDirRetention("sometimes")
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatalf("expected validation error")
|
||||
}
|
||||
|
||||
message := err.Error()
|
||||
for _, expected := range []string{
|
||||
"primary llm timeout seconds",
|
||||
"total llm concurrency",
|
||||
"proposal llm concurrency",
|
||||
"validation llm concurrency must be less than or equal to total llm concurrency",
|
||||
"validation max prompt tokens",
|
||||
"min section tokens",
|
||||
"grammar confidence threshold",
|
||||
"work dir retention",
|
||||
"unsupported output schema",
|
||||
} {
|
||||
if !strings.Contains(message, expected) {
|
||||
t.Fatalf("expected error to contain %q, got %q", expected, message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveValidationLLMInheritance(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.PrimaryLLM.APIKey = "primary-key"
|
||||
cfg.PrimaryLLM.Model = "primary-model"
|
||||
cfg.PrimaryLLM.BaseURL = "https://primary.example/v1"
|
||||
cfg.PrimaryLLM.TimeoutSeconds = 111
|
||||
cfg.PrimaryLLM.MaxRetries = 2
|
||||
cfg.TotalLLMConcurrency = 7
|
||||
cfg.syncLegacyConcurrencyAliases()
|
||||
|
||||
effective := cfg.EffectiveValidationLLMConfig()
|
||||
if effective.APIKey != "primary-key" || effective.Model != "primary-model" || effective.BaseURL != "https://primary.example/v1" || effective.TimeoutSeconds != 111 || effective.MaxRetries != 2 || effective.Concurrency != 7 {
|
||||
t.Fatalf("unexpected inherited config: %#v", effective)
|
||||
}
|
||||
|
||||
validationTimeout := 222
|
||||
validationRetries := 9
|
||||
cfg.ValidationLLM.APIKey = "validation-key"
|
||||
cfg.ValidationLLM.Model = "validation-model"
|
||||
cfg.ValidationLLM.BaseURL = "https://validation.example/v1"
|
||||
cfg.ValidationLLM.TimeoutSeconds = &validationTimeout
|
||||
cfg.ValidationLLM.MaxRetries = &validationRetries
|
||||
validationConcurrency := 4
|
||||
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||
cfg.syncLegacyConcurrencyAliases()
|
||||
|
||||
effective = cfg.EffectiveValidationLLMConfig()
|
||||
if effective.APIKey != "validation-key" || effective.Model != "validation-model" || effective.BaseURL != "https://validation.example/v1" || effective.TimeoutSeconds != 222 || effective.MaxRetries != 9 || effective.Concurrency != 4 {
|
||||
t.Fatalf("unexpected overridden validation config: %#v", effective)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationLLMConcurrencyCannotExceedTotal(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.TotalLLMConcurrency = 2
|
||||
cfg.ProposalLLMConcurrency = 2
|
||||
validationConcurrency := 3
|
||||
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Fatal("expected validation error when validation llm concurrency exceeds total")
|
||||
}
|
||||
|
||||
validationConcurrency = 2
|
||||
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("expected equal concurrency to validate, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProposalLLMConcurrencyCannotExceedTotal(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.TotalLLMConcurrency = 2
|
||||
cfg.ProposalLLMConcurrency = 3
|
||||
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Fatal("expected validation error when proposal llm concurrency exceeds total")
|
||||
}
|
||||
|
||||
cfg.ProposalLLMConcurrency = 2
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("expected equal concurrency to validate, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLITotalLLMConcurrencyOverrideDrivesEffectiveValidationConcurrencyWhenValidationUnset(t *testing.T) {
|
||||
cfg := Default()
|
||||
totalLLMConcurrency := 6
|
||||
if err := cfg.ApplyCLIOverrides(CLIOverrides{TotalLLMConcurrency: &totalLLMConcurrency}); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
if cfg.ValidationLLMConcurrency != nil {
|
||||
t.Fatalf("expected validation concurrency to remain unset, got %#v", cfg.ValidationLLMConcurrency)
|
||||
}
|
||||
if cfg.EffectiveValidationLLMConcurrency() != 6 {
|
||||
t.Fatalf("expected inherited validation concurrency 6, got %d", cfg.EffectiveValidationLLMConcurrency())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactedConfig(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.PrimaryLLM.APIKey = "secret-primary"
|
||||
cfg.ValidationLLM.APIKey = "secret-validation"
|
||||
|
||||
redacted := cfg.Redacted()
|
||||
|
||||
if redacted.PrimaryLLM.APIKey != redactedSecret {
|
||||
t.Fatalf("expected primary api key to be redacted, got %q", redacted.PrimaryLLM.APIKey)
|
||||
}
|
||||
if redacted.ValidationLLM.APIKey != redactedSecret {
|
||||
t.Fatalf("expected validation api key to be redacted, got %q", redacted.ValidationLLM.APIKey)
|
||||
}
|
||||
if cfg.PrimaryLLM.APIKey != "secret-primary" {
|
||||
t.Fatalf("redaction should not mutate original config")
|
||||
}
|
||||
}
|
||||
|
||||
func mapLookup(values map[string]string) func(string) (string, bool) {
|
||||
return func(key string) (string, bool) {
|
||||
value, ok := values[key]
|
||||
return value, ok
|
||||
}
|
||||
}
|
||||
268
internal/core/config/env.go
Normal file
268
internal/core/config/env.go
Normal file
@@ -0,0 +1,268 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultConfigPath = "/etc/audita/config.yml"
|
||||
DefaultConfigPathUsrLocal = "/usr/local/etc/audita/config.yml"
|
||||
)
|
||||
|
||||
var DefaultConfigSearchPaths = []string{
|
||||
DefaultConfigPathUsrLocal,
|
||||
DefaultConfigPath,
|
||||
}
|
||||
|
||||
func LoadFromEnv() (Config, error) {
|
||||
cfg := Default()
|
||||
if err := cfg.applyEnvOverrides(os.LookupEnv); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func loadFromLookup(lookup func(string) (string, bool)) (Config, error) {
|
||||
cfg := Default()
|
||||
if err := cfg.applyEnvOverrides(lookup); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func (c *Config) ApplyEnvOverrides() error {
|
||||
return c.applyEnvOverrides(os.LookupEnv)
|
||||
}
|
||||
|
||||
func (c *Config) applyEnvOverrides(lookup func(string) (string, bool)) error {
|
||||
if c == nil {
|
||||
return fmt.Errorf("config must not be nil")
|
||||
}
|
||||
|
||||
cfg := c
|
||||
if raw, ok := lookup("AUDITA_MODULES"); ok {
|
||||
modules, err := ParseModulesCSV(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_MODULES: %w", err)
|
||||
}
|
||||
cfg.Modules = modules
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_LLM_API_KEY"); ok {
|
||||
cfg.PrimaryLLM.APIKey = raw
|
||||
} else if raw, ok := lookup("OPENROUTER_API_KEY"); ok {
|
||||
cfg.PrimaryLLM.APIKey = raw
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_LLM_API_KEY"); ok {
|
||||
cfg.ValidationLLM.APIKey = raw
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_MODEL"); ok {
|
||||
cfg.PrimaryLLM.Model = raw
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_MODEL"); ok {
|
||||
cfg.ValidationLLM.Model = raw
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_BASE_URL"); ok {
|
||||
cfg.PrimaryLLM.BaseURL = raw
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_BASE_URL"); ok {
|
||||
cfg.ValidationLLM.BaseURL = raw
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_LLM_TIMEOUT_SECONDS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_LLM_TIMEOUT_SECONDS: %w", err)
|
||||
}
|
||||
cfg.PrimaryLLM.TimeoutSeconds = value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS: %w", err)
|
||||
}
|
||||
cfg.ValidationLLM.TimeoutSeconds = &value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_MAX_RETRIES"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_MAX_RETRIES: %w", err)
|
||||
}
|
||||
cfg.PrimaryLLM.MaxRetries = value
|
||||
}
|
||||
totalConcurrencySet := false
|
||||
if raw, ok := lookup("AUDITA_TOTAL_LLM_CONCURRENCY"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_TOTAL_LLM_CONCURRENCY: %w", err)
|
||||
}
|
||||
cfg.TotalLLMConcurrency = value
|
||||
totalConcurrencySet = true
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_LLM_CONCURRENCY"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_LLM_CONCURRENCY: %w", err)
|
||||
}
|
||||
if !totalConcurrencySet {
|
||||
cfg.TotalLLMConcurrency = value
|
||||
totalConcurrencySet = true
|
||||
}
|
||||
}
|
||||
|
||||
proposalConcurrencySet := false
|
||||
if raw, ok := lookup("AUDITA_PROPOSAL_LLM_CONCURRENCY"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_PROPOSAL_LLM_CONCURRENCY: %w", err)
|
||||
}
|
||||
cfg.ProposalLLMConcurrency = value
|
||||
proposalConcurrencySet = true
|
||||
}
|
||||
if totalConcurrencySet && !proposalConcurrencySet {
|
||||
cfg.ProposalLLMConcurrency = cfg.TotalLLMConcurrency
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_MAX_RETRIES"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_VALIDATION_MAX_RETRIES: %w", err)
|
||||
}
|
||||
cfg.ValidationLLM.MaxRetries = &value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_LLM_CONCURRENCY"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_VALIDATION_LLM_CONCURRENCY: %w", err)
|
||||
}
|
||||
cfg.ValidationLLMConcurrency = &value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_MAX_PROMPT_TOKENS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_VALIDATION_MAX_PROMPT_TOKENS: %w", err)
|
||||
}
|
||||
cfg.ValidationMaxPromptTokens = value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_MAX_SECTION_TOKENS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_MAX_SECTION_TOKENS: %w", err)
|
||||
}
|
||||
cfg.MaxSectionTokens = value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_MIN_SECTION_TOKENS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_MIN_SECTION_TOKENS: %w", err)
|
||||
}
|
||||
cfg.MinSectionTokens = value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_TARGET_SECTIONS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_TARGET_SECTIONS: %w", err)
|
||||
}
|
||||
cfg.TargetSections = &value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD: %w", err)
|
||||
}
|
||||
cfg.Thresholds.Glossary = value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD: %w", err)
|
||||
}
|
||||
cfg.Thresholds.Grammar = value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD: %w", err)
|
||||
}
|
||||
cfg.Thresholds.Homophones = value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD: %w", err)
|
||||
}
|
||||
cfg.Thresholds.SpokenWord = value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_GAP"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_GAP: %w", err)
|
||||
}
|
||||
cfg.Normalization.MaxSegmentGap = value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_NORMALIZE_ELLIPSIS_GAP"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_NORMALIZE_ELLIPSIS_GAP: %w", err)
|
||||
}
|
||||
cfg.Normalization.EllipsisGap = value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION: %w", err)
|
||||
}
|
||||
cfg.Normalization.MaxSegmentDuration = value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS: %w", err)
|
||||
}
|
||||
cfg.Normalization.MaxSegmentTokens = value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_WORK_DIR"); ok {
|
||||
cfg.WorkDir = raw
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_WORK_DIR_RETENTION"); ok {
|
||||
cfg.WorkDirRetention = WorkDirRetention(raw)
|
||||
}
|
||||
|
||||
cfg.syncLegacyConcurrencyAliases()
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseInt(raw string) (int, error) {
|
||||
value, err := strconv.Atoi(raw)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("must be an integer")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func parseFloat(raw string) (float64, error) {
|
||||
value, err := strconv.ParseFloat(raw, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("must be a number")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
342
internal/core/config/file_config.go
Normal file
342
internal/core/config/file_config.go
Normal file
@@ -0,0 +1,342 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const SupportedFileConfigVersion = 1
|
||||
|
||||
var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
type FileConfig struct {
|
||||
Version int `yaml:"version"`
|
||||
Pipeline *FileConfigPipeline `yaml:"pipeline,omitempty"`
|
||||
Output *FileConfigOutput `yaml:"output,omitempty"`
|
||||
LLM *FileConfigLLM `yaml:"llm,omitempty"`
|
||||
Concurrency *FileConfigConcurrency `yaml:"concurrency,omitempty"`
|
||||
Chunking *FileConfigChunking `yaml:"chunking,omitempty"`
|
||||
Normalization *FileConfigNormalization `yaml:"normalization,omitempty"`
|
||||
Thresholds *FileConfigThresholds `yaml:"thresholds,omitempty"`
|
||||
Context *FileConfigContext `yaml:"context,omitempty"`
|
||||
Diagnostics *FileConfigDiagnostics `yaml:"diagnostics,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigPipeline struct {
|
||||
Modules []string `yaml:"modules,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigOutput struct {
|
||||
Schema *string `yaml:"schema,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigLLM struct {
|
||||
Proposal *FileConfigLLMTarget `yaml:"proposal,omitempty"`
|
||||
Validation *FileConfigLLMTarget `yaml:"validation,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigLLMTarget struct {
|
||||
BaseURL *string `yaml:"base_url,omitempty"`
|
||||
Model *string `yaml:"model,omitempty"`
|
||||
APIKeyEnv *string `yaml:"api_key_env,omitempty"`
|
||||
Timeout *fileConfigDurationOrInt `yaml:"timeout,omitempty"`
|
||||
MaxRetries *int `yaml:"max_retries,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigConcurrency struct {
|
||||
TotalLLM *int `yaml:"total_llm,omitempty"`
|
||||
ProposalLLM *int `yaml:"proposal_llm,omitempty"`
|
||||
ValidationLLM *int `yaml:"validation_llm,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigChunking struct {
|
||||
TargetSections *int `yaml:"target_sections,omitempty"`
|
||||
MaxSectionTokens *int `yaml:"max_section_tokens,omitempty"`
|
||||
MinSectionTokens *int `yaml:"min_section_tokens,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigNormalization struct {
|
||||
MaxSegmentGap *fileConfigDurationOrFloat `yaml:"max_segment_gap,omitempty"`
|
||||
EllipsisGap *fileConfigDurationOrFloat `yaml:"ellipsis_gap,omitempty"`
|
||||
MaxSegmentDuration *fileConfigDurationOrFloat `yaml:"max_segment_duration,omitempty"`
|
||||
MaxSegmentTokens *int `yaml:"max_segment_tokens,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigThresholds struct {
|
||||
Glossary *float64 `yaml:"glossary,omitempty"`
|
||||
Homophones *float64 `yaml:"homophones,omitempty"`
|
||||
SpokenWord *float64 `yaml:"spoken_word,omitempty"`
|
||||
Grammar *float64 `yaml:"grammar,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigContext struct {
|
||||
Description *string `yaml:"description,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigDiagnostics struct {
|
||||
WorkDir *string `yaml:"work_dir,omitempty"`
|
||||
Retention *string `yaml:"retention,omitempty"`
|
||||
}
|
||||
|
||||
type fileConfigDurationOrInt struct {
|
||||
seconds int
|
||||
}
|
||||
|
||||
func (v *fileConfigDurationOrInt) UnmarshalYAML(node *yaml.Node) error {
|
||||
switch node.Kind {
|
||||
case yaml.ScalarNode:
|
||||
if node.Tag == "!!int" {
|
||||
var n int
|
||||
if err := node.Decode(&n); err != nil {
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
v.seconds = n
|
||||
return nil
|
||||
}
|
||||
|
||||
var s string
|
||||
if err := node.Decode(&s); err != nil {
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
d, err := time.ParseDuration(strings.TrimSpace(s))
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid duration %q", s)
|
||||
}
|
||||
if d <= 0 {
|
||||
v.seconds = int(d / time.Second)
|
||||
return nil
|
||||
}
|
||||
if d%time.Second != 0 {
|
||||
return fmt.Errorf("duration %q must resolve to whole seconds", s)
|
||||
}
|
||||
v.seconds = int(d / time.Second)
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
}
|
||||
|
||||
func (v fileConfigDurationOrInt) Seconds() int { return v.seconds }
|
||||
|
||||
type fileConfigDurationOrFloat struct {
|
||||
seconds float64
|
||||
}
|
||||
|
||||
func (v *fileConfigDurationOrFloat) UnmarshalYAML(node *yaml.Node) error {
|
||||
switch node.Kind {
|
||||
case yaml.ScalarNode:
|
||||
if node.Tag == "!!int" || node.Tag == "!!float" {
|
||||
var f float64
|
||||
if err := node.Decode(&f); err != nil {
|
||||
return fmt.Errorf("must be a numeric seconds value or duration string")
|
||||
}
|
||||
v.seconds = f
|
||||
return nil
|
||||
}
|
||||
var s string
|
||||
if err := node.Decode(&s); err != nil {
|
||||
return fmt.Errorf("must be a numeric seconds value or duration string")
|
||||
}
|
||||
d, err := time.ParseDuration(strings.TrimSpace(s))
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid duration %q", s)
|
||||
}
|
||||
v.seconds = d.Seconds()
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("must be a numeric seconds value or duration string")
|
||||
}
|
||||
}
|
||||
|
||||
func (v fileConfigDurationOrFloat) Seconds() float64 { return v.seconds }
|
||||
|
||||
func LoadFileConfig(path string) (FileConfig, error) {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return FileConfig{}, fmt.Errorf("read config file %q: %w", path, err)
|
||||
}
|
||||
cfg, err := ParseFileConfigYAML(b)
|
||||
if err != nil {
|
||||
return FileConfig{}, fmt.Errorf("parse config file %q: %w", path, err)
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func ParseFileConfigYAML(data []byte) (FileConfig, error) {
|
||||
var fileCfg FileConfig
|
||||
dec := yaml.NewDecoder(strings.NewReader(string(data)))
|
||||
dec.KnownFields(true)
|
||||
if err := dec.Decode(&fileCfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("decode yaml: %w", err)
|
||||
}
|
||||
if fileCfg.Version == 0 {
|
||||
return FileConfig{}, fmt.Errorf("config version is required")
|
||||
}
|
||||
if fileCfg.Version != SupportedFileConfigVersion {
|
||||
return FileConfig{}, fmt.Errorf("unsupported config version %d", fileCfg.Version)
|
||||
}
|
||||
return fileCfg, nil
|
||||
}
|
||||
|
||||
func (c *Config) ApplyFileConfig(fileCfg FileConfig) error {
|
||||
return c.applyFileConfigWithLookup(fileCfg, os.LookupEnv)
|
||||
}
|
||||
|
||||
func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
|
||||
if c == nil {
|
||||
return fmt.Errorf("config must not be nil")
|
||||
}
|
||||
if fileCfg.Version != SupportedFileConfigVersion {
|
||||
return fmt.Errorf("unsupported config version %d", fileCfg.Version)
|
||||
}
|
||||
|
||||
if fileCfg.Pipeline != nil && len(fileCfg.Pipeline.Modules) > 0 {
|
||||
c.Modules = append([]string(nil), fileCfg.Pipeline.Modules...)
|
||||
}
|
||||
if fileCfg.Output != nil && fileCfg.Output.Schema != nil {
|
||||
c.OutputSchema = strings.TrimSpace(*fileCfg.Output.Schema)
|
||||
}
|
||||
|
||||
if fileCfg.LLM != nil {
|
||||
if fileCfg.LLM.Proposal != nil {
|
||||
if fileCfg.LLM.Proposal.BaseURL != nil {
|
||||
c.PrimaryLLM.BaseURL = *fileCfg.LLM.Proposal.BaseURL
|
||||
}
|
||||
if fileCfg.LLM.Proposal.Model != nil {
|
||||
c.PrimaryLLM.Model = *fileCfg.LLM.Proposal.Model
|
||||
}
|
||||
if fileCfg.LLM.Proposal.Timeout != nil {
|
||||
c.PrimaryLLM.TimeoutSeconds = fileCfg.LLM.Proposal.Timeout.Seconds()
|
||||
}
|
||||
if fileCfg.LLM.Proposal.MaxRetries != nil {
|
||||
c.PrimaryLLM.MaxRetries = *fileCfg.LLM.Proposal.MaxRetries
|
||||
}
|
||||
if fileCfg.LLM.Proposal.APIKeyEnv != nil {
|
||||
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Proposal.APIKeyEnv, lookup)
|
||||
if err != nil {
|
||||
return fmt.Errorf("llm.proposal.api_key_env: %w", err)
|
||||
}
|
||||
c.PrimaryLLM.APIKey = apiKey
|
||||
}
|
||||
}
|
||||
if fileCfg.LLM.Validation != nil {
|
||||
if fileCfg.LLM.Validation.BaseURL != nil {
|
||||
c.ValidationLLM.BaseURL = *fileCfg.LLM.Validation.BaseURL
|
||||
}
|
||||
if fileCfg.LLM.Validation.Model != nil {
|
||||
c.ValidationLLM.Model = *fileCfg.LLM.Validation.Model
|
||||
}
|
||||
if fileCfg.LLM.Validation.Timeout != nil {
|
||||
v := fileCfg.LLM.Validation.Timeout.Seconds()
|
||||
c.ValidationLLM.TimeoutSeconds = &v
|
||||
}
|
||||
if fileCfg.LLM.Validation.MaxRetries != nil {
|
||||
v := *fileCfg.LLM.Validation.MaxRetries
|
||||
c.ValidationLLM.MaxRetries = &v
|
||||
}
|
||||
if fileCfg.LLM.Validation.APIKeyEnv != nil {
|
||||
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Validation.APIKeyEnv, lookup)
|
||||
if err != nil {
|
||||
return fmt.Errorf("llm.validation.api_key_env: %w", err)
|
||||
}
|
||||
c.ValidationLLM.APIKey = apiKey
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if fileCfg.Concurrency != nil {
|
||||
if fileCfg.Concurrency.TotalLLM != nil {
|
||||
c.TotalLLMConcurrency = *fileCfg.Concurrency.TotalLLM
|
||||
}
|
||||
if fileCfg.Concurrency.ProposalLLM != nil {
|
||||
c.ProposalLLMConcurrency = *fileCfg.Concurrency.ProposalLLM
|
||||
}
|
||||
if fileCfg.Concurrency.ValidationLLM != nil {
|
||||
v := *fileCfg.Concurrency.ValidationLLM
|
||||
c.ValidationLLMConcurrency = &v
|
||||
}
|
||||
}
|
||||
|
||||
if fileCfg.Chunking != nil {
|
||||
if fileCfg.Chunking.TargetSections != nil {
|
||||
v := *fileCfg.Chunking.TargetSections
|
||||
c.TargetSections = &v
|
||||
}
|
||||
if fileCfg.Chunking.MaxSectionTokens != nil {
|
||||
c.MaxSectionTokens = *fileCfg.Chunking.MaxSectionTokens
|
||||
}
|
||||
if fileCfg.Chunking.MinSectionTokens != nil {
|
||||
c.MinSectionTokens = *fileCfg.Chunking.MinSectionTokens
|
||||
}
|
||||
}
|
||||
|
||||
if fileCfg.Normalization != nil {
|
||||
if fileCfg.Normalization.MaxSegmentGap != nil {
|
||||
c.Normalization.MaxSegmentGap = fileCfg.Normalization.MaxSegmentGap.Seconds()
|
||||
}
|
||||
if fileCfg.Normalization.EllipsisGap != nil {
|
||||
c.Normalization.EllipsisGap = fileCfg.Normalization.EllipsisGap.Seconds()
|
||||
}
|
||||
if fileCfg.Normalization.MaxSegmentDuration != nil {
|
||||
c.Normalization.MaxSegmentDuration = fileCfg.Normalization.MaxSegmentDuration.Seconds()
|
||||
}
|
||||
if fileCfg.Normalization.MaxSegmentTokens != nil {
|
||||
c.Normalization.MaxSegmentTokens = *fileCfg.Normalization.MaxSegmentTokens
|
||||
}
|
||||
}
|
||||
|
||||
if fileCfg.Thresholds != nil {
|
||||
if fileCfg.Thresholds.Glossary != nil {
|
||||
c.Thresholds.Glossary = *fileCfg.Thresholds.Glossary
|
||||
}
|
||||
if fileCfg.Thresholds.Homophones != nil {
|
||||
c.Thresholds.Homophones = *fileCfg.Thresholds.Homophones
|
||||
}
|
||||
if fileCfg.Thresholds.SpokenWord != nil {
|
||||
c.Thresholds.SpokenWord = *fileCfg.Thresholds.SpokenWord
|
||||
}
|
||||
if fileCfg.Thresholds.Grammar != nil {
|
||||
c.Thresholds.Grammar = *fileCfg.Thresholds.Grammar
|
||||
}
|
||||
}
|
||||
|
||||
if fileCfg.Context != nil && fileCfg.Context.Description != nil {
|
||||
c.TranscriptDescription = strings.TrimSpace(*fileCfg.Context.Description)
|
||||
}
|
||||
|
||||
if fileCfg.Diagnostics != nil {
|
||||
if fileCfg.Diagnostics.WorkDir != nil {
|
||||
c.WorkDir = *fileCfg.Diagnostics.WorkDir
|
||||
}
|
||||
if fileCfg.Diagnostics.Retention != nil {
|
||||
c.WorkDirRetention = WorkDirRetention(*fileCfg.Diagnostics.Retention)
|
||||
}
|
||||
}
|
||||
|
||||
c.syncLegacyConcurrencyAliases()
|
||||
if err := c.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
|
||||
name := strings.TrimSpace(envName)
|
||||
if name == "" {
|
||||
return "", fmt.Errorf("must not be empty")
|
||||
}
|
||||
if !envVarNamePattern.MatchString(name) {
|
||||
return "", fmt.Errorf("must be an environment variable name")
|
||||
}
|
||||
if strings.Contains(name, string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("must be an environment variable name")
|
||||
}
|
||||
v, _ := lookup(name)
|
||||
return v, nil
|
||||
}
|
||||
290
internal/core/config/file_config_test.go
Normal file
290
internal/core/config/file_config_test.go
Normal file
@@ -0,0 +1,290 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseFileConfigYAMLValid(t *testing.T) {
|
||||
raw := `
|
||||
version: 1
|
||||
pipeline:
|
||||
modules: [glossary, homophones, grammar]
|
||||
output:
|
||||
schema: audita-v1
|
||||
llm:
|
||||
proposal:
|
||||
base_url: https://example.test/v1
|
||||
model: provider/model-a
|
||||
api_key_env: AUDITA_PROPOSAL_KEY
|
||||
timeout: 2m
|
||||
max_retries: 4
|
||||
validation:
|
||||
base_url: https://example.test/validation
|
||||
model: provider/model-b
|
||||
api_key_env: AUDITA_VALIDATION_KEY
|
||||
timeout: 45
|
||||
max_retries: 3
|
||||
concurrency:
|
||||
total_llm: 8
|
||||
proposal_llm: 4
|
||||
validation_llm: 2
|
||||
chunking:
|
||||
target_sections: 6
|
||||
max_section_tokens: 9000
|
||||
min_section_tokens: 3000
|
||||
normalization:
|
||||
max_segment_gap: 1.5s
|
||||
ellipsis_gap: 2
|
||||
max_segment_duration: 45s
|
||||
max_segment_tokens: 1500
|
||||
thresholds:
|
||||
glossary: 0.9
|
||||
homophones: 0.7
|
||||
spoken_word: 0.8
|
||||
grammar: 0.75
|
||||
context:
|
||||
description: " crowd scene with many proper nouns "
|
||||
diagnostics:
|
||||
work_dir: /tmp/audita-config
|
||||
retention: always
|
||||
`
|
||||
cfg, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML error: %v", err)
|
||||
}
|
||||
if cfg.Version != 1 {
|
||||
t.Fatalf("expected version 1, got %d", cfg.Version)
|
||||
}
|
||||
if cfg.Pipeline == nil || len(cfg.Pipeline.Modules) != 3 {
|
||||
t.Fatalf("unexpected pipeline modules: %#v", cfg.Pipeline)
|
||||
}
|
||||
if cfg.Output == nil || cfg.Output.Schema == nil || *cfg.Output.Schema != "audita-v1" {
|
||||
t.Fatalf("expected output schema audita-v1, got %#v", cfg.Output)
|
||||
}
|
||||
if cfg.LLM == nil || cfg.LLM.Proposal == nil || cfg.LLM.Validation == nil {
|
||||
t.Fatalf("expected llm proposal+validation blocks")
|
||||
}
|
||||
if cfg.LLM.Proposal.Timeout == nil || cfg.LLM.Proposal.Timeout.Seconds() != 120 {
|
||||
t.Fatalf("expected proposal timeout 120s, got %#v", cfg.LLM.Proposal.Timeout)
|
||||
}
|
||||
if cfg.LLM.Validation.Timeout == nil || cfg.LLM.Validation.Timeout.Seconds() != 45 {
|
||||
t.Fatalf("expected validation timeout 45s, got %#v", cfg.LLM.Validation.Timeout)
|
||||
}
|
||||
if cfg.Normalization == nil || cfg.Normalization.MaxSegmentGap == nil || cfg.Normalization.MaxSegmentGap.Seconds() != 1.5 {
|
||||
t.Fatalf("expected parsed duration for normalization max_segment_gap")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigYAMLRejectsUnknownField(t *testing.T) {
|
||||
raw := `
|
||||
version: 1
|
||||
pipeline:
|
||||
modules: [grammar]
|
||||
output:
|
||||
unknown: v1
|
||||
`
|
||||
_, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err == nil {
|
||||
t.Fatalf("expected unknown field error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "field unknown not found") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigYAMLRejectsMissingVersion(t *testing.T) {
|
||||
raw := `pipeline: {modules: [grammar]}`
|
||||
_, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err == nil {
|
||||
t.Fatalf("expected missing version error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "config version is required") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigYAMLRejectsUnsupportedVersion(t *testing.T) {
|
||||
raw := `version: 2`
|
||||
_, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err == nil {
|
||||
t.Fatalf("expected unsupported version error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "unsupported config version 2") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigParsesAndMergesFields(t *testing.T) {
|
||||
raw := `
|
||||
version: 1
|
||||
pipeline:
|
||||
modules: [spoken_word, grammar]
|
||||
output:
|
||||
schema: audita-v1
|
||||
llm:
|
||||
proposal:
|
||||
model: provider/new-proposal
|
||||
api_key_env: PROPOSAL_KEY_NAME
|
||||
timeout: 90s
|
||||
max_retries: 5
|
||||
validation:
|
||||
model: provider/new-validation
|
||||
api_key_env: VALIDATION_KEY_NAME
|
||||
timeout: 150
|
||||
max_retries: 6
|
||||
concurrency:
|
||||
total_llm: 7
|
||||
proposal_llm: 3
|
||||
validation_llm: 2
|
||||
chunking:
|
||||
target_sections: 9
|
||||
thresholds:
|
||||
glossary: 0.91
|
||||
homophones: 0.61
|
||||
spoken_word: 0.71
|
||||
grammar: 0.81
|
||||
diagnostics:
|
||||
retention: never
|
||||
`
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML error: %v", err)
|
||||
}
|
||||
|
||||
cfg := Default()
|
||||
lookup := func(name string) (string, bool) {
|
||||
switch name {
|
||||
case "PROPOSAL_KEY_NAME":
|
||||
return "proposal-secret", true
|
||||
case "VALIDATION_KEY_NAME":
|
||||
return "validation-secret", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
if err := cfg.applyFileConfigWithLookup(fileCfg, lookup); err != nil {
|
||||
t.Fatalf("applyFileConfigWithLookup error: %v", err)
|
||||
}
|
||||
if strings.Join(cfg.Modules, ",") != "spoken_word,grammar" {
|
||||
t.Fatalf("unexpected modules: %#v", cfg.Modules)
|
||||
}
|
||||
if cfg.OutputSchema != "audita-v1" {
|
||||
t.Fatalf("unexpected output schema: %q", cfg.OutputSchema)
|
||||
}
|
||||
if cfg.PrimaryLLM.Model != "provider/new-proposal" {
|
||||
t.Fatalf("unexpected proposal model: %q", cfg.PrimaryLLM.Model)
|
||||
}
|
||||
if cfg.PrimaryLLM.APIKey != "proposal-secret" {
|
||||
t.Fatalf("expected proposal key from api_key_env lookup, got %q", cfg.PrimaryLLM.APIKey)
|
||||
}
|
||||
if cfg.PrimaryLLM.TimeoutSeconds != 90 {
|
||||
t.Fatalf("unexpected proposal timeout: %d", cfg.PrimaryLLM.TimeoutSeconds)
|
||||
}
|
||||
if cfg.ValidationLLM.Model != "provider/new-validation" {
|
||||
t.Fatalf("unexpected validation model: %q", cfg.ValidationLLM.Model)
|
||||
}
|
||||
if cfg.ValidationLLM.APIKey != "validation-secret" {
|
||||
t.Fatalf("expected validation key from api_key_env lookup, got %q", cfg.ValidationLLM.APIKey)
|
||||
}
|
||||
if cfg.ValidationLLM.TimeoutSeconds == nil || *cfg.ValidationLLM.TimeoutSeconds != 150 {
|
||||
t.Fatalf("unexpected validation timeout: %#v", cfg.ValidationLLM.TimeoutSeconds)
|
||||
}
|
||||
if cfg.TotalLLMConcurrency != 7 || cfg.ProposalLLMConcurrency != 3 {
|
||||
t.Fatalf("unexpected llm concurrency values: total=%d proposal=%d", cfg.TotalLLMConcurrency, cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
if cfg.ValidationLLMConcurrency == nil || *cfg.ValidationLLMConcurrency != 2 {
|
||||
t.Fatalf("unexpected validation llm concurrency: %#v", cfg.ValidationLLMConcurrency)
|
||||
}
|
||||
if cfg.TargetSections == nil || *cfg.TargetSections != 9 {
|
||||
t.Fatalf("unexpected target sections: %#v", cfg.TargetSections)
|
||||
}
|
||||
if cfg.WorkDirRetention != WorkDirRetentionNever {
|
||||
t.Fatalf("unexpected retention: %q", cfg.WorkDirRetention)
|
||||
}
|
||||
if cfg.PrimaryLLM.Concurrency != 7 {
|
||||
t.Fatalf("expected legacy alias to sync, got %d", cfg.PrimaryLLM.Concurrency)
|
||||
}
|
||||
if cfg.ValidationLLM.Concurrency == nil || *cfg.ValidationLLM.Concurrency != 2 {
|
||||
t.Fatalf("expected validation alias to sync, got %#v", cfg.ValidationLLM.Concurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigContextDescriptionTrim(t *testing.T) {
|
||||
raw := `
|
||||
version: 1
|
||||
context:
|
||||
description: " scene context "
|
||||
`
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML error: %v", err)
|
||||
}
|
||||
cfg := Default()
|
||||
if err := cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{})); err != nil {
|
||||
t.Fatalf("applyFileConfigWithLookup error: %v", err)
|
||||
}
|
||||
if cfg.TranscriptDescription != "scene context" {
|
||||
t.Fatalf("unexpected transcript description: %q", cfg.TranscriptDescription)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigRejectsInvalidAPIKeyEnvName(t *testing.T) {
|
||||
raw := `
|
||||
version: 1
|
||||
llm:
|
||||
proposal:
|
||||
api_key_env: "not a var name"
|
||||
`
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML error: %v", err)
|
||||
}
|
||||
cfg := Default()
|
||||
err = cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{}))
|
||||
if err == nil {
|
||||
t.Fatalf("expected api_key_env validation error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "environment variable name") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigDurationParsingErrors(t *testing.T) {
|
||||
raw := `
|
||||
version: 1
|
||||
llm:
|
||||
proposal:
|
||||
timeout: "1.5s"
|
||||
`
|
||||
_, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err == nil {
|
||||
t.Fatalf("expected duration parse error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "whole seconds") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileConfigReadsFromPath(t *testing.T) {
|
||||
p := writeTempFileConfig(t, "version: 1\n")
|
||||
cfg, err := LoadFileConfig(p)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadFileConfig error: %v", err)
|
||||
}
|
||||
if cfg.Version != 1 {
|
||||
t.Fatalf("expected version 1, got %d", cfg.Version)
|
||||
}
|
||||
}
|
||||
|
||||
func writeTempFileConfig(t *testing.T, contents string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
path := dir + "/config.yaml"
|
||||
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
|
||||
t.Fatalf("write config file: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
159
internal/core/config/flags.go
Normal file
159
internal/core/config/flags.go
Normal file
@@ -0,0 +1,159 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type CLIOverrides struct {
|
||||
ModulesCSV *string
|
||||
OutputSchema *string
|
||||
PrimaryLLMAPIKey *string
|
||||
ValidationLLMAPIKey *string
|
||||
PrimaryModel *string
|
||||
ValidationModel *string
|
||||
PrimaryBaseURL *string
|
||||
ValidationBaseURL *string
|
||||
PrimaryLLMTimeoutSeconds *int
|
||||
TotalLLMConcurrency *int
|
||||
ProposalLLMConcurrency *int
|
||||
PrimaryLLMConcurrency *int
|
||||
ValidationLLMTimeoutSeconds *int
|
||||
MaxRetries *int
|
||||
ValidationMaxRetries *int
|
||||
ValidationLLMConcurrency *int
|
||||
ValidationMaxPromptTokens *int
|
||||
MaxSectionTokens *int
|
||||
MinSectionTokens *int
|
||||
TargetSections *int
|
||||
GlossaryConfidenceThreshold *float64
|
||||
GrammarConfidenceThreshold *float64
|
||||
HomophonesConfidenceThreshold *float64
|
||||
SpokenWordConfidenceThreshold *float64
|
||||
NormalizeMaxSegmentGap *float64
|
||||
NormalizeEllipsisGap *float64
|
||||
NormalizeMaxSegmentDuration *float64
|
||||
NormalizeMaxSegmentTokens *int
|
||||
TranscriptDescription *string
|
||||
WorkDir *string
|
||||
WorkDirRetention *string
|
||||
}
|
||||
|
||||
func (c *Config) ApplyCLIOverrides(overrides CLIOverrides) error {
|
||||
if overrides.ModulesCSV != nil {
|
||||
modules, err := ParseModulesCSV(*overrides.ModulesCSV)
|
||||
if err != nil {
|
||||
return fmt.Errorf("--modules: %w", err)
|
||||
}
|
||||
c.Modules = modules
|
||||
}
|
||||
if overrides.OutputSchema != nil {
|
||||
c.OutputSchema = strings.TrimSpace(*overrides.OutputSchema)
|
||||
}
|
||||
|
||||
if overrides.PrimaryLLMAPIKey != nil {
|
||||
c.PrimaryLLM.APIKey = *overrides.PrimaryLLMAPIKey
|
||||
}
|
||||
if overrides.ValidationLLMAPIKey != nil {
|
||||
c.ValidationLLM.APIKey = *overrides.ValidationLLMAPIKey
|
||||
}
|
||||
if overrides.PrimaryModel != nil {
|
||||
c.PrimaryLLM.Model = *overrides.PrimaryModel
|
||||
}
|
||||
if overrides.ValidationModel != nil {
|
||||
c.ValidationLLM.Model = *overrides.ValidationModel
|
||||
}
|
||||
if overrides.PrimaryBaseURL != nil {
|
||||
c.PrimaryLLM.BaseURL = *overrides.PrimaryBaseURL
|
||||
}
|
||||
if overrides.ValidationBaseURL != nil {
|
||||
c.ValidationLLM.BaseURL = *overrides.ValidationBaseURL
|
||||
}
|
||||
if overrides.PrimaryLLMTimeoutSeconds != nil {
|
||||
c.PrimaryLLM.TimeoutSeconds = *overrides.PrimaryLLMTimeoutSeconds
|
||||
}
|
||||
totalConcurrencySet := false
|
||||
if overrides.TotalLLMConcurrency != nil {
|
||||
c.TotalLLMConcurrency = *overrides.TotalLLMConcurrency
|
||||
totalConcurrencySet = true
|
||||
}
|
||||
// Backward-compatible alias: --llm-concurrency maps to total concurrency
|
||||
// only when --total-llm-concurrency is not set in the same CLI invocation.
|
||||
if overrides.PrimaryLLMConcurrency != nil && !totalConcurrencySet {
|
||||
c.TotalLLMConcurrency = *overrides.PrimaryLLMConcurrency
|
||||
totalConcurrencySet = true
|
||||
}
|
||||
proposalConcurrencySet := false
|
||||
if overrides.ProposalLLMConcurrency != nil {
|
||||
c.ProposalLLMConcurrency = *overrides.ProposalLLMConcurrency
|
||||
proposalConcurrencySet = true
|
||||
}
|
||||
if totalConcurrencySet && !proposalConcurrencySet {
|
||||
c.ProposalLLMConcurrency = c.TotalLLMConcurrency
|
||||
}
|
||||
if overrides.ValidationLLMTimeoutSeconds != nil {
|
||||
value := *overrides.ValidationLLMTimeoutSeconds
|
||||
c.ValidationLLM.TimeoutSeconds = &value
|
||||
}
|
||||
if overrides.MaxRetries != nil {
|
||||
c.PrimaryLLM.MaxRetries = *overrides.MaxRetries
|
||||
}
|
||||
if overrides.ValidationMaxRetries != nil {
|
||||
value := *overrides.ValidationMaxRetries
|
||||
c.ValidationLLM.MaxRetries = &value
|
||||
}
|
||||
if overrides.ValidationLLMConcurrency != nil {
|
||||
value := *overrides.ValidationLLMConcurrency
|
||||
c.ValidationLLMConcurrency = &value
|
||||
}
|
||||
if overrides.ValidationMaxPromptTokens != nil {
|
||||
c.ValidationMaxPromptTokens = *overrides.ValidationMaxPromptTokens
|
||||
}
|
||||
if overrides.MaxSectionTokens != nil {
|
||||
c.MaxSectionTokens = *overrides.MaxSectionTokens
|
||||
}
|
||||
if overrides.MinSectionTokens != nil {
|
||||
c.MinSectionTokens = *overrides.MinSectionTokens
|
||||
}
|
||||
if overrides.TargetSections != nil {
|
||||
value := *overrides.TargetSections
|
||||
c.TargetSections = &value
|
||||
}
|
||||
if overrides.GlossaryConfidenceThreshold != nil {
|
||||
c.Thresholds.Glossary = *overrides.GlossaryConfidenceThreshold
|
||||
}
|
||||
if overrides.GrammarConfidenceThreshold != nil {
|
||||
c.Thresholds.Grammar = *overrides.GrammarConfidenceThreshold
|
||||
}
|
||||
if overrides.HomophonesConfidenceThreshold != nil {
|
||||
c.Thresholds.Homophones = *overrides.HomophonesConfidenceThreshold
|
||||
}
|
||||
if overrides.SpokenWordConfidenceThreshold != nil {
|
||||
c.Thresholds.SpokenWord = *overrides.SpokenWordConfidenceThreshold
|
||||
}
|
||||
if overrides.NormalizeMaxSegmentGap != nil {
|
||||
c.Normalization.MaxSegmentGap = *overrides.NormalizeMaxSegmentGap
|
||||
}
|
||||
if overrides.NormalizeEllipsisGap != nil {
|
||||
c.Normalization.EllipsisGap = *overrides.NormalizeEllipsisGap
|
||||
}
|
||||
if overrides.NormalizeMaxSegmentDuration != nil {
|
||||
c.Normalization.MaxSegmentDuration = *overrides.NormalizeMaxSegmentDuration
|
||||
}
|
||||
if overrides.NormalizeMaxSegmentTokens != nil {
|
||||
c.Normalization.MaxSegmentTokens = *overrides.NormalizeMaxSegmentTokens
|
||||
}
|
||||
if overrides.TranscriptDescription != nil {
|
||||
c.TranscriptDescription = strings.TrimSpace(*overrides.TranscriptDescription)
|
||||
}
|
||||
if overrides.WorkDir != nil {
|
||||
c.WorkDir = *overrides.WorkDir
|
||||
}
|
||||
if overrides.WorkDirRetention != nil {
|
||||
c.WorkDirRetention = WorkDirRetention(*overrides.WorkDirRetention)
|
||||
}
|
||||
|
||||
c.syncLegacyConcurrencyAliases()
|
||||
|
||||
return c.Validate()
|
||||
}
|
||||
17
internal/core/config/redaction.go
Normal file
17
internal/core/config/redaction.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package config
|
||||
|
||||
const redactedSecret = "[REDACTED]"
|
||||
|
||||
func (c Config) Redacted() Config {
|
||||
redacted := c
|
||||
redacted.PrimaryLLM.APIKey = redactSecret(redacted.PrimaryLLM.APIKey)
|
||||
redacted.ValidationLLM.APIKey = redactSecret(redacted.ValidationLLM.APIKey)
|
||||
return redacted
|
||||
}
|
||||
|
||||
func redactSecret(value string) string {
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
return redactedSecret
|
||||
}
|
||||
133
internal/core/config/validation.go
Normal file
133
internal/core/config/validation.go
Normal file
@@ -0,0 +1,133 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (c Config) Validate() error {
|
||||
var issues []string
|
||||
|
||||
if len(c.Modules) == 0 {
|
||||
issues = append(issues, "modules must not be empty")
|
||||
}
|
||||
for _, module := range c.Modules {
|
||||
if strings.TrimSpace(module) == "" {
|
||||
issues = append(issues, "modules must not contain empty values")
|
||||
break
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(c.OutputSchema) == "" {
|
||||
issues = append(issues, "output schema must not be empty")
|
||||
} else {
|
||||
switch strings.TrimSpace(c.OutputSchema) {
|
||||
case "bare-segments", "audita-v1":
|
||||
default:
|
||||
issues = append(issues, fmt.Sprintf("unsupported output schema %q", c.OutputSchema))
|
||||
}
|
||||
}
|
||||
|
||||
if c.PrimaryLLM.TimeoutSeconds <= 0 {
|
||||
issues = append(issues, "primary llm timeout seconds must be greater than zero")
|
||||
}
|
||||
if c.PrimaryLLM.MaxRetries < 0 {
|
||||
issues = append(issues, "max retries must be zero or greater")
|
||||
}
|
||||
if c.TotalLLMConcurrency <= 0 {
|
||||
issues = append(issues, "total llm concurrency must be greater than zero")
|
||||
}
|
||||
if c.ProposalLLMConcurrency <= 0 {
|
||||
issues = append(issues, "proposal llm concurrency must be greater than zero")
|
||||
}
|
||||
if c.ProposalLLMConcurrency > c.TotalLLMConcurrency {
|
||||
issues = append(issues, "proposal llm concurrency must be less than or equal to total llm concurrency")
|
||||
}
|
||||
|
||||
if c.ValidationLLM.TimeoutSeconds != nil && *c.ValidationLLM.TimeoutSeconds <= 0 {
|
||||
issues = append(issues, "validation llm timeout seconds must be greater than zero")
|
||||
}
|
||||
if c.ValidationLLM.MaxRetries != nil && *c.ValidationLLM.MaxRetries < 0 {
|
||||
issues = append(issues, "validation max retries must be zero or greater")
|
||||
}
|
||||
if c.ValidationLLMConcurrency != nil && *c.ValidationLLMConcurrency <= 0 {
|
||||
issues = append(issues, "validation llm concurrency must be greater than zero")
|
||||
}
|
||||
if c.ValidationLLMConcurrency != nil && *c.ValidationLLMConcurrency > c.TotalLLMConcurrency {
|
||||
issues = append(issues, "validation llm concurrency must be less than or equal to total llm concurrency")
|
||||
}
|
||||
|
||||
if c.ValidationMaxPromptTokens <= 0 {
|
||||
issues = append(issues, "validation max prompt tokens must be greater than zero")
|
||||
}
|
||||
if c.MaxSectionTokens <= 0 {
|
||||
issues = append(issues, "max section tokens must be greater than zero")
|
||||
}
|
||||
if c.MinSectionTokens <= 0 {
|
||||
issues = append(issues, "min section tokens must be greater than zero")
|
||||
}
|
||||
if c.MinSectionTokens > c.MaxSectionTokens {
|
||||
issues = append(issues, "min section tokens must be less than or equal to max section tokens")
|
||||
}
|
||||
if c.TargetSections != nil && *c.TargetSections <= 0 {
|
||||
issues = append(issues, "target sections must be greater than zero when set")
|
||||
}
|
||||
|
||||
if err := validateConfidence("glossary", c.Thresholds.Glossary); err != nil {
|
||||
issues = append(issues, err.Error())
|
||||
}
|
||||
if err := validateConfidence("grammar", c.Thresholds.Grammar); err != nil {
|
||||
issues = append(issues, err.Error())
|
||||
}
|
||||
if err := validateConfidence("homophones", c.Thresholds.Homophones); err != nil {
|
||||
issues = append(issues, err.Error())
|
||||
}
|
||||
if err := validateConfidence("spoken-word", c.Thresholds.SpokenWord); err != nil {
|
||||
issues = append(issues, err.Error())
|
||||
}
|
||||
|
||||
if c.Normalization.MaxSegmentGap < 0 {
|
||||
issues = append(issues, "normalize max segment gap must be zero or greater")
|
||||
}
|
||||
if c.Normalization.EllipsisGap < 0 {
|
||||
issues = append(issues, "normalize ellipsis gap must be zero or greater")
|
||||
}
|
||||
if c.Normalization.MaxSegmentDuration <= 0 {
|
||||
issues = append(issues, "normalize max segment duration must be greater than zero")
|
||||
}
|
||||
if c.Normalization.MaxSegmentTokens <= 0 {
|
||||
issues = append(issues, "normalize max segment tokens must be greater than zero")
|
||||
}
|
||||
if len(strings.TrimSpace(c.TranscriptDescription)) > DefaultTranscriptDescriptionMaxChars {
|
||||
issues = append(issues, fmt.Sprintf("transcript description must be %d characters or fewer", DefaultTranscriptDescriptionMaxChars))
|
||||
}
|
||||
|
||||
if strings.TrimSpace(c.WorkDir) == "" {
|
||||
issues = append(issues, "work dir must not be empty")
|
||||
}
|
||||
|
||||
if err := validateRetention(c.WorkDirRetention); err != nil {
|
||||
issues = append(issues, err.Error())
|
||||
}
|
||||
|
||||
if len(issues) > 0 {
|
||||
return fmt.Errorf("invalid config: %s", strings.Join(issues, "; "))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateConfidence(name string, threshold float64) error {
|
||||
if threshold < 0.0 || threshold > 1.0 {
|
||||
return fmt.Errorf("%s confidence threshold must be between 0.0 and 1.0", name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateRetention(retention WorkDirRetention) error {
|
||||
switch retention {
|
||||
case WorkDirRetentionAuto, WorkDirRetentionAlways, WorkDirRetentionNever:
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("work dir retention must be one of: auto, always, never")
|
||||
}
|
||||
}
|
||||
72
internal/core/diagnostics/retention_test.go
Normal file
72
internal/core/diagnostics/retention_test.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestShouldRetainRunDirectoryMatrix(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input RetentionDecisionInput
|
||||
want bool
|
||||
}{
|
||||
{name: "always success keeps", input: RetentionDecisionInput{RetentionMode: "always", RunSucceeded: true}, want: true},
|
||||
{name: "always failure keeps", input: RetentionDecisionInput{RetentionMode: "always", RunSucceeded: false}, want: true},
|
||||
{name: "never success keeps", input: RetentionDecisionInput{RetentionMode: "never", RunSucceeded: true}, want: true},
|
||||
{name: "never failure keeps", input: RetentionDecisionInput{RetentionMode: "never", RunSucceeded: false}, want: true},
|
||||
{name: "auto success no skips removes", input: RetentionDecisionInput{RetentionMode: "auto", RunSucceeded: true, HasSkippedCorrections: false}, want: false},
|
||||
{name: "auto success skips keeps", input: RetentionDecisionInput{RetentionMode: "auto", RunSucceeded: true, HasSkippedCorrections: true}, want: true},
|
||||
{name: "auto failure keeps", input: RetentionDecisionInput{RetentionMode: "auto", RunSucceeded: false}, want: true},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := ShouldRetainRunDirectory(tc.input)
|
||||
if got != tc.want {
|
||||
t.Fatalf("unexpected retain decision: got=%v want=%v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyRetentionRemovesWhenDecisionSaysRemove(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
runPath := filepath.Join(workDir, "run-test")
|
||||
if err := os.Mkdir(runPath, 0o755); err != nil {
|
||||
t.Fatalf("mkdir run path: %v", err)
|
||||
}
|
||||
|
||||
runDir := &RunDirectory{path: runPath, retention: "auto"}
|
||||
err := runDir.ApplyRetention(RetentionDecisionInput{RunSucceeded: true, HasSkippedCorrections: false})
|
||||
if err != nil {
|
||||
t.Fatalf("ApplyRetention failed: %v", err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(runPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected run directory removed, stat err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyRetentionReturnsRemovalError(t *testing.T) {
|
||||
parent := t.TempDir()
|
||||
runPath := filepath.Join(parent, "run-test")
|
||||
if err := os.Mkdir(runPath, 0o755); err != nil {
|
||||
t.Fatalf("mkdir run path: %v", err)
|
||||
}
|
||||
|
||||
// Make parent non-writable so removing child fails.
|
||||
if err := os.Chmod(parent, 0o500); err != nil {
|
||||
t.Fatalf("chmod parent: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = os.Chmod(parent, 0o700)
|
||||
})
|
||||
|
||||
runDir := &RunDirectory{path: runPath, retention: "auto"}
|
||||
err := runDir.ApplyRetention(RetentionDecisionInput{RunSucceeded: true, HasSkippedCorrections: false})
|
||||
if err == nil {
|
||||
t.Fatalf("expected removal error, got nil")
|
||||
}
|
||||
}
|
||||
242
internal/core/diagnostics/run_dir.go
Normal file
242
internal/core/diagnostics/run_dir.go
Normal file
@@ -0,0 +1,242 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
// RunDirectory represents a per-run diagnostics directory
|
||||
type RunDirectory struct {
|
||||
path string
|
||||
retention string
|
||||
createdAt time.Time
|
||||
}
|
||||
|
||||
type RetentionDecisionInput struct {
|
||||
RetentionMode string
|
||||
RunSucceeded bool
|
||||
HasSkippedCorrections bool
|
||||
}
|
||||
|
||||
func ShouldRetainRunDirectory(input RetentionDecisionInput) bool {
|
||||
// Failed runs are always retained.
|
||||
if !input.RunSucceeded {
|
||||
return true
|
||||
}
|
||||
|
||||
switch input.RetentionMode {
|
||||
case "always":
|
||||
return true
|
||||
case "never":
|
||||
return true
|
||||
case "auto":
|
||||
return input.HasSkippedCorrections
|
||||
default:
|
||||
// Be conservative for unknown values.
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// InvocationMetadata captures non-secret invocation details for diagnostics.
|
||||
type InvocationMetadata struct {
|
||||
Operation string `json:"operation"`
|
||||
TranscriptPath string `json:"transcript_path"`
|
||||
GlossaryPath string `json:"glossary_path"`
|
||||
OutputPath string `json:"output_path,omitempty"`
|
||||
ReportJSONPath string `json:"report_json_path,omitempty"`
|
||||
ConfigPath string `json:"config_path,omitempty"`
|
||||
ConfigSource string `json:"config_source,omitempty"`
|
||||
ConfigVersion *int `json:"config_version,omitempty"`
|
||||
TranscriptDescription string `json:"transcript_description,omitempty"`
|
||||
Modules []string `json:"modules"`
|
||||
RunID string `json:"run_id"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
}
|
||||
|
||||
// NewRunDirectory creates a new run directory under the configured work dir
|
||||
func NewRunDirectory(workDir, retention string) (*RunDirectory, error) {
|
||||
if workDir == "" {
|
||||
workDir = ".audita-runs"
|
||||
}
|
||||
|
||||
// Create work directory if it doesn't exist
|
||||
if err := os.MkdirAll(workDir, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create work directory %q: %w", workDir, err)
|
||||
}
|
||||
|
||||
// Create a unique run directory identifier.
|
||||
runID := fmt.Sprintf("run-%d", time.Now().UTC().UnixNano())
|
||||
runPath := filepath.Join(workDir, runID)
|
||||
|
||||
if err := os.Mkdir(runPath, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create run directory %q: %w", runPath, err)
|
||||
}
|
||||
|
||||
return &RunDirectory{
|
||||
path: runPath,
|
||||
retention: retention,
|
||||
createdAt: time.Now().UTC(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Path returns the run directory path
|
||||
func (r *RunDirectory) Path() string {
|
||||
return r.path
|
||||
}
|
||||
|
||||
func (r *RunDirectory) runID() string {
|
||||
return filepath.Base(r.path)
|
||||
}
|
||||
|
||||
// WriteInvocationMetadata writes invocation metadata for this run.
|
||||
func (r *RunDirectory) WriteInvocationMetadata(metadata InvocationMetadata) error {
|
||||
if metadata.RunID == "" {
|
||||
metadata.RunID = r.runID()
|
||||
}
|
||||
if metadata.StartedAt.IsZero() {
|
||||
metadata.StartedAt = r.createdAt
|
||||
}
|
||||
|
||||
path := filepath.Join(r.path, "invocation.json")
|
||||
bytes, err := json.MarshalIndent(metadata, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal invocation metadata: %w", err)
|
||||
}
|
||||
bytes = append(bytes, '\n')
|
||||
if err := os.WriteFile(path, bytes, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write invocation metadata: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteEffectiveConfig writes redacted effective config metadata for this run.
|
||||
func (r *RunDirectory) WriteEffectiveConfig(cfg config.Config) error {
|
||||
path := filepath.Join(r.path, "effective-config.json")
|
||||
redacted := cfg.Redacted()
|
||||
bytes, err := json.MarshalIndent(redacted, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal effective config: %w", err)
|
||||
}
|
||||
bytes = append(bytes, '\n')
|
||||
if err := os.WriteFile(path, bytes, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write effective config: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteSourceTranscript writes the source transcript artifact
|
||||
func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript, raw []byte) error {
|
||||
// Write raw source for reference
|
||||
sourcePath := filepath.Join(r.path, "source-transcript.json")
|
||||
if err := os.WriteFile(sourcePath, raw, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write source transcript: %w", err)
|
||||
}
|
||||
|
||||
// Write parsed source for debugging
|
||||
parsedPath := filepath.Join(r.path, "source-transcript-parsed.json")
|
||||
parsedBytes, err := json.MarshalIndent(transcript, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal parsed source transcript: %w", err)
|
||||
}
|
||||
parsedBytes = append(parsedBytes, '\n')
|
||||
if err := os.WriteFile(parsedPath, parsedBytes, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write parsed source transcript: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteNormalizedTranscript writes the normalized transcript artifact
|
||||
func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript) error {
|
||||
normalizedPath := filepath.Join(r.path, "normalized-transcript.json")
|
||||
bytes, err := schema.TranscriptToJSON(transcript)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to serialize normalized transcript: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(normalizedPath, bytes, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write normalized transcript: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteNormalizationSummary writes the normalization summary artifact
|
||||
func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.NormalizationSummary) error {
|
||||
summaryPath := filepath.Join(r.path, "normalization-summary.json")
|
||||
bytes, err := json.MarshalIndent(summary, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal normalization summary: %w", err)
|
||||
}
|
||||
bytes = append(bytes, '\n')
|
||||
if err := os.WriteFile(summaryPath, bytes, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write normalization summary: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteReport writes the authoritative report artifact
|
||||
func (r *RunDirectory) WriteReport(report reporting.ProcessReport) error {
|
||||
reportPath := filepath.Join(r.path, "report.json")
|
||||
bytes, err := json.MarshalIndent(report, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal report: %w", err)
|
||||
}
|
||||
bytes = append(bytes, '\n')
|
||||
if err := os.WriteFile(reportPath, bytes, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write report: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteErrorLog writes an error log on failure
|
||||
func (r *RunDirectory) WriteErrorLog(errorMessage string) error {
|
||||
errorPath := filepath.Join(r.path, "error.log")
|
||||
return os.WriteFile(errorPath, []byte(errorMessage+"\n"), 0o644)
|
||||
}
|
||||
|
||||
// WriteChunkingSummary writes the chunking summary artifact
|
||||
func (r *RunDirectory) WriteChunkingSummary(summary *chunking.DetailedSummary) error {
|
||||
summaryPath := filepath.Join(r.path, "chunking-summary.json")
|
||||
bytes, err := json.MarshalIndent(summary, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal chunking summary: %w", err)
|
||||
}
|
||||
bytes = append(bytes, '\n')
|
||||
if err := os.WriteFile(summaryPath, bytes, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write chunking summary: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *RunDirectory) WriteJSONArtifact(name string, payload any) error {
|
||||
artifactPath := filepath.Join(r.path, name)
|
||||
bytes, err := json.MarshalIndent(payload, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal %s: %w", name, err)
|
||||
}
|
||||
bytes = append(bytes, '\n')
|
||||
if err := os.WriteFile(artifactPath, bytes, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write %s: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *RunDirectory) ApplyRetention(input RetentionDecisionInput) error {
|
||||
decision := input
|
||||
if decision.RetentionMode == "" {
|
||||
decision.RetentionMode = r.retention
|
||||
}
|
||||
|
||||
if ShouldRetainRunDirectory(decision) {
|
||||
return nil
|
||||
}
|
||||
return os.RemoveAll(r.path)
|
||||
}
|
||||
29
internal/core/io/files.go
Normal file
29
internal/core/io/files.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package io
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func ReadRequiredFile(path string, label string) ([]byte, error) {
|
||||
contents, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read %s file %q: %w", label, path, err)
|
||||
}
|
||||
return contents, nil
|
||||
}
|
||||
|
||||
func ValidateWellFormedJSON(path string, raw []byte) error {
|
||||
if !json.Valid(raw) {
|
||||
return fmt.Errorf("transcript file %q is not valid JSON", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func WriteFile(path string, contents []byte) error {
|
||||
if err := os.WriteFile(path, contents, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write output file %q: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
206
internal/core/normalization/normalize.go
Normal file
206
internal/core/normalization/normalize.go
Normal file
@@ -0,0 +1,206 @@
|
||||
package normalization
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
// NormalizationConfig holds configuration parameters for transcript normalization
|
||||
type NormalizationConfig struct {
|
||||
MaxSegmentGap float64 // Maximum gap between segments to consider for merging (seconds)
|
||||
EllipsisGap float64 // Gap threshold above which to insert ellipsis (seconds)
|
||||
MaxSegmentDuration float64 // Maximum duration for a merged segment (seconds)
|
||||
MaxSegmentTokens int // Maximum token estimate for a merged segment
|
||||
}
|
||||
|
||||
// NormalizationSummary records the results and statistics of normalization
|
||||
type NormalizationSummary struct {
|
||||
InputSegmentCount int `json:"input_segment_count"`
|
||||
OutputSegmentCount int `json:"output_segment_count"`
|
||||
MergesPerformed int `json:"merges_performed"`
|
||||
IDsReassigned int `json:"ids_reassigned"`
|
||||
SkippedMerges struct {
|
||||
DifferentSpeakers int `json:"different_speakers"`
|
||||
GapTooLarge int `json:"gap_too_large"`
|
||||
DurationExceeded int `json:"duration_exceeded"`
|
||||
TokenLimitExceeded int `json:"token_limit_exceeded"`
|
||||
} `json:"skipped_merges"`
|
||||
}
|
||||
|
||||
// NormalizeTranscript performs deterministic normalization on a transcript
|
||||
type NormalizeTranscript struct {
|
||||
config NormalizationConfig
|
||||
estimator *SimpleTokenEstimator
|
||||
}
|
||||
|
||||
// NewNormalizer creates a new transcript normalizer with the given configuration
|
||||
func NewNormalizer(config NormalizationConfig) *NormalizeTranscript {
|
||||
return &NormalizeTranscript{
|
||||
config: config,
|
||||
estimator: &SimpleTokenEstimator{},
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize performs deterministic normalization on the given transcript
|
||||
func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.Transcript, *NormalizationSummary) {
|
||||
summary := &NormalizationSummary{
|
||||
InputSegmentCount: len(transcript.Segments),
|
||||
}
|
||||
|
||||
if len(transcript.Segments) == 0 {
|
||||
return transcript, summary
|
||||
}
|
||||
|
||||
// Sort segments chronologically
|
||||
sortedSegments := make([]schema.Segment, len(transcript.Segments))
|
||||
copy(sortedSegments, transcript.Segments)
|
||||
sort.Slice(sortedSegments, func(i, j int) bool {
|
||||
return sortedSegments[i].Start < sortedSegments[j].Start
|
||||
})
|
||||
|
||||
// Merge same-speaker adjacent segments
|
||||
var normalizedSegments []schema.Segment
|
||||
var currentSegment schema.Segment
|
||||
|
||||
for i, segment := range sortedSegments {
|
||||
if i == 0 {
|
||||
// Initialize with first segment
|
||||
currentSegment = segment
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if we should merge with current segment
|
||||
gap := segment.Start - currentSegment.End
|
||||
merge, reason := shouldMergeWithReason(n.config, n.estimator, ¤tSegment, &segment, gap)
|
||||
if merge {
|
||||
summary.MergesPerformed++
|
||||
currentSegment = mergeSegments(¤tSegment, &segment, gap, n.config.EllipsisGap)
|
||||
} else {
|
||||
// Track the reason for not merging
|
||||
switch reason {
|
||||
case "different_speakers":
|
||||
summary.SkippedMerges.DifferentSpeakers++
|
||||
case "gap_too_large":
|
||||
summary.SkippedMerges.GapTooLarge++
|
||||
case "duration_exceeded":
|
||||
summary.SkippedMerges.DurationExceeded++
|
||||
case "token_limit_exceeded":
|
||||
summary.SkippedMerges.TokenLimitExceeded++
|
||||
}
|
||||
// Finalize current segment and start new one
|
||||
normalizedSegments = append(normalizedSegments, currentSegment)
|
||||
currentSegment = segment
|
||||
}
|
||||
}
|
||||
|
||||
// Add the last segment
|
||||
if len(currentSegment.Text) > 0 {
|
||||
normalizedSegments = append(normalizedSegments, currentSegment)
|
||||
}
|
||||
|
||||
// Reassign sequential IDs starting at 1
|
||||
for i, segment := range normalizedSegments {
|
||||
if segment.ID != i+1 {
|
||||
summary.IDsReassigned++
|
||||
}
|
||||
normalizedSegments[i].ID = i + 1
|
||||
}
|
||||
|
||||
summary.OutputSegmentCount = len(normalizedSegments)
|
||||
|
||||
return &schema.Transcript{Segments: normalizedSegments}, summary
|
||||
}
|
||||
|
||||
func shouldMerge(config NormalizationConfig, estimator *SimpleTokenEstimator,
|
||||
current, next *schema.Segment, gap float64) bool {
|
||||
|
||||
// Don't merge if different speakers
|
||||
if current.Speaker != next.Speaker {
|
||||
return false
|
||||
}
|
||||
|
||||
// Don't merge if gap is too large
|
||||
if gap > config.MaxSegmentGap {
|
||||
return false
|
||||
}
|
||||
|
||||
// Calculate what the merged segment would look like
|
||||
mergedText := current.Text
|
||||
if gap >= config.EllipsisGap {
|
||||
mergedText += "... "
|
||||
} else {
|
||||
mergedText += " "
|
||||
}
|
||||
mergedText += next.Text
|
||||
|
||||
mergedDuration := next.End - current.Start
|
||||
|
||||
// Don't merge if duration would be exceeded
|
||||
if mergedDuration > config.MaxSegmentDuration {
|
||||
return false
|
||||
}
|
||||
|
||||
// Don't merge if token estimate would be exceeded
|
||||
tokenEstimate := estimator.EstimateTokens(mergedText)
|
||||
if tokenEstimate > config.MaxSegmentTokens {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func shouldMergeWithReason(config NormalizationConfig, estimator *SimpleTokenEstimator,
|
||||
current, next *schema.Segment, gap float64) (bool, string) {
|
||||
|
||||
if current.Speaker != next.Speaker {
|
||||
return false, "different_speakers"
|
||||
}
|
||||
if gap > config.MaxSegmentGap {
|
||||
return false, "gap_too_large"
|
||||
}
|
||||
|
||||
mergedText := current.Text
|
||||
if gap >= config.EllipsisGap {
|
||||
mergedText += "... "
|
||||
} else {
|
||||
mergedText += " "
|
||||
}
|
||||
mergedText += next.Text
|
||||
|
||||
mergedDuration := next.End - current.Start
|
||||
if mergedDuration > config.MaxSegmentDuration {
|
||||
return false, "duration_exceeded"
|
||||
}
|
||||
|
||||
tokenEstimate := estimator.EstimateTokens(mergedText)
|
||||
if tokenEstimate > config.MaxSegmentTokens {
|
||||
return false, "token_limit_exceeded"
|
||||
}
|
||||
|
||||
return true, ""
|
||||
}
|
||||
|
||||
func mergeSegments(current, next *schema.Segment, gap, ellipsisGap float64) schema.Segment {
|
||||
mergedText := current.Text
|
||||
if gap >= ellipsisGap {
|
||||
mergedText += "... "
|
||||
} else {
|
||||
mergedText += " "
|
||||
}
|
||||
mergedText += next.Text
|
||||
|
||||
// Merge categories
|
||||
categories := make([]string, 0, len(current.Categories)+len(next.Categories))
|
||||
categories = append(categories, current.Categories...)
|
||||
categories = append(categories, next.Categories...)
|
||||
|
||||
return schema.Segment{
|
||||
ID: current.ID, // Will be reassigned later
|
||||
Speaker: current.Speaker,
|
||||
Start: current.Start,
|
||||
End: next.End,
|
||||
Text: mergedText,
|
||||
Categories: categories,
|
||||
}
|
||||
}
|
||||
684
internal/core/normalization/normalize_test.go
Normal file
684
internal/core/normalization/normalize_test.go
Normal file
@@ -0,0 +1,684 @@
|
||||
package normalization
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
func TestNormalizationNoOp(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
|
||||
{ID: 2, Speaker: "Bob", Start: 2.0, End: 3.0, Text: "Hi"},
|
||||
},
|
||||
}
|
||||
|
||||
normalized, summary := normalizer.Normalize(transcript)
|
||||
|
||||
// Should be no-op since different speakers
|
||||
if len(normalized.Segments) != 2 {
|
||||
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
|
||||
}
|
||||
if summary.InputSegmentCount != 2 {
|
||||
t.Errorf("expected input count 2, got %d", summary.InputSegmentCount)
|
||||
}
|
||||
if summary.OutputSegmentCount != 2 {
|
||||
t.Errorf("expected output count 2, got %d", summary.OutputSegmentCount)
|
||||
}
|
||||
if summary.MergesPerformed != 0 {
|
||||
t.Errorf("expected 0 merges, got %d", summary.MergesPerformed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationSameSpeakerMergeWithinGap(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
|
||||
{ID: 2, Speaker: "Alice", Start: 1.5, End: 2.5, Text: "world"},
|
||||
},
|
||||
}
|
||||
|
||||
normalized, summary := normalizer.Normalize(transcript)
|
||||
|
||||
// Should merge since same speaker and small gap
|
||||
if len(normalized.Segments) != 1 {
|
||||
t.Errorf("expected 1 merged segment, got %d", len(normalized.Segments))
|
||||
}
|
||||
if summary.MergesPerformed != 1 {
|
||||
t.Errorf("expected 1 merge, got %d", summary.MergesPerformed)
|
||||
}
|
||||
if !strings.Contains(normalized.Segments[0].Text, "Hello") || !strings.Contains(normalized.Segments[0].Text, "world") {
|
||||
t.Errorf("expected merged text to contain both parts, got %q", normalized.Segments[0].Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationEllipsisInsertionAboveThreshold(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 0.5,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
|
||||
{ID: 2, Speaker: "Alice", Start: 2.0, End: 3.0, Text: "world"},
|
||||
},
|
||||
}
|
||||
|
||||
normalized, _ := normalizer.Normalize(transcript)
|
||||
|
||||
// Should merge with ellipsis since gap (1.0) > ellipsis threshold (0.5)
|
||||
if len(normalized.Segments) != 1 {
|
||||
t.Errorf("expected 1 merged segment, got %d", len(normalized.Segments))
|
||||
}
|
||||
if !strings.Contains(normalized.Segments[0].Text, "...") {
|
||||
t.Errorf("expected ellipsis in merged text, got %q", normalized.Segments[0].Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationNoMergeDifferentSpeakers(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
|
||||
{ID: 2, Speaker: "Bob", Start: 1.5, End: 2.5, Text: "Hi"},
|
||||
},
|
||||
}
|
||||
|
||||
normalized, summary := normalizer.Normalize(transcript)
|
||||
|
||||
// Should NOT merge since different speakers
|
||||
if len(normalized.Segments) != 2 {
|
||||
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
|
||||
}
|
||||
if summary.SkippedMerges.DifferentSpeakers != 1 {
|
||||
t.Errorf("expected 1 skipped merge for different speakers, got %d", summary.SkippedMerges.DifferentSpeakers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationNoMergeWhenMaxDurationExceeded(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 1.0, // Very small max duration
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "Alice", Start: 0.0, End: 0.5, Text: "Hello"},
|
||||
{ID: 2, Speaker: "Alice", Start: 0.6, End: 1.5, Text: "world"},
|
||||
},
|
||||
}
|
||||
|
||||
normalized, summary := normalizer.Normalize(transcript)
|
||||
|
||||
// Should NOT merge since merged duration (1.5) > max duration (1.0)
|
||||
if len(normalized.Segments) != 2 {
|
||||
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
|
||||
}
|
||||
if summary.SkippedMerges.DurationExceeded != 1 {
|
||||
t.Errorf("expected 1 skipped merge for duration exceeded, got %d", summary.SkippedMerges.DurationExceeded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationNoMergeWhenTokenLimitExceeded(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 1, // Very small token limit
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
|
||||
{ID: 2, Speaker: "Alice", Start: 1.5, End: 2.5, Text: "world"},
|
||||
},
|
||||
}
|
||||
|
||||
normalized, summary := normalizer.Normalize(transcript)
|
||||
|
||||
// Should NOT merge since merged token count would exceed limit
|
||||
if len(normalized.Segments) != 2 {
|
||||
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
|
||||
}
|
||||
if summary.SkippedMerges.TokenLimitExceeded != 1 {
|
||||
t.Errorf("expected 1 skipped merge for token limit exceeded, got %d", summary.SkippedMerges.TokenLimitExceeded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationChronologicalOrdering(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 2, Speaker: "Alice", Start: 5.0, End: 6.0, Text: "Later"},
|
||||
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "First"},
|
||||
{ID: 3, Speaker: "Alice", Start: 2.0, End: 3.0, Text: "Middle"},
|
||||
},
|
||||
}
|
||||
|
||||
normalized, _ := normalizer.Normalize(transcript)
|
||||
|
||||
// Should be sorted chronologically
|
||||
if len(normalized.Segments) != 1 {
|
||||
t.Errorf("expected 1 merged segment, got %d", len(normalized.Segments))
|
||||
}
|
||||
if normalized.Segments[0].Start != 0.0 {
|
||||
t.Errorf("expected first segment to start at 0.0, got %f", normalized.Segments[0].Start)
|
||||
}
|
||||
if normalized.Segments[0].End != 6.0 {
|
||||
t.Errorf("expected merged segment to end at 6.0, got %f", normalized.Segments[0].End)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationSequentialIDReassignment(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 5, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
|
||||
{ID: 10, Speaker: "Bob", Start: 2.0, End: 3.0, Text: "Hi"},
|
||||
},
|
||||
}
|
||||
|
||||
normalized, summary := normalizer.Normalize(transcript)
|
||||
|
||||
// Should have sequential IDs starting at 1
|
||||
if len(normalized.Segments) != 2 {
|
||||
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
|
||||
}
|
||||
if normalized.Segments[0].ID != 1 {
|
||||
t.Errorf("expected first segment ID 1, got %d", normalized.Segments[0].ID)
|
||||
}
|
||||
if normalized.Segments[1].ID != 2 {
|
||||
t.Errorf("expected second segment ID 2, got %d", normalized.Segments[1].ID)
|
||||
}
|
||||
if summary.IDsReassigned != 2 {
|
||||
t.Errorf("expected 2 IDs reassigned, got %d", summary.IDsReassigned)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationCategoryPreservation(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello", Categories: []string{"greeting"}},
|
||||
{ID: 2, Speaker: "Alice", Start: 1.5, End: 2.5, Text: "world", Categories: []string{"response"}},
|
||||
},
|
||||
}
|
||||
|
||||
normalized, _ := normalizer.Normalize(transcript)
|
||||
|
||||
// Should merge and preserve both categories
|
||||
if len(normalized.Segments) != 1 {
|
||||
t.Errorf("expected 1 merged segment, got %d", len(normalized.Segments))
|
||||
}
|
||||
if len(normalized.Segments[0].Categories) != 2 {
|
||||
t.Errorf("expected 2 categories, got %d", len(normalized.Segments[0].Categories))
|
||||
}
|
||||
if normalized.Segments[0].Categories[0] != "greeting" || normalized.Segments[0].Categories[1] != "response" {
|
||||
t.Errorf("expected preserved categories, got %v", normalized.Segments[0].Categories)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationGoldenBareArrayTranscript(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
raw, err := os.ReadFile("testdata/bare_array_transcript.json")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test fixture: %v", err)
|
||||
}
|
||||
|
||||
transcript, err := schema.ParseSourceTranscriptJSON(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse source transcript: %v", err)
|
||||
}
|
||||
|
||||
converted := &schema.Transcript{
|
||||
Segments: make([]schema.Segment, len(transcript.Segments)),
|
||||
}
|
||||
for i, s := range transcript.Segments {
|
||||
id := i + 1
|
||||
if s.ID != nil {
|
||||
id = *s.ID
|
||||
}
|
||||
converted.Segments[i] = schema.Segment{
|
||||
ID: id,
|
||||
Speaker: s.Speaker,
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
Text: s.Text,
|
||||
Categories: s.Categories,
|
||||
}
|
||||
}
|
||||
|
||||
normalized, _ := normalizer.Normalize(converted)
|
||||
|
||||
// Compare with golden output semantically
|
||||
expectedRaw, err := os.ReadFile("testdata/bare_array_transcript.golden.json")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read golden fixture: %v", err)
|
||||
}
|
||||
|
||||
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse golden transcript: %v", err)
|
||||
}
|
||||
|
||||
assertTranscriptsEqual(t, normalized, expectedTranscript)
|
||||
}
|
||||
|
||||
func TestNormalizationGoldenObjectWithSegmentsTranscript(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
raw, err := os.ReadFile("testdata/object_with_segments_transcript.json")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test fixture: %v", err)
|
||||
}
|
||||
|
||||
transcript, err := schema.ParseSourceTranscriptJSON(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse source transcript: %v", err)
|
||||
}
|
||||
|
||||
converted := &schema.Transcript{
|
||||
Segments: make([]schema.Segment, len(transcript.Segments)),
|
||||
}
|
||||
for i, s := range transcript.Segments {
|
||||
id := i + 1
|
||||
if s.ID != nil {
|
||||
id = *s.ID
|
||||
}
|
||||
converted.Segments[i] = schema.Segment{
|
||||
ID: id,
|
||||
Speaker: s.Speaker,
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
Text: s.Text,
|
||||
Categories: s.Categories,
|
||||
}
|
||||
}
|
||||
|
||||
normalized, _ := normalizer.Normalize(converted)
|
||||
|
||||
// Compare with golden output semantically
|
||||
expectedRaw, err := os.ReadFile("testdata/object_with_segments_transcript.golden.json")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read golden fixture: %v", err)
|
||||
}
|
||||
|
||||
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse golden transcript: %v", err)
|
||||
}
|
||||
|
||||
assertTranscriptsEqual(t, normalized, expectedTranscript)
|
||||
}
|
||||
|
||||
func TestNormalizationGoldenTranscriptWithCategories(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
raw, err := os.ReadFile("testdata/transcript_with_categories.json")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test fixture: %v", err)
|
||||
}
|
||||
|
||||
transcript, err := schema.ParseSourceTranscriptJSON(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse source transcript: %v", err)
|
||||
}
|
||||
|
||||
converted := &schema.Transcript{
|
||||
Segments: make([]schema.Segment, len(transcript.Segments)),
|
||||
}
|
||||
for i, s := range transcript.Segments {
|
||||
id := i + 1
|
||||
if s.ID != nil {
|
||||
id = *s.ID
|
||||
}
|
||||
converted.Segments[i] = schema.Segment{
|
||||
ID: id,
|
||||
Speaker: s.Speaker,
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
Text: s.Text,
|
||||
Categories: s.Categories,
|
||||
}
|
||||
}
|
||||
|
||||
normalized, _ := normalizer.Normalize(converted)
|
||||
|
||||
// Compare with golden output semantically
|
||||
expectedRaw, err := os.ReadFile("testdata/transcript_with_categories.golden.json")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read golden fixture: %v", err)
|
||||
}
|
||||
|
||||
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse golden transcript: %v", err)
|
||||
}
|
||||
|
||||
assertTranscriptsEqual(t, normalized, expectedTranscript)
|
||||
}
|
||||
|
||||
func TestNormalizationGoldenTranscriptWithOriginalIDs(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
raw, err := os.ReadFile("testdata/transcript_with_original_ids.json")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test fixture: %v", err)
|
||||
}
|
||||
|
||||
transcript, err := schema.ParseSourceTranscriptJSON(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse source transcript: %v", err)
|
||||
}
|
||||
|
||||
converted := &schema.Transcript{
|
||||
Segments: make([]schema.Segment, len(transcript.Segments)),
|
||||
}
|
||||
for i, s := range transcript.Segments {
|
||||
id := i + 1
|
||||
if s.ID != nil {
|
||||
id = *s.ID
|
||||
}
|
||||
converted.Segments[i] = schema.Segment{
|
||||
ID: id,
|
||||
Speaker: s.Speaker,
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
Text: s.Text,
|
||||
Categories: s.Categories,
|
||||
}
|
||||
}
|
||||
|
||||
normalized, _ := normalizer.Normalize(converted)
|
||||
|
||||
// Compare with golden output semantically
|
||||
expectedRaw, err := os.ReadFile("testdata/transcript_with_original_ids.golden.json")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read golden fixture: %v", err)
|
||||
}
|
||||
|
||||
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse golden transcript: %v", err)
|
||||
}
|
||||
|
||||
assertTranscriptsEqual(t, normalized, expectedTranscript)
|
||||
}
|
||||
|
||||
func TestNormalizationGoldenSameSpeakerMerge(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
raw, err := os.ReadFile("testdata/transcript_same_speaker_merge.json")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test fixture: %v", err)
|
||||
}
|
||||
|
||||
transcript, err := schema.ParseSourceTranscriptJSON(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse source transcript: %v", err)
|
||||
}
|
||||
|
||||
converted := &schema.Transcript{
|
||||
Segments: make([]schema.Segment, len(transcript.Segments)),
|
||||
}
|
||||
for i, s := range transcript.Segments {
|
||||
id := i + 1
|
||||
if s.ID != nil {
|
||||
id = *s.ID
|
||||
}
|
||||
converted.Segments[i] = schema.Segment{
|
||||
ID: id,
|
||||
Speaker: s.Speaker,
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
Text: s.Text,
|
||||
Categories: s.Categories,
|
||||
}
|
||||
}
|
||||
|
||||
normalized, _ := normalizer.Normalize(converted)
|
||||
|
||||
// Compare with golden output semantically
|
||||
expectedRaw, err := os.ReadFile("testdata/transcript_same_speaker_merge.golden.json")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read golden fixture: %v", err)
|
||||
}
|
||||
|
||||
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse golden transcript: %v", err)
|
||||
}
|
||||
|
||||
assertTranscriptsEqual(t, normalized, expectedTranscript)
|
||||
}
|
||||
|
||||
func TestNormalizationGoldenEllipsisMerge(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
raw, err := os.ReadFile("testdata/transcript_ellipsis_merge.json")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test fixture: %v", err)
|
||||
}
|
||||
|
||||
transcript, err := schema.ParseSourceTranscriptJSON(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse source transcript: %v", err)
|
||||
}
|
||||
|
||||
converted := &schema.Transcript{
|
||||
Segments: make([]schema.Segment, len(transcript.Segments)),
|
||||
}
|
||||
for i, s := range transcript.Segments {
|
||||
id := i + 1
|
||||
if s.ID != nil {
|
||||
id = *s.ID
|
||||
}
|
||||
converted.Segments[i] = schema.Segment{
|
||||
ID: id,
|
||||
Speaker: s.Speaker,
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
Text: s.Text,
|
||||
Categories: s.Categories,
|
||||
}
|
||||
}
|
||||
|
||||
normalized, _ := normalizer.Normalize(converted)
|
||||
|
||||
// Compare with golden output semantically
|
||||
expectedRaw, err := os.ReadFile("testdata/transcript_ellipsis_merge.golden.json")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read golden fixture: %v", err)
|
||||
}
|
||||
|
||||
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse golden transcript: %v", err)
|
||||
}
|
||||
|
||||
assertTranscriptsEqual(t, normalized, expectedTranscript)
|
||||
}
|
||||
|
||||
func TestNormalizationGoldenDifferentSpeakersNoMerge(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
raw, err := os.ReadFile("testdata/transcript_different_speakers_no_merge.json")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test fixture: %v", err)
|
||||
}
|
||||
|
||||
transcript, err := schema.ParseSourceTranscriptJSON(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse source transcript: %v", err)
|
||||
}
|
||||
|
||||
converted := &schema.Transcript{
|
||||
Segments: make([]schema.Segment, len(transcript.Segments)),
|
||||
}
|
||||
for i, s := range transcript.Segments {
|
||||
id := i + 1
|
||||
if s.ID != nil {
|
||||
id = *s.ID
|
||||
}
|
||||
converted.Segments[i] = schema.Segment{
|
||||
ID: id,
|
||||
Speaker: s.Speaker,
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
Text: s.Text,
|
||||
Categories: s.Categories,
|
||||
}
|
||||
}
|
||||
|
||||
normalized, _ := normalizer.Normalize(converted)
|
||||
|
||||
// Compare with golden output semantically
|
||||
expectedRaw, err := os.ReadFile("testdata/transcript_different_speakers_no_merge.golden.json")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read golden fixture: %v", err)
|
||||
}
|
||||
|
||||
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse golden transcript: %v", err)
|
||||
}
|
||||
|
||||
assertTranscriptsEqual(t, normalized, expectedTranscript)
|
||||
}
|
||||
|
||||
func assertTranscriptsEqual(t *testing.T, actual, expected *schema.Transcript) {
|
||||
t.Helper()
|
||||
|
||||
if len(actual.Segments) != len(expected.Segments) {
|
||||
t.Errorf("segment count mismatch: expected %d, got %d", len(expected.Segments), len(actual.Segments))
|
||||
return
|
||||
}
|
||||
|
||||
for i := range actual.Segments {
|
||||
a := actual.Segments[i]
|
||||
e := expected.Segments[i]
|
||||
|
||||
if a.ID != e.ID {
|
||||
t.Errorf("segment %d: ID mismatch: expected %d, got %d", i, e.ID, a.ID)
|
||||
}
|
||||
if a.Speaker != e.Speaker {
|
||||
t.Errorf("segment %d: speaker mismatch: expected %q, got %q", i, e.Speaker, a.Speaker)
|
||||
}
|
||||
if a.Start != e.Start {
|
||||
t.Errorf("segment %d: start mismatch: expected %f, got %f", i, e.Start, a.Start)
|
||||
}
|
||||
if a.End != e.End {
|
||||
t.Errorf("segment %d: end mismatch: expected %f, got %f", i, e.End, a.End)
|
||||
}
|
||||
if a.Text != e.Text {
|
||||
t.Errorf("segment %d: text mismatch: expected %q, got %q", i, e.Text, a.Text)
|
||||
}
|
||||
if len(a.Categories) != len(e.Categories) {
|
||||
t.Errorf("segment %d: categories count mismatch: expected %d, got %d", i, len(e.Categories), len(a.Categories))
|
||||
continue
|
||||
}
|
||||
for j, cat := range a.Categories {
|
||||
if j >= len(e.Categories) || cat != e.Categories[j] {
|
||||
t.Errorf("segment %d: category %d mismatch: expected %q, got %q", i, j, e.Categories[j], cat)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
19
internal/core/normalization/testdata/bare_array_transcript.golden.json
vendored
Normal file
19
internal/core/normalization/testdata/bare_array_transcript.golden.json
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "Alice",
|
||||
"start": 0.0,
|
||||
"end": 1.5,
|
||||
"text": "Hello world."
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"speaker": "Bob",
|
||||
"start": 2.0,
|
||||
"end": 3.5,
|
||||
"text": "Hi there.",
|
||||
"categories": [
|
||||
"greeting"
|
||||
]
|
||||
}
|
||||
]
|
||||
17
internal/core/normalization/testdata/bare_array_transcript.json
vendored
Normal file
17
internal/core/normalization/testdata/bare_array_transcript.json
vendored
Normal file
@@ -0,0 +1,17 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "Alice",
|
||||
"start": 0.0,
|
||||
"end": 1.5,
|
||||
"text": "Hello world."
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"speaker": "Bob",
|
||||
"start": 2.0,
|
||||
"end": 3.5,
|
||||
"text": "Hi there.",
|
||||
"categories": ["greeting"]
|
||||
}
|
||||
]
|
||||
19
internal/core/normalization/testdata/comprehensive_glossary.yaml
vendored
Normal file
19
internal/core/normalization/testdata/comprehensive_glossary.yaml
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
glossary:
|
||||
- name: Jesters
|
||||
category: faction
|
||||
summary: A faction name.
|
||||
aliases:
|
||||
- Jester
|
||||
- Jest
|
||||
- name: Popov
|
||||
category: character
|
||||
summary: A character name.
|
||||
aliases:
|
||||
- Hrank
|
||||
- Pop
|
||||
- name: Audita
|
||||
category: system
|
||||
summary: The transcript polishing system.
|
||||
aliases:
|
||||
- Audit
|
||||
- Auditor
|
||||
19
internal/core/normalization/testdata/object_with_segments_transcript.golden.json
vendored
Normal file
19
internal/core/normalization/testdata/object_with_segments_transcript.golden.json
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "Alice",
|
||||
"start": 0.0,
|
||||
"end": 1.5,
|
||||
"text": "Hello world."
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"speaker": "Bob",
|
||||
"start": 2.0,
|
||||
"end": 3.5,
|
||||
"text": "Hi there.",
|
||||
"categories": [
|
||||
"greeting"
|
||||
]
|
||||
}
|
||||
]
|
||||
19
internal/core/normalization/testdata/object_with_segments_transcript.json
vendored
Normal file
19
internal/core/normalization/testdata/object_with_segments_transcript.json
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"segments": [
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "Alice",
|
||||
"start": 0.0,
|
||||
"end": 1.5,
|
||||
"text": "Hello world."
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"speaker": "Bob",
|
||||
"start": 2.0,
|
||||
"end": 3.5,
|
||||
"text": "Hi there.",
|
||||
"categories": ["greeting"]
|
||||
}
|
||||
]
|
||||
}
|
||||
16
internal/core/normalization/testdata/transcript_different_speakers_no_merge.golden.json
vendored
Normal file
16
internal/core/normalization/testdata/transcript_different_speakers_no_merge.golden.json
vendored
Normal file
@@ -0,0 +1,16 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "Alice",
|
||||
"start": 0.0,
|
||||
"end": 1.0,
|
||||
"text": "Hello"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"speaker": "Bob",
|
||||
"start": 1.5,
|
||||
"end": 2.5,
|
||||
"text": "Hi"
|
||||
}
|
||||
]
|
||||
16
internal/core/normalization/testdata/transcript_different_speakers_no_merge.json
vendored
Normal file
16
internal/core/normalization/testdata/transcript_different_speakers_no_merge.json
vendored
Normal file
@@ -0,0 +1,16 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "Alice",
|
||||
"start": 0.0,
|
||||
"end": 1.0,
|
||||
"text": "Hello"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"speaker": "Bob",
|
||||
"start": 1.5,
|
||||
"end": 2.5,
|
||||
"text": "Hi"
|
||||
}
|
||||
]
|
||||
9
internal/core/normalization/testdata/transcript_ellipsis_merge.golden.json
vendored
Normal file
9
internal/core/normalization/testdata/transcript_ellipsis_merge.golden.json
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "Alice",
|
||||
"start": 0.0,
|
||||
"end": 3.0,
|
||||
"text": "Hello... world"
|
||||
}
|
||||
]
|
||||
16
internal/core/normalization/testdata/transcript_ellipsis_merge.json
vendored
Normal file
16
internal/core/normalization/testdata/transcript_ellipsis_merge.json
vendored
Normal file
@@ -0,0 +1,16 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "Alice",
|
||||
"start": 0.0,
|
||||
"end": 1.0,
|
||||
"text": "Hello"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"speaker": "Alice",
|
||||
"start": 2.0,
|
||||
"end": 3.0,
|
||||
"text": "world"
|
||||
}
|
||||
]
|
||||
9
internal/core/normalization/testdata/transcript_same_speaker_merge.golden.json
vendored
Normal file
9
internal/core/normalization/testdata/transcript_same_speaker_merge.golden.json
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "Alice",
|
||||
"start": 0.0,
|
||||
"end": 2.2,
|
||||
"text": "Hello world"
|
||||
}
|
||||
]
|
||||
16
internal/core/normalization/testdata/transcript_same_speaker_merge.json
vendored
Normal file
16
internal/core/normalization/testdata/transcript_same_speaker_merge.json
vendored
Normal file
@@ -0,0 +1,16 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "Alice",
|
||||
"start": 0.0,
|
||||
"end": 1.0,
|
||||
"text": "Hello"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"speaker": "Alice",
|
||||
"start": 1.2,
|
||||
"end": 2.2,
|
||||
"text": "world"
|
||||
}
|
||||
]
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user