Compare commits
103 Commits
python-fin
...
v1.0.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 018d08fd4c | |||
| 3a0a0b3940 | |||
| 99ab2f181b | |||
| 4e4801dc98 | |||
| 900ad74958 | |||
| 28d5201a69 | |||
| e5944f9875 | |||
| 6344fc91ba | |||
| 7f3a91cc9e | |||
| 72fb021453 | |||
| 40e8b54d3b | |||
| 76651333b1 | |||
| 0b01c3a83d | |||
| 0630d36734 | |||
| 52c2697040 | |||
| f790c1441c | |||
| 56f9b28f4b | |||
| 222222f449 | |||
| 99391cd18b | |||
| 84be774b34 | |||
| e053f7e124 | |||
| 13029dbb33 | |||
| 938bfe88c1 | |||
| fa1bd237d1 | |||
| 3d7057b437 | |||
| 32c8c8b446 | |||
| a3655f5540 | |||
| a84941d681 | |||
| 46b7356a3b | |||
| 39208cd119 | |||
| 6dae15656d | |||
| b103bb2e7d | |||
| 2b2a3fc024 | |||
| 52ffe42e73 | |||
| 3b160cf05b | |||
| 0687982822 | |||
| ff4ed82239 | |||
| 037121e9ce | |||
| d6126bf52b | |||
| 1bc5936681 | |||
| 9a77a0cd0b | |||
| ebbd2c8a63 | |||
| de99467ede | |||
| 20f612215f | |||
| 3d45571bb0 | |||
| 1afd753fad | |||
| a85a7e204e | |||
| 509436cc4a | |||
| a48f6da1f4 | |||
| df96f9fdf6 | |||
| 390daa8b84 | |||
| af84249da0 | |||
| cad172a758 | |||
| fb59cb21b9 | |||
| 68e2d9b549 | |||
| 185f7ca2b6 | |||
| 7ccadc6bd6 | |||
| a9f7fa27ff | |||
| dbf3605712 | |||
| 543a7ff8ef | |||
| fc3a7b7a67 | |||
| b360493cdc | |||
| 12202508bf | |||
| 6d9a4bd017 | |||
| 426864eedb | |||
| 0b17a6fbeb | |||
| aeb31f1c0d | |||
| 28fe899aa1 | |||
| 30606f5c49 | |||
| db880ed868 | |||
| 5217093be2 | |||
| 0e83991537 | |||
| 3e8d19cccd | |||
| c1193e3450 | |||
| 73249b63d8 | |||
| c3087aeda6 | |||
| 0452a605ad | |||
| b9b7384123 | |||
| 726acc47e1 | |||
| 5c78b1d5d9 | |||
| f461922b9b | |||
| c58d307ba7 | |||
| 1eb93481e0 | |||
| b997e7c97c | |||
| 12fd541669 | |||
| 10377876e4 | |||
| d847168ecd | |||
| 14e51698c2 | |||
| aeb9c4f062 | |||
| e2ae7f77d8 | |||
| 0b1b670baf | |||
| 3cfa4b6e8a | |||
| 950edc01f2 | |||
| ea8def423e | |||
| 95fe8c32fa | |||
| 08b7531149 | |||
| 2cf2d390da | |||
| 8f3c2ec5fd | |||
| 9427c4e6cc | |||
| 6424d7db4f | |||
| 87e560dd3d | |||
| 09fc6fd364 | |||
| 2e47c8a1b6 |
61
.gitignore
vendored
61
.gitignore
vendored
@@ -1,3 +1,4 @@
|
||||
# --> Python
|
||||
.DS_Store
|
||||
.venv/
|
||||
__pycache__/
|
||||
@@ -9,3 +10,63 @@ dist/
|
||||
build/
|
||||
*.egg-info/
|
||||
|
||||
# ---> Codex
|
||||
.codex
|
||||
AGENTS.md
|
||||
|
||||
# ---> Go
|
||||
# If you prefer the allow list template instead of the deny list, see community template:
|
||||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||
#
|
||||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
|
||||
# Dependency directories (remove the comment below to include it)
|
||||
# vendor/
|
||||
|
||||
# Go workspace file
|
||||
go.work
|
||||
go.work.sum
|
||||
|
||||
# env file
|
||||
.env
|
||||
|
||||
# Compiled binaries and test configuration
|
||||
narratio
|
||||
local-test
|
||||
pipeline.yml
|
||||
bin/
|
||||
|
||||
# Local run artifacts
|
||||
.audita-runs/
|
||||
report.json
|
||||
corrected.json
|
||||
normalized.json
|
||||
|
||||
# Coverage artifacts
|
||||
coverage.out
|
||||
coverage.txt
|
||||
|
||||
# ---> VisualStudioCode
|
||||
.vscode/*
|
||||
!.vscode/settings.json
|
||||
!.vscode/tasks.json
|
||||
!.vscode/launch.json
|
||||
!.vscode/extensions.json
|
||||
!.vscode/*.code-snippets
|
||||
|
||||
# Local History for Visual Studio Code
|
||||
.history/
|
||||
|
||||
# Built Visual Studio Code Extensions
|
||||
*.vsix
|
||||
|
||||
50
.woodpecker/release.yml
Normal file
50
.woodpecker/release.yml
Normal file
@@ -0,0 +1,50 @@
|
||||
when:
|
||||
- event: tag
|
||||
|
||||
steps:
|
||||
- name: build-release-assets
|
||||
image: golang:1.25
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
|
||||
version="$CI_COMMIT_TAG"
|
||||
dist="dist"
|
||||
pkg="gitea.maximumdirect.net/eric/audita/cmd/audita"
|
||||
|
||||
rm -rf "$dist"
|
||||
mkdir -p "$dist"
|
||||
|
||||
build_binary() {
|
||||
goos="$1"
|
||||
goarch="$2"
|
||||
suffix="$3"
|
||||
output="$dist/audita-$version-$goos-$goarch$suffix"
|
||||
|
||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \
|
||||
go build -trimpath -ldflags "-s -w -X gitea.maximumdirect.net/eric/audita/internal/buildinfo.Version=$version" \
|
||||
-o "$output" "$pkg"
|
||||
}
|
||||
|
||||
build_binary linux amd64 ""
|
||||
build_binary linux arm64 ""
|
||||
build_binary darwin amd64 ""
|
||||
build_binary darwin arm64 ""
|
||||
build_binary windows amd64 ".exe"
|
||||
build_binary windows arm64 ".exe"
|
||||
|
||||
- name: publish-release
|
||||
image: woodpeckerci/plugin-release
|
||||
depends_on:
|
||||
- build-release-assets
|
||||
settings:
|
||||
api_key:
|
||||
from_secret: GITEA_RELEASE_TOKEN
|
||||
files:
|
||||
- dist/audita-*
|
||||
checksum: sha256
|
||||
checksum-file: SHA256SUMS
|
||||
checksum-flatten: true
|
||||
file-exists: skip
|
||||
overwrite: false
|
||||
prerelease: false
|
||||
2
LICENSE
2
LICENSE
@@ -1,4 +1,4 @@
|
||||
Copyright (c) 2026 eric.
|
||||
Copyright (c) 2026 Eric Rakestraw.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
||||
|
||||
|
||||
154
README.md
154
README.md
@@ -1,148 +1,46 @@
|
||||
# Audita
|
||||
|
||||
Audita is a framework-first transcript correction application. The public `audita` package provides:
|
||||
Audita is a CLI that polishes transcript JSON using glossary-aware and LLM-backed correction modules.
|
||||
|
||||
- deterministic transcript normalization
|
||||
- token-batched module orchestration
|
||||
- concrete `glossary`, `homophones`, `spoken_word`, and `grammar` modules built on reusable proposal / validator contracts
|
||||
- structured run reporting and work-dir diagnostics
|
||||
## Quickstart
|
||||
|
||||
The previous working implementation has been preserved as `audita_prototype` inside this repository. Its full regression suite lives under `tests/audita_prototype`.
|
||||
|
||||
## Development
|
||||
|
||||
This project is set up for `uv`.
|
||||
Build:
|
||||
|
||||
```sh
|
||||
uv sync --extra dev
|
||||
uv run pytest
|
||||
go build -o ./bin/audita ./cmd/audita
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
Process a transcript with the current framework implementation:
|
||||
Run the shortest useful command:
|
||||
|
||||
```sh
|
||||
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
audita process ./transcript.json --glossary ./glossary.yaml --output ./corrected.json
|
||||
```
|
||||
|
||||
The framework currently runs this default module sequence:
|
||||
Notes:
|
||||
- the transcript JSON path is required as a positional argument;
|
||||
- `--glossary` is required;
|
||||
- without `--output`, corrected transcript JSON is written to stdout.
|
||||
|
||||
1. `glossary`
|
||||
2. `homophones`
|
||||
3. `glossary`
|
||||
4. `spoken_word`
|
||||
5. `grammar`
|
||||
## Configuration
|
||||
|
||||
Resolved run instance names are auto-numbered for repeats, so the default report pipeline is:
|
||||
Audita loads defaults, optional file config, environment overrides, then CLI overrides.
|
||||
|
||||
1. `glossary_1`
|
||||
2. `homophones`
|
||||
3. `glossary_2`
|
||||
4. `spoken_word`
|
||||
5. `grammar`
|
||||
|
||||
The default module sequence is fully implemented today:
|
||||
|
||||
- `glossary` proposes glossary-supported acoustic corrections
|
||||
- `homophones` proposes conservative homophone and mistranscription corrections
|
||||
- `spoken_word` proposes conservative dysfluency cleanup
|
||||
- `grammar` proposes punctuation, capitalization, and spacing cleanup only
|
||||
|
||||
To run a custom module sequence, pass `--modules`:
|
||||
Use these commands to validate and inspect config:
|
||||
|
||||
```sh
|
||||
uv run audita process transcript.json --glossary glossary.yaml --modules grammar --output corrected.json
|
||||
audita config validate --config ./audita.yml
|
||||
audita config print-effective --config ./audita.yml
|
||||
```
|
||||
|
||||
To also write a structured JSON report:
|
||||
## Documentation
|
||||
|
||||
```sh
|
||||
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json --report-json report.json
|
||||
```
|
||||
|
||||
From a checked-out repository, you can also use the root launcher:
|
||||
|
||||
```sh
|
||||
./audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
```
|
||||
|
||||
For a system-wide command, install the source tree under `/usr/local/src/audita`, sync dependencies there, and symlink the root launcher into your `PATH`:
|
||||
|
||||
```sh
|
||||
cd /usr/local/src/audita
|
||||
uv sync --extra dev
|
||||
ln -s /usr/local/src/audita/audita /usr/local/bin/audita
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
```
|
||||
|
||||
Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr.
|
||||
`--report-json` writes a separate machine-readable run report and never mixes report data into stdout.
|
||||
|
||||
Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Default OpenRouter runs require LLM API credentials, because the `glossary`, `homophones`, `spoken_word`, and `grammar` modules make real LLM calls. Self-hosted or other non-default OpenAI-compatible endpoints may not require credentials. `AUDITA_LLM_API_KEY` and `--llm-api-key` are the preferred provider-neutral credential surfaces, while `OPENROUTER_API_KEY` remains supported as a backward-compatible fallback.
|
||||
|
||||
| Environment variable | CLI flag | Default | Purpose |
|
||||
| --- | --- | --- | --- |
|
||||
| `AUDITA_MODULES` | `--modules` | `glossary,homophones,glossary,spoken_word,grammar` | Comma-separated logical module keys to run; CLI overrides the environment value |
|
||||
| `AUDITA_LLM_API_KEY` | `--llm-api-key` | unset | Preferred provider-neutral LLM API credential; required for the default OpenRouter endpoint and optional for non-default endpoints; CLI overrides both environment-key variants |
|
||||
| `AUDITA_VALIDATION_LLM_API_KEY` | `--validation-llm-api-key` | unset | Validation-phase LLM API credential; defaults to the primary LLM API key and is optional for non-default validation endpoints |
|
||||
| `AUDITA_MODEL` | `--model` | `openrouter/google/gemma-4-31b-it` | LLM model name sent to the configured OpenAI-compatible endpoint |
|
||||
| `AUDITA_VALIDATION_MODEL` | `--validation-model` | unset | Validation-phase LLM model; defaults to `AUDITA_MODEL` |
|
||||
| `AUDITA_BASE_URL` | `--base-url` | `https://openrouter.ai/api/v1` | OpenAI-compatible API base URL |
|
||||
| `AUDITA_VALIDATION_BASE_URL` | `--validation-base-url` | unset | Validation-phase OpenAI-compatible API base URL; defaults to `AUDITA_BASE_URL` |
|
||||
| `AUDITA_LLM_TIMEOUT_SECONDS` | `--llm-timeout-seconds` | `600` | Per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint |
|
||||
| `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | `--validation-llm-timeout-seconds` | unset | Validation-phase per-request timeout in seconds; defaults to `AUDITA_LLM_TIMEOUT_SECONDS` |
|
||||
| `AUDITA_VALIDATION_MAX_PROMPT_TOKENS` | `--validation-max-prompt-tokens` | `2048` | Maximum estimated tokens per validation-phase LLM prompt batch |
|
||||
| `AUDITA_TARGET_SECTIONS` | `--target-sections` | unset | Exact number of contiguous proposal-stage transcript sections; errors if min/max token bounds cannot be satisfied |
|
||||
| `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses |
|
||||
| `AUDITA_VALIDATION_MAX_RETRIES` | `--validation-max-retries` | unset | Validation-phase structured-output retries; defaults to `AUDITA_MAX_RETRIES` |
|
||||
| `AUDITA_VALIDATION_LLM_CONCURRENCY` | `--validation-llm-concurrency` | unset | Validation-phase LLM concurrency; defaults to `AUDITA_LLM_CONCURRENCY` |
|
||||
| `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `8192` | Maximum estimated tokens per proposal-stage transcript section |
|
||||
| `AUDITA_MIN_SECTION_TOKENS` | `--min-section-tokens` | `2048` | Minimum estimated tokens per proposal-stage transcript section when balancing for concurrency |
|
||||
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation |
|
||||
| `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | `--grammar-confidence-threshold` | `0.8` | Minimum confidence required for grammar proposals to survive validation |
|
||||
| `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | `--homophones-confidence-threshold` | `0.8` | Minimum confidence required for homophone proposals to survive validation |
|
||||
| `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | `--spoken-word-confidence-threshold` | `0.8` | Minimum confidence required for spoken-word proposals to survive validation |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | `--normalize-max-segment-gap` | `4.0` | Same-speaker gaps eligible for deterministic merging |
|
||||
| `AUDITA_NORMALIZE_ELLIPSIS_GAP` | `--normalize-ellipsis-gap` | `3.5` | Same-speaker gaps above this value are joined with ` ... ` |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | `--normalize-max-segment-duration` | `60.0` | Maximum merged segment duration |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | `--normalize-max-segment-tokens` | `2048` | Maximum merged segment prompt payload size |
|
||||
| `AUDITA_WORK_DIR` | `--work-dir` | `/tmp/audita` | Per-run scratch diagnostics directory |
|
||||
| `AUDITA_WORK_DIR_RETENTION` | `--work-dir-retention` | `auto` | Whether to retain the per-run work directory: `auto`, `always`, or `never` |
|
||||
|
||||
Set `AUDITA_MODULES=grammar` to run only the grammar module by default, or override it per command with `--modules`.
|
||||
|
||||
Validation-phase LLM settings inherit from the primary `AUDITA_*` LLM settings by default. Set any of the `AUDITA_VALIDATION_*` values only when you want LLM-backed validators to use a different model, endpoint, credential, timeout, retry budget, or concurrency level.
|
||||
|
||||
OpenRouter remains the default out of the box:
|
||||
|
||||
```sh
|
||||
export AUDITA_LLM_API_KEY=your-openrouter-key
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
```
|
||||
|
||||
You can point Audita at any OpenAI-compatible endpoint by changing `AUDITA_BASE_URL` and, if needed, `AUDITA_MODEL`. For example, a local vLLM server:
|
||||
|
||||
```sh
|
||||
export AUDITA_BASE_URL=http://localhost:8000/v1
|
||||
export AUDITA_MODEL=meta-llama/Llama-3.1-8B-Instruct
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
```
|
||||
|
||||
If your self-hosted endpoint requires authentication, you can still set `AUDITA_LLM_API_KEY`; Audita simply no longer requires it for non-default endpoints.
|
||||
|
||||
Or the actual OpenAI API:
|
||||
|
||||
```sh
|
||||
export AUDITA_LLM_API_KEY=your-openai-key
|
||||
export AUDITA_BASE_URL=https://api.openai.com/v1
|
||||
export AUDITA_MODEL=gpt-4.1-mini
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
```
|
||||
|
||||
`AUDITA_WORK_DIR` stores per-run diagnostics while processing. Under the default `AUDITA_WORK_DIR_RETENTION=auto`, clean successful runs are removed, while failed runs and successful runs with final skipped corrections are preserved. Use `always` to keep every run directory and `never` to remove successful run directories even when skips remain.
|
||||
Failed runs always preserve the run directory and include an authoritative `report.json` alongside normalization and prompt/response diagnostics.
|
||||
|
||||
## Prototype Archive
|
||||
|
||||
The archived prototype remains importable as `audita_prototype` and is still covered by its original regression suite. This is intentional: the new `audita` package is a framework-oriented rewrite, not a thin wrapper around the old code.
|
||||
- CLI reference: [`docs/cli.md`](docs/cli.md)
|
||||
- Configuration reference: [`docs/config.md`](docs/config.md)
|
||||
- Operations guide: [`docs/operations.md`](docs/operations.md)
|
||||
- Troubleshooting: [`docs/troubleshooting.md`](docs/troubleshooting.md)
|
||||
- Subprocess integration: [`docs/integrations/subprocess.md`](docs/integrations/subprocess.md)
|
||||
- OpenAI-compatible LLM integration: [`docs/integrations/openai-compatible-llm.md`](docs/integrations/openai-compatible-llm.md)
|
||||
- Transcript and glossary file integration: [`docs/integrations/transcript-glossary-files.md`](docs/integrations/transcript-glossary-files.md)
|
||||
- Development workflow: [`docs/policy/development.md`](docs/policy/development.md)
|
||||
- Architecture policy: [`docs/policy/architecture.md`](docs/policy/architecture.md)
|
||||
- Documentation policy: [`docs/policy/documentation.md`](docs/policy/documentation.md)
|
||||
|
||||
147
audita
147
audita
@@ -1,147 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
DEFAULT_WORK_DIR = "/tmp/audita"
|
||||
_SECRET_FLAGS = {"--llm-api-key", "--validation-llm-api-key"}
|
||||
|
||||
|
||||
def _redact_argv(argv: list[str]) -> list[str]:
|
||||
redacted: list[str] = []
|
||||
index = 0
|
||||
while index < len(argv):
|
||||
arg = argv[index]
|
||||
matched_flag = next((flag for flag in _SECRET_FLAGS if arg == flag or arg.startswith(flag + "=")), None)
|
||||
if matched_flag is None:
|
||||
redacted.append(arg)
|
||||
index += 1
|
||||
continue
|
||||
if arg == matched_flag:
|
||||
redacted.append(arg)
|
||||
if index + 1 < len(argv):
|
||||
redacted.append("[REDACTED]")
|
||||
index += 2
|
||||
else:
|
||||
index += 1
|
||||
continue
|
||||
redacted.append(f"{matched_flag}=[REDACTED]")
|
||||
index += 1
|
||||
return redacted
|
||||
|
||||
|
||||
def _resolve_work_root(argv: list[str]) -> Path:
|
||||
for index, arg in enumerate(argv):
|
||||
if arg == "--work-dir" and index + 1 < len(argv):
|
||||
return Path(argv[index + 1])
|
||||
if arg.startswith("--work-dir="):
|
||||
return Path(arg.split("=", 1)[1])
|
||||
return Path(os.environ.get("AUDITA_WORK_DIR") or DEFAULT_WORK_DIR)
|
||||
|
||||
|
||||
def _create_run_dir(root: Path) -> Path:
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
|
||||
run_dir = root / f"run-{timestamp}-{uuid4().hex[:8]}"
|
||||
run_dir.mkdir(parents=False, exist_ok=False)
|
||||
return run_dir
|
||||
|
||||
|
||||
def _capture_run_dirs(root: Path) -> set[str]:
|
||||
if not root.exists():
|
||||
return set()
|
||||
return {path.name for path in root.iterdir() if path.is_dir() and path.name.startswith("run-")}
|
||||
|
||||
|
||||
def _find_new_run_dir(root: Path, before: set[str]) -> Optional[Path]:
|
||||
if not root.exists():
|
||||
return None
|
||||
candidates = [
|
||||
path for path in root.iterdir() if path.is_dir() and path.name.startswith("run-") and path.name not in before
|
||||
]
|
||||
if not candidates:
|
||||
return None
|
||||
return max(candidates, key=lambda path: path.name)
|
||||
|
||||
|
||||
def _write_launcher_error_log(
|
||||
path: Path,
|
||||
*,
|
||||
message: str,
|
||||
exit_code: int,
|
||||
argv: list[str],
|
||||
command: Optional[list[str]],
|
||||
) -> None:
|
||||
payload = {
|
||||
"timestamp": datetime.utcnow().isoformat() + "Z",
|
||||
"message": message,
|
||||
"exit_code": exit_code,
|
||||
"argv": argv,
|
||||
"cwd": os.getcwd(),
|
||||
"command": command,
|
||||
}
|
||||
path.write_text(
|
||||
"Audita Launcher Diagnostics\n" + json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _emit_console_line(message: str) -> None:
|
||||
for stream in (sys.stderr, sys.stdout):
|
||||
if stream is None:
|
||||
continue
|
||||
try:
|
||||
stream.write(f"{message}\n")
|
||||
stream.flush()
|
||||
return
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
|
||||
|
||||
def main() -> int:
|
||||
argv = list(sys.argv[1:])
|
||||
work_root = _resolve_work_root(argv)
|
||||
redacted_argv = _redact_argv(argv)
|
||||
uv = shutil.which("uv")
|
||||
if uv is None:
|
||||
run_dir = _create_run_dir(work_root)
|
||||
error_log = run_dir / "error.log"
|
||||
message = "uv is required to run this launcher. Install uv and run `uv sync` in the Audita project."
|
||||
_write_launcher_error_log(error_log, message=message, exit_code=1, argv=redacted_argv, command=None)
|
||||
_emit_console_line(f"audita: error: {message}")
|
||||
_emit_console_line("audita: exit code: 1")
|
||||
_emit_console_line(f"audita: run directory: {run_dir}")
|
||||
_emit_console_line(f"audita: error log: {error_log}")
|
||||
return 1
|
||||
|
||||
project_root = Path(__file__).resolve().parent
|
||||
command = [uv, "run", "--project", str(project_root), "python", "-m", "audita", *sys.argv[1:]]
|
||||
before = _capture_run_dirs(work_root)
|
||||
result = subprocess.run(command, cwd=project_root, check=False)
|
||||
if result.returncode == 0:
|
||||
return 0
|
||||
if _find_new_run_dir(work_root, before) is None:
|
||||
run_dir = _create_run_dir(work_root)
|
||||
error_log = run_dir / "error.log"
|
||||
_write_launcher_error_log(
|
||||
error_log,
|
||||
message=f"Audita subprocess exited with status {result.returncode}.",
|
||||
exit_code=result.returncode,
|
||||
argv=redacted_argv,
|
||||
command=_redact_argv(command),
|
||||
)
|
||||
_emit_console_line(f"audita: subprocess exited with status {result.returncode}")
|
||||
_emit_console_line(f"audita: run directory: {run_dir}")
|
||||
_emit_console_line(f"audita: error log: {error_log}")
|
||||
return result.returncode
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
11
cmd/audita/main.go
Normal file
11
cmd/audita/main.go
Normal file
@@ -0,0 +1,11 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/cli"
|
||||
)
|
||||
|
||||
func main() {
|
||||
os.Exit(cli.Run(os.Args[1:], os.Stdout, os.Stderr))
|
||||
}
|
||||
677
cmd/audita/main_integration_test.go
Normal file
677
cmd/audita/main_integration_test.go
Normal file
@@ -0,0 +1,677 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/cli"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/testsupport"
|
||||
)
|
||||
|
||||
func TestHelperProcess(t *testing.T) {
|
||||
if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
|
||||
return
|
||||
}
|
||||
|
||||
sep := -1
|
||||
for i, arg := range os.Args {
|
||||
if arg == "--" {
|
||||
sep = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if sep == -1 {
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
cli.ConfigureSubprocessTestHooksFromEnv()
|
||||
code := cli.Run(os.Args[sep+1:], os.Stdout, os.Stderr)
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func TestProcessHelpSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(t, "process", "--help")
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if !strings.Contains(result.stdout, "Usage:") || !strings.Contains(result.stdout, "--glossary") {
|
||||
t.Fatalf("unexpected help stdout: %q", result.stdout)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSuccessWithOutputSubprocess(t *testing.T) {
|
||||
outputPath := filepath.Join(t.TempDir(), "corrected.json")
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output",
|
||||
outputPath,
|
||||
)
|
||||
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout when --output is set, got %q", result.stdout)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
|
||||
inputBytes := readFile(t, fixturePath("tiny_transcript.json"))
|
||||
outputBytes := readFile(t, outputPath)
|
||||
assertJSONSemanticallyEqual(t, inputBytes, outputBytes)
|
||||
}
|
||||
|
||||
func TestProcessSuccessWithoutOutputSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
|
||||
inputBytes := readFile(t, fixturePath("tiny_transcript.json"))
|
||||
assertJSONSemanticallyEqual(t, inputBytes, []byte(result.stdout))
|
||||
}
|
||||
|
||||
func TestProcessSuccessWithAuditaV1OutputSchemaSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output-schema",
|
||||
"audita-v1",
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
var out struct {
|
||||
Schema string `json:"schema"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(result.stdout), &out); err != nil {
|
||||
t.Fatalf("expected valid audita-v1 JSON output: %v", err)
|
||||
}
|
||||
if out.Schema != "audita-v1" {
|
||||
t.Fatalf("expected audita-v1 schema, got %q", out.Schema)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMissingTranscriptSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(t, "process", "--glossary", fixturePath("tiny_glossary.yaml"))
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "expected exactly 1 transcript JSON path argument") {
|
||||
t.Fatalf("expected actionable missing transcript error, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMalformedJSONSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("malformed_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "is not valid JSON") {
|
||||
t.Fatalf("expected malformed JSON error, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMissingTranscriptFileSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
filepath.Join(t.TempDir(), "missing-transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "transcript_read") {
|
||||
t.Fatalf("expected transcript_read failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMissingGlossaryFileSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
filepath.Join(t.TempDir(), "missing-glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "glossary_read") {
|
||||
t.Fatalf("expected glossary_read failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureTranscriptSchemaSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
schemaFixturePath("transcript_empty_speaker.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "transcript_schema") {
|
||||
t.Fatalf("expected transcript_schema failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMalformedGlossaryYAMLSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
schemaFixturePath("glossary_malformed.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "glossary_schema") {
|
||||
t.Fatalf("expected glossary_schema failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureUnreadableTranscriptSubprocess(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("portable unreadable-file permissions are not reliable on windows")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
transcriptPath := filepath.Join(dir, "transcript.json")
|
||||
if err := os.WriteFile(transcriptPath, []byte(`[]`), 0o000); err != nil {
|
||||
t.Fatalf("write unreadable transcript: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.Chmod(transcriptPath, 0o644) })
|
||||
if _, err := os.ReadFile(transcriptPath); err == nil {
|
||||
t.Skip("unable to make transcript unreadable on this platform/user")
|
||||
}
|
||||
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
transcriptPath,
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "transcript_read") {
|
||||
t.Fatalf("expected transcript_read failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureUnwritableOutputSubprocess(t *testing.T) {
|
||||
outputDir := t.TempDir()
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output",
|
||||
outputDir,
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "failed to write output file") {
|
||||
t.Fatalf("expected write failure message, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureUnwritableReportJSONSubprocess(t *testing.T) {
|
||||
reportDir := t.TempDir()
|
||||
outputPath := filepath.Join(t.TempDir(), "out.json")
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output",
|
||||
outputPath,
|
||||
"--report-json",
|
||||
reportDir,
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "failed to write report JSON file") {
|
||||
t.Fatalf("expected report write failure message, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSuccessReportJSONSubprocess(t *testing.T) {
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--report-json",
|
||||
reportPath,
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
if !json.Valid([]byte(result.stdout)) {
|
||||
t.Fatalf("expected transcript JSON only on stdout, got %q", result.stdout)
|
||||
}
|
||||
report := readFile(t, reportPath)
|
||||
if !json.Valid(report) {
|
||||
t.Fatalf("expected valid report JSON, got %q", string(report))
|
||||
}
|
||||
// Ensure report JSON is not printed to stdout.
|
||||
if strings.Contains(result.stdout, `"default_pipeline"`) {
|
||||
t.Fatalf("report JSON leaked to stdout: %q", result.stdout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSuccessLargeTranscriptSubprocess(t *testing.T) {
|
||||
transcriptPath := writeLargeTranscriptFixture(t, 320)
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
transcriptPath,
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
if !json.Valid([]byte(result.stdout)) {
|
||||
t.Fatalf("expected valid transcript JSON on stdout")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMalformedStructuredLLMResponseViaSubprocessHook(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "malformed_structured"},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"grammar",
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected zero exit code, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if !json.Valid([]byte(result.stdout)) {
|
||||
t.Fatalf("expected transcript JSON on stdout, got %q", result.stdout)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
report := readFile(t, reportPath)
|
||||
if !json.Valid(report) {
|
||||
t.Fatalf("expected valid success report JSON")
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
|
||||
t.Fatalf("did not expect error.log, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureBackendLLMViaSubprocessHook(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "backend_error"},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"grammar",
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "synthetic backend failure") {
|
||||
t.Fatalf("expected backend failure details, got %q", result.stderr)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "diagnostics:") {
|
||||
t.Fatalf("expected diagnostics path in stderr, got %q", result.stderr)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(onlyRunDir(t, workDir), "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log in retained failed run: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMidPipelinePreservesPartialReportsSubprocess(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "mid_pipeline_fail"},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"glossary,homophones,glossary,spoken_word,grammar",
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
reportRaw := readFile(t, reportPath)
|
||||
var report struct {
|
||||
Status string `json:"status"`
|
||||
ErrorPhase string `json:"error_phase"`
|
||||
ModuleResults []struct {
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
Status string `json:"status"`
|
||||
} `json:"module_results"`
|
||||
}
|
||||
if err := json.Unmarshal(reportRaw, &report); err != nil {
|
||||
t.Fatalf("unmarshal report: %v", err)
|
||||
}
|
||||
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
|
||||
t.Fatalf("expected failed runner_execution report, got %+v", report)
|
||||
}
|
||||
if len(report.ModuleResults) == 0 {
|
||||
t.Fatalf("expected partial module results in failure report")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessCancellationViaSubprocessTimeoutHook(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
|
||||
result := runCLISubprocessContext(t, ctx,
|
||||
map[string]string{
|
||||
"AUDITA_SUBPROCESS_TEST_LLM_MODE": "block_until_cancel",
|
||||
"AUDITA_SUBPROCESS_TEST_RUN_TIMEOUT_MS": "120",
|
||||
},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"grammar",
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.stdout != "" {
|
||||
if result.stderr == "" {
|
||||
t.Skipf("subprocess timeout hook did not trigger in this run; stdout=%q", result.stdout)
|
||||
}
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "context deadline exceeded") {
|
||||
t.Fatalf("expected context deadline error, got %q", result.stderr)
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log for canceled run: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil {
|
||||
t.Fatalf("expected report.json for canceled run: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSubprocessNoSecretLeakInOutputsAndDiagnostics(t *testing.T) {
|
||||
secret := "subprocess-secret"
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
outputPath := filepath.Join(t.TempDir(), "out.json")
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{
|
||||
"AUDITA_LLM_API_KEY": secret,
|
||||
"AUDITA_VALIDATION_LLM_API_KEY": secret,
|
||||
},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output",
|
||||
outputPath,
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if strings.Contains(result.stdout, secret) || strings.Contains(result.stderr, secret) {
|
||||
t.Fatalf("secret leaked in subprocess stdio")
|
||||
}
|
||||
assertNoSecretInFile(t, reportPath, secret)
|
||||
assertNoSecretInTree(t, onlyRunDir(t, workDir), secret)
|
||||
}
|
||||
|
||||
type subprocessResult struct {
|
||||
stdout string
|
||||
stderr string
|
||||
exitCode int
|
||||
}
|
||||
|
||||
func runCLISubprocess(t *testing.T, args ...string) subprocessResult {
|
||||
t.Helper()
|
||||
return runCLISubprocessWithEnv(t, nil, args...)
|
||||
}
|
||||
|
||||
func runCLISubprocessWithEnv(t *testing.T, extraEnv map[string]string, args ...string) subprocessResult {
|
||||
t.Helper()
|
||||
return runCLISubprocessContext(t, context.Background(), extraEnv, args...)
|
||||
}
|
||||
|
||||
func runCLISubprocessContext(t *testing.T, ctx context.Context, extraEnv map[string]string, args ...string) subprocessResult {
|
||||
t.Helper()
|
||||
cmdArgs := append([]string{"-test.run=TestHelperProcess", "--"}, args...)
|
||||
cmd := exec.CommandContext(ctx, os.Args[0], cmdArgs...)
|
||||
env := append(filterAuditaEnv(os.Environ()), "GO_WANT_HELPER_PROCESS=1")
|
||||
for k, v := range extraEnv {
|
||||
env = append(env, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
cmd.Env = env
|
||||
var stdoutBuf bytes.Buffer
|
||||
var stderrBuf bytes.Buffer
|
||||
cmd.Stdout = &stdoutBuf
|
||||
cmd.Stderr = &stderrBuf
|
||||
|
||||
err := cmd.Run()
|
||||
result := subprocessResult{
|
||||
stdout: stdoutBuf.String(),
|
||||
stderr: stderrBuf.String(),
|
||||
}
|
||||
if err == nil {
|
||||
return result
|
||||
}
|
||||
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
result.exitCode = exitErr.ExitCode()
|
||||
return result
|
||||
}
|
||||
|
||||
t.Fatalf("subprocess execution failed: %v", err)
|
||||
return subprocessResult{}
|
||||
}
|
||||
|
||||
func filterAuditaEnv(env []string) []string {
|
||||
filtered := make([]string, 0, len(env))
|
||||
for _, entry := range env {
|
||||
key := entry
|
||||
if idx := strings.IndexByte(entry, '='); idx >= 0 {
|
||||
key = entry[:idx]
|
||||
}
|
||||
if strings.HasPrefix(key, "AUDITA_") || key == "OPENROUTER_API_KEY" {
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, entry)
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
func fixturePath(name string) string {
|
||||
return filepath.Join("..", "..", "internal", "cli", "testdata", name)
|
||||
}
|
||||
|
||||
func schemaFixturePath(name string) string {
|
||||
return filepath.Join("..", "..", "internal", "core", "schema", "testdata", name)
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) []byte {
|
||||
return testsupport.ReadFile(t, path)
|
||||
}
|
||||
|
||||
func assertJSONSemanticallyEqual(t *testing.T, expected []byte, actual []byte) {
|
||||
t.Helper()
|
||||
if !json.Valid(actual) {
|
||||
t.Fatalf("actual output is not valid JSON: %q", string(actual))
|
||||
}
|
||||
|
||||
var expectedValue any
|
||||
var actualValue any
|
||||
if err := json.Unmarshal(expected, &expectedValue); err != nil {
|
||||
t.Fatalf("failed to unmarshal expected JSON: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(actual, &actualValue); err != nil {
|
||||
t.Fatalf("failed to unmarshal actual JSON: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(expectedValue, actualValue) {
|
||||
t.Fatalf("JSON content mismatch: expected %q got %q", string(expected), string(actual))
|
||||
}
|
||||
}
|
||||
|
||||
func writeLargeTranscriptFixture(t *testing.T, segments int) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "large-transcript.json")
|
||||
rows := make([]string, 0, segments)
|
||||
for i := 0; i < segments; i++ {
|
||||
rows = append(rows, fmt.Sprintf(`{"id":%d,"speaker":"Speaker%d","start":%s,"end":%s,"text":"Segment %d has enough words to exercise stdout and pipe buffering safely."}`,
|
||||
i+1,
|
||||
(i%4)+1,
|
||||
strconv.FormatFloat(float64(i)*1.1, 'f', 1, 64),
|
||||
strconv.FormatFloat(float64(i)*1.1+1.0, 'f', 1, 64),
|
||||
i+1,
|
||||
))
|
||||
}
|
||||
payload := "[\n " + strings.Join(rows, ",\n ") + "\n]\n"
|
||||
if err := os.WriteFile(path, []byte(payload), 0o644); err != nil {
|
||||
t.Fatalf("write large transcript fixture: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func onlyRunDir(t *testing.T, workDir string) string {
|
||||
return testsupport.OnlyRunDir(t, workDir)
|
||||
}
|
||||
|
||||
func assertNoSecretInFile(t *testing.T, path, secret string) {
|
||||
testsupport.AssertNoSecretInFile(t, path, secret)
|
||||
}
|
||||
|
||||
func assertNoSecretInTree(t *testing.T, root, secret string) {
|
||||
testsupport.AssertNoSecretInTree(t, root, secret)
|
||||
}
|
||||
186
docs/cli.md
Normal file
186
docs/cli.md
Normal file
@@ -0,0 +1,186 @@
|
||||
# Audita CLI Reference
|
||||
|
||||
## Shortest Useful Command
|
||||
|
||||
```sh
|
||||
audita process <transcript.json> --glossary <glossary.yaml> --output <corrected.json>
|
||||
```
|
||||
|
||||
This command validates input files, runs the configured correction pipeline, and writes corrected transcript JSON.
|
||||
|
||||
## Command Overview
|
||||
|
||||
- `audita process`: process one transcript JSON file.
|
||||
- `audita config validate`: validate a versioned YAML config file.
|
||||
- `audita config print-effective`: print redacted effective config JSON.
|
||||
|
||||
General help:
|
||||
|
||||
```sh
|
||||
audita --help
|
||||
audita process --help
|
||||
audita config --help
|
||||
```
|
||||
|
||||
## `process`
|
||||
|
||||
Usage:
|
||||
|
||||
```sh
|
||||
audita process <transcript.json> [flags]
|
||||
```
|
||||
|
||||
Input requirements:
|
||||
- exactly one transcript JSON positional argument is required;
|
||||
- `--glossary <path>` is required.
|
||||
|
||||
Config path selection for `process`:
|
||||
1. `--config <path>`
|
||||
2. `AUDITA_CONFIG`
|
||||
3. `/usr/local/etc/audita/config.yml` (if present)
|
||||
4. `/etc/audita/config.yml` (if present)
|
||||
|
||||
For precedence and full config schema, see [`docs/config.md`](config.md).
|
||||
|
||||
### `process` Flag Reference
|
||||
|
||||
Core I/O flags:
|
||||
- `--config <path>`: path to versioned YAML config file.
|
||||
- `--glossary <path>`: glossary YAML input path (required).
|
||||
- `--output <path>`: corrected transcript JSON output file path.
|
||||
- `--report-json <path>`: machine-readable report JSON output path.
|
||||
- `--output-schema <key>`: output schema key (`bare-segments` or `audita-v1`).
|
||||
- `--modules <csv>`: comma-separated module sequence override.
|
||||
|
||||
Primary LLM flags:
|
||||
- `--llm-api-key <value>`: primary LLM API key.
|
||||
- `--model <name>`: primary LLM model name.
|
||||
- `--base-url <url>`: primary OpenAI-compatible base URL.
|
||||
- `--llm-timeout-seconds <int>`: primary timeout in seconds.
|
||||
- `--max-retries <int>`: primary structured-output retries.
|
||||
|
||||
Validation LLM flags:
|
||||
- `--validation-llm-api-key <value>`: validation LLM API key.
|
||||
- `--validation-model <name>`: validation LLM model name.
|
||||
- `--validation-base-url <url>`: validation OpenAI-compatible base URL.
|
||||
- `--validation-llm-timeout-seconds <int>`: validation timeout in seconds.
|
||||
- `--validation-max-retries <int>`: validation structured-output retries.
|
||||
- `--validation-max-prompt-tokens <int>`: validation max prompt tokens.
|
||||
|
||||
Concurrency flags:
|
||||
- `--total-llm-concurrency <int>`: total concurrent proposal+validation LLM calls.
|
||||
- `--proposal-llm-concurrency <int>`: concurrent proposal-generation LLM calls.
|
||||
- `--validation-llm-concurrency <int>`: concurrent validation LLM calls.
|
||||
- `--llm-concurrency <int>`: alias for `--total-llm-concurrency`.
|
||||
|
||||
Chunking and normalization flags:
|
||||
- `--target-sections <int>`: target number of transcript sections.
|
||||
- `--max-section-tokens <int>`: maximum section tokens.
|
||||
- `--min-section-tokens <int>`: minimum section tokens.
|
||||
- `--normalize-max-segment-gap <float>`: maximum same-speaker merge gap.
|
||||
- `--normalize-ellipsis-gap <float>`: gap threshold for ellipsis insertion.
|
||||
- `--normalize-max-segment-duration <float>`: maximum merged segment duration.
|
||||
- `--normalize-max-segment-tokens <int>`: maximum merged segment token estimate.
|
||||
|
||||
Threshold flags:
|
||||
- `--glossary-confidence-threshold <float>`
|
||||
- `--homophones-confidence-threshold <float>`
|
||||
- `--spoken-word-confidence-threshold <float>`
|
||||
- `--grammar-confidence-threshold <float>`
|
||||
|
||||
Context and diagnostics flags:
|
||||
- `--transcript-description <text>`: background context for prompts; does not override transcript content.
|
||||
- `--work-dir <path>`: per-run diagnostics work directory.
|
||||
- `--work-dir-retention <auto|always|never>`: run-directory retention policy.
|
||||
|
||||
### `process` Output and Exit Behavior
|
||||
|
||||
- With `--output`: stdout is expected to be empty on success.
|
||||
- Without `--output`: stdout contains transcript JSON only on success.
|
||||
- `--report-json` writes a file and is never printed to stdout.
|
||||
- Stderr is human-readable diagnostics/errors.
|
||||
- On failures after diagnostics initialization, stderr includes the diagnostics directory path.
|
||||
|
||||
Exit behavior:
|
||||
- `0`: success.
|
||||
- `1`: runtime failure during processing/reporting/output paths.
|
||||
- `2`: CLI usage or configuration input error.
|
||||
|
||||
Integration references:
|
||||
- subprocess contract: [`docs/integrations/subprocess.md`](integrations/subprocess.md)
|
||||
- transcript/glossary file contract: [`docs/integrations/transcript-glossary-files.md`](integrations/transcript-glossary-files.md)
|
||||
|
||||
### `process` Examples
|
||||
|
||||
Write corrected transcript to a file:
|
||||
|
||||
```sh
|
||||
audita process transcript.json \
|
||||
--glossary glossary.yaml \
|
||||
--output corrected.json
|
||||
```
|
||||
|
||||
Emit transcript JSON to stdout:
|
||||
|
||||
```sh
|
||||
audita process transcript.json --glossary glossary.yaml
|
||||
```
|
||||
|
||||
Use explicit config and write report JSON:
|
||||
|
||||
```sh
|
||||
audita process transcript.json \
|
||||
--glossary glossary.yaml \
|
||||
--config audita.yml \
|
||||
--output corrected.json \
|
||||
--report-json report.json
|
||||
```
|
||||
|
||||
Override the module sequence:
|
||||
|
||||
```sh
|
||||
audita process transcript.json \
|
||||
--glossary glossary.yaml \
|
||||
--modules glossary,homophones,grammar \
|
||||
--output corrected.json
|
||||
```
|
||||
|
||||
## `config validate`
|
||||
|
||||
Usage:
|
||||
|
||||
```sh
|
||||
audita config validate --config <path>
|
||||
```
|
||||
|
||||
Behavior:
|
||||
- validates defaults merged with file config;
|
||||
- does not apply environment overrides;
|
||||
- prints `config is valid` on success.
|
||||
|
||||
Errors:
|
||||
- `--config` is required;
|
||||
- positional arguments are rejected;
|
||||
- validation failures are printed to stderr.
|
||||
|
||||
## `config print-effective`
|
||||
|
||||
Usage:
|
||||
|
||||
```sh
|
||||
audita config print-effective [--config <path>]
|
||||
```
|
||||
|
||||
Config path selection:
|
||||
1. `--config <path>` when provided
|
||||
2. `AUDITA_CONFIG`
|
||||
3. `/usr/local/etc/audita/config.yml` (if present)
|
||||
4. `/etc/audita/config.yml` (if present)
|
||||
|
||||
Behavior:
|
||||
- merges defaults, optional config file, and environment overrides;
|
||||
- prints redacted JSON to stdout.
|
||||
|
||||
Errors:
|
||||
- positional arguments are rejected;
|
||||
- resolution or parse failures are printed to stderr.
|
||||
238
docs/config.md
Normal file
238
docs/config.md
Normal file
@@ -0,0 +1,238 @@
|
||||
# Audita Configuration
|
||||
|
||||
## Scope
|
||||
|
||||
This is the canonical configuration reference for Audita.
|
||||
|
||||
It documents:
|
||||
- config path resolution;
|
||||
- effective precedence across defaults, file config, environment, and CLI;
|
||||
- supported `version: 1` YAML schema;
|
||||
- environment overrides;
|
||||
- CLI override relationship;
|
||||
- validation and secrets behavior.
|
||||
|
||||
For CLI command syntax, see [`docs/cli.md`](cli.md).
|
||||
For OpenAI-compatible endpoint behavior, see [`docs/integrations/openai-compatible-llm.md`](integrations/openai-compatible-llm.md).
|
||||
For transcript/glossary input file contracts, see [`docs/integrations/transcript-glossary-files.md`](integrations/transcript-glossary-files.md).
|
||||
|
||||
## Loading Model
|
||||
|
||||
Path resolution for `audita process` and `audita config print-effective`:
|
||||
1. `--config <path>`
|
||||
2. `AUDITA_CONFIG`
|
||||
3. `/usr/local/etc/audita/config.yml` (if present)
|
||||
4. `/etc/audita/config.yml` (if present)
|
||||
|
||||
Missing explicit path behavior:
|
||||
- missing `--config` target is an error;
|
||||
- missing `AUDITA_CONFIG` target is an error.
|
||||
|
||||
Missing default-path files are non-fatal.
|
||||
|
||||
## Effective Precedence
|
||||
|
||||
`audita process`:
|
||||
1. defaults
|
||||
2. file config
|
||||
3. environment overrides
|
||||
4. CLI overrides
|
||||
|
||||
`audita config print-effective`:
|
||||
1. defaults
|
||||
2. file config
|
||||
3. environment overrides
|
||||
|
||||
`audita config validate`:
|
||||
1. defaults
|
||||
2. file config
|
||||
|
||||
`config validate` is intentionally file-only (no environment overrides).
|
||||
|
||||
## Defaults
|
||||
|
||||
Current defaults:
|
||||
- modules: `glossary,homophones,glossary,spoken_word,grammar`
|
||||
- output schema: `bare-segments`
|
||||
- primary model: `openrouter/google/gemma-4-31b-it`
|
||||
- primary base URL: `https://openrouter.ai/api/v1`
|
||||
- primary timeout: `600` seconds
|
||||
- max retries: `3`
|
||||
- total/proposal LLM concurrency: `1`
|
||||
- validation max prompt tokens: `2048`
|
||||
- max section tokens: `8192`
|
||||
- min section tokens: `2048`
|
||||
- confidence thresholds: `0.8`
|
||||
- normalization max segment gap: `4.0`
|
||||
- normalization ellipsis gap: `3.5`
|
||||
- normalization max segment duration: `60.0`
|
||||
- normalization max segment tokens: `2048`
|
||||
- transcript description: empty
|
||||
- work dir: `/tmp/audita`
|
||||
- work dir retention: `auto`
|
||||
|
||||
## YAML Schema (`version: 1`)
|
||||
|
||||
Supported file version:
|
||||
- `version: 1` (required)
|
||||
|
||||
Unknown YAML fields are rejected.
|
||||
|
||||
```yaml
|
||||
version: 1
|
||||
|
||||
pipeline:
|
||||
modules: [glossary, homophones, glossary, spoken_word, grammar]
|
||||
|
||||
output:
|
||||
schema: bare-segments
|
||||
|
||||
llm:
|
||||
proposal:
|
||||
base_url: https://openrouter.ai/api/v1
|
||||
model: openrouter/google/gemma-4-31b-it
|
||||
api_key_env: AUDITA_LLM_API_KEY
|
||||
timeout: 600s
|
||||
max_retries: 3
|
||||
validation:
|
||||
base_url: https://openrouter.ai/api/v1
|
||||
model: openrouter/google/gemma-4-31b-it
|
||||
api_key_env: AUDITA_VALIDATION_LLM_API_KEY
|
||||
timeout: 600
|
||||
max_retries: 3
|
||||
|
||||
concurrency:
|
||||
total_llm: 1
|
||||
proposal_llm: 1
|
||||
validation_llm: 1
|
||||
|
||||
chunking:
|
||||
target_sections: 8
|
||||
max_section_tokens: 8192
|
||||
min_section_tokens: 2048
|
||||
|
||||
normalization:
|
||||
max_segment_gap: 4s
|
||||
ellipsis_gap: 3.5s
|
||||
max_segment_duration: 60s
|
||||
max_segment_tokens: 2048
|
||||
|
||||
thresholds:
|
||||
glossary: 0.8
|
||||
homophones: 0.8
|
||||
spoken_word: 0.8
|
||||
grammar: 0.8
|
||||
|
||||
context:
|
||||
description: optional background context
|
||||
|
||||
diagnostics:
|
||||
work_dir: /tmp/audita
|
||||
retention: auto
|
||||
```
|
||||
|
||||
Duration-parsing behavior:
|
||||
- `llm.*.timeout`: integer seconds or duration string; duration strings must resolve to whole seconds.
|
||||
- `normalization.*` duration-like fields: numeric seconds or duration string.
|
||||
|
||||
## Environment Overrides
|
||||
|
||||
Modules:
|
||||
- `AUDITA_MODULES`
|
||||
|
||||
Config path:
|
||||
- `AUDITA_CONFIG`
|
||||
|
||||
Primary LLM:
|
||||
- `AUDITA_LLM_API_KEY` (falls back to `OPENROUTER_API_KEY` when unset)
|
||||
- `AUDITA_MODEL`
|
||||
- `AUDITA_BASE_URL`
|
||||
- `AUDITA_LLM_TIMEOUT_SECONDS`
|
||||
- `AUDITA_MAX_RETRIES`
|
||||
|
||||
Validation LLM:
|
||||
- `AUDITA_VALIDATION_LLM_API_KEY`
|
||||
- `AUDITA_VALIDATION_MODEL`
|
||||
- `AUDITA_VALIDATION_BASE_URL`
|
||||
- `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS`
|
||||
- `AUDITA_VALIDATION_MAX_RETRIES`
|
||||
- `AUDITA_VALIDATION_MAX_PROMPT_TOKENS`
|
||||
|
||||
Concurrency:
|
||||
- `AUDITA_TOTAL_LLM_CONCURRENCY`
|
||||
- `AUDITA_PROPOSAL_LLM_CONCURRENCY`
|
||||
- `AUDITA_VALIDATION_LLM_CONCURRENCY`
|
||||
- `AUDITA_LLM_CONCURRENCY` (legacy alias for total)
|
||||
|
||||
Chunking:
|
||||
- `AUDITA_MAX_SECTION_TOKENS`
|
||||
- `AUDITA_MIN_SECTION_TOKENS`
|
||||
- `AUDITA_TARGET_SECTIONS`
|
||||
|
||||
Thresholds:
|
||||
- `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD`
|
||||
- `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD`
|
||||
- `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD`
|
||||
- `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD`
|
||||
|
||||
Normalization:
|
||||
- `AUDITA_NORMALIZE_MAX_SEGMENT_GAP`
|
||||
- `AUDITA_NORMALIZE_ELLIPSIS_GAP`
|
||||
- `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION`
|
||||
- `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS`
|
||||
|
||||
Diagnostics:
|
||||
- `AUDITA_WORK_DIR`
|
||||
- `AUDITA_WORK_DIR_RETENTION` (`auto`, `always`, `never`)
|
||||
|
||||
Transcript description:
|
||||
- no `AUDITA_*` environment variable is currently defined.
|
||||
|
||||
## CLI Override Relationship
|
||||
|
||||
CLI flags override file and environment values for `audita process`.
|
||||
|
||||
The CLI supports canonical total concurrency (`--total-llm-concurrency`) and legacy alias (`--llm-concurrency`):
|
||||
- when both are provided at the same precedence layer, canonical total wins;
|
||||
- if proposal concurrency is not explicitly set and total is set via environment or CLI, proposal concurrency inherits that total;
|
||||
- validation concurrency inherits total only when validation concurrency is unset.
|
||||
|
||||
For full flag syntax, see [`docs/cli.md`](cli.md).
|
||||
|
||||
## Validation Rules
|
||||
|
||||
Validation includes:
|
||||
- supported module keys only;
|
||||
- supported output schema keys only (`bare-segments`, `audita-v1`);
|
||||
- positive timeout/concurrency/token constraints;
|
||||
- `proposal_llm <= total_llm` and `validation_llm <= total_llm` when validation is set;
|
||||
- confidence thresholds in `[0.0, 1.0]`;
|
||||
- transcript description length `<= 500` characters;
|
||||
- non-empty work dir;
|
||||
- work-dir retention in `auto|always|never`.
|
||||
|
||||
## Secrets
|
||||
|
||||
Recommended secret handling:
|
||||
- use `llm.proposal.api_key_env` and `llm.validation.api_key_env` in file config;
|
||||
- use `AUDITA_*_API_KEY` environment overrides or CLI key flags when needed.
|
||||
|
||||
`api_key_env` fields contain environment variable names, not secret values.
|
||||
|
||||
Redaction behavior:
|
||||
- `audita config print-effective` redacts resolved API keys.
|
||||
- diagnostics and report paths redact configured secret values.
|
||||
|
||||
## Examples
|
||||
|
||||
- Minimal config: [`examples/minimal-config.yml`](../examples/minimal-config.yml)
|
||||
- Production-style config: [`examples/production-config.yml`](../examples/production-config.yml)
|
||||
- Tiny transcript input: [`examples/tiny-transcript.json`](../examples/tiny-transcript.json)
|
||||
- Tiny glossary input: [`examples/tiny-glossary.yaml`](../examples/tiny-glossary.yaml)
|
||||
|
||||
Validate the config examples:
|
||||
|
||||
```sh
|
||||
audita config validate --config examples/minimal-config.yml
|
||||
audita config validate --config examples/production-config.yml
|
||||
```
|
||||
119
docs/integrations/openai-compatible-llm.md
Normal file
119
docs/integrations/openai-compatible-llm.md
Normal file
@@ -0,0 +1,119 @@
|
||||
# OpenAI-Compatible LLM Integration
|
||||
|
||||
## Scope
|
||||
|
||||
This document defines the external LLM endpoint contract Audita currently uses.
|
||||
|
||||
It covers:
|
||||
- endpoint and auth expectations;
|
||||
- structured request and response shape;
|
||||
- retry and timeout behavior;
|
||||
- diagnostics and secret redaction.
|
||||
|
||||
For user-facing CLI flags and config keys, see [`docs/cli.md`](../cli.md) and [`docs/config.md`](../config.md).
|
||||
|
||||
## Endpoint Contract
|
||||
|
||||
Audita sends HTTPS `POST` requests to:
|
||||
|
||||
- `<base_url>/chat/completions`
|
||||
|
||||
`base_url` comes from primary or validation LLM config and is required.
|
||||
|
||||
## Authentication Contract
|
||||
|
||||
When an API key is configured, Audita sends:
|
||||
|
||||
- `Authorization: Bearer <api_key>`
|
||||
|
||||
When no API key is configured, the `Authorization` header is omitted.
|
||||
|
||||
## Request Shape
|
||||
|
||||
Audita sends a chat-completions payload with:
|
||||
- `model`;
|
||||
- `messages` (role/content pairs);
|
||||
- `response_format` using JSON Schema strict mode.
|
||||
|
||||
Representative shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "example-model",
|
||||
"messages": [
|
||||
{"role": "system", "content": "..."},
|
||||
{"role": "user", "content": "..."}
|
||||
],
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "correction_set",
|
||||
"strict": true,
|
||||
"schema": {"type": "object"}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Behavioral requirements enforced by Audita:
|
||||
- `model` must resolve to a non-empty value;
|
||||
- each message must have non-empty `role` and `content`;
|
||||
- `response_format.type` is always `json_schema`;
|
||||
- `response_format.json_schema.name` and `schema` must be present;
|
||||
- request schema JSON must be valid JSON.
|
||||
|
||||
## Response Handling Contract
|
||||
|
||||
Audita expects a successful JSON response with at least one choice and assistant content that can be interpreted as JSON.
|
||||
|
||||
Supported assistant content forms:
|
||||
- string containing JSON;
|
||||
- raw JSON value.
|
||||
|
||||
Audita then decodes the JSON against the expected structured output type.
|
||||
|
||||
Current structured schema identities used by Audita runtime:
|
||||
- `correction_set`
|
||||
- `validator_decision_set`
|
||||
|
||||
## Retries and Timeouts
|
||||
|
||||
Retry behavior:
|
||||
- default max retries is `3` when unset;
|
||||
- retries apply to retryable transport/decode/server-side errors;
|
||||
- HTTP `429` and `5xx` responses are retryable;
|
||||
- retry stops immediately when context is canceled or deadline expires.
|
||||
|
||||
Timeout behavior:
|
||||
- request timeout is derived from configured LLM timeout settings;
|
||||
- timeout/cancellation propagate through HTTP requests and return nonzero process failures.
|
||||
|
||||
## Error Behavior
|
||||
|
||||
Non-2xx responses fail the request.
|
||||
|
||||
Error message extraction behavior:
|
||||
- if provider JSON includes `error.message`, Audita surfaces that message;
|
||||
- else if provider JSON includes top-level `message`, Audita surfaces that;
|
||||
- otherwise Audita surfaces status code plus response body text.
|
||||
|
||||
Malformed or incompatible structured responses fail safely and are surfaced as runtime errors or validator/proposal warnings depending on call site.
|
||||
|
||||
## Secret Redaction
|
||||
|
||||
Configured LLM secrets are redacted from:
|
||||
- surfaced adapter/runtime errors;
|
||||
- LLM diagnostics request/response/error artifacts;
|
||||
- effective config/report artifacts that include LLM configuration material.
|
||||
|
||||
Redaction marker:
|
||||
- `[REDACTED]`
|
||||
|
||||
## Compatibility Boundaries
|
||||
|
||||
This integration documentation applies only to the implemented OpenAI-compatible chat completions flow.
|
||||
|
||||
Not part of current behavior:
|
||||
- provider SDK integration;
|
||||
- non-OpenAI-compatible API contracts;
|
||||
- server-side model routing features beyond explicitly configured model/base URL.
|
||||
99
docs/integrations/subprocess.md
Normal file
99
docs/integrations/subprocess.md
Normal file
@@ -0,0 +1,99 @@
|
||||
# Subprocess Integration
|
||||
|
||||
## Scope
|
||||
|
||||
This document describes how a parent process should invoke Audita as a subprocess.
|
||||
|
||||
It covers:
|
||||
- invocation shape;
|
||||
- stdout/stderr behavior;
|
||||
- output/report file behavior;
|
||||
- diagnostics and exit behavior.
|
||||
|
||||
For full CLI and config references, see [`docs/cli.md`](../cli.md) and [`docs/config.md`](../config.md).
|
||||
|
||||
## Recommended Invocation
|
||||
|
||||
Use explicit output and report paths for machine workflows:
|
||||
|
||||
```sh
|
||||
audita process <transcript.json> \
|
||||
--glossary <glossary.yaml> \
|
||||
--output <output-transcript.json> \
|
||||
--report-json <report.json>
|
||||
```
|
||||
|
||||
Optional commonly used flags:
|
||||
- `--config <path>`
|
||||
- `--output-schema <bare-segments|audita-v1>`
|
||||
- `--work-dir <dir>`
|
||||
- `--work-dir-retention <always|auto|never>`
|
||||
- `--transcript-description <text>`
|
||||
|
||||
## Stdout Contract
|
||||
|
||||
On success:
|
||||
- with `--output`: stdout is expected to be empty;
|
||||
- without `--output`: stdout contains transcript JSON only.
|
||||
|
||||
`--report-json` output is never written to stdout.
|
||||
|
||||
## Stderr Contract
|
||||
|
||||
Stderr is human-readable status/error output.
|
||||
|
||||
On failures:
|
||||
- stderr includes a concise top-level error;
|
||||
- when diagnostics are initialized, stderr includes diagnostics directory path.
|
||||
|
||||
Do not treat stderr as a machine-stable JSON channel.
|
||||
|
||||
## Output and Report File Contract
|
||||
|
||||
Transcript output:
|
||||
- `--output` writes corrected transcript JSON to the provided path;
|
||||
- output write failures return nonzero.
|
||||
|
||||
Report output:
|
||||
- `--report-json` writes machine-readable process report JSON to the provided path;
|
||||
- run diagnostics also attempt to write their own `report.json`;
|
||||
- report write failures return nonzero;
|
||||
- on failure paths, report writing is best-effort and does not mask the primary run error.
|
||||
|
||||
## Diagnostics Contract
|
||||
|
||||
When run-directory initialization succeeds, per-run diagnostics artifacts are written under the configured work directory.
|
||||
|
||||
Typical artifacts include:
|
||||
- `source-transcript.json`
|
||||
- `source-transcript-parsed.json`
|
||||
- `normalized-transcript.json`
|
||||
- `normalization-summary.json`
|
||||
- `chunking-summary.json`
|
||||
- `invocation.json`
|
||||
- `effective-config.json`
|
||||
- `utilization-diagnostics.json`
|
||||
- `correction-ledger.json`
|
||||
- `report.json`
|
||||
- `error.log` (failure)
|
||||
|
||||
Retention behavior is controlled by `--work-dir-retention` / config.
|
||||
|
||||
## Exit Behavior
|
||||
|
||||
Exit codes:
|
||||
- `0`: success;
|
||||
- `1`: runtime processing/output/report failure;
|
||||
- `2`: CLI usage or configuration input error.
|
||||
|
||||
Treat any nonzero as subprocess failure.
|
||||
|
||||
## Parent-Process Guidance
|
||||
|
||||
For reliable orchestration:
|
||||
- read stdout and stderr concurrently to avoid pipe blocking;
|
||||
- prefer `--output` and `--report-json` for machine parsing;
|
||||
- use timeout/cancellation in the parent process;
|
||||
- inspect diagnostics path and `report.json`/`error.log` on failure.
|
||||
|
||||
For input file contracts, see [`docs/integrations/transcript-glossary-files.md`](transcript-glossary-files.md).
|
||||
98
docs/integrations/transcript-glossary-files.md
Normal file
98
docs/integrations/transcript-glossary-files.md
Normal file
@@ -0,0 +1,98 @@
|
||||
# Transcript and Glossary File Integration
|
||||
|
||||
## Scope
|
||||
|
||||
This document defines the input file contracts for:
|
||||
- transcript JSON;
|
||||
- glossary YAML.
|
||||
|
||||
These files are loaded and validated before processing begins.
|
||||
|
||||
## Transcript JSON Contract
|
||||
|
||||
Audita accepts either top-level shape:
|
||||
- JSON array of segments; or
|
||||
- JSON object with a `segments` array.
|
||||
|
||||
Segment fields:
|
||||
- `id` (optional integer in source form);
|
||||
- `speaker` (required non-empty string);
|
||||
- `start` (required finite non-negative number);
|
||||
- `end` (required finite non-negative number, `>= start`);
|
||||
- `text` (required non-empty string);
|
||||
- `categories` (optional string array; entries must be non-empty).
|
||||
|
||||
Additional rules:
|
||||
- transcript must contain at least one segment;
|
||||
- duplicate segment IDs are rejected when IDs are present.
|
||||
|
||||
Example (`examples/tiny-transcript.json`):
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "A",
|
||||
"start": 0.0,
|
||||
"end": 1.2,
|
||||
"text": "hello world"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
## Glossary YAML Contract
|
||||
|
||||
Audita expects top-level `glossary` list entries.
|
||||
|
||||
Entry fields:
|
||||
- `name` (required non-empty string);
|
||||
- `category` (required non-empty string);
|
||||
- `summary` (required non-empty string);
|
||||
- `aliases` (optional list of strings; entries must be non-empty);
|
||||
- `plural` (optional string).
|
||||
|
||||
Additional rules:
|
||||
- glossary must contain at least one entry.
|
||||
|
||||
Example (`examples/tiny-glossary.yaml`):
|
||||
|
||||
```yaml
|
||||
glossary:
|
||||
- name: Audita
|
||||
aliases:
|
||||
- audita
|
||||
category: product
|
||||
summary: The Audita transcript correction CLI.
|
||||
```
|
||||
|
||||
## Validation Failure Behavior
|
||||
|
||||
Representative transcript validation failures:
|
||||
- invalid JSON;
|
||||
- unsupported top-level shape;
|
||||
- empty `speaker` or `text`;
|
||||
- invalid times (`NaN`, `Inf`, negative, or `end < start`);
|
||||
- duplicate IDs;
|
||||
- empty transcript array.
|
||||
|
||||
Representative glossary validation failures:
|
||||
- invalid YAML;
|
||||
- empty or missing glossary entries;
|
||||
- missing required entry fields;
|
||||
- empty alias values.
|
||||
|
||||
These failures surface as schema errors and the process exits nonzero.
|
||||
|
||||
## CLI Usage
|
||||
|
||||
Minimal invocation:
|
||||
|
||||
```sh
|
||||
audita process ./transcript.json --glossary ./glossary.yaml --output ./corrected.json
|
||||
```
|
||||
|
||||
See also:
|
||||
- [`docs/cli.md`](../cli.md)
|
||||
- [`docs/config.md`](../config.md)
|
||||
- [`examples/tiny-transcript.json`](../../examples/tiny-transcript.json)
|
||||
- [`examples/tiny-glossary.yaml`](../../examples/tiny-glossary.yaml)
|
||||
79
docs/internal/diagnostics-reporting.md
Normal file
79
docs/internal/diagnostics-reporting.md
Normal file
@@ -0,0 +1,79 @@
|
||||
# Audita Diagnostics and Reporting
|
||||
|
||||
## Scope
|
||||
|
||||
This document describes diagnostics artifacts, process report mapping, and correction ledger generation.
|
||||
|
||||
## Run Directory Ownership
|
||||
|
||||
`internal/core/diagnostics` owns run-directory creation, artifact writes, and retention decisions.
|
||||
|
||||
Stable artifact names include:
|
||||
- `source-transcript.json`
|
||||
- `source-transcript-parsed.json`
|
||||
- `normalized-transcript.json`
|
||||
- `normalization-summary.json`
|
||||
- `chunking-summary.json`
|
||||
- `utilization-diagnostics.json`
|
||||
- `correction-ledger.json`
|
||||
- `invocation.json`
|
||||
- `effective-config.json`
|
||||
- `report.json`
|
||||
- `error.log` (failure)
|
||||
|
||||
## Process Report Mapping
|
||||
|
||||
`internal/framework/processreport` maps runner/CLI execution facts into `reporting.ProcessReport`.
|
||||
|
||||
Report metadata fields include:
|
||||
- `report_schema_name` (`audita-process-report`)
|
||||
- `report_schema_version` (`v1`)
|
||||
- `output_schema`
|
||||
- `config_version` (when file config exists)
|
||||
|
||||
The report includes:
|
||||
- top-level status/error phase/error message;
|
||||
- normalization/chunking summaries;
|
||||
- diagnostics metadata paths;
|
||||
- per-module results and module summary.
|
||||
|
||||
## Correction Ledger
|
||||
|
||||
`internal/framework/processreport/BuildCorrectionLedger` flattens run results into `correction-ledger.json` entries.
|
||||
|
||||
Dispositions:
|
||||
- `applied`
|
||||
- `skipped`
|
||||
- `rejected`
|
||||
- `failed`
|
||||
|
||||
Validator decisions are split into deterministic and LLM-backed groups using validator metadata classification.
|
||||
|
||||
## Report Write Paths
|
||||
|
||||
- run directory always attempts to write `report.json` when possible;
|
||||
- optional `--report-json` writes an external report file;
|
||||
- on failure paths, report writing is best-effort and does not mask primary run errors.
|
||||
|
||||
## Retention Interaction
|
||||
|
||||
Current retention behavior:
|
||||
- failed runs are retained;
|
||||
- `always` keeps successful runs;
|
||||
- `auto` removes only clean successful runs;
|
||||
- `never` currently retains successful runs in current implementation.
|
||||
|
||||
## Redaction
|
||||
|
||||
Redacted data expectations:
|
||||
- effective config artifact uses config redaction;
|
||||
- diagnostics payloads and surfaced errors use LLM secret redaction;
|
||||
- reports should not include raw API key values.
|
||||
|
||||
## Key Tests
|
||||
|
||||
- `internal/core/diagnostics/*_test.go`
|
||||
- `internal/framework/processreport/*_test.go`
|
||||
- `internal/core/reporting/report_test.go`
|
||||
- `internal/cli/run_test.go`
|
||||
- `cmd/audita/main_integration_test.go`
|
||||
67
docs/internal/llm-runtime.md
Normal file
67
docs/internal/llm-runtime.md
Normal file
@@ -0,0 +1,67 @@
|
||||
# Audita LLM Runtime
|
||||
|
||||
## Scope
|
||||
|
||||
This document describes the structured LLM runtime and scheduler behavior.
|
||||
|
||||
## Client Boundary
|
||||
|
||||
All runtime LLM calls go through `contracts.StructuredLLMClient`.
|
||||
|
||||
Primary adapter:
|
||||
- `internal/framework/llm/OpenAICompatibleClient`
|
||||
|
||||
## Request/Response Behavior
|
||||
|
||||
The OpenAI-compatible adapter sends chat completions requests with:
|
||||
- model;
|
||||
- messages;
|
||||
- `response_format.type = json_schema`;
|
||||
- strict schema envelope (`name`, `schema`, `strict=true`).
|
||||
|
||||
The response is decoded into the requested structured output target.
|
||||
|
||||
## Response Schema Registry
|
||||
|
||||
Structured response schemas are registered in `internal/framework/responseschema`:
|
||||
- `correction_set`
|
||||
- `validator_decision_set`
|
||||
|
||||
Each schema includes stable diagnostics metadata (`id`, `version`, `name`, `sha256`).
|
||||
|
||||
## Retries and Error Handling
|
||||
|
||||
Adapter retries apply to retryable conditions (for example transport/decoding/retryable status classes) up to configured `max_retries`.
|
||||
|
||||
Errors are sanitized to redact configured API-key values before surfacing.
|
||||
|
||||
Malformed structured output detection is shared through `internal/framework/structuredoutput` and is used by:
|
||||
- proposal generation;
|
||||
- LLM-backed validators.
|
||||
|
||||
## Scheduling and Concurrency
|
||||
|
||||
`internal/framework/llm/Scheduler` provides FIFO, context-aware permit gating.
|
||||
|
||||
Runner composes scheduler limits across:
|
||||
- total LLM concurrency;
|
||||
- proposal LLM concurrency;
|
||||
- validation LLM concurrency.
|
||||
|
||||
Scheduler release is guarded to avoid permit leaks on cancellation/error.
|
||||
|
||||
## Diagnostics and Redaction
|
||||
|
||||
`internal/framework/llm/DiagnosticsWriter` writes request/response/error artifacts.
|
||||
|
||||
Configured secrets are derived from `llm.ConfiguredSecrets(cfg)` and redacted from:
|
||||
- diagnostics payloads;
|
||||
- surfaced runtime/adapter errors.
|
||||
|
||||
## Key Tests
|
||||
|
||||
- `internal/framework/llm/openai_compatible_client_test.go`
|
||||
- `internal/framework/llm/scheduler_test.go`
|
||||
- `internal/framework/llm/diagnostics_test.go`
|
||||
- `internal/framework/responseschema/registry_test.go`
|
||||
- `internal/framework/structuredoutput/malformed_test.go`
|
||||
58
docs/internal/modules.md
Normal file
58
docs/internal/modules.md
Normal file
@@ -0,0 +1,58 @@
|
||||
# Audita Modules
|
||||
|
||||
## Scope
|
||||
|
||||
This document covers module contracts and built-in module packages.
|
||||
|
||||
## Module Contract
|
||||
|
||||
Modules implement `contracts.TranscriptModule`:
|
||||
- `Key()`
|
||||
- `ReplacementPolicy()`
|
||||
- `Validators()`
|
||||
- `Propose(ctx, req)`
|
||||
|
||||
Runner resolves configured module specs to module instances through `internal/framework/modules`.
|
||||
|
||||
## Built-In Modules
|
||||
|
||||
Current module packages:
|
||||
- `internal/modules/glossary`
|
||||
- `internal/modules/homophones`
|
||||
- `internal/modules/spoken_word`
|
||||
- `internal/modules/grammar`
|
||||
|
||||
Current replacement policies:
|
||||
- `glossary`: `replace_all`
|
||||
- `homophones`: `require_unique`
|
||||
- `spoken_word`: `require_unique`
|
||||
- `grammar`: `require_unique`
|
||||
|
||||
## Proposal Generation Ownership
|
||||
|
||||
Shared proposal-generation plumbing is centralized in:
|
||||
- `internal/framework/proposal_generation`
|
||||
|
||||
Module packages own:
|
||||
- prompt selection (`internal/prompts` prompt IDs);
|
||||
- module-specific prompt payload construction.
|
||||
|
||||
Shared prompt helpers live in `internal/framework/promptcontext`.
|
||||
|
||||
## Validator Chain Ownership
|
||||
|
||||
Built-in chains are resolved in `internal/validators` per module key.
|
||||
Module packages call the built-in chain resolver at construction.
|
||||
|
||||
## Failure and Warning Behavior
|
||||
|
||||
- module setup failures surface as `runner_setup` or module setup errors;
|
||||
- module runtime failures surface as `runner_execution` with partial module results preserved;
|
||||
- malformed structured proposal payloads are downgraded to warnings and section-level proposal rejection.
|
||||
|
||||
## Key Tests
|
||||
|
||||
- `internal/modules/*/module_test.go`
|
||||
- `internal/framework/modules/registry_test.go`
|
||||
- `internal/framework/proposal_generation/*_test.go`
|
||||
- `internal/cli/run_test.go` (pipeline/report integration)
|
||||
46
docs/internal/output-schemas.md
Normal file
46
docs/internal/output-schemas.md
Normal file
@@ -0,0 +1,46 @@
|
||||
# Audita Output Schemas
|
||||
|
||||
## Scope
|
||||
|
||||
This document describes the implemented transcript output schema registry.
|
||||
|
||||
## Registry Ownership
|
||||
|
||||
Output schema registry is owned by `internal/core/outputschema`.
|
||||
|
||||
Supported schema keys:
|
||||
- `bare-segments`
|
||||
- `audita-v1`
|
||||
|
||||
## Schemas
|
||||
|
||||
`bare-segments`:
|
||||
- top-level JSON array of transcript segments.
|
||||
|
||||
`audita-v1`:
|
||||
- top-level JSON object with:
|
||||
- `schema: "audita-v1"`
|
||||
- `version: "v1"`
|
||||
- `segments: [...]`
|
||||
|
||||
Segment fields include `id`, `speaker`, `start`, `end`, `text`, and optional `categories`.
|
||||
|
||||
## Validation and Resolution
|
||||
|
||||
Config validation and runtime resolution both reject unsupported schema keys.
|
||||
|
||||
Unknown schema keys fail with `unsupported output schema` before output emission.
|
||||
|
||||
## Output Emission
|
||||
|
||||
The selected schema is used by `audita process` when writing:
|
||||
- output file (`--output`) or
|
||||
- stdout (when no `--output`).
|
||||
|
||||
Report metadata records selected `output_schema`.
|
||||
|
||||
## Key Tests
|
||||
|
||||
- `internal/core/outputschema/registry_test.go`
|
||||
- `internal/core/config/config_test.go`
|
||||
- `internal/cli/run_test.go`
|
||||
92
docs/internal/overview.md
Normal file
92
docs/internal/overview.md
Normal file
@@ -0,0 +1,92 @@
|
||||
# Audita Internal Overview
|
||||
|
||||
## Scope
|
||||
|
||||
This document is the internal architecture entry point for developers and coding agents.
|
||||
|
||||
It summarizes:
|
||||
- package boundaries;
|
||||
- the main `process` execution path;
|
||||
- where to add new code safely.
|
||||
|
||||
## Package Map
|
||||
|
||||
CLI and command orchestration:
|
||||
- `internal/cli`
|
||||
|
||||
Core deterministic components:
|
||||
- `internal/core/config`
|
||||
- `internal/core/schema`
|
||||
- `internal/core/normalization`
|
||||
- `internal/core/chunking`
|
||||
- `internal/core/outputschema`
|
||||
- `internal/core/diagnostics`
|
||||
- `internal/core/reporting`
|
||||
- `internal/core/modulecatalog`
|
||||
|
||||
Framework orchestration and contracts:
|
||||
- `internal/framework/contracts`
|
||||
- `internal/framework/modules`
|
||||
- `internal/framework/proposal_generation`
|
||||
- `internal/framework/proposals`
|
||||
- `internal/framework/runner`
|
||||
- `internal/framework/validators`
|
||||
- `internal/framework/llm`
|
||||
- `internal/framework/responseschema`
|
||||
- `internal/framework/structuredoutput`
|
||||
- `internal/framework/processreport`
|
||||
- `internal/framework/promptcontext`
|
||||
- `internal/framework/stagename`
|
||||
|
||||
Domain implementations:
|
||||
- `internal/modules/*`
|
||||
- `internal/validators/*`
|
||||
- `internal/prompts`
|
||||
|
||||
## Main Execution Path (`audita process`)
|
||||
|
||||
High-level flow:
|
||||
1. CLI loads effective config and validates CLI requirements.
|
||||
2. Run directory is created and invocation/effective config artifacts are written.
|
||||
3. Transcript/glossary files are loaded and parsed.
|
||||
4. Transcript is normalized and chunked.
|
||||
5. `runner.Run` executes configured module instances.
|
||||
6. Proposals are validated, applied deterministically, and serialized in selected output schema.
|
||||
7. Process report, utilization diagnostics, correction ledger, and retention decisions are finalized.
|
||||
|
||||
## Boundary Summary
|
||||
|
||||
- `internal/core/*` owns deterministic, reusable logic and persistence-independent rules.
|
||||
- `internal/framework/*` owns orchestration contracts and reusable runtime plumbing.
|
||||
- `internal/modules/*` owns module-specific proposal behavior and prompt usage.
|
||||
- `internal/validators/*` owns validator composition and built-in chain assembly.
|
||||
- `internal/prompts` owns embedded prompt assets and metadata registry.
|
||||
|
||||
## Where To Add New Code
|
||||
|
||||
Add config fields:
|
||||
- `internal/core/config`
|
||||
|
||||
Add module behavior:
|
||||
- one package under `internal/modules/<module_key>`
|
||||
- registration/wiring through `internal/framework/modules` and config module list
|
||||
|
||||
Add validators:
|
||||
- implementation under `internal/validators/<validator_key>`
|
||||
- registry/chain wiring in `internal/validators`
|
||||
|
||||
Add runtime orchestration behavior:
|
||||
- `internal/framework/*` (runner/proposal/validator/LLM plumbing)
|
||||
|
||||
Add CLI surface:
|
||||
- `internal/cli`
|
||||
|
||||
## Related Internal Docs
|
||||
|
||||
- [`docs/internal/pipeline.md`](pipeline.md)
|
||||
- [`docs/internal/modules.md`](modules.md)
|
||||
- [`docs/internal/validators.md`](validators.md)
|
||||
- [`docs/internal/llm-runtime.md`](llm-runtime.md)
|
||||
- [`docs/internal/diagnostics-reporting.md`](diagnostics-reporting.md)
|
||||
- [`docs/internal/prompts.md`](prompts.md)
|
||||
- [`docs/internal/output-schemas.md`](output-schemas.md)
|
||||
79
docs/internal/pipeline.md
Normal file
79
docs/internal/pipeline.md
Normal file
@@ -0,0 +1,79 @@
|
||||
# Audita Internal Pipeline
|
||||
|
||||
## Scope
|
||||
|
||||
This document describes the implemented `audita process` pipeline.
|
||||
|
||||
## Inputs
|
||||
|
||||
Pipeline inputs are:
|
||||
- effective config (`internal/core/config`);
|
||||
- transcript JSON (`internal/core/schema`);
|
||||
- glossary YAML (`internal/core/schema`).
|
||||
|
||||
## Pipeline Phases
|
||||
|
||||
1. Input loading and schema validation
|
||||
- transcript and glossary files are read and parsed.
|
||||
- schema failures stop the run with `transcript_schema` or `glossary_schema`.
|
||||
|
||||
2. Normalization
|
||||
- canonical transcript segments are normalized by configured gap/duration/token settings.
|
||||
- normalization summary artifacts are written.
|
||||
|
||||
3. Chunking
|
||||
- normalized transcript is chunked with configured max/min tokens and target sections.
|
||||
|
||||
4. Module proposal generation
|
||||
- runner executes configured module instances in sequence.
|
||||
- each module proposes corrections per section.
|
||||
- per-section proposal generation can run concurrently.
|
||||
|
||||
5. Validator filtering
|
||||
- validators run on candidate proposals before apply.
|
||||
- deterministic validators run before LLM-backed validators.
|
||||
- LLM validator inputs are batched by max prompt token limit.
|
||||
|
||||
6. Deterministic apply
|
||||
- approved proposals are applied via replacement policy.
|
||||
- applied/skipped/rejected outcomes are recorded.
|
||||
|
||||
7. Output and reporting
|
||||
- final transcript is serialized with selected output schema.
|
||||
- report, utilization diagnostics, and correction ledger are written.
|
||||
- retention policy is applied to run directory.
|
||||
|
||||
## Runner Outputs
|
||||
|
||||
`runner.Run` returns:
|
||||
- final transcript;
|
||||
- per-module results;
|
||||
- utilization diagnostics.
|
||||
|
||||
CLI/reporting then map this into process report and diagnostics artifacts.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
Representative failure phases include:
|
||||
- `run_dir_creation`
|
||||
- `transcript_read`, `glossary_read`
|
||||
- `transcript_schema`, `glossary_schema`
|
||||
- `chunking`
|
||||
- `runner_setup`, `runner_execution`
|
||||
- `output_schema`, `serialization`, `output_write`, `stdout_write`
|
||||
|
||||
When diagnostics are available, failure stderr includes diagnostics path.
|
||||
|
||||
## Invariants
|
||||
|
||||
- module execution order follows configured module sequence;
|
||||
- proposal/validator nondeterminism is isolated before deterministic apply;
|
||||
- proposal indices are assigned deterministically by section order;
|
||||
- output/report artifacts are generated from run results, not speculative state.
|
||||
|
||||
## Key Tests
|
||||
|
||||
- `internal/framework/runner/runner_test.go`
|
||||
- `internal/framework/proposal_generation/*_test.go`
|
||||
- `internal/cli/run_test.go`
|
||||
- `cmd/audita/main_integration_test.go`
|
||||
62
docs/internal/prompts.md
Normal file
62
docs/internal/prompts.md
Normal file
@@ -0,0 +1,62 @@
|
||||
# Audita Prompt Registry
|
||||
|
||||
## Scope
|
||||
|
||||
This document describes embedded prompt assets, prompt metadata, and rendering behavior.
|
||||
|
||||
## Registry Ownership
|
||||
|
||||
Prompt registry lives in `internal/prompts` and embeds assets under `internal/prompts/assets/**`.
|
||||
|
||||
Registered prompt IDs:
|
||||
- `modules.glossary.proposal`
|
||||
- `modules.homophones.proposal`
|
||||
- `modules.spoken_word.proposal`
|
||||
- `modules.grammar.proposal`
|
||||
- `validators.spoken_form_plausibility`
|
||||
- `validators.meaning_reversal_review`
|
||||
- `validators.editorial_review`
|
||||
- `validators.grammar_review`
|
||||
- `validators.spoken_word_review`
|
||||
|
||||
## Metadata Model
|
||||
|
||||
Each prompt has metadata:
|
||||
- `prompt_id`
|
||||
- `prompt_version`
|
||||
- `prompt_source`
|
||||
- `embedded_path`
|
||||
- `sha256`
|
||||
|
||||
Current source/version values:
|
||||
- `prompt_source = builtin`
|
||||
- `prompt_version = v1`
|
||||
|
||||
## Rendering
|
||||
|
||||
`prompts.RenderUserSystem(promptID, data)` renders system/user templates.
|
||||
|
||||
Template behavior:
|
||||
- uses Go `text/template`;
|
||||
- `missingkey=error` is enabled;
|
||||
- output is trimmed.
|
||||
|
||||
A shared hardening fragment is embedded once and referenced by prompt templates.
|
||||
|
||||
## Prompt Context Inputs
|
||||
|
||||
Shared prompt payload helpers:
|
||||
- transcript section JSON (`internal/framework/promptcontext/MarshalTranscriptSectionJSON`)
|
||||
- transcript description block (`TranscriptDescriptionBlock`)
|
||||
|
||||
Modules and LLM validators provide typed data maps to render prompt assets.
|
||||
|
||||
## Diagnostics Integration
|
||||
|
||||
Prompt metadata is attached to proposal/validator diagnostics request metadata using `Metadata.DiagnosticsMap()`.
|
||||
|
||||
## Key Tests
|
||||
|
||||
- `internal/prompts/registry_test.go`
|
||||
- `internal/framework/promptcontext/*_test.go`
|
||||
- module and validator prompt builder tests
|
||||
72
docs/internal/validators.md
Normal file
72
docs/internal/validators.md
Normal file
@@ -0,0 +1,72 @@
|
||||
# Audita Validators
|
||||
|
||||
## Scope
|
||||
|
||||
This document describes validator composition, execution order, and decision handling.
|
||||
|
||||
## Ownership
|
||||
|
||||
Built-in validator keys and chains:
|
||||
- `internal/validators`
|
||||
|
||||
Shared validator runtime mechanics:
|
||||
- `internal/framework/validators`
|
||||
|
||||
Execution-class metadata:
|
||||
- `internal/validators/metadata`
|
||||
|
||||
## Built-In Validator Keys
|
||||
|
||||
Deterministic:
|
||||
- `proposal_shape`
|
||||
- `confidence_threshold`
|
||||
- `original_text_presence`
|
||||
- `non_empty_corrected_text`
|
||||
- `no_effect`
|
||||
- `protected_terms`
|
||||
|
||||
LLM-backed:
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
- `editorial_review`
|
||||
|
||||
## Built-In Chains
|
||||
|
||||
Module chains are defined in `internal/validators/chains.go`.
|
||||
Glossary, homophones, spoken_word, and grammar each resolve a fixed ordered chain.
|
||||
|
||||
## Runtime Execution
|
||||
|
||||
For each module section:
|
||||
1. run deterministic validators;
|
||||
2. run LLM-backed validators;
|
||||
3. record decisions and warnings;
|
||||
4. carry only approved proposals forward.
|
||||
|
||||
Decision cardinality is enforced: each candidate proposal must receive exactly one decision per validator.
|
||||
|
||||
## LLM Validator Batching
|
||||
|
||||
LLM validators:
|
||||
- build canonical validation request payloads;
|
||||
- batch by `validation_max_prompt_tokens`;
|
||||
- call structured LLM client using response schema registry.
|
||||
|
||||
Oversized single proposals are rejected with `validator_input_too_large`.
|
||||
Malformed LLM validator responses are downgraded to warnings and rejected batch decisions.
|
||||
|
||||
## Decision and Rejection Reporting
|
||||
|
||||
Runner records:
|
||||
- `validator_decisions`
|
||||
- `validator_rejected`
|
||||
- warning records (including malformed response warnings)
|
||||
|
||||
Correction ledger classifies deterministic vs LLM validator decisions using canonical metadata classes.
|
||||
|
||||
## Key Tests
|
||||
|
||||
- `internal/validators/*_test.go`
|
||||
- `internal/framework/validators/*_test.go`
|
||||
- `internal/framework/processreport/correction_ledger_test.go`
|
||||
- `internal/cli/run_test.go`
|
||||
113
docs/operations.md
Normal file
113
docs/operations.md
Normal file
@@ -0,0 +1,113 @@
|
||||
# Audita Operations
|
||||
|
||||
## Scope
|
||||
|
||||
This document covers operational behavior for `audita process` as currently implemented:
|
||||
- run lifecycle;
|
||||
- output and report files;
|
||||
- diagnostics artifacts;
|
||||
- run-directory retention behavior;
|
||||
- failure inspection and recovery.
|
||||
|
||||
For command syntax, see [`docs/cli.md`](cli.md).
|
||||
|
||||
## Process Run Lifecycle
|
||||
|
||||
A `process` run performs these high-level steps:
|
||||
1. load effective config (defaults + optional file + env + CLI);
|
||||
2. create a per-run diagnostics directory;
|
||||
3. load transcript JSON and glossary YAML;
|
||||
4. parse/validate input schemas;
|
||||
5. normalize transcript and compute chunking;
|
||||
6. run configured modules/validators;
|
||||
7. serialize output schema and write transcript output;
|
||||
8. build and write process report;
|
||||
9. apply run-directory retention.
|
||||
|
||||
If a failure happens after diagnostics initialization, the run writes failure details and returns nonzero.
|
||||
|
||||
## Output Files
|
||||
|
||||
Transcript output:
|
||||
- when `--output <path>` is set, corrected transcript JSON is written to that file;
|
||||
- when `--output` is omitted, corrected transcript JSON is written to stdout.
|
||||
|
||||
Report output:
|
||||
- when `--report-json <path>` is set, Audita writes a process report JSON file;
|
||||
- the run directory also writes its own `report.json` artifact.
|
||||
|
||||
On success with `--output`, stdout is expected to be empty.
|
||||
|
||||
## Diagnostics Directory
|
||||
|
||||
By default, runs use `work_dir` from effective config (default `/tmp/audita`).
|
||||
Each run directory is created under the work dir using a generated ID like `run-<unix-nanos>`.
|
||||
|
||||
Top-level diagnostics artifacts:
|
||||
- `source-transcript.json`
|
||||
- `source-transcript-parsed.json`
|
||||
- `normalized-transcript.json`
|
||||
- `normalization-summary.json`
|
||||
- `chunking-summary.json`
|
||||
- `utilization-diagnostics.json`
|
||||
- `correction-ledger.json`
|
||||
- `invocation.json`
|
||||
- `effective-config.json` (redacted)
|
||||
- `report.json`
|
||||
- `error.log` (failure runs)
|
||||
|
||||
Report diagnostics metadata includes resolved paths to these artifacts.
|
||||
|
||||
## Correction Ledger and Utilization Diagnostics
|
||||
|
||||
`correction-ledger.json` records correction dispositions:
|
||||
- `applied`
|
||||
- `skipped`
|
||||
- `rejected`
|
||||
- `failed`
|
||||
|
||||
`utilization-diagnostics.json` records effective concurrency and execution timing summaries for run/module/validator activity.
|
||||
|
||||
## Retention Behavior
|
||||
|
||||
Retention is controlled by `work_dir_retention` (`auto|always|never`).
|
||||
|
||||
Current behavior:
|
||||
- failed runs are always retained;
|
||||
- `always`: successful runs are retained;
|
||||
- `auto`: successful runs are retained only when skipped/rejected corrections occurred; clean successful runs are removed;
|
||||
- `never`: successful runs are currently retained (same net retention outcome as `always` in current implementation).
|
||||
|
||||
Even when a successful run directory is removed under `auto`, an explicit `--report-json` file is still preserved at its target path.
|
||||
|
||||
## Failure Inspection
|
||||
|
||||
For failed runs:
|
||||
1. read stderr for the top-level failure and diagnostics path;
|
||||
2. open `error.log` in the reported run directory;
|
||||
3. inspect run `report.json` (`status`, `error_phase`, `error_message`);
|
||||
4. inspect related artifacts referenced by report diagnostics metadata.
|
||||
|
||||
Typical `error_phase` values include:
|
||||
- `transcript_read`
|
||||
- `glossary_read`
|
||||
- `transcript_schema`
|
||||
- `glossary_schema`
|
||||
- `chunking`
|
||||
- `runner_setup`
|
||||
- `runner_execution`
|
||||
- `output_schema`
|
||||
- `serialization`
|
||||
- `output_write`
|
||||
- `stdout_write`
|
||||
|
||||
## Recovery Guidance
|
||||
|
||||
Safe recovery pattern:
|
||||
1. correct the immediate input/config/output-path problem;
|
||||
2. rerun with `--work-dir-retention always` during debugging;
|
||||
3. once stable, restore your normal retention mode.
|
||||
|
||||
Not implemented:
|
||||
- resume/checkpoint APIs
|
||||
- remote diagnostics/report storage
|
||||
207
docs/policy/architecture.md
Normal file
207
docs/policy/architecture.md
Normal file
@@ -0,0 +1,207 @@
|
||||
# Architecture Policy
|
||||
|
||||
## Purpose
|
||||
|
||||
This document defines Audita's development architecture and invariants for maintainers and LLM coding agents. It describes how the project is intended to be changed safely, based on behavior implemented in this repository today.
|
||||
|
||||
User-facing behavior belongs in the README and focused runtime docs. Future or proposed work belongs only under `docs/roadmap/`.
|
||||
|
||||
## Project Shape
|
||||
|
||||
Audita is a single-process Go CLI for transcript polishing. The executable entrypoint is `cmd/audita`; command handling lives in `internal/cli`.
|
||||
|
||||
The implemented `audita process` flow is:
|
||||
|
||||
1. load effective config;
|
||||
2. read and validate transcript JSON and glossary YAML;
|
||||
3. normalize transcript segments;
|
||||
4. chunk the working transcript into sections;
|
||||
5. resolve configured module instances;
|
||||
6. run correction modules and validator chains;
|
||||
7. apply approved proposals deterministically;
|
||||
8. write transcript output, reports, and diagnostics artifacts.
|
||||
|
||||
The current built-in modules are `glossary`, `homophones`, `spoken_word`, and `grammar`. The default configured module sequence repeats `glossary`.
|
||||
|
||||
For external behavior and compatibility details, prefer links to existing behavior docs:
|
||||
|
||||
- [CLI reference](../cli.md)
|
||||
- [Configuration](../config.md)
|
||||
- [Operations](../operations.md)
|
||||
- [Troubleshooting](../troubleshooting.md)
|
||||
- [Integration docs](../integrations/subprocess.md)
|
||||
- [Internal docs](../internal/overview.md)
|
||||
|
||||
## Core Design Principles
|
||||
|
||||
- **Hexagonal architecture:** keep domain behavior behind narrow internal contracts. CLI, filesystem, config loading, diagnostics writing, and LLM transport are adapters around the core processing flow.
|
||||
- **Composable modules and validators:** correction stages and validators should remain small, explicit, and independently testable.
|
||||
- **Deterministic orchestration around LLM calls:** LLM responses are nondeterministic inputs. Proposal indexing, validator ordering, proposal application, reports, and output serialization must remain deterministic.
|
||||
- **Bounded and observable concurrency:** use the implemented schedulers and configured concurrency limits for LLM call sites. Preserve utilization diagnostics when changing scheduling or orchestration.
|
||||
- **Conservative correction behavior:** validate proposed corrections before application; apply accepted proposals through deterministic apply-time safety checks.
|
||||
- **Standard-library-first:** prefer the Go standard library. Narrow third-party dependencies are acceptable when they materially improve maintainability, such as `gopkg.in/yaml.v3` for YAML parsing.
|
||||
- **Current-behavior documentation:** non-roadmap docs must describe implemented behavior only.
|
||||
|
||||
## Architectural Boundaries
|
||||
|
||||
`internal/core` owns domain data handling and stable runtime contracts that do not require CLI or provider transport knowledge:
|
||||
|
||||
- config defaults, loading, validation, redaction, and catalogs;
|
||||
- transcript and glossary schemas;
|
||||
- normalization and chunking;
|
||||
- output-schema encoding;
|
||||
- diagnostics artifact naming and run-directory helpers;
|
||||
- public process report shapes.
|
||||
|
||||
`internal/framework` owns orchestration contracts and reusable runtime mechanics:
|
||||
|
||||
- module and validator interfaces;
|
||||
- proposal generation, proposal application, and prompt context;
|
||||
- runner orchestration;
|
||||
- LLM scheduler, OpenAI-compatible adapter, redaction helpers, and diagnostics writers;
|
||||
- structured response schema registry;
|
||||
- process report and correction-ledger assembly.
|
||||
|
||||
`internal/modules/*` owns module-specific correction stages. `internal/validators/*` owns built-in validator implementations, registry, chains, and execution-class metadata. `internal/prompts` owns embedded prompt assets and prompt metadata.
|
||||
|
||||
`internal/cli` owns command parsing, exit codes, stdout/stderr behavior, config command behavior, filesystem input/output wiring, and top-level process orchestration. CLI concerns should not move into modules, validators, or schema logic.
|
||||
|
||||
Tests should stay close to the behavior they protect. Shared test helpers are acceptable when they remove clear duplication without hiding module-specific behavior.
|
||||
|
||||
## Modules and Validators
|
||||
|
||||
Modules implement `contracts.TranscriptModule`. A module must provide:
|
||||
|
||||
- a stable key;
|
||||
- a replacement policy;
|
||||
- a validator chain;
|
||||
- proposal generation from explicit request inputs.
|
||||
|
||||
Module packages should stay separate. Do not collapse module-specific prompts, scope, or validation choices into a broad generic stage abstraction.
|
||||
|
||||
Validators implement the shared validator contract and return one decision per candidate proposal. Deterministic validators and LLM-backed validators are both composable chain elements. Validator identity and execution class metadata are stable enough to affect ordering, diagnostics, reports, and correction-ledger classification.
|
||||
|
||||
Future module or validator changes should preserve:
|
||||
|
||||
- explicit inputs and outputs;
|
||||
- no hidden global state;
|
||||
- explicit config dependencies;
|
||||
- deterministic proposal index handling;
|
||||
- validation before final mutation;
|
||||
- stable reason codes and validator keys where already exposed.
|
||||
|
||||
## LLM Integration and Concurrency
|
||||
|
||||
LLM calls are external effects behind narrow contracts. Production structured completions use `contracts.StructuredLLMClient`; the implemented provider adapter is OpenAI-compatible HTTP code in `internal/framework/llm`.
|
||||
|
||||
Structured response schemas are registered in `internal/framework/responseschema`. Provider-side schema enforcement is not a substitute for local validation: Audita still validates proposal structure, validator decision cardinality, and apply-time safety.
|
||||
|
||||
Concurrency is bounded by configured scheduler limits:
|
||||
|
||||
- total LLM concurrency;
|
||||
- proposal LLM concurrency;
|
||||
- validation LLM concurrency.
|
||||
|
||||
The scheduler is context-aware and releases permits on success, failure, and cancellation. Runner code may collect section-level work concurrently, but transcript mutation is applied later in deterministic proposal-index order.
|
||||
|
||||
Diagnostics for LLM interactions should be useful for debugging without leaking configured secrets. Use the existing redaction helpers and `llm.ConfiguredSecrets`.
|
||||
|
||||
## State, Inputs, and Outputs
|
||||
|
||||
Audita does not implement resume, checkpoint, manifest, or remote storage behavior. Runtime state is in memory plus per-run diagnostics artifacts written under the configured work directory.
|
||||
|
||||
Transcript input accepts the implemented JSON forms documented in the public contract. Parsed source transcripts are normalized into Audita's internal transcript shape before chunking and module execution.
|
||||
|
||||
Proposals and validator decisions are intermediate runtime data. Approved proposals are applied through `internal/framework/proposals`, which clones transcript state, orders by proposal index, and records applied or skipped changes.
|
||||
|
||||
Transcript output is encoded through `internal/core/outputschema`. Reports and correction ledgers are machine-readable artifacts derived from runner outputs; their public shape should not be changed casually.
|
||||
|
||||
## Configuration and CLI Boundaries
|
||||
|
||||
Config behavior is owned by `internal/core/config`; command usage and process wiring are owned by `internal/cli`.
|
||||
|
||||
`audita process` uses implemented precedence: defaults, config file, environment, then CLI flags. `config validate` validates defaults plus a file config and intentionally does not apply environment overrides. `config print-effective` applies defaults, file config, and environment overrides, then prints redacted JSON.
|
||||
|
||||
Do not duplicate full CLI or config reference material here. Use [Configuration](../config.md), [CLI reference](../cli.md), [Operations](../operations.md), and integration docs under [`docs/integrations/`](../integrations/subprocess.md) for current external behavior.
|
||||
|
||||
When adding config fields or CLI flags, update:
|
||||
|
||||
- config defaults, file/env/CLI application, and validation;
|
||||
- CLI flag extraction if applicable;
|
||||
- redaction when secrets are involved;
|
||||
- tests for precedence and source-specific behavior;
|
||||
- user-facing docs if external behavior changes.
|
||||
|
||||
## Errors, Logging, and Diagnostics
|
||||
|
||||
Errors should be phase-specific enough for CLI users and subprocess callers. The CLI writes human-readable errors to stderr and preserves transcript JSON-only stdout behavior on successful stdout output.
|
||||
|
||||
Run diagnostics are best-effort after run-directory creation. Failed runs are retained. Successful run retention follows the implemented work-dir retention policy.
|
||||
|
||||
Diagnostics and reports must not leak configured LLM secrets. Config redaction and LLM payload/error redaction are separate responsibilities and should remain separate.
|
||||
|
||||
Process reports, diagnostics metadata, utilization diagnostics, and correction ledgers are part of the public contract. Prefer additive, compatible changes.
|
||||
|
||||
## Testing Expectations
|
||||
|
||||
Use targeted package tests for touched behavior and `go test ./...` for substantial changes.
|
||||
|
||||
When changing modules, inspect or add:
|
||||
|
||||
- package-local module tests under `internal/modules/*`;
|
||||
- prompt rendering or proposal-generation tests when prompt inputs change;
|
||||
- parity or release fixtures when public output behavior changes.
|
||||
|
||||
When changing validators, inspect or add:
|
||||
|
||||
- validator package tests;
|
||||
- registry and chain tests under `internal/validators`;
|
||||
- framework validator tests for batching, malformed output, diagnostics, and cardinality.
|
||||
|
||||
When changing LLM integration or concurrency, inspect or add:
|
||||
|
||||
- `internal/framework/llm` scheduler/client/redaction tests;
|
||||
- `internal/framework/runner` orchestration and utilization tests;
|
||||
- structured-output malformed classification tests.
|
||||
|
||||
When changing config, CLI, schema, output, reports, or diagnostics, inspect or add:
|
||||
|
||||
- `internal/core/config` tests;
|
||||
- CLI tests under `internal/cli`;
|
||||
- schema and output-schema tests under `internal/core`;
|
||||
- report, diagnostics, parity, and release-fixture tests.
|
||||
|
||||
## Dependency Policy
|
||||
|
||||
Audita should remain dependency-light. Prefer standard-library solutions for CLI parsing, HTTP, JSON, filesystem, synchronization, and tests.
|
||||
|
||||
Third-party dependencies should be narrow, justified, and preferably de facto standard for their purpose. YAML parsing is the current direct dependency exception.
|
||||
|
||||
Do not add broad frameworks for CLI, dependency injection, workflow orchestration, logging, or plugin systems without a concrete implemented need and focused tests.
|
||||
|
||||
## Documentation Expectations
|
||||
|
||||
Follow [Documentation Policy](./documentation.md). Architecture policy must stay concise and aligned with implemented behavior.
|
||||
|
||||
Do not use architecture docs as changelogs. Do not describe planned modules, adapters, modes, persistence, or configuration unless they are implemented. Put future work under `docs/roadmap/`.
|
||||
|
||||
## Architectural Invariants
|
||||
|
||||
- Keep LLM transport behind `StructuredLLMClient` and framework adapter boundaries.
|
||||
- Keep correction modules narrowly scoped and package-separated.
|
||||
- Keep validators modular, composable, and identified by stable keys.
|
||||
- Keep CLI/config/filesystem concerns out of module and validator domain logic.
|
||||
- Preserve deterministic transcript mutation and output handling around nondeterministic LLM calls.
|
||||
- Keep LLM concurrency bounded, configurable, and observable where implemented.
|
||||
- Keep run diagnostics and reports redacted and machine-readable.
|
||||
- Keep public CLI, config, output-schema, diagnostics, report, prompt, module, and validator contracts stable unless a change is explicit and tested.
|
||||
- Prefer small shared helpers over broad rewrites.
|
||||
- Avoid new dependencies unless they are narrow and clearly justified.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- No plugin framework is implemented.
|
||||
- No generic workflow engine is implemented.
|
||||
- No resume, checkpoint, manifest, or remote storage system is implemented.
|
||||
- No multi-process service mode is implemented.
|
||||
- No provider SDK abstraction beyond the current structured LLM client contract and OpenAI-compatible HTTP adapter is implemented.
|
||||
124
docs/policy/development.md
Normal file
124
docs/policy/development.md
Normal file
@@ -0,0 +1,124 @@
|
||||
# Audita Development Workflow
|
||||
|
||||
## Scope
|
||||
|
||||
This is the canonical contributor workflow for Audita maintainers and coding agents.
|
||||
|
||||
It defines:
|
||||
- repository layout and boundaries;
|
||||
- setup and test commands;
|
||||
- expectations for code changes;
|
||||
- how to add config, CLI, modules, validators, docs, and examples.
|
||||
|
||||
## Setup
|
||||
|
||||
Prerequisites:
|
||||
- Go `1.24` or newer.
|
||||
|
||||
Common commands:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go build ./cmd/audita
|
||||
```
|
||||
|
||||
## Repository Layout
|
||||
|
||||
Top-level areas:
|
||||
- `cmd/audita`: executable entrypoint.
|
||||
- `internal/cli`: command parsing and process/config command orchestration.
|
||||
- `internal/core`: deterministic config/schema/normalization/chunking/output/diagnostics/reporting logic.
|
||||
- `internal/framework`: runner orchestration, contracts, proposal generation/application, validators runtime, LLM runtime, response schemas.
|
||||
- `internal/modules/*`: module-specific correction behavior.
|
||||
- `internal/validators/*`: validator implementations, chains, and metadata.
|
||||
- `internal/prompts`: embedded prompts and prompt metadata.
|
||||
- `docs/`: canonical documentation.
|
||||
- `examples/`: maintained copyable inputs/configs.
|
||||
|
||||
## Change Workflow
|
||||
|
||||
1. Confirm scope and behavior contract before editing.
|
||||
2. Make focused changes in the appropriate ownership area.
|
||||
3. Add or update tests for changed behavior.
|
||||
4. Run targeted package tests for touched areas.
|
||||
5. Run `go test ./...` for substantial changes.
|
||||
6. Update docs/examples when external behavior changes.
|
||||
|
||||
## How To Add or Change Configuration
|
||||
|
||||
1. Add fields/defaults/validation under `internal/core/config`.
|
||||
2. Apply source precedence correctly (defaults, file, env, CLI for `process`).
|
||||
3. Ensure `config validate` remains file-only and `config print-effective` remains redacted.
|
||||
4. Update tests in `internal/core/config` and related CLI tests.
|
||||
5. Update [`docs/config.md`](../config.md) and relevant examples under `examples/`.
|
||||
|
||||
## How To Add or Change CLI Behavior
|
||||
|
||||
1. Implement parsing/wiring in `internal/cli`.
|
||||
2. Keep stdout/stderr and exit behavior compatible unless intentional and documented.
|
||||
3. Update CLI tests under `internal/cli` and integration tests under `cmd/audita`.
|
||||
4. Update [`docs/cli.md`](../cli.md) and related integration docs.
|
||||
|
||||
## How To Add or Change Modules
|
||||
|
||||
1. Add or update one module package under `internal/modules/<module_key>`.
|
||||
2. Keep module-specific prompt ownership in the module + `internal/prompts`.
|
||||
3. Wire module registration/catalog resolution through framework/core module catalog code.
|
||||
4. Verify replacement policy and validator chain selection.
|
||||
5. Add/update module tests and proposal-generation tests.
|
||||
6. Update internal docs when behavior/contracts change.
|
||||
|
||||
## How To Add or Change Validators
|
||||
|
||||
1. Implement validator behavior in `internal/validators` and shared runtime pieces in `internal/framework/validators` only when needed.
|
||||
2. Preserve stable validator keys and decision semantics where already exposed.
|
||||
3. Keep deterministic vs LLM-backed execution-class behavior explicit.
|
||||
4. Add/update validator, chain, batching, and malformed-output tests.
|
||||
5. Update validator documentation when external or developer-facing behavior changes.
|
||||
|
||||
## Documentation and Examples Expectations
|
||||
|
||||
- Keep one canonical home per topic (see [`docs/policy/documentation.md`](documentation.md)).
|
||||
- Do not document future/unimplemented behavior outside `docs/roadmap/`.
|
||||
- Keep command examples and config/examples in sync with current code.
|
||||
- Keep examples secret-free and copyable.
|
||||
|
||||
## Practical Validation Checklist
|
||||
|
||||
Use this checklist for meaningful runtime-impacting changes:
|
||||
|
||||
1. Run core tests:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
```
|
||||
|
||||
2. Verify config commands and examples:
|
||||
|
||||
```sh
|
||||
go run ./cmd/audita config validate --config examples/minimal-config.yml
|
||||
go run ./cmd/audita config validate --config examples/production-config.yml
|
||||
go run ./cmd/audita config print-effective --config examples/minimal-config.yml
|
||||
```
|
||||
|
||||
3. Re-check subprocess/runtime contract when touching CLI/process/report paths:
|
||||
- `--output` success keeps stdout empty;
|
||||
- no `--output` success writes transcript JSON to stdout;
|
||||
- `--report-json` writes file output and is not written to stdout;
|
||||
- failures return nonzero and include diagnostics path when available.
|
||||
|
||||
4. Re-check diagnostics/report/redaction when touching LLM, reporting, or diagnostics code:
|
||||
- report schema metadata fields remain present;
|
||||
- diagnostics artifact paths remain valid;
|
||||
- configured secret values remain redacted in reports/diagnostics/errors.
|
||||
|
||||
5. Re-check output schema behavior when touching serialization/schema code:
|
||||
- default `bare-segments` behavior remains correct unless intentionally changed;
|
||||
- `audita-v1` behavior remains correct unless intentionally changed;
|
||||
- unsupported schemas fail validation/resolve paths clearly.
|
||||
|
||||
## Commit Discipline
|
||||
|
||||
- Keep commits scoped and reviewable.
|
||||
- Avoid mixing unrelated refactors with behavior changes.
|
||||
- Use concise plain-English commit messages.
|
||||
356
docs/policy/documentation.md
Normal file
356
docs/policy/documentation.md
Normal file
@@ -0,0 +1,356 @@
|
||||
# Go Project Documentation Policy
|
||||
|
||||
## Purpose
|
||||
|
||||
Project documentation must help four audiences:
|
||||
|
||||
1. users who need to run the application;
|
||||
2. administrators/operators who need to configure and operate it;
|
||||
3. developers who need to understand and change it safely;
|
||||
4. LLM coding agents that need clear scope, boundaries, and invariants.
|
||||
|
||||
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
||||
|
||||
## Core Rules
|
||||
|
||||
### 1. Keep docs concise
|
||||
|
||||
Each document should cover a defined scope and only the essentials for that scope.
|
||||
|
||||
Avoid:
|
||||
- long background explanations;
|
||||
- repeated reference material;
|
||||
- implementation detail in user-facing docs;
|
||||
- aspirational language outside roadmap docs;
|
||||
- verbose examples where one minimal example is clearer.
|
||||
|
||||
### 2. Document only implemented behavior outside roadmap files
|
||||
|
||||
Unimplemented, planned, aspirational, experimental, or future work may be described only under:
|
||||
|
||||
- `docs/roadmap/`
|
||||
|
||||
No other documentation file, including `README.md`, should describe code, features, modules, stages, commands, config fields, or behaviors that do not currently exist.
|
||||
|
||||
If a feature is partial, non-roadmap docs may describe only the implemented portion and its current boundary.
|
||||
|
||||
### 3. Use canonical homes
|
||||
|
||||
Each type of information should have one canonical location.
|
||||
|
||||
Canonical homes:
|
||||
|
||||
- project purpose and quickstart: `README.md`
|
||||
- development principles: `docs/policy/architecture.md`
|
||||
- configuration reference: `docs/config.md`
|
||||
- CLI reference: `docs/cli.md`
|
||||
- operations and recovery: `docs/operations.md`
|
||||
- troubleshooting: `docs/troubleshooting.md`
|
||||
- implemented internals: `docs/internal/`
|
||||
- future work: `docs/roadmap/`
|
||||
- contributor workflow: `docs/policy/development.md`
|
||||
- copyable examples: `examples/`
|
||||
|
||||
Other files should summarize briefly and link to the canonical source.
|
||||
|
||||
### 4. Keep examples real
|
||||
|
||||
Examples should be valid, maintained, and free of secrets.
|
||||
|
||||
Where practical:
|
||||
- example configs should load successfully;
|
||||
- example commands should match real CLI syntax;
|
||||
- important examples should be covered by tests.
|
||||
|
||||
## Documentation Profiles
|
||||
|
||||
All projects require:
|
||||
|
||||
- `README.md`
|
||||
- `docs/policy/architecture.md`
|
||||
|
||||
Additional docs depend on the project.
|
||||
|
||||
### Small library
|
||||
|
||||
Recommended:
|
||||
- `docs/policy/development.md`, if contributor conventions are non-obvious
|
||||
|
||||
### Simple CLI
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/policy/development.md`
|
||||
|
||||
### Config-driven CLI
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`
|
||||
- `docs/config.md`
|
||||
|
||||
Recommended:
|
||||
- `examples/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
### Stateful or operator-facing application
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- `examples/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
### Modular, staged, service-oriented, or orchestration application
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
- `docs/internal/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- validated examples under `examples/`
|
||||
|
||||
## Required Documents
|
||||
|
||||
### README.md
|
||||
|
||||
**Audience:** users, administrators, operators
|
||||
|
||||
The README is the outward-facing project orientation page.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. concise description;
|
||||
2. elevator pitch;
|
||||
3. shortest useful command or usage example;
|
||||
4. links to targeted docs.
|
||||
|
||||
The README should be short. It is not a manual.
|
||||
|
||||
The “shortest useful command” means the simplest command that performs the project’s core use case. (It does not mean `app --help`.)
|
||||
|
||||
### docs/policy/architecture.md
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
`docs/policy/architecture.md` is required for every project.
|
||||
|
||||
It is an inward-facing development policy document. It should describe how the project is intended to be built and changed.
|
||||
|
||||
It should include:
|
||||
|
||||
- project shape;
|
||||
- core design principles;
|
||||
- package and boundary philosophy;
|
||||
- state/persistence philosophy, if applicable;
|
||||
- external integration philosophy, if applicable;
|
||||
- error-handling and logging principles;
|
||||
- testing expectations;
|
||||
- documentation expectations;
|
||||
- architectural invariants;
|
||||
- explicit non-goals, if useful.
|
||||
|
||||
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
|
||||
|
||||
### docs/policy/development.md
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for projects maintained by humans and LLM coding agents.
|
||||
|
||||
It should include:
|
||||
|
||||
- repository layout;
|
||||
- build/test commands;
|
||||
- coding conventions;
|
||||
- dependency policy;
|
||||
- how to add config fields;
|
||||
- how to add CLI flags;
|
||||
- how to add stages/modules/adapters, if applicable;
|
||||
- how to update examples;
|
||||
- documentation update expectations.
|
||||
|
||||
### docs/config.md
|
||||
|
||||
**Audience:** administrators, operators, advanced users
|
||||
|
||||
Required for applications with configuration files.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. config file locations and discovery precedence;
|
||||
2. minimal working config;
|
||||
3. production-oriented config;
|
||||
4. full configuration reference;
|
||||
5. secrets handling, if applicable;
|
||||
6. links to maintained examples.
|
||||
|
||||
The full configuration reference should be canonical.
|
||||
|
||||
### docs/cli.md
|
||||
|
||||
**Audience:** users, administrators, operators
|
||||
|
||||
Required for CLI applications.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. shortest useful command;
|
||||
2. command overview;
|
||||
3. complete flag reference;
|
||||
4. common workflows;
|
||||
5. diagnostic or recovery commands, if applicable.
|
||||
|
||||
Explain when commands are useful, not just their syntax.
|
||||
|
||||
### docs/operations.md
|
||||
|
||||
**Audience:** administrators, operators
|
||||
|
||||
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
|
||||
|
||||
It should cover:
|
||||
|
||||
- normal workflow;
|
||||
- filesystem layout;
|
||||
- remote storage layout, if applicable;
|
||||
- logs and manifests;
|
||||
- resume/retry behavior;
|
||||
- cleanup behavior;
|
||||
- archive/backup behavior;
|
||||
- safe recovery procedures;
|
||||
- operational caveats.
|
||||
|
||||
### docs/troubleshooting.md
|
||||
|
||||
**Audience:** administrators, operators
|
||||
|
||||
Recommended once recurring failure modes exist.
|
||||
|
||||
Each entry should include:
|
||||
|
||||
- symptom;
|
||||
- likely cause;
|
||||
- diagnostic command or inspection step;
|
||||
- safe fix;
|
||||
- relevant links.
|
||||
|
||||
### docs/internal/
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for modular, staged, service-oriented, or orchestration projects.
|
||||
|
||||
This directory describes implemented internal components. It is not the roadmap.
|
||||
|
||||
Use one file per major component where useful.
|
||||
|
||||
Each component doc should include:
|
||||
|
||||
1. purpose;
|
||||
2. inputs and outputs;
|
||||
3. boundaries;
|
||||
4. config fields used;
|
||||
5. external adapters used;
|
||||
6. state or manifest behavior, if applicable;
|
||||
7. skip/resume behavior, if applicable;
|
||||
8. failure behavior;
|
||||
9. tests to inspect before changing;
|
||||
10. architectural invariants.
|
||||
|
||||
### docs/roadmap/
|
||||
|
||||
**Audience:** maintainers, developers, LLM coding agents
|
||||
|
||||
This is the only place for planned, future, aspirational, experimental, or unimplemented work.
|
||||
|
||||
Roadmap docs should clearly distinguish:
|
||||
|
||||
- proposed work;
|
||||
- accepted plans;
|
||||
- deferred ideas;
|
||||
- rejected ideas;
|
||||
- implementation prompts or task breakdowns, if useful.
|
||||
|
||||
Roadmap docs should not be confused with current behavior.
|
||||
|
||||
### docs/integrations/
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
||||
|
||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
|
||||
|
||||
Use one file per integration where useful.
|
||||
|
||||
## Examples Directory
|
||||
|
||||
Projects with non-trivial configuration or workflows should include `examples/`.
|
||||
|
||||
Useful examples include:
|
||||
|
||||
- minimal working config;
|
||||
- production-oriented config;
|
||||
- full annotated config;
|
||||
- local development config;
|
||||
- remote/object-storage config;
|
||||
- minimal session/input file.
|
||||
|
||||
Examples should be valid, maintained, tested when practical, and linked from relevant docs.
|
||||
|
||||
## Security and Privacy
|
||||
|
||||
Docs and examples must not include:
|
||||
|
||||
- real API keys;
|
||||
- tokens;
|
||||
- passwords;
|
||||
- private keys;
|
||||
- private environment dumps;
|
||||
- sensitive user data;
|
||||
- raw private transcripts;
|
||||
- private infrastructure details unless intentionally public.
|
||||
|
||||
Document secret-handling mechanisms, not actual secret values.
|
||||
|
||||
## Maintenance Rules
|
||||
|
||||
When docs change, verify the affected behavior.
|
||||
|
||||
Where practical:
|
||||
|
||||
- load example config files in tests;
|
||||
- test CLI examples or command parser behavior;
|
||||
- validate documented flags against real flags;
|
||||
- remove stale references;
|
||||
- update links after renames;
|
||||
- keep roadmap content out of non-roadmap docs.
|
||||
|
||||
If documentation and code disagree, fix the documentation and/or open a roadmap item; do not leave aspirational behavior in current-behavior docs.
|
||||
|
||||
Documentation is complete only when it matches the current code.
|
||||
|
||||
## Documentation Change Checklist
|
||||
|
||||
Before merging documentation changes, verify:
|
||||
|
||||
- README is concise and orientation-focused.
|
||||
- `docs/policy/architecture.md` describes development principles.
|
||||
- Future work appears only under `docs/roadmap/`.
|
||||
- User-facing docs avoid unnecessary internals.
|
||||
- Developer-facing docs preserve boundaries and invariants.
|
||||
- Config examples match the schema.
|
||||
- CLI examples match real commands and flags.
|
||||
- Defaults appear in the canonical config reference.
|
||||
- No secrets or private data are included.
|
||||
- Links are accurate.
|
||||
153
docs/troubleshooting.md
Normal file
153
docs/troubleshooting.md
Normal file
@@ -0,0 +1,153 @@
|
||||
# Audita Troubleshooting
|
||||
|
||||
## Scope
|
||||
|
||||
This guide lists recurring implemented failure modes for `audita process` and `audita config`.
|
||||
|
||||
For each entry: symptom, likely cause, inspect, and fix.
|
||||
|
||||
## Config Validation Fails
|
||||
|
||||
Symptom:
|
||||
- `audita config validate --config <path>` exits nonzero.
|
||||
|
||||
Likely causes:
|
||||
- missing `version`;
|
||||
- unsupported config version;
|
||||
- unknown YAML field;
|
||||
- unsupported module key or output schema;
|
||||
- invalid numeric/range/concurrency/retention values.
|
||||
|
||||
Inspect:
|
||||
1. rerun `audita config validate --config <path>` and read stderr.
|
||||
2. if needed, inspect effective config with `audita config print-effective --config <path>`.
|
||||
|
||||
Fix:
|
||||
- set `version: 1`;
|
||||
- remove unknown fields;
|
||||
- use supported module keys and output schemas (`bare-segments`, `audita-v1`);
|
||||
- correct invalid values to satisfy validation constraints.
|
||||
|
||||
## Config File Resolution Errors
|
||||
|
||||
Symptom:
|
||||
- `audita process` fails before processing with config-related errors like `config file not found`.
|
||||
|
||||
Likely causes:
|
||||
- `--config` points to a missing path;
|
||||
- `AUDITA_CONFIG` points to a missing path;
|
||||
- unreadable config path.
|
||||
|
||||
Inspect:
|
||||
1. confirm `--config` or `AUDITA_CONFIG` path exists;
|
||||
2. run `audita config validate --config <path>` directly.
|
||||
|
||||
Fix:
|
||||
- correct the path or unset invalid `AUDITA_CONFIG`;
|
||||
- fix permissions for the config file.
|
||||
|
||||
## Transcript or Glossary Schema Errors
|
||||
|
||||
Symptom:
|
||||
- stderr includes `transcript_schema` or `glossary_schema` and run exits nonzero.
|
||||
|
||||
Likely causes:
|
||||
- transcript is not valid JSON or has invalid segment fields;
|
||||
- glossary is not valid YAML or has missing required glossary entry fields.
|
||||
|
||||
Inspect:
|
||||
1. check stderr for parser/validation details;
|
||||
2. if diagnostics were created, inspect `error.log` and run `report.json` (`error_phase`);
|
||||
3. inspect `source-transcript.json` and `source-transcript-parsed.json` in the run directory.
|
||||
|
||||
Fix:
|
||||
- correct transcript JSON shape/content;
|
||||
- correct glossary YAML shape/content and required entry fields;
|
||||
- rerun validation with known-good tiny examples for comparison:
|
||||
- `examples/tiny-transcript.json`
|
||||
- `examples/tiny-glossary.yaml`
|
||||
|
||||
## LLM Runtime/Backend Failures
|
||||
|
||||
Symptom:
|
||||
- stderr includes `runner_execution` (or backend timeout/error details) and nonzero exit.
|
||||
|
||||
Likely causes:
|
||||
- unreachable/failed LLM endpoint;
|
||||
- timeout/cancellation;
|
||||
- runtime module execution failure.
|
||||
|
||||
Inspect:
|
||||
1. inspect stderr for backend message details;
|
||||
2. inspect run `report.json` (`error_phase`, `module_results`);
|
||||
3. inspect diagnostics payloads and `error.log`.
|
||||
|
||||
Fix:
|
||||
- verify model/base URL/API key settings;
|
||||
- increase timeout if needed;
|
||||
- rerun with `--work-dir-retention always` while debugging.
|
||||
|
||||
## Output File Write Failure
|
||||
|
||||
Symptom:
|
||||
- stderr includes `failed to write output file` and run exits nonzero.
|
||||
|
||||
Likely causes:
|
||||
- output path directory missing;
|
||||
- insufficient filesystem permissions;
|
||||
- invalid output target path.
|
||||
|
||||
Inspect:
|
||||
1. check `--output` target directory exists and is writable;
|
||||
2. inspect run diagnostics `error.log` and report `error_phase`.
|
||||
|
||||
Fix:
|
||||
- write to a valid writable path;
|
||||
- create missing directories;
|
||||
- adjust permissions.
|
||||
|
||||
## Report File Write Failure
|
||||
|
||||
Symptom:
|
||||
- stderr includes `failed to write report JSON file` and run exits nonzero.
|
||||
|
||||
Likely causes:
|
||||
- invalid or unwritable `--report-json` target path.
|
||||
|
||||
Inspect:
|
||||
1. verify parent directory exists and is writable;
|
||||
2. inspect diagnostics `error.log` for `report_write` context.
|
||||
|
||||
Fix:
|
||||
- choose a writable report path;
|
||||
- create missing directories;
|
||||
- rerun.
|
||||
|
||||
## Unsupported Output Schema
|
||||
|
||||
Symptom:
|
||||
- stderr includes `unsupported output schema` and run exits nonzero.
|
||||
|
||||
Likely causes:
|
||||
- unsupported `--output-schema` value;
|
||||
- unsupported `output.schema` in config.
|
||||
|
||||
Inspect:
|
||||
1. check CLI/config schema key;
|
||||
2. run `audita config validate --config <path>` when config is involved.
|
||||
|
||||
Fix:
|
||||
- use `bare-segments` or `audita-v1`.
|
||||
|
||||
## Diagnostics Directory Lookup
|
||||
|
||||
Symptom:
|
||||
- run fails and you need artifacts for debugging.
|
||||
|
||||
Inspect:
|
||||
1. read stderr for `audita process: diagnostics: <run-dir>`;
|
||||
2. open `<run-dir>/report.json` and `<run-dir>/error.log`;
|
||||
3. use diagnostics paths embedded in report metadata for artifact lookup.
|
||||
|
||||
Fix:
|
||||
- rerun with `--work-dir-retention always` to preserve run directories during investigation.
|
||||
6
examples/minimal-config.yml
Normal file
6
examples/minimal-config.yml
Normal file
@@ -0,0 +1,6 @@
|
||||
version: 1
|
||||
output:
|
||||
schema: bare-segments
|
||||
llm:
|
||||
proposal:
|
||||
api_key_env: AUDITA_LLM_API_KEY
|
||||
41
examples/production-config.yml
Normal file
41
examples/production-config.yml
Normal file
@@ -0,0 +1,41 @@
|
||||
version: 1
|
||||
pipeline:
|
||||
modules: [glossary, homophones, glossary, spoken_word, grammar]
|
||||
output:
|
||||
schema: audita-v1
|
||||
llm:
|
||||
proposal:
|
||||
base_url: https://openrouter.ai/api/v1
|
||||
model: openrouter/google/gemma-4-31b-it
|
||||
api_key_env: AUDITA_LLM_API_KEY
|
||||
timeout: 120s
|
||||
max_retries: 3
|
||||
validation:
|
||||
base_url: https://openrouter.ai/api/v1
|
||||
model: openrouter/google/gemma-4-31b-it
|
||||
api_key_env: AUDITA_VALIDATION_LLM_API_KEY
|
||||
timeout: 120
|
||||
max_retries: 3
|
||||
concurrency:
|
||||
total_llm: 2
|
||||
proposal_llm: 2
|
||||
validation_llm: 1
|
||||
chunking:
|
||||
target_sections: 8
|
||||
max_section_tokens: 8192
|
||||
min_section_tokens: 2048
|
||||
normalization:
|
||||
max_segment_gap: 4s
|
||||
ellipsis_gap: 3.5s
|
||||
max_segment_duration: 60s
|
||||
max_segment_tokens: 2048
|
||||
thresholds:
|
||||
glossary: 0.8
|
||||
homophones: 0.8
|
||||
spoken_word: 0.8
|
||||
grammar: 0.8
|
||||
context:
|
||||
description: "General context for domain vocabulary and speaker names."
|
||||
diagnostics:
|
||||
work_dir: /tmp/audita
|
||||
retention: auto
|
||||
6
examples/tiny-glossary.yaml
Normal file
6
examples/tiny-glossary.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
glossary:
|
||||
- name: Audita
|
||||
aliases:
|
||||
- audita
|
||||
category: product
|
||||
summary: The Audita transcript correction CLI.
|
||||
9
examples/tiny-transcript.json
Normal file
9
examples/tiny-transcript.json
Normal file
@@ -0,0 +1,9 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "A",
|
||||
"start": 0.0,
|
||||
"end": 1.2,
|
||||
"text": "hello world"
|
||||
}
|
||||
]
|
||||
11
go.mod
Normal file
11
go.mod
Normal file
@@ -0,0 +1,11 @@
|
||||
module gitea.maximumdirect.net/eric/audita
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require gopkg.in/yaml.v3 v3.0.1
|
||||
|
||||
require (
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
)
|
||||
17
go.sum
Normal file
17
go.sum
Normal file
@@ -0,0 +1,17 @@
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
338
internal/cli/parity_test.go
Normal file
338
internal/cli/parity_test.go
Normal file
@@ -0,0 +1,338 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
)
|
||||
|
||||
type parityFixtureCase struct {
|
||||
Name string `json:"name"`
|
||||
TranscriptFile string `json:"transcript_file"`
|
||||
GlossaryFile string `json:"glossary_file"`
|
||||
ModulesCSV string `json:"modules_csv,omitempty"`
|
||||
ProposalResponsesFile string `json:"proposal_responses_file,omitempty"`
|
||||
ValidationResponsesFile string `json:"validation_responses_file,omitempty"`
|
||||
Env map[string]string `json:"env,omitempty"`
|
||||
Expect parityExpectation `json:"expect"`
|
||||
}
|
||||
|
||||
type parityExpectation struct {
|
||||
ExitCode int `json:"exit_code"`
|
||||
Status string `json:"status,omitempty"`
|
||||
ErrorPhase string `json:"error_phase,omitempty"`
|
||||
StdoutMode string `json:"stdout_mode,omitempty"` // empty|json
|
||||
StderrContains string `json:"stderr_contains,omitempty"`
|
||||
OutputTranscriptFile string `json:"output_transcript_file,omitempty"`
|
||||
ModuleInstances []string `json:"module_instances,omitempty"`
|
||||
ModuleCount int `json:"module_count,omitempty"`
|
||||
TotalAppliedChanges int `json:"total_applied_changes,omitempty"`
|
||||
TotalSkippedChanges int `json:"total_skipped_changes,omitempty"`
|
||||
FailedModuleInstance string `json:"failed_module_instance,omitempty"`
|
||||
ValidatorRejectedReasonCodes []string `json:"validator_rejected_reason_codes,omitempty"`
|
||||
ApplicationSkipReasonCodes []string `json:"application_skip_reason_codes,omitempty"`
|
||||
RequireErrorLog bool `json:"require_error_log,omitempty"`
|
||||
SecretMarkers []string `json:"secret_markers,omitempty"`
|
||||
ExpectedProposalCalls []string `json:"expected_proposal_calls,omitempty"`
|
||||
ExpectedValidationCalls []string `json:"expected_validation_calls,omitempty"`
|
||||
ModuleAppliedCounts []int `json:"module_applied_counts,omitempty"`
|
||||
ModuleRejectedCounts []int `json:"module_rejected_counts,omitempty"`
|
||||
ModuleSkipCounts []int `json:"module_skip_counts,omitempty"`
|
||||
MinResponsePayloadArtifacts int `json:"min_response_payload_artifacts,omitempty"`
|
||||
}
|
||||
|
||||
func TestParityFixtures(t *testing.T) {
|
||||
casePaths, err := filepath.Glob(parityFixturePath("*.case.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("glob parity fixtures: %v", err)
|
||||
}
|
||||
if len(casePaths) == 0 {
|
||||
t.Fatal("expected at least one parity fixture case")
|
||||
}
|
||||
|
||||
for _, casePath := range casePaths {
|
||||
fx := loadParityFixtureCase(t, casePath)
|
||||
t.Run(fx.Name, func(t *testing.T) {
|
||||
runParityFixtureCase(t, filepath.Dir(casePath), fx)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func loadParityFixtureCase(t *testing.T, casePath string) parityFixtureCase {
|
||||
t.Helper()
|
||||
var fx parityFixtureCase
|
||||
raw := readFile(t, casePath)
|
||||
if err := json.Unmarshal(raw, &fx); err != nil {
|
||||
t.Fatalf("parse parity case %q: %v", casePath, err)
|
||||
}
|
||||
if strings.TrimSpace(fx.Name) == "" {
|
||||
t.Fatalf("parity case %q missing name", casePath)
|
||||
}
|
||||
return fx
|
||||
}
|
||||
|
||||
func runParityFixtureCase(t *testing.T, caseDir string, fx parityFixtureCase) {
|
||||
t.Helper()
|
||||
for k, v := range fx.Env {
|
||||
t.Setenv(k, v)
|
||||
}
|
||||
|
||||
proposalClient := &fakeStructuredLLMClient{}
|
||||
validationClient := &fakeStructuredLLMClient{}
|
||||
|
||||
if strings.TrimSpace(fx.ProposalResponsesFile) != "" {
|
||||
raw := readFile(t, filepath.Join(caseDir, fx.ProposalResponsesFile))
|
||||
if err := json.Unmarshal(raw, &proposalClient.proposalResponses); err != nil {
|
||||
t.Fatalf("parse proposal responses: %v", err)
|
||||
}
|
||||
processProposalLLMClient = proposalClient
|
||||
}
|
||||
if strings.TrimSpace(fx.ValidationResponsesFile) != "" {
|
||||
raw := readFile(t, filepath.Join(caseDir, fx.ValidationResponsesFile))
|
||||
if err := json.Unmarshal(raw, &validationClient.validationResponses); err != nil {
|
||||
t.Fatalf("parse validation responses: %v", err)
|
||||
}
|
||||
processValidationLLMClient = validationClient
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
processProposalLLMClient = nil
|
||||
processValidationLLMClient = nil
|
||||
})
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
workDir := t.TempDir()
|
||||
outputPath := filepath.Join(t.TempDir(), "out.json")
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
args := []string{
|
||||
"process",
|
||||
filepath.Join(caseDir, fx.TranscriptFile),
|
||||
"--glossary",
|
||||
filepath.Join(caseDir, fx.GlossaryFile),
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
}
|
||||
// Keep stdout shape deterministic for parity tests.
|
||||
if fx.Expect.StdoutMode != "json" {
|
||||
args = append(args, "--output", outputPath)
|
||||
}
|
||||
if strings.TrimSpace(fx.ModulesCSV) != "" {
|
||||
args = append(args, "--modules", fx.ModulesCSV)
|
||||
}
|
||||
|
||||
exitCode := Run(args, &stdout, &stderr)
|
||||
if exitCode != fx.Expect.ExitCode {
|
||||
t.Fatalf("expected exit code %d, got %d stderr=%q", fx.Expect.ExitCode, exitCode, stderr.String())
|
||||
}
|
||||
|
||||
switch fx.Expect.StdoutMode {
|
||||
case "json":
|
||||
if _, err := json.Marshal(stdout.String()); err != nil {
|
||||
t.Fatalf("unexpected stdout marshal error: %v", err)
|
||||
}
|
||||
if !json.Valid(stdout.Bytes()) {
|
||||
t.Fatalf("expected JSON stdout, got %q", stdout.String())
|
||||
}
|
||||
default:
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("expected empty stdout, got %q", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
if fx.Expect.StderrContains != "" && !strings.Contains(stderr.String(), fx.Expect.StderrContains) {
|
||||
t.Fatalf("expected stderr to contain %q, got %q", fx.Expect.StderrContains, stderr.String())
|
||||
}
|
||||
|
||||
report := readProcessReport(t, reportPath)
|
||||
assertParityReport(t, report, fx.Expect)
|
||||
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
runDirReport := readProcessReport(t, filepath.Join(runDir, "report.json"))
|
||||
assertParityReport(t, runDirReport, fx.Expect)
|
||||
|
||||
if fx.Expect.RequireErrorLog {
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if strings.TrimSpace(fx.Expect.OutputTranscriptFile) != "" && fx.Expect.ExitCode == 0 {
|
||||
got := readFile(t, outputPath)
|
||||
want := readFile(t, filepath.Join(caseDir, fx.Expect.OutputTranscriptFile))
|
||||
assertJSONSemanticEqual(t, want, got)
|
||||
}
|
||||
|
||||
if len(fx.Expect.ExpectedProposalCalls) > 0 && !reflect.DeepEqual(proposalClient.calls, fx.Expect.ExpectedProposalCalls) {
|
||||
t.Fatalf("unexpected proposal calls: got %v want %v", proposalClient.calls, fx.Expect.ExpectedProposalCalls)
|
||||
}
|
||||
if len(fx.Expect.ExpectedValidationCalls) > 0 && !reflect.DeepEqual(validationClient.calls, fx.Expect.ExpectedValidationCalls) {
|
||||
t.Fatalf("unexpected validation calls: got %v want %v", validationClient.calls, fx.Expect.ExpectedValidationCalls)
|
||||
}
|
||||
|
||||
if len(fx.Expect.SecretMarkers) > 0 {
|
||||
assertNoSecretMarkers(t, reportPath, fx.Expect.SecretMarkers)
|
||||
assertNoSecretMarkersInTree(t, runDir, fx.Expect.SecretMarkers)
|
||||
}
|
||||
if fx.Expect.MinResponsePayloadArtifacts > 0 {
|
||||
matches, err := filepath.Glob(filepath.Join(runDir, "*", "*response-payload.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("glob response payload artifacts: %v", err)
|
||||
}
|
||||
if len(matches) < fx.Expect.MinResponsePayloadArtifacts {
|
||||
t.Fatalf("expected at least %d response payload artifacts, got %d", fx.Expect.MinResponsePayloadArtifacts, len(matches))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertParityReport(t *testing.T, report reporting.ProcessReport, exp parityExpectation) {
|
||||
t.Helper()
|
||||
if exp.Status != "" && report.Status != exp.Status {
|
||||
t.Fatalf("expected report status %q, got %q", exp.Status, report.Status)
|
||||
}
|
||||
if exp.ErrorPhase != "" && report.ErrorPhase != exp.ErrorPhase {
|
||||
t.Fatalf("expected report error_phase %q, got %q", exp.ErrorPhase, report.ErrorPhase)
|
||||
}
|
||||
if len(exp.ModuleInstances) > 0 {
|
||||
got := make([]string, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, mr.ModuleInstance)
|
||||
}
|
||||
if !reflect.DeepEqual(got, exp.ModuleInstances) {
|
||||
t.Fatalf("unexpected module instances: got %v want %v", got, exp.ModuleInstances)
|
||||
}
|
||||
}
|
||||
if exp.ModuleCount > 0 {
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.ModuleCount != exp.ModuleCount {
|
||||
t.Fatalf("expected module_count=%d, got %+v", exp.ModuleCount, report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
if exp.TotalAppliedChanges > 0 {
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.TotalAppliedChanges != exp.TotalAppliedChanges {
|
||||
t.Fatalf("expected total_applied_changes=%d, got %+v", exp.TotalAppliedChanges, report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
if exp.TotalSkippedChanges > 0 {
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.TotalSkippedChanges != exp.TotalSkippedChanges {
|
||||
t.Fatalf("expected total_skipped_changes=%d, got %+v", exp.TotalSkippedChanges, report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
if exp.FailedModuleInstance != "" {
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != exp.FailedModuleInstance {
|
||||
t.Fatalf("expected failed_module_instance=%q, got %+v", exp.FailedModuleInstance, report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
|
||||
if len(exp.ValidatorRejectedReasonCodes) > 0 {
|
||||
got := collectValidatorRejectedReasonCodes(report.ModuleResults)
|
||||
if !reflect.DeepEqual(got, exp.ValidatorRejectedReasonCodes) {
|
||||
t.Fatalf("unexpected validator rejected reason codes: got %v want %v", got, exp.ValidatorRejectedReasonCodes)
|
||||
}
|
||||
}
|
||||
if len(exp.ApplicationSkipReasonCodes) > 0 {
|
||||
got := collectApplicationSkipReasonCodes(report.ModuleResults)
|
||||
if !reflect.DeepEqual(got, exp.ApplicationSkipReasonCodes) {
|
||||
t.Fatalf("unexpected application skip reason codes: got %v want %v", got, exp.ApplicationSkipReasonCodes)
|
||||
}
|
||||
}
|
||||
if len(exp.ModuleAppliedCounts) > 0 {
|
||||
got := make([]int, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, len(mr.AppliedChanges))
|
||||
}
|
||||
if !reflect.DeepEqual(got, exp.ModuleAppliedCounts) {
|
||||
t.Fatalf("unexpected per-module applied counts: got %v want %v", got, exp.ModuleAppliedCounts)
|
||||
}
|
||||
}
|
||||
if len(exp.ModuleRejectedCounts) > 0 {
|
||||
got := make([]int, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, len(mr.ValidatorRejected))
|
||||
}
|
||||
if !reflect.DeepEqual(got, exp.ModuleRejectedCounts) {
|
||||
t.Fatalf("unexpected per-module rejected counts: got %v want %v", got, exp.ModuleRejectedCounts)
|
||||
}
|
||||
}
|
||||
if len(exp.ModuleSkipCounts) > 0 {
|
||||
got := make([]int, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, len(mr.SkippedChanges))
|
||||
}
|
||||
if !reflect.DeepEqual(got, exp.ModuleSkipCounts) {
|
||||
t.Fatalf("unexpected per-module skip counts: got %v want %v", got, exp.ModuleSkipCounts)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func collectValidatorRejectedReasonCodes(results []reporting.ModuleReport) []string {
|
||||
out := make([]string, 0)
|
||||
for _, mr := range results {
|
||||
for _, vr := range mr.ValidatorRejected {
|
||||
out = append(out, vr.ReasonCode)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func collectApplicationSkipReasonCodes(results []reporting.ModuleReport) []string {
|
||||
out := make([]string, 0)
|
||||
for _, mr := range results {
|
||||
for _, sk := range mr.SkippedChanges {
|
||||
out = append(out, string(sk.SkipReason))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func assertJSONSemanticEqual(t *testing.T, expected []byte, actual []byte) {
|
||||
t.Helper()
|
||||
var exp any
|
||||
var act any
|
||||
if err := json.Unmarshal(expected, &exp); err != nil {
|
||||
t.Fatalf("unmarshal expected json: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(actual, &act); err != nil {
|
||||
t.Fatalf("unmarshal actual json: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(exp, act) {
|
||||
t.Fatalf("JSON mismatch\nexpected=%s\nactual=%s", string(expected), string(actual))
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoSecretMarkers(t *testing.T, filePath string, markers []string) {
|
||||
t.Helper()
|
||||
raw := string(readFile(t, filePath))
|
||||
for _, marker := range markers {
|
||||
if marker != "" && strings.Contains(raw, marker) {
|
||||
t.Fatalf("secret marker %q leaked in %s", marker, filePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoSecretMarkersInTree(t *testing.T, root string, markers []string) {
|
||||
t.Helper()
|
||||
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil || d == nil || d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
raw := string(readFile(t, path))
|
||||
for _, marker := range markers {
|
||||
if marker != "" && strings.Contains(raw, marker) {
|
||||
t.Fatalf("secret marker %q leaked in %s", marker, path)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func parityFixturePath(name string) string {
|
||||
return filepath.Join("testdata", "parity", name)
|
||||
}
|
||||
121
internal/cli/process_flags.go
Normal file
121
internal/cli/process_flags.go
Normal file
@@ -0,0 +1,121 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"flag"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
)
|
||||
|
||||
type processOverrideBinding func(*config.CLIOverrides, processFlags)
|
||||
|
||||
var processOverrideBindings = map[string]processOverrideBinding{
|
||||
"modules": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ModulesCSV = flags.modules
|
||||
},
|
||||
"output-schema": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.OutputSchema = flags.outputSchema
|
||||
},
|
||||
"llm-api-key": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.PrimaryLLMAPIKey = flags.llmAPIKey
|
||||
},
|
||||
"validation-llm-api-key": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationLLMAPIKey = flags.validationLLMAPIKey
|
||||
},
|
||||
"model": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.PrimaryModel = flags.model
|
||||
},
|
||||
"validation-model": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationModel = flags.validationModel
|
||||
},
|
||||
"base-url": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.PrimaryBaseURL = flags.baseURL
|
||||
},
|
||||
"validation-base-url": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationBaseURL = flags.validationBaseURL
|
||||
},
|
||||
"llm-timeout-seconds": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.PrimaryLLMTimeoutSeconds = flags.llmTimeoutSeconds
|
||||
},
|
||||
"total-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.TotalLLMConcurrency = flags.totalLLMConcurrency
|
||||
},
|
||||
"proposal-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ProposalLLMConcurrency = flags.proposalLLMConcurrency
|
||||
},
|
||||
"llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.PrimaryLLMConcurrency = flags.llmConcurrency
|
||||
},
|
||||
"validation-llm-timeout-seconds": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationLLMTimeoutSeconds = flags.validationLLMTimeoutSeconds
|
||||
},
|
||||
"max-retries": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.MaxRetries = flags.maxRetries
|
||||
},
|
||||
"validation-max-retries": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationMaxRetries = flags.validationMaxRetries
|
||||
},
|
||||
"validation-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationLLMConcurrency = flags.validationLLMConcurrency
|
||||
},
|
||||
"validation-max-prompt-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationMaxPromptTokens = flags.validationMaxPromptTokens
|
||||
},
|
||||
"max-section-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.MaxSectionTokens = flags.maxSectionTokens
|
||||
},
|
||||
"min-section-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.MinSectionTokens = flags.minSectionTokens
|
||||
},
|
||||
"target-sections": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.TargetSections = flags.targetSections
|
||||
},
|
||||
"glossary-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.GlossaryConfidenceThreshold = flags.glossaryConfidenceThreshold
|
||||
},
|
||||
"grammar-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.GrammarConfidenceThreshold = flags.grammarConfidenceThreshold
|
||||
},
|
||||
"homophones-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.HomophonesConfidenceThreshold = flags.homophonesConfidenceThreshold
|
||||
},
|
||||
"spoken-word-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.SpokenWordConfidenceThreshold = flags.spokenWordConfidenceThreshold
|
||||
},
|
||||
"normalize-max-segment-gap": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.NormalizeMaxSegmentGap = flags.normalizeMaxSegmentGap
|
||||
},
|
||||
"normalize-ellipsis-gap": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.NormalizeEllipsisGap = flags.normalizeEllipsisGap
|
||||
},
|
||||
"normalize-max-segment-duration": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.NormalizeMaxSegmentDuration = flags.normalizeMaxSegmentDuration
|
||||
},
|
||||
"normalize-max-segment-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.NormalizeMaxSegmentTokens = flags.normalizeMaxSegmentTokens
|
||||
},
|
||||
"transcript-description": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.TranscriptDescription = flags.transcriptDescription
|
||||
},
|
||||
"work-dir": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.WorkDir = flags.workDir
|
||||
},
|
||||
"work-dir-retention": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.WorkDirRetention = flags.workDirRetention
|
||||
},
|
||||
}
|
||||
|
||||
func processCLIOverrides(fs *flag.FlagSet, flags processFlags) (config.CLIOverrides, bool) {
|
||||
overrides := config.CLIOverrides{}
|
||||
explicitModules := false
|
||||
fs.Visit(func(f *flag.Flag) {
|
||||
if f.Name == "modules" {
|
||||
explicitModules = true
|
||||
}
|
||||
binding, ok := processOverrideBindings[f.Name]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
binding(&overrides, flags)
|
||||
})
|
||||
return overrides, explicitModules
|
||||
}
|
||||
433
internal/cli/process_flags_test.go
Normal file
433
internal/cli/process_flags_test.go
Normal file
@@ -0,0 +1,433 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"io"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
)
|
||||
|
||||
func TestProcessCLIOverridesMapsEveryConfigMutatingFlag(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
flagName string
|
||||
value string
|
||||
wantExplicitModules bool
|
||||
assertOverrideFields func(t *testing.T, overrides config.CLIOverrides)
|
||||
}{
|
||||
{
|
||||
name: "modules",
|
||||
flagName: "modules",
|
||||
value: "grammar,glossary",
|
||||
wantExplicitModules: true,
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "ModulesCSV", overrides.ModulesCSV, "grammar,glossary")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "output schema",
|
||||
flagName: "output-schema",
|
||||
value: "audita-v1",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "OutputSchema", overrides.OutputSchema, "audita-v1")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "primary api key",
|
||||
flagName: "llm-api-key",
|
||||
value: "primary-key",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "PrimaryLLMAPIKey", overrides.PrimaryLLMAPIKey, "primary-key")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation api key",
|
||||
flagName: "validation-llm-api-key",
|
||||
value: "validation-key",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "ValidationLLMAPIKey", overrides.ValidationLLMAPIKey, "validation-key")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "primary model",
|
||||
flagName: "model",
|
||||
value: "primary-model",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "PrimaryModel", overrides.PrimaryModel, "primary-model")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation model",
|
||||
flagName: "validation-model",
|
||||
value: "validation-model",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "ValidationModel", overrides.ValidationModel, "validation-model")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "primary base url",
|
||||
flagName: "base-url",
|
||||
value: "https://primary.example.test",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "PrimaryBaseURL", overrides.PrimaryBaseURL, "https://primary.example.test")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation base url",
|
||||
flagName: "validation-base-url",
|
||||
value: "https://validation.example.test",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "ValidationBaseURL", overrides.ValidationBaseURL, "https://validation.example.test")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "primary timeout",
|
||||
flagName: "llm-timeout-seconds",
|
||||
value: "101",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "PrimaryLLMTimeoutSeconds", overrides.PrimaryLLMTimeoutSeconds, 101)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "total concurrency",
|
||||
flagName: "total-llm-concurrency",
|
||||
value: "5",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "TotalLLMConcurrency", overrides.TotalLLMConcurrency, 5)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "proposal concurrency",
|
||||
flagName: "proposal-llm-concurrency",
|
||||
value: "3",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "ProposalLLMConcurrency", overrides.ProposalLLMConcurrency, 3)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "legacy concurrency alias",
|
||||
flagName: "llm-concurrency",
|
||||
value: "4",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "PrimaryLLMConcurrency", overrides.PrimaryLLMConcurrency, 4)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation timeout",
|
||||
flagName: "validation-llm-timeout-seconds",
|
||||
value: "202",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "ValidationLLMTimeoutSeconds", overrides.ValidationLLMTimeoutSeconds, 202)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "max retries",
|
||||
flagName: "max-retries",
|
||||
value: "6",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "MaxRetries", overrides.MaxRetries, 6)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation max retries",
|
||||
flagName: "validation-max-retries",
|
||||
value: "7",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "ValidationMaxRetries", overrides.ValidationMaxRetries, 7)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation concurrency",
|
||||
flagName: "validation-llm-concurrency",
|
||||
value: "8",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "ValidationLLMConcurrency", overrides.ValidationLLMConcurrency, 8)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation max prompt tokens",
|
||||
flagName: "validation-max-prompt-tokens",
|
||||
value: "4096",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "ValidationMaxPromptTokens", overrides.ValidationMaxPromptTokens, 4096)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "max section tokens",
|
||||
flagName: "max-section-tokens",
|
||||
value: "9000",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "MaxSectionTokens", overrides.MaxSectionTokens, 9000)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "min section tokens",
|
||||
flagName: "min-section-tokens",
|
||||
value: "1000",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "MinSectionTokens", overrides.MinSectionTokens, 1000)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "target sections",
|
||||
flagName: "target-sections",
|
||||
value: "12",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "TargetSections", overrides.TargetSections, 12)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "glossary threshold",
|
||||
flagName: "glossary-confidence-threshold",
|
||||
value: "0.91",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "GlossaryConfidenceThreshold", overrides.GlossaryConfidenceThreshold, 0.91)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "grammar threshold",
|
||||
flagName: "grammar-confidence-threshold",
|
||||
value: "0.92",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "GrammarConfidenceThreshold", overrides.GrammarConfidenceThreshold, 0.92)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "homophones threshold",
|
||||
flagName: "homophones-confidence-threshold",
|
||||
value: "0.93",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "HomophonesConfidenceThreshold", overrides.HomophonesConfidenceThreshold, 0.93)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "spoken word threshold",
|
||||
flagName: "spoken-word-confidence-threshold",
|
||||
value: "0.94",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "SpokenWordConfidenceThreshold", overrides.SpokenWordConfidenceThreshold, 0.94)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "normalize max segment gap",
|
||||
flagName: "normalize-max-segment-gap",
|
||||
value: "1.2",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "NormalizeMaxSegmentGap", overrides.NormalizeMaxSegmentGap, 1.2)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "normalize ellipsis gap",
|
||||
flagName: "normalize-ellipsis-gap",
|
||||
value: "2.3",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "NormalizeEllipsisGap", overrides.NormalizeEllipsisGap, 2.3)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "normalize max segment duration",
|
||||
flagName: "normalize-max-segment-duration",
|
||||
value: "45.6",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "NormalizeMaxSegmentDuration", overrides.NormalizeMaxSegmentDuration, 45.6)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "normalize max segment tokens",
|
||||
flagName: "normalize-max-segment-tokens",
|
||||
value: "321",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "NormalizeMaxSegmentTokens", overrides.NormalizeMaxSegmentTokens, 321)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "transcript description",
|
||||
flagName: "transcript-description",
|
||||
value: "podcast episode",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "TranscriptDescription", overrides.TranscriptDescription, "podcast episode")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "work dir",
|
||||
flagName: "work-dir",
|
||||
value: "/tmp/custom-audita",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "WorkDir", overrides.WorkDir, "/tmp/custom-audita")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "work dir retention",
|
||||
flagName: "work-dir-retention",
|
||||
value: "always",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "WorkDirRetention", overrides.WorkDirRetention, "always")
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fs, flags := newProcessFlagSet(config.Default(), io.Discard)
|
||||
if err := fs.Parse([]string{"--" + tc.flagName, tc.value}); err != nil {
|
||||
t.Fatalf("parse flag: %v", err)
|
||||
}
|
||||
|
||||
overrides, explicitModules := processCLIOverrides(fs, flags)
|
||||
if explicitModules != tc.wantExplicitModules {
|
||||
t.Fatalf("explicitModules=%v, want %v", explicitModules, tc.wantExplicitModules)
|
||||
}
|
||||
tc.assertOverrideFields(t, overrides)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessCLIOverridesIgnoresNonConfigFlags(t *testing.T) {
|
||||
fs, flags := newProcessFlagSet(config.Default(), io.Discard)
|
||||
if err := fs.Parse([]string{
|
||||
"--config", "/tmp/config.yml",
|
||||
"--glossary", "/tmp/glossary.yml",
|
||||
"--output", "/tmp/output.json",
|
||||
"--report-json", "/tmp/report.json",
|
||||
}); err != nil {
|
||||
t.Fatalf("parse flags: %v", err)
|
||||
}
|
||||
|
||||
overrides, explicitModules := processCLIOverrides(fs, flags)
|
||||
if explicitModules {
|
||||
t.Fatal("non-config flags should not mark modules explicit")
|
||||
}
|
||||
assertNoCLIOverrides(t, overrides)
|
||||
}
|
||||
|
||||
func TestNewProcessFlagSetDefaultsReflectEffectiveConfig(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
cfg.Modules = []string{"grammar", "glossary"}
|
||||
cfg.OutputSchema = "audita-v1"
|
||||
cfg.PrimaryLLM.APIKey = "primary-key"
|
||||
cfg.ValidationLLM.APIKey = "validation-key"
|
||||
cfg.PrimaryLLM.Model = "primary-model"
|
||||
cfg.ValidationLLM.Model = "validation-model"
|
||||
cfg.PrimaryLLM.BaseURL = "https://primary.example.test"
|
||||
cfg.ValidationLLM.BaseURL = "https://validation.example.test"
|
||||
cfg.PrimaryLLM.TimeoutSeconds = 101
|
||||
cfg.TotalLLMConcurrency = 5
|
||||
cfg.ProposalLLMConcurrency = 3
|
||||
cfg.PrimaryLLM.MaxRetries = 6
|
||||
cfg.ValidationMaxPromptTokens = 4096
|
||||
cfg.MaxSectionTokens = 9000
|
||||
cfg.MinSectionTokens = 1000
|
||||
cfg.Thresholds.Glossary = 0.91
|
||||
cfg.Thresholds.Grammar = 0.92
|
||||
cfg.Thresholds.Homophones = 0.93
|
||||
cfg.Thresholds.SpokenWord = 0.94
|
||||
cfg.Normalization.MaxSegmentGap = 1.2
|
||||
cfg.Normalization.EllipsisGap = 2.3
|
||||
cfg.Normalization.MaxSegmentDuration = 45.6
|
||||
cfg.Normalization.MaxSegmentTokens = 321
|
||||
cfg.TranscriptDescription = "podcast episode"
|
||||
cfg.WorkDir = "/tmp/custom-audita"
|
||||
cfg.WorkDirRetention = config.WorkDirRetentionAlways
|
||||
|
||||
validationTimeout := 202
|
||||
validationRetries := 7
|
||||
validationConcurrency := 8
|
||||
targetSections := 12
|
||||
cfg.ValidationLLM.TimeoutSeconds = &validationTimeout
|
||||
cfg.ValidationLLM.MaxRetries = &validationRetries
|
||||
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||
cfg.TargetSections = &targetSections
|
||||
|
||||
_, flags := newProcessFlagSet(cfg, io.Discard)
|
||||
|
||||
assertStringOverride(t, "modules default", flags.modules, "grammar,glossary")
|
||||
assertStringOverride(t, "output schema default", flags.outputSchema, "audita-v1")
|
||||
assertStringOverride(t, "primary api key default", flags.llmAPIKey, "primary-key")
|
||||
assertStringOverride(t, "validation api key default", flags.validationLLMAPIKey, "validation-key")
|
||||
assertStringOverride(t, "primary model default", flags.model, "primary-model")
|
||||
assertStringOverride(t, "validation model default", flags.validationModel, "validation-model")
|
||||
assertStringOverride(t, "primary base url default", flags.baseURL, "https://primary.example.test")
|
||||
assertStringOverride(t, "validation base url default", flags.validationBaseURL, "https://validation.example.test")
|
||||
assertIntOverride(t, "primary timeout default", flags.llmTimeoutSeconds, 101)
|
||||
assertIntOverride(t, "total concurrency default", flags.totalLLMConcurrency, 5)
|
||||
assertIntOverride(t, "proposal concurrency default", flags.proposalLLMConcurrency, 3)
|
||||
assertIntOverride(t, "legacy concurrency alias default", flags.llmConcurrency, 5)
|
||||
assertIntOverride(t, "validation timeout default", flags.validationLLMTimeoutSeconds, validationTimeout)
|
||||
assertIntOverride(t, "max retries default", flags.maxRetries, 6)
|
||||
assertIntOverride(t, "validation max retries default", flags.validationMaxRetries, validationRetries)
|
||||
assertIntOverride(t, "validation concurrency default", flags.validationLLMConcurrency, validationConcurrency)
|
||||
assertIntOverride(t, "validation max prompt tokens default", flags.validationMaxPromptTokens, 4096)
|
||||
assertIntOverride(t, "max section tokens default", flags.maxSectionTokens, 9000)
|
||||
assertIntOverride(t, "min section tokens default", flags.minSectionTokens, 1000)
|
||||
assertIntOverride(t, "target sections default", flags.targetSections, targetSections)
|
||||
assertFloatOverride(t, "glossary threshold default", flags.glossaryConfidenceThreshold, 0.91)
|
||||
assertFloatOverride(t, "grammar threshold default", flags.grammarConfidenceThreshold, 0.92)
|
||||
assertFloatOverride(t, "homophones threshold default", flags.homophonesConfidenceThreshold, 0.93)
|
||||
assertFloatOverride(t, "spoken word threshold default", flags.spokenWordConfidenceThreshold, 0.94)
|
||||
assertFloatOverride(t, "normalize max segment gap default", flags.normalizeMaxSegmentGap, 1.2)
|
||||
assertFloatOverride(t, "normalize ellipsis gap default", flags.normalizeEllipsisGap, 2.3)
|
||||
assertFloatOverride(t, "normalize max segment duration default", flags.normalizeMaxSegmentDuration, 45.6)
|
||||
assertIntOverride(t, "normalize max segment tokens default", flags.normalizeMaxSegmentTokens, 321)
|
||||
assertStringOverride(t, "transcript description default", flags.transcriptDescription, "podcast episode")
|
||||
assertStringOverride(t, "work dir default", flags.workDir, "/tmp/custom-audita")
|
||||
assertStringOverride(t, "work dir retention default", flags.workDirRetention, "always")
|
||||
}
|
||||
|
||||
func TestNewProcessFlagSetUsesFallbackDefaultsForUnsetOptionalConfig(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
|
||||
_, flags := newProcessFlagSet(cfg, io.Discard)
|
||||
|
||||
assertIntOverride(t, "validation timeout fallback", flags.validationLLMTimeoutSeconds, cfg.PrimaryLLM.TimeoutSeconds)
|
||||
assertIntOverride(t, "validation retries fallback", flags.validationMaxRetries, cfg.PrimaryLLM.MaxRetries)
|
||||
assertIntOverride(t, "validation concurrency fallback", flags.validationLLMConcurrency, cfg.TotalLLMConcurrency)
|
||||
assertIntOverride(t, "target sections fallback", flags.targetSections, 0)
|
||||
}
|
||||
|
||||
func assertStringOverride(t *testing.T, name string, got *string, want string) {
|
||||
t.Helper()
|
||||
if got == nil || *got != want {
|
||||
t.Fatalf("%s=%v, want %q", name, pointerValue(got), want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertIntOverride(t *testing.T, name string, got *int, want int) {
|
||||
t.Helper()
|
||||
if got == nil || *got != want {
|
||||
t.Fatalf("%s=%v, want %d", name, pointerValue(got), want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertFloatOverride(t *testing.T, name string, got *float64, want float64) {
|
||||
t.Helper()
|
||||
if got == nil || *got != want {
|
||||
t.Fatalf("%s=%v, want %v", name, pointerValue(got), want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoCLIOverrides(t *testing.T, overrides config.CLIOverrides) {
|
||||
t.Helper()
|
||||
value := reflect.ValueOf(overrides)
|
||||
typ := value.Type()
|
||||
for i := 0; i < value.NumField(); i++ {
|
||||
field := value.Field(i)
|
||||
if field.Kind() != reflect.Ptr {
|
||||
t.Fatalf("unexpected non-pointer CLIOverrides field %s", typ.Field(i).Name)
|
||||
}
|
||||
if !field.IsNil() {
|
||||
t.Fatalf("expected no CLI overrides, field %s was set", typ.Field(i).Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func pointerValue[T any](ptr *T) any {
|
||||
if ptr == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
if stringer, ok := any(*ptr).(interface{ String() string }); ok {
|
||||
return strings.TrimSpace(stringer.String())
|
||||
}
|
||||
return *ptr
|
||||
}
|
||||
469
internal/cli/release_fixtures_test.go
Normal file
469
internal/cli/release_fixtures_test.go
Normal file
@@ -0,0 +1,469 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
)
|
||||
|
||||
type releaseFixtureExpectations struct {
|
||||
MustApplyTexts []string `json:"must_apply_texts"`
|
||||
MustNotApplyTexts []string `json:"must_not_apply_texts"`
|
||||
ProtectedTerms []string `json:"protected_terms"`
|
||||
ExpectedModuleInstance []string `json:"expected_module_instances"`
|
||||
MinimumCounts struct {
|
||||
Applied int `json:"applied"`
|
||||
Rejected int `json:"rejected"`
|
||||
Skipped int `json:"skipped"`
|
||||
} `json:"minimum_counts"`
|
||||
}
|
||||
|
||||
func TestReleaseFixtureDefaultPipelineReadiness(t *testing.T) {
|
||||
base := fixturePath(filepath.Join("release", "default-release"))
|
||||
|
||||
var expectations releaseFixtureExpectations
|
||||
if err := json.Unmarshal(readFile(t, base+".expectations.json"), &expectations); err != nil {
|
||||
t.Fatalf("unmarshal release expectations: %v", err)
|
||||
}
|
||||
|
||||
proposalResponses := readProposalResponses(t, base+".proposals.json")
|
||||
validationResponses := readValidationResponses(t, base+".validations.json")
|
||||
|
||||
// First pass: default full pipeline with deterministic fake LLM responses.
|
||||
first := runReleaseFixturePass(t, releaseRunConfig{
|
||||
transcriptPath: base + ".transcript.json",
|
||||
glossaryPath: base + ".glossary.yaml",
|
||||
outputSchema: "bare-segments",
|
||||
proposalResponses: proposalResponses,
|
||||
validationResponses: validationResponses,
|
||||
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
|
||||
reportSchemaName: reporting.DefaultProcessReportSchemaName,
|
||||
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
||||
expectedOutputSchema: "bare-segments",
|
||||
expectModuleInstances: expectations.ExpectedModuleInstance,
|
||||
expectUtilizationPaths: true,
|
||||
})
|
||||
|
||||
gotTranscript := mustReadTranscript(t, first.outputPath)
|
||||
expectFinalTranscriptContains(t, gotTranscript, expectations.MustApplyTexts)
|
||||
expectFinalTranscriptDoesNotContain(t, gotTranscript, expectations.MustNotApplyTexts)
|
||||
expectFinalTranscriptContains(t, gotTranscript, expectations.ProtectedTerms)
|
||||
|
||||
assertReleaseCounts(t, first.report, expectations)
|
||||
assertPromptAndSchemaMetadataPresent(t, first.runDir)
|
||||
assertReleaseLedgerShape(t, first.report)
|
||||
assertReleaseUtilizationShape(t, first.report)
|
||||
assertStableValidatorKeysPresent(t, first.report)
|
||||
assertStdoutStderrContract(t, first.stdout, first.stderr)
|
||||
assertNoSecretMarkersInTree(t, first.runDir, []string{"release-secret"})
|
||||
assertNoSecretMarkers(t, first.reportPath, []string{"release-secret"})
|
||||
|
||||
// Output schema check: audita-v1 object payload.
|
||||
auditaV1 := runReleaseFixturePass(t, releaseRunConfig{
|
||||
transcriptPath: base + ".transcript.json",
|
||||
glossaryPath: base + ".glossary.yaml",
|
||||
outputSchema: "audita-v1",
|
||||
proposalResponses: proposalResponses,
|
||||
validationResponses: validationResponses,
|
||||
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
|
||||
reportSchemaName: reporting.DefaultProcessReportSchemaName,
|
||||
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
||||
expectedOutputSchema: "audita-v1",
|
||||
expectModuleInstances: expectations.ExpectedModuleInstance,
|
||||
expectUtilizationPaths: true,
|
||||
})
|
||||
assertAuditaV1OutputShape(t, auditaV1.outputPath)
|
||||
|
||||
// Idempotence-oriented second pass:
|
||||
// run again on first output with deterministic no-op responses.
|
||||
noOpProposals := make([]proposal_generation.StructuredCorrectionSet, 5)
|
||||
for i := range noOpProposals {
|
||||
noOpProposals[i] = proposal_generation.StructuredCorrectionSet{Corrections: nil}
|
||||
}
|
||||
second := runReleaseFixturePass(t, releaseRunConfig{
|
||||
transcriptPath: first.outputPath,
|
||||
glossaryPath: base + ".glossary.yaml",
|
||||
outputSchema: "bare-segments",
|
||||
proposalResponses: noOpProposals,
|
||||
validationResponses: nil,
|
||||
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
|
||||
reportSchemaName: reporting.DefaultProcessReportSchemaName,
|
||||
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
||||
expectedOutputSchema: "bare-segments",
|
||||
expectModuleInstances: expectations.ExpectedModuleInstance,
|
||||
expectUtilizationPaths: true,
|
||||
})
|
||||
firstSegments := mustReadTranscript(t, first.outputPath)
|
||||
secondSegments := mustReadTranscript(t, second.outputPath)
|
||||
if !reflect.DeepEqual(firstSegments, secondSegments) {
|
||||
t.Fatalf("expected idempotent second pass transcript; first=%+v second=%+v", firstSegments, secondSegments)
|
||||
}
|
||||
if second.report.ModulesSummary == nil {
|
||||
t.Fatalf("expected modules summary on second pass")
|
||||
}
|
||||
if second.report.ModulesSummary.TotalAppliedChanges != 0 {
|
||||
t.Fatalf("expected no-op second pass (0 applied), got %+v", second.report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
|
||||
type releaseRunConfig struct {
|
||||
transcriptPath string
|
||||
glossaryPath string
|
||||
outputSchema string
|
||||
proposalResponses []proposal_generation.StructuredCorrectionSet
|
||||
validationResponses []validators.LLMValidationResponse
|
||||
expectedProposalCalls []string
|
||||
reportSchemaName string
|
||||
reportSchemaVersion string
|
||||
expectedOutputSchema string
|
||||
expectModuleInstances []string
|
||||
expectUtilizationPaths bool
|
||||
}
|
||||
|
||||
type releaseRunResult struct {
|
||||
stdout string
|
||||
stderr string
|
||||
outputPath string
|
||||
reportPath string
|
||||
report reporting.ProcessReport
|
||||
runDir string
|
||||
}
|
||||
|
||||
func runReleaseFixturePass(t *testing.T, cfg releaseRunConfig) releaseRunResult {
|
||||
t.Helper()
|
||||
|
||||
processProposalLLMClient = &fakeStructuredLLMClient{proposalResponses: append([]proposal_generation.StructuredCorrectionSet(nil), cfg.proposalResponses...)}
|
||||
processValidationLLMClient = &fakeStructuredLLMClient{validationResponses: append([]validators.LLMValidationResponse(nil), cfg.validationResponses...)}
|
||||
t.Cleanup(func() {
|
||||
processProposalLLMClient = nil
|
||||
processValidationLLMClient = nil
|
||||
})
|
||||
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
outputPath := filepath.Join(t.TempDir(), "out.json")
|
||||
configPath := writeFile(t, "release-config.yml", "version: 1\n")
|
||||
|
||||
args := []string{
|
||||
"process",
|
||||
cfg.transcriptPath,
|
||||
"--glossary",
|
||||
cfg.glossaryPath,
|
||||
"--config",
|
||||
configPath,
|
||||
"--output",
|
||||
outputPath,
|
||||
"--output-schema",
|
||||
cfg.outputSchema,
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
}
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
exitCode := Run(args, &stdout, &stderr)
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||
}
|
||||
|
||||
report := readProcessReport(t, reportPath)
|
||||
if report.ReportMetadata.ReportSchemaName != cfg.reportSchemaName || report.ReportMetadata.ReportSchemaVersion != cfg.reportSchemaVersion {
|
||||
t.Fatalf("unexpected report schema metadata: %+v", report.ReportMetadata)
|
||||
}
|
||||
if report.ReportMetadata.OutputSchema != cfg.expectedOutputSchema {
|
||||
t.Fatalf("unexpected output schema metadata: got %q want %q", report.ReportMetadata.OutputSchema, cfg.expectedOutputSchema)
|
||||
}
|
||||
if len(cfg.expectModuleInstances) > 0 {
|
||||
got := make([]string, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, mr.ModuleInstance)
|
||||
}
|
||||
if !reflect.DeepEqual(got, cfg.expectModuleInstances) {
|
||||
t.Fatalf("unexpected module instances: got %v want %v", got, cfg.expectModuleInstances)
|
||||
}
|
||||
}
|
||||
if report.Diagnostics == nil {
|
||||
t.Fatalf("expected diagnostics metadata")
|
||||
}
|
||||
if cfg.expectUtilizationPaths {
|
||||
if report.Diagnostics.UtilizationSummaryPath == "" || report.Diagnostics.CorrectionLedgerPath == "" {
|
||||
t.Fatalf("expected utilization/ledger artifact paths in report diagnostics: %+v", report.Diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil {
|
||||
t.Fatalf("expected run-dir report: %v", err)
|
||||
}
|
||||
|
||||
if c, ok := processProposalLLMClient.(*fakeStructuredLLMClient); ok {
|
||||
if !reflect.DeepEqual(c.calls, cfg.expectedProposalCalls) {
|
||||
t.Fatalf("unexpected proposal call order: got %v want %v", c.calls, cfg.expectedProposalCalls)
|
||||
}
|
||||
}
|
||||
|
||||
return releaseRunResult{
|
||||
stdout: stdout.String(),
|
||||
stderr: stderr.String(),
|
||||
outputPath: outputPath,
|
||||
reportPath: reportPath,
|
||||
report: report,
|
||||
runDir: runDir,
|
||||
}
|
||||
}
|
||||
|
||||
func readProposalResponses(t *testing.T, path string) []proposal_generation.StructuredCorrectionSet {
|
||||
t.Helper()
|
||||
var out []proposal_generation.StructuredCorrectionSet
|
||||
if err := json.Unmarshal(readFile(t, path), &out); err != nil {
|
||||
t.Fatalf("unmarshal proposal responses: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func readValidationResponses(t *testing.T, path string) []validators.LLMValidationResponse {
|
||||
t.Helper()
|
||||
var out []validators.LLMValidationResponse
|
||||
if err := json.Unmarshal(readFile(t, path), &out); err != nil {
|
||||
t.Fatalf("unmarshal validation responses: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mustReadTranscript(t *testing.T, path string) []schema.Segment {
|
||||
t.Helper()
|
||||
transcript, err := schema.ParseTranscriptJSON(readFile(t, path))
|
||||
if err != nil {
|
||||
t.Fatalf("parse transcript output: %v", err)
|
||||
}
|
||||
return transcript.Segments
|
||||
}
|
||||
|
||||
func expectFinalTranscriptContains(t *testing.T, segments []schema.Segment, needles []string) {
|
||||
t.Helper()
|
||||
joined := flattenTranscriptText(segments)
|
||||
for _, needle := range needles {
|
||||
if !strings.Contains(joined, needle) {
|
||||
t.Fatalf("expected transcript to contain %q, got %q", needle, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func expectFinalTranscriptDoesNotContain(t *testing.T, segments []schema.Segment, needles []string) {
|
||||
t.Helper()
|
||||
joined := flattenTranscriptText(segments)
|
||||
for _, needle := range needles {
|
||||
if strings.Contains(joined, needle) {
|
||||
t.Fatalf("expected transcript to not contain %q, got %q", needle, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func flattenTranscriptText(segments []schema.Segment) string {
|
||||
parts := make([]string, 0, len(segments))
|
||||
for _, s := range segments {
|
||||
parts = append(parts, s.Text)
|
||||
}
|
||||
return strings.Join(parts, "\n")
|
||||
}
|
||||
|
||||
func assertReleaseCounts(t *testing.T, report reporting.ProcessReport, exp releaseFixtureExpectations) {
|
||||
t.Helper()
|
||||
if report.ModulesSummary == nil {
|
||||
t.Fatalf("expected modules_summary")
|
||||
}
|
||||
if report.ModulesSummary.TotalAppliedChanges < exp.MinimumCounts.Applied {
|
||||
t.Fatalf("expected at least %d applied changes, got %+v", exp.MinimumCounts.Applied, report.ModulesSummary)
|
||||
}
|
||||
validatorRejected := 0
|
||||
skipped := 0
|
||||
for _, mr := range report.ModuleResults {
|
||||
validatorRejected += len(mr.ValidatorRejected)
|
||||
skipped += len(mr.SkippedChanges)
|
||||
}
|
||||
if validatorRejected < exp.MinimumCounts.Rejected {
|
||||
t.Fatalf("expected at least %d validator rejections, got %d", exp.MinimumCounts.Rejected, validatorRejected)
|
||||
}
|
||||
if skipped < exp.MinimumCounts.Skipped {
|
||||
t.Fatalf("expected at least %d application skips, got %d", exp.MinimumCounts.Skipped, skipped)
|
||||
}
|
||||
}
|
||||
|
||||
func assertReleaseUtilizationShape(t *testing.T, report reporting.ProcessReport) {
|
||||
t.Helper()
|
||||
var payload struct {
|
||||
EffectiveConcurrency struct {
|
||||
TotalLLM int `json:"total_llm"`
|
||||
} `json:"effective_concurrency"`
|
||||
RunTiming struct {
|
||||
SchedulerQueueWaitMS int64 `json:"scheduler_queue_wait_ms"`
|
||||
LLMExecutionTimeMS int64 `json:"llm_execution_time_ms"`
|
||||
DeterministicValidationMS int64 `json:"deterministic_validation_time_ms"`
|
||||
} `json:"run_timing"`
|
||||
Modules []map[string]any `json:"modules"`
|
||||
Validators []map[string]any `json:"validators"`
|
||||
}
|
||||
if err := json.Unmarshal(readFile(t, report.Diagnostics.UtilizationSummaryPath), &payload); err != nil {
|
||||
t.Fatalf("unmarshal utilization diagnostics: %v", err)
|
||||
}
|
||||
if payload.EffectiveConcurrency.TotalLLM <= 0 {
|
||||
t.Fatalf("expected positive total llm concurrency, got %+v", payload.EffectiveConcurrency)
|
||||
}
|
||||
if payload.RunTiming.SchedulerQueueWaitMS < 0 || payload.RunTiming.LLMExecutionTimeMS < 0 || payload.RunTiming.DeterministicValidationMS < 0 {
|
||||
t.Fatalf("expected non-negative run timing values, got %+v", payload.RunTiming)
|
||||
}
|
||||
if len(payload.Modules) == 0 {
|
||||
t.Fatalf("expected module timing summaries")
|
||||
}
|
||||
if len(payload.Validators) == 0 {
|
||||
t.Fatalf("expected validator timing summaries")
|
||||
}
|
||||
}
|
||||
|
||||
func assertReleaseLedgerShape(t *testing.T, report reporting.ProcessReport) {
|
||||
t.Helper()
|
||||
var entries []struct {
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ProposalIndex int `json:"proposal_index"`
|
||||
Disposition string `json:"disposition"`
|
||||
DispositionReason string `json:"disposition_reason_code"`
|
||||
OriginalText string `json:"original_text"`
|
||||
ProposedCorrected string `json:"proposed_corrected_text"`
|
||||
ReplacementPolicy string `json:"replacement_policy"`
|
||||
DeterministicResults []struct {
|
||||
ValidatorKey string `json:"validator_key"`
|
||||
} `json:"deterministic_validator_decisions"`
|
||||
LLMResults []struct {
|
||||
ValidatorKey string `json:"validator_key"`
|
||||
} `json:"llm_validator_decisions"`
|
||||
}
|
||||
if err := json.Unmarshal(readFile(t, report.Diagnostics.CorrectionLedgerPath), &entries); err != nil {
|
||||
t.Fatalf("unmarshal correction ledger: %v", err)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
t.Fatalf("expected correction ledger entries")
|
||||
}
|
||||
hasApplied := false
|
||||
hasRejected := false
|
||||
hasSkipped := false
|
||||
for _, entry := range entries {
|
||||
if entry.ModuleInstance == "" || entry.ModuleKey == "" {
|
||||
t.Fatalf("expected module identity in ledger entry: %+v", entry)
|
||||
}
|
||||
switch entry.Disposition {
|
||||
case "applied":
|
||||
hasApplied = true
|
||||
case "rejected":
|
||||
hasRejected = true
|
||||
case "skipped":
|
||||
hasSkipped = true
|
||||
}
|
||||
}
|
||||
if !hasApplied || !hasRejected {
|
||||
t.Fatalf("expected applied and rejected entries in correction ledger, got %+v", entries)
|
||||
}
|
||||
if !hasSkipped {
|
||||
// Some deterministic fixture paths do not trigger apply-time skips;
|
||||
// rejections are still captured separately from application skips.
|
||||
}
|
||||
}
|
||||
|
||||
func assertPromptAndSchemaMetadataPresent(t *testing.T, runDir string) {
|
||||
t.Helper()
|
||||
metadataPaths, err := filepath.Glob(filepath.Join(runDir, "*", "*request-metadata.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("glob request metadata artifacts: %v", err)
|
||||
}
|
||||
if len(metadataPaths) == 0 {
|
||||
t.Fatalf("expected request metadata artifacts with prompt metadata")
|
||||
}
|
||||
|
||||
foundPromptMetadata := false
|
||||
foundSchemaMetadata := false
|
||||
for _, path := range metadataPaths {
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(readFile(t, path), &payload); err != nil {
|
||||
t.Fatalf("unmarshal request metadata artifact %q: %v", path, err)
|
||||
}
|
||||
if pm, ok := payload["prompt_metadata"].(map[string]any); ok {
|
||||
if pm["prompt_id"] != nil && pm["prompt_version"] != nil && pm["sha256"] != nil {
|
||||
foundPromptMetadata = true
|
||||
}
|
||||
}
|
||||
if sm, ok := payload["response_schema"].(map[string]any); ok {
|
||||
if sm["id"] != nil && sm["version"] != nil && sm["name"] != nil && sm["sha256"] != nil {
|
||||
foundSchemaMetadata = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !foundPromptMetadata {
|
||||
t.Fatalf("expected prompt metadata in request metadata artifacts")
|
||||
}
|
||||
if !foundSchemaMetadata {
|
||||
t.Fatalf("expected structured response schema metadata in request metadata artifacts")
|
||||
}
|
||||
}
|
||||
|
||||
func assertStableValidatorKeysPresent(t *testing.T, report reporting.ProcessReport) {
|
||||
t.Helper()
|
||||
seen := map[string]bool{}
|
||||
for _, module := range report.ModuleResults {
|
||||
for _, decision := range module.ValidatorDecisions {
|
||||
seen[decision.ValidatorName] = true
|
||||
}
|
||||
for _, rejected := range module.ValidatorRejected {
|
||||
seen[rejected.ValidatorName] = true
|
||||
}
|
||||
}
|
||||
expectedAny := []string{
|
||||
"confidence_threshold",
|
||||
"original_text_presence",
|
||||
"no_effect",
|
||||
}
|
||||
for _, key := range expectedAny {
|
||||
if !seen[key] {
|
||||
t.Fatalf("expected stable validator key %q in report decisions/rejections; seen=%v", key, seen)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertStdoutStderrContract(t *testing.T, stdout, stderr string) {
|
||||
t.Helper()
|
||||
if stdout != "" {
|
||||
t.Fatalf("expected empty stdout with --output, got %q", stdout)
|
||||
}
|
||||
if strings.Contains(stderr, `"module_results"`) || strings.Contains(stderr, `"report_metadata"`) {
|
||||
t.Fatalf("stderr should remain human-readable, not report JSON: %q", stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func assertAuditaV1OutputShape(t *testing.T, outputPath string) {
|
||||
t.Helper()
|
||||
var payload struct {
|
||||
Schema string `json:"schema"`
|
||||
Version string `json:"version"`
|
||||
Segments []schema.Segment `json:"segments"`
|
||||
}
|
||||
if err := json.Unmarshal(readFile(t, outputPath), &payload); err != nil {
|
||||
t.Fatalf("unmarshal audita-v1 output: %v", err)
|
||||
}
|
||||
if payload.Schema != "audita-v1" || payload.Version != "v1" {
|
||||
t.Fatalf("unexpected audita-v1 metadata: %+v", payload)
|
||||
}
|
||||
if len(payload.Segments) == 0 {
|
||||
t.Fatalf("expected non-empty audita-v1 segments")
|
||||
}
|
||||
}
|
||||
867
internal/cli/run.go
Normal file
867
internal/cli/run.go
Normal file
@@ -0,0 +1,867 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
||||
coreio "gitea.maximumdirect.net/eric/audita/internal/core/io"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/processreport"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
)
|
||||
|
||||
type noOpStructuredLLMClient struct{}
|
||||
|
||||
func (c noOpStructuredLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
_ = ctx
|
||||
_ = req
|
||||
switch target := out.(type) {
|
||||
case *validators.LLMValidationResponse:
|
||||
*target = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{}}
|
||||
case *proposal_generation.StructuredCorrectionSet:
|
||||
*target = proposal_generation.StructuredCorrectionSet{Corrections: nil}
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
}
|
||||
|
||||
func shouldUseNoOpLLMClientForTests() bool {
|
||||
return strings.HasSuffix(filepath.Base(os.Args[0]), ".test") || os.Getenv("GO_WANT_HELPER_PROCESS") == "1"
|
||||
}
|
||||
|
||||
type processInvocation struct {
|
||||
TranscriptPath string
|
||||
GlossaryPath string
|
||||
OutputPath string
|
||||
ReportJSONPath string
|
||||
Config config.Config
|
||||
ConfigPath string
|
||||
ConfigSource string
|
||||
ConfigVersion *int
|
||||
ExplicitModules bool
|
||||
}
|
||||
|
||||
var processModuleFactory runner.ModuleFactory
|
||||
var processProposalLLMClient contracts.StructuredLLMClient
|
||||
var processProposalLLMScheduler runner.ValidationScheduler
|
||||
var processValidationLLMClient contracts.StructuredLLMClient
|
||||
var processValidationLLMScheduler runner.ValidationScheduler
|
||||
var processRunnerContext = func() (context.Context, context.CancelFunc) {
|
||||
return context.Background(), func() {}
|
||||
}
|
||||
|
||||
var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) {
|
||||
runDir, err := diagnostics.NewRunDirectory(inv.Config.WorkDir, string(inv.Config.WorkDirRetention))
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, fmt.Errorf("run_dir_creation: %w", err)
|
||||
}
|
||||
|
||||
fail := func(phase string, err error, runOutput *runner.RunOutput) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("%s: %v", phase, err))
|
||||
return nil, nil, runOutput, runDir, fmt.Errorf("%s: %w", phase, err)
|
||||
}
|
||||
|
||||
if err := runDir.WriteInvocationMetadata(diagnostics.InvocationMetadata{
|
||||
Operation: "process",
|
||||
TranscriptPath: inv.TranscriptPath,
|
||||
GlossaryPath: inv.GlossaryPath,
|
||||
OutputPath: inv.OutputPath,
|
||||
ReportJSONPath: inv.ReportJSONPath,
|
||||
ConfigPath: inv.ConfigPath,
|
||||
ConfigSource: inv.ConfigSource,
|
||||
ConfigVersion: inv.ConfigVersion,
|
||||
TranscriptDescription: inv.Config.TranscriptDescription,
|
||||
Modules: append([]string(nil), inv.Config.Modules...),
|
||||
}); err != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("invocation_metadata: %v", err))
|
||||
}
|
||||
if err := runDir.WriteEffectiveConfig(inv.Config); err != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("effective_config: %v", err))
|
||||
}
|
||||
|
||||
transcriptBytes, err := coreio.ReadRequiredFile(inv.TranscriptPath, "transcript")
|
||||
if err != nil {
|
||||
return fail("transcript_read", err, nil)
|
||||
}
|
||||
|
||||
glossaryBytes, err := coreio.ReadRequiredFile(inv.GlossaryPath, "glossary")
|
||||
if err != nil {
|
||||
return fail("glossary_read", err, nil)
|
||||
}
|
||||
|
||||
sourceTranscript, err := schema.ParseSourceTranscriptJSON(transcriptBytes)
|
||||
if err != nil {
|
||||
return fail("transcript_schema", err, nil)
|
||||
}
|
||||
|
||||
glossary, err := schema.ParseGlossaryYAML(glossaryBytes)
|
||||
if err != nil {
|
||||
return fail("glossary_schema", err, nil)
|
||||
}
|
||||
|
||||
if err := runDir.WriteSourceTranscript(sourceTranscript, transcriptBytes); err != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("source_artifact: %v", err))
|
||||
}
|
||||
|
||||
canonical := sourceToCanonicalTranscript(sourceTranscript)
|
||||
normalizer := normalization.NewNormalizer(normalization.NormalizationConfig{
|
||||
MaxSegmentGap: inv.Config.Normalization.MaxSegmentGap,
|
||||
EllipsisGap: inv.Config.Normalization.EllipsisGap,
|
||||
MaxSegmentDuration: inv.Config.Normalization.MaxSegmentDuration,
|
||||
MaxSegmentTokens: inv.Config.Normalization.MaxSegmentTokens,
|
||||
})
|
||||
|
||||
normalizedTranscript, normSummary := normalizer.Normalize(canonical)
|
||||
|
||||
if err := runDir.WriteNormalizedTranscript(normalizedTranscript); err != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("normalized_artifact: %v", err))
|
||||
}
|
||||
if err := runDir.WriteNormalizationSummary(normSummary); err != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("normalization_summary: %v", err))
|
||||
}
|
||||
|
||||
// Compute chunks after normalization
|
||||
chunker := chunking.NewChunker(chunking.ChunkingConfig{
|
||||
MaxSectionTokens: inv.Config.MaxSectionTokens,
|
||||
MinSectionTokens: inv.Config.MinSectionTokens,
|
||||
TargetSections: inv.Config.TargetSections,
|
||||
})
|
||||
|
||||
sections, chunkErr := chunker.ChunkTranscript(normalizedTranscript)
|
||||
if chunkErr != nil {
|
||||
return fail("chunking", chunkErr, nil)
|
||||
}
|
||||
|
||||
chunkConfig := chunking.ChunkingConfig{
|
||||
MaxSectionTokens: inv.Config.MaxSectionTokens,
|
||||
MinSectionTokens: inv.Config.MinSectionTokens,
|
||||
TargetSections: inv.Config.TargetSections,
|
||||
}
|
||||
chunkSummary := chunking.ComputeSummary(sections, chunkConfig)
|
||||
chunkDetailedSummary := chunking.ComputeDetailedSummary(sections, chunkConfig)
|
||||
|
||||
if err := runDir.WriteChunkingSummary(&chunkDetailedSummary); err != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("chunking_summary: %v", err))
|
||||
}
|
||||
|
||||
workingTranscript := normalizedTranscript
|
||||
var runOutput *runner.RunOutput
|
||||
moduleFactory := processModuleFactory
|
||||
if moduleFactory == nil {
|
||||
moduleFactory = modules.NewFactory(modules.Dependencies{
|
||||
Config: &inv.Config,
|
||||
Glossary: glossary,
|
||||
DiagnosticsDir: runDir.Path(),
|
||||
})
|
||||
}
|
||||
if moduleFactory != nil {
|
||||
proposalLLMClient := processProposalLLMClient
|
||||
validationLLMClient := processValidationLLMClient
|
||||
proposalScheduler := processProposalLLMScheduler
|
||||
validationScheduler := processValidationLLMScheduler
|
||||
|
||||
if processModuleFactory == nil {
|
||||
// Production runtime path: construct clients/schedulers from config.
|
||||
if proposalLLMClient == nil {
|
||||
if shouldUseNoOpLLMClientForTests() {
|
||||
proposalLLMClient = noOpStructuredLLMClient{}
|
||||
} else {
|
||||
primaryCfg := llm.ResolvePrimaryConfig(inv.Config)
|
||||
client, clientErr := llm.NewOpenAICompatibleClient(primaryCfg.ToOpenAICompatibleClientConfig(nil))
|
||||
if clientErr != nil {
|
||||
return fail("runner_setup", clientErr, nil)
|
||||
}
|
||||
proposalLLMClient = client
|
||||
}
|
||||
}
|
||||
if validationLLMClient == nil {
|
||||
if shouldUseNoOpLLMClientForTests() {
|
||||
validationLLMClient = noOpStructuredLLMClient{}
|
||||
} else {
|
||||
validationCfg := llm.ResolveValidationConfig(inv.Config)
|
||||
client, clientErr := llm.NewOpenAICompatibleClient(validationCfg.ToOpenAICompatibleClientConfig(nil))
|
||||
if clientErr != nil {
|
||||
return fail("runner_setup", clientErr, nil)
|
||||
}
|
||||
validationLLMClient = client
|
||||
}
|
||||
}
|
||||
globalScheduler := proposalScheduler
|
||||
if globalScheduler == nil {
|
||||
s, sErr := llm.NewScheduler(inv.Config.TotalLLMConcurrency)
|
||||
if sErr != nil {
|
||||
return fail("runner_setup", sErr, nil)
|
||||
}
|
||||
globalScheduler = s
|
||||
}
|
||||
if proposalScheduler == nil {
|
||||
proposalScheduler = globalScheduler
|
||||
if inv.Config.EffectiveProposalLLMConcurrency() < inv.Config.TotalLLMConcurrency {
|
||||
s, sErr := llm.NewScheduler(inv.Config.EffectiveProposalLLMConcurrency())
|
||||
if sErr != nil {
|
||||
return fail("runner_setup", sErr, nil)
|
||||
}
|
||||
proposalScheduler = composeSchedulers(globalScheduler, s)
|
||||
}
|
||||
}
|
||||
if validationScheduler == nil {
|
||||
validationScheduler = globalScheduler
|
||||
if inv.Config.ValidationLLMConcurrency != nil && inv.Config.EffectiveValidationLLMConcurrency() < inv.Config.TotalLLMConcurrency {
|
||||
s, sErr := llm.NewScheduler(inv.Config.EffectiveValidationLLMConcurrency())
|
||||
if sErr != nil {
|
||||
return fail("runner_setup", sErr, nil)
|
||||
}
|
||||
validationScheduler = composeSchedulers(globalScheduler, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
moduleSpecs, err := contracts.ResolveModuleRunSpecs(inv.Config.Modules)
|
||||
if err != nil {
|
||||
return fail("runner_setup", err, nil)
|
||||
}
|
||||
|
||||
runCtx, cancelRun := processRunnerContext()
|
||||
defer cancelRun()
|
||||
runnerResult, runErr := runner.New(moduleFactory).Run(runCtx, runner.RunInput{
|
||||
Config: &inv.Config,
|
||||
Transcript: normalizedTranscript,
|
||||
Glossary: glossary,
|
||||
ModuleSpecs: moduleSpecs,
|
||||
EffectiveConcurrency: runner.EffectiveConcurrencyLimits{
|
||||
TotalLLM: inv.Config.TotalLLMConcurrency,
|
||||
ProposalLLM: inv.Config.EffectiveProposalLLMConcurrency(),
|
||||
ValidationLLM: inv.Config.EffectiveValidationLLMConcurrency(),
|
||||
},
|
||||
ProposalLLMClient: proposalLLMClient,
|
||||
ProposalLLMScheduler: proposalScheduler,
|
||||
ProposalDiagnosticsDir: runDir.Path(),
|
||||
ValidationLLMClient: validationLLMClient,
|
||||
ValidationLLMScheduler: validationScheduler,
|
||||
ValidationDiagnosticsDir: runDir.Path(),
|
||||
})
|
||||
runOutput = &runnerResult
|
||||
if runErr != nil {
|
||||
return fail("runner_execution", runErr, runOutput)
|
||||
}
|
||||
workingTranscript = runnerResult.FinalTranscript
|
||||
}
|
||||
|
||||
encoderDef, err := outputschema.Resolve(inv.Config.OutputSchema)
|
||||
if err != nil {
|
||||
return fail("output_schema", err, runOutput)
|
||||
}
|
||||
outputBytes, err := encoderDef.Encoder(workingTranscript)
|
||||
if err != nil {
|
||||
return fail("serialization", err, runOutput)
|
||||
}
|
||||
|
||||
if strings.TrimSpace(inv.OutputPath) != "" {
|
||||
if err := coreio.WriteFile(inv.OutputPath, outputBytes); err != nil {
|
||||
return fail("output_write", err, runOutput)
|
||||
}
|
||||
return normSummary, &chunkSummary, runOutput, runDir, nil
|
||||
}
|
||||
|
||||
if _, err := stdout.Write(outputBytes); err != nil {
|
||||
return fail("stdout_write", err, runOutput)
|
||||
}
|
||||
|
||||
return normSummary, &chunkSummary, runOutput, runDir, nil
|
||||
}
|
||||
|
||||
func sourceToCanonicalTranscript(source *schema.SourceTranscript) *schema.Transcript {
|
||||
segments := make([]schema.Segment, len(source.Segments))
|
||||
for i, s := range source.Segments {
|
||||
id := i + 1
|
||||
if s.ID != nil {
|
||||
id = *s.ID
|
||||
}
|
||||
segments[i] = schema.Segment{
|
||||
ID: id,
|
||||
Speaker: s.Speaker,
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
Text: s.Text,
|
||||
Categories: s.Categories,
|
||||
}
|
||||
}
|
||||
return &schema.Transcript{Segments: segments}
|
||||
}
|
||||
|
||||
type chainedScheduler struct {
|
||||
schedulers []runner.ValidationScheduler
|
||||
}
|
||||
|
||||
func (s chainedScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
|
||||
if len(s.schedulers) == 0 {
|
||||
return fn(ctx)
|
||||
}
|
||||
run := fn
|
||||
for i := len(s.schedulers) - 1; i >= 0; i-- {
|
||||
scheduler := s.schedulers[i]
|
||||
next := run
|
||||
run = func(callCtx context.Context) error {
|
||||
return scheduler.Run(callCtx, next)
|
||||
}
|
||||
}
|
||||
return run(ctx)
|
||||
}
|
||||
|
||||
func composeSchedulers(schedulers ...runner.ValidationScheduler) runner.ValidationScheduler {
|
||||
filtered := make([]runner.ValidationScheduler, 0, len(schedulers))
|
||||
for _, scheduler := range schedulers {
|
||||
if scheduler != nil {
|
||||
filtered = append(filtered, scheduler)
|
||||
}
|
||||
}
|
||||
switch len(filtered) {
|
||||
case 0:
|
||||
return nil
|
||||
case 1:
|
||||
return filtered[0]
|
||||
default:
|
||||
return chainedScheduler{schedulers: filtered}
|
||||
}
|
||||
}
|
||||
|
||||
// Run executes the Audita CLI with the provided arguments and streams.
|
||||
func Run(args []string, stdout, stderr io.Writer) int {
|
||||
if len(args) == 0 {
|
||||
writeRootUsage(stdout)
|
||||
return 0
|
||||
}
|
||||
|
||||
if isHelpCommand(args) {
|
||||
writeRootUsage(stdout)
|
||||
return 0
|
||||
}
|
||||
|
||||
if args[0] == "process" {
|
||||
return runProcess(args[1:], stdout, stderr)
|
||||
}
|
||||
if args[0] == "config" {
|
||||
return runConfig(args[1:], stdout, stderr)
|
||||
}
|
||||
|
||||
fmt.Fprintf(stderr, "audita: unknown command %q\n\n", args[0])
|
||||
writeRootUsage(stderr)
|
||||
return 2
|
||||
}
|
||||
|
||||
func runProcess(args []string, stdout, stderr io.Writer) int {
|
||||
startedAt := time.Now().UTC()
|
||||
|
||||
configPathOverride, configPathOverrideSet, err := findConfigPathOverride(args)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
effectiveConfig, err := config.LoadEffectiveConfig(configPathOverride, configPathOverrideSet)
|
||||
if err != nil {
|
||||
var effectiveConfigErr *config.EffectiveConfigError
|
||||
if errors.As(err, &effectiveConfigErr) {
|
||||
switch effectiveConfigErr.Kind {
|
||||
case config.EffectiveConfigErrorLoadFile, config.EffectiveConfigErrorApplyFile:
|
||||
fmt.Fprintf(stderr, "audita process: invalid config file: %v\n", effectiveConfigErr)
|
||||
case config.EffectiveConfigErrorApplyEnv:
|
||||
fmt.Fprintf(stderr, "audita process: invalid environment configuration: %v\n", effectiveConfigErr)
|
||||
default:
|
||||
fmt.Fprintf(stderr, "audita process: %v\n", effectiveConfigErr)
|
||||
}
|
||||
} else {
|
||||
fmt.Fprintf(stderr, "audita process: %v\n", err)
|
||||
}
|
||||
return 2
|
||||
}
|
||||
|
||||
cfg := effectiveConfig.Config
|
||||
configPath := effectiveConfig.ConfigPath
|
||||
configSource := effectiveConfig.ConfigSource
|
||||
configVersion := effectiveConfig.ConfigVersion
|
||||
|
||||
fs, pFlags := newProcessFlagSet(cfg, stderr)
|
||||
|
||||
if isHelpCommand(args) || hasHelpFlag(args) {
|
||||
writeProcessUsage(stdout, fs)
|
||||
return 0
|
||||
}
|
||||
|
||||
parseArgs := args
|
||||
transcriptFromFront := ""
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
transcriptFromFront = args[0]
|
||||
parseArgs = args[1:]
|
||||
}
|
||||
|
||||
if err := fs.Parse(parseArgs); err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
writeProcessUsage(stdout, fs)
|
||||
return 0
|
||||
}
|
||||
return 2
|
||||
}
|
||||
|
||||
overrides, explicitModules := processCLIOverrides(fs, pFlags)
|
||||
|
||||
if err := cfg.ApplyCLIOverrides(overrides); err != nil {
|
||||
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
|
||||
remaining := fs.Args()
|
||||
positional := make([]string, 0, len(remaining)+1)
|
||||
if transcriptFromFront != "" {
|
||||
positional = append(positional, transcriptFromFront)
|
||||
}
|
||||
positional = append(positional, remaining...)
|
||||
|
||||
if len(positional) != 1 {
|
||||
fmt.Fprintln(stderr, "audita process: expected exactly 1 transcript JSON path argument")
|
||||
return 2
|
||||
}
|
||||
if strings.TrimSpace(*pFlags.glossaryPath) == "" {
|
||||
fmt.Fprintln(stderr, "audita process: --glossary is required")
|
||||
return 2
|
||||
}
|
||||
|
||||
inv := processInvocation{
|
||||
TranscriptPath: positional[0],
|
||||
GlossaryPath: *pFlags.glossaryPath,
|
||||
OutputPath: *pFlags.outputPath,
|
||||
ReportJSONPath: *pFlags.reportJSONPath,
|
||||
Config: cfg,
|
||||
ConfigPath: configPath,
|
||||
ConfigSource: configSource,
|
||||
ConfigVersion: configVersion,
|
||||
ExplicitModules: explicitModules,
|
||||
}
|
||||
|
||||
normSummary, chunkSummary, runOutput, runDir, runErr := processRunner(inv, stdout)
|
||||
completedAt := time.Now().UTC()
|
||||
|
||||
if runErr != nil {
|
||||
if runDir != nil && runOutput != nil {
|
||||
if runOutput.Utilization != nil {
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
||||
}
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
|
||||
RunDirectoryPath: runDir.Path(),
|
||||
RunOutput: runOutput,
|
||||
}))
|
||||
}
|
||||
errorPhase, errorMessage := extractErrorPhase(runErr)
|
||||
report := processreport.Build(processReportInput("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput))
|
||||
|
||||
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
||||
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
||||
fmt.Fprintf(stderr, "audita process: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
if runDir != nil {
|
||||
_ = runDir.WriteReport(report)
|
||||
_ = runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
||||
RunSucceeded: false,
|
||||
})
|
||||
}
|
||||
|
||||
fmt.Fprintf(stderr, "audita process: %v\n", runErr)
|
||||
if runDir != nil {
|
||||
fmt.Fprintf(stderr, "audita process: diagnostics: %s\n", runDir.Path())
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
if runDir != nil && runOutput != nil {
|
||||
if runOutput.Utilization != nil {
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
||||
}
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
|
||||
RunDirectoryPath: runDir.Path(),
|
||||
RunOutput: runOutput,
|
||||
}))
|
||||
}
|
||||
|
||||
report := processreport.Build(processReportInput("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput))
|
||||
|
||||
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
||||
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
||||
if runDir != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("report_write: %v", err))
|
||||
_ = runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
||||
RunSucceeded: false,
|
||||
})
|
||||
}
|
||||
fmt.Fprintf(stderr, "audita process: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
if runDir != nil {
|
||||
_ = runDir.WriteReport(report)
|
||||
if err := runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
||||
RunSucceeded: true,
|
||||
HasSkippedCorrections: processreport.HasSkippedCorrections(runOutput),
|
||||
}); err != nil {
|
||||
fmt.Fprintf(stderr, "audita process: failed to apply work-dir retention: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
func runConfig(args []string, stdout, stderr io.Writer) int {
|
||||
if len(args) == 0 || isHelpCommand(args) || hasHelpFlag(args) {
|
||||
writeConfigUsage(stdout)
|
||||
return 0
|
||||
}
|
||||
|
||||
switch args[0] {
|
||||
case "validate":
|
||||
return runConfigValidate(args[1:], stdout, stderr)
|
||||
case "print-effective":
|
||||
return runConfigPrintEffective(args[1:], stdout, stderr)
|
||||
default:
|
||||
fmt.Fprintf(stderr, "audita config: unknown command %q\n\n", args[0])
|
||||
writeConfigUsage(stderr)
|
||||
return 2
|
||||
}
|
||||
}
|
||||
|
||||
func runConfigValidate(args []string, stdout, stderr io.Writer) int {
|
||||
fs := flag.NewFlagSet("config validate", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
configPath := fs.String("config", "", "Path to versioned YAML config file")
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
writeConfigValidateUsage(stdout)
|
||||
return 0
|
||||
}
|
||||
return 2
|
||||
}
|
||||
if strings.TrimSpace(*configPath) == "" {
|
||||
fmt.Fprintln(stderr, "audita config validate: --config is required")
|
||||
return 2
|
||||
}
|
||||
if len(fs.Args()) != 0 {
|
||||
fmt.Fprintln(stderr, "audita config validate: unexpected positional arguments")
|
||||
return 2
|
||||
}
|
||||
|
||||
fileCfg, err := config.LoadFileConfig(strings.TrimSpace(*configPath))
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
cfg := config.Default()
|
||||
if err := cfg.ApplyFileConfig(fileCfg); err != nil {
|
||||
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
fmt.Fprintln(stdout, "config is valid")
|
||||
return 0
|
||||
}
|
||||
|
||||
func runConfigPrintEffective(args []string, stdout, stderr io.Writer) int {
|
||||
fs := flag.NewFlagSet("config print-effective", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
configPath := fs.String("config", "", "Path to versioned YAML config file")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
writeConfigPrintEffectiveUsage(stdout)
|
||||
return 0
|
||||
}
|
||||
return 2
|
||||
}
|
||||
if len(fs.Args()) != 0 {
|
||||
fmt.Fprintln(stderr, "audita config print-effective: unexpected positional arguments")
|
||||
return 2
|
||||
}
|
||||
|
||||
configPathValue := strings.TrimSpace(*configPath)
|
||||
configPathSet := configPathValue != ""
|
||||
effectiveConfig, err := config.LoadEffectiveConfig(configPathValue, configPathSet)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
|
||||
cfg := effectiveConfig.Config
|
||||
|
||||
redacted := cfg.Redacted()
|
||||
out, err := json.MarshalIndent(redacted, "", " ")
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
out = append(out, '\n')
|
||||
if _, err := stdout.Write(out); err != nil {
|
||||
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func extractErrorPhase(err error) (phase string, message string) {
|
||||
msg := err.Error()
|
||||
if strings.Contains(msg, ": ") {
|
||||
parts := strings.SplitN(msg, ": ", 2)
|
||||
if len(parts) == 2 {
|
||||
return parts[0], parts[1]
|
||||
}
|
||||
}
|
||||
return "", msg
|
||||
}
|
||||
|
||||
func processReportInput(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) processreport.BuildInput {
|
||||
runDirectoryPath := ""
|
||||
if runDir != nil {
|
||||
runDirectoryPath = runDir.Path()
|
||||
}
|
||||
return processreport.BuildInput{
|
||||
Status: status,
|
||||
TranscriptPath: inv.TranscriptPath,
|
||||
GlossaryPath: inv.GlossaryPath,
|
||||
OutputPath: inv.OutputPath,
|
||||
Modules: inv.Config.Modules,
|
||||
OutputSchema: inv.Config.OutputSchema,
|
||||
ConfigVersion: inv.ConfigVersion,
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
ErrorMessage: errorMessage,
|
||||
ErrorPhase: errorPhase,
|
||||
RunDirectoryPath: runDirectoryPath,
|
||||
NormalizationSummary: normalizationSummary,
|
||||
ChunkingSummary: chunkingSummary,
|
||||
RunOutput: runOutput,
|
||||
}
|
||||
}
|
||||
|
||||
type processFlags struct {
|
||||
configPath *string
|
||||
glossaryPath *string
|
||||
outputPath *string
|
||||
reportJSONPath *string
|
||||
modules *string
|
||||
outputSchema *string
|
||||
llmAPIKey *string
|
||||
validationLLMAPIKey *string
|
||||
model *string
|
||||
validationModel *string
|
||||
baseURL *string
|
||||
validationBaseURL *string
|
||||
llmTimeoutSeconds *int
|
||||
totalLLMConcurrency *int
|
||||
proposalLLMConcurrency *int
|
||||
llmConcurrency *int
|
||||
validationLLMTimeoutSeconds *int
|
||||
validationMaxPromptTokens *int
|
||||
targetSections *int
|
||||
maxRetries *int
|
||||
validationMaxRetries *int
|
||||
validationLLMConcurrency *int
|
||||
maxSectionTokens *int
|
||||
minSectionTokens *int
|
||||
glossaryConfidenceThreshold *float64
|
||||
grammarConfidenceThreshold *float64
|
||||
homophonesConfidenceThreshold *float64
|
||||
spokenWordConfidenceThreshold *float64
|
||||
normalizeMaxSegmentGap *float64
|
||||
normalizeEllipsisGap *float64
|
||||
normalizeMaxSegmentDuration *float64
|
||||
normalizeMaxSegmentTokens *int
|
||||
transcriptDescription *string
|
||||
workDir *string
|
||||
workDirRetention *string
|
||||
}
|
||||
|
||||
func newProcessFlagSet(cfg config.Config, stderr io.Writer) (*flag.FlagSet, processFlags) {
|
||||
fs := flag.NewFlagSet("process", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
|
||||
validationTimeoutSecondsDefault := cfg.PrimaryLLM.TimeoutSeconds
|
||||
if cfg.ValidationLLM.TimeoutSeconds != nil {
|
||||
validationTimeoutSecondsDefault = *cfg.ValidationLLM.TimeoutSeconds
|
||||
}
|
||||
|
||||
validationMaxRetriesDefault := cfg.PrimaryLLM.MaxRetries
|
||||
if cfg.ValidationLLM.MaxRetries != nil {
|
||||
validationMaxRetriesDefault = *cfg.ValidationLLM.MaxRetries
|
||||
}
|
||||
|
||||
validationLLMConcurrencyDefault := cfg.TotalLLMConcurrency
|
||||
if cfg.ValidationLLMConcurrency != nil {
|
||||
validationLLMConcurrencyDefault = *cfg.ValidationLLMConcurrency
|
||||
}
|
||||
|
||||
targetSectionsDefault := 0
|
||||
if cfg.TargetSections != nil {
|
||||
targetSectionsDefault = *cfg.TargetSections
|
||||
}
|
||||
|
||||
pFlags := processFlags{
|
||||
configPath: fs.String("config", "", "Path to versioned YAML config file"),
|
||||
glossaryPath: fs.String("glossary", "", "Path to glossary YAML file"),
|
||||
outputPath: fs.String("output", "", "Path to corrected transcript JSON output file"),
|
||||
reportJSONPath: fs.String("report-json", "", "Path to machine-readable report JSON output file"),
|
||||
modules: fs.String("modules", strings.Join(cfg.Modules, ","), "Comma-separated module sequence override"),
|
||||
outputSchema: fs.String("output-schema", cfg.OutputSchema, "Output schema: bare-segments|audita-v1"),
|
||||
llmAPIKey: fs.String("llm-api-key", cfg.PrimaryLLM.APIKey, "Primary LLM API key"),
|
||||
validationLLMAPIKey: fs.String("validation-llm-api-key", cfg.ValidationLLM.APIKey, "Validation LLM API key"),
|
||||
model: fs.String("model", cfg.PrimaryLLM.Model, "Primary LLM model name"),
|
||||
validationModel: fs.String("validation-model", cfg.ValidationLLM.Model, "Validation LLM model name"),
|
||||
baseURL: fs.String("base-url", cfg.PrimaryLLM.BaseURL, "Primary OpenAI-compatible base URL"),
|
||||
validationBaseURL: fs.String("validation-base-url", cfg.ValidationLLM.BaseURL, "Validation OpenAI-compatible base URL"),
|
||||
llmTimeoutSeconds: fs.Int("llm-timeout-seconds", cfg.PrimaryLLM.TimeoutSeconds, "Primary LLM timeout in seconds"),
|
||||
totalLLMConcurrency: fs.Int("total-llm-concurrency", cfg.TotalLLMConcurrency, "Total concurrent LLM calls across proposal and validation"),
|
||||
proposalLLMConcurrency: fs.Int("proposal-llm-concurrency", cfg.EffectiveProposalLLMConcurrency(), "Concurrent proposal-generation LLM calls"),
|
||||
llmConcurrency: fs.Int("llm-concurrency", cfg.TotalLLMConcurrency, "Alias for --total-llm-concurrency"),
|
||||
validationLLMTimeoutSeconds: fs.Int("validation-llm-timeout-seconds", validationTimeoutSecondsDefault, "Validation LLM timeout in seconds"),
|
||||
validationMaxPromptTokens: fs.Int("validation-max-prompt-tokens", cfg.ValidationMaxPromptTokens, "Validation max prompt tokens"),
|
||||
targetSections: fs.Int("target-sections", targetSectionsDefault, "Target number of transcript sections"),
|
||||
maxRetries: fs.Int("max-retries", cfg.PrimaryLLM.MaxRetries, "Maximum structured-output retries"),
|
||||
validationMaxRetries: fs.Int("validation-max-retries", validationMaxRetriesDefault, "Validation structured-output retries"),
|
||||
validationLLMConcurrency: fs.Int("validation-llm-concurrency", validationLLMConcurrencyDefault, "Concurrent validation LLM calls (inherits total when unset)"),
|
||||
maxSectionTokens: fs.Int("max-section-tokens", cfg.MaxSectionTokens, "Maximum section tokens"),
|
||||
minSectionTokens: fs.Int("min-section-tokens", cfg.MinSectionTokens, "Minimum section tokens"),
|
||||
glossaryConfidenceThreshold: fs.Float64("glossary-confidence-threshold", cfg.Thresholds.Glossary, "Glossary confidence threshold"),
|
||||
grammarConfidenceThreshold: fs.Float64("grammar-confidence-threshold", cfg.Thresholds.Grammar, "Grammar confidence threshold"),
|
||||
homophonesConfidenceThreshold: fs.Float64("homophones-confidence-threshold", cfg.Thresholds.Homophones, "Homophones confidence threshold"),
|
||||
spokenWordConfidenceThreshold: fs.Float64("spoken-word-confidence-threshold", cfg.Thresholds.SpokenWord, "Spoken-word confidence threshold"),
|
||||
normalizeMaxSegmentGap: fs.Float64("normalize-max-segment-gap", cfg.Normalization.MaxSegmentGap, "Maximum same-speaker merge gap"),
|
||||
normalizeEllipsisGap: fs.Float64("normalize-ellipsis-gap", cfg.Normalization.EllipsisGap, "Gap threshold for ellipsis insertion"),
|
||||
normalizeMaxSegmentDuration: fs.Float64("normalize-max-segment-duration", cfg.Normalization.MaxSegmentDuration, "Maximum merged segment duration"),
|
||||
normalizeMaxSegmentTokens: fs.Int("normalize-max-segment-tokens", cfg.Normalization.MaxSegmentTokens, "Maximum merged segment token estimate"),
|
||||
transcriptDescription: fs.String("transcript-description", cfg.TranscriptDescription, "Brief background context for LLM prompts; does not override transcript content"),
|
||||
workDir: fs.String("work-dir", cfg.WorkDir, "Per-run work directory"),
|
||||
workDirRetention: fs.String("work-dir-retention", string(cfg.WorkDirRetention), "Work-dir retention policy: auto|always|never"),
|
||||
}
|
||||
|
||||
return fs, pFlags
|
||||
}
|
||||
|
||||
func findConfigPathOverride(args []string) (path string, set bool, err error) {
|
||||
for i := 0; i < len(args); i++ {
|
||||
arg := strings.TrimSpace(args[i])
|
||||
if arg == "" {
|
||||
continue
|
||||
}
|
||||
if arg == "--config" {
|
||||
if i+1 >= len(args) {
|
||||
return "", false, fmt.Errorf("--config requires a path")
|
||||
}
|
||||
return strings.TrimSpace(args[i+1]), true, nil
|
||||
}
|
||||
if strings.HasPrefix(arg, "--config=") {
|
||||
return strings.TrimSpace(strings.TrimPrefix(arg, "--config=")), true, nil
|
||||
}
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
func isHelpCommand(args []string) bool {
|
||||
if len(args) == 0 {
|
||||
return false
|
||||
}
|
||||
if len(args) == 1 {
|
||||
switch args[0] {
|
||||
case "help", "-h", "--help":
|
||||
return true
|
||||
}
|
||||
}
|
||||
if len(args) == 2 && args[0] == "help" {
|
||||
switch args[1] {
|
||||
case "process":
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasHelpFlag(args []string) bool {
|
||||
for _, arg := range args {
|
||||
if arg == "-h" || arg == "--help" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func writeRootUsage(w io.Writer) {
|
||||
fmt.Fprintln(w, "Audita is a transcript processing CLI.")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Usage:")
|
||||
fmt.Fprintln(w, " audita <command> [options]")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Commands:")
|
||||
fmt.Fprintln(w, " process Process a transcript JSON file")
|
||||
fmt.Fprintln(w, " config Validate and inspect config")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Example:")
|
||||
fmt.Fprintln(w, " audita process transcript.json --glossary glossary.yaml --output corrected.json")
|
||||
}
|
||||
|
||||
func writeConfigUsage(w io.Writer) {
|
||||
fmt.Fprintln(w, "Validate and inspect Audita config.")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Usage:")
|
||||
fmt.Fprintln(w, " audita config <command> [flags]")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Commands:")
|
||||
fmt.Fprintln(w, " validate Validate a versioned YAML config file")
|
||||
fmt.Fprintln(w, " print-effective Print redacted effective config JSON (defaults + config file + env)")
|
||||
}
|
||||
|
||||
func writeConfigValidateUsage(w io.Writer) {
|
||||
fmt.Fprintln(w, "Validate a versioned YAML config file.")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Usage:")
|
||||
fmt.Fprintln(w, " audita config validate --config <path>")
|
||||
}
|
||||
|
||||
func writeConfigPrintEffectiveUsage(w io.Writer) {
|
||||
fmt.Fprintln(w, "Print redacted effective config JSON.")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Usage:")
|
||||
fmt.Fprintln(w, " audita config print-effective [--config <path>]")
|
||||
}
|
||||
|
||||
func writeProcessUsage(w io.Writer, fs *flag.FlagSet) {
|
||||
fmt.Fprintln(w, "Process a transcript JSON file.")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Usage:")
|
||||
fmt.Fprintln(w, " audita process <transcript.json> [flags]")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Flags:")
|
||||
fs.VisitAll(func(f *flag.Flag) {
|
||||
fmt.Fprintf(w, " --%s\n", f.Name)
|
||||
})
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Example:")
|
||||
fmt.Fprintln(w, " audita process transcript.json --glossary glossary.yaml --output corrected.json")
|
||||
}
|
||||
4354
internal/cli/run_test.go
Normal file
4354
internal/cli/run_test.go
Normal file
File diff suppressed because it is too large
Load Diff
110
internal/cli/subprocess_test_hooks.go
Normal file
110
internal/cli/subprocess_test_hooks.go
Normal file
@@ -0,0 +1,110 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
)
|
||||
|
||||
const (
|
||||
subprocessTestLLMModeEnv = "AUDITA_SUBPROCESS_TEST_LLM_MODE"
|
||||
subprocessTestRunTimeoutMSEnv = "AUDITA_SUBPROCESS_TEST_RUN_TIMEOUT_MS"
|
||||
)
|
||||
|
||||
// ConfigureSubprocessTestHooksFromEnv enables deterministic test-only hooks for
|
||||
// subprocess integration tests that run through the Go test binary helper path.
|
||||
func ConfigureSubprocessTestHooksFromEnv() {
|
||||
mode := strings.TrimSpace(os.Getenv(subprocessTestLLMModeEnv))
|
||||
timeoutMSRaw := strings.TrimSpace(os.Getenv(subprocessTestRunTimeoutMSEnv))
|
||||
// Only activate in explicit subprocess test mode.
|
||||
if mode == "" && timeoutMSRaw == "" {
|
||||
return
|
||||
}
|
||||
|
||||
if mode != "" {
|
||||
client := &subprocessTestLLMClient{mode: mode}
|
||||
processProposalLLMClient = client
|
||||
processValidationLLMClient = client
|
||||
}
|
||||
|
||||
if timeoutMSRaw == "" {
|
||||
return
|
||||
}
|
||||
timeoutMS, err := strconv.Atoi(timeoutMSRaw)
|
||||
if err != nil || timeoutMS <= 0 {
|
||||
return
|
||||
}
|
||||
processRunnerContext = func() (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.Background(), time.Duration(timeoutMS)*time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
type subprocessTestLLMClient struct {
|
||||
mode string
|
||||
mu sync.Mutex
|
||||
proposals int
|
||||
}
|
||||
|
||||
func (c *subprocessTestLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
_ = req
|
||||
|
||||
switch c.mode {
|
||||
case "backend_error":
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("synthetic backend failure")
|
||||
case "block_until_cancel":
|
||||
<-ctx.Done()
|
||||
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
||||
case "malformed_structured":
|
||||
switch target := out.(type) {
|
||||
case *proposal_generation.StructuredCorrectionSet:
|
||||
*target = proposal_generation.StructuredCorrectionSet{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y", Confidence: 0.99},
|
||||
},
|
||||
}
|
||||
case *validators.LLMValidationResponse:
|
||||
*target = validators.LLMValidationResponse{
|
||||
Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 999, Approved: true, Confidence: 0.9, Reason: "bad index"},
|
||||
},
|
||||
}
|
||||
}
|
||||
case "mid_pipeline_fail":
|
||||
if _, ok := out.(*proposal_generation.StructuredCorrectionSet); ok {
|
||||
c.mu.Lock()
|
||||
c.proposals++
|
||||
proposalCall := c.proposals
|
||||
c.mu.Unlock()
|
||||
|
||||
if proposalCall >= 3 {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("synthetic mid-pipeline failure")
|
||||
}
|
||||
}
|
||||
switch target := out.(type) {
|
||||
case *proposal_generation.StructuredCorrectionSet:
|
||||
*target = proposal_generation.StructuredCorrectionSet{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99},
|
||||
},
|
||||
}
|
||||
case *validators.LLMValidationResponse:
|
||||
*target = validators.LLMValidationResponse{Validations: nil}
|
||||
}
|
||||
default:
|
||||
switch target := out.(type) {
|
||||
case *proposal_generation.StructuredCorrectionSet:
|
||||
*target = proposal_generation.StructuredCorrectionSet{Corrections: nil}
|
||||
case *validators.LLMValidationResponse:
|
||||
*target = validators.LLMValidationResponse{Validations: nil}
|
||||
}
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
}
|
||||
1
internal/cli/testdata/malformed_transcript.json
vendored
Normal file
1
internal/cli/testdata/malformed_transcript.json
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"segments":[{"id":1,"text":"oops"}
|
||||
22
internal/cli/testdata/parity/application-skip-ambiguous.case.json
vendored
Normal file
22
internal/cli/testdata/parity/application-skip-ambiguous.case.json
vendored
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"name": "ambiguous_match_rejected_before_application",
|
||||
"transcript_file": "application-skip-ambiguous.transcript.json",
|
||||
"glossary_file": "default-handoff.glossary.yaml",
|
||||
"modules_csv": "homophones",
|
||||
"proposal_responses_file": "application-skip-ambiguous.proposals.json",
|
||||
"validation_responses_file": "application-skip-ambiguous.validations.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "application-skip-ambiguous.expected-transcript.json",
|
||||
"module_instances": ["homophones"],
|
||||
"module_count": 1,
|
||||
"total_applied_changes": 0,
|
||||
"total_skipped_changes": 1,
|
||||
"module_applied_counts": [0],
|
||||
"module_rejected_counts": [1],
|
||||
"module_skip_counts": [0],
|
||||
"validator_rejected_reason_codes": ["ambiguous_original_text"],
|
||||
"expected_proposal_calls": ["homophones:proposal"]
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/application-skip-ambiguous.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/application-skip-ambiguous.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"the site near another site"}
|
||||
]
|
||||
3
internal/cli/testdata/parity/application-skip-ambiguous.proposals.json
vendored
Normal file
3
internal/cli/testdata/parity/application-skip-ambiguous.proposals.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]}
|
||||
]
|
||||
3
internal/cli/testdata/parity/application-skip-ambiguous.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/application-skip-ambiguous.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"the site near another site"}
|
||||
]
|
||||
4
internal/cli/testdata/parity/application-skip-ambiguous.validations.json
vendored
Normal file
4
internal/cli/testdata/parity/application-skip-ambiguous.validations.json
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
28
internal/cli/testdata/parity/default-full-pipeline.case.json
vendored
Normal file
28
internal/cli/testdata/parity/default-full-pipeline.case.json
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "default_full_pipeline_shape_and_reports",
|
||||
"transcript_file": "default-full-pipeline.transcript.json",
|
||||
"glossary_file": "default-full-pipeline.glossary.yaml",
|
||||
"proposal_responses_file": "default-full-pipeline.proposals.json",
|
||||
"validation_responses_file": "default-full-pipeline.validations.json",
|
||||
"env": {
|
||||
"AUDITA_LLM_API_KEY": "parity-secret",
|
||||
"AUDITA_VALIDATION_LLM_API_KEY": "parity-secret"
|
||||
},
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "default-full-pipeline.expected-transcript.json",
|
||||
"module_instances": ["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
|
||||
"module_count": 5,
|
||||
"total_applied_changes": 3,
|
||||
"total_skipped_changes": 3,
|
||||
"secret_markers": ["parity-secret"],
|
||||
"expected_proposal_calls": [
|
||||
"glossary_1:proposal",
|
||||
"homophones:proposal",
|
||||
"glossary_2:proposal",
|
||||
"spoken_word:proposal",
|
||||
"grammar:proposal"
|
||||
]
|
||||
}
|
||||
}
|
||||
9
internal/cli/testdata/parity/default-full-pipeline.expected-transcript.json
vendored
Normal file
9
internal/cli/testdata/parity/default-full-pipeline.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "Alice",
|
||||
"start": 0,
|
||||
"end": 1,
|
||||
"text": "Hello, there were Jesters hmm"
|
||||
}
|
||||
]
|
||||
7
internal/cli/testdata/parity/default-full-pipeline.glossary.yaml
vendored
Normal file
7
internal/cli/testdata/parity/default-full-pipeline.glossary.yaml
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
glossary:
|
||||
- name: Jesters
|
||||
aliases:
|
||||
- jester
|
||||
plural: jesters
|
||||
category: faction
|
||||
summary: A protected in-world faction term.
|
||||
28
internal/cli/testdata/parity/default-full-pipeline.proposals.json
vendored
Normal file
28
internal/cli/testdata/parity/default-full-pipeline.proposals.json
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
[
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "Jesters", "corrected_text": "jesters", "confidence": 0.99},
|
||||
{"id": 1, "original_text": "Jesters", "corrected_text": "JESTERX", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "jesters", "corrected_text": "JESTERS", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "uh", "corrected_text": "hmm", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
|
||||
]
|
||||
}
|
||||
]
|
||||
3
internal/cli/testdata/parity/default-full-pipeline.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/default-full-pipeline.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
|
||||
]
|
||||
12
internal/cli/testdata/parity/default-full-pipeline.validations.json
vendored
Normal file
12
internal/cli/testdata/parity/default-full-pipeline.validations.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject cleanup"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "parity-secret"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
27
internal/cli/testdata/parity/default-handoff.case.json
vendored
Normal file
27
internal/cli/testdata/parity/default-handoff.case.json
vendored
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"name": "default_pipeline_handoff_and_module_order",
|
||||
"transcript_file": "default-handoff.transcript.json",
|
||||
"glossary_file": "default-handoff.glossary.yaml",
|
||||
"proposal_responses_file": "default-handoff.proposals.json",
|
||||
"validation_responses_file": "default-handoff.validations.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "default-handoff.expected-transcript.json",
|
||||
"module_instances": ["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
|
||||
"module_count": 5,
|
||||
"total_applied_changes": 5,
|
||||
"total_skipped_changes": 0,
|
||||
"module_applied_counts": [1, 1, 1, 1, 1],
|
||||
"module_rejected_counts": [0, 0, 0, 0, 0],
|
||||
"module_skip_counts": [0, 0, 0, 0, 0],
|
||||
"expected_proposal_calls": [
|
||||
"glossary_1:proposal",
|
||||
"homophones:proposal",
|
||||
"glossary_2:proposal",
|
||||
"spoken_word:proposal",
|
||||
"grammar:proposal"
|
||||
],
|
||||
"min_response_payload_artifacts": 15
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/default-handoff.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/default-handoff.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"Hello, there were Jesters at the Sight um"}
|
||||
]
|
||||
6
internal/cli/testdata/parity/default-handoff.glossary.yaml
vendored
Normal file
6
internal/cli/testdata/parity/default-handoff.glossary.yaml
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
glossary:
|
||||
- name: Jesters
|
||||
aliases: [jester]
|
||||
plural: jesters
|
||||
category: faction
|
||||
summary: A protected in-world faction term.
|
||||
7
internal/cli/testdata/parity/default-handoff.proposals.json
vendored
Normal file
7
internal/cli/testdata/parity/default-handoff.proposals.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
[
|
||||
{"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "sight", "corrected_text": "Sight", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "um um", "corrected_text": "um", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}]}
|
||||
]
|
||||
3
internal/cli/testdata/parity/default-handoff.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/default-handoff.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures at the site um um"}
|
||||
]
|
||||
12
internal/cli/testdata/parity/default-handoff.validations.json
vendored
Normal file
12
internal/cli/testdata/parity/default-handoff.validations.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
17
internal/cli/testdata/parity/deterministic-validator-low-confidence.case.json
vendored
Normal file
17
internal/cli/testdata/parity/deterministic-validator-low-confidence.case.json
vendored
Normal file
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"name": "deterministic_validator_low_confidence",
|
||||
"transcript_file": "deterministic-validator-low-confidence.transcript.json",
|
||||
"glossary_file": "default-full-pipeline.glossary.yaml",
|
||||
"modules_csv": "grammar",
|
||||
"proposal_responses_file": "deterministic-validator-low-confidence.proposals.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "deterministic-validator-low-confidence.expected-transcript.json",
|
||||
"module_instances": ["grammar"],
|
||||
"module_count": 1,
|
||||
"total_skipped_changes": 1,
|
||||
"validator_rejected_reason_codes": ["low_confidence"],
|
||||
"expected_proposal_calls": ["grammar:proposal"]
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/deterministic-validator-low-confidence.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/deterministic-validator-low-confidence.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"hello , world"}
|
||||
]
|
||||
7
internal/cli/testdata/parity/deterministic-validator-low-confidence.proposals.json
vendored
Normal file
7
internal/cli/testdata/parity/deterministic-validator-low-confidence.proposals.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
[
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.1}
|
||||
]
|
||||
}
|
||||
]
|
||||
3
internal/cli/testdata/parity/deterministic-validator-low-confidence.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/deterministic-validator-low-confidence.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , world"}
|
||||
]
|
||||
12
internal/cli/testdata/parity/glossary-schema-error.case.json
vendored
Normal file
12
internal/cli/testdata/parity/glossary-schema-error.case.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"name": "glossary_schema_handling",
|
||||
"transcript_file": "default-full-pipeline.transcript.json",
|
||||
"glossary_file": "glossary-schema-error.glossary.yaml",
|
||||
"expect": {
|
||||
"exit_code": 1,
|
||||
"status": "failed",
|
||||
"error_phase": "glossary_schema",
|
||||
"stderr_contains": "glossary_schema",
|
||||
"require_error_log": true
|
||||
}
|
||||
}
|
||||
2
internal/cli/testdata/parity/glossary-schema-error.glossary.yaml
vendored
Normal file
2
internal/cli/testdata/parity/glossary-schema-error.glossary.yaml
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
glossary:
|
||||
- name: MissingCategoryAndSummary
|
||||
19
internal/cli/testdata/parity/llm-validator-rejection.case.json
vendored
Normal file
19
internal/cli/testdata/parity/llm-validator-rejection.case.json
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "llm_validator_decision_handling",
|
||||
"transcript_file": "llm-validator-rejection.transcript.json",
|
||||
"glossary_file": "default-full-pipeline.glossary.yaml",
|
||||
"modules_csv": "grammar",
|
||||
"proposal_responses_file": "llm-validator-rejection.proposals.json",
|
||||
"validation_responses_file": "llm-validator-rejection.validations.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "llm-validator-rejection.expected-transcript.json",
|
||||
"module_instances": ["grammar"],
|
||||
"module_count": 1,
|
||||
"total_skipped_changes": 1,
|
||||
"validator_rejected_reason_codes": ["llm_rejected"],
|
||||
"expected_proposal_calls": ["grammar:proposal"],
|
||||
"expected_validation_calls": ["grammar:section-0000:editorial_review:batch-0000"]
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/llm-validator-rejection.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/llm-validator-rejection.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"hello , world"}
|
||||
]
|
||||
7
internal/cli/testdata/parity/llm-validator-rejection.proposals.json
vendored
Normal file
7
internal/cli/testdata/parity/llm-validator-rejection.proposals.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
[
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
|
||||
]
|
||||
}
|
||||
]
|
||||
3
internal/cli/testdata/parity/llm-validator-rejection.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/llm-validator-rejection.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , world"}
|
||||
]
|
||||
7
internal/cli/testdata/parity/llm-validator-rejection.validations.json
vendored
Normal file
7
internal/cli/testdata/parity/llm-validator-rejection.validations.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
[
|
||||
{
|
||||
"validations": [
|
||||
{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject stylistic overreach"}
|
||||
]
|
||||
}
|
||||
]
|
||||
23
internal/cli/testdata/parity/mid-pipeline-failure.case.json
vendored
Normal file
23
internal/cli/testdata/parity/mid-pipeline-failure.case.json
vendored
Normal file
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "mid_pipeline_failure_partial_progress",
|
||||
"transcript_file": "default-handoff.transcript.json",
|
||||
"glossary_file": "default-handoff.glossary.yaml",
|
||||
"proposal_responses_file": "mid-pipeline-failure.proposals.json",
|
||||
"validation_responses_file": "mid-pipeline-failure.validations.json",
|
||||
"expect": {
|
||||
"exit_code": 1,
|
||||
"status": "failed",
|
||||
"error_phase": "runner_execution",
|
||||
"stderr_contains": "runner_execution",
|
||||
"module_instances": ["glossary_1", "homophones", "glossary_2"],
|
||||
"module_count": 3,
|
||||
"total_applied_changes": 2,
|
||||
"total_skipped_changes": 0,
|
||||
"failed_module_instance": "glossary_2",
|
||||
"module_applied_counts": [1, 1, 0],
|
||||
"module_rejected_counts": [0, 0, 0],
|
||||
"module_skip_counts": [0, 0, 0],
|
||||
"require_error_log": true,
|
||||
"expected_proposal_calls": ["glossary_1:proposal", "homophones:proposal", "glossary_2:proposal"]
|
||||
}
|
||||
}
|
||||
4
internal/cli/testdata/parity/mid-pipeline-failure.proposals.json
vendored
Normal file
4
internal/cli/testdata/parity/mid-pipeline-failure.proposals.json
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
[
|
||||
{"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]}
|
||||
]
|
||||
6
internal/cli/testdata/parity/mid-pipeline-failure.validations.json
vendored
Normal file
6
internal/cli/testdata/parity/mid-pipeline-failure.validations.json
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
21
internal/cli/testdata/parity/protected-term-rejection.case.json
vendored
Normal file
21
internal/cli/testdata/parity/protected-term-rejection.case.json
vendored
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "protected_glossary_term_behavior",
|
||||
"transcript_file": "protected-term-rejection.transcript.json",
|
||||
"glossary_file": "default-handoff.glossary.yaml",
|
||||
"modules_csv": "homophones",
|
||||
"proposal_responses_file": "protected-term-rejection.proposals.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "protected-term-rejection.expected-transcript.json",
|
||||
"module_instances": ["homophones"],
|
||||
"module_count": 1,
|
||||
"total_applied_changes": 0,
|
||||
"total_skipped_changes": 1,
|
||||
"module_applied_counts": [0],
|
||||
"module_rejected_counts": [1],
|
||||
"module_skip_counts": [0],
|
||||
"validator_rejected_reason_codes": ["protected_glossary_term"],
|
||||
"expected_proposal_calls": ["homophones:proposal"]
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/protected-term-rejection.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/protected-term-rejection.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"The Jesters entered the hall."}
|
||||
]
|
||||
3
internal/cli/testdata/parity/protected-term-rejection.proposals.json
vendored
Normal file
3
internal/cli/testdata/parity/protected-term-rejection.proposals.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"corrections": [{"id": 1, "original_text": "Jesters", "corrected_text": "Gestures", "confidence": 0.99}]}
|
||||
]
|
||||
3
internal/cli/testdata/parity/protected-term-rejection.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/protected-term-rejection.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"The Jesters entered the hall."}
|
||||
]
|
||||
12
internal/cli/testdata/parity/transcript-schema-error.case.json
vendored
Normal file
12
internal/cli/testdata/parity/transcript-schema-error.case.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"name": "transcript_schema_handling",
|
||||
"transcript_file": "transcript-schema-error.transcript.json",
|
||||
"glossary_file": "default-full-pipeline.glossary.yaml",
|
||||
"expect": {
|
||||
"exit_code": 1,
|
||||
"status": "failed",
|
||||
"error_phase": "transcript_schema",
|
||||
"stderr_contains": "transcript_schema",
|
||||
"require_error_log": true
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/transcript-schema-error.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/transcript-schema-error.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"","start":0.0,"end":1.0,"text":"bad"}
|
||||
]
|
||||
24
internal/cli/testdata/release/default-release.expectations.json
vendored
Normal file
24
internal/cli/testdata/release/default-release.expectations.json
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"must_apply_texts": [
|
||||
"Hello, there were Jesters hmm"
|
||||
],
|
||||
"must_not_apply_texts": [
|
||||
"JESTERX",
|
||||
"there were gestures"
|
||||
],
|
||||
"protected_terms": [
|
||||
"Jesters"
|
||||
],
|
||||
"expected_module_instances": [
|
||||
"glossary_1",
|
||||
"homophones",
|
||||
"glossary_2",
|
||||
"spoken_word",
|
||||
"grammar"
|
||||
],
|
||||
"minimum_counts": {
|
||||
"applied": 1,
|
||||
"rejected": 1,
|
||||
"skipped": 0
|
||||
}
|
||||
}
|
||||
9
internal/cli/testdata/release/default-release.expected-transcript.json
vendored
Normal file
9
internal/cli/testdata/release/default-release.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "Alice",
|
||||
"start": 0,
|
||||
"end": 1,
|
||||
"text": "Hello, there were Jesters hmm"
|
||||
}
|
||||
]
|
||||
7
internal/cli/testdata/release/default-release.glossary.yaml
vendored
Normal file
7
internal/cli/testdata/release/default-release.glossary.yaml
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
glossary:
|
||||
- name: Jesters
|
||||
aliases:
|
||||
- jester
|
||||
plural: jesters
|
||||
category: faction
|
||||
summary: A protected in-world faction term.
|
||||
28
internal/cli/testdata/release/default-release.proposals.json
vendored
Normal file
28
internal/cli/testdata/release/default-release.proposals.json
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
[
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "Jesters", "corrected_text": "jesters", "confidence": 0.99},
|
||||
{"id": 1, "original_text": "Jesters", "corrected_text": "JESTERX", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "jesters", "corrected_text": "JESTERS", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "uh", "corrected_text": "hmm", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
|
||||
]
|
||||
}
|
||||
]
|
||||
3
internal/cli/testdata/release/default-release.transcript.json
vendored
Normal file
3
internal/cli/testdata/release/default-release.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
|
||||
]
|
||||
12
internal/cli/testdata/release/default-release.validations.json
vendored
Normal file
12
internal/cli/testdata/release/default-release.validations.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject cleanup"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "release-secret"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
6
internal/cli/testdata/tiny_glossary.yaml
vendored
Normal file
6
internal/cli/testdata/tiny_glossary.yaml
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
glossary:
|
||||
- name: Audita
|
||||
aliases:
|
||||
- audita
|
||||
category: product
|
||||
summary: The Audita transcript correction CLI.
|
||||
9
internal/cli/testdata/tiny_transcript.json
vendored
Normal file
9
internal/cli/testdata/tiny_transcript.json
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "A",
|
||||
"start": 0.0,
|
||||
"end": 1.2,
|
||||
"text": "hello world"
|
||||
}
|
||||
]
|
||||
312
internal/core/chunking/sections.go
Normal file
312
internal/core/chunking/sections.go
Normal file
@@ -0,0 +1,312 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
// Section represents a contiguous chunk of transcript segments with metadata.
|
||||
type Section struct {
|
||||
// Index is the 0-based section index within the chunked transcript
|
||||
Index int `json:"section_index"`
|
||||
|
||||
// StartSegmentID is the ID of the first segment in this section
|
||||
StartSegmentID int `json:"start_segment_id"`
|
||||
|
||||
// EndSegmentID is the ID of the last segment in this section
|
||||
EndSegmentID int `json:"end_segment_id"`
|
||||
|
||||
// EstimatedTokens is the approximate token count for this section
|
||||
EstimatedTokens int `json:"estimated_tokens"`
|
||||
|
||||
// Segments contains the segments in this section, in order
|
||||
Segments []schema.Segment `json:"segments"`
|
||||
}
|
||||
|
||||
// ChunkingConfig holds configuration for transcript chunking.
|
||||
type ChunkingConfig struct {
|
||||
// MaxSectionTokens is the maximum allowed tokens per section
|
||||
MaxSectionTokens int
|
||||
|
||||
// MinSectionTokens is a validated soft lower-bound setting retained for
|
||||
// configuration/reporting compatibility.
|
||||
MinSectionTokens int
|
||||
|
||||
// TargetSections is an optional target number of sections
|
||||
// If nil, section count is derived from total/max token budgeting.
|
||||
TargetSections *int
|
||||
}
|
||||
|
||||
// Chunker performs deterministic chunking of normalized transcript segments.
|
||||
type Chunker struct {
|
||||
config ChunkingConfig
|
||||
estimator TokenEstimator
|
||||
}
|
||||
|
||||
// NewChunker creates a new chunker with the given configuration.
|
||||
func NewChunker(config ChunkingConfig) *Chunker {
|
||||
return &Chunker{
|
||||
config: config,
|
||||
estimator: NewSimpleTokenEstimator(),
|
||||
}
|
||||
}
|
||||
|
||||
// NewChunkerWithEstimator creates a new chunker with a custom estimator.
|
||||
func NewChunkerWithEstimator(config ChunkingConfig, estimator TokenEstimator) *Chunker {
|
||||
return &Chunker{
|
||||
config: config,
|
||||
estimator: estimator,
|
||||
}
|
||||
}
|
||||
|
||||
// ChunkTranscript divides a normalized transcript into contiguous token-bounded
|
||||
// sections using a deterministic balanced forward pass.
|
||||
//
|
||||
// Behavior:
|
||||
// - preserve segment order and never split segments;
|
||||
// - estimate per-segment tokens once, then compute total;
|
||||
// - derive desired section count from ceil(total/max_section_tokens), unless
|
||||
// target_sections is explicitly set;
|
||||
// - prefer section sizes near ceil(total/section_count) while never exceeding
|
||||
// max_section_tokens unless a section consists of a single oversized segment.
|
||||
//
|
||||
// Returns an error if explicit target_sections is impossible under constraints.
|
||||
// The input transcript is never mutated.
|
||||
func (c *Chunker) ChunkTranscript(transcript *schema.Transcript) ([]Section, error) {
|
||||
if transcript == nil || len(transcript.Segments) == 0 {
|
||||
return []Section{}, nil
|
||||
}
|
||||
|
||||
// Validate configuration
|
||||
if c.config.MaxSectionTokens <= 0 {
|
||||
return nil, fmt.Errorf("max_section_tokens must be positive, got %d", c.config.MaxSectionTokens)
|
||||
}
|
||||
|
||||
if c.config.MinSectionTokens < 0 {
|
||||
return nil, fmt.Errorf("min_section_tokens must be non-negative, got %d", c.config.MinSectionTokens)
|
||||
}
|
||||
|
||||
if c.config.MinSectionTokens > c.config.MaxSectionTokens {
|
||||
return nil, fmt.Errorf("min_section_tokens (%d) cannot exceed max_section_tokens (%d)",
|
||||
c.config.MinSectionTokens, c.config.MaxSectionTokens)
|
||||
}
|
||||
|
||||
// Calculate token counts for each segment (deterministic).
|
||||
segmentTokens := make([]int, len(transcript.Segments))
|
||||
totalTokens := 0
|
||||
for i, seg := range transcript.Segments {
|
||||
segmentTokens[i] = c.estimator.EstimateTokens(seg.Text)
|
||||
totalTokens += segmentTokens[i]
|
||||
}
|
||||
|
||||
minPossibleSections := c.calculateMinPossibleSections(segmentTokens)
|
||||
|
||||
var desiredSections int
|
||||
useExplicitTarget := false
|
||||
if c.config.TargetSections != nil {
|
||||
desiredSections = *c.config.TargetSections
|
||||
useExplicitTarget = true
|
||||
if desiredSections <= 0 {
|
||||
return nil, fmt.Errorf("target_sections must be positive, got %d", desiredSections)
|
||||
}
|
||||
if err := c.validateTargetSections(desiredSections, segmentTokens); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
desiredSections = ceilDiv(totalTokens, c.config.MaxSectionTokens)
|
||||
if desiredSections < minPossibleSections {
|
||||
desiredSections = minPossibleSections
|
||||
}
|
||||
if desiredSections < 1 {
|
||||
desiredSections = 1
|
||||
}
|
||||
if desiredSections > len(transcript.Segments) {
|
||||
desiredSections = len(transcript.Segments)
|
||||
}
|
||||
}
|
||||
|
||||
targetTokensPerSection := ceilDiv(totalTokens, desiredSections)
|
||||
if useExplicitTarget {
|
||||
return c.buildSectionsWithExplicitTarget(
|
||||
transcript.Segments,
|
||||
segmentTokens,
|
||||
desiredSections,
|
||||
targetTokensPerSection,
|
||||
)
|
||||
}
|
||||
|
||||
return c.buildSectionsBalanced(transcript.Segments, segmentTokens, targetTokensPerSection), nil
|
||||
}
|
||||
|
||||
// validateTargetSections checks if the target section count is achievable.
|
||||
func (c *Chunker) validateTargetSections(target int, segmentTokens []int) error {
|
||||
// Maximum possible sections: limited by segment count
|
||||
maxPossibleSections := len(segmentTokens)
|
||||
|
||||
if target > maxPossibleSections {
|
||||
return fmt.Errorf(
|
||||
"target_sections (%d) is impossible: cannot have more sections than segments (%d)",
|
||||
target, maxPossibleSections)
|
||||
}
|
||||
|
||||
// Minimum possible sections: each segment must fit within max bounds
|
||||
minPossibleSections := c.calculateMinPossibleSections(segmentTokens)
|
||||
|
||||
if target < minPossibleSections {
|
||||
return fmt.Errorf(
|
||||
"target_sections (%d) is impossible: need at least %d sections to respect max_section_tokens (%d)",
|
||||
target, minPossibleSections, c.config.MaxSectionTokens)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildSectionsBalanced creates sections with a deterministic single-pass policy.
|
||||
func (c *Chunker) buildSectionsBalanced(segments []schema.Segment, segmentTokens []int, targetTokensPerSection int) []Section {
|
||||
var sections []Section
|
||||
var currentSegments []schema.Segment
|
||||
currentTokens := 0
|
||||
|
||||
for i, seg := range segments {
|
||||
tokens := segmentTokens[i]
|
||||
|
||||
// Empty section: always accept the next segment, including oversized.
|
||||
if len(currentSegments) == 0 {
|
||||
currentSegments = append(currentSegments, seg)
|
||||
currentTokens = tokens
|
||||
continue
|
||||
}
|
||||
|
||||
// If adding next segment would exceed max, close current section.
|
||||
if currentTokens+tokens > c.config.MaxSectionTokens {
|
||||
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
|
||||
currentSegments = []schema.Segment{seg}
|
||||
currentTokens = tokens
|
||||
continue
|
||||
}
|
||||
|
||||
// Prefer staying near target tokens per section.
|
||||
if targetTokensPerSection == 0 || currentTokens < targetTokensPerSection {
|
||||
currentSegments = append(currentSegments, seg)
|
||||
currentTokens += tokens
|
||||
continue
|
||||
}
|
||||
|
||||
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
|
||||
currentSegments = []schema.Segment{seg}
|
||||
currentTokens = tokens
|
||||
}
|
||||
|
||||
if len(currentSegments) > 0 {
|
||||
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
|
||||
}
|
||||
|
||||
return sections
|
||||
}
|
||||
|
||||
// buildSection creates a Section from segments.
|
||||
func (c *Chunker) buildSection(index int, segments []schema.Segment, tokens int) Section {
|
||||
return Section{
|
||||
Index: index,
|
||||
StartSegmentID: segments[0].ID,
|
||||
EndSegmentID: segments[len(segments)-1].ID,
|
||||
EstimatedTokens: tokens,
|
||||
Segments: segments,
|
||||
}
|
||||
}
|
||||
|
||||
// calculateMinPossibleSections calculates the minimum number of sections needed
|
||||
// to ensure no section exceeds max tokens.
|
||||
func (c *Chunker) calculateMinPossibleSections(segmentTokens []int) int {
|
||||
sections := 0
|
||||
currentTokens := 0
|
||||
|
||||
for _, tokens := range segmentTokens {
|
||||
if tokens > c.config.MaxSectionTokens {
|
||||
// Each oversized segment needs its own section
|
||||
if currentTokens > 0 {
|
||||
sections++
|
||||
currentTokens = 0
|
||||
}
|
||||
sections++
|
||||
} else if currentTokens+tokens > c.config.MaxSectionTokens {
|
||||
sections++
|
||||
currentTokens = tokens
|
||||
} else {
|
||||
currentTokens += tokens
|
||||
}
|
||||
}
|
||||
|
||||
if currentTokens > 0 {
|
||||
sections++
|
||||
}
|
||||
|
||||
return sections
|
||||
}
|
||||
|
||||
// buildSectionsWithExplicitTarget builds exactly desiredSections when feasible.
|
||||
func (c *Chunker) buildSectionsWithExplicitTarget(
|
||||
segments []schema.Segment,
|
||||
segmentTokens []int,
|
||||
desiredSections int,
|
||||
targetTokensPerSection int,
|
||||
) ([]Section, error) {
|
||||
n := len(segments)
|
||||
cursor := 0
|
||||
sections := make([]Section, 0, desiredSections)
|
||||
|
||||
for sectionIdx := 0; sectionIdx < desiredSections; sectionIdx++ {
|
||||
if cursor >= n {
|
||||
break
|
||||
}
|
||||
|
||||
remainingSectionsAfter := desiredSections - sectionIdx - 1
|
||||
currentSegments := []schema.Segment{segments[cursor]}
|
||||
currentTokens := segmentTokens[cursor]
|
||||
cursor++
|
||||
|
||||
for cursor < n {
|
||||
remainingSegments := n - cursor
|
||||
|
||||
// Reserve one segment per future section to avoid empty sections.
|
||||
if remainingSegments == remainingSectionsAfter {
|
||||
break
|
||||
}
|
||||
|
||||
nextTokens := segmentTokens[cursor]
|
||||
if currentTokens+nextTokens > c.config.MaxSectionTokens {
|
||||
break
|
||||
}
|
||||
if targetTokensPerSection == 0 || currentTokens < targetTokensPerSection {
|
||||
currentSegments = append(currentSegments, segments[cursor])
|
||||
currentTokens += nextTokens
|
||||
cursor++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
|
||||
}
|
||||
|
||||
if cursor != n || len(sections) != desiredSections {
|
||||
return nil, fmt.Errorf(
|
||||
"target_sections (%d) is impossible under current constraints (got %d sections)",
|
||||
desiredSections,
|
||||
len(sections),
|
||||
)
|
||||
}
|
||||
|
||||
return sections, nil
|
||||
}
|
||||
|
||||
func ceilDiv(numerator int, denominator int) int {
|
||||
if denominator <= 0 {
|
||||
return 0
|
||||
}
|
||||
if numerator <= 0 {
|
||||
return 0
|
||||
}
|
||||
return (numerator + denominator - 1) / denominator
|
||||
}
|
||||
518
internal/core/chunking/sections_test.go
Normal file
518
internal/core/chunking/sections_test.go
Normal file
@@ -0,0 +1,518 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
// mapTokenEstimator provides deterministic per-segment token counts for tests.
|
||||
type mapTokenEstimator struct {
|
||||
byText map[string]int
|
||||
}
|
||||
|
||||
func (e *mapTokenEstimator) EstimateTokens(text string) int {
|
||||
if e.byText == nil {
|
||||
return 0
|
||||
}
|
||||
if tokens, ok := e.byText[text]; ok {
|
||||
return tokens
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func makeSegments(texts []string) []schema.Segment {
|
||||
segments := make([]schema.Segment, len(texts))
|
||||
for i, text := range texts {
|
||||
segments[i] = schema.Segment{
|
||||
ID: i + 1,
|
||||
Speaker: "DM",
|
||||
Start: float64(i * 10),
|
||||
End: float64(i*10 + 5),
|
||||
Text: text,
|
||||
}
|
||||
}
|
||||
return segments
|
||||
}
|
||||
|
||||
func makeTranscript(segments []schema.Segment) *schema.Transcript {
|
||||
return &schema.Transcript{Segments: segments}
|
||||
}
|
||||
|
||||
func intPtr(i int) *int {
|
||||
return &i
|
||||
}
|
||||
|
||||
func assertSegmentCoverageAndOrder(t *testing.T, input []schema.Segment, sections []Section) {
|
||||
t.Helper()
|
||||
|
||||
seen := make([]schema.Segment, 0, len(input))
|
||||
for _, sec := range sections {
|
||||
seen = append(seen, sec.Segments...)
|
||||
}
|
||||
|
||||
if len(seen) != len(input) {
|
||||
t.Fatalf("expected %d total segment occurrences, got %d", len(input), len(seen))
|
||||
}
|
||||
|
||||
for i := range input {
|
||||
if seen[i].ID != input[i].ID {
|
||||
t.Fatalf("segment order mismatch at index %d: got id=%d want id=%d", i, seen[i].ID, input[i].ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertSectionMetadataConsistent(t *testing.T, sections []Section) {
|
||||
t.Helper()
|
||||
|
||||
for i, sec := range sections {
|
||||
if sec.Index != i {
|
||||
t.Fatalf("section %d: expected index=%d got=%d", i, i, sec.Index)
|
||||
}
|
||||
if len(sec.Segments) == 0 {
|
||||
t.Fatalf("section %d: section must not be empty", i)
|
||||
}
|
||||
if sec.StartSegmentID != sec.Segments[0].ID {
|
||||
t.Fatalf("section %d: start_segment_id mismatch", i)
|
||||
}
|
||||
if sec.EndSegmentID != sec.Segments[len(sec.Segments)-1].ID {
|
||||
t.Fatalf("section %d: end_segment_id mismatch", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertMaxBoundExceptSingletonOversized(t *testing.T, sections []Section, max int) {
|
||||
t.Helper()
|
||||
|
||||
for i, sec := range sections {
|
||||
if sec.EstimatedTokens <= max {
|
||||
continue
|
||||
}
|
||||
if len(sec.Segments) != 1 {
|
||||
t.Fatalf("section %d exceeds max tokens (%d>%d) with %d segments", i, sec.EstimatedTokens, max, len(sec.Segments))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func imbalance(sections []Section) int {
|
||||
if len(sections) == 0 {
|
||||
return 0
|
||||
}
|
||||
minTokens := sections[0].EstimatedTokens
|
||||
maxTokens := sections[0].EstimatedTokens
|
||||
for _, sec := range sections {
|
||||
if sec.EstimatedTokens < minTokens {
|
||||
minTokens = sec.EstimatedTokens
|
||||
}
|
||||
if sec.EstimatedTokens > maxTokens {
|
||||
maxTokens = sec.EstimatedTokens
|
||||
}
|
||||
}
|
||||
return maxTokens - minTokens
|
||||
}
|
||||
|
||||
func greedyMaxFillSections(segments []schema.Segment, tokens []int, max int) []Section {
|
||||
sections := make([]Section, 0)
|
||||
var current []schema.Segment
|
||||
currentTokens := 0
|
||||
|
||||
for i, seg := range segments {
|
||||
tok := tokens[i]
|
||||
if len(current) == 0 {
|
||||
current = append(current, seg)
|
||||
currentTokens = tok
|
||||
continue
|
||||
}
|
||||
if currentTokens+tok > max {
|
||||
sections = append(sections, Section{
|
||||
Index: len(sections),
|
||||
StartSegmentID: current[0].ID,
|
||||
EndSegmentID: current[len(current)-1].ID,
|
||||
EstimatedTokens: currentTokens,
|
||||
Segments: append([]schema.Segment(nil), current...),
|
||||
})
|
||||
current = []schema.Segment{seg}
|
||||
currentTokens = tok
|
||||
continue
|
||||
}
|
||||
current = append(current, seg)
|
||||
currentTokens += tok
|
||||
}
|
||||
|
||||
if len(current) > 0 {
|
||||
sections = append(sections, Section{
|
||||
Index: len(sections),
|
||||
StartSegmentID: current[0].ID,
|
||||
EndSegmentID: current[len(current)-1].ID,
|
||||
EstimatedTokens: currentTokens,
|
||||
Segments: append([]schema.Segment(nil), current...),
|
||||
})
|
||||
}
|
||||
|
||||
return sections
|
||||
}
|
||||
|
||||
func TestChunkEmptyTranscript(t *testing.T) {
|
||||
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 10})
|
||||
|
||||
sections, err := chunker.ChunkTranscript(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript(nil): %v", err)
|
||||
}
|
||||
if len(sections) != 0 {
|
||||
t.Fatalf("expected 0 sections for nil transcript, got %d", len(sections))
|
||||
}
|
||||
|
||||
sections, err = chunker.ChunkTranscript(makeTranscript(nil))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript(empty): %v", err)
|
||||
}
|
||||
if len(sections) != 0 {
|
||||
t.Fatalf("expected 0 sections for empty transcript, got %d", len(sections))
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkSingleSegment(t *testing.T) {
|
||||
segments := makeSegments([]string{"s1"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 10},
|
||||
&mapTokenEstimator{byText: map[string]int{"s1": 7}},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("expected 1 section, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 7 {
|
||||
t.Fatalf("expected estimated_tokens=7, got %d", sections[0].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertSectionMetadataConsistent(t, sections)
|
||||
}
|
||||
|
||||
func TestChunkSingleOversizedSegment(t *testing.T) {
|
||||
segments := makeSegments([]string{"big"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 5},
|
||||
&mapTokenEstimator{byText: map[string]int{"big": 120}},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("expected 1 section, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 120 {
|
||||
t.Fatalf("expected oversized singleton section, got %d", sections[0].EstimatedTokens)
|
||||
}
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 50)
|
||||
}
|
||||
|
||||
func TestChunkTotalBelowMaxSingleSection(t *testing.T) {
|
||||
segments := makeSegments([]string{"a", "b", "c"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 5},
|
||||
&ConstTokenEstimator{Tokens: 10},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("expected 1 section, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 30 {
|
||||
t.Fatalf("expected 30 section tokens, got %d", sections[0].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
}
|
||||
|
||||
func TestChunkTotalExactlyDivisibleByMax(t *testing.T) {
|
||||
segments := makeSegments([]string{"a", "b", "c", "d"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1},
|
||||
&ConstTokenEstimator{Tokens: 5},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 2 {
|
||||
t.Fatalf("expected 2 sections, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 10 || sections[1].EstimatedTokens != 10 {
|
||||
t.Fatalf("expected [10,10] tokens, got [%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 10)
|
||||
}
|
||||
|
||||
func TestChunkTotalNotDivisibleByMax(t *testing.T) {
|
||||
segments := makeSegments([]string{"a", "b", "c", "d", "e"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1},
|
||||
&ConstTokenEstimator{Tokens: 5},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 3 {
|
||||
t.Fatalf("expected 3 sections, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 10 || sections[1].EstimatedTokens != 10 || sections[2].EstimatedTokens != 5 {
|
||||
t.Fatalf("expected [10,10,5] tokens, got [%d,%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens, sections[2].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 10)
|
||||
}
|
||||
|
||||
func TestChunkTargetSectionsPrecedenceAndSuccess(t *testing.T) {
|
||||
segments := makeSegments([]string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10"})
|
||||
target := 3
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 200, MinSectionTokens: 1, TargetSections: &target},
|
||||
&ConstTokenEstimator{Tokens: 10},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != target {
|
||||
t.Fatalf("expected %d sections from explicit target, got %d", target, len(sections))
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertSectionMetadataConsistent(t, sections)
|
||||
}
|
||||
|
||||
func TestChunkTargetSectionsImpossibleTooMany(t *testing.T) {
|
||||
segments := makeSegments([]string{"1", "2", "3", "4", "5"})
|
||||
target := 10
|
||||
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 1, TargetSections: &target})
|
||||
|
||||
_, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err == nil {
|
||||
t.Fatal("expected error for impossible target_sections")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "cannot have more sections than segments") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkTargetSectionsImpossibleTooFew(t *testing.T) {
|
||||
segments := makeSegments([]string{"1", "2", "3"})
|
||||
target := 1
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1, TargetSections: &target},
|
||||
&ConstTokenEstimator{Tokens: 30},
|
||||
)
|
||||
|
||||
_, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err == nil {
|
||||
t.Fatal("expected error for impossible target_sections")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "need at least 3 sections") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkManySmallSegmentsBalanced(t *testing.T) {
|
||||
texts := []string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"}
|
||||
segments := makeSegments(texts)
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1},
|
||||
&ConstTokenEstimator{Tokens: 10},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 3 {
|
||||
t.Fatalf("expected 3 sections, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 40 || sections[1].EstimatedTokens != 40 || sections[2].EstimatedTokens != 30 {
|
||||
t.Fatalf("expected [40,40,30], got [%d,%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens, sections[2].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 50)
|
||||
}
|
||||
|
||||
func TestChunkMixedLargeAndSmallSegments(t *testing.T) {
|
||||
segments := makeSegments([]string{"big1", "s1", "s2", "s3", "big2", "s4"})
|
||||
estimator := &mapTokenEstimator{byText: map[string]int{
|
||||
"big1": 120,
|
||||
"s1": 10,
|
||||
"s2": 10,
|
||||
"s3": 10,
|
||||
"big2": 120,
|
||||
"s4": 10,
|
||||
}}
|
||||
chunker := NewChunkerWithEstimator(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 1}, estimator)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 4 {
|
||||
t.Fatalf("expected 4 sections, got %d", len(sections))
|
||||
}
|
||||
if len(sections[0].Segments) != 1 || sections[0].Segments[0].Text != "big1" {
|
||||
t.Fatalf("expected first oversized segment in singleton section, got %+v", sections[0].Segments)
|
||||
}
|
||||
if len(sections[2].Segments) != 1 || sections[2].Segments[0].Text != "big2" {
|
||||
t.Fatalf("expected second oversized segment in singleton section, got %+v", sections[2].Segments)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 100)
|
||||
}
|
||||
|
||||
func TestChunkDeterministicOrdering(t *testing.T) {
|
||||
segments := makeSegments([]string{"a", "b", "c", "d", "e", "f"})
|
||||
chunkerCfg := ChunkingConfig{MaxSectionTokens: 15, MinSectionTokens: 1}
|
||||
estimator := &ConstTokenEstimator{Tokens: 5}
|
||||
|
||||
var first []Section
|
||||
for i := 0; i < 5; i++ {
|
||||
chunker := NewChunkerWithEstimator(chunkerCfg, estimator)
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("iteration %d: %v", i, err)
|
||||
}
|
||||
if i == 0 {
|
||||
first = sections
|
||||
continue
|
||||
}
|
||||
if len(sections) != len(first) {
|
||||
t.Fatalf("iteration %d: section count mismatch (%d vs %d)", i, len(sections), len(first))
|
||||
}
|
||||
for j := range sections {
|
||||
if sections[j].Index != first[j].Index ||
|
||||
sections[j].StartSegmentID != first[j].StartSegmentID ||
|
||||
sections[j].EndSegmentID != first[j].EndSegmentID ||
|
||||
sections[j].EstimatedTokens != first[j].EstimatedTokens ||
|
||||
len(sections[j].Segments) != len(first[j].Segments) {
|
||||
t.Fatalf("iteration %d section %d mismatch", i, j)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkNoMutationOfInput(t *testing.T) {
|
||||
segments := makeSegments([]string{"original one", "original two"})
|
||||
transcript := makeTranscript(segments)
|
||||
original := make([]string, len(transcript.Segments))
|
||||
for i := range transcript.Segments {
|
||||
original[i] = transcript.Segments[i].Text
|
||||
}
|
||||
|
||||
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1})
|
||||
if _, err := chunker.ChunkTranscript(transcript); err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
for i := range transcript.Segments {
|
||||
if transcript.Segments[i].Text != original[i] {
|
||||
t.Fatalf("segment %d mutated", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkConfigValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config ChunkingConfig
|
||||
errContains string
|
||||
}{
|
||||
{
|
||||
name: "zero max tokens",
|
||||
config: ChunkingConfig{MaxSectionTokens: 0, MinSectionTokens: 1},
|
||||
errContains: "max_section_tokens must be positive",
|
||||
},
|
||||
{
|
||||
name: "negative max tokens",
|
||||
config: ChunkingConfig{MaxSectionTokens: -1, MinSectionTokens: 1},
|
||||
errContains: "max_section_tokens must be positive",
|
||||
},
|
||||
{
|
||||
name: "negative min tokens",
|
||||
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: -1},
|
||||
errContains: "min_section_tokens must be non-negative",
|
||||
},
|
||||
{
|
||||
name: "min exceeds max",
|
||||
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 11},
|
||||
errContains: "min_section_tokens (11) cannot exceed max_section_tokens (10)",
|
||||
},
|
||||
{
|
||||
name: "zero target sections",
|
||||
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1, TargetSections: intPtr(0)},
|
||||
errContains: "target_sections must be positive",
|
||||
},
|
||||
{
|
||||
name: "negative target sections",
|
||||
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1, TargetSections: intPtr(-1)},
|
||||
errContains: "target_sections must be positive",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
chunker := NewChunker(tt.config)
|
||||
_, err := chunker.ChunkTranscript(makeTranscript(makeSegments([]string{"x"})))
|
||||
if err == nil {
|
||||
t.Fatalf("expected error containing %q", tt.errContains)
|
||||
}
|
||||
if !strings.Contains(err.Error(), tt.errContains) {
|
||||
t.Fatalf("expected error containing %q, got %q", tt.errContains, err.Error())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkBalancedAlgorithmBeatsGreedyMaxFillOnUnevenTranscript(t *testing.T) {
|
||||
segments := makeSegments([]string{"s1", "s2", "s3", "s4", "s5", "s6"})
|
||||
tokenMap := map[string]int{
|
||||
"s1": 50,
|
||||
"s2": 10,
|
||||
"s3": 10,
|
||||
"s4": 10,
|
||||
"s5": 10,
|
||||
"s6": 10,
|
||||
}
|
||||
estimator := &mapTokenEstimator{byText: tokenMap}
|
||||
chunker := NewChunkerWithEstimator(ChunkingConfig{MaxSectionTokens: 80, MinSectionTokens: 1}, estimator)
|
||||
|
||||
balancedSections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
|
||||
tokens := make([]int, 0, len(segments))
|
||||
for _, seg := range segments {
|
||||
tokens = append(tokens, tokenMap[seg.Text])
|
||||
}
|
||||
greedySections := greedyMaxFillSections(segments, tokens, 80)
|
||||
|
||||
balancedImbalance := imbalance(balancedSections)
|
||||
greedyImbalance := imbalance(greedySections)
|
||||
if balancedImbalance >= greedyImbalance {
|
||||
t.Fatalf(
|
||||
"expected balanced chunking to improve over greedy max-fill; balanced=%d greedy=%d",
|
||||
balancedImbalance,
|
||||
greedyImbalance,
|
||||
)
|
||||
}
|
||||
|
||||
assertSegmentCoverageAndOrder(t, segments, balancedSections)
|
||||
assertMaxBoundExceptSingletonOversized(t, balancedSections, 80)
|
||||
}
|
||||
84
internal/core/chunking/summary.go
Normal file
84
internal/core/chunking/summary.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package chunking
|
||||
|
||||
// Summary provides a concise overview of chunking results for reports
|
||||
type Summary struct {
|
||||
ChunkCount int `json:"chunk_count"`
|
||||
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
|
||||
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
|
||||
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
|
||||
TargetSections *int `json:"target_sections,omitempty"`
|
||||
MaxSectionTokens int `json:"max_section_tokens"`
|
||||
MinSectionTokens int `json:"min_section_tokens"`
|
||||
}
|
||||
|
||||
// ChunkSummary represents a single chunk's metadata for diagnostics
|
||||
type ChunkSummary struct {
|
||||
Index int `json:"index"`
|
||||
StartSegmentID int `json:"start_segment_id"`
|
||||
EndSegmentID int `json:"end_segment_id"`
|
||||
EstimatedTokens int `json:"estimated_tokens"`
|
||||
SegmentCount int `json:"segment_count"`
|
||||
}
|
||||
|
||||
// DetailedSummary provides per-chunk details for diagnostics
|
||||
type DetailedSummary struct {
|
||||
Summary `json:",inline"`
|
||||
Chunks []ChunkSummary `json:"chunks"`
|
||||
}
|
||||
|
||||
// ComputeSummary creates a Summary from sections and config
|
||||
func ComputeSummary(sections []Section, config ChunkingConfig) Summary {
|
||||
if len(sections) == 0 {
|
||||
return Summary{
|
||||
ChunkCount: 0,
|
||||
MaxSectionTokens: config.MaxSectionTokens,
|
||||
MinSectionTokens: config.MinSectionTokens,
|
||||
TargetSections: config.TargetSections,
|
||||
}
|
||||
}
|
||||
|
||||
minTokens := sections[0].EstimatedTokens
|
||||
maxTokens := sections[0].EstimatedTokens
|
||||
totalTokens := 0
|
||||
|
||||
for _, sec := range sections {
|
||||
if sec.EstimatedTokens < minTokens {
|
||||
minTokens = sec.EstimatedTokens
|
||||
}
|
||||
if sec.EstimatedTokens > maxTokens {
|
||||
maxTokens = sec.EstimatedTokens
|
||||
}
|
||||
totalTokens += sec.EstimatedTokens
|
||||
}
|
||||
|
||||
return Summary{
|
||||
ChunkCount: len(sections),
|
||||
MinEstimatedTokens: minTokens,
|
||||
MaxEstimatedTokens: maxTokens,
|
||||
TotalEstimatedTokens: totalTokens,
|
||||
TargetSections: config.TargetSections,
|
||||
MaxSectionTokens: config.MaxSectionTokens,
|
||||
MinSectionTokens: config.MinSectionTokens,
|
||||
}
|
||||
}
|
||||
|
||||
// ComputeDetailedSummary creates a DetailedSummary from sections and config
|
||||
func ComputeDetailedSummary(sections []Section, config ChunkingConfig) DetailedSummary {
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
chunks := make([]ChunkSummary, len(sections))
|
||||
for i, sec := range sections {
|
||||
chunks[i] = ChunkSummary{
|
||||
Index: sec.Index,
|
||||
StartSegmentID: sec.StartSegmentID,
|
||||
EndSegmentID: sec.EndSegmentID,
|
||||
EstimatedTokens: sec.EstimatedTokens,
|
||||
SegmentCount: len(sec.Segments),
|
||||
}
|
||||
}
|
||||
|
||||
return DetailedSummary{
|
||||
Summary: summary,
|
||||
Chunks: chunks,
|
||||
}
|
||||
}
|
||||
148
internal/core/chunking/summary_test.go
Normal file
148
internal/core/chunking/summary_test.go
Normal file
@@ -0,0 +1,148 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
func TestComputeSummary(t *testing.T) {
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
}
|
||||
|
||||
sections := []Section{
|
||||
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
|
||||
{Index: 1, EstimatedTokens: 50, StartSegmentID: 3, EndSegmentID: 4},
|
||||
{Index: 2, EstimatedTokens: 20, StartSegmentID: 5, EndSegmentID: 5},
|
||||
}
|
||||
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
if summary.ChunkCount != 3 {
|
||||
t.Errorf("expected chunk_count=3, got %d", summary.ChunkCount)
|
||||
}
|
||||
if summary.MinEstimatedTokens != 20 {
|
||||
t.Errorf("expected min_estimated_tokens=20, got %d", summary.MinEstimatedTokens)
|
||||
}
|
||||
if summary.MaxEstimatedTokens != 50 {
|
||||
t.Errorf("expected max_estimated_tokens=50, got %d", summary.MaxEstimatedTokens)
|
||||
}
|
||||
if summary.TotalEstimatedTokens != 100 {
|
||||
t.Errorf("expected total_estimated_tokens=100, got %d", summary.TotalEstimatedTokens)
|
||||
}
|
||||
if summary.MaxSectionTokens != 100 {
|
||||
t.Errorf("expected max_section_tokens=100, got %d", summary.MaxSectionTokens)
|
||||
}
|
||||
if summary.MinSectionTokens != 10 {
|
||||
t.Errorf("expected min_section_tokens=10, got %d", summary.MinSectionTokens)
|
||||
}
|
||||
if summary.TargetSections != nil {
|
||||
t.Errorf("expected target_sections=nil, got %v", summary.TargetSections)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeSummaryWithTarget(t *testing.T) {
|
||||
target := 5
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
TargetSections: &target,
|
||||
}
|
||||
|
||||
sections := []Section{
|
||||
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
|
||||
}
|
||||
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
if summary.TargetSections == nil || *summary.TargetSections != 5 {
|
||||
t.Errorf("expected target_sections=5, got %v", summary.TargetSections)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeSummaryEmptySections(t *testing.T) {
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
}
|
||||
|
||||
sections := []Section{}
|
||||
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
if summary.ChunkCount != 0 {
|
||||
t.Errorf("expected chunk_count=0, got %d", summary.ChunkCount)
|
||||
}
|
||||
if summary.MinEstimatedTokens != 0 {
|
||||
t.Errorf("expected min_estimated_tokens=0 for empty, got %d", summary.MinEstimatedTokens)
|
||||
}
|
||||
if summary.MaxSectionTokens != 100 {
|
||||
t.Errorf("expected max_section_tokens preserved, got %d", summary.MaxSectionTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeDetailedSummary(t *testing.T) {
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
}
|
||||
|
||||
sections := []Section{
|
||||
{
|
||||
Index: 0,
|
||||
EstimatedTokens: 30,
|
||||
StartSegmentID: 1,
|
||||
EndSegmentID: 2,
|
||||
Segments: make([]schema.Segment, 2), // 2 segments
|
||||
},
|
||||
{
|
||||
Index: 1,
|
||||
EstimatedTokens: 50,
|
||||
StartSegmentID: 3,
|
||||
EndSegmentID: 5,
|
||||
Segments: make([]schema.Segment, 3), // 3 segments
|
||||
},
|
||||
}
|
||||
|
||||
detailed := ComputeDetailedSummary(sections, config)
|
||||
|
||||
if detailed.ChunkCount != 2 {
|
||||
t.Errorf("expected chunk_count=2, got %d", detailed.ChunkCount)
|
||||
}
|
||||
if len(detailed.Chunks) != 2 {
|
||||
t.Fatalf("expected 2 chunk entries, got %d", len(detailed.Chunks))
|
||||
}
|
||||
|
||||
// Check first chunk
|
||||
if detailed.Chunks[0].Index != 0 {
|
||||
t.Errorf("expected chunk[0].index=0, got %d", detailed.Chunks[0].Index)
|
||||
}
|
||||
if detailed.Chunks[0].StartSegmentID != 1 {
|
||||
t.Errorf("expected chunk[0].start_segment_id=1, got %d", detailed.Chunks[0].StartSegmentID)
|
||||
}
|
||||
if detailed.Chunks[0].EndSegmentID != 2 {
|
||||
t.Errorf("expected chunk[0].end_segment_id=2, got %d", detailed.Chunks[0].EndSegmentID)
|
||||
}
|
||||
if detailed.Chunks[0].EstimatedTokens != 30 {
|
||||
t.Errorf("expected chunk[0].estimated_tokens=30, got %d", detailed.Chunks[0].EstimatedTokens)
|
||||
}
|
||||
if detailed.Chunks[0].SegmentCount != 2 {
|
||||
t.Errorf("expected chunk[0].segment_count=2, got %d", detailed.Chunks[0].SegmentCount)
|
||||
}
|
||||
|
||||
// Check second chunk
|
||||
if detailed.Chunks[1].Index != 1 {
|
||||
t.Errorf("expected chunk[1].index=1, got %d", detailed.Chunks[1].Index)
|
||||
}
|
||||
if detailed.Chunks[1].StartSegmentID != 3 {
|
||||
t.Errorf("expected chunk[1].start_segment_id=3, got %d", detailed.Chunks[1].StartSegmentID)
|
||||
}
|
||||
if detailed.Chunks[1].EndSegmentID != 5 {
|
||||
t.Errorf("expected chunk[1].end_segment_id=5, got %d", detailed.Chunks[1].EndSegmentID)
|
||||
}
|
||||
if detailed.Chunks[1].SegmentCount != 3 {
|
||||
t.Errorf("expected chunk[1].segment_count=3, got %d", detailed.Chunks[1].SegmentCount)
|
||||
}
|
||||
}
|
||||
55
internal/core/chunking/tokens.go
Normal file
55
internal/core/chunking/tokens.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// TokenEstimator provides a deterministic token estimation suitable for prompt budgeting.
|
||||
// The estimator is approximate but stable, isolated, and replaceable.
|
||||
type TokenEstimator interface {
|
||||
EstimateTokens(text string) int
|
||||
}
|
||||
|
||||
// SimpleTokenEstimator provides a basic deterministic token estimation.
|
||||
// This uses a simple heuristic based on word count and punctuation.
|
||||
type SimpleTokenEstimator struct{}
|
||||
|
||||
// NewSimpleTokenEstimator creates a new simple token estimator.
|
||||
func NewSimpleTokenEstimator() *SimpleTokenEstimator {
|
||||
return &SimpleTokenEstimator{}
|
||||
}
|
||||
|
||||
// EstimateTokens provides a rough estimate of the number of tokens in the given text.
|
||||
// This implementation uses a simple heuristic: count words and punctuation as tokens.
|
||||
// The estimate is deterministic and stable for the same input text.
|
||||
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
|
||||
if text == "" {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Simple heuristic: split on whitespace and count non-empty segments
|
||||
words := strings.Fields(text)
|
||||
tokenCount := len(words)
|
||||
|
||||
// Add some estimate for punctuation that might be separate tokens
|
||||
punctuationCount := 0
|
||||
for _, r := range text {
|
||||
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
|
||||
punctuationCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Rough estimate: each word is a token, plus half the punctuation as separate tokens
|
||||
return tokenCount + (punctuationCount / 2)
|
||||
}
|
||||
|
||||
// ConstTokenEstimator returns a constant token count for testing purposes.
|
||||
type ConstTokenEstimator struct {
|
||||
Tokens int
|
||||
}
|
||||
|
||||
// EstimateTokens returns the configured constant token count.
|
||||
func (e *ConstTokenEstimator) EstimateTokens(text string) int {
|
||||
return e.Tokens
|
||||
}
|
||||
59
internal/core/chunking/tokens_test.go
Normal file
59
internal/core/chunking/tokens_test.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSimpleTokenEstimator(t *testing.T) {
|
||||
estimator := NewSimpleTokenEstimator()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
text string
|
||||
expected int
|
||||
}{
|
||||
{"empty string", "", 0},
|
||||
{"single word", "hello", 1},
|
||||
{"two words", "hello world", 2},
|
||||
{"with punctuation", "hello, world!", 3}, // 2 words + 2 punctuation/2 = 3
|
||||
{"multiple sentences", "Hello world. This is a test.", 7}, // 7 words + 2 punctuation/2 = 8? Actually "Hello world." has 3 punctuation
|
||||
{"with apostrophes", "don't won't can't", 3},
|
||||
{"with hyphens", "well-known state-of-the-art", 2}, // hyphens don't count
|
||||
{"unicode text", "café naïve", 2},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := estimator.EstimateTokens(tt.text)
|
||||
if got != tt.expected {
|
||||
t.Errorf("EstimateTokens(%q) = %d, want %d", tt.text, got, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConstTokenEstimator(t *testing.T) {
|
||||
estimator := &ConstTokenEstimator{Tokens: 42}
|
||||
|
||||
if got := estimator.EstimateTokens("any text"); got != 42 {
|
||||
t.Errorf("ConstTokenEstimator.EstimateTokens = %d, want 42", got)
|
||||
}
|
||||
|
||||
if got := estimator.EstimateTokens(""); got != 42 {
|
||||
t.Errorf("ConstTokenEstimator.EstimateTokens(empty) = %d, want 42", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokenEstimatorDeterminism(t *testing.T) {
|
||||
estimator := NewSimpleTokenEstimator()
|
||||
text := "The quick brown fox jumps over the lazy dog. Hello, world!"
|
||||
|
||||
// Run multiple times and verify same result
|
||||
first := estimator.EstimateTokens(text)
|
||||
for i := 0; i < 10; i++ {
|
||||
got := estimator.EstimateTokens(text)
|
||||
if got != first {
|
||||
t.Errorf("EstimateTokens not deterministic: iteration %d got %d, first was %d", i, got, first)
|
||||
}
|
||||
}
|
||||
}
|
||||
171
internal/core/config/apply_helpers.go
Normal file
171
internal/core/config/apply_helpers.go
Normal file
@@ -0,0 +1,171 @@
|
||||
package config
|
||||
|
||||
import "strings"
|
||||
|
||||
type llmTargetPatch struct {
|
||||
apiKey *string
|
||||
model *string
|
||||
baseURL *string
|
||||
timeoutSeconds *int
|
||||
maxRetries *int
|
||||
}
|
||||
|
||||
type concurrencyPatch struct {
|
||||
totalLLM *int
|
||||
legacyTotalLLM *int
|
||||
proposalLLM *int
|
||||
validationLLM *int
|
||||
inheritProposal bool
|
||||
allowLegacyAlias bool
|
||||
}
|
||||
|
||||
type chunkingPatch struct {
|
||||
targetSections *int
|
||||
maxSectionTokens *int
|
||||
minSectionTokens *int
|
||||
}
|
||||
|
||||
type thresholdsPatch struct {
|
||||
glossary *float64
|
||||
grammar *float64
|
||||
homophones *float64
|
||||
spokenWord *float64
|
||||
}
|
||||
|
||||
type normalizationPatch struct {
|
||||
maxSegmentGap *float64
|
||||
ellipsisGap *float64
|
||||
maxSegmentDuration *float64
|
||||
maxSegmentTokens *int
|
||||
}
|
||||
|
||||
type contextPatch struct {
|
||||
transcriptDescription *string
|
||||
}
|
||||
|
||||
type diagnosticsPatch struct {
|
||||
workDir *string
|
||||
workDirRetention *string
|
||||
}
|
||||
|
||||
func (c *Config) applyPrimaryLLMTargetPatch(patch llmTargetPatch) {
|
||||
if patch.apiKey != nil {
|
||||
c.PrimaryLLM.APIKey = *patch.apiKey
|
||||
}
|
||||
if patch.model != nil {
|
||||
c.PrimaryLLM.Model = *patch.model
|
||||
}
|
||||
if patch.baseURL != nil {
|
||||
c.PrimaryLLM.BaseURL = *patch.baseURL
|
||||
}
|
||||
if patch.timeoutSeconds != nil {
|
||||
c.PrimaryLLM.TimeoutSeconds = *patch.timeoutSeconds
|
||||
}
|
||||
if patch.maxRetries != nil {
|
||||
c.PrimaryLLM.MaxRetries = *patch.maxRetries
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Config) applyValidationLLMTargetPatch(patch llmTargetPatch) {
|
||||
if patch.apiKey != nil {
|
||||
c.ValidationLLM.APIKey = *patch.apiKey
|
||||
}
|
||||
if patch.model != nil {
|
||||
c.ValidationLLM.Model = *patch.model
|
||||
}
|
||||
if patch.baseURL != nil {
|
||||
c.ValidationLLM.BaseURL = *patch.baseURL
|
||||
}
|
||||
if patch.timeoutSeconds != nil {
|
||||
value := *patch.timeoutSeconds
|
||||
c.ValidationLLM.TimeoutSeconds = &value
|
||||
}
|
||||
if patch.maxRetries != nil {
|
||||
value := *patch.maxRetries
|
||||
c.ValidationLLM.MaxRetries = &value
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Config) applyConcurrencyPatch(patch concurrencyPatch) {
|
||||
totalSet := false
|
||||
if patch.totalLLM != nil {
|
||||
c.TotalLLMConcurrency = *patch.totalLLM
|
||||
totalSet = true
|
||||
}
|
||||
if patch.allowLegacyAlias && patch.legacyTotalLLM != nil && !totalSet {
|
||||
c.TotalLLMConcurrency = *patch.legacyTotalLLM
|
||||
totalSet = true
|
||||
}
|
||||
|
||||
proposalSet := false
|
||||
if patch.proposalLLM != nil {
|
||||
c.ProposalLLMConcurrency = *patch.proposalLLM
|
||||
proposalSet = true
|
||||
}
|
||||
if patch.inheritProposal && totalSet && !proposalSet {
|
||||
c.ProposalLLMConcurrency = c.TotalLLMConcurrency
|
||||
}
|
||||
|
||||
if patch.validationLLM != nil {
|
||||
value := *patch.validationLLM
|
||||
c.ValidationLLMConcurrency = &value
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Config) applyChunkingPatch(patch chunkingPatch) {
|
||||
if patch.targetSections != nil {
|
||||
value := *patch.targetSections
|
||||
c.TargetSections = &value
|
||||
}
|
||||
if patch.maxSectionTokens != nil {
|
||||
c.MaxSectionTokens = *patch.maxSectionTokens
|
||||
}
|
||||
if patch.minSectionTokens != nil {
|
||||
c.MinSectionTokens = *patch.minSectionTokens
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Config) applyThresholdsPatch(patch thresholdsPatch) {
|
||||
if patch.glossary != nil {
|
||||
c.Thresholds.Glossary = *patch.glossary
|
||||
}
|
||||
if patch.grammar != nil {
|
||||
c.Thresholds.Grammar = *patch.grammar
|
||||
}
|
||||
if patch.homophones != nil {
|
||||
c.Thresholds.Homophones = *patch.homophones
|
||||
}
|
||||
if patch.spokenWord != nil {
|
||||
c.Thresholds.SpokenWord = *patch.spokenWord
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Config) applyNormalizationPatch(patch normalizationPatch) {
|
||||
if patch.maxSegmentGap != nil {
|
||||
c.Normalization.MaxSegmentGap = *patch.maxSegmentGap
|
||||
}
|
||||
if patch.ellipsisGap != nil {
|
||||
c.Normalization.EllipsisGap = *patch.ellipsisGap
|
||||
}
|
||||
if patch.maxSegmentDuration != nil {
|
||||
c.Normalization.MaxSegmentDuration = *patch.maxSegmentDuration
|
||||
}
|
||||
if patch.maxSegmentTokens != nil {
|
||||
c.Normalization.MaxSegmentTokens = *patch.maxSegmentTokens
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Config) applyContextPatch(patch contextPatch) {
|
||||
if patch.transcriptDescription != nil {
|
||||
c.TranscriptDescription = strings.TrimSpace(*patch.transcriptDescription)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Config) applyDiagnosticsPatch(patch diagnosticsPatch) {
|
||||
if patch.workDir != nil {
|
||||
c.WorkDir = *patch.workDir
|
||||
}
|
||||
if patch.workDirRetention != nil {
|
||||
c.WorkDirRetention = WorkDirRetention(*patch.workDirRetention)
|
||||
}
|
||||
}
|
||||
199
internal/core/config/config.go
Normal file
199
internal/core/config/config.go
Normal file
@@ -0,0 +1,199 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||
)
|
||||
|
||||
type WorkDirRetention string
|
||||
|
||||
const (
|
||||
WorkDirRetentionAuto WorkDirRetention = "auto"
|
||||
WorkDirRetentionAlways WorkDirRetention = "always"
|
||||
WorkDirRetentionNever WorkDirRetention = "never"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultModulesCSV = modulecatalog.KeyGlossary + "," + modulecatalog.KeyHomophones + "," + modulecatalog.KeyGlossary + "," + modulecatalog.KeySpokenWord + "," + modulecatalog.KeyGrammar
|
||||
DefaultOutputSchema = "bare-segments"
|
||||
DefaultPrimaryModel = "openrouter/google/gemma-4-31b-it"
|
||||
DefaultPrimaryBaseURL = "https://openrouter.ai/api/v1"
|
||||
DefaultPrimaryLLMTimeoutSeconds = 600
|
||||
DefaultMaxRetries = 3
|
||||
DefaultLLMConcurrency = 1
|
||||
DefaultValidationMaxPromptTokens = 2048
|
||||
DefaultMaxSectionTokens = 8192
|
||||
DefaultMinSectionTokens = 2048
|
||||
DefaultConfidenceThreshold = 0.8
|
||||
DefaultNormalizeMaxSegmentGap = 4.0
|
||||
DefaultNormalizeEllipsisGap = 3.5
|
||||
DefaultNormalizeMaxSegmentDuration = 60.0
|
||||
DefaultNormalizeMaxSegmentTokens = 2048
|
||||
DefaultTranscriptDescriptionMaxChars = 500
|
||||
DefaultWorkDir = "/tmp/audita"
|
||||
DefaultWorkDirRetention WorkDirRetention = WorkDirRetentionAuto
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Modules []string
|
||||
OutputSchema string
|
||||
PrimaryLLM LLMConfig
|
||||
ValidationLLM ValidationLLMConfig
|
||||
TotalLLMConcurrency int
|
||||
ProposalLLMConcurrency int
|
||||
ValidationLLMConcurrency *int
|
||||
ValidationMaxPromptTokens int
|
||||
MaxSectionTokens int
|
||||
MinSectionTokens int
|
||||
TargetSections *int
|
||||
Thresholds ConfidenceThresholds
|
||||
Normalization NormalizationConfig
|
||||
TranscriptDescription string
|
||||
WorkDir string
|
||||
WorkDirRetention WorkDirRetention
|
||||
}
|
||||
|
||||
type LLMConfig struct {
|
||||
APIKey string
|
||||
Model string
|
||||
BaseURL string
|
||||
TimeoutSeconds int
|
||||
MaxRetries int
|
||||
// Concurrency is retained as a backward-compatible alias for
|
||||
// TotalLLMConcurrency.
|
||||
Concurrency int
|
||||
}
|
||||
|
||||
type ValidationLLMConfig struct {
|
||||
APIKey string
|
||||
Model string
|
||||
BaseURL string
|
||||
TimeoutSeconds *int
|
||||
MaxRetries *int
|
||||
// Concurrency is retained as a backward-compatible alias for
|
||||
// ValidationLLMConcurrency.
|
||||
Concurrency *int
|
||||
}
|
||||
|
||||
type ConfidenceThresholds struct {
|
||||
Glossary float64
|
||||
Grammar float64
|
||||
Homophones float64
|
||||
SpokenWord float64
|
||||
}
|
||||
|
||||
type NormalizationConfig struct {
|
||||
MaxSegmentGap float64
|
||||
EllipsisGap float64
|
||||
MaxSegmentDuration float64
|
||||
MaxSegmentTokens int
|
||||
}
|
||||
|
||||
func Default() Config {
|
||||
modules, _ := ParseModulesCSV(DefaultModulesCSV)
|
||||
|
||||
return Config{
|
||||
Modules: modules,
|
||||
OutputSchema: DefaultOutputSchema,
|
||||
PrimaryLLM: LLMConfig{
|
||||
Model: DefaultPrimaryModel,
|
||||
BaseURL: DefaultPrimaryBaseURL,
|
||||
TimeoutSeconds: DefaultPrimaryLLMTimeoutSeconds,
|
||||
MaxRetries: DefaultMaxRetries,
|
||||
Concurrency: DefaultLLMConcurrency,
|
||||
},
|
||||
ValidationLLM: ValidationLLMConfig{},
|
||||
TotalLLMConcurrency: DefaultLLMConcurrency,
|
||||
ProposalLLMConcurrency: DefaultLLMConcurrency,
|
||||
ValidationLLMConcurrency: nil,
|
||||
ValidationMaxPromptTokens: DefaultValidationMaxPromptTokens,
|
||||
MaxSectionTokens: DefaultMaxSectionTokens,
|
||||
MinSectionTokens: DefaultMinSectionTokens,
|
||||
TargetSections: nil,
|
||||
Thresholds: ConfidenceThresholds{
|
||||
Glossary: DefaultConfidenceThreshold,
|
||||
Grammar: DefaultConfidenceThreshold,
|
||||
Homophones: DefaultConfidenceThreshold,
|
||||
SpokenWord: DefaultConfidenceThreshold,
|
||||
},
|
||||
Normalization: NormalizationConfig{
|
||||
MaxSegmentGap: DefaultNormalizeMaxSegmentGap,
|
||||
EllipsisGap: DefaultNormalizeEllipsisGap,
|
||||
MaxSegmentDuration: DefaultNormalizeMaxSegmentDuration,
|
||||
MaxSegmentTokens: DefaultNormalizeMaxSegmentTokens,
|
||||
},
|
||||
WorkDir: DefaultWorkDir,
|
||||
WorkDirRetention: DefaultWorkDirRetention,
|
||||
}
|
||||
}
|
||||
|
||||
func ParseModulesCSV(raw string) ([]string, error) {
|
||||
parts := strings.Split(raw, ",")
|
||||
modules := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
trimmed := strings.TrimSpace(part)
|
||||
if trimmed == "" {
|
||||
return nil, fmt.Errorf("modules list contains an empty value")
|
||||
}
|
||||
modules = append(modules, trimmed)
|
||||
}
|
||||
if len(modules) == 0 {
|
||||
return nil, fmt.Errorf("modules list must not be empty")
|
||||
}
|
||||
return modules, nil
|
||||
}
|
||||
|
||||
func (c Config) EffectiveValidationLLMConfig() LLMConfig {
|
||||
effective := c.PrimaryLLM
|
||||
|
||||
if c.ValidationLLM.APIKey != "" {
|
||||
effective.APIKey = c.ValidationLLM.APIKey
|
||||
}
|
||||
if c.ValidationLLM.Model != "" {
|
||||
effective.Model = c.ValidationLLM.Model
|
||||
}
|
||||
if c.ValidationLLM.BaseURL != "" {
|
||||
effective.BaseURL = c.ValidationLLM.BaseURL
|
||||
}
|
||||
if c.ValidationLLM.TimeoutSeconds != nil {
|
||||
effective.TimeoutSeconds = *c.ValidationLLM.TimeoutSeconds
|
||||
}
|
||||
if c.ValidationLLM.MaxRetries != nil {
|
||||
effective.MaxRetries = *c.ValidationLLM.MaxRetries
|
||||
}
|
||||
effective.Concurrency = c.EffectiveValidationLLMConcurrency()
|
||||
|
||||
return effective
|
||||
}
|
||||
|
||||
func (c Config) EffectiveValidationLLMConcurrency() int {
|
||||
if c.ValidationLLMConcurrency != nil {
|
||||
return *c.ValidationLLMConcurrency
|
||||
}
|
||||
return c.TotalLLMConcurrency
|
||||
}
|
||||
|
||||
func (c Config) EffectiveProposalLLMConcurrency() int {
|
||||
if c.ProposalLLMConcurrency > 0 {
|
||||
return c.ProposalLLMConcurrency
|
||||
}
|
||||
return c.TotalLLMConcurrency
|
||||
}
|
||||
|
||||
func (c *Config) syncLegacyConcurrencyAliases() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.PrimaryLLM.Concurrency = c.TotalLLMConcurrency
|
||||
c.ValidationLLM.Concurrency = intPtr(c.ValidationLLMConcurrency)
|
||||
}
|
||||
|
||||
func intPtr(v *int) *int {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
x := *v
|
||||
return &x
|
||||
}
|
||||
705
internal/core/config/config_test.go
Normal file
705
internal/core/config/config_test.go
Normal file
@@ -0,0 +1,705 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
|
||||
)
|
||||
|
||||
func TestDefaultConfigValues(t *testing.T) {
|
||||
cfg := Default()
|
||||
|
||||
if got, want := strings.Join(cfg.Modules, ","), DefaultModulesCSV; got != want {
|
||||
t.Fatalf("modules mismatch: got %q want %q", got, want)
|
||||
}
|
||||
if cfg.OutputSchema != DefaultOutputSchema {
|
||||
t.Fatalf("unexpected default output schema: %q", cfg.OutputSchema)
|
||||
}
|
||||
if cfg.PrimaryLLM.Model != DefaultPrimaryModel {
|
||||
t.Fatalf("unexpected default primary model: %q", cfg.PrimaryLLM.Model)
|
||||
}
|
||||
if cfg.PrimaryLLM.BaseURL != DefaultPrimaryBaseURL {
|
||||
t.Fatalf("unexpected default primary base url: %q", cfg.PrimaryLLM.BaseURL)
|
||||
}
|
||||
if cfg.PrimaryLLM.TimeoutSeconds != DefaultPrimaryLLMTimeoutSeconds {
|
||||
t.Fatalf("unexpected default timeout seconds: %d", cfg.PrimaryLLM.TimeoutSeconds)
|
||||
}
|
||||
if cfg.PrimaryLLM.MaxRetries != DefaultMaxRetries {
|
||||
t.Fatalf("unexpected default max retries: %d", cfg.PrimaryLLM.MaxRetries)
|
||||
}
|
||||
if cfg.TotalLLMConcurrency != DefaultLLMConcurrency {
|
||||
t.Fatalf("unexpected default total llm concurrency: %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != DefaultLLMConcurrency {
|
||||
t.Fatalf("unexpected default proposal llm concurrency: %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
if cfg.ValidationLLMConcurrency != nil {
|
||||
t.Fatalf("expected validation llm concurrency to be unset by default")
|
||||
}
|
||||
if cfg.PrimaryLLM.Concurrency != cfg.TotalLLMConcurrency {
|
||||
t.Fatalf("expected primary llm concurrency alias to mirror total, got primary=%d total=%d", cfg.PrimaryLLM.Concurrency, cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ValidationLLM.TimeoutSeconds != nil {
|
||||
t.Fatalf("expected validation timeout to be unset by default")
|
||||
}
|
||||
if cfg.ValidationLLM.MaxRetries != nil {
|
||||
t.Fatalf("expected validation max retries to be unset by default")
|
||||
}
|
||||
if cfg.ValidationLLM.Concurrency != nil {
|
||||
t.Fatalf("expected legacy validation llm concurrency alias to be unset by default")
|
||||
}
|
||||
if cfg.TargetSections != nil {
|
||||
t.Fatalf("expected target sections to be unset by default")
|
||||
}
|
||||
if cfg.WorkDir != DefaultWorkDir {
|
||||
t.Fatalf("unexpected default work dir: %q", cfg.WorkDir)
|
||||
}
|
||||
if cfg.TranscriptDescription != "" {
|
||||
t.Fatalf("expected default transcript description to be empty, got %q", cfg.TranscriptDescription)
|
||||
}
|
||||
if cfg.WorkDirRetention != DefaultWorkDirRetention {
|
||||
t.Fatalf("unexpected default work dir retention: %q", cfg.WorkDirRetention)
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("default config should validate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromEnvOverridesAndFallback(t *testing.T) {
|
||||
env := map[string]string{
|
||||
"AUDITA_MODEL": "openai/gpt-4.1-mini",
|
||||
"AUDITA_BASE_URL": "https://api.openai.com/v1",
|
||||
"AUDITA_LLM_TIMEOUT_SECONDS": "120",
|
||||
"AUDITA_MAX_RETRIES": "7",
|
||||
"AUDITA_TOTAL_LLM_CONCURRENCY": "6",
|
||||
"AUDITA_PROPOSAL_LLM_CONCURRENCY": "4",
|
||||
"AUDITA_VALIDATION_LLM_CONCURRENCY": "2",
|
||||
"AUDITA_VALIDATION_MAX_PROMPT_TOKENS": "4096",
|
||||
"AUDITA_MAX_SECTION_TOKENS": "9000",
|
||||
"AUDITA_MIN_SECTION_TOKENS": "3000",
|
||||
"AUDITA_TARGET_SECTIONS": "5",
|
||||
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.9",
|
||||
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.7",
|
||||
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.6",
|
||||
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.5",
|
||||
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "2.5",
|
||||
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "2.0",
|
||||
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "30.0",
|
||||
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "1024",
|
||||
"AUDITA_WORK_DIR": "/var/tmp/audita",
|
||||
"AUDITA_WORK_DIR_RETENTION": "always",
|
||||
"OPENROUTER_API_KEY": "fallback-key",
|
||||
}
|
||||
|
||||
cfg, err := loadFromLookup(mapLookup(env))
|
||||
if err != nil {
|
||||
t.Fatalf("loadFromLookup returned error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.PrimaryLLM.APIKey != "fallback-key" {
|
||||
t.Fatalf("expected OPENROUTER_API_KEY fallback, got %q", cfg.PrimaryLLM.APIKey)
|
||||
}
|
||||
if cfg.PrimaryLLM.Model != env["AUDITA_MODEL"] {
|
||||
t.Fatalf("unexpected model: %q", cfg.PrimaryLLM.Model)
|
||||
}
|
||||
if cfg.PrimaryLLM.BaseURL != env["AUDITA_BASE_URL"] {
|
||||
t.Fatalf("unexpected base url: %q", cfg.PrimaryLLM.BaseURL)
|
||||
}
|
||||
if cfg.TargetSections == nil || *cfg.TargetSections != 5 {
|
||||
t.Fatalf("unexpected target sections: %#v", cfg.TargetSections)
|
||||
}
|
||||
if cfg.TotalLLMConcurrency != 6 {
|
||||
t.Fatalf("unexpected total llm concurrency: %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 4 {
|
||||
t.Fatalf("unexpected proposal llm concurrency: %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
if cfg.ValidationLLMConcurrency == nil || *cfg.ValidationLLMConcurrency != 2 {
|
||||
t.Fatalf("unexpected validation llm concurrency: %#v", cfg.ValidationLLMConcurrency)
|
||||
}
|
||||
if cfg.PrimaryLLM.Concurrency != 6 {
|
||||
t.Fatalf("expected primary alias concurrency 6, got %d", cfg.PrimaryLLM.Concurrency)
|
||||
}
|
||||
if cfg.ValidationLLM.Concurrency == nil || *cfg.ValidationLLM.Concurrency != 2 {
|
||||
t.Fatalf("expected validation alias concurrency 2, got %#v", cfg.ValidationLLM.Concurrency)
|
||||
}
|
||||
if cfg.WorkDirRetention != WorkDirRetentionAlways {
|
||||
t.Fatalf("unexpected work dir retention: %q", cfg.WorkDirRetention)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromEnvLegacyLLMConcurrencyAliasForTotalAndProposal(t *testing.T) {
|
||||
env := map[string]string{
|
||||
"AUDITA_LLM_CONCURRENCY": "5",
|
||||
}
|
||||
|
||||
cfg, err := loadFromLookup(mapLookup(env))
|
||||
if err != nil {
|
||||
t.Fatalf("loadFromLookup returned error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.TotalLLMConcurrency != 5 {
|
||||
t.Fatalf("expected total concurrency from legacy alias, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 5 {
|
||||
t.Fatalf("expected proposal concurrency to inherit legacy total, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromEnvCanonicalTotalWinsLegacyAlias(t *testing.T) {
|
||||
env := map[string]string{
|
||||
"AUDITA_TOTAL_LLM_CONCURRENCY": "4",
|
||||
"AUDITA_LLM_CONCURRENCY": "9",
|
||||
}
|
||||
|
||||
cfg, err := loadFromLookup(mapLookup(env))
|
||||
if err != nil {
|
||||
t.Fatalf("loadFromLookup returned error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.TotalLLMConcurrency != 4 {
|
||||
t.Fatalf("expected canonical total to win over legacy alias, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 4 {
|
||||
t.Fatalf("expected proposal to inherit canonical total when unset, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromEnvUsesAuditaLLMAPIKeyOverFallback(t *testing.T) {
|
||||
env := map[string]string{
|
||||
"AUDITA_LLM_API_KEY": "primary-key",
|
||||
"OPENROUTER_API_KEY": "fallback-key",
|
||||
}
|
||||
|
||||
cfg, err := loadFromLookup(mapLookup(env))
|
||||
if err != nil {
|
||||
t.Fatalf("loadFromLookup returned error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.PrimaryLLM.APIKey != "primary-key" {
|
||||
t.Fatalf("expected AUDITA_LLM_API_KEY to win, got %q", cfg.PrimaryLLM.APIKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCLIOverridesPrecedence(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.PrimaryLLM.Model = "env-model"
|
||||
cfg.WorkDir = "/env/work"
|
||||
|
||||
model := "cli-model"
|
||||
workDir := "/cli/work"
|
||||
modules := "grammar"
|
||||
outputSchema := "audita-v1"
|
||||
totalLLMConcurrency := 5
|
||||
proposalLLMConcurrency := 3
|
||||
overrides := CLIOverrides{
|
||||
PrimaryModel: &model,
|
||||
WorkDir: &workDir,
|
||||
ModulesCSV: &modules,
|
||||
OutputSchema: &outputSchema,
|
||||
TotalLLMConcurrency: &totalLLMConcurrency,
|
||||
ProposalLLMConcurrency: &proposalLLMConcurrency,
|
||||
}
|
||||
|
||||
if err := cfg.ApplyCLIOverrides(overrides); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
|
||||
if cfg.PrimaryLLM.Model != "cli-model" {
|
||||
t.Fatalf("expected CLI model override, got %q", cfg.PrimaryLLM.Model)
|
||||
}
|
||||
if cfg.WorkDir != "/cli/work" {
|
||||
t.Fatalf("expected CLI work dir override, got %q", cfg.WorkDir)
|
||||
}
|
||||
if !reflect.DeepEqual(cfg.Modules, []string{"grammar"}) {
|
||||
t.Fatalf("unexpected modules: %#v", cfg.Modules)
|
||||
}
|
||||
if cfg.OutputSchema != "audita-v1" {
|
||||
t.Fatalf("expected CLI output schema override, got %q", cfg.OutputSchema)
|
||||
}
|
||||
if cfg.TotalLLMConcurrency != 5 {
|
||||
t.Fatalf("expected CLI total concurrency override, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 3 {
|
||||
t.Fatalf("expected CLI proposal concurrency override, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCLIOverridesTrimsTranscriptDescription(t *testing.T) {
|
||||
cfg := Default()
|
||||
description := " background context about speakers "
|
||||
if err := cfg.ApplyCLIOverrides(CLIOverrides{TranscriptDescription: &description}); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
if cfg.TranscriptDescription != "background context about speakers" {
|
||||
t.Fatalf("unexpected transcript description trim result: %q", cfg.TranscriptDescription)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigSourcesApplySharedEffectiveFieldsConsistently(t *testing.T) {
|
||||
fileCfg := mustParseFileConfigYAML(t, `
|
||||
version: 1
|
||||
output:
|
||||
schema: " audita-v1 "
|
||||
llm:
|
||||
proposal:
|
||||
base_url: https://proposal.example.test/v1
|
||||
model: provider/proposal
|
||||
timeout: 101
|
||||
max_retries: 5
|
||||
validation:
|
||||
base_url: https://validation.example.test/v1
|
||||
model: provider/validation
|
||||
timeout: 202
|
||||
max_retries: 6
|
||||
chunking:
|
||||
target_sections: 7
|
||||
max_section_tokens: 9000
|
||||
min_section_tokens: 1000
|
||||
thresholds:
|
||||
glossary: 0.91
|
||||
grammar: 0.92
|
||||
homophones: 0.93
|
||||
spoken_word: 0.94
|
||||
normalization:
|
||||
max_segment_gap: 1.2
|
||||
ellipsis_gap: 2.3
|
||||
max_segment_duration: 45.6
|
||||
max_segment_tokens: 321
|
||||
context:
|
||||
description: " shared context "
|
||||
diagnostics:
|
||||
work_dir: /tmp/audita-shared
|
||||
retention: always
|
||||
`)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
apply func(*Config) error
|
||||
}{
|
||||
{
|
||||
name: "file",
|
||||
apply: func(cfg *Config) error {
|
||||
return cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{}))
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "env",
|
||||
apply: func(cfg *Config) error {
|
||||
return cfg.applyEnvOverrides(mapLookup(map[string]string{
|
||||
"AUDITA_MODEL": "provider/proposal",
|
||||
"AUDITA_BASE_URL": "https://proposal.example.test/v1",
|
||||
"AUDITA_LLM_TIMEOUT_SECONDS": "101",
|
||||
"AUDITA_MAX_RETRIES": "5",
|
||||
"AUDITA_VALIDATION_MODEL": "provider/validation",
|
||||
"AUDITA_VALIDATION_BASE_URL": "https://validation.example.test/v1",
|
||||
"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "202",
|
||||
"AUDITA_VALIDATION_MAX_RETRIES": "6",
|
||||
"AUDITA_TARGET_SECTIONS": "7",
|
||||
"AUDITA_MAX_SECTION_TOKENS": "9000",
|
||||
"AUDITA_MIN_SECTION_TOKENS": "1000",
|
||||
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.91",
|
||||
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.92",
|
||||
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.93",
|
||||
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.94",
|
||||
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "1.2",
|
||||
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "2.3",
|
||||
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.6",
|
||||
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "321",
|
||||
"AUDITA_WORK_DIR": "/tmp/audita-shared",
|
||||
"AUDITA_WORK_DIR_RETENTION": "always",
|
||||
}))
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "cli",
|
||||
apply: func(cfg *Config) error {
|
||||
outputSchema := " audita-v1 "
|
||||
proposalModel := "provider/proposal"
|
||||
proposalBaseURL := "https://proposal.example.test/v1"
|
||||
proposalTimeout := 101
|
||||
proposalMaxRetries := 5
|
||||
validationModel := "provider/validation"
|
||||
validationBaseURL := "https://validation.example.test/v1"
|
||||
validationTimeout := 202
|
||||
validationMaxRetries := 6
|
||||
targetSections := 7
|
||||
maxSectionTokens := 9000
|
||||
minSectionTokens := 1000
|
||||
glossaryThreshold := 0.91
|
||||
grammarThreshold := 0.92
|
||||
homophonesThreshold := 0.93
|
||||
spokenWordThreshold := 0.94
|
||||
normalizeMaxSegmentGap := 1.2
|
||||
normalizeEllipsisGap := 2.3
|
||||
normalizeMaxSegmentDuration := 45.6
|
||||
normalizeMaxSegmentTokens := 321
|
||||
description := " shared context "
|
||||
workDir := "/tmp/audita-shared"
|
||||
workDirRetention := "always"
|
||||
return cfg.ApplyCLIOverrides(CLIOverrides{
|
||||
OutputSchema: &outputSchema,
|
||||
PrimaryModel: &proposalModel,
|
||||
PrimaryBaseURL: &proposalBaseURL,
|
||||
PrimaryLLMTimeoutSeconds: &proposalTimeout,
|
||||
MaxRetries: &proposalMaxRetries,
|
||||
ValidationModel: &validationModel,
|
||||
ValidationBaseURL: &validationBaseURL,
|
||||
ValidationLLMTimeoutSeconds: &validationTimeout,
|
||||
ValidationMaxRetries: &validationMaxRetries,
|
||||
TargetSections: &targetSections,
|
||||
MaxSectionTokens: &maxSectionTokens,
|
||||
MinSectionTokens: &minSectionTokens,
|
||||
GlossaryConfidenceThreshold: &glossaryThreshold,
|
||||
GrammarConfidenceThreshold: &grammarThreshold,
|
||||
HomophonesConfidenceThreshold: &homophonesThreshold,
|
||||
SpokenWordConfidenceThreshold: &spokenWordThreshold,
|
||||
NormalizeMaxSegmentGap: &normalizeMaxSegmentGap,
|
||||
NormalizeEllipsisGap: &normalizeEllipsisGap,
|
||||
NormalizeMaxSegmentDuration: &normalizeMaxSegmentDuration,
|
||||
NormalizeMaxSegmentTokens: &normalizeMaxSegmentTokens,
|
||||
TranscriptDescription: &description,
|
||||
WorkDir: &workDir,
|
||||
WorkDirRetention: &workDirRetention,
|
||||
})
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cfg := Default()
|
||||
if err := tc.apply(&cfg); err != nil {
|
||||
t.Fatalf("apply config source: %v", err)
|
||||
}
|
||||
assertSharedEffectiveFields(t, cfg, sharedEffectiveFieldOptions{
|
||||
wantOutputSchemaOverride: tc.name != "env",
|
||||
wantTranscriptDescriptionPatch: tc.name != "env",
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLIAPIKeyOverrideIsDirectValue(t *testing.T) {
|
||||
cfg := Default()
|
||||
apiKey := "NOT_AN_ENV_VAR_NAME"
|
||||
validationAPIKey := "also direct"
|
||||
|
||||
if err := cfg.ApplyCLIOverrides(CLIOverrides{PrimaryLLMAPIKey: &apiKey, ValidationLLMAPIKey: &validationAPIKey}); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
if cfg.PrimaryLLM.APIKey != apiKey {
|
||||
t.Fatalf("expected direct primary api key, got %q", cfg.PrimaryLLM.APIKey)
|
||||
}
|
||||
if cfg.ValidationLLM.APIKey != validationAPIKey {
|
||||
t.Fatalf("expected direct validation api key, got %q", cfg.ValidationLLM.APIKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigTotalConcurrencyDoesNotChangeProposalWhenProposalUnset(t *testing.T) {
|
||||
fileCfg := mustParseFileConfigYAML(t, `
|
||||
version: 1
|
||||
concurrency:
|
||||
total_llm: 4
|
||||
`)
|
||||
cfg := Default()
|
||||
|
||||
if err := cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{})); err != nil {
|
||||
t.Fatalf("applyFileConfigWithLookup failed: %v", err)
|
||||
}
|
||||
|
||||
if cfg.TotalLLMConcurrency != 4 {
|
||||
t.Fatalf("expected file total concurrency 4, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != DefaultLLMConcurrency {
|
||||
t.Fatalf("expected file config to preserve proposal concurrency when unset, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationRejectsOverlyLongTranscriptDescription(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.TranscriptDescription = strings.Repeat("a", DefaultTranscriptDescriptionMaxChars+1)
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatalf("expected transcript description length validation error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "transcript description must be 500 characters or fewer") {
|
||||
t.Fatalf("unexpected validation error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCLIOverridesLegacyLLMConcurrencyAlias(t *testing.T) {
|
||||
cfg := Default()
|
||||
aliasConcurrency := 6
|
||||
|
||||
if err := cfg.ApplyCLIOverrides(CLIOverrides{PrimaryLLMConcurrency: &aliasConcurrency}); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
|
||||
if cfg.TotalLLMConcurrency != 6 {
|
||||
t.Fatalf("expected legacy --llm-concurrency alias to set total, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 6 {
|
||||
t.Fatalf("expected proposal to inherit aliased total when unset, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCLIOverridesCanonicalTotalWinsLegacyAlias(t *testing.T) {
|
||||
cfg := Default()
|
||||
canonicalTotal := 4
|
||||
legacyAlias := 9
|
||||
|
||||
if err := cfg.ApplyCLIOverrides(CLIOverrides{TotalLLMConcurrency: &canonicalTotal, PrimaryLLMConcurrency: &legacyAlias}); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
|
||||
if cfg.TotalLLMConcurrency != 4 {
|
||||
t.Fatalf("expected canonical total concurrency to win, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 4 {
|
||||
t.Fatalf("expected proposal to inherit canonical total when proposal is unset, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationFailures(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.OutputSchema = "unknown-schema"
|
||||
cfg.PrimaryLLM.TimeoutSeconds = -1
|
||||
cfg.TotalLLMConcurrency = 0
|
||||
cfg.ProposalLLMConcurrency = 0
|
||||
validationConcurrency := 5
|
||||
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||
cfg.ValidationMaxPromptTokens = 0
|
||||
cfg.MaxSectionTokens = 100
|
||||
cfg.MinSectionTokens = 200
|
||||
cfg.Thresholds.Grammar = 1.5
|
||||
cfg.WorkDirRetention = WorkDirRetention("sometimes")
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatalf("expected validation error")
|
||||
}
|
||||
|
||||
message := err.Error()
|
||||
for _, expected := range []string{
|
||||
"primary llm timeout seconds",
|
||||
"total llm concurrency",
|
||||
"proposal llm concurrency",
|
||||
"validation llm concurrency must be less than or equal to total llm concurrency",
|
||||
"validation max prompt tokens",
|
||||
"min section tokens",
|
||||
"grammar confidence threshold",
|
||||
"work dir retention",
|
||||
"unsupported output schema",
|
||||
} {
|
||||
if !strings.Contains(message, expected) {
|
||||
t.Fatalf("expected error to contain %q, got %q", expected, message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationRejectsUnsupportedModuleKey(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.Modules = []string{modulecatalog.KeyGlossary, "made_up"}
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatalf("expected validation error for unsupported module key")
|
||||
}
|
||||
if !strings.Contains(err.Error(), `unsupported module key "made_up"`) {
|
||||
t.Fatalf("expected unsupported module key error, got %q", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationAllowsRepeatedSupportedModuleKeys(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.Modules = []string{modulecatalog.KeyGlossary, modulecatalog.KeyGlossary, modulecatalog.KeyGrammar}
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("expected repeated supported module keys to validate, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationAcceptsAllSupportedOutputSchemas(t *testing.T) {
|
||||
for _, schemaKey := range outputschema.SupportedKeys() {
|
||||
cfg := Default()
|
||||
cfg.OutputSchema = schemaKey
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("expected output schema %q to validate, got %v", schemaKey, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveValidationLLMInheritance(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.PrimaryLLM.APIKey = "primary-key"
|
||||
cfg.PrimaryLLM.Model = "primary-model"
|
||||
cfg.PrimaryLLM.BaseURL = "https://primary.example/v1"
|
||||
cfg.PrimaryLLM.TimeoutSeconds = 111
|
||||
cfg.PrimaryLLM.MaxRetries = 2
|
||||
cfg.TotalLLMConcurrency = 7
|
||||
cfg.syncLegacyConcurrencyAliases()
|
||||
|
||||
effective := cfg.EffectiveValidationLLMConfig()
|
||||
if effective.APIKey != "primary-key" || effective.Model != "primary-model" || effective.BaseURL != "https://primary.example/v1" || effective.TimeoutSeconds != 111 || effective.MaxRetries != 2 || effective.Concurrency != 7 {
|
||||
t.Fatalf("unexpected inherited config: %#v", effective)
|
||||
}
|
||||
|
||||
validationTimeout := 222
|
||||
validationRetries := 9
|
||||
cfg.ValidationLLM.APIKey = "validation-key"
|
||||
cfg.ValidationLLM.Model = "validation-model"
|
||||
cfg.ValidationLLM.BaseURL = "https://validation.example/v1"
|
||||
cfg.ValidationLLM.TimeoutSeconds = &validationTimeout
|
||||
cfg.ValidationLLM.MaxRetries = &validationRetries
|
||||
validationConcurrency := 4
|
||||
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||
cfg.syncLegacyConcurrencyAliases()
|
||||
|
||||
effective = cfg.EffectiveValidationLLMConfig()
|
||||
if effective.APIKey != "validation-key" || effective.Model != "validation-model" || effective.BaseURL != "https://validation.example/v1" || effective.TimeoutSeconds != 222 || effective.MaxRetries != 9 || effective.Concurrency != 4 {
|
||||
t.Fatalf("unexpected overridden validation config: %#v", effective)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationLLMConcurrencyCannotExceedTotal(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.TotalLLMConcurrency = 2
|
||||
cfg.ProposalLLMConcurrency = 2
|
||||
validationConcurrency := 3
|
||||
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Fatal("expected validation error when validation llm concurrency exceeds total")
|
||||
}
|
||||
|
||||
validationConcurrency = 2
|
||||
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("expected equal concurrency to validate, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProposalLLMConcurrencyCannotExceedTotal(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.TotalLLMConcurrency = 2
|
||||
cfg.ProposalLLMConcurrency = 3
|
||||
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Fatal("expected validation error when proposal llm concurrency exceeds total")
|
||||
}
|
||||
|
||||
cfg.ProposalLLMConcurrency = 2
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("expected equal concurrency to validate, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLITotalLLMConcurrencyOverrideDrivesEffectiveValidationConcurrencyWhenValidationUnset(t *testing.T) {
|
||||
cfg := Default()
|
||||
totalLLMConcurrency := 6
|
||||
if err := cfg.ApplyCLIOverrides(CLIOverrides{TotalLLMConcurrency: &totalLLMConcurrency}); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
if cfg.ValidationLLMConcurrency != nil {
|
||||
t.Fatalf("expected validation concurrency to remain unset, got %#v", cfg.ValidationLLMConcurrency)
|
||||
}
|
||||
if cfg.EffectiveValidationLLMConcurrency() != 6 {
|
||||
t.Fatalf("expected inherited validation concurrency 6, got %d", cfg.EffectiveValidationLLMConcurrency())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactedConfig(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.PrimaryLLM.APIKey = "secret-primary"
|
||||
cfg.ValidationLLM.APIKey = "secret-validation"
|
||||
|
||||
redacted := cfg.Redacted()
|
||||
|
||||
if redacted.PrimaryLLM.APIKey != redactedSecret {
|
||||
t.Fatalf("expected primary api key to be redacted, got %q", redacted.PrimaryLLM.APIKey)
|
||||
}
|
||||
if redacted.ValidationLLM.APIKey != redactedSecret {
|
||||
t.Fatalf("expected validation api key to be redacted, got %q", redacted.ValidationLLM.APIKey)
|
||||
}
|
||||
if cfg.PrimaryLLM.APIKey != "secret-primary" {
|
||||
t.Fatalf("redaction should not mutate original config")
|
||||
}
|
||||
}
|
||||
|
||||
func mapLookup(values map[string]string) func(string) (string, bool) {
|
||||
return func(key string) (string, bool) {
|
||||
value, ok := values[key]
|
||||
return value, ok
|
||||
}
|
||||
}
|
||||
|
||||
func mustParseFileConfigYAML(t *testing.T, raw string) FileConfig {
|
||||
t.Helper()
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML failed: %v", err)
|
||||
}
|
||||
return fileCfg
|
||||
}
|
||||
|
||||
type sharedEffectiveFieldOptions struct {
|
||||
wantOutputSchemaOverride bool
|
||||
wantTranscriptDescriptionPatch bool
|
||||
}
|
||||
|
||||
func assertSharedEffectiveFields(t *testing.T, cfg Config, opts sharedEffectiveFieldOptions) {
|
||||
t.Helper()
|
||||
wantOutputSchema := DefaultOutputSchema
|
||||
if opts.wantOutputSchemaOverride {
|
||||
wantOutputSchema = "audita-v1"
|
||||
}
|
||||
if cfg.OutputSchema != wantOutputSchema {
|
||||
t.Fatalf("unexpected output schema: %q", cfg.OutputSchema)
|
||||
}
|
||||
if cfg.PrimaryLLM.Model != "provider/proposal" ||
|
||||
cfg.PrimaryLLM.BaseURL != "https://proposal.example.test/v1" ||
|
||||
cfg.PrimaryLLM.TimeoutSeconds != 101 ||
|
||||
cfg.PrimaryLLM.MaxRetries != 5 {
|
||||
t.Fatalf("unexpected primary llm config: %+v", cfg.PrimaryLLM)
|
||||
}
|
||||
if cfg.ValidationLLM.Model != "provider/validation" ||
|
||||
cfg.ValidationLLM.BaseURL != "https://validation.example.test/v1" ||
|
||||
cfg.ValidationLLM.TimeoutSeconds == nil ||
|
||||
*cfg.ValidationLLM.TimeoutSeconds != 202 ||
|
||||
cfg.ValidationLLM.MaxRetries == nil ||
|
||||
*cfg.ValidationLLM.MaxRetries != 6 {
|
||||
t.Fatalf("unexpected validation llm config: %+v", cfg.ValidationLLM)
|
||||
}
|
||||
if cfg.TargetSections == nil || *cfg.TargetSections != 7 ||
|
||||
cfg.MaxSectionTokens != 9000 ||
|
||||
cfg.MinSectionTokens != 1000 {
|
||||
t.Fatalf("unexpected chunking config: target=%v max=%d min=%d", cfg.TargetSections, cfg.MaxSectionTokens, cfg.MinSectionTokens)
|
||||
}
|
||||
if cfg.Thresholds.Glossary != 0.91 ||
|
||||
cfg.Thresholds.Grammar != 0.92 ||
|
||||
cfg.Thresholds.Homophones != 0.93 ||
|
||||
cfg.Thresholds.SpokenWord != 0.94 {
|
||||
t.Fatalf("unexpected thresholds: %+v", cfg.Thresholds)
|
||||
}
|
||||
if cfg.Normalization.MaxSegmentGap != 1.2 ||
|
||||
cfg.Normalization.EllipsisGap != 2.3 ||
|
||||
cfg.Normalization.MaxSegmentDuration != 45.6 ||
|
||||
cfg.Normalization.MaxSegmentTokens != 321 {
|
||||
t.Fatalf("unexpected normalization: %+v", cfg.Normalization)
|
||||
}
|
||||
wantDescription := ""
|
||||
if opts.wantTranscriptDescriptionPatch {
|
||||
wantDescription = "shared context"
|
||||
}
|
||||
if cfg.TranscriptDescription != wantDescription {
|
||||
t.Fatalf("unexpected transcript description: %q", cfg.TranscriptDescription)
|
||||
}
|
||||
if cfg.WorkDir != "/tmp/audita-shared" ||
|
||||
cfg.WorkDirRetention != WorkDirRetentionAlways {
|
||||
t.Fatalf("unexpected diagnostics config: work_dir=%q retention=%q", cfg.WorkDir, cfg.WorkDirRetention)
|
||||
}
|
||||
}
|
||||
119
internal/core/config/effective_config.go
Normal file
119
internal/core/config/effective_config.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type EffectiveConfigErrorKind string
|
||||
|
||||
const (
|
||||
EffectiveConfigErrorResolvePath EffectiveConfigErrorKind = "resolve_path"
|
||||
EffectiveConfigErrorLoadFile EffectiveConfigErrorKind = "load_file"
|
||||
EffectiveConfigErrorApplyFile EffectiveConfigErrorKind = "apply_file"
|
||||
EffectiveConfigErrorApplyEnv EffectiveConfigErrorKind = "apply_env"
|
||||
)
|
||||
|
||||
type EffectiveConfigError struct {
|
||||
Kind EffectiveConfigErrorKind
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *EffectiveConfigError) Error() string {
|
||||
if e == nil || e.Err == nil {
|
||||
return ""
|
||||
}
|
||||
return e.Err.Error()
|
||||
}
|
||||
|
||||
func (e *EffectiveConfigError) Unwrap() error {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return e.Err
|
||||
}
|
||||
|
||||
type EffectiveConfig struct {
|
||||
Config Config
|
||||
ConfigPath string
|
||||
ConfigSource string
|
||||
ConfigVersion *int
|
||||
}
|
||||
|
||||
func ResolveConfigPath(cliConfigPath string, cliConfigPathSet bool) (path string, source string, err error) {
|
||||
return resolveConfigPathWithLookup(cliConfigPath, cliConfigPathSet, os.LookupEnv, os.Stat, DefaultConfigSearchPaths)
|
||||
}
|
||||
|
||||
func LoadEffectiveConfig(cliConfigPath string, cliConfigPathSet bool) (EffectiveConfig, error) {
|
||||
return loadEffectiveConfigWithLookup(cliConfigPath, cliConfigPathSet, os.LookupEnv, os.Stat, DefaultConfigSearchPaths)
|
||||
}
|
||||
|
||||
func loadEffectiveConfigWithLookup(cliConfigPath string, cliConfigPathSet bool, lookup func(string) (string, bool), statPath func(string) (os.FileInfo, error), defaultSearchPaths []string) (EffectiveConfig, error) {
|
||||
configPath, configSource, err := resolveConfigPathWithLookup(cliConfigPath, cliConfigPathSet, lookup, statPath, defaultSearchPaths)
|
||||
if err != nil {
|
||||
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorResolvePath, Err: err}
|
||||
}
|
||||
|
||||
cfg := Default()
|
||||
var configVersion *int
|
||||
if configPath != "" {
|
||||
fileCfg, fileErr := LoadFileConfig(configPath)
|
||||
if fileErr != nil {
|
||||
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorLoadFile, Err: fileErr}
|
||||
}
|
||||
if applyErr := cfg.ApplyFileConfig(fileCfg); applyErr != nil {
|
||||
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorApplyFile, Err: applyErr}
|
||||
}
|
||||
configVersion = &fileCfg.Version
|
||||
}
|
||||
if applyEnvErr := cfg.applyEnvOverrides(lookup); applyEnvErr != nil {
|
||||
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorApplyEnv, Err: applyEnvErr}
|
||||
}
|
||||
|
||||
return EffectiveConfig{
|
||||
Config: cfg,
|
||||
ConfigPath: configPath,
|
||||
ConfigSource: configSource,
|
||||
ConfigVersion: configVersion,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func resolveConfigPathWithLookup(cliConfigPath string, cliConfigPathSet bool, lookup func(string) (string, bool), statPath func(string) (os.FileInfo, error), defaultSearchPaths []string) (path string, source string, err error) {
|
||||
if cliConfigPathSet {
|
||||
path = strings.TrimSpace(cliConfigPath)
|
||||
if path == "" {
|
||||
return "", "", fmt.Errorf("--config requires a non-empty path")
|
||||
}
|
||||
if _, statErr := statPath(path); statErr != nil {
|
||||
if os.IsNotExist(statErr) {
|
||||
return "", "", fmt.Errorf("config file not found: %s", path)
|
||||
}
|
||||
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
|
||||
}
|
||||
return path, "flag", nil
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_CONFIG"); ok {
|
||||
path = strings.TrimSpace(raw)
|
||||
if path == "" {
|
||||
return "", "", fmt.Errorf("AUDITA_CONFIG must not be empty")
|
||||
}
|
||||
if _, statErr := statPath(path); statErr != nil {
|
||||
if os.IsNotExist(statErr) {
|
||||
return "", "", fmt.Errorf("config file not found: %s", path)
|
||||
}
|
||||
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
|
||||
}
|
||||
return path, "env", nil
|
||||
}
|
||||
|
||||
for _, defaultPath := range defaultSearchPaths {
|
||||
if _, statErr := statPath(defaultPath); statErr == nil {
|
||||
return defaultPath, "default", nil
|
||||
} else if !os.IsNotExist(statErr) {
|
||||
return "", "", fmt.Errorf("cannot access config file %s: %w", defaultPath, statErr)
|
||||
}
|
||||
}
|
||||
return "", "", nil
|
||||
}
|
||||
163
internal/core/config/effective_config_test.go
Normal file
163
internal/core/config/effective_config_test.go
Normal file
@@ -0,0 +1,163 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestResolveConfigPathWithLookupMatrix(t *testing.T) {
|
||||
statFor := func(existing map[string]bool) func(string) (os.FileInfo, error) {
|
||||
return func(path string) (os.FileInfo, error) {
|
||||
if existing[path] {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
cliPath string
|
||||
cliPathSet bool
|
||||
lookup func(string) (string, bool)
|
||||
stat func(string) (os.FileInfo, error)
|
||||
defaultSearchPaths []string
|
||||
wantPath string
|
||||
wantSource string
|
||||
wantErrContains string
|
||||
}{
|
||||
{
|
||||
name: "explicit config path",
|
||||
cliPath: "/tmp/explicit.yml",
|
||||
cliPathSet: true,
|
||||
lookup: func(string) (string, bool) { return "", false },
|
||||
stat: statFor(map[string]bool{"/tmp/explicit.yml": true}),
|
||||
defaultSearchPaths: []string{
|
||||
"/usr/local/etc/audita/config.yml",
|
||||
"/etc/audita/config.yml",
|
||||
},
|
||||
wantPath: "/tmp/explicit.yml",
|
||||
wantSource: "flag",
|
||||
},
|
||||
{
|
||||
name: "env config path",
|
||||
cliPathSet: false,
|
||||
lookup: func(key string) (string, bool) {
|
||||
if key == "AUDITA_CONFIG" {
|
||||
return "/tmp/from-env.yml", true
|
||||
}
|
||||
return "", false
|
||||
},
|
||||
stat: statFor(map[string]bool{"/tmp/from-env.yml": true}),
|
||||
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
|
||||
wantPath: "/tmp/from-env.yml",
|
||||
wantSource: "env",
|
||||
},
|
||||
{
|
||||
name: "default search path",
|
||||
cliPathSet: false,
|
||||
lookup: func(string) (string, bool) { return "", false },
|
||||
stat: statFor(map[string]bool{
|
||||
"/usr/local/etc/audita/config.yml": true,
|
||||
"/etc/audita/config.yml": true,
|
||||
}),
|
||||
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
|
||||
wantPath: "/usr/local/etc/audita/config.yml",
|
||||
wantSource: "default",
|
||||
},
|
||||
{
|
||||
name: "explicit missing path",
|
||||
cliPath: "/tmp/missing.yml",
|
||||
cliPathSet: true,
|
||||
lookup: func(string) (string, bool) { return "", false },
|
||||
stat: statFor(map[string]bool{}),
|
||||
defaultSearchPaths: []string{
|
||||
"/usr/local/etc/audita/config.yml",
|
||||
"/etc/audita/config.yml",
|
||||
},
|
||||
wantErrContains: "config file not found",
|
||||
},
|
||||
{
|
||||
name: "missing env path",
|
||||
cliPathSet: false,
|
||||
lookup: func(key string) (string, bool) {
|
||||
if key == "AUDITA_CONFIG" {
|
||||
return "/tmp/missing-from-env.yml", true
|
||||
}
|
||||
return "", false
|
||||
},
|
||||
stat: statFor(map[string]bool{}),
|
||||
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
|
||||
wantErrContains: "config file not found",
|
||||
},
|
||||
{
|
||||
name: "missing default paths",
|
||||
cliPathSet: false,
|
||||
lookup: func(string) (string, bool) { return "", false },
|
||||
stat: statFor(map[string]bool{}),
|
||||
defaultSearchPaths: []string{
|
||||
"/usr/local/etc/audita/config.yml",
|
||||
"/etc/audita/config.yml",
|
||||
},
|
||||
wantPath: "",
|
||||
wantSource: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
gotPath, gotSource, err := resolveConfigPathWithLookup(tc.cliPath, tc.cliPathSet, tc.lookup, tc.stat, tc.defaultSearchPaths)
|
||||
if tc.wantErrContains != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantErrContains) {
|
||||
t.Fatalf("expected error containing %q, got %v", tc.wantErrContains, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if gotPath != tc.wantPath || gotSource != tc.wantSource {
|
||||
t.Fatalf("unexpected result: got path=%q source=%q, want path=%q source=%q", gotPath, gotSource, tc.wantPath, tc.wantSource)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadEffectiveConfigWithLookupAppliesDefaultsFileThenEnv(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
configPath := filepath.Join(tempDir, "config.yml")
|
||||
configYAML := "version: 1\nllm:\n proposal:\n model: file-model\n"
|
||||
if err := os.WriteFile(configPath, []byte(configYAML), 0o644); err != nil {
|
||||
t.Fatalf("write config file: %v", err)
|
||||
}
|
||||
|
||||
lookup := func(key string) (string, bool) {
|
||||
switch key {
|
||||
case "AUDITA_CONFIG":
|
||||
return configPath, true
|
||||
case "AUDITA_MODEL":
|
||||
return "env-model", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
result, err := loadEffectiveConfigWithLookup("", false, lookup, os.Stat, DefaultConfigSearchPaths)
|
||||
if err != nil {
|
||||
t.Fatalf("loadEffectiveConfigWithLookup error: %v", err)
|
||||
}
|
||||
if result.ConfigPath != configPath {
|
||||
t.Fatalf("unexpected config path: %q", result.ConfigPath)
|
||||
}
|
||||
if result.ConfigSource != "env" {
|
||||
t.Fatalf("unexpected config source: %q", result.ConfigSource)
|
||||
}
|
||||
if result.ConfigVersion == nil || *result.ConfigVersion != SupportedFileConfigVersion {
|
||||
t.Fatalf("unexpected config version: %#v", result.ConfigVersion)
|
||||
}
|
||||
if result.Config.PrimaryLLM.Model != "env-model" {
|
||||
t.Fatalf("expected env override to win over file value, got %q", result.Config.PrimaryLLM.Model)
|
||||
}
|
||||
}
|
||||
269
internal/core/config/env.go
Normal file
269
internal/core/config/env.go
Normal file
@@ -0,0 +1,269 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultConfigPath = "/etc/audita/config.yml"
|
||||
DefaultConfigPathUsrLocal = "/usr/local/etc/audita/config.yml"
|
||||
)
|
||||
|
||||
var DefaultConfigSearchPaths = []string{
|
||||
DefaultConfigPathUsrLocal,
|
||||
DefaultConfigPath,
|
||||
}
|
||||
|
||||
func LoadFromEnv() (Config, error) {
|
||||
cfg := Default()
|
||||
if err := cfg.applyEnvOverrides(os.LookupEnv); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func loadFromLookup(lookup func(string) (string, bool)) (Config, error) {
|
||||
cfg := Default()
|
||||
if err := cfg.applyEnvOverrides(lookup); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func (c *Config) ApplyEnvOverrides() error {
|
||||
return c.applyEnvOverrides(os.LookupEnv)
|
||||
}
|
||||
|
||||
func (c *Config) applyEnvOverrides(lookup func(string) (string, bool)) error {
|
||||
if c == nil {
|
||||
return fmt.Errorf("config must not be nil")
|
||||
}
|
||||
|
||||
cfg := c
|
||||
if raw, ok := lookup("AUDITA_MODULES"); ok {
|
||||
modules, err := ParseModulesCSV(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_MODULES: %w", err)
|
||||
}
|
||||
cfg.Modules = modules
|
||||
}
|
||||
|
||||
primaryLLM := llmTargetPatch{}
|
||||
if raw, ok := lookup("AUDITA_LLM_API_KEY"); ok {
|
||||
primaryLLM.apiKey = &raw
|
||||
} else if raw, ok := lookup("OPENROUTER_API_KEY"); ok {
|
||||
primaryLLM.apiKey = &raw
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_MODEL"); ok {
|
||||
primaryLLM.model = &raw
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_BASE_URL"); ok {
|
||||
primaryLLM.baseURL = &raw
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_LLM_TIMEOUT_SECONDS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_LLM_TIMEOUT_SECONDS: %w", err)
|
||||
}
|
||||
primaryLLM.timeoutSeconds = &value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_MAX_RETRIES"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_MAX_RETRIES: %w", err)
|
||||
}
|
||||
primaryLLM.maxRetries = &value
|
||||
}
|
||||
cfg.applyPrimaryLLMTargetPatch(primaryLLM)
|
||||
|
||||
validationLLM := llmTargetPatch{}
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_LLM_API_KEY"); ok {
|
||||
validationLLM.apiKey = &raw
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_MODEL"); ok {
|
||||
validationLLM.model = &raw
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_BASE_URL"); ok {
|
||||
validationLLM.baseURL = &raw
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS: %w", err)
|
||||
}
|
||||
validationLLM.timeoutSeconds = &value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_MAX_RETRIES"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_VALIDATION_MAX_RETRIES: %w", err)
|
||||
}
|
||||
validationLLM.maxRetries = &value
|
||||
}
|
||||
cfg.applyValidationLLMTargetPatch(validationLLM)
|
||||
|
||||
concurrency := concurrencyPatch{
|
||||
inheritProposal: true,
|
||||
allowLegacyAlias: true,
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_TOTAL_LLM_CONCURRENCY"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_TOTAL_LLM_CONCURRENCY: %w", err)
|
||||
}
|
||||
concurrency.totalLLM = &value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_LLM_CONCURRENCY"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_LLM_CONCURRENCY: %w", err)
|
||||
}
|
||||
concurrency.legacyTotalLLM = &value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_PROPOSAL_LLM_CONCURRENCY"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_PROPOSAL_LLM_CONCURRENCY: %w", err)
|
||||
}
|
||||
concurrency.proposalLLM = &value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_LLM_CONCURRENCY"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_VALIDATION_LLM_CONCURRENCY: %w", err)
|
||||
}
|
||||
concurrency.validationLLM = &value
|
||||
}
|
||||
cfg.applyConcurrencyPatch(concurrency)
|
||||
|
||||
if raw, ok := lookup("AUDITA_VALIDATION_MAX_PROMPT_TOKENS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_VALIDATION_MAX_PROMPT_TOKENS: %w", err)
|
||||
}
|
||||
cfg.ValidationMaxPromptTokens = value
|
||||
}
|
||||
|
||||
chunking := chunkingPatch{}
|
||||
if raw, ok := lookup("AUDITA_MAX_SECTION_TOKENS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_MAX_SECTION_TOKENS: %w", err)
|
||||
}
|
||||
chunking.maxSectionTokens = &value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_MIN_SECTION_TOKENS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_MIN_SECTION_TOKENS: %w", err)
|
||||
}
|
||||
chunking.minSectionTokens = &value
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_TARGET_SECTIONS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_TARGET_SECTIONS: %w", err)
|
||||
}
|
||||
chunking.targetSections = &value
|
||||
}
|
||||
cfg.applyChunkingPatch(chunking)
|
||||
|
||||
thresholds := thresholdsPatch{}
|
||||
if raw, ok := lookup("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD: %w", err)
|
||||
}
|
||||
thresholds.glossary = &value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD: %w", err)
|
||||
}
|
||||
thresholds.grammar = &value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD: %w", err)
|
||||
}
|
||||
thresholds.homophones = &value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD: %w", err)
|
||||
}
|
||||
thresholds.spokenWord = &value
|
||||
}
|
||||
cfg.applyThresholdsPatch(thresholds)
|
||||
|
||||
normalization := normalizationPatch{}
|
||||
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_GAP"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_GAP: %w", err)
|
||||
}
|
||||
normalization.maxSegmentGap = &value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_NORMALIZE_ELLIPSIS_GAP"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_NORMALIZE_ELLIPSIS_GAP: %w", err)
|
||||
}
|
||||
normalization.ellipsisGap = &value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION"); ok {
|
||||
value, err := parseFloat(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION: %w", err)
|
||||
}
|
||||
normalization.maxSegmentDuration = &value
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS"); ok {
|
||||
value, err := parseInt(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS: %w", err)
|
||||
}
|
||||
normalization.maxSegmentTokens = &value
|
||||
}
|
||||
cfg.applyNormalizationPatch(normalization)
|
||||
|
||||
diagnostics := diagnosticsPatch{}
|
||||
if raw, ok := lookup("AUDITA_WORK_DIR"); ok {
|
||||
diagnostics.workDir = &raw
|
||||
}
|
||||
if raw, ok := lookup("AUDITA_WORK_DIR_RETENTION"); ok {
|
||||
diagnostics.workDirRetention = &raw
|
||||
}
|
||||
cfg.applyDiagnosticsPatch(diagnostics)
|
||||
|
||||
cfg.syncLegacyConcurrencyAliases()
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseInt(raw string) (int, error) {
|
||||
value, err := strconv.Atoi(raw)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("must be an integer")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func parseFloat(raw string) (float64, error) {
|
||||
value, err := strconv.ParseFloat(raw, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("must be a number")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
322
internal/core/config/file_config.go
Normal file
322
internal/core/config/file_config.go
Normal file
@@ -0,0 +1,322 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const SupportedFileConfigVersion = 1
|
||||
|
||||
var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
type FileConfig struct {
|
||||
Version int `yaml:"version"`
|
||||
Pipeline *FileConfigPipeline `yaml:"pipeline,omitempty"`
|
||||
Output *FileConfigOutput `yaml:"output,omitempty"`
|
||||
LLM *FileConfigLLM `yaml:"llm,omitempty"`
|
||||
Concurrency *FileConfigConcurrency `yaml:"concurrency,omitempty"`
|
||||
Chunking *FileConfigChunking `yaml:"chunking,omitempty"`
|
||||
Normalization *FileConfigNormalization `yaml:"normalization,omitempty"`
|
||||
Thresholds *FileConfigThresholds `yaml:"thresholds,omitempty"`
|
||||
Context *FileConfigContext `yaml:"context,omitempty"`
|
||||
Diagnostics *FileConfigDiagnostics `yaml:"diagnostics,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigPipeline struct {
|
||||
Modules []string `yaml:"modules,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigOutput struct {
|
||||
Schema *string `yaml:"schema,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigLLM struct {
|
||||
Proposal *FileConfigLLMTarget `yaml:"proposal,omitempty"`
|
||||
Validation *FileConfigLLMTarget `yaml:"validation,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigLLMTarget struct {
|
||||
BaseURL *string `yaml:"base_url,omitempty"`
|
||||
Model *string `yaml:"model,omitempty"`
|
||||
APIKeyEnv *string `yaml:"api_key_env,omitempty"`
|
||||
Timeout *fileConfigDurationOrInt `yaml:"timeout,omitempty"`
|
||||
MaxRetries *int `yaml:"max_retries,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigConcurrency struct {
|
||||
TotalLLM *int `yaml:"total_llm,omitempty"`
|
||||
ProposalLLM *int `yaml:"proposal_llm,omitempty"`
|
||||
ValidationLLM *int `yaml:"validation_llm,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigChunking struct {
|
||||
TargetSections *int `yaml:"target_sections,omitempty"`
|
||||
MaxSectionTokens *int `yaml:"max_section_tokens,omitempty"`
|
||||
MinSectionTokens *int `yaml:"min_section_tokens,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigNormalization struct {
|
||||
MaxSegmentGap *fileConfigDurationOrFloat `yaml:"max_segment_gap,omitempty"`
|
||||
EllipsisGap *fileConfigDurationOrFloat `yaml:"ellipsis_gap,omitempty"`
|
||||
MaxSegmentDuration *fileConfigDurationOrFloat `yaml:"max_segment_duration,omitempty"`
|
||||
MaxSegmentTokens *int `yaml:"max_segment_tokens,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigThresholds struct {
|
||||
Glossary *float64 `yaml:"glossary,omitempty"`
|
||||
Homophones *float64 `yaml:"homophones,omitempty"`
|
||||
SpokenWord *float64 `yaml:"spoken_word,omitempty"`
|
||||
Grammar *float64 `yaml:"grammar,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigContext struct {
|
||||
Description *string `yaml:"description,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigDiagnostics struct {
|
||||
WorkDir *string `yaml:"work_dir,omitempty"`
|
||||
Retention *string `yaml:"retention,omitempty"`
|
||||
}
|
||||
|
||||
type fileConfigDurationOrInt struct {
|
||||
seconds int
|
||||
}
|
||||
|
||||
func (v *fileConfigDurationOrInt) UnmarshalYAML(node *yaml.Node) error {
|
||||
switch node.Kind {
|
||||
case yaml.ScalarNode:
|
||||
if node.Tag == "!!int" {
|
||||
var n int
|
||||
if err := node.Decode(&n); err != nil {
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
v.seconds = n
|
||||
return nil
|
||||
}
|
||||
|
||||
var s string
|
||||
if err := node.Decode(&s); err != nil {
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
d, err := time.ParseDuration(strings.TrimSpace(s))
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid duration %q", s)
|
||||
}
|
||||
if d <= 0 {
|
||||
v.seconds = int(d / time.Second)
|
||||
return nil
|
||||
}
|
||||
if d%time.Second != 0 {
|
||||
return fmt.Errorf("duration %q must resolve to whole seconds", s)
|
||||
}
|
||||
v.seconds = int(d / time.Second)
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
}
|
||||
|
||||
func (v fileConfigDurationOrInt) Seconds() int { return v.seconds }
|
||||
|
||||
type fileConfigDurationOrFloat struct {
|
||||
seconds float64
|
||||
}
|
||||
|
||||
func (v *fileConfigDurationOrFloat) UnmarshalYAML(node *yaml.Node) error {
|
||||
switch node.Kind {
|
||||
case yaml.ScalarNode:
|
||||
if node.Tag == "!!int" || node.Tag == "!!float" {
|
||||
var f float64
|
||||
if err := node.Decode(&f); err != nil {
|
||||
return fmt.Errorf("must be a numeric seconds value or duration string")
|
||||
}
|
||||
v.seconds = f
|
||||
return nil
|
||||
}
|
||||
var s string
|
||||
if err := node.Decode(&s); err != nil {
|
||||
return fmt.Errorf("must be a numeric seconds value or duration string")
|
||||
}
|
||||
d, err := time.ParseDuration(strings.TrimSpace(s))
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid duration %q", s)
|
||||
}
|
||||
v.seconds = d.Seconds()
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("must be a numeric seconds value or duration string")
|
||||
}
|
||||
}
|
||||
|
||||
func (v fileConfigDurationOrFloat) Seconds() float64 { return v.seconds }
|
||||
|
||||
func LoadFileConfig(path string) (FileConfig, error) {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return FileConfig{}, fmt.Errorf("read config file %q: %w", path, err)
|
||||
}
|
||||
cfg, err := ParseFileConfigYAML(b)
|
||||
if err != nil {
|
||||
return FileConfig{}, fmt.Errorf("parse config file %q: %w", path, err)
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func ParseFileConfigYAML(data []byte) (FileConfig, error) {
|
||||
var fileCfg FileConfig
|
||||
dec := yaml.NewDecoder(strings.NewReader(string(data)))
|
||||
dec.KnownFields(true)
|
||||
if err := dec.Decode(&fileCfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("decode yaml: %w", err)
|
||||
}
|
||||
if fileCfg.Version == 0 {
|
||||
return FileConfig{}, fmt.Errorf("config version is required")
|
||||
}
|
||||
if fileCfg.Version != SupportedFileConfigVersion {
|
||||
return FileConfig{}, fmt.Errorf("unsupported config version %d", fileCfg.Version)
|
||||
}
|
||||
return fileCfg, nil
|
||||
}
|
||||
|
||||
func (c *Config) ApplyFileConfig(fileCfg FileConfig) error {
|
||||
return c.applyFileConfigWithLookup(fileCfg, os.LookupEnv)
|
||||
}
|
||||
|
||||
func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
|
||||
if c == nil {
|
||||
return fmt.Errorf("config must not be nil")
|
||||
}
|
||||
if fileCfg.Version != SupportedFileConfigVersion {
|
||||
return fmt.Errorf("unsupported config version %d", fileCfg.Version)
|
||||
}
|
||||
|
||||
if fileCfg.Pipeline != nil && len(fileCfg.Pipeline.Modules) > 0 {
|
||||
c.Modules = append([]string(nil), fileCfg.Pipeline.Modules...)
|
||||
}
|
||||
if fileCfg.Output != nil && fileCfg.Output.Schema != nil {
|
||||
c.OutputSchema = strings.TrimSpace(*fileCfg.Output.Schema)
|
||||
}
|
||||
|
||||
if fileCfg.LLM != nil {
|
||||
if fileCfg.LLM.Proposal != nil {
|
||||
patch := llmTargetPatch{
|
||||
model: fileCfg.LLM.Proposal.Model,
|
||||
baseURL: fileCfg.LLM.Proposal.BaseURL,
|
||||
maxRetries: fileCfg.LLM.Proposal.MaxRetries,
|
||||
}
|
||||
if fileCfg.LLM.Proposal.Timeout != nil {
|
||||
timeoutSeconds := fileCfg.LLM.Proposal.Timeout.Seconds()
|
||||
patch.timeoutSeconds = &timeoutSeconds
|
||||
}
|
||||
if fileCfg.LLM.Proposal.APIKeyEnv != nil {
|
||||
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Proposal.APIKeyEnv, lookup)
|
||||
if err != nil {
|
||||
return fmt.Errorf("llm.proposal.api_key_env: %w", err)
|
||||
}
|
||||
patch.apiKey = &apiKey
|
||||
}
|
||||
c.applyPrimaryLLMTargetPatch(patch)
|
||||
}
|
||||
if fileCfg.LLM.Validation != nil {
|
||||
patch := llmTargetPatch{
|
||||
model: fileCfg.LLM.Validation.Model,
|
||||
baseURL: fileCfg.LLM.Validation.BaseURL,
|
||||
maxRetries: fileCfg.LLM.Validation.MaxRetries,
|
||||
}
|
||||
if fileCfg.LLM.Validation.Timeout != nil {
|
||||
timeoutSeconds := fileCfg.LLM.Validation.Timeout.Seconds()
|
||||
patch.timeoutSeconds = &timeoutSeconds
|
||||
}
|
||||
if fileCfg.LLM.Validation.APIKeyEnv != nil {
|
||||
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Validation.APIKeyEnv, lookup)
|
||||
if err != nil {
|
||||
return fmt.Errorf("llm.validation.api_key_env: %w", err)
|
||||
}
|
||||
patch.apiKey = &apiKey
|
||||
}
|
||||
c.applyValidationLLMTargetPatch(patch)
|
||||
}
|
||||
}
|
||||
|
||||
if fileCfg.Concurrency != nil {
|
||||
c.applyConcurrencyPatch(concurrencyPatch{
|
||||
totalLLM: fileCfg.Concurrency.TotalLLM,
|
||||
proposalLLM: fileCfg.Concurrency.ProposalLLM,
|
||||
validationLLM: fileCfg.Concurrency.ValidationLLM,
|
||||
})
|
||||
}
|
||||
|
||||
if fileCfg.Chunking != nil {
|
||||
c.applyChunkingPatch(chunkingPatch{
|
||||
targetSections: fileCfg.Chunking.TargetSections,
|
||||
maxSectionTokens: fileCfg.Chunking.MaxSectionTokens,
|
||||
minSectionTokens: fileCfg.Chunking.MinSectionTokens,
|
||||
})
|
||||
}
|
||||
|
||||
if fileCfg.Normalization != nil {
|
||||
patch := normalizationPatch{
|
||||
maxSegmentTokens: fileCfg.Normalization.MaxSegmentTokens,
|
||||
}
|
||||
if fileCfg.Normalization.MaxSegmentGap != nil {
|
||||
maxSegmentGap := fileCfg.Normalization.MaxSegmentGap.Seconds()
|
||||
patch.maxSegmentGap = &maxSegmentGap
|
||||
}
|
||||
if fileCfg.Normalization.EllipsisGap != nil {
|
||||
ellipsisGap := fileCfg.Normalization.EllipsisGap.Seconds()
|
||||
patch.ellipsisGap = &ellipsisGap
|
||||
}
|
||||
if fileCfg.Normalization.MaxSegmentDuration != nil {
|
||||
maxSegmentDuration := fileCfg.Normalization.MaxSegmentDuration.Seconds()
|
||||
patch.maxSegmentDuration = &maxSegmentDuration
|
||||
}
|
||||
c.applyNormalizationPatch(patch)
|
||||
}
|
||||
|
||||
if fileCfg.Thresholds != nil {
|
||||
c.applyThresholdsPatch(thresholdsPatch{
|
||||
glossary: fileCfg.Thresholds.Glossary,
|
||||
grammar: fileCfg.Thresholds.Grammar,
|
||||
homophones: fileCfg.Thresholds.Homophones,
|
||||
spokenWord: fileCfg.Thresholds.SpokenWord,
|
||||
})
|
||||
}
|
||||
|
||||
if fileCfg.Context != nil && fileCfg.Context.Description != nil {
|
||||
c.applyContextPatch(contextPatch{transcriptDescription: fileCfg.Context.Description})
|
||||
}
|
||||
|
||||
if fileCfg.Diagnostics != nil {
|
||||
c.applyDiagnosticsPatch(diagnosticsPatch{
|
||||
workDir: fileCfg.Diagnostics.WorkDir,
|
||||
workDirRetention: fileCfg.Diagnostics.Retention,
|
||||
})
|
||||
}
|
||||
|
||||
c.syncLegacyConcurrencyAliases()
|
||||
if err := c.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
|
||||
name := strings.TrimSpace(envName)
|
||||
if name == "" {
|
||||
return "", fmt.Errorf("must not be empty")
|
||||
}
|
||||
if !envVarNamePattern.MatchString(name) {
|
||||
return "", fmt.Errorf("must be an environment variable name")
|
||||
}
|
||||
if strings.Contains(name, string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("must be an environment variable name")
|
||||
}
|
||||
v, _ := lookup(name)
|
||||
return v, nil
|
||||
}
|
||||
290
internal/core/config/file_config_test.go
Normal file
290
internal/core/config/file_config_test.go
Normal file
@@ -0,0 +1,290 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseFileConfigYAMLValid(t *testing.T) {
|
||||
raw := `
|
||||
version: 1
|
||||
pipeline:
|
||||
modules: [glossary, homophones, grammar]
|
||||
output:
|
||||
schema: audita-v1
|
||||
llm:
|
||||
proposal:
|
||||
base_url: https://example.test/v1
|
||||
model: provider/model-a
|
||||
api_key_env: AUDITA_PROPOSAL_KEY
|
||||
timeout: 2m
|
||||
max_retries: 4
|
||||
validation:
|
||||
base_url: https://example.test/validation
|
||||
model: provider/model-b
|
||||
api_key_env: AUDITA_VALIDATION_KEY
|
||||
timeout: 45
|
||||
max_retries: 3
|
||||
concurrency:
|
||||
total_llm: 8
|
||||
proposal_llm: 4
|
||||
validation_llm: 2
|
||||
chunking:
|
||||
target_sections: 6
|
||||
max_section_tokens: 9000
|
||||
min_section_tokens: 3000
|
||||
normalization:
|
||||
max_segment_gap: 1.5s
|
||||
ellipsis_gap: 2
|
||||
max_segment_duration: 45s
|
||||
max_segment_tokens: 1500
|
||||
thresholds:
|
||||
glossary: 0.9
|
||||
homophones: 0.7
|
||||
spoken_word: 0.8
|
||||
grammar: 0.75
|
||||
context:
|
||||
description: " crowd scene with many proper nouns "
|
||||
diagnostics:
|
||||
work_dir: /tmp/audita-config
|
||||
retention: always
|
||||
`
|
||||
cfg, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML error: %v", err)
|
||||
}
|
||||
if cfg.Version != 1 {
|
||||
t.Fatalf("expected version 1, got %d", cfg.Version)
|
||||
}
|
||||
if cfg.Pipeline == nil || len(cfg.Pipeline.Modules) != 3 {
|
||||
t.Fatalf("unexpected pipeline modules: %#v", cfg.Pipeline)
|
||||
}
|
||||
if cfg.Output == nil || cfg.Output.Schema == nil || *cfg.Output.Schema != "audita-v1" {
|
||||
t.Fatalf("expected output schema audita-v1, got %#v", cfg.Output)
|
||||
}
|
||||
if cfg.LLM == nil || cfg.LLM.Proposal == nil || cfg.LLM.Validation == nil {
|
||||
t.Fatalf("expected llm proposal+validation blocks")
|
||||
}
|
||||
if cfg.LLM.Proposal.Timeout == nil || cfg.LLM.Proposal.Timeout.Seconds() != 120 {
|
||||
t.Fatalf("expected proposal timeout 120s, got %#v", cfg.LLM.Proposal.Timeout)
|
||||
}
|
||||
if cfg.LLM.Validation.Timeout == nil || cfg.LLM.Validation.Timeout.Seconds() != 45 {
|
||||
t.Fatalf("expected validation timeout 45s, got %#v", cfg.LLM.Validation.Timeout)
|
||||
}
|
||||
if cfg.Normalization == nil || cfg.Normalization.MaxSegmentGap == nil || cfg.Normalization.MaxSegmentGap.Seconds() != 1.5 {
|
||||
t.Fatalf("expected parsed duration for normalization max_segment_gap")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigYAMLRejectsUnknownField(t *testing.T) {
|
||||
raw := `
|
||||
version: 1
|
||||
pipeline:
|
||||
modules: [grammar]
|
||||
output:
|
||||
unknown: v1
|
||||
`
|
||||
_, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err == nil {
|
||||
t.Fatalf("expected unknown field error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "field unknown not found") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigYAMLRejectsMissingVersion(t *testing.T) {
|
||||
raw := `pipeline: {modules: [grammar]}`
|
||||
_, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err == nil {
|
||||
t.Fatalf("expected missing version error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "config version is required") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigYAMLRejectsUnsupportedVersion(t *testing.T) {
|
||||
raw := `version: 2`
|
||||
_, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err == nil {
|
||||
t.Fatalf("expected unsupported version error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "unsupported config version 2") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigParsesAndMergesFields(t *testing.T) {
|
||||
raw := `
|
||||
version: 1
|
||||
pipeline:
|
||||
modules: [spoken_word, grammar]
|
||||
output:
|
||||
schema: audita-v1
|
||||
llm:
|
||||
proposal:
|
||||
model: provider/new-proposal
|
||||
api_key_env: PROPOSAL_KEY_NAME
|
||||
timeout: 90s
|
||||
max_retries: 5
|
||||
validation:
|
||||
model: provider/new-validation
|
||||
api_key_env: VALIDATION_KEY_NAME
|
||||
timeout: 150
|
||||
max_retries: 6
|
||||
concurrency:
|
||||
total_llm: 7
|
||||
proposal_llm: 3
|
||||
validation_llm: 2
|
||||
chunking:
|
||||
target_sections: 9
|
||||
thresholds:
|
||||
glossary: 0.91
|
||||
homophones: 0.61
|
||||
spoken_word: 0.71
|
||||
grammar: 0.81
|
||||
diagnostics:
|
||||
retention: never
|
||||
`
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML error: %v", err)
|
||||
}
|
||||
|
||||
cfg := Default()
|
||||
lookup := func(name string) (string, bool) {
|
||||
switch name {
|
||||
case "PROPOSAL_KEY_NAME":
|
||||
return "proposal-secret", true
|
||||
case "VALIDATION_KEY_NAME":
|
||||
return "validation-secret", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
if err := cfg.applyFileConfigWithLookup(fileCfg, lookup); err != nil {
|
||||
t.Fatalf("applyFileConfigWithLookup error: %v", err)
|
||||
}
|
||||
if strings.Join(cfg.Modules, ",") != "spoken_word,grammar" {
|
||||
t.Fatalf("unexpected modules: %#v", cfg.Modules)
|
||||
}
|
||||
if cfg.OutputSchema != "audita-v1" {
|
||||
t.Fatalf("unexpected output schema: %q", cfg.OutputSchema)
|
||||
}
|
||||
if cfg.PrimaryLLM.Model != "provider/new-proposal" {
|
||||
t.Fatalf("unexpected proposal model: %q", cfg.PrimaryLLM.Model)
|
||||
}
|
||||
if cfg.PrimaryLLM.APIKey != "proposal-secret" {
|
||||
t.Fatalf("expected proposal key from api_key_env lookup, got %q", cfg.PrimaryLLM.APIKey)
|
||||
}
|
||||
if cfg.PrimaryLLM.TimeoutSeconds != 90 {
|
||||
t.Fatalf("unexpected proposal timeout: %d", cfg.PrimaryLLM.TimeoutSeconds)
|
||||
}
|
||||
if cfg.ValidationLLM.Model != "provider/new-validation" {
|
||||
t.Fatalf("unexpected validation model: %q", cfg.ValidationLLM.Model)
|
||||
}
|
||||
if cfg.ValidationLLM.APIKey != "validation-secret" {
|
||||
t.Fatalf("expected validation key from api_key_env lookup, got %q", cfg.ValidationLLM.APIKey)
|
||||
}
|
||||
if cfg.ValidationLLM.TimeoutSeconds == nil || *cfg.ValidationLLM.TimeoutSeconds != 150 {
|
||||
t.Fatalf("unexpected validation timeout: %#v", cfg.ValidationLLM.TimeoutSeconds)
|
||||
}
|
||||
if cfg.TotalLLMConcurrency != 7 || cfg.ProposalLLMConcurrency != 3 {
|
||||
t.Fatalf("unexpected llm concurrency values: total=%d proposal=%d", cfg.TotalLLMConcurrency, cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
if cfg.ValidationLLMConcurrency == nil || *cfg.ValidationLLMConcurrency != 2 {
|
||||
t.Fatalf("unexpected validation llm concurrency: %#v", cfg.ValidationLLMConcurrency)
|
||||
}
|
||||
if cfg.TargetSections == nil || *cfg.TargetSections != 9 {
|
||||
t.Fatalf("unexpected target sections: %#v", cfg.TargetSections)
|
||||
}
|
||||
if cfg.WorkDirRetention != WorkDirRetentionNever {
|
||||
t.Fatalf("unexpected retention: %q", cfg.WorkDirRetention)
|
||||
}
|
||||
if cfg.PrimaryLLM.Concurrency != 7 {
|
||||
t.Fatalf("expected legacy alias to sync, got %d", cfg.PrimaryLLM.Concurrency)
|
||||
}
|
||||
if cfg.ValidationLLM.Concurrency == nil || *cfg.ValidationLLM.Concurrency != 2 {
|
||||
t.Fatalf("expected validation alias to sync, got %#v", cfg.ValidationLLM.Concurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigContextDescriptionTrim(t *testing.T) {
|
||||
raw := `
|
||||
version: 1
|
||||
context:
|
||||
description: " scene context "
|
||||
`
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML error: %v", err)
|
||||
}
|
||||
cfg := Default()
|
||||
if err := cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{})); err != nil {
|
||||
t.Fatalf("applyFileConfigWithLookup error: %v", err)
|
||||
}
|
||||
if cfg.TranscriptDescription != "scene context" {
|
||||
t.Fatalf("unexpected transcript description: %q", cfg.TranscriptDescription)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigRejectsInvalidAPIKeyEnvName(t *testing.T) {
|
||||
raw := `
|
||||
version: 1
|
||||
llm:
|
||||
proposal:
|
||||
api_key_env: "not a var name"
|
||||
`
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML error: %v", err)
|
||||
}
|
||||
cfg := Default()
|
||||
err = cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{}))
|
||||
if err == nil {
|
||||
t.Fatalf("expected api_key_env validation error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "environment variable name") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigDurationParsingErrors(t *testing.T) {
|
||||
raw := `
|
||||
version: 1
|
||||
llm:
|
||||
proposal:
|
||||
timeout: "1.5s"
|
||||
`
|
||||
_, err := ParseFileConfigYAML([]byte(raw))
|
||||
if err == nil {
|
||||
t.Fatalf("expected duration parse error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "whole seconds") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFileConfigReadsFromPath(t *testing.T) {
|
||||
p := writeTempFileConfig(t, "version: 1\n")
|
||||
cfg, err := LoadFileConfig(p)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadFileConfig error: %v", err)
|
||||
}
|
||||
if cfg.Version != 1 {
|
||||
t.Fatalf("expected version 1, got %d", cfg.Version)
|
||||
}
|
||||
}
|
||||
|
||||
func writeTempFileConfig(t *testing.T, contents string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
path := dir + "/config.yaml"
|
||||
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
|
||||
t.Fatalf("write config file: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
106
internal/core/config/flags.go
Normal file
106
internal/core/config/flags.go
Normal file
@@ -0,0 +1,106 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type CLIOverrides struct {
|
||||
ModulesCSV *string
|
||||
OutputSchema *string
|
||||
PrimaryLLMAPIKey *string
|
||||
ValidationLLMAPIKey *string
|
||||
PrimaryModel *string
|
||||
ValidationModel *string
|
||||
PrimaryBaseURL *string
|
||||
ValidationBaseURL *string
|
||||
PrimaryLLMTimeoutSeconds *int
|
||||
TotalLLMConcurrency *int
|
||||
ProposalLLMConcurrency *int
|
||||
PrimaryLLMConcurrency *int
|
||||
ValidationLLMTimeoutSeconds *int
|
||||
MaxRetries *int
|
||||
ValidationMaxRetries *int
|
||||
ValidationLLMConcurrency *int
|
||||
ValidationMaxPromptTokens *int
|
||||
MaxSectionTokens *int
|
||||
MinSectionTokens *int
|
||||
TargetSections *int
|
||||
GlossaryConfidenceThreshold *float64
|
||||
GrammarConfidenceThreshold *float64
|
||||
HomophonesConfidenceThreshold *float64
|
||||
SpokenWordConfidenceThreshold *float64
|
||||
NormalizeMaxSegmentGap *float64
|
||||
NormalizeEllipsisGap *float64
|
||||
NormalizeMaxSegmentDuration *float64
|
||||
NormalizeMaxSegmentTokens *int
|
||||
TranscriptDescription *string
|
||||
WorkDir *string
|
||||
WorkDirRetention *string
|
||||
}
|
||||
|
||||
func (c *Config) ApplyCLIOverrides(overrides CLIOverrides) error {
|
||||
if overrides.ModulesCSV != nil {
|
||||
modules, err := ParseModulesCSV(*overrides.ModulesCSV)
|
||||
if err != nil {
|
||||
return fmt.Errorf("--modules: %w", err)
|
||||
}
|
||||
c.Modules = modules
|
||||
}
|
||||
if overrides.OutputSchema != nil {
|
||||
c.OutputSchema = strings.TrimSpace(*overrides.OutputSchema)
|
||||
}
|
||||
|
||||
c.applyPrimaryLLMTargetPatch(llmTargetPatch{
|
||||
apiKey: overrides.PrimaryLLMAPIKey,
|
||||
model: overrides.PrimaryModel,
|
||||
baseURL: overrides.PrimaryBaseURL,
|
||||
timeoutSeconds: overrides.PrimaryLLMTimeoutSeconds,
|
||||
maxRetries: overrides.MaxRetries,
|
||||
})
|
||||
c.applyValidationLLMTargetPatch(llmTargetPatch{
|
||||
apiKey: overrides.ValidationLLMAPIKey,
|
||||
model: overrides.ValidationModel,
|
||||
baseURL: overrides.ValidationBaseURL,
|
||||
timeoutSeconds: overrides.ValidationLLMTimeoutSeconds,
|
||||
maxRetries: overrides.ValidationMaxRetries,
|
||||
})
|
||||
c.applyConcurrencyPatch(concurrencyPatch{
|
||||
totalLLM: overrides.TotalLLMConcurrency,
|
||||
legacyTotalLLM: overrides.PrimaryLLMConcurrency,
|
||||
proposalLLM: overrides.ProposalLLMConcurrency,
|
||||
validationLLM: overrides.ValidationLLMConcurrency,
|
||||
inheritProposal: true,
|
||||
allowLegacyAlias: true,
|
||||
})
|
||||
|
||||
if overrides.ValidationMaxPromptTokens != nil {
|
||||
c.ValidationMaxPromptTokens = *overrides.ValidationMaxPromptTokens
|
||||
}
|
||||
c.applyChunkingPatch(chunkingPatch{
|
||||
targetSections: overrides.TargetSections,
|
||||
maxSectionTokens: overrides.MaxSectionTokens,
|
||||
minSectionTokens: overrides.MinSectionTokens,
|
||||
})
|
||||
c.applyThresholdsPatch(thresholdsPatch{
|
||||
glossary: overrides.GlossaryConfidenceThreshold,
|
||||
grammar: overrides.GrammarConfidenceThreshold,
|
||||
homophones: overrides.HomophonesConfidenceThreshold,
|
||||
spokenWord: overrides.SpokenWordConfidenceThreshold,
|
||||
})
|
||||
c.applyNormalizationPatch(normalizationPatch{
|
||||
maxSegmentGap: overrides.NormalizeMaxSegmentGap,
|
||||
ellipsisGap: overrides.NormalizeEllipsisGap,
|
||||
maxSegmentDuration: overrides.NormalizeMaxSegmentDuration,
|
||||
maxSegmentTokens: overrides.NormalizeMaxSegmentTokens,
|
||||
})
|
||||
c.applyContextPatch(contextPatch{transcriptDescription: overrides.TranscriptDescription})
|
||||
c.applyDiagnosticsPatch(diagnosticsPatch{
|
||||
workDir: overrides.WorkDir,
|
||||
workDirRetention: overrides.WorkDirRetention,
|
||||
})
|
||||
|
||||
c.syncLegacyConcurrencyAliases()
|
||||
|
||||
return c.Validate()
|
||||
}
|
||||
17
internal/core/config/redaction.go
Normal file
17
internal/core/config/redaction.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package config
|
||||
|
||||
const redactedSecret = "[REDACTED]"
|
||||
|
||||
func (c Config) Redacted() Config {
|
||||
redacted := c
|
||||
redacted.PrimaryLLM.APIKey = redactSecret(redacted.PrimaryLLM.APIKey)
|
||||
redacted.ValidationLLM.APIKey = redactSecret(redacted.ValidationLLM.APIKey)
|
||||
return redacted
|
||||
}
|
||||
|
||||
func redactSecret(value string) string {
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
return redactedSecret
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user