Compare commits
107 Commits
python-fin
...
v1.1.0
| Author | SHA1 | Date | |
|---|---|---|---|
| c5627460d0 | |||
| 3c68ec9d05 | |||
| e8b8fbd509 | |||
| 588ca36dfc | |||
| 018d08fd4c | |||
| 3a0a0b3940 | |||
| 99ab2f181b | |||
| 4e4801dc98 | |||
| 900ad74958 | |||
| 28d5201a69 | |||
| e5944f9875 | |||
| 6344fc91ba | |||
| 7f3a91cc9e | |||
| 72fb021453 | |||
| 40e8b54d3b | |||
| 76651333b1 | |||
| 0b01c3a83d | |||
| 0630d36734 | |||
| 52c2697040 | |||
| f790c1441c | |||
| 56f9b28f4b | |||
| 222222f449 | |||
| 99391cd18b | |||
| 84be774b34 | |||
| e053f7e124 | |||
| 13029dbb33 | |||
| 938bfe88c1 | |||
| fa1bd237d1 | |||
| 3d7057b437 | |||
| 32c8c8b446 | |||
| a3655f5540 | |||
| a84941d681 | |||
| 46b7356a3b | |||
| 39208cd119 | |||
| 6dae15656d | |||
| b103bb2e7d | |||
| 2b2a3fc024 | |||
| 52ffe42e73 | |||
| 3b160cf05b | |||
| 0687982822 | |||
| ff4ed82239 | |||
| 037121e9ce | |||
| d6126bf52b | |||
| 1bc5936681 | |||
| 9a77a0cd0b | |||
| ebbd2c8a63 | |||
| de99467ede | |||
| 20f612215f | |||
| 3d45571bb0 | |||
| 1afd753fad | |||
| a85a7e204e | |||
| 509436cc4a | |||
| a48f6da1f4 | |||
| df96f9fdf6 | |||
| 390daa8b84 | |||
| af84249da0 | |||
| cad172a758 | |||
| fb59cb21b9 | |||
| 68e2d9b549 | |||
| 185f7ca2b6 | |||
| 7ccadc6bd6 | |||
| a9f7fa27ff | |||
| dbf3605712 | |||
| 543a7ff8ef | |||
| fc3a7b7a67 | |||
| b360493cdc | |||
| 12202508bf | |||
| 6d9a4bd017 | |||
| 426864eedb | |||
| 0b17a6fbeb | |||
| aeb31f1c0d | |||
| 28fe899aa1 | |||
| 30606f5c49 | |||
| db880ed868 | |||
| 5217093be2 | |||
| 0e83991537 | |||
| 3e8d19cccd | |||
| c1193e3450 | |||
| 73249b63d8 | |||
| c3087aeda6 | |||
| 0452a605ad | |||
| b9b7384123 | |||
| 726acc47e1 | |||
| 5c78b1d5d9 | |||
| f461922b9b | |||
| c58d307ba7 | |||
| 1eb93481e0 | |||
| b997e7c97c | |||
| 12fd541669 | |||
| 10377876e4 | |||
| d847168ecd | |||
| 14e51698c2 | |||
| aeb9c4f062 | |||
| e2ae7f77d8 | |||
| 0b1b670baf | |||
| 3cfa4b6e8a | |||
| 950edc01f2 | |||
| ea8def423e | |||
| 95fe8c32fa | |||
| 08b7531149 | |||
| 2cf2d390da | |||
| 8f3c2ec5fd | |||
| 9427c4e6cc | |||
| 6424d7db4f | |||
| 87e560dd3d | |||
| 09fc6fd364 | |||
| 2e47c8a1b6 |
60
.gitignore
vendored
60
.gitignore
vendored
@@ -1,3 +1,4 @@
|
||||
# --> Python
|
||||
.DS_Store
|
||||
.venv/
|
||||
__pycache__/
|
||||
@@ -9,3 +10,62 @@ dist/
|
||||
build/
|
||||
*.egg-info/
|
||||
|
||||
# ---> Codex
|
||||
.codex
|
||||
|
||||
# ---> Go
|
||||
# If you prefer the allow list template instead of the deny list, see community template:
|
||||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||
#
|
||||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
|
||||
# Dependency directories (remove the comment below to include it)
|
||||
# vendor/
|
||||
|
||||
# Go workspace file
|
||||
go.work
|
||||
go.work.sum
|
||||
|
||||
# env file
|
||||
.env
|
||||
|
||||
# Compiled binaries and test configuration
|
||||
narratio
|
||||
local-test
|
||||
pipeline.yml
|
||||
bin/
|
||||
|
||||
# Local run artifacts
|
||||
.audita-runs/
|
||||
report.json
|
||||
corrected.json
|
||||
normalized.json
|
||||
|
||||
# Coverage artifacts
|
||||
coverage.out
|
||||
coverage.txt
|
||||
|
||||
# ---> VisualStudioCode
|
||||
.vscode/*
|
||||
!.vscode/settings.json
|
||||
!.vscode/tasks.json
|
||||
!.vscode/launch.json
|
||||
!.vscode/extensions.json
|
||||
!.vscode/*.code-snippets
|
||||
|
||||
# Local History for Visual Studio Code
|
||||
.history/
|
||||
|
||||
# Built Visual Studio Code Extensions
|
||||
*.vsix
|
||||
|
||||
25
.woodpecker/release.yml
Normal file
25
.woodpecker/release.yml
Normal file
@@ -0,0 +1,25 @@
|
||||
when:
|
||||
- event: tag
|
||||
|
||||
steps:
|
||||
- name: build-release-assets
|
||||
image: golang:1.25
|
||||
commands:
|
||||
- bash scripts/build-release.sh "$CI_COMMIT_TAG" dist
|
||||
|
||||
- name: publish-release
|
||||
image: woodpeckerci/plugin-release
|
||||
depends_on:
|
||||
- build-release-assets
|
||||
settings:
|
||||
api_key:
|
||||
from_secret: GITEA_RELEASE_TOKEN
|
||||
files:
|
||||
- dist/audita-*-linux-amd64
|
||||
- dist/audita-*-linux-arm64
|
||||
checksum: sha256
|
||||
checksum-file: SHA256SUMS
|
||||
checksum-flatten: true
|
||||
file-exists: skip
|
||||
overwrite: false
|
||||
prerelease: false
|
||||
4
AGENTS.md
Normal file
4
AGENTS.md
Normal file
@@ -0,0 +1,4 @@
|
||||
Please review `docs/development.md` for initial orientation in this repository
|
||||
and follow its task-specific reading guide.
|
||||
|
||||
Review and follow `docs/policy/agents.md` before recommending, planning, or implementing any changes to this repository.
|
||||
2
LICENSE
2
LICENSE
@@ -1,4 +1,4 @@
|
||||
Copyright (c) 2026 eric.
|
||||
Copyright (c) 2026 Eric Rakestraw.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
||||
|
||||
|
||||
162
README.md
162
README.md
@@ -1,148 +1,44 @@
|
||||
# Audita
|
||||
|
||||
Audita is a framework-first transcript correction application. The public `audita` package provides:
|
||||
Audita polishes transcript JSON using glossary-aware and LLM-backed correction
|
||||
modules. It validates proposed corrections and produces corrected text plus
|
||||
reports for review and subprocess integration.
|
||||
|
||||
- deterministic transcript normalization
|
||||
- token-batched module orchestration
|
||||
- concrete `glossary`, `homophones`, `spoken_word`, and `grammar` modules built on reusable proposal / validator contracts
|
||||
- structured run reporting and work-dir diagnostics
|
||||
## Quickstart
|
||||
|
||||
The previous working implementation has been preserved as `audita_prototype` inside this repository. Its full regression suite lives under `tests/audita_prototype`.
|
||||
|
||||
## Development
|
||||
|
||||
This project is set up for `uv`.
|
||||
From a checkout, build with Go 1.24 or newer:
|
||||
|
||||
```sh
|
||||
uv sync --extra dev
|
||||
uv run pytest
|
||||
go build -o ./bin/audita ./cmd/audita
|
||||
```
|
||||
|
||||
## Usage
|
||||
Processing requires a reachable [compatible LLM endpoint](docs/integrations/openai-compatible-llm.md)
|
||||
and access to its configured model. Supply its credentials through the
|
||||
[configuration mechanisms](docs/config.md#secrets); the
|
||||
[minimal config](examples/minimal-config.yml) reads `AUDITA_LLM_API_KEY` and uses
|
||||
the [default model and endpoint](docs/config.md#llm-settings). Set that variable
|
||||
in your environment before using an endpoint that requires authentication.
|
||||
For another endpoint or model, edit a copy of the config.
|
||||
|
||||
Process a transcript with the current framework implementation:
|
||||
Run the included small input with the minimal config:
|
||||
|
||||
```sh
|
||||
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
./bin/audita process examples/tiny-transcript.json \
|
||||
--config examples/minimal-config.yml \
|
||||
--glossary examples/tiny-glossary.yaml \
|
||||
--output corrected.json
|
||||
```
|
||||
|
||||
The framework currently runs this default module sequence:
|
||||
This makes LLM requests. Inherited configuration environment overrides still
|
||||
apply; see [precedence](docs/config.md#effective-precedence).
|
||||
|
||||
1. `glossary`
|
||||
2. `homophones`
|
||||
3. `glossary`
|
||||
4. `spoken_word`
|
||||
5. `grammar`
|
||||
## Documentation
|
||||
|
||||
Resolved run instance names are auto-numbered for repeats, so the default report pipeline is:
|
||||
|
||||
1. `glossary_1`
|
||||
2. `homophones`
|
||||
3. `glossary_2`
|
||||
4. `spoken_word`
|
||||
5. `grammar`
|
||||
|
||||
The default module sequence is fully implemented today:
|
||||
|
||||
- `glossary` proposes glossary-supported acoustic corrections
|
||||
- `homophones` proposes conservative homophone and mistranscription corrections
|
||||
- `spoken_word` proposes conservative dysfluency cleanup
|
||||
- `grammar` proposes punctuation, capitalization, and spacing cleanup only
|
||||
|
||||
To run a custom module sequence, pass `--modules`:
|
||||
|
||||
```sh
|
||||
uv run audita process transcript.json --glossary glossary.yaml --modules grammar --output corrected.json
|
||||
```
|
||||
|
||||
To also write a structured JSON report:
|
||||
|
||||
```sh
|
||||
uv run audita process transcript.json --glossary glossary.yaml --output corrected.json --report-json report.json
|
||||
```
|
||||
|
||||
From a checked-out repository, you can also use the root launcher:
|
||||
|
||||
```sh
|
||||
./audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
```
|
||||
|
||||
For a system-wide command, install the source tree under `/usr/local/src/audita`, sync dependencies there, and symlink the root launcher into your `PATH`:
|
||||
|
||||
```sh
|
||||
cd /usr/local/src/audita
|
||||
uv sync --extra dev
|
||||
ln -s /usr/local/src/audita/audita /usr/local/bin/audita
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
```
|
||||
|
||||
Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr.
|
||||
`--report-json` writes a separate machine-readable run report and never mixes report data into stdout.
|
||||
|
||||
Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Default OpenRouter runs require LLM API credentials, because the `glossary`, `homophones`, `spoken_word`, and `grammar` modules make real LLM calls. Self-hosted or other non-default OpenAI-compatible endpoints may not require credentials. `AUDITA_LLM_API_KEY` and `--llm-api-key` are the preferred provider-neutral credential surfaces, while `OPENROUTER_API_KEY` remains supported as a backward-compatible fallback.
|
||||
|
||||
| Environment variable | CLI flag | Default | Purpose |
|
||||
| --- | --- | --- | --- |
|
||||
| `AUDITA_MODULES` | `--modules` | `glossary,homophones,glossary,spoken_word,grammar` | Comma-separated logical module keys to run; CLI overrides the environment value |
|
||||
| `AUDITA_LLM_API_KEY` | `--llm-api-key` | unset | Preferred provider-neutral LLM API credential; required for the default OpenRouter endpoint and optional for non-default endpoints; CLI overrides both environment-key variants |
|
||||
| `AUDITA_VALIDATION_LLM_API_KEY` | `--validation-llm-api-key` | unset | Validation-phase LLM API credential; defaults to the primary LLM API key and is optional for non-default validation endpoints |
|
||||
| `AUDITA_MODEL` | `--model` | `openrouter/google/gemma-4-31b-it` | LLM model name sent to the configured OpenAI-compatible endpoint |
|
||||
| `AUDITA_VALIDATION_MODEL` | `--validation-model` | unset | Validation-phase LLM model; defaults to `AUDITA_MODEL` |
|
||||
| `AUDITA_BASE_URL` | `--base-url` | `https://openrouter.ai/api/v1` | OpenAI-compatible API base URL |
|
||||
| `AUDITA_VALIDATION_BASE_URL` | `--validation-base-url` | unset | Validation-phase OpenAI-compatible API base URL; defaults to `AUDITA_BASE_URL` |
|
||||
| `AUDITA_LLM_TIMEOUT_SECONDS` | `--llm-timeout-seconds` | `600` | Per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint |
|
||||
| `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | `--validation-llm-timeout-seconds` | unset | Validation-phase per-request timeout in seconds; defaults to `AUDITA_LLM_TIMEOUT_SECONDS` |
|
||||
| `AUDITA_VALIDATION_MAX_PROMPT_TOKENS` | `--validation-max-prompt-tokens` | `2048` | Maximum estimated tokens per validation-phase LLM prompt batch |
|
||||
| `AUDITA_TARGET_SECTIONS` | `--target-sections` | unset | Exact number of contiguous proposal-stage transcript sections; errors if min/max token bounds cannot be satisfied |
|
||||
| `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses |
|
||||
| `AUDITA_VALIDATION_MAX_RETRIES` | `--validation-max-retries` | unset | Validation-phase structured-output retries; defaults to `AUDITA_MAX_RETRIES` |
|
||||
| `AUDITA_VALIDATION_LLM_CONCURRENCY` | `--validation-llm-concurrency` | unset | Validation-phase LLM concurrency; defaults to `AUDITA_LLM_CONCURRENCY` |
|
||||
| `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `8192` | Maximum estimated tokens per proposal-stage transcript section |
|
||||
| `AUDITA_MIN_SECTION_TOKENS` | `--min-section-tokens` | `2048` | Minimum estimated tokens per proposal-stage transcript section when balancing for concurrency |
|
||||
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation |
|
||||
| `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | `--grammar-confidence-threshold` | `0.8` | Minimum confidence required for grammar proposals to survive validation |
|
||||
| `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | `--homophones-confidence-threshold` | `0.8` | Minimum confidence required for homophone proposals to survive validation |
|
||||
| `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | `--spoken-word-confidence-threshold` | `0.8` | Minimum confidence required for spoken-word proposals to survive validation |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | `--normalize-max-segment-gap` | `4.0` | Same-speaker gaps eligible for deterministic merging |
|
||||
| `AUDITA_NORMALIZE_ELLIPSIS_GAP` | `--normalize-ellipsis-gap` | `3.5` | Same-speaker gaps above this value are joined with ` ... ` |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | `--normalize-max-segment-duration` | `60.0` | Maximum merged segment duration |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | `--normalize-max-segment-tokens` | `2048` | Maximum merged segment prompt payload size |
|
||||
| `AUDITA_WORK_DIR` | `--work-dir` | `/tmp/audita` | Per-run scratch diagnostics directory |
|
||||
| `AUDITA_WORK_DIR_RETENTION` | `--work-dir-retention` | `auto` | Whether to retain the per-run work directory: `auto`, `always`, or `never` |
|
||||
|
||||
Set `AUDITA_MODULES=grammar` to run only the grammar module by default, or override it per command with `--modules`.
|
||||
|
||||
Validation-phase LLM settings inherit from the primary `AUDITA_*` LLM settings by default. Set any of the `AUDITA_VALIDATION_*` values only when you want LLM-backed validators to use a different model, endpoint, credential, timeout, retry budget, or concurrency level.
|
||||
|
||||
OpenRouter remains the default out of the box:
|
||||
|
||||
```sh
|
||||
export AUDITA_LLM_API_KEY=your-openrouter-key
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
```
|
||||
|
||||
You can point Audita at any OpenAI-compatible endpoint by changing `AUDITA_BASE_URL` and, if needed, `AUDITA_MODEL`. For example, a local vLLM server:
|
||||
|
||||
```sh
|
||||
export AUDITA_BASE_URL=http://localhost:8000/v1
|
||||
export AUDITA_MODEL=meta-llama/Llama-3.1-8B-Instruct
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
```
|
||||
|
||||
If your self-hosted endpoint requires authentication, you can still set `AUDITA_LLM_API_KEY`; Audita simply no longer requires it for non-default endpoints.
|
||||
|
||||
Or the actual OpenAI API:
|
||||
|
||||
```sh
|
||||
export AUDITA_LLM_API_KEY=your-openai-key
|
||||
export AUDITA_BASE_URL=https://api.openai.com/v1
|
||||
export AUDITA_MODEL=gpt-4.1-mini
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
```
|
||||
|
||||
`AUDITA_WORK_DIR` stores per-run diagnostics while processing. Under the default `AUDITA_WORK_DIR_RETENTION=auto`, clean successful runs are removed, while failed runs and successful runs with final skipped corrections are preserved. Use `always` to keep every run directory and `never` to remove successful run directories even when skips remain.
|
||||
Failed runs always preserve the run directory and include an authoritative `report.json` alongside normalization and prompt/response diagnostics.
|
||||
|
||||
## Prototype Archive
|
||||
|
||||
The archived prototype remains importable as `audita_prototype` and is still covered by its original regression suite. This is intentional: the new `audita` package is a framework-oriented rewrite, not a thin wrapper around the old code.
|
||||
- [CLI reference](docs/cli.md)
|
||||
- [Configuration and maintained examples](docs/config.md)
|
||||
- [Operations and artifact lifecycle](docs/operations.md)
|
||||
- [Transcript and glossary inputs](docs/integrations/transcript-glossary-files.md)
|
||||
- [Output files and report formats](docs/integrations/output-files.md)
|
||||
- [Subprocess integration](docs/integrations/subprocess.md)
|
||||
- [LLM endpoint contract](docs/integrations/openai-compatible-llm.md)
|
||||
- [Contributor guide](docs/development.md)
|
||||
|
||||
147
audita
147
audita
@@ -1,147 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
DEFAULT_WORK_DIR = "/tmp/audita"
|
||||
_SECRET_FLAGS = {"--llm-api-key", "--validation-llm-api-key"}
|
||||
|
||||
|
||||
def _redact_argv(argv: list[str]) -> list[str]:
|
||||
redacted: list[str] = []
|
||||
index = 0
|
||||
while index < len(argv):
|
||||
arg = argv[index]
|
||||
matched_flag = next((flag for flag in _SECRET_FLAGS if arg == flag or arg.startswith(flag + "=")), None)
|
||||
if matched_flag is None:
|
||||
redacted.append(arg)
|
||||
index += 1
|
||||
continue
|
||||
if arg == matched_flag:
|
||||
redacted.append(arg)
|
||||
if index + 1 < len(argv):
|
||||
redacted.append("[REDACTED]")
|
||||
index += 2
|
||||
else:
|
||||
index += 1
|
||||
continue
|
||||
redacted.append(f"{matched_flag}=[REDACTED]")
|
||||
index += 1
|
||||
return redacted
|
||||
|
||||
|
||||
def _resolve_work_root(argv: list[str]) -> Path:
|
||||
for index, arg in enumerate(argv):
|
||||
if arg == "--work-dir" and index + 1 < len(argv):
|
||||
return Path(argv[index + 1])
|
||||
if arg.startswith("--work-dir="):
|
||||
return Path(arg.split("=", 1)[1])
|
||||
return Path(os.environ.get("AUDITA_WORK_DIR") or DEFAULT_WORK_DIR)
|
||||
|
||||
|
||||
def _create_run_dir(root: Path) -> Path:
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
|
||||
run_dir = root / f"run-{timestamp}-{uuid4().hex[:8]}"
|
||||
run_dir.mkdir(parents=False, exist_ok=False)
|
||||
return run_dir
|
||||
|
||||
|
||||
def _capture_run_dirs(root: Path) -> set[str]:
|
||||
if not root.exists():
|
||||
return set()
|
||||
return {path.name for path in root.iterdir() if path.is_dir() and path.name.startswith("run-")}
|
||||
|
||||
|
||||
def _find_new_run_dir(root: Path, before: set[str]) -> Optional[Path]:
|
||||
if not root.exists():
|
||||
return None
|
||||
candidates = [
|
||||
path for path in root.iterdir() if path.is_dir() and path.name.startswith("run-") and path.name not in before
|
||||
]
|
||||
if not candidates:
|
||||
return None
|
||||
return max(candidates, key=lambda path: path.name)
|
||||
|
||||
|
||||
def _write_launcher_error_log(
|
||||
path: Path,
|
||||
*,
|
||||
message: str,
|
||||
exit_code: int,
|
||||
argv: list[str],
|
||||
command: Optional[list[str]],
|
||||
) -> None:
|
||||
payload = {
|
||||
"timestamp": datetime.utcnow().isoformat() + "Z",
|
||||
"message": message,
|
||||
"exit_code": exit_code,
|
||||
"argv": argv,
|
||||
"cwd": os.getcwd(),
|
||||
"command": command,
|
||||
}
|
||||
path.write_text(
|
||||
"Audita Launcher Diagnostics\n" + json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _emit_console_line(message: str) -> None:
|
||||
for stream in (sys.stderr, sys.stdout):
|
||||
if stream is None:
|
||||
continue
|
||||
try:
|
||||
stream.write(f"{message}\n")
|
||||
stream.flush()
|
||||
return
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
|
||||
|
||||
def main() -> int:
|
||||
argv = list(sys.argv[1:])
|
||||
work_root = _resolve_work_root(argv)
|
||||
redacted_argv = _redact_argv(argv)
|
||||
uv = shutil.which("uv")
|
||||
if uv is None:
|
||||
run_dir = _create_run_dir(work_root)
|
||||
error_log = run_dir / "error.log"
|
||||
message = "uv is required to run this launcher. Install uv and run `uv sync` in the Audita project."
|
||||
_write_launcher_error_log(error_log, message=message, exit_code=1, argv=redacted_argv, command=None)
|
||||
_emit_console_line(f"audita: error: {message}")
|
||||
_emit_console_line("audita: exit code: 1")
|
||||
_emit_console_line(f"audita: run directory: {run_dir}")
|
||||
_emit_console_line(f"audita: error log: {error_log}")
|
||||
return 1
|
||||
|
||||
project_root = Path(__file__).resolve().parent
|
||||
command = [uv, "run", "--project", str(project_root), "python", "-m", "audita", *sys.argv[1:]]
|
||||
before = _capture_run_dirs(work_root)
|
||||
result = subprocess.run(command, cwd=project_root, check=False)
|
||||
if result.returncode == 0:
|
||||
return 0
|
||||
if _find_new_run_dir(work_root, before) is None:
|
||||
run_dir = _create_run_dir(work_root)
|
||||
error_log = run_dir / "error.log"
|
||||
_write_launcher_error_log(
|
||||
error_log,
|
||||
message=f"Audita subprocess exited with status {result.returncode}.",
|
||||
exit_code=result.returncode,
|
||||
argv=redacted_argv,
|
||||
command=_redact_argv(command),
|
||||
)
|
||||
_emit_console_line(f"audita: subprocess exited with status {result.returncode}")
|
||||
_emit_console_line(f"audita: run directory: {run_dir}")
|
||||
_emit_console_line(f"audita: error log: {error_log}")
|
||||
return result.returncode
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
11
cmd/audita/main.go
Normal file
11
cmd/audita/main.go
Normal file
@@ -0,0 +1,11 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/cli"
|
||||
)
|
||||
|
||||
func main() {
|
||||
os.Exit(cli.Run(os.Args[1:], os.Stdout, os.Stderr))
|
||||
}
|
||||
686
cmd/audita/main_integration_test.go
Normal file
686
cmd/audita/main_integration_test.go
Normal file
@@ -0,0 +1,686 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/cli"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/testsupport"
|
||||
)
|
||||
|
||||
func TestHelperProcess(t *testing.T) {
|
||||
if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
|
||||
return
|
||||
}
|
||||
|
||||
sep := -1
|
||||
for i, arg := range os.Args {
|
||||
if arg == "--" {
|
||||
sep = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if sep == -1 {
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
cli.ConfigureSubprocessTestHooksFromEnv()
|
||||
code := cli.Run(os.Args[sep+1:], os.Stdout, os.Stderr)
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func TestProcessHelpSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(t, "process", "--help")
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if !strings.Contains(result.stdout, "Usage:") || !strings.Contains(result.stdout, "--glossary") {
|
||||
t.Fatalf("unexpected help stdout: %q", result.stdout)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSuccessWithOutputSubprocess(t *testing.T) {
|
||||
outputPath := filepath.Join(t.TempDir(), "corrected.json")
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output",
|
||||
outputPath,
|
||||
)
|
||||
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout when --output is set, got %q", result.stdout)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
|
||||
inputBytes := readFile(t, fixturePath("tiny_transcript.json"))
|
||||
outputBytes := readFile(t, outputPath)
|
||||
assertJSONSemanticallyEqual(t, inputBytes, outputBytes)
|
||||
}
|
||||
|
||||
func TestProcessSuccessWithoutOutputSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
|
||||
inputBytes := readFile(t, fixturePath("tiny_transcript.json"))
|
||||
assertJSONSemanticallyEqual(t, inputBytes, []byte(result.stdout))
|
||||
}
|
||||
|
||||
func TestProcessSuccessWithAuditaV1OutputSchemaSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output-schema",
|
||||
"audita-v1",
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
var out struct {
|
||||
Schema string `json:"schema"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(result.stdout), &out); err != nil {
|
||||
t.Fatalf("expected valid audita-v1 JSON output: %v", err)
|
||||
}
|
||||
if out.Schema != "audita-v1" {
|
||||
t.Fatalf("expected audita-v1 schema, got %q", out.Schema)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMissingTranscriptSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(t, "process", "--glossary", fixturePath("tiny_glossary.yaml"))
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "expected exactly 1 transcript JSON path argument") {
|
||||
t.Fatalf("expected actionable missing transcript error, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMalformedJSONSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("malformed_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "is not valid JSON") {
|
||||
t.Fatalf("expected malformed JSON error, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMissingTranscriptFileSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
filepath.Join(t.TempDir(), "missing-transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "transcript_read") {
|
||||
t.Fatalf("expected transcript_read failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMissingGlossaryFileSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
filepath.Join(t.TempDir(), "missing-glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "glossary_read") {
|
||||
t.Fatalf("expected glossary_read failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureTranscriptSchemaSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
schemaFixturePath("transcript_empty_speaker.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "transcript_schema") {
|
||||
t.Fatalf("expected transcript_schema failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMalformedGlossaryYAMLSubprocess(t *testing.T) {
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
schemaFixturePath("glossary_malformed.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "glossary_schema") {
|
||||
t.Fatalf("expected glossary_schema failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureUnreadableTranscriptSubprocess(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("portable unreadable-file permissions are not reliable on windows")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
transcriptPath := filepath.Join(dir, "transcript.json")
|
||||
if err := os.WriteFile(transcriptPath, []byte(`[]`), 0o000); err != nil {
|
||||
t.Fatalf("write unreadable transcript: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.Chmod(transcriptPath, 0o644) })
|
||||
if _, err := os.ReadFile(transcriptPath); err == nil {
|
||||
t.Skip("unable to make transcript unreadable on this platform/user")
|
||||
}
|
||||
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
transcriptPath,
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "transcript_read") {
|
||||
t.Fatalf("expected transcript_read failure, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureUnwritableOutputSubprocess(t *testing.T) {
|
||||
outputDir := t.TempDir()
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output",
|
||||
outputDir,
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "failed to write output file") {
|
||||
t.Fatalf("expected write failure message, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureUnwritableReportJSONSubprocess(t *testing.T) {
|
||||
reportDir := t.TempDir()
|
||||
outputPath := filepath.Join(t.TempDir(), "out.json")
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output",
|
||||
outputPath,
|
||||
"--report-json",
|
||||
reportDir,
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "failed to write report JSON file") {
|
||||
t.Fatalf("expected report write failure message, got %q", result.stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSuccessReportJSONSubprocess(t *testing.T) {
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--report-json",
|
||||
reportPath,
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
if !json.Valid([]byte(result.stdout)) {
|
||||
t.Fatalf("expected transcript JSON only on stdout, got %q", result.stdout)
|
||||
}
|
||||
report := readFile(t, reportPath)
|
||||
if !json.Valid(report) {
|
||||
t.Fatalf("expected valid report JSON, got %q", string(report))
|
||||
}
|
||||
// Ensure report JSON is not printed to stdout.
|
||||
if strings.Contains(result.stdout, `"default_pipeline"`) {
|
||||
t.Fatalf("report JSON leaked to stdout: %q", result.stdout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSuccessLargeTranscriptSubprocess(t *testing.T) {
|
||||
transcriptPath := writeLargeTranscriptFixture(t, 320)
|
||||
result := runCLISubprocess(
|
||||
t,
|
||||
"process",
|
||||
transcriptPath,
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected exit 0, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
if !json.Valid([]byte(result.stdout)) {
|
||||
t.Fatalf("expected valid transcript JSON on stdout")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMalformedStructuredLLMResponseViaSubprocessHook(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "malformed_structured"},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"grammar",
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected zero exit code, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if !json.Valid([]byte(result.stdout)) {
|
||||
t.Fatalf("expected transcript JSON on stdout, got %q", result.stdout)
|
||||
}
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
report := readFile(t, reportPath)
|
||||
if !json.Valid(report) {
|
||||
t.Fatalf("expected valid success report JSON")
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
|
||||
t.Fatalf("did not expect error.log, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureBackendLLMViaSubprocessHook(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "backend_error"},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"grammar",
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "synthetic backend failure") {
|
||||
t.Fatalf("expected backend failure details, got %q", result.stderr)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "diagnostics:") {
|
||||
t.Fatalf("expected diagnostics path in stderr, got %q", result.stderr)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(onlyRunDir(t, workDir), "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log in retained failed run: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFailureMidPipelinePreservesPartialReportsSubprocess(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{"AUDITA_SUBPROCESS_TEST_LLM_MODE": "mid_pipeline_fail"},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"glossary,homophones,glossary,spoken_word,grammar",
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
reportRaw := readFile(t, reportPath)
|
||||
var report struct {
|
||||
Status string `json:"status"`
|
||||
ErrorPhase string `json:"error_phase"`
|
||||
ModuleResults []struct {
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
Status string `json:"status"`
|
||||
} `json:"module_results"`
|
||||
}
|
||||
if err := json.Unmarshal(reportRaw, &report); err != nil {
|
||||
t.Fatalf("unmarshal report: %v", err)
|
||||
}
|
||||
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
|
||||
t.Fatalf("expected failed runner_execution report, got %+v", report)
|
||||
}
|
||||
if len(report.ModuleResults) == 0 {
|
||||
t.Fatalf("expected partial module results in failure report")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessCancellationViaSubprocessTimeoutHook(t *testing.T) {
|
||||
workDir := t.TempDir()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
|
||||
result := runCLISubprocessContext(t, ctx,
|
||||
map[string]string{
|
||||
"AUDITA_SUBPROCESS_TEST_LLM_MODE": "block_until_cancel",
|
||||
"AUDITA_SUBPROCESS_TEST_RUN_TIMEOUT_MS": "120",
|
||||
},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"grammar",
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.stdout != "" {
|
||||
if result.stderr == "" {
|
||||
t.Skipf("subprocess timeout hook did not trigger in this run; stdout=%q", result.stdout)
|
||||
}
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "context deadline exceeded") {
|
||||
t.Fatalf("expected context deadline error, got %q", result.stderr)
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log for canceled run: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil {
|
||||
t.Fatalf("expected report.json for canceled run: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessSubprocessNoSecretLeakInOutputsAndDiagnostics(t *testing.T) {
|
||||
secret := "subprocess-secret"
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
outputPath := filepath.Join(t.TempDir(), "out.json")
|
||||
result := runCLISubprocessWithEnv(t,
|
||||
map[string]string{
|
||||
"AUDITA_LLM_API_KEY": secret,
|
||||
"AUDITA_VALIDATION_LLM_API_KEY": secret,
|
||||
},
|
||||
"process",
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--output",
|
||||
outputPath,
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if strings.Contains(result.stdout, secret) || strings.Contains(result.stderr, secret) {
|
||||
t.Fatalf("secret leaked in subprocess stdio")
|
||||
}
|
||||
assertNoSecretInFile(t, reportPath, secret)
|
||||
assertNoSecretInTree(t, onlyRunDir(t, workDir), secret)
|
||||
}
|
||||
|
||||
type subprocessResult struct {
|
||||
stdout string
|
||||
stderr string
|
||||
exitCode int
|
||||
}
|
||||
|
||||
func runCLISubprocess(t *testing.T, args ...string) subprocessResult {
|
||||
t.Helper()
|
||||
return runCLISubprocessWithEnv(t, nil, args...)
|
||||
}
|
||||
|
||||
func runCLISubprocessWithEnv(t *testing.T, extraEnv map[string]string, args ...string) subprocessResult {
|
||||
t.Helper()
|
||||
return runCLISubprocessContext(t, context.Background(), extraEnv, args...)
|
||||
}
|
||||
|
||||
func runCLISubprocessContext(t *testing.T, ctx context.Context, extraEnv map[string]string, args ...string) subprocessResult {
|
||||
t.Helper()
|
||||
cmdArgs := append([]string{"-test.run=TestHelperProcess", "--"}, args...)
|
||||
cmd := exec.CommandContext(ctx, os.Args[0], cmdArgs...)
|
||||
env := append(filterAuditaEnv(os.Environ()), "GO_WANT_HELPER_PROCESS=1")
|
||||
// Use defaults without discovering the host's installed configuration.
|
||||
// Explicit test environment overrides and --config retain their precedence.
|
||||
if _, supplied := extraEnv["AUDITA_CONFIG"]; !supplied {
|
||||
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||
if err := os.WriteFile(configPath, []byte("version: 1\n"), 0600); err != nil {
|
||||
t.Fatalf("write baseline config: %v", err)
|
||||
}
|
||||
env = append(env, "AUDITA_CONFIG="+configPath)
|
||||
}
|
||||
for k, v := range extraEnv {
|
||||
env = append(env, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
cmd.Env = env
|
||||
var stdoutBuf bytes.Buffer
|
||||
var stderrBuf bytes.Buffer
|
||||
cmd.Stdout = &stdoutBuf
|
||||
cmd.Stderr = &stderrBuf
|
||||
|
||||
err := cmd.Run()
|
||||
result := subprocessResult{
|
||||
stdout: stdoutBuf.String(),
|
||||
stderr: stderrBuf.String(),
|
||||
}
|
||||
if err == nil {
|
||||
return result
|
||||
}
|
||||
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
result.exitCode = exitErr.ExitCode()
|
||||
return result
|
||||
}
|
||||
|
||||
t.Fatalf("subprocess execution failed: %v", err)
|
||||
return subprocessResult{}
|
||||
}
|
||||
|
||||
func filterAuditaEnv(env []string) []string {
|
||||
filtered := make([]string, 0, len(env))
|
||||
for _, entry := range env {
|
||||
key := entry
|
||||
if idx := strings.IndexByte(entry, '='); idx >= 0 {
|
||||
key = entry[:idx]
|
||||
}
|
||||
if strings.HasPrefix(key, "AUDITA_") || key == "OPENROUTER_API_KEY" {
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, entry)
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
func fixturePath(name string) string {
|
||||
return filepath.Join("..", "..", "internal", "cli", "testdata", name)
|
||||
}
|
||||
|
||||
func schemaFixturePath(name string) string {
|
||||
return filepath.Join("..", "..", "internal", "core", "schema", "testdata", name)
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) []byte {
|
||||
return testsupport.ReadFile(t, path)
|
||||
}
|
||||
|
||||
func assertJSONSemanticallyEqual(t *testing.T, expected []byte, actual []byte) {
|
||||
t.Helper()
|
||||
if !json.Valid(actual) {
|
||||
t.Fatalf("actual output is not valid JSON: %q", string(actual))
|
||||
}
|
||||
|
||||
var expectedValue any
|
||||
var actualValue any
|
||||
if err := json.Unmarshal(expected, &expectedValue); err != nil {
|
||||
t.Fatalf("failed to unmarshal expected JSON: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(actual, &actualValue); err != nil {
|
||||
t.Fatalf("failed to unmarshal actual JSON: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(expectedValue, actualValue) {
|
||||
t.Fatalf("JSON content mismatch: expected %q got %q", string(expected), string(actual))
|
||||
}
|
||||
}
|
||||
|
||||
func writeLargeTranscriptFixture(t *testing.T, segments int) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "large-transcript.json")
|
||||
rows := make([]string, 0, segments)
|
||||
for i := 0; i < segments; i++ {
|
||||
rows = append(rows, fmt.Sprintf(`{"id":%d,"speaker":"Speaker%d","start":%s,"end":%s,"text":"Segment %d has enough words to exercise stdout and pipe buffering safely."}`,
|
||||
i+1,
|
||||
(i%4)+1,
|
||||
strconv.FormatFloat(float64(i)*1.1, 'f', 1, 64),
|
||||
strconv.FormatFloat(float64(i)*1.1+1.0, 'f', 1, 64),
|
||||
i+1,
|
||||
))
|
||||
}
|
||||
payload := "[\n " + strings.Join(rows, ",\n ") + "\n]\n"
|
||||
if err := os.WriteFile(path, []byte(payload), 0o644); err != nil {
|
||||
t.Fatalf("write large transcript fixture: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func onlyRunDir(t *testing.T, workDir string) string {
|
||||
return testsupport.OnlyRunDir(t, workDir)
|
||||
}
|
||||
|
||||
func assertNoSecretInFile(t *testing.T, path, secret string) {
|
||||
testsupport.AssertNoSecretInFile(t, path, secret)
|
||||
}
|
||||
|
||||
func assertNoSecretInTree(t *testing.T, root, secret string) {
|
||||
testsupport.AssertNoSecretInTree(t, root, secret)
|
||||
}
|
||||
7
docs/adr/README.md
Normal file
7
docs/adr/README.md
Normal file
@@ -0,0 +1,7 @@
|
||||
# Architectural Decision Records
|
||||
|
||||
No ADRs have been recorded here. Use the
|
||||
[documentation policy's ADR lifecycle](../policy/documentation.md#architecture-decision-records)
|
||||
for significant decisions. Current architecture belongs in the
|
||||
[architecture policy](../policy/architecture.md); implementation status belongs in
|
||||
[roadmap documents](../roadmap/).
|
||||
198
docs/cli.md
Normal file
198
docs/cli.md
Normal file
@@ -0,0 +1,198 @@
|
||||
# Audita CLI Reference
|
||||
|
||||
## Shortest Useful Command
|
||||
|
||||
```sh
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
```
|
||||
|
||||
This command runs the core workflow with your input files. For a copyable example
|
||||
using maintained inputs and explicit prerequisites, see the [README](../README.md).
|
||||
Commands here assume `audita` is on `PATH`; use `./bin/audita` after a local build.
|
||||
|
||||
## Command Overview
|
||||
|
||||
- `audita process`: process one transcript JSON file.
|
||||
- `audita config validate`: validate a versioned YAML config file.
|
||||
- `audita config print-effective`: print redacted effective config JSON.
|
||||
|
||||
General help:
|
||||
|
||||
```sh
|
||||
audita --help
|
||||
audita process --help
|
||||
audita config --help
|
||||
```
|
||||
|
||||
## `process`
|
||||
|
||||
Usage:
|
||||
|
||||
```sh
|
||||
audita process <transcript.json> [flags]
|
||||
```
|
||||
|
||||
Input requirements:
|
||||
- exactly one transcript JSON positional argument is required;
|
||||
- `--glossary <path>` is required.
|
||||
|
||||
Place the transcript path first, followed by flags, as shown above. Flag values
|
||||
may use `--name value` or `--name=value`. Unknown flags or extra positional
|
||||
arguments are errors. Flags before a final transcript path are also accepted;
|
||||
do not interleave further flags after a positional argument consumed by the flag
|
||||
parser.
|
||||
|
||||
[Configuration discovery and precedence](config.md#loading-model) determine the
|
||||
initial settings; explicit process flags supply overrides. Configuration is loaded
|
||||
before process help is printed, so invalid discovered configuration can also make
|
||||
process help fail. Input formats belong to the
|
||||
[input-file contract](integrations/transcript-glossary-files.md).
|
||||
|
||||
### `process` Flag Reference
|
||||
|
||||
Core I/O flags:
|
||||
- `--config <path>`: path to versioned YAML config file.
|
||||
- `--glossary <path>`: glossary YAML input path (required).
|
||||
- `--output <path>`: corrected transcript JSON output file path.
|
||||
- `--report-json <path>`: machine-readable report JSON output path.
|
||||
- `--output-schema <key>`: output encoder selection; see [configuration](config.md#pipeline-and-output).
|
||||
- `--modules <csv>`: comma-separated module sequence override.
|
||||
|
||||
Primary LLM flags:
|
||||
- `--llm-api-key <value>`: primary LLM API key.
|
||||
- `--model <name>`: primary LLM model name.
|
||||
- `--base-url <url>`: primary OpenAI-compatible base URL.
|
||||
- `--llm-timeout-seconds <int>`: primary timeout in seconds.
|
||||
- `--max-retries <int>`: primary structured-output retries.
|
||||
|
||||
Validation LLM flags:
|
||||
- `--validation-llm-api-key <value>`: validation LLM API key.
|
||||
- `--validation-model <name>`: validation LLM model name.
|
||||
- `--validation-base-url <url>`: validation OpenAI-compatible base URL.
|
||||
- `--validation-llm-timeout-seconds <int>`: validation timeout in seconds.
|
||||
- `--validation-max-retries <int>`: validation structured-output retries.
|
||||
- `--validation-max-prompt-tokens <int>`: validation max prompt tokens.
|
||||
|
||||
Concurrency flags:
|
||||
- `--total-llm-concurrency <int>`: total concurrent proposal+validation LLM calls.
|
||||
- `--proposal-llm-concurrency <int>`: concurrent proposal-generation LLM calls.
|
||||
- `--validation-llm-concurrency <int>`: concurrent validation LLM calls.
|
||||
- `--llm-concurrency <int>`: alias for `--total-llm-concurrency`.
|
||||
|
||||
Chunking and normalization flags:
|
||||
- `--target-sections <int>`: target number of transcript sections.
|
||||
- `--max-section-tokens <int>`: maximum section tokens.
|
||||
- `--min-section-tokens <int>`: minimum section tokens.
|
||||
- `--normalize-max-segment-gap <float>`: maximum same-speaker merge gap.
|
||||
- `--normalize-ellipsis-gap <float>`: gap threshold for ellipsis insertion.
|
||||
- `--normalize-max-segment-duration <float>`: maximum merged segment duration.
|
||||
- `--normalize-max-segment-tokens <int>`: maximum merged segment token estimate.
|
||||
|
||||
Threshold flags:
|
||||
- `--glossary-confidence-threshold <float>`
|
||||
- `--homophones-confidence-threshold <float>`
|
||||
- `--spoken-word-confidence-threshold <float>`
|
||||
- `--grammar-confidence-threshold <float>`
|
||||
|
||||
Context and diagnostics flags:
|
||||
- `--transcript-description <text>`: background context for prompts; does not override transcript content.
|
||||
- `--work-dir <path>`: per-run diagnostics work directory.
|
||||
- `--work-dir-retention <value>`: retention selection; see [configuration](config.md#thresholds-context-and-diagnostics).
|
||||
|
||||
Flag descriptions identify their settings; defaults, accepted keys, numeric
|
||||
constraints, and inheritance rules are owned by [configuration](config.md).
|
||||
Normalization time flags use numeric seconds; LLM timeout flags use integer seconds.
|
||||
|
||||
### `process` Output and Exit Behavior
|
||||
|
||||
- With `--output`: stdout is expected to be empty on success.
|
||||
- Without `--output`: stdout contains transcript JSON only on success.
|
||||
- `--report-json` writes a file and is never printed to stdout.
|
||||
- Stderr is human-readable diagnostics/errors.
|
||||
- Processing failures include the diagnostics directory path when initialization
|
||||
succeeded. Later report-write or retention errors can omit that path.
|
||||
- Output files may already exist when a later reporting or cleanup failure occurs.
|
||||
- Rejected/skipped corrections and warnings can occur in a successful run; inspect
|
||||
the [report format](integrations/output-files.md#process-report) for details.
|
||||
- Diagnostic wording is human-readable and is not a machine-stable JSON channel.
|
||||
|
||||
Exit behavior:
|
||||
- `0`: success.
|
||||
- `1`: runtime failure during processing/reporting/output or retention handling.
|
||||
- `2`: CLI usage or configuration input error.
|
||||
|
||||
Integration references:
|
||||
- subprocess contract: [`docs/integrations/subprocess.md`](integrations/subprocess.md)
|
||||
- transcript/glossary file contract: [`docs/integrations/transcript-glossary-files.md`](integrations/transcript-glossary-files.md)
|
||||
|
||||
### `process` Examples
|
||||
|
||||
Write corrected transcript to a file:
|
||||
|
||||
```sh
|
||||
audita process transcript.json \
|
||||
--glossary glossary.yaml \
|
||||
--output corrected.json
|
||||
```
|
||||
|
||||
Emit transcript JSON to stdout:
|
||||
|
||||
```sh
|
||||
audita process transcript.json --glossary glossary.yaml
|
||||
```
|
||||
|
||||
Use explicit config and write report JSON:
|
||||
|
||||
```sh
|
||||
audita process transcript.json \
|
||||
--glossary glossary.yaml \
|
||||
--config audita.yml \
|
||||
--output corrected.json \
|
||||
--report-json report.json
|
||||
```
|
||||
|
||||
Override the module sequence:
|
||||
|
||||
```sh
|
||||
audita process transcript.json \
|
||||
--glossary glossary.yaml \
|
||||
--modules glossary,homophones,grammar \
|
||||
--output corrected.json
|
||||
```
|
||||
|
||||
## Configuration Commands
|
||||
|
||||
### `config validate`
|
||||
|
||||
```sh
|
||||
audita config validate --config examples/minimal-config.yml
|
||||
audita config validate --config examples/production-config.yml
|
||||
```
|
||||
|
||||
The only command-specific flag is `--config <path>`, which is required.
|
||||
Positional arguments are rejected. Success prints `config is valid` to stdout
|
||||
and returns zero; usage, file-read, parse, and validation errors go to stderr and
|
||||
return `2`. See [applied configuration sources](config.md#effective-precedence)
|
||||
and [validation boundaries](config.md#examples).
|
||||
|
||||
### `config print-effective`
|
||||
|
||||
```sh
|
||||
audita config print-effective --config examples/minimal-config.yml
|
||||
```
|
||||
|
||||
The only command-specific flag is optional `--config <path>`. Positional
|
||||
arguments are rejected. Without an explicit file, [discovery](config.md#loading-model)
|
||||
applies. Success prints redacted configuration JSON to stdout and returns zero.
|
||||
Usage and configuration errors return `2`; serialization or stdout-write failures
|
||||
return `1`.
|
||||
|
||||
The printed JSON reflects the effective in-memory configuration; it is not a YAML
|
||||
configuration file and is not the process-report schema. Use it to inspect merged
|
||||
settings, not as a file to pass back to the YAML loader.
|
||||
|
||||
## Help
|
||||
|
||||
Root invocation without arguments and root help print usage successfully. Use
|
||||
`--help` or `-h` for subcommand help. Help normally returns zero; process help has
|
||||
the configuration-loading caveat described above. Unknown commands return `2`.
|
||||
188
docs/config.md
Normal file
188
docs/config.md
Normal file
@@ -0,0 +1,188 @@
|
||||
# Audita Configuration
|
||||
|
||||
This is the canonical reference for discovery, settings, defaults, environment
|
||||
mapping, and validation. [CLI syntax](cli.md), [artifact lifecycle](operations.md),
|
||||
and [file formats](integrations/output-files.md) have separate owners.
|
||||
|
||||
## Loading Model
|
||||
|
||||
For processing and effective-config inspection, the first applicable source
|
||||
selects one YAML file:
|
||||
|
||||
1. Explicit CLI config path.
|
||||
2. `AUDITA_CONFIG`.
|
||||
3. `/usr/local/etc/audita/config.yml`, if present.
|
||||
4. `/etc/audita/config.yml`, if present.
|
||||
|
||||
These files are alternatives, not layers merged together. Missing system files
|
||||
are skipped. An empty or missing explicitly selected path is an error, as is an
|
||||
inaccessible selected file. Without a selected file, built-in defaults apply.
|
||||
The validation command requires an explicit file and does not use discovery.
|
||||
|
||||
## Effective Precedence
|
||||
|
||||
| Operation | Applied sources, lowest to highest precedence |
|
||||
| --- | --- |
|
||||
| Process | Built-in defaults, selected file, environment overrides, explicit CLI overrides |
|
||||
| Print effective configuration | Built-in defaults, selected file, environment overrides |
|
||||
| Validate configuration file | Built-in defaults, explicit file |
|
||||
|
||||
File validation does not apply configuration environment overrides, but it still
|
||||
resolves environment variables named by `api_key_env` fields. Each configuration
|
||||
layer is validated as it is applied: a later override cannot repair an invalid
|
||||
earlier layer. [Config commands](cli.md#configuration-commands) provide inspection
|
||||
and validation without processing a transcript or contacting an LLM.
|
||||
|
||||
## File Schema
|
||||
|
||||
The file is YAML with required `version: 1`. Unknown fields are rejected.
|
||||
The remaining sections and fields below are optional. Omitted values preserve
|
||||
lower-precedence settings, except for the documented inheritance rules.
|
||||
Complete copyable files are in [examples](#examples).
|
||||
|
||||
### Pipeline and Output
|
||||
|
||||
| YAML field | Default | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `pipeline.modules` | `[glossary, homophones, glossary, spoken_word, grammar]` | Ordered module sequence. Supported keys are `glossary`, `homophones`, `spoken_word`, and `grammar`; repeated keys are allowed. An omitted or empty YAML list leaves the sequence unchanged. |
|
||||
| `output.schema` | `bare-segments` | Output encoder selection: `bare-segments` or `audita-v1`. See [output formats](integrations/output-files.md#corrected-transcript). |
|
||||
|
||||
Module keys select correction stages. Validator chains are built into each module;
|
||||
there is no configuration field for selecting individual validator keys.
|
||||
|
||||
### LLM Settings
|
||||
|
||||
The same fields are supported under `llm.proposal` and `llm.validation`.
|
||||
|
||||
| Field suffix | Proposal default | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `base_url` | `https://openrouter.ai/api/v1` | Endpoint base URL; the [LLM contract](integrations/openai-compatible-llm.md#endpoint-and-authentication) defines how it is used. |
|
||||
| `model` | `openrouter/google/gemma-4-31b-it` | Model identifier sent to the endpoint. Availability is determined by that endpoint. |
|
||||
| `api_key_env` | Unset | Name of an environment variable containing the API key; see [secrets](#secrets). |
|
||||
| `timeout` | `600` seconds | Positive integer seconds or a duration string resolving to whole seconds, such as `120s`. |
|
||||
| `max_retries` | `3` | Non-negative retry count after the initial attempt; retry conditions belong to the [LLM contract](integrations/openai-compatible-llm.md#retries-and-timeouts). |
|
||||
|
||||
Validation settings inherit the effective proposal settings individually. Empty
|
||||
validation model, base URL, or API key values inherit the proposal value; omitted
|
||||
validation timeout and retry count inherit, while an explicit retry count of zero
|
||||
is retained in configuration. The current HTTP adapter interprets a retry count
|
||||
of zero as its default of three retries, so zero does not disable retries.
|
||||
No API key is required by local config validation. Model and base-URL
|
||||
usability are checked during client setup/requests, not by an online config check.
|
||||
|
||||
### Concurrency
|
||||
|
||||
| YAML field | Default | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `concurrency.total_llm` | `1` | Positive total concurrent LLM-call limit. |
|
||||
| `concurrency.proposal_llm` | `1` | Positive proposal-call limit, no greater than total. |
|
||||
| `concurrency.validation_llm` | Inherits total | Positive validation-call limit, no greater than total. |
|
||||
|
||||
In a YAML file, setting total alone does not change proposal concurrency. At the
|
||||
environment or CLI layer, setting total without setting proposal at that same
|
||||
layer also sets proposal to that total, even if a lower layer supplied a proposal
|
||||
value. Unset validation concurrency inherits the final total. The canonical total
|
||||
setting wins over its legacy alias when both appear at the same layer.
|
||||
|
||||
Validation prompt budgeting is available through environment or CLI only:
|
||||
`AUDITA_VALIDATION_MAX_PROMPT_TOKENS` has default `2048` and must be positive.
|
||||
There is no YAML field for it.
|
||||
|
||||
### Chunking and Normalization
|
||||
|
||||
| YAML field | Default | Meaning and validation |
|
||||
| --- | --- | --- |
|
||||
| `chunking.target_sections` | Unset | Optional positive target count. Processing rejects targets impossible for the normalized transcript and token constraints. |
|
||||
| `chunking.max_section_tokens` | `8192` | Positive maximum section token estimate. |
|
||||
| `chunking.min_section_tokens` | `2048` | Positive minimum used in chunk balancing; no greater than maximum. |
|
||||
| `normalization.max_segment_gap` | `4.0` seconds | Non-negative gap limit for same-speaker merging. |
|
||||
| `normalization.ellipsis_gap` | `3.5` seconds | Non-negative gap threshold for inserting an ellipsis during merging. |
|
||||
| `normalization.max_segment_duration` | `60.0` seconds | Positive merged-segment duration limit. |
|
||||
| `normalization.max_segment_tokens` | `2048` | Positive merged-segment token estimate limit. |
|
||||
|
||||
The three normalization time fields accept numeric seconds or YAML duration
|
||||
strings. Their environment and CLI overrides use numeric seconds. Section token
|
||||
counts are estimates; config validation does not test whether a particular input
|
||||
can satisfy the selected chunking target.
|
||||
|
||||
### Thresholds, Context, and Diagnostics
|
||||
|
||||
| YAML field | Default | Meaning and validation |
|
||||
| --- | --- | --- |
|
||||
| `thresholds.glossary` | `0.8` | Glossary correction confidence threshold, in `[0, 1]`. |
|
||||
| `thresholds.homophones` | `0.8` | Homophone correction confidence threshold, in `[0, 1]`. |
|
||||
| `thresholds.spoken_word` | `0.8` | Spoken-word correction confidence threshold, in `[0, 1]`. |
|
||||
| `thresholds.grammar` | `0.8` | Grammar correction confidence threshold, in `[0, 1]`. |
|
||||
| `context.description` | Empty | Trimmed background context for prompts; limited to 500 bytes of UTF-8 text after trimming. It does not replace transcript content. |
|
||||
| `diagnostics.work_dir` | `/tmp/audita` | Non-empty parent directory for run artifacts. |
|
||||
| `diagnostics.retention` | `auto` | Accepted values: `auto`, `always`, `never`. Exact lifecycle behavior, including the current behavior of `never`, belongs to [operations](operations.md#retention-behavior). |
|
||||
|
||||
## Environment Overrides
|
||||
|
||||
Values are read when the variable is set. Empty values are not generally equivalent
|
||||
to unset values: numeric overrides must parse, and empty strings may clear a
|
||||
setting or fail validation. Numeric environment values use integers or floating
|
||||
point seconds as indicated above, not duration strings.
|
||||
|
||||
| Environment variable | Setting |
|
||||
| --- | --- |
|
||||
| `AUDITA_CONFIG` | Config file selection |
|
||||
| `AUDITA_MODULES` | Module sequence as comma-separated keys |
|
||||
| `AUDITA_LLM_API_KEY` | Direct proposal API key |
|
||||
| `OPENROUTER_API_KEY` | Proposal API key fallback when `AUDITA_LLM_API_KEY` is unset |
|
||||
| `AUDITA_MODEL` | Proposal model |
|
||||
| `AUDITA_BASE_URL` | Proposal endpoint |
|
||||
| `AUDITA_LLM_TIMEOUT_SECONDS` | Proposal timeout |
|
||||
| `AUDITA_MAX_RETRIES` | Proposal retries |
|
||||
| `AUDITA_VALIDATION_LLM_API_KEY` | Direct validation API key |
|
||||
| `AUDITA_VALIDATION_MODEL` | Validation model |
|
||||
| `AUDITA_VALIDATION_BASE_URL` | Validation endpoint |
|
||||
| `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | Validation timeout |
|
||||
| `AUDITA_VALIDATION_MAX_RETRIES` | Validation retries |
|
||||
| `AUDITA_TOTAL_LLM_CONCURRENCY` | Total concurrency |
|
||||
| `AUDITA_LLM_CONCURRENCY` | Legacy total-concurrency alias |
|
||||
| `AUDITA_PROPOSAL_LLM_CONCURRENCY` | Proposal concurrency |
|
||||
| `AUDITA_VALIDATION_LLM_CONCURRENCY` | Validation concurrency |
|
||||
| `AUDITA_VALIDATION_MAX_PROMPT_TOKENS` | Validation prompt budget |
|
||||
| `AUDITA_TARGET_SECTIONS` | Section target |
|
||||
| `AUDITA_MAX_SECTION_TOKENS` | Section maximum |
|
||||
| `AUDITA_MIN_SECTION_TOKENS` | Section minimum |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | Merge gap |
|
||||
| `AUDITA_NORMALIZE_ELLIPSIS_GAP` | Ellipsis gap |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | Merged duration limit |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | Merged token estimate limit |
|
||||
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | Glossary threshold |
|
||||
| `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | Homophone threshold |
|
||||
| `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | Spoken-word threshold |
|
||||
| `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | Grammar threshold |
|
||||
| `AUDITA_WORK_DIR` | Diagnostics directory |
|
||||
| `AUDITA_WORK_DIR_RETENTION` | Retention selection |
|
||||
|
||||
There is no environment override for output schema or transcript description.
|
||||
Use file configuration or the corresponding [CLI flags](cli.md#process-flag-reference).
|
||||
|
||||
## Secrets
|
||||
|
||||
File config accepts environment variable names through `api_key_env`, not literal
|
||||
API-key fields. Names must be non-empty and match an environment-variable
|
||||
identifier; an unset named variable resolves to an empty value. Runtime environment
|
||||
key overrides take precedence over keys resolved from a file. CLI key flags supply
|
||||
direct values and take precedence over environment settings.
|
||||
|
||||
Effective-config inspection redacts API keys. Configured LLM secrets are also
|
||||
redacted by the runtime's error and diagnostic handling, but transcript content
|
||||
is still sensitive: see [artifact handling](operations.md#artifact-handling).
|
||||
|
||||
## Examples
|
||||
|
||||
- [Minimal configuration](../examples/minimal-config.yml): starts from defaults
|
||||
and obtains the proposal key from the environment.
|
||||
- [Production-oriented configuration](../examples/production-config.yml): illustrates
|
||||
both LLM targets, concurrency, chunking, normalization, thresholds, and diagnostics.
|
||||
Its explicit section target is intended for larger inputs, not the tiny sample.
|
||||
- [Tiny transcript](../examples/tiny-transcript.json) and
|
||||
[tiny glossary](../examples/tiny-glossary.yaml): inputs for the README quickstart.
|
||||
|
||||
Use the [CLI configuration commands](cli.md#configuration-commands) to validate
|
||||
files and inspect effective settings. Validation checks local structure and
|
||||
constraints; it does not verify credentials, model availability, or endpoint access.
|
||||
49
docs/development.md
Normal file
49
docs/development.md
Normal file
@@ -0,0 +1,49 @@
|
||||
# Developing Audita
|
||||
|
||||
Audita is a Go CLI for transcript polishing. Start with the
|
||||
[README](../README.md) for product orientation and the
|
||||
[internal overview](internal/overview.md) for the implemented component map.
|
||||
|
||||
## Before Changing the Repository
|
||||
|
||||
Coding agents must read and follow the [agent policy](policy/agents.md) before
|
||||
recommending, planning, or implementing changes. Read the
|
||||
[architecture policy](policy/architecture.md) for boundaries and invariants,
|
||||
then use the task guide below rather than reading every reference indiscriminately.
|
||||
|
||||
Confirm the requested scope and behavior contract, make focused changes in the
|
||||
owning component, and update the relevant tests and canonical documentation
|
||||
when behavior changes. The [testing policy](policy/testing.md) governs test
|
||||
sufficiency; the [documentation policy](policy/documentation.md) governs content
|
||||
ownership. Commit discipline and agent Git restrictions are in the
|
||||
[agent policy](policy/agents.md#git-commit-policy).
|
||||
|
||||
## Baseline Validation
|
||||
|
||||
Use Go 1.24 or newer. From the repository root:
|
||||
|
||||
```sh
|
||||
go build -o ./bin/audita ./cmd/audita
|
||||
go test ./...
|
||||
```
|
||||
|
||||
Run targeted package tests for touched behavior and the complete suite for
|
||||
substantial changes. Documentation-only work normally needs affected-link,
|
||||
example, and whitespace checks; use `git diff --check`. Detailed component
|
||||
validation belongs with the change recipes below.
|
||||
|
||||
## Task-Specific Reading Guide
|
||||
|
||||
| Task | Read |
|
||||
| --- | --- |
|
||||
| Locate components or change orchestration | [Internal overview](internal/overview.md), then [pipeline](internal/pipeline.md) |
|
||||
| Change configuration or CLI behavior | [Configuration contract](config.md), [CLI contract](cli.md), and [configuration/CLI change recipes](internal/cli-config.md) |
|
||||
| Change modules or validators | [Modules](internal/modules.md) or [validators](internal/validators.md); follow their prompt, registry, and test references |
|
||||
| Change LLM transport or concurrency | [LLM integration contract](integrations/openai-compatible-llm.md), [LLM runtime](internal/llm-runtime.md), and [pipeline](internal/pipeline.md) |
|
||||
| Change prompts | [Prompt registry](internal/prompts.md) and the consuming [module](internal/modules.md) or [validator](internal/validators.md) |
|
||||
| Change input/output formats | [Input file contracts](integrations/transcript-glossary-files.md), [output-file contract](integrations/output-files.md), [encoder internals](internal/output-schemas.md), and [configuration](config.md) |
|
||||
| Change reports or diagnostics | [Operations](operations.md), [output-file contract](integrations/output-files.md), [subprocess integration](integrations/subprocess.md), and [diagnostics/reporting internals](internal/diagnostics-reporting.md) |
|
||||
| Change tests | [Testing policy](policy/testing.md) and the affected component's internal documentation |
|
||||
| Change docs or examples | [Documentation policy](policy/documentation.md), the canonical topic owner, and [example validation](internal/cli-config.md#validating-examples) |
|
||||
| Prepare a release | [Release policy](policy/release.md) and [release procedure](release.md) |
|
||||
| Record a significant architectural decision | [ADR lifecycle](policy/documentation.md#architecture-decision-records) and [ADR index](adr/README.md) |
|
||||
76
docs/integrations/openai-compatible-llm.md
Normal file
76
docs/integrations/openai-compatible-llm.md
Normal file
@@ -0,0 +1,76 @@
|
||||
# OpenAI-Compatible LLM Integration
|
||||
|
||||
This document owns the external protocol used by Audita's structured LLM adapter.
|
||||
Settings, inheritance, defaults, and credential supply belong to
|
||||
[configuration](../config.md#llm-settings); implementation and scheduling belong to
|
||||
[LLM runtime internals](../internal/llm-runtime.md).
|
||||
|
||||
## Endpoint and Authentication
|
||||
|
||||
Audita sends an HTTP `POST` to the configured base URL with trailing slashes
|
||||
removed and `/chat/completions` appended. Supply an HTTP or HTTPS base URL that
|
||||
makes this the intended endpoint; the adapter does not require HTTPS in code.
|
||||
Requests have `Content-Type: application/json`. A non-empty configured API key
|
||||
adds `Authorization: Bearer <api_key>`; otherwise that header is omitted.
|
||||
|
||||
The endpoint must support the configured model and structured JSON-schema
|
||||
responses. Local configuration validation does not probe that support.
|
||||
|
||||
## Request Shape
|
||||
|
||||
The payload contains `model`, an array of `messages` with string `role`/`content`,
|
||||
and `response_format`. The structured-output portion has this illustrative shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "correction_set",
|
||||
"strict": true,
|
||||
"schema": {"type": "object"}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The actual schema is supplied for the call; the fragment above is not a complete
|
||||
request schema. Runtime calls use `correction_set` and `validator_decision_set`
|
||||
identities. The adapter requires a non-empty resolved model, non-empty message
|
||||
roles/content, a schema name, and valid schema JSON. It sends strict mode to the
|
||||
provider; local typed decoding and proposal/decision checks still apply afterward.
|
||||
|
||||
## Response Handling Contract
|
||||
|
||||
A successful response must be a JSON envelope with at least one `choices` item.
|
||||
Audita reads the first choice's `message.content`, accepting a non-empty string
|
||||
containing JSON or a non-null raw JSON value that can decode into the expected
|
||||
output type. Empty content, missing choices, and incompatible payloads fail.
|
||||
The adapter also reads optional `model` and token-usage metadata.
|
||||
|
||||
There is no streaming protocol or provider SDK dependency in this integration.
|
||||
Audita uses this chat-completions contract, not other provider API shapes.
|
||||
|
||||
## Retries and Timeouts
|
||||
|
||||
The effective adapter retry count is additional attempts after the initial request;
|
||||
see [configuration](../config.md#llm-settings) for the current zero-value behavior.
|
||||
Transport/read errors, malformed response envelopes/content, typed decode failures,
|
||||
HTTP `429`, and server-error responses are retryable. Other non-success HTTP
|
||||
responses and a response missing choices are not retried by the adapter.
|
||||
Cancellation or an expired context stops retrying.
|
||||
|
||||
The configured HTTP timeout applies per request; it is not a whole-process time
|
||||
budget. A parent process can impose its own overall timeout. See
|
||||
[configuration](../config.md#llm-settings) for timeout and retry values and
|
||||
[subprocess guidance](subprocess.md) for parent responsibilities.
|
||||
|
||||
## Error Behavior
|
||||
|
||||
For non-success responses, error detail comes from provider `error.message`, then
|
||||
a top-level `message`, otherwise the HTTP status and response body. Configured
|
||||
keys are redacted from surfaced errors and LLM diagnostic payloads.
|
||||
|
||||
Malformed structured responses can become section/batch warnings and rejected
|
||||
proposals rather than fatal run errors, depending on the call site. Other adapter
|
||||
failures can terminate processing with partial results. Consult the
|
||||
[report contract](output-files.md#module-outcomes) for those outcomes and
|
||||
[operations](../operations.md#artifact-handling) for handling diagnostic contents.
|
||||
186
docs/integrations/output-files.md
Normal file
186
docs/integrations/output-files.md
Normal file
@@ -0,0 +1,186 @@
|
||||
# Output Files and Report Formats
|
||||
|
||||
This document owns externally consumed corrected-transcript and report formats,
|
||||
and describes the correction ledger and utilization diagnostics. It documents
|
||||
current serialized shapes without making every debugging artifact a versioned
|
||||
compatibility promise. Physical filenames, placement, retention, and permissions
|
||||
belong to [operations](../operations.md); invocation and output channels belong to
|
||||
the [CLI](../cli.md#process-output-and-exit-behavior).
|
||||
|
||||
## Corrected Transcript
|
||||
|
||||
The [output setting](../config.md#pipeline-and-output) selects the envelope:
|
||||
|
||||
| Encoder | JSON shape |
|
||||
| --- | --- |
|
||||
| `bare-segments` | Top-level array of segment objects |
|
||||
| `audita-v1` | Object with `schema: "audita-v1"`, `version: "v1"`, and `segments` containing the segment array |
|
||||
|
||||
Both emit these segment fields:
|
||||
|
||||
| Field | Type | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `id` | Integer | Identifier in the normalized transcript |
|
||||
| `speaker` | String | Speaker label |
|
||||
| `start`, `end` | Numbers | Start/end times in seconds |
|
||||
| `text` | String | Text after normalization and accepted corrections |
|
||||
| `categories` | Array of strings, optional | Segment categories; omitted when empty |
|
||||
|
||||
Input IDs and segment boundaries need not survive normalization unchanged;
|
||||
normalization can merge segments and reassign IDs. The output describes the
|
||||
resulting transcript, not a lossless copy of the source JSON. Input acceptance
|
||||
rules belong to the [input-file contract](transcript-glossary-files.md).
|
||||
|
||||
## Process Report
|
||||
|
||||
The report is a JSON object. `report_metadata` identifies its schema separately
|
||||
from the transcript output envelope:
|
||||
|
||||
| Metadata field | Type / current value |
|
||||
| --- | --- |
|
||||
| `report_schema_name` | String: `audita-process-report` |
|
||||
| `report_schema_version` | String: `v1` |
|
||||
| `output_schema` | String identifying the selected transcript encoder |
|
||||
| `config_version` | Optional integer, present when file configuration was loaded |
|
||||
|
||||
The remaining top-level fields are:
|
||||
|
||||
| Field | Type and meaning |
|
||||
| --- | --- |
|
||||
| `phase` | String; currently `default_pipeline` |
|
||||
| `status` | String: `success` or `failed` |
|
||||
| `operation` | String; currently `process` |
|
||||
| `transcript_path`, `glossary_path` | Input path strings |
|
||||
| `output_path` | Optional transcript destination string |
|
||||
| `modules` | Array of configured module keys, including repeats |
|
||||
| `started_at`, `completed_at` | RFC 3339 timestamps, possibly fractional; completion is an optional field populated by completed process runs |
|
||||
| `error_phase`, `error_message` | Optional failure classification and human-readable detail |
|
||||
| `input_segment_count`, `normalized_segment_count` | Optional integer normalization counts |
|
||||
| `normalization_merges`, `normalization_id_reassignments` | Optional integer transformation counts |
|
||||
| `normalization_skipped` | Object of optional integer counters: `different_speakers`, `gap_too_large`, `duration_exceeded`, `token_limit_exceeded`; can be empty |
|
||||
| `chunking` | Optional section summary described below |
|
||||
| `diagnostics` | Optional artifact-path metadata described below |
|
||||
| `modules_summary` | Optional aggregate module outcome |
|
||||
| `module_results` | Optional array of module outcomes |
|
||||
|
||||
Failures can preserve partial module results. Missing summaries indicate unavailable
|
||||
information, not necessarily zero work. Early configuration failures occur before
|
||||
report construction. Report success describes processing results; a later file-write
|
||||
or cleanup error can still cause a nonzero CLI exit. Use the exit result and report
|
||||
together as explained in [subprocess guidance](subprocess.md).
|
||||
|
||||
Error phases identify where processing failed, for example `transcript_read`,
|
||||
`glossary_read`, `transcript_schema`, `glossary_schema`, `chunking`, `runner_setup`,
|
||||
`runner_execution`, `output_schema`, `serialization`, `output_write`, and
|
||||
`stdout_write`. This is not an exhaustive enumeration or a promise that every
|
||||
late failure is represented in a successfully written report. Error messages are
|
||||
human-readable, not structured fields to parse for control flow.
|
||||
|
||||
### Chunking and Diagnostics Metadata
|
||||
|
||||
`chunking` contains integer fields `chunk_count`, `min_estimated_chunk_tokens`,
|
||||
`max_estimated_chunk_tokens`, `total_estimated_transcript_tokens`,
|
||||
`max_section_tokens`, and `min_section_tokens`, plus optional integer
|
||||
`target_sections`. Token counts are estimates.
|
||||
|
||||
`diagnostics` contains optional path strings:
|
||||
|
||||
- `directory_path`
|
||||
- `source_transcript_path`, `parsed_source_transcript_path`, `normalized_transcript_path`
|
||||
- `normalization_summary_path`, `chunking_summary_path`
|
||||
- `utilization_summary_path`, `correction_ledger_path`
|
||||
- `invocation_metadata_path`, `redacted_effective_config_path`, `error_log_path`
|
||||
|
||||
The [operations guide](../operations.md#diagnostics-directory) defines the physical
|
||||
artifacts and path-lifetime caveats.
|
||||
|
||||
### Module Outcomes
|
||||
|
||||
`modules_summary` contains integer `module_count`, `total_applied_changes`, and
|
||||
`total_skipped_changes`, plus optional string `failed_module_instance`.
|
||||
`total_skipped_changes` counts both apply-time skipped changes and validator
|
||||
rejections. Their detailed records are stored in separate arrays.
|
||||
|
||||
Each `module_results` object contains:
|
||||
|
||||
| Field | Type and meaning |
|
||||
| --- | --- |
|
||||
| `module_key`, `module_instance` | Strings; the instance distinguishes repeated occurrences of a configured module |
|
||||
| `replacement_policy` | Optional string describing the module's replacement policy |
|
||||
| `status` | String describing module outcome (`success` or `failed`) |
|
||||
| `proposal_count` | Integer |
|
||||
| `started_at`, `completed_at` | Optional timestamps |
|
||||
| `error_message` | Optional failure detail |
|
||||
| `warnings` | Optional array of warning records |
|
||||
| `validator_decisions` | Optional array of decision records |
|
||||
| `validator_rejected` | Optional array of rejected-correction records |
|
||||
| `applied_changes`, `skipped_changes` | Optional arrays of apply outcomes |
|
||||
|
||||
A validator decision has `validator_name` (string), `proposal_index` (integer),
|
||||
`approved` (boolean), `reason_code` and `message` (strings), and optional
|
||||
`diagnostic_artifact_path` (string).
|
||||
|
||||
Rejected corrections include `validator_name`, `proposal_index`, `module_key`,
|
||||
`module_instance`, `target_segment_id`, `original_text`, `corrected_text`,
|
||||
`reason_code`, and `message`. Applied changes include the same correction identity
|
||||
and text fields without validator/reason/message fields, plus integer
|
||||
`replacement_count`. Skipped changes include correction identity/text fields,
|
||||
`skip_reason`, and `message`.
|
||||
|
||||
A warning has string `scope`, `reason_code`, and `message`; optional integer
|
||||
`section_index` and `batch_index`; and optional string `validator_name` and
|
||||
`diagnostic_artifact_path`. Warnings, skipped changes, and rejected corrections
|
||||
can occur without a failed run. Proposal indexes associate outcomes with their
|
||||
proposal within a module instance; they are not transcript segment IDs.
|
||||
|
||||
## Correction Ledger
|
||||
|
||||
The ledger is an array of correction entries; it can be `null` when no module
|
||||
results are available and empty when modules produced no entries. Unlike the
|
||||
process report, it has no separate schema-version envelope.
|
||||
|
||||
Each entry has required `module_key`, `module_instance`, integer `proposal_index`,
|
||||
and `disposition`. The current dispositions are:
|
||||
|
||||
| Disposition | Meaning |
|
||||
| --- | --- |
|
||||
| `applied` | The proposed change was applied |
|
||||
| `skipped` | The proposal reached application but was skipped |
|
||||
| `rejected` | A validator rejected the proposal |
|
||||
| `failed` | Module-level failure record, not necessarily an individual proposal |
|
||||
|
||||
Optional fields are `run_id`, integer `segment_id`, `original_text`,
|
||||
`proposed_corrected_text`, `applied_corrected_text`, `replacement_policy`,
|
||||
`disposition_reason_code`, and `disposition_message`. Optional validator arrays
|
||||
are `deterministic_validator_decisions` and `llm_validator_decisions`; each record
|
||||
contains string `validator_key`, boolean `approved`, string `reason_code`, and
|
||||
optional string `message`.
|
||||
|
||||
Module failure entries can have the zero-value proposal index and no correction
|
||||
text or segment. Do not interpret them as evidence that proposal zero failed.
|
||||
Empty optional values are omitted. The ledger records disposition, not a replay
|
||||
or resume protocol.
|
||||
|
||||
## Utilization Diagnostics
|
||||
|
||||
This unversioned JSON object describes observations from the runtime, not service
|
||||
limits or performance guarantees:
|
||||
|
||||
- `effective_concurrency`: integer `total_llm`, `proposal_llm`, `validation_llm`.
|
||||
- `run_timing`: millisecond integers `run_wall_time_ms`, `scheduler_queue_wait_ms`,
|
||||
`llm_execution_time_ms`, `deterministic_validation_time_ms`; integer
|
||||
`max_in_flight_llm_calls` and `average_in_flight_llm_calls`.
|
||||
- `llm_calls`: integer `total_proposal_calls` and `total_validation_calls`.
|
||||
- Optional `modules`: records with `module_key`, `module_instance`, integer
|
||||
`proposal_llm_calls` and `validation_llm_calls`, and millisecond integer
|
||||
`scheduler_queue_wait_ms`, `llm_execution_time_ms`,
|
||||
`deterministic_validation_time_ms`, and `module_wall_time_ms`.
|
||||
- Optional `validators`: records with `module_key`, `module_instance`,
|
||||
`validator_key`, boolean `llm_backed`, integer `calls`, and millisecond integer
|
||||
`elapsed_ms`.
|
||||
|
||||
Aggregated execution/wait durations can overlap in concurrent work; they need
|
||||
not sum to wall time. The observed average concurrency is serialized as an integer.
|
||||
Other source snapshots, prompt payloads, and diagnostic files are debugging
|
||||
material; their placement is documented in operations without defining a separate
|
||||
public schema for every file.
|
||||
40
docs/integrations/subprocess.md
Normal file
40
docs/integrations/subprocess.md
Normal file
@@ -0,0 +1,40 @@
|
||||
# Subprocess Integration
|
||||
|
||||
This guide explains how a parent process can invoke Audita and consume its results.
|
||||
The [CLI reference](../cli.md) owns command syntax, streams, and exit semantics;
|
||||
[configuration](../config.md) owns discovery and precedence.
|
||||
|
||||
## Invocation
|
||||
|
||||
Use an argument array with the transcript path and required glossary argument,
|
||||
following the [process examples](../cli.md#process-examples). For machine workflows,
|
||||
select explicit transcript and report destinations rather than depending on the
|
||||
working directory or run-directory retention. Supplying a configuration path also
|
||||
makes file selection predictable; inherited environment overrides still apply.
|
||||
|
||||
Resolve the executable deliberately and set the child's working directory so
|
||||
relative input/output paths and reported artifact paths have a known base.
|
||||
Read stdout and stderr concurrently to avoid pipe blocking, and impose timeout
|
||||
or cancellation in the parent process when needed.
|
||||
|
||||
## Result Handling
|
||||
|
||||
Interpret the exit result according to the
|
||||
[CLI output and exit contract](../cli.md#process-output-and-exit-behavior).
|
||||
Consume transcript/report data through the
|
||||
[output-file contract](output-files.md), and treat stderr as human-readable detail.
|
||||
The report supplements the exit result: it may be absent after early failures,
|
||||
and output may have been written before a later reporting or cleanup failure.
|
||||
|
||||
If processing fails, preserve available evidence and follow
|
||||
[operations recovery](../operations.md#failure-inspection-and-recovery).
|
||||
For reports retained outside the run directory, account for the
|
||||
[artifact lifecycle](../operations.md#retention-behavior) before dereferencing
|
||||
reported paths. Neither the existence of an output file nor a diagnostics path
|
||||
alone establishes successful completion.
|
||||
|
||||
## Related Contracts
|
||||
|
||||
- [Transcript and glossary inputs](transcript-glossary-files.md)
|
||||
- [LLM endpoint compatibility](openai-compatible-llm.md)
|
||||
- [Configuration inspection commands](../cli.md#configuration-commands)
|
||||
50
docs/integrations/transcript-glossary-files.md
Normal file
50
docs/integrations/transcript-glossary-files.md
Normal file
@@ -0,0 +1,50 @@
|
||||
# Transcript and Glossary Input Files
|
||||
|
||||
This document owns accepted input formats. Use the [CLI reference](../cli.md) for
|
||||
invocation and error exit semantics, [operations](../operations.md) for artifact
|
||||
handling, and [output files](output-files.md) for corrected-transcript serialization.
|
||||
|
||||
## Transcript JSON Contract
|
||||
|
||||
Accepted top-level shapes are an array of segments or an object containing a
|
||||
`segments` array. The array must contain at least one segment. Unknown object
|
||||
fields are ignored by the current parser; an input envelope does not require a
|
||||
schema/version marker.
|
||||
|
||||
| Segment field | Type and validation |
|
||||
| --- | --- |
|
||||
| `id` | Optional integer (or null). Supplied IDs must be unique; missing IDs are accepted. |
|
||||
| `speaker` | Non-empty string |
|
||||
| `start`, `end` | Numeric seconds, finite and non-negative; end must be at least start |
|
||||
| `text` | Non-empty string |
|
||||
| `categories` | Optional array of non-empty strings |
|
||||
|
||||
Supply explicit start and end times for meaningful timing. The current decoder
|
||||
uses zero for missing or null timestamps rather than enforcing their presence.
|
||||
String validation checks empty values, not whitespace trimming. Normalization can
|
||||
merge segments and reassign IDs, so source IDs need not match output IDs.
|
||||
|
||||
The [tiny transcript](../../examples/tiny-transcript.json) is a complete copyable
|
||||
input. Invalid JSON, unsupported top-level shapes, empty transcripts, invalid field
|
||||
types, duplicate supplied IDs, and the validation failures above are rejected.
|
||||
|
||||
## Glossary YAML Contract
|
||||
|
||||
The top-level object contains a `glossary` list with at least one entry.
|
||||
|
||||
| Entry field | Type and validation |
|
||||
| --- | --- |
|
||||
| `name` | Non-empty string |
|
||||
| `category` | Non-empty string |
|
||||
| `summary` | Non-empty string |
|
||||
| `aliases` | Optional list of non-empty strings |
|
||||
| `plural` | Optional string |
|
||||
|
||||
The parser checks empty strings without trimming them. Unknown YAML fields are
|
||||
ignored by the glossary parser, unlike the stricter
|
||||
[configuration-file schema](../config.md#file-schema).
|
||||
|
||||
The [tiny glossary](../../examples/tiny-glossary.yaml) is a complete copyable input.
|
||||
Malformed YAML, an empty/missing glossary, invalid shapes, missing required text,
|
||||
and empty aliases are rejected. These formats describe input acceptance, not
|
||||
additional vocabulary restrictions.
|
||||
64
docs/internal/cli-config.md
Normal file
64
docs/internal/cli-config.md
Normal file
@@ -0,0 +1,64 @@
|
||||
# Configuration and CLI Change Recipes
|
||||
|
||||
This document owns contributor editing and validation steps for configuration,
|
||||
CLI wiring, and maintained examples. Definitions of behavior belong to the
|
||||
[configuration](../config.md) and [CLI](../cli.md) contracts.
|
||||
|
||||
## Changing Configuration
|
||||
|
||||
1. Update fields, defaults, loading, and validation in `internal/core/config`.
|
||||
2. Update file, environment, and CLI mapping as applicable, including CLI flag
|
||||
extraction. Preserve the source-specific behavior defined in the
|
||||
[configuration contract](../config.md#effective-precedence), including the
|
||||
distinct validation and effective-config commands.
|
||||
3. Update redaction when secrets are involved.
|
||||
4. Review configuration tests and related CLI tests for precedence, validation,
|
||||
and source-specific behavior. Add or adjust tests for the changed contract
|
||||
according to the [testing policy](../policy/testing.md).
|
||||
5. Update the configuration reference and affected [examples](../../examples/).
|
||||
|
||||
## Changing CLI Behavior
|
||||
|
||||
1. Implement parsing and wiring in `internal/cli`.
|
||||
2. Preserve [stream and exit contracts](../cli.md) unless a change is intentional
|
||||
and documented. Review [subprocess integration](../integrations/subprocess.md)
|
||||
when changing output, error, or report handling.
|
||||
3. Review `internal/cli` tests and `cmd/audita` subprocess tests; update coverage
|
||||
for the changed behavior.
|
||||
4. Update the CLI reference and affected integration guidance.
|
||||
|
||||
## Validating Examples
|
||||
|
||||
Keep complete copyable files in [examples](../../examples/) and link to them from
|
||||
their canonical contract documentation. Keep them valid and secret-free.
|
||||
From the repository root, validate the maintained configuration examples:
|
||||
|
||||
```sh
|
||||
go run ./cmd/audita config validate --config examples/minimal-config.yml
|
||||
go run ./cmd/audita config validate --config examples/production-config.yml
|
||||
go run ./cmd/audita config print-effective --config examples/minimal-config.yml
|
||||
```
|
||||
|
||||
The effective-config command incorporates environment settings; use a controlled
|
||||
environment when comparing its results. Check affected CLI examples against the
|
||||
parser and use schema tests for maintained transcript/glossary inputs. Example
|
||||
validation should not invoke a live LLM merely to check documentation.
|
||||
|
||||
## Runtime Change Checks
|
||||
|
||||
Select checks according to the [testing policy](../policy/testing.md):
|
||||
|
||||
- Configuration or CLI changes: inspect `internal/core/config` tests,
|
||||
`internal/cli` tests, and `cmd/audita` integration tests.
|
||||
- Process/output changes: verify stdout, stderr, exit, output-file, and report
|
||||
behavior against the [CLI contract](../cli.md) and
|
||||
[subprocess guidance](../integrations/subprocess.md), including failures.
|
||||
- Report or diagnostics changes: review [report assembly tests and redaction
|
||||
checks](diagnostics-reporting.md#key-tests), metadata fields, and artifact paths.
|
||||
- Schema or serialization changes: review [output-schema tests](output-schemas.md#key-tests)
|
||||
and `internal/core/schema` tests, including supported formats and rejection of
|
||||
unsupported schemas.
|
||||
- Changes affecting assembled output: inspect CLI parity and release fixtures.
|
||||
|
||||
Use the [contributor guide](../development.md#baseline-validation) for baseline
|
||||
commands. These recipes supplement, rather than duplicate, public contracts.
|
||||
74
docs/internal/diagnostics-reporting.md
Normal file
74
docs/internal/diagnostics-reporting.md
Normal file
@@ -0,0 +1,74 @@
|
||||
# Diagnostics and Report Assembly
|
||||
|
||||
## Scope
|
||||
|
||||
This document owns implementation flow and collaborators. Physical artifacts and
|
||||
retention belong to [operations](../operations.md). Serialized reports, ledger
|
||||
records, and utilization shapes belong to the
|
||||
[output-file contract](../integrations/output-files.md).
|
||||
|
||||
## Run Directory and Artifact Writers
|
||||
|
||||
`internal/core/diagnostics` creates run directories, writes snapshots and metadata,
|
||||
and evaluates retention. CLI orchestration supplies the selected configuration,
|
||||
paths, run outcome, and correction-presence information. Artifact names are
|
||||
centralized in diagnostics; their operator-facing inventory is in
|
||||
[operations](../operations.md#diagnostics-directory).
|
||||
|
||||
Snapshot writes are attempted at the relevant processing phases. Several diagnostic
|
||||
writes are best-effort, so report metadata can identify a location without proving
|
||||
that a file was written. The CLI distinguishes primary processing errors from
|
||||
secondary diagnostic/report write errors.
|
||||
|
||||
## Process Report Mapping
|
||||
|
||||
`internal/framework/processreport.Build` maps invocation facts, normalization and
|
||||
chunking summaries, and runner output into `reporting.ProcessReport`. Core reporting
|
||||
owns the Go data structures and JSON writer. The builder attaches diagnostic
|
||||
metadata when a run-directory path is available.
|
||||
|
||||
Module mapping copies outcomes from runner results, collects summary counts, and
|
||||
preserves available partial module results on failure. The CLI failure path does
|
||||
not pass normalization or chunking summaries to report construction even when
|
||||
some earlier processing completed. Consumer interpretation belongs to the
|
||||
[process-report contract](../integrations/output-files.md#process-report).
|
||||
|
||||
## Correction Ledger and Utilization
|
||||
|
||||
`BuildCorrectionLedger` flattens module changes, rejections, and failures into
|
||||
ledger entries. It associates validator records by proposal index within each
|
||||
module, separates execution classes through validator metadata, and sorts entries
|
||||
by module instance, proposal index, and disposition.
|
||||
|
||||
Runner instrumentation collects utilization; CLI orchestration writes it when
|
||||
run output is available. Report assembly does not independently measure calls.
|
||||
For the serialized forms see [ledger](../integrations/output-files.md#correction-ledger)
|
||||
and [utilization](../integrations/output-files.md#utilization-diagnostics).
|
||||
|
||||
## CLI Coordination and Retention
|
||||
|
||||
CLI orchestration writes caller-selected reports and run-local artifacts before
|
||||
asking diagnostics to apply retention. `HasSkippedCorrections` derives the
|
||||
retention input from runner outcomes. Keep the exact lifecycle definition in
|
||||
[operations](../operations.md#retention-behavior), not in another internal table.
|
||||
|
||||
On a processing failure, secondary report-write errors do not replace the original
|
||||
error. On the success path, an external report-write failure returns before the
|
||||
run-local report write; retention errors occur later. This ordering explains why
|
||||
artifacts and process exit status must be assessed together.
|
||||
|
||||
## Redaction
|
||||
|
||||
Effective configuration uses config redaction. LLM diagnostic and error paths
|
||||
use `llm.ConfiguredSecrets` and the runtime's redaction helpers. Keep these separate
|
||||
mechanisms aligned with the [architecture requirement](../policy/architecture.md#errors-and-secret-handling).
|
||||
Operational handling of transcript-bearing artifacts belongs to
|
||||
[operations](../operations.md#artifact-handling).
|
||||
|
||||
## Key Tests
|
||||
|
||||
- `internal/core/diagnostics/*_test.go`
|
||||
- `internal/framework/processreport/*_test.go`
|
||||
- `internal/core/reporting/report_test.go`
|
||||
- `internal/cli/run_test.go`
|
||||
- `cmd/audita/main_integration_test.go`
|
||||
68
docs/internal/llm-runtime.md
Normal file
68
docs/internal/llm-runtime.md
Normal file
@@ -0,0 +1,68 @@
|
||||
# Audita LLM Runtime
|
||||
|
||||
## Scope
|
||||
|
||||
This document describes the structured LLM runtime and scheduler behavior.
|
||||
|
||||
## Client Boundary
|
||||
|
||||
All runtime LLM calls go through `contracts.StructuredLLMClient`.
|
||||
|
||||
Primary adapter:
|
||||
- `internal/framework/llm/OpenAICompatibleClient`
|
||||
|
||||
## Adapter and Response Schema Registry
|
||||
|
||||
The adapter implements the [LLM wire contract](../integrations/openai-compatible-llm.md).
|
||||
It maps internal structured requests to HTTP messages and decodes provider content
|
||||
into the supplied Go output target. Endpoint, authentication, response shapes, and
|
||||
retry conditions are defined in that contract rather than repeated here.
|
||||
|
||||
`internal/framework/responseschema` stores the per-call response schema definitions
|
||||
and diagnostic metadata. Proposal generation and validation choose their schema
|
||||
before invoking the client. Provider strict mode does not replace local checks.
|
||||
|
||||
## Error Classification
|
||||
|
||||
The adapter marks retryable errors internally and uses context state and attempt
|
||||
count to decide whether to retry. It sanitizes errors before returning them.
|
||||
`internal/framework/structuredoutput` classifies malformed output for proposal
|
||||
and validator call sites, which can turn those errors into warnings and rejected
|
||||
section/batch work. Other errors propagate through the runner.
|
||||
|
||||
## Scheduling and Concurrency
|
||||
|
||||
`internal/framework/llm/Scheduler` provides FIFO, context-aware permit gating.
|
||||
|
||||
CLI orchestration composes global and stage schedulers; runner instrumentation
|
||||
measures them. Limit definitions and inheritance belong to
|
||||
[configuration](../config.md#concurrency).
|
||||
|
||||
Scheduler release is guarded to avoid permit leaks on cancellation/error. Runner
|
||||
collects utilization observations; their serialized shape is defined by the
|
||||
[output-file contract](../integrations/output-files.md#utilization-diagnostics).
|
||||
|
||||
## Diagnostics and Redaction
|
||||
|
||||
`internal/framework/llm/DiagnosticsWriter` writes request/response/error artifacts.
|
||||
|
||||
Configured secrets are derived from `llm.ConfiguredSecrets(cfg)` and redacted from:
|
||||
- diagnostics payloads;
|
||||
- surfaced runtime/adapter errors.
|
||||
|
||||
## Changing LLM Integration or Scheduling
|
||||
|
||||
Review client, scheduler, redaction, and structured-output malformed-response
|
||||
coverage alongside runner orchestration and utilization tests. Use the
|
||||
[testing policy](../policy/testing.md) to select sufficient checks; preserve the
|
||||
[LLM integration contract](../integrations/openai-compatible-llm.md) and
|
||||
[architecture invariants](../policy/architecture.md#correction-and-concurrency-invariants).
|
||||
|
||||
## Key Tests
|
||||
|
||||
- `internal/framework/llm/openai_compatible_client_test.go`
|
||||
- `internal/framework/llm/scheduler_test.go`
|
||||
- `internal/framework/llm/diagnostics_test.go`
|
||||
- `internal/framework/responseschema/registry_test.go`
|
||||
- `internal/framework/structuredoutput/malformed_test.go`
|
||||
- `internal/framework/runner/*_test.go` (orchestration and utilization)
|
||||
76
docs/internal/modules.md
Normal file
76
docs/internal/modules.md
Normal file
@@ -0,0 +1,76 @@
|
||||
# Audita Modules
|
||||
|
||||
## Scope
|
||||
|
||||
This document covers internal module contracts and packages. User-selectable keys
|
||||
and the default sequence belong to [configuration](../config.md#pipeline-and-output).
|
||||
|
||||
## Module Contract
|
||||
|
||||
Modules implement `contracts.TranscriptModule`:
|
||||
- `Key()`
|
||||
- `ReplacementPolicy()`
|
||||
- `Validators()`
|
||||
- `Propose(ctx, req)`
|
||||
|
||||
Runner resolves configured module specs to module instances through `internal/framework/modules`.
|
||||
|
||||
## Built-In Modules
|
||||
|
||||
Current module packages:
|
||||
- `internal/modules/glossary`
|
||||
- `internal/modules/homophones`
|
||||
- `internal/modules/spoken_word`
|
||||
- `internal/modules/grammar`
|
||||
|
||||
Current replacement policies:
|
||||
- `glossary`: `replace_all`
|
||||
- `homophones`: `require_unique`
|
||||
- `spoken_word`: `require_unique`
|
||||
- `grammar`: `require_unique`
|
||||
|
||||
## Proposal Generation Ownership
|
||||
|
||||
Shared proposal-generation plumbing is centralized in:
|
||||
- `internal/framework/proposal_generation`
|
||||
|
||||
Module packages own:
|
||||
- prompt selection (`internal/prompts` prompt IDs);
|
||||
- module-specific prompt payload construction.
|
||||
|
||||
Shared prompt helpers live in `internal/framework/promptcontext`.
|
||||
|
||||
## Validator Chain Ownership
|
||||
|
||||
Built-in chains are resolved in `internal/validators` per module key.
|
||||
Module packages call the built-in chain resolver at construction.
|
||||
|
||||
## Failure and Warning Behavior
|
||||
|
||||
Module construction failures propagate to orchestration; execution failures can
|
||||
preserve partial module results. Malformed structured proposal payloads become
|
||||
warnings and section-level rejection. Consumer-visible outcomes are defined by
|
||||
the [report contract](../integrations/output-files.md#module-outcomes).
|
||||
|
||||
## Changing Modules
|
||||
|
||||
1. Add or update the narrowly scoped package under `internal/modules/<module_key>`.
|
||||
2. Keep module-specific prompt ownership in the module and `internal/prompts`;
|
||||
see [prompt rendering and metadata](prompts.md).
|
||||
3. Wire module registration and catalog resolution through framework module
|
||||
registration and the core module catalog. Update selectable keys in the
|
||||
[configuration contract](../config.md) when they change.
|
||||
4. Verify replacement policy and validator-chain selection against the
|
||||
[architecture requirements](../policy/architecture.md#correction-and-concurrency-invariants).
|
||||
5. Review module, prompt-rendering, and proposal-generation tests. Inspect parity
|
||||
and release fixtures when public output changes, choosing sufficient coverage
|
||||
under the [testing policy](../policy/testing.md).
|
||||
6. Update this document for implementation changes and the appropriate public
|
||||
contract owner for externally visible changes.
|
||||
|
||||
## Key Tests
|
||||
|
||||
- `internal/modules/*/module_test.go`
|
||||
- `internal/framework/modules/registry_test.go`
|
||||
- `internal/framework/proposal_generation/*_test.go`
|
||||
- `internal/cli/run_test.go` (pipeline/report integration)
|
||||
34
docs/internal/output-schemas.md
Normal file
34
docs/internal/output-schemas.md
Normal file
@@ -0,0 +1,34 @@
|
||||
# Output Encoding Internals
|
||||
|
||||
## Scope and Ownership
|
||||
|
||||
`internal/core/outputschema` owns encoder registration and resolution. External
|
||||
JSON envelopes and segment fields are defined by the
|
||||
[output-file contract](../integrations/output-files.md#corrected-transcript);
|
||||
selectable values and defaults belong to
|
||||
[configuration](../config.md#pipeline-and-output).
|
||||
|
||||
## Resolution and Encoding
|
||||
|
||||
Configuration validation checks registry support. CLI orchestration resolves the
|
||||
selected encoder, passes the final transcript to it, and routes the encoded bytes
|
||||
to the destination defined by the [CLI contract](../cli.md#process-output-and-exit-behavior).
|
||||
An unsupported key fails validation or resolution rather than selecting an
|
||||
implicit fallback.
|
||||
|
||||
The registry delegates bare segment serialization to the transcript schema
|
||||
package and uses an envelope encoder for the versioned form. Report construction
|
||||
receives the selected key separately; see [report assembly](diagnostics-reporting.md).
|
||||
|
||||
## Changing an Encoder
|
||||
|
||||
Update registration, configuration validation, and serialization together. Inspect
|
||||
round-trip and shape tests, then update the external output-file contract if the
|
||||
serialized form changes. Keep internal helper changes separate from decisions
|
||||
about externally visible compatibility.
|
||||
|
||||
## Key Tests
|
||||
|
||||
- `internal/core/outputschema/registry_test.go`
|
||||
- `internal/core/config/config_test.go`
|
||||
- `internal/cli/run_test.go`
|
||||
94
docs/internal/overview.md
Normal file
94
docs/internal/overview.md
Normal file
@@ -0,0 +1,94 @@
|
||||
# Audita Internal Overview
|
||||
|
||||
## Scope
|
||||
|
||||
This document owns the implemented component map. For normative boundaries, see
|
||||
[architecture policy](../policy/architecture.md); for task-specific reading and
|
||||
baseline commands, see the [contributor guide](../development.md).
|
||||
|
||||
## Package Map
|
||||
|
||||
Executable and command orchestration:
|
||||
- `cmd/audita`: executable entry point.
|
||||
- `internal/cli`: argument parsing, config commands, streams, filesystem wiring, and process orchestration.
|
||||
|
||||
Core deterministic components:
|
||||
- `internal/core/config`
|
||||
- `internal/core/schema`
|
||||
- `internal/core/normalization`
|
||||
- `internal/core/chunking`
|
||||
- `internal/core/outputschema`
|
||||
- `internal/core/diagnostics`
|
||||
- `internal/core/reporting`
|
||||
- `internal/core/modulecatalog`
|
||||
|
||||
Framework orchestration and contracts:
|
||||
- `internal/framework/contracts`
|
||||
- `internal/framework/modules`
|
||||
- `internal/framework/proposal_generation`
|
||||
- `internal/framework/proposals`
|
||||
- `internal/framework/runner`
|
||||
- `internal/framework/validators`
|
||||
- `internal/framework/llm`
|
||||
- `internal/framework/responseschema`
|
||||
- `internal/framework/structuredoutput`
|
||||
- `internal/framework/processreport`
|
||||
- `internal/framework/promptcontext`
|
||||
- `internal/framework/stagename`
|
||||
|
||||
Domain implementations:
|
||||
- `internal/modules/*`
|
||||
- `internal/validators/*`
|
||||
- `internal/prompts`
|
||||
|
||||
Supporting areas:
|
||||
- `docs/`: canonical documentation.
|
||||
- `examples/`: maintained copyable inputs and configuration.
|
||||
|
||||
## Current Scope
|
||||
|
||||
Runtime state is held in memory plus per-run diagnostics artifacts. Audita does
|
||||
not currently implement a plugin framework, generic workflow engine,
|
||||
resume/checkpoint/manifest system, remote storage, or multi-process service mode.
|
||||
Its provider adapter is OpenAI-compatible HTTP behind the structured LLM client
|
||||
contract. These are current implementation boundaries, not bans on future work.
|
||||
|
||||
## Main Execution Path (`audita process`)
|
||||
|
||||
The CLI prepares input and configuration, invokes the runner, and writes results.
|
||||
The [pipeline](pipeline.md) owns detailed state transitions; the
|
||||
[operations guide](../operations.md) owns the operator workflow and physical
|
||||
artifact lifecycle.
|
||||
|
||||
## Component Responsibilities
|
||||
|
||||
Core components implement configuration defaults/loading/validation/redaction,
|
||||
transcript and glossary schemas, normalization and chunking, output encoding,
|
||||
diagnostics artifact helpers, and public report structures.
|
||||
|
||||
Framework components implement module and validator contracts, proposal
|
||||
generation and application, prompt context, runner orchestration, LLM scheduling
|
||||
and transport, response schemas, and process-report/correction-ledger assembly.
|
||||
Domain packages provide correction modules, built-in validator chains and
|
||||
execution metadata, and embedded prompt assets.
|
||||
|
||||
These describe the implementation; dependency and ownership requirements belong
|
||||
to the [architecture policy](../policy/architecture.md#ownership-and-dependency-direction).
|
||||
|
||||
## Change Recipes
|
||||
|
||||
- [Configuration, CLI, and example changes](cli-config.md)
|
||||
- [Module changes](modules.md#changing-modules)
|
||||
- [Validator changes](validators.md#changing-validators)
|
||||
- Runtime orchestration changes: use [pipeline](pipeline.md), [LLM runtime](llm-runtime.md),
|
||||
or [diagnostics/reporting](diagnostics-reporting.md) and their test references.
|
||||
|
||||
## Related Internal Docs
|
||||
|
||||
- [`docs/internal/pipeline.md`](pipeline.md)
|
||||
- [`docs/internal/modules.md`](modules.md)
|
||||
- [`docs/internal/validators.md`](validators.md)
|
||||
- [`docs/internal/llm-runtime.md`](llm-runtime.md)
|
||||
- [`docs/internal/diagnostics-reporting.md`](diagnostics-reporting.md)
|
||||
- [`docs/internal/prompts.md`](prompts.md)
|
||||
- [`docs/internal/output-schemas.md`](output-schemas.md)
|
||||
66
docs/internal/pipeline.md
Normal file
66
docs/internal/pipeline.md
Normal file
@@ -0,0 +1,66 @@
|
||||
# Audita Internal Pipeline
|
||||
|
||||
## Scope
|
||||
|
||||
This document describes processing collaborators and state transitions. External
|
||||
input definitions belong to the [input-file contract](../integrations/transcript-glossary-files.md),
|
||||
settings to [configuration](../config.md), and serialized results to the
|
||||
[output-file contract](../integrations/output-files.md).
|
||||
|
||||
## Inputs
|
||||
|
||||
Pipeline inputs are:
|
||||
- effective config (`internal/core/config`);
|
||||
- transcript JSON (`internal/core/schema`);
|
||||
- glossary YAML (`internal/core/schema`).
|
||||
|
||||
## Pipeline Phases
|
||||
|
||||
1. **Input loading:** CLI filesystem adapters read inputs; schema parsing produces
|
||||
source transcript and glossary objects. Source segments are converted to the
|
||||
canonical transcript shape before normalization.
|
||||
2. **Normalization:** the normalizer sorts and merges segments using its configured
|
||||
time/token rules, reassigns IDs, and produces summary counters.
|
||||
3. **Chunking:** the chunker partitions the normalized transcript and returns
|
||||
sections and token estimates, or fails when constraints are impossible.
|
||||
4. **Proposal generation:** the runner executes configured module instances in
|
||||
sequence. Sections can generate proposals concurrently; results are collected
|
||||
for deterministic indexing.
|
||||
5. **Validation:** deterministic validators run before LLM-backed validators.
|
||||
LLM validation is batched by prompt budget; candidate decisions feed application.
|
||||
6. **Application:** `internal/framework/proposals` clones transcript state and
|
||||
applies approved proposals in index order through replacement-policy checks.
|
||||
7. **Serialization and reporting:** the CLI encodes the final transcript, maps
|
||||
runner output into reports, writes artifacts, and delegates retention to
|
||||
diagnostics. See [report assembly](diagnostics-reporting.md).
|
||||
|
||||
## Runner Outputs
|
||||
|
||||
`runner.Run` returns:
|
||||
- final transcript;
|
||||
- per-module results;
|
||||
- utilization diagnostics.
|
||||
|
||||
CLI/reporting then map this into process report and diagnostics artifacts.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
CLI orchestration labels failures at the phase boundary. Runner execution failures
|
||||
can return completed or partial module results, which report assembly preserves.
|
||||
[Report fields](../integrations/output-files.md#process-report) define how consumers
|
||||
see these outcomes; [CLI semantics](../cli.md#process-output-and-exit-behavior)
|
||||
define streams and exits. Diagnostic write failures do not always abort processing.
|
||||
|
||||
## Ordering and State
|
||||
|
||||
The runner mutates its working transcript between module instances, after collecting
|
||||
section results and validating proposals. Proposal indexes follow section order;
|
||||
reports are derived from actual run results. Global requirements are owned by the
|
||||
[architecture policy](../policy/architecture.md#correction-and-concurrency-invariants).
|
||||
|
||||
## Key Tests
|
||||
|
||||
- `internal/framework/runner/runner_test.go`
|
||||
- `internal/framework/proposal_generation/*_test.go`
|
||||
- `internal/cli/run_test.go`
|
||||
- `cmd/audita/main_integration_test.go`
|
||||
62
docs/internal/prompts.md
Normal file
62
docs/internal/prompts.md
Normal file
@@ -0,0 +1,62 @@
|
||||
# Audita Prompt Registry
|
||||
|
||||
## Scope
|
||||
|
||||
This document describes embedded prompt assets, prompt metadata, and rendering behavior.
|
||||
|
||||
## Registry Ownership
|
||||
|
||||
Prompt registry lives in `internal/prompts` and embeds assets under `internal/prompts/assets/**`.
|
||||
|
||||
Registered prompt IDs:
|
||||
- `modules.glossary.proposal`
|
||||
- `modules.homophones.proposal`
|
||||
- `modules.spoken_word.proposal`
|
||||
- `modules.grammar.proposal`
|
||||
- `validators.spoken_form_plausibility`
|
||||
- `validators.meaning_reversal_review`
|
||||
- `validators.editorial_review`
|
||||
- `validators.grammar_review`
|
||||
- `validators.spoken_word_review`
|
||||
|
||||
## Metadata Model
|
||||
|
||||
Each prompt has metadata:
|
||||
- `prompt_id`
|
||||
- `prompt_version`
|
||||
- `prompt_source`
|
||||
- `embedded_path`
|
||||
- `sha256`
|
||||
|
||||
Current source/version values:
|
||||
- `prompt_source = builtin`
|
||||
- `prompt_version = v1`
|
||||
|
||||
## Rendering
|
||||
|
||||
`prompts.RenderUserSystem(promptID, data)` renders system/user templates.
|
||||
|
||||
Template behavior:
|
||||
- uses Go `text/template`;
|
||||
- `missingkey=error` is enabled;
|
||||
- output is trimmed.
|
||||
|
||||
A shared hardening fragment is embedded once and referenced by prompt templates.
|
||||
|
||||
## Prompt Context Inputs
|
||||
|
||||
Shared prompt payload helpers:
|
||||
- transcript section JSON (`internal/framework/promptcontext/MarshalTranscriptSectionJSON`)
|
||||
- transcript description block (`TranscriptDescriptionBlock`)
|
||||
|
||||
Modules and LLM validators provide typed data maps to render prompt assets.
|
||||
|
||||
## Diagnostics Integration
|
||||
|
||||
Prompt metadata is attached to proposal/validator diagnostics request metadata using `Metadata.DiagnosticsMap()`.
|
||||
|
||||
## Key Tests
|
||||
|
||||
- `internal/prompts/registry_test.go`
|
||||
- `internal/framework/promptcontext/*_test.go`
|
||||
- module and validator prompt builder tests
|
||||
87
docs/internal/validators.md
Normal file
87
docs/internal/validators.md
Normal file
@@ -0,0 +1,87 @@
|
||||
# Audita Validators
|
||||
|
||||
## Scope
|
||||
|
||||
This document describes internal validator composition, execution order, and
|
||||
decision handling. These registry keys identify implementation components; they
|
||||
are not independently selectable through [configuration](../config.md#pipeline-and-output).
|
||||
|
||||
## Ownership
|
||||
|
||||
Built-in validator keys and chains:
|
||||
- `internal/validators`
|
||||
|
||||
Shared validator runtime mechanics:
|
||||
- `internal/framework/validators`
|
||||
|
||||
Execution-class metadata:
|
||||
- `internal/validators/metadata`
|
||||
|
||||
## Built-In Validator Keys
|
||||
|
||||
Deterministic:
|
||||
- `proposal_shape`
|
||||
- `confidence_threshold`
|
||||
- `original_text_presence`
|
||||
- `non_empty_corrected_text`
|
||||
- `no_effect`
|
||||
- `protected_terms`
|
||||
|
||||
LLM-backed:
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
- `editorial_review`
|
||||
|
||||
## Built-In Chains
|
||||
|
||||
Module chains are defined in `internal/validators/chains.go`.
|
||||
Glossary, homophones, spoken_word, and grammar each resolve a fixed ordered chain.
|
||||
|
||||
## Runtime Execution
|
||||
|
||||
For each module section:
|
||||
1. run deterministic validators;
|
||||
2. run LLM-backed validators;
|
||||
3. record decisions and warnings;
|
||||
4. carry only approved proposals forward.
|
||||
|
||||
Decision cardinality is enforced: each candidate proposal must receive exactly one decision per validator.
|
||||
|
||||
## LLM Validator Batching
|
||||
|
||||
LLM validators:
|
||||
- build canonical validation request payloads;
|
||||
- batch using the [configured prompt budget](../config.md#concurrency);
|
||||
- call structured LLM client using response schema registry.
|
||||
|
||||
Oversized single proposals are rejected with `validator_input_too_large`.
|
||||
Malformed LLM validator responses are downgraded to warnings and rejected batch decisions.
|
||||
|
||||
## Decision and Rejection Reporting
|
||||
|
||||
Runner collects decisions, rejected candidates, and warning records. Ledger
|
||||
assembly groups decisions by deterministic versus LLM-backed execution metadata.
|
||||
The serialized names and fields belong to the
|
||||
[report and ledger contract](../integrations/output-files.md); implementation mapping
|
||||
is covered by [report assembly](diagnostics-reporting.md).
|
||||
|
||||
## Changing Validators
|
||||
|
||||
1. Implement validator behavior under `internal/validators/<validator_key>`;
|
||||
update registry and chain wiring in `internal/validators`.
|
||||
2. Put shared runtime mechanics in `internal/framework/validators` only when
|
||||
needed. Keep deterministic versus LLM-backed execution metadata explicit.
|
||||
3. Preserve exposed identities and decision semantics under the
|
||||
[architecture policy](../policy/architecture.md#correction-and-concurrency-invariants).
|
||||
4. Review validator, registry, chain, batching, malformed-output, diagnostics,
|
||||
and decision-cardinality tests; update meaningful coverage under the
|
||||
[testing policy](../policy/testing.md).
|
||||
5. Update implementation guidance here and any affected external contract in its
|
||||
[canonical owner](../policy/documentation.md#canonical-ownership).
|
||||
|
||||
## Key Tests
|
||||
|
||||
- `internal/validators/*_test.go`
|
||||
- `internal/framework/validators/*_test.go`
|
||||
- `internal/framework/processreport/correction_ledger_test.go`
|
||||
- `internal/cli/run_test.go`
|
||||
110
docs/operations.md
Normal file
110
docs/operations.md
Normal file
@@ -0,0 +1,110 @@
|
||||
# Audita Operations
|
||||
|
||||
This document owns runtime workflow, physical artifact placement, retention, and
|
||||
recovery. Use the [CLI reference](cli.md) for invocation and stream/exit semantics,
|
||||
[configuration](config.md) for settings, and [output-file contracts](integrations/output-files.md)
|
||||
for logical formats.
|
||||
|
||||
## Normal Workflow
|
||||
|
||||
1. Choose a configuration and validate it using the
|
||||
[configuration commands](cli.md#configuration-commands).
|
||||
2. Prepare input files conforming to the [input contract](integrations/transcript-glossary-files.md).
|
||||
3. Run the [processing command](cli.md#process-examples), choosing explicit output
|
||||
and report destinations when those files must survive run-directory cleanup.
|
||||
4. Inspect the corrected transcript and process report. Review rejected or skipped
|
||||
corrections when the report indicates them; successful execution does not mean
|
||||
every proposed correction was applied.
|
||||
|
||||
Processing creates a run directory, loads and normalizes the inputs, chunks the
|
||||
transcript, executes the correction pipeline, writes results, and applies
|
||||
retention. Early configuration failures happen before run-directory creation.
|
||||
Failures after initialization attempt to preserve diagnostic evidence; individual
|
||||
artifact writes can fail, so a run directory need not contain every listed file.
|
||||
|
||||
## Output Files
|
||||
|
||||
The caller chooses transcript and optional external report destinations using the
|
||||
[CLI](cli.md#process-output-and-exit-behavior). These are separate from the per-run
|
||||
diagnostic directory unless the caller deliberately places them inside it.
|
||||
Choose existing, writable parent directories. Output/report files can be overwritten
|
||||
by subsequent runs using the same destinations.
|
||||
|
||||
A later report or cleanup failure can occur after transcript output was written.
|
||||
Inspect the process result and available report before treating the files as a
|
||||
completed run. Retention of the diagnostic directory does not archive caller-selected
|
||||
output files elsewhere.
|
||||
|
||||
## Diagnostics Directory
|
||||
|
||||
The [diagnostics configuration](config.md#thresholds-context-and-diagnostics)
|
||||
selects the parent work directory. Runs create children named like
|
||||
`run-<unix-nanoseconds>`. The runtime creates the work directory if needed.
|
||||
Report paths may be relative when the supplied work directory is relative;
|
||||
resolve them using the invocation's working directory.
|
||||
|
||||
| Artifact in the run directory | Purpose |
|
||||
| --- | --- |
|
||||
| `source-transcript.json` | Original transcript input |
|
||||
| `source-transcript-parsed.json` | Parsed source representation |
|
||||
| `normalized-transcript.json` | Transcript after normalization |
|
||||
| `normalization-summary.json` | Normalization counters |
|
||||
| `chunking-summary.json` | Section/token summary |
|
||||
| `invocation.json` | Invocation metadata |
|
||||
| `effective-config.json` | Redacted configuration snapshot |
|
||||
| `report.json` | Run's [process report](integrations/output-files.md#process-report) |
|
||||
| `correction-ledger.json` | [Correction dispositions](integrations/output-files.md#correction-ledger) |
|
||||
| `utilization-diagnostics.json` | [Concurrency and timing observations](integrations/output-files.md#utilization-diagnostics) |
|
||||
| `error.log` | Failure details when written |
|
||||
|
||||
LLM request, response, and error diagnostics are also written within the run
|
||||
directory. Their nested naming and payloads are debugging details. Prefer paths
|
||||
provided by the report and its decision/warning records over reconstructing those
|
||||
paths. The report describes intended artifact locations, not proof that every
|
||||
write succeeded or that the files still exist after retention.
|
||||
|
||||
## Retention Behavior
|
||||
|
||||
[Configuration](config.md#thresholds-context-and-diagnostics) owns accepted
|
||||
retention values and the default. Current lifecycle behavior is:
|
||||
|
||||
| Run outcome / selection | Directory outcome |
|
||||
| --- | --- |
|
||||
| Failed run | Retained regardless of selection |
|
||||
| Successful run, `always` | Retained |
|
||||
| Successful run, `auto` | Retained if corrections were skipped or rejected; otherwise removed |
|
||||
| Successful run, `never` | Currently retained, the same outcome as `always` |
|
||||
|
||||
The name `never` does not currently mean deletion. Warnings alone do not trigger
|
||||
retention under `auto` unless they also produce skipped or rejected corrections.
|
||||
An external report outside the run directory survives automatic cleanup, but its
|
||||
artifact paths can then point to removed files.
|
||||
|
||||
## Artifact Handling
|
||||
|
||||
Run artifacts can contain full transcripts, glossary-related context, and LLM
|
||||
prompts/responses. Configured-key redaction does not anonymize those contents.
|
||||
Choose a work directory whose ownership and permissions fit the transcript data,
|
||||
and retain or share artifacts intentionally. Credential-supply mechanisms are
|
||||
owned by [configuration](config.md#secrets).
|
||||
|
||||
For a debugging run, select retained diagnostics using the
|
||||
[retention flag](cli.md#process-flag-reference). Copy needed artifacts before
|
||||
removing completed run directories manually. Avoid deleting a directory while
|
||||
its run is active. Audita does not provide a separate archival/backup or remote
|
||||
artifact storage facility.
|
||||
|
||||
## Failure Inspection and Recovery
|
||||
|
||||
1. Read stderr and the exit classification defined by the [CLI](cli.md#process-output-and-exit-behavior).
|
||||
2. Locate the external report, or the run directory identified by the error when
|
||||
available. If a late error omits the path, inspect the selected work directory.
|
||||
3. Read the available report and error log. Use
|
||||
[report status and error fields](integrations/output-files.md#process-report)
|
||||
to identify the failed phase, then inspect referenced artifacts.
|
||||
4. Correct the concrete input, configuration, endpoint, or filesystem issue and
|
||||
rerun the command with diagnostics retained if further investigation is needed.
|
||||
|
||||
Each invocation starts a new run. There is no implemented resume/checkpoint API;
|
||||
recovery is a rerun, not continuation from an artifact. Failures during early
|
||||
configuration or directory creation may leave no report or diagnostic files.
|
||||
103
docs/policy/agents.md
Normal file
103
docs/policy/agents.md
Normal file
@@ -0,0 +1,103 @@
|
||||
# Agent Policy
|
||||
|
||||
**Base Policy Version:** 1.0
|
||||
|
||||
**Last Updated**: 2026-09-23
|
||||
|
||||
## Purpose
|
||||
|
||||
Agents working in this repository must read and adhere to the following house rules.
|
||||
|
||||
## Implementation Discipline
|
||||
|
||||
**KEY PRINCIPLES:** Write the simplest idiomatic code that satisfies the user's
|
||||
request. Do not implicitly strengthen requirements, restrictions, or guarantees
|
||||
beyond what the user actually requested.
|
||||
|
||||
**Keep implementation machinery proportionate.** Use straightforward,
|
||||
idiomatic code and introduce only the machinery needed to support the requested
|
||||
behavior and flexibility. Optimize for conceptual simplicity and
|
||||
maintainability, not for the fewest lines, fewest options, or narrowest product
|
||||
surface.
|
||||
|
||||
**Do not invent restrictions.** Do not reject, prohibit, constrain, cap,
|
||||
harden, or otherwise narrow behavior unless required by the user, an explicit
|
||||
repository contract, or an unavoidable correctness constraint. Preserve
|
||||
operator and caller choice where policy has not been established. If an
|
||||
additional restriction appears advisable, present it as a decision rather than
|
||||
silently implementing it.
|
||||
|
||||
**Do not invent guarantees.** Do not strengthen public contracts, guarantees,
|
||||
or invariants beyond those expressly requested by the user.
|
||||
|
||||
**Use the standard library.** For example: in almost every case, the best
|
||||
approach to copying files is to use the standard library. Writing custom code to
|
||||
implement power-loss protection at the application level, or to guard against
|
||||
bad actors with privileged and concurrent access to the host's filesystem,
|
||||
is wasteful, unnecessary, and not idiomatic. Stick to the simple, standard, and
|
||||
idiomatic approach unless another path is expressly requested by the user.
|
||||
|
||||
**An implementation detail is not repository policy.** Tests and documentation
|
||||
must not elevate incidental behavior into a durable guarantee or restriction
|
||||
unless that behavior was expressly selected as part of the contract.
|
||||
|
||||
## Codebase Discovery
|
||||
|
||||
This project uses codebase-memory-mcp to maintain a knowledge graph of the
|
||||
codebase. Always prefer its graph tools over grep, glob, or file search for code discovery.
|
||||
If the project is not indexed, run `index_repository` first.
|
||||
|
||||
Use this order:
|
||||
|
||||
1. `search_graph` to find functions, classes, routes, and variables by pattern.
|
||||
2. `trace_path` to trace callers and callees.
|
||||
3. `get_code_snippet` to read a specific function or class.
|
||||
4. `query_graph` for complex graph queries.
|
||||
5. `get_architecture` for the high-level project summary.
|
||||
|
||||
Fall back to grep or glob for string literals, error messages, configuration
|
||||
values, non-code files, or when the graph tools return insufficient results.
|
||||
|
||||
Examples:
|
||||
|
||||
- Find a handler: `search_graph(name_pattern=".*OrderHandler.*")`.
|
||||
- Find callers: `trace_path(function_name="OrderHandler", direction="inbound")`.
|
||||
- Read source: `get_code_snippet(qualified_name="pkg/orders.OrderHandler")`.
|
||||
|
||||
## Git Commit Policy
|
||||
|
||||
This policy applies to all coding agents and agent harnesses working in this
|
||||
repository, not only Codex.
|
||||
|
||||
By default, create a commit after completing each coherent unit of work, using a
|
||||
concise, plain-English message describing what changed. This policy is standing
|
||||
authorization to create those local commits; a separate user request is not
|
||||
required. An explicit user instruction to leave work uncommitted takes precedence.
|
||||
|
||||
A unit of work is a focused, reviewable outcome, including its relevant tests and
|
||||
documentation. Do not commit after every intermediate edit or create empty commits.
|
||||
Complete the appropriate validation before committing. If work is incomplete or
|
||||
validation is blocked, report that state rather than presenting it as completed.
|
||||
|
||||
Before committing:
|
||||
|
||||
1. Run `git status --short` and review the relevant diff, including any staged
|
||||
changes. Identify changes that predate the current unit of work.
|
||||
2. Exclude secrets, local environment files, generated junk, and unrelated changes.
|
||||
3. Stage only the files or hunks belonging to the completed unit. Use `git add .`
|
||||
only when every included change belongs to that unit. Do not disturb unrelated
|
||||
staged or unstaged work.
|
||||
4. Review `git diff --staged` and run appropriate checks for the change. Do not
|
||||
repeat checks already passed for unchanged content merely to create a commit.
|
||||
5. Review `git log --oneline -n 10` and use
|
||||
`git commit -m "<concise plain-English commit message>"`.
|
||||
6. Report the commit identifier and any remaining uncommitted work.
|
||||
|
||||
If the unit cannot be safely separated from pre-existing work, leave it uncommitted
|
||||
and explain the overlap rather than silently including that work. If a commit
|
||||
fails, preserve the changes and report the failure; do not bypass hooks.
|
||||
|
||||
Default commit authorization does not authorize pushing, publishing, amending,
|
||||
resetting, reverting, or rewriting history. Those actions require a separate user
|
||||
instruction. Do not use `git commit -a` or `git commit --no-verify` to bypass the
|
||||
staging and validation discipline above.
|
||||
105
docs/policy/architecture.md
Normal file
105
docs/policy/architecture.md
Normal file
@@ -0,0 +1,105 @@
|
||||
# Architecture Policy
|
||||
|
||||
**Base Policy Version:** 1.0
|
||||
|
||||
**Last Updated**: 2026-09-23
|
||||
|
||||
## Purpose and System Shape
|
||||
|
||||
Audita is a single-process Go CLI for transcript polishing. Its architecture
|
||||
separates deterministic transcript processing from external LLM effects through
|
||||
narrow contracts and adapters. Correction modules and validators remain explicit,
|
||||
composable components in an ordered processing pipeline.
|
||||
|
||||
This policy owns architectural requirements. The [internal overview](../internal/overview.md)
|
||||
owns the implemented component inventory and links to runtime mechanics.
|
||||
[CLI](../cli.md), [configuration](../config.md), [operations](../operations.md),
|
||||
and [integration documentation](../integrations/subprocess.md) own external contracts.
|
||||
|
||||
## Ownership and Dependency Direction
|
||||
|
||||
- **Core:** owns domain data handling and reusable configuration, schema,
|
||||
normalization, chunking, output, diagnostics, and reporting contracts. Domain
|
||||
behavior must not require CLI or provider-transport knowledge.
|
||||
- **Framework:** owns orchestration contracts and shared runtime mechanics for
|
||||
modules, validators, proposals, LLM adapters, scheduling, and report assembly.
|
||||
It composes domain operations through explicit inputs and outputs.
|
||||
- **Modules:** own narrowly scoped correction behavior and module-specific prompt
|
||||
choices. Keep module packages separate rather than collapsing their distinct
|
||||
responsibilities into a broad generic stage abstraction.
|
||||
- **Validators:** own validation behavior and composition. Deterministic and
|
||||
LLM-backed validators remain explicit, independently composable elements.
|
||||
- **Prompts:** own embedded prompt assets and metadata used by the runtime.
|
||||
- **CLI and adapters:** own command parsing, streams, filesystem wiring, and
|
||||
transport. Keep these concerns outside module, validator, and schema logic.
|
||||
|
||||
LLM transport belongs behind `StructuredLLMClient` and framework adapter
|
||||
boundaries. Prefer small shared helpers over broad rewrites. Keep module and
|
||||
validator inputs, outputs, and configuration dependencies explicit, without
|
||||
hidden global state.
|
||||
|
||||
## Correction and Concurrency Invariants
|
||||
|
||||
- Treat LLM responses as nondeterministic inputs. Preserve deterministic proposal
|
||||
indexing, validator ordering, transcript mutation, reports, and serialization.
|
||||
- Validate corrections before final mutation, including local proposal checks,
|
||||
decision cardinality, and apply-time safety. Provider-side schema enforcement
|
||||
does not replace these checks.
|
||||
- Keep concurrency bounded, configurable, context-aware, and observable through
|
||||
the existing scheduler and utilization diagnostics. Preserve permit release on
|
||||
success, failure, and cancellation.
|
||||
- Keep module and validator identities, reason codes, and execution-class
|
||||
distinctions stable where exposed as contracts.
|
||||
|
||||
The [pipeline](../internal/pipeline.md), [modules](../internal/modules.md),
|
||||
[validators](../internal/validators.md), and [LLM runtime](../internal/llm-runtime.md)
|
||||
describe how these requirements are implemented.
|
||||
|
||||
## State and Compatibility
|
||||
|
||||
Keep transcript mutation and output handling deterministic around external
|
||||
calls. Reports and correction ledgers are derived from processing results.
|
||||
Preserve the separation between transient processing state and filesystem
|
||||
adapters; physical artifact placement and retention belong to
|
||||
[operations](../operations.md).
|
||||
|
||||
Public CLI, configuration, output-schema, diagnostics, report, prompt, module,
|
||||
and validator contracts should remain stable unless a change is deliberate and
|
||||
tested. Prefer additive, compatible report changes. Current implementation
|
||||
limits are documented in the [internal overview](../internal/overview.md#current-scope);
|
||||
they are not additional prohibitions on future development.
|
||||
|
||||
## Errors and Secret Handling
|
||||
|
||||
Errors should identify the failed processing phase sufficiently for users and
|
||||
subprocess callers. Preserve the stream and exit behavior defined by the
|
||||
[CLI contract](../cli.md) and the artifact handling defined by
|
||||
[operations](../operations.md).
|
||||
|
||||
Diagnostics, reports, and surfaced errors must not leak configured LLM secrets.
|
||||
Configuration redaction and LLM payload/error redaction have separate
|
||||
responsibilities and should remain separate. Implementation details belong in
|
||||
[diagnostics and reporting](../internal/diagnostics-reporting.md).
|
||||
|
||||
## Dependencies
|
||||
|
||||
Keep Audita dependency-light and prefer the Go standard library for CLI parsing,
|
||||
HTTP, JSON, filesystems, synchronization, and tests. Third-party dependencies
|
||||
should be narrow, justified, and preferably established for their purpose.
|
||||
YAML parsing is the existing direct dependency exception.
|
||||
|
||||
Do not introduce broad CLI, dependency-injection, workflow, logging, or plugin
|
||||
frameworks without a concrete implemented need and focused tests.
|
||||
|
||||
## Testing and Documentation
|
||||
|
||||
Follow the [testing policy](testing.md) for risk-based sufficiency and stable test
|
||||
boundaries. Keep tests close to the behavior they protect; shared helpers should
|
||||
remove duplication without obscuring component-specific behavior. Relevant
|
||||
internal docs identify component tests and change recipes; the
|
||||
[contributor guide](../development.md) provides baseline validation commands.
|
||||
|
||||
Follow the [documentation policy](documentation.md) for ownership, current versus
|
||||
future behavior, and the ADR lifecycle. Keep architecture concise and normative;
|
||||
use the roadmap for implementation plans rather than maintaining a changelog or
|
||||
future-feature specification here.
|
||||
164
docs/policy/documentation.md
Normal file
164
docs/policy/documentation.md
Normal file
@@ -0,0 +1,164 @@
|
||||
# Documentation Policy
|
||||
|
||||
**Base Policy Version:** 1.1
|
||||
|
||||
**Last Updated**: 2026-09-23
|
||||
|
||||
## Purpose
|
||||
|
||||
This policy assigns each documentation topic to one canonical owner. Its goal is
|
||||
to keep this application's documentation accurate, concise, discoverable, and resistant
|
||||
to drift for users, operators, developers, integrators, and LLM coding agents.
|
||||
|
||||
## Core Rules
|
||||
|
||||
### One Canonical Owner
|
||||
|
||||
Each authoritative fact belongs in one document. A non-owning document may give
|
||||
a short, stable summary for orientation, but it must link to the canonical owner
|
||||
instead of repeating volatile details.
|
||||
|
||||
Volatile details include commands, flags, configuration fields and defaults,
|
||||
module keys, schemas, file names, paths, status codes, retry behavior, and
|
||||
runtime guarantees. If readers could reasonably treat a statement as a
|
||||
contract, maintain it only in the owning document.
|
||||
|
||||
### Current And Future Behavior
|
||||
|
||||
Outside `docs/roadmap/`, documentation describes implemented behavior only.
|
||||
Partial features may be described only to their implemented boundary.
|
||||
|
||||
ADRs are the narrow exception: an ADR may record an accepted architectural
|
||||
decision before implementation, but acceptance must not be presented as proof
|
||||
that the behavior exists. The roadmap owns implementation status and sequencing
|
||||
until the decision is implemented. Current architecture, user, operator,
|
||||
integration, and internal documentation are updated when the behavior lands.
|
||||
|
||||
### Audience And Detail
|
||||
|
||||
Write for the document's stated audience and include only the detail needed for
|
||||
its owned topic. User and operator docs should not expose implementation detail.
|
||||
Developer docs should link to user-facing and external contracts rather than
|
||||
restate them.
|
||||
|
||||
### Examples
|
||||
|
||||
Complete copyable files belong in `examples/`. Documentation may use the
|
||||
smallest illustrative snippet needed to explain its owned topic, but should link
|
||||
to maintained examples instead of embedding a second complete copy.
|
||||
|
||||
Examples must be valid, secret-free, and tested where practical. Commands and
|
||||
configuration used in documentation should match the application.
|
||||
|
||||
### Security And Privacy
|
||||
|
||||
Documentation and examples must not contain real credentials, private keys,
|
||||
private environment dumps, sensitive source material, or private infrastructure
|
||||
details unless intentionally public. Document secret-handling mechanisms, not
|
||||
secret values.
|
||||
|
||||
## Canonical Ownership
|
||||
|
||||
| Topic | Canonical owner | Owned content | Content owned elsewhere |
|
||||
| --- | --- | --- | --- |
|
||||
| Product orientation and minimal end-to-end quickstart | `README.md` | What this application is, why it is useful, one shortest successful invocation, and links onward. | Complete command reference, configuration reference, operational procedures, implementation detail. |
|
||||
| Contributor entry point | `docs/development.md` | Task-oriented reading guide, minimal contributor orientation, baseline validation commands, and links to canonical docs. | Package inventory, architecture rules, subsystem behavior, detailed change recipes. |
|
||||
| Application architecture policy | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, architectural boundaries, invariants, safety properties, and non-goals. | Concrete package inventory, implementation mechanics, contributor procedures, decision history, future work. |
|
||||
| Agent policy | `docs/policy/agents.md` | Implementation discipline for coding agents, including proportionality, idiomatic simplicity, code discovery, Git commit restrictions, and rules against inventing requirements, restrictions, guarantees, or repository policy. | Application contracts, architecture, testing policy, and contributor procedures. |
|
||||
| Documentation policy | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
|
||||
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk-based sufficiency, test boundaries, doubles, coverage guidance, regression-test policy, and criteria for adding, rewriting, or deleting tests. | Subsystem behavior, application contracts, subsystem-specific test inventories, and implementation plans. |
|
||||
| Release policy | `docs/policy/release.md` | Versioning and compatibility rules, release integrity and immutability requirements, release-documentation policy, required gates, publication controls, verification requirements, and exception policy. | Repository-specific commands, product installation guidance, CLI version semantics, historical release summaries, and CI implementation detail. |
|
||||
| Repository-specific release procedure | `docs/release.md` | Maintainer prerequisites, candidate preparation, concrete validation and publication commands, repository and registry guards, deliverable publication, consumer-facing verification, and failure recovery. | General release requirements, product installation guidance, historical release summaries, and current compatibility contracts. |
|
||||
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
|
||||
| Configuration contract | `docs/config.md` | Discovery and precedence, file schema, fields, defaults, environment overrides, validation rules, and user-selectable module or validator keys. | Complete example files, CLI syntax, runtime state lifecycle, module implementation details. |
|
||||
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
|
||||
| Integration contracts for external components | `docs/integrations/`, including `docs/integrations/output-files.md` for serialized outputs | External file formats and protocols, upstream and downstream contracts, logical output bundle paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
||||
| Inventory of implemented internal packages | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
||||
| Behavior and architecture of internal packages | Other files under `docs/internal/` | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
||||
| Contributor change recipes | Relevant files under `docs/internal/`, including `docs/internal/cli-config.md`, `docs/internal/modules.md`, and `docs/internal/validators.md` | Concrete editing and validation steps for each component. | Contributor orientation, global policy, and definitions of external contracts. |
|
||||
| Architectural decision records | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
|
||||
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status; sequencing; and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
||||
| Release-note history | `docs/releases/` | One checked-in historical summary for each source release made under the procedure. The note at the immutable tag is that release's record. | Current commands, behavior, contracts, and compatibility definitions. |
|
||||
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other files intended to be copied or run. | Field-by-field reference, command reference, prose explanation. |
|
||||
|
||||
Documents that do not exist are required only when the corresponding interface
|
||||
or responsibility exists. Do not create placeholder API, consumer, integration,
|
||||
or operations documents for behavior the application does not have. Audita has
|
||||
retired its standalone troubleshooting guide; concrete operating and recovery
|
||||
procedures belong in `docs/operations.md`.
|
||||
|
||||
## Boundary Rules
|
||||
|
||||
### Orientation
|
||||
|
||||
The README owns product orientation. The developer guide routes contributors.
|
||||
Architecture owns normative structure. Internal overview owns the current
|
||||
concrete component map. These documents may link to one another but should not
|
||||
maintain parallel package or behavior descriptions.
|
||||
|
||||
### Commands, Configuration, And Operations
|
||||
|
||||
CLI documentation answers how to invoke the application. Configuration
|
||||
documentation answers what settings mean. Operations answers what happens to
|
||||
runtime state and how to operate or recover the application. When a workflow
|
||||
crosses these topics, choose the document that owns the task and link to the
|
||||
other contracts.
|
||||
|
||||
### Releases
|
||||
|
||||
The release policy defines requirements and invariants that every release must
|
||||
satisfy. The repository-specific release procedure defines the commands and
|
||||
workflow maintainers use to satisfy that policy. Release notes are immutable
|
||||
historical summaries, not current-state contract owners: the checked-in note at
|
||||
an immutable tag records that release, while current canonical documentation
|
||||
must change with the behavior it describes. Do not use a release procedure or
|
||||
release note to weaken policy, redefine current behavior, or defer required
|
||||
documentation updates.
|
||||
|
||||
### Contracts And Implementation
|
||||
|
||||
Integration and API documents define externally observable shapes and
|
||||
semantics. Internal documents explain how this application implements or consumes those
|
||||
contracts. Internal docs may name a field, file, or protocol to identify a
|
||||
dependency, but must link to its canonical contract for the definition.
|
||||
|
||||
### Security Topics
|
||||
|
||||
This policy owns what documentation and examples may contain. Architecture owns
|
||||
application security invariants. Configuration owns credential-supply
|
||||
mechanisms. Operations owns permissions and handling of sensitive runtime
|
||||
artifacts. Internal docs own implementation mechanisms only.
|
||||
|
||||
## Architecture Decision Records
|
||||
|
||||
Use sequentially numbered ADR filenames such as
|
||||
`0001-record-architecture-decisions.md`. Follow the lightweight Nygard format:
|
||||
|
||||
1. title;
|
||||
2. status;
|
||||
3. date;
|
||||
4. context;
|
||||
5. decision;
|
||||
6. alternatives considered;
|
||||
7. consequences.
|
||||
|
||||
Treat the decision content of an accepted ADR as immutable. When a decision
|
||||
changes, create a new ADR and update the earlier ADR's status to superseded.
|
||||
Rejected architectural alternatives belong in the ADR; rejected product ideas
|
||||
belong in the roadmap.
|
||||
|
||||
## Maintenance
|
||||
|
||||
When behavior changes, update its canonical owner in the same change. If
|
||||
ownership moves, remove the old definition and replace it with a link where
|
||||
navigation remains useful.
|
||||
|
||||
Before completing documentation work:
|
||||
|
||||
- verify affected behavior and examples;
|
||||
- check commands, flags, fields, defaults, schemas, and paths against their
|
||||
implementation;
|
||||
- keep unimplemented behavior in the roadmap, subject to the ADR exception;
|
||||
- remove stale references and validate links;
|
||||
- confirm that non-owning documents summarize and link rather than redefine;
|
||||
- confirm that no secrets or sensitive private data were added.
|
||||
168
docs/policy/release.md
Normal file
168
docs/policy/release.md
Normal file
@@ -0,0 +1,168 @@
|
||||
# Release Policy
|
||||
|
||||
**Base Policy Version:** 1.0
|
||||
|
||||
**Last Updated**: 2026-09-23
|
||||
|
||||
## Purpose
|
||||
|
||||
This policy defines the minimum requirements for selecting, validating,
|
||||
publishing, verifying, and correcting releases. Its goal is to make every
|
||||
release intentional, reproducible from an exact source revision, documented
|
||||
for consumers, and safe from retrospective replacement.
|
||||
|
||||
The repository-specific commands that implement these requirements belong in
|
||||
`docs/release.md`. Historical release summaries belong in `docs/releases/`.
|
||||
This is an Audita-specific adaptation of the base policy, not a requirement to
|
||||
implement the standard unchanged. Adapt it further when Audita's needs warrant;
|
||||
the canonical procedure must agree with the adopted policy.
|
||||
|
||||
## Release Model
|
||||
|
||||
A release identifies one exact source commit with an immutable version tag and
|
||||
its checked-in release note. The primary release branch is `main`.
|
||||
|
||||
Release success means that the intended version tag has been created and
|
||||
successfully pushed to the authoritative remote with its source commit. The
|
||||
release procedure ends at that point. It must never wait for CI to complete or
|
||||
require successful CI status, artifact publication, or consumer installation as
|
||||
a completion gate.
|
||||
|
||||
CI builds and publishes binaries and checksums asynchronously from the tagged
|
||||
source. Delayed or failed automation does not reverse release success. Artifact
|
||||
availability is a separate status and must not be claimed merely because the tag
|
||||
was pushed. Published artifacts must be traceable to the tagged source.
|
||||
|
||||
Automation must not silently change the source candidate, move a published tag,
|
||||
or replace an existing published artifact.
|
||||
|
||||
## Supported Platforms
|
||||
|
||||
| Platform | Support |
|
||||
| --- | --- |
|
||||
| Linux amd64 | Required production target |
|
||||
| Linux arm64 | Required production target |
|
||||
| macOS arm64 | Best effort for local development; failures do not block a release or Linux artifact publication |
|
||||
|
||||
Windows and macOS amd64 are unsupported. Neither their builds nor tests are
|
||||
release requirements. Native macOS testing and access to a Mac are not release
|
||||
prerequisites.
|
||||
|
||||
## Versioning And Compatibility
|
||||
|
||||
Stable releases use Semantic Versioning in the form `vMAJOR.MINOR.PATCH`, with
|
||||
no leading zeroes. Each version must be unused locally and on the authoritative
|
||||
remote before publication begins.
|
||||
|
||||
Unless an adopted policy explicitly supports and defines prereleases,
|
||||
prerelease versions are not permitted.
|
||||
|
||||
For releases at or after `v1.0.0`:
|
||||
|
||||
- a major release may make intentional incompatible changes;
|
||||
- a minor release may add backward-compatible behavior; and
|
||||
- a patch release may make backward-compatible fixes.
|
||||
|
||||
Before `v1.0.0`, a minor release may change a documented public, operational,
|
||||
configuration, data, integration, or compatibility contract when the release
|
||||
note explains the impact and required consumer action. A patch release must
|
||||
not intentionally break those contracts within its minor line.
|
||||
|
||||
A repository with additional compatibility promises must define them in its
|
||||
adopted policy or canonical contract documentation. A release note may explain
|
||||
a compatibility change, but it must not redefine the current compatibility
|
||||
contract.
|
||||
|
||||
## Release Documentation
|
||||
|
||||
Every release made under this policy must include
|
||||
`docs/releases/<version>.md` in the tagged source tree. The filename must use
|
||||
the complete version tag, and the document must begin with a title containing
|
||||
the product name and that exact version.
|
||||
|
||||
Release notes must contain a short summary and any relevant compatibility or
|
||||
upgrade instructions. Fixed sections and explicit no-action boilerplate are not
|
||||
required. Keep notes proportionate to the change.
|
||||
|
||||
Release notes are immutable historical summaries. They should link to current
|
||||
canonical documentation for exact behavior. Every affected canonical document
|
||||
must be updated in the same candidate; a release note must not substitute for
|
||||
current documentation or defer required documentation changes.
|
||||
|
||||
## Candidate Preparation And Validation
|
||||
|
||||
Use the minimum gates needed for confidence in Audita's release. Provide one
|
||||
local command for the selected automated checks, following the
|
||||
[testing policy](testing.md). Run the deterministic, offline default test suite
|
||||
once for the candidate and verify that both required Linux targets build.
|
||||
Avoid duplicate test runs solely to satisfy separate release phases.
|
||||
|
||||
Additional checks must address a concrete risk not already covered. Do not
|
||||
inherit a mandatory linting, documentation audit, platform matrix, or live-service
|
||||
checklist solely from the base policy. Documentation migration validation is
|
||||
separate work, not automatically a recurring release gate.
|
||||
|
||||
Automate mechanical guards. Do not require manual approvals, sign-offs, laptop
|
||||
testing, or post-publication inspection as release gates. Version selection and
|
||||
a concise release summary remain preparation inputs, not separate approval rounds.
|
||||
|
||||
## Publication Guards
|
||||
|
||||
The procedure must automate the small set of checks needed to publish the
|
||||
intended source under a fresh version:
|
||||
|
||||
- valid, unused version on the authoritative remote;
|
||||
- clean worktree and index on the primary release branch;
|
||||
- the exact candidate that passed the local checks, including its matching note;
|
||||
- publication of that source commit and the exact intended tag.
|
||||
|
||||
Do not require remote CI success. Avoid an additional manual confirmation for
|
||||
each mechanical guard. Publish explicit refs rather than all local tags.
|
||||
A successful push is the completion signal; a failed or ambiguous push must be
|
||||
resolved against remote ref state before declaring success or retrying.
|
||||
|
||||
## Immutability And Correction
|
||||
|
||||
Published version tags and published deliverables are immutable. Do not move,
|
||||
overwrite, recreate, or delete a published tag. Do not replace an artifact,
|
||||
image, package, checksum, signature, provenance statement, or release note
|
||||
under an already published version.
|
||||
|
||||
If validation fails before publication, correct and commit the candidate, then
|
||||
rerun the release command. Validation must apply to the exact candidate. An
|
||||
unpublished local tag may be removed after confirming that it was never published
|
||||
or used by a release channel.
|
||||
|
||||
If a defect is discovered after publication, leave the published release intact. Correct the defect on the primary branch, choose a
|
||||
new patch version or other semantically appropriate new version, write a new
|
||||
matching release note, and perform the complete release procedure again.
|
||||
|
||||
A distribution channel's supported yank, deprecation, or advisory mechanism
|
||||
may warn consumers away from a defective release, but it must not replace that
|
||||
release's contents or reuse its version.
|
||||
|
||||
## Repository Release Procedure
|
||||
|
||||
Maintain [docs/release.md](../release.md) as a short, executable maintainer runbook.
|
||||
It must identify prerequisites, authoritative remote, version and note preparation,
|
||||
one local validation command, automated source/tag guards, exact publication
|
||||
commands, and failure handling. It must name the asynchronous automation and link
|
||||
to this policy for supported platforms and release-success semantics.
|
||||
|
||||
Describe how to diagnose or retry failed artifact delivery separately from release
|
||||
completion. Missing artifacts may be published later from the same immutable tag;
|
||||
existing published artifacts must not be overwritten. A source defect requires a
|
||||
new version. No polling or CI waiting belongs in the release command.
|
||||
|
||||
## Audita Adaptations
|
||||
|
||||
The maintainer approved these deviations from the base policy on 2026-09-23:
|
||||
|
||||
- Audita's needs govern policy adaptation rather than strict template compliance.
|
||||
- Only Linux amd64 and arm64 are required; macOS arm64 is best effort.
|
||||
- A successful tag push completes the release; asynchronous CI cannot block it.
|
||||
- Gates are minimal, automated where practical, and avoid redundant validation
|
||||
and human intervention.
|
||||
|
||||
The [release procedure](../release.md) and its linked scripts implement these
|
||||
requirements. CI delivery remains asynchronous.
|
||||
300
docs/policy/testing.md
Normal file
300
docs/policy/testing.md
Normal file
@@ -0,0 +1,300 @@
|
||||
# Testing Policy
|
||||
|
||||
**Base Policy Version:** 1.0
|
||||
|
||||
**Last Updated**: 2026-09-23
|
||||
|
||||
## Purpose
|
||||
|
||||
Our tests exist to make **incorrect changes expensive and correct changes cheap**.
|
||||
|
||||
We do not optimize for test count, line coverage, exhaustive isolation, or the fewest possible tests. We optimize for sufficient confidence in important behavior while imposing as little unnecessary friction as possible on future development.
|
||||
|
||||
## Every test has a cost
|
||||
|
||||
Testing is not an unqualified good. Every test imposes both an immediate cost and a continuing lifetime cost.
|
||||
|
||||
A test must be:
|
||||
|
||||
- written and reviewed;
|
||||
- understood by future maintainers and coding agents;
|
||||
- executed in local and CI workflows;
|
||||
- diagnosed when it fails;
|
||||
- updated when legitimate behavior changes;
|
||||
- maintained as fixtures, APIs, and dependencies evolve; and
|
||||
- removed or rewritten when it becomes redundant, brittle, misleading, or obsolete.
|
||||
|
||||
Tests also create cognitive and architectural friction. They can constrain refactoring, duplicate policy, slow feedback loops, add noise to failures, and cause harmless implementation changes to require unrelated edits across the suite.
|
||||
|
||||
A test is warranted only when the confidence it provides justifies these costs.
|
||||
|
||||
Apply this cost-benefit analysis at two levels:
|
||||
|
||||
1. **Per test:** What realistic defect does this test detect, how consequential would that defect be, and is that protection worth the test's lifetime cost?
|
||||
2. **Across the suite:** Does this collection provide materially more confidence than a smaller, simpler suite would?
|
||||
|
||||
The preferred test suite is a **lean suite that provides sufficient confidence in the risks that matter, without redundant or low-value tests**. We seek sufficient confidence with the least unnecessary testing friction, not the fewest possible tests.
|
||||
|
||||
Some friction is intentional. Tests should make dangerous changes—such as breaking compatibility, corrupting data, violating security boundaries, or reintroducing subtle bugs—require deliberate review. They should not make ordinary internal changes needlessly expensive.
|
||||
|
||||
The cost of a test is not a reason to omit testing by default. Do not cite maintenance cost abstractly. When omitting a plausible test, be able to state why the protected failure is low-risk, already covered, obvious, reversible, or cheaper to detect elsewhere. For consequential, subtle, or difficult-to-observe behavior, the presumption should favor testing.
|
||||
|
||||
## Default testing style
|
||||
|
||||
Use a **classical/Detroit-style** approach:
|
||||
|
||||
- Test observable behavior, resulting state, contracts, and invariants.
|
||||
- Use real internal collaborators when they are fast and deterministic.
|
||||
- Use fakes, stubs, or mocks primarily at expensive, nondeterministic, destructive, or external boundaries.
|
||||
- Prefer package-level behavioral tests over tests coupled to private helpers or internal call sequences.
|
||||
- Treat exact collaborator interactions as testable behavior only when the interaction itself is a requirement.
|
||||
|
||||
Examples of appropriate seams include clocks, randomness, subprocesses, remote APIs, object storage, email, and paid LLM calls.
|
||||
|
||||
## Test execution requirements
|
||||
|
||||
Tests in the default suite must be deterministic, offline, and independent of real credentials. They must not invoke paid APIs or depend on mutable external services. Tests that require live infrastructure must be explicitly opt-in and clearly separated from the default suite.
|
||||
|
||||
Control clocks, randomness, environment variables, and other process-global or machine-specific state when they affect behavior. Tests should be safe to run repeatedly and alongside other tests without depending on execution order or state left by an earlier test.
|
||||
|
||||
## What deserves tests
|
||||
|
||||
To the extent the following features are present in the application, prioritize tests for:
|
||||
|
||||
1. Public and package-level contracts.
|
||||
2. Domain rules and important invariants.
|
||||
3. Boundary conditions and malformed input.
|
||||
4. Failure handling, cancellation, retries, recovery, and partial success.
|
||||
5. Serialization, schemas, compatibility, and round trips.
|
||||
6. Previously observed or plausible regressions.
|
||||
7. Representative integration and end-to-end workflows.
|
||||
|
||||
A package-level contract is behavior relied upon by another package or major collaborator, not every observable detail of a package implementation.
|
||||
|
||||
For behavior involving **data integrity, destructive operations, compatibility, security, concurrency, idempotency, or recovery**, presume that durable tests are required unless the behavior is already credibly protected at another layer.
|
||||
|
||||
Do not add tests merely because a function, branch, or line exists. Do not add a test when the same meaningful risk is already adequately protected elsewhere.
|
||||
|
||||
## Choose the right test boundary
|
||||
|
||||
Test through the narrowest stable boundary that expresses the behavior clearly.
|
||||
|
||||
This is often the package API, but it may instead be:
|
||||
|
||||
- a smaller pure function when dense domain logic is most clearly isolated there;
|
||||
- a package-level operation when several internal collaborators jointly produce the behavior; or
|
||||
- a larger integration boundary when correctness emerges from interaction with a real dependency.
|
||||
|
||||
Do not force all behavior through oversized end-to-end tests. Do not test every private helper merely because it exists. Choose the boundary that gives durable confidence with the least incidental coupling.
|
||||
|
||||
## Test behavior, not implementation
|
||||
|
||||
A test should protect a decision, contract, or invariant. A test must not memorialize the current implementation.
|
||||
|
||||
Before adding or retaining a test, ask:
|
||||
|
||||
> What realistic defect would this test catch?
|
||||
|
||||
A test is suspect when its main purpose is to detect that someone:
|
||||
|
||||
- changed an internal constant;
|
||||
- renamed or split a private helper;
|
||||
- reordered equivalent internal operations;
|
||||
- changed incidental formatting;
|
||||
- replaced one correct algorithm with another; or
|
||||
- refactored internal object structure without changing behavior.
|
||||
|
||||
Refactoring should normally require no test edits unless the refactored structure is itself part of the contract.
|
||||
|
||||
A test can be factually correct and still have negative value. Accurately describing current behavior is not sufficient; the protected behavior must be important enough to justify the future friction.
|
||||
|
||||
## Expected effects of different changes
|
||||
|
||||
Use the following expectations when evaluating test failures and test maintenance:
|
||||
|
||||
| Change | Expected effect on tests |
|
||||
|---|---|
|
||||
| Internal refactor that preserves behavior | Existing tests should normally remain unchanged and continue to pass. |
|
||||
| Change to an internal default with no contractual significance | Behavioral tests should normally remain unchanged; tests should derive expectations from configuration or relationships rather than duplicate the old value. |
|
||||
| Intentional change to public behavior, policy, schema, or compatibility guarantees | The relevant tests should be reviewed and changed deliberately. |
|
||||
| Accidental violation of a contract or invariant | Tests should fail; fix the production code rather than rewriting the tests to accept the defect. |
|
||||
|
||||
A test failing is not the same as a test needing to be edited. Many tests may correctly fail because of one production defect. The maintenance smell is a correct internal change that requires unrelated expectation updates throughout the suite.
|
||||
|
||||
## Separate mechanism from policy
|
||||
|
||||
Configurable thresholds and defaults must not be duplicated throughout the test suite.
|
||||
|
||||
For example, do not encode an internal concurrency limit indirectly:
|
||||
|
||||
```go
|
||||
// Production policy:
|
||||
const maxConcurrency = 4
|
||||
|
||||
// Brittle test:
|
||||
err := startProcesses(5)
|
||||
require.Error(t, err)
|
||||
```
|
||||
|
||||
Instead, test the mechanism relationally:
|
||||
|
||||
```go
|
||||
const limit = 2
|
||||
runner := NewRunner(limit)
|
||||
|
||||
require.NoError(t, runner.Start(limit))
|
||||
require.ErrorIs(t, runner.Start(limit+1), ErrTooMuchConcurrency)
|
||||
```
|
||||
|
||||
The test should prove:
|
||||
|
||||
- the configured limit is accepted; and
|
||||
- one beyond the configured limit is rejected.
|
||||
|
||||
The production default should be tested exactly only when its literal value is itself a public, operational, safety, protocol, or compatibility requirement.
|
||||
|
||||
Apply the same rule to limits, timeouts, capacities, retry counts, and ranges: test relationships and behavior, not duplicated literals.
|
||||
|
||||
For concurrency limits, test both kinds of behavior when relevant:
|
||||
|
||||
1. **Configuration enforcement:** invalid or excessive requested values are handled correctly.
|
||||
2. **Runtime enforcement:** observed peak concurrency never exceeds the configured limit.
|
||||
|
||||
Use a test-controlled limit and measure the behavior relative to that limit. Do not merely assert today's default value.
|
||||
|
||||
## Avoid semantic duplication across layers
|
||||
|
||||
Each behavior should have a clear test owner.
|
||||
|
||||
- Parser tests own parsing cases.
|
||||
- Validator tests own validation rules.
|
||||
- Domain tests own transformations and invariants.
|
||||
- Adapter tests own external integration behavior.
|
||||
- Orchestrator tests own coordination and failure propagation.
|
||||
- CLI tests own argument and configuration mapping.
|
||||
- End-to-end tests prove that representative assembled workflows work.
|
||||
|
||||
Higher-level tests should not repeat every lower-level case. A single intentional policy change should not require unrelated edits across many test files.
|
||||
|
||||
Tests that are individually reasonable may still be collectively redundant. Evaluate the marginal value of each additional test in light of the protection already provided by the rest of the suite.
|
||||
|
||||
## Use test doubles deliberately
|
||||
|
||||
Choose the least elaborate test double that provides the required control or observation.
|
||||
|
||||
As a default:
|
||||
|
||||
1. Prefer real collaborators when they are fast and deterministic.
|
||||
2. Use small in-memory fakes when realistic stateful behavior is helpful.
|
||||
3. Use stubs when a dependency only needs to provide controlled responses.
|
||||
4. Use mocks when the interaction itself is contractual.
|
||||
|
||||
Mocks are appropriate when the contract includes facts such as:
|
||||
|
||||
- a notification is sent exactly once;
|
||||
- a transaction is committed only after successful writes;
|
||||
- cancellation reaches a subprocess;
|
||||
- an expensive API is called no more than once; or
|
||||
- a security audit event is emitted.
|
||||
|
||||
Do not use mocks merely to isolate every object or reproduce the implementation's call graph.
|
||||
|
||||
## Go-specific guidance
|
||||
|
||||
Use:
|
||||
|
||||
- table-driven tests for meaningful behavioral categories and boundaries;
|
||||
- `t.TempDir()` for real filesystem behavior;
|
||||
- `httptest.Server` for realistic HTTP interactions;
|
||||
- fuzz tests for parsers, normalization, path handling, and broad input spaces;
|
||||
- golden files only when the complete output is intentionally stable;
|
||||
- integration tests where correctness depends on component interaction; and
|
||||
- a small number of representative end-to-end tests.
|
||||
|
||||
Avoid exact error-string assertions unless the wording is itself contractual. Prefer `errors.Is`, `errors.As`, typed errors, or structured error fields.
|
||||
|
||||
At CLI boundaries, prefer exit classifications, structured output, and the smallest stable semantic fragment needed to identify the error. Do not snapshot complete diagnostic wording unless it is contractual.
|
||||
|
||||
Golden-file updates must require an explicit local flag. CI must not update golden files automatically, and reviewers must inspect the semantic diff before accepting an update.
|
||||
|
||||
Keep tests readable and direct. Test helpers and fixture frameworks must earn their own maintenance cost; do not build elaborate test infrastructure for small or isolated needs.
|
||||
|
||||
## Coverage
|
||||
|
||||
Coverage is a diagnostic, not a target.
|
||||
|
||||
Use it to find untested critical branches and unexpectedly weak packages. Do not write low-value tests solely to increase a percentage, and do not infer test quality from coverage alone.
|
||||
|
||||
Pure domain logic will often warrant higher coverage than CLI wiring or external adapters. Uneven coverage is acceptable when it reflects risk.
|
||||
|
||||
Increasing coverage is valuable only when the newly covered behavior protects a meaningful risk at an acceptable cost.
|
||||
|
||||
## Regression tests
|
||||
|
||||
A bug fix should normally include a regression test that fails before the fix and passes afterward.
|
||||
|
||||
Retain the test when the defect could realistically recur and its consequences justify the ongoing cost. Prefer the narrowest durable test of the violated contract or invariant; do not preserve accidental implementation details from the original bug.
|
||||
|
||||
Not every historical bug requires a permanent test. If the underlying design has made recurrence impossible, the test has become redundant, or a stronger invariant test now subsumes it, remove or consolidate it.
|
||||
|
||||
## Deleting or rewriting tests
|
||||
|
||||
Tests are maintained code, not permanent historical artifacts.
|
||||
|
||||
Delete or rewrite a test when its maintenance cost exceeds the confidence it provides.
|
||||
|
||||
Strong candidates include tests that:
|
||||
|
||||
- require updates after harmless internal changes;
|
||||
- directly assert private constants without protecting a real contract;
|
||||
- duplicate the same policy across several layers;
|
||||
- verify mock choreography rather than outcomes;
|
||||
- snapshot large amounts of incidental output;
|
||||
- test trivial private helpers already exercised through stable package behavior;
|
||||
- protect risks already covered more effectively elsewhere;
|
||||
- are flaky, misleading, obsolete, or disproportionately expensive to diagnose; or
|
||||
- no longer correspond to a plausible failure mode.
|
||||
|
||||
Several brittle tests may encode one genuine requirement. Replace them with one durable behavior-level or invariant test rather than preserving all of them.
|
||||
|
||||
Deleting a low-value test can improve the quality of the suite by reducing noise, maintenance burden, and friction around legitimate change.
|
||||
|
||||
## Reviewing a proposed test
|
||||
|
||||
Use the following questions when the value, boundary, or durability of a proposed test is not self-evident. Significant test additions should be reviewable against them, but written answers are not required for every routine test.
|
||||
|
||||
1. What realistic defect would it catch?
|
||||
2. How likely is that defect?
|
||||
3. How consequential would it be?
|
||||
4. Is the behavior already protected elsewhere?
|
||||
5. At which layer should this behavior be owned?
|
||||
6. Does the test assert a durable contract or an incidental implementation detail?
|
||||
7. Could the implementation be refactored without changing the behavior and without editing this test?
|
||||
8. What should cause this test to fail?
|
||||
9. What legitimate changes should not cause this test to fail?
|
||||
10. What ongoing maintenance, execution, and diagnostic cost will the test impose?
|
||||
11. Is there a smaller or more direct test that protects the same risk?
|
||||
|
||||
Do not add the test when its expected lifetime cost exceeds its expected protective value.
|
||||
|
||||
When deciding not to test plausible behavior, record or be able to explain why the risk is low, already protected, obvious, reversible, or cheaper to detect elsewhere.
|
||||
|
||||
## Definition of sufficient
|
||||
|
||||
A test suite is sufficient when:
|
||||
|
||||
- important contracts and invariants are protected;
|
||||
- meaningful boundaries and failure modes are exercised;
|
||||
- realistic and consequential regressions are credibly protected against silent recurrence;
|
||||
- behavior involving data integrity, destructive operations, compatibility, security, concurrency, idempotency, and recovery is credibly protected;
|
||||
- important external boundaries have realistic integration coverage;
|
||||
- representative complete workflows are tested;
|
||||
- failures provide useful signal rather than redundant noise;
|
||||
- legitimate internal changes usually do not require test edits; and
|
||||
- additional tests would mostly repeat existing protection or preserve inconsequential implementation details.
|
||||
|
||||
Sufficiency is a risk judgment, not a coverage percentage or test count. Reassess it as the application, its users, and the consequences of failure evolve.
|
||||
|
||||
The governing rule is:
|
||||
|
||||
> Test heavily where failure is consequential, subtle, or difficult to detect after the fact. Test lightly where failure is obvious, reversible, and inexpensive—and retain no test whose lifetime cost exceeds the confidence it provides.
|
||||
130
docs/release.md
Normal file
130
docs/release.md
Normal file
@@ -0,0 +1,130 @@
|
||||
# Releasing Audita
|
||||
|
||||
The [release policy](policy/release.md) defines versioning, supported platforms,
|
||||
immutability, and success. This runbook implements it with
|
||||
[scripts/release.sh](../scripts/release.sh). Publication ends after a successful
|
||||
source/tag push; binary delivery runs asynchronously.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Use Bash, Git, and Go 1.25 or newer (CI uses Go 1.25), with dependencies available
|
||||
locally or downloadable. Tests themselves are offline. Work from a
|
||||
checkout of `main` with permission to push that branch and tags to `origin`.
|
||||
For this repository, `origin` is `https://gitea.maximumdirect.net/eric/audita.git`.
|
||||
The script uses the checkout's configured `origin`; set it correctly before use.
|
||||
The remote must support atomic pushes. No CI credentials or Mac are needed locally.
|
||||
|
||||
CI uses the existing `GITEA_RELEASE_TOKEN` secret for asynchronous asset delivery.
|
||||
macOS arm64 is a best-effort development environment; no macOS binary is published.
|
||||
|
||||
## Prepare the Candidate
|
||||
|
||||
Choose an unused version following the policy. Write
|
||||
`docs/releases/<version>.md`, beginning with `# Audita <version>`, followed by a
|
||||
short summary and any compatibility or upgrade instructions relevant to the
|
||||
release. There are no mandatory boilerplate sections. Update affected canonical
|
||||
documentation with behavior changes.
|
||||
|
||||
Commit the intended changes, including the note, on `main`. The release script
|
||||
requires a clean index and worktree and does not create commits. There is no
|
||||
separate approval round. Published notes describe their tagged release; existing
|
||||
release history does not need to be reconstructed.
|
||||
|
||||
## Validate and Publish
|
||||
|
||||
From the repository root, substitute the chosen version:
|
||||
|
||||
```sh
|
||||
bash scripts/release.sh --publish v0.2.0
|
||||
```
|
||||
|
||||
This single invocation:
|
||||
|
||||
1. Checks the version, branch, clean candidate, matching note title, and tag absence.
|
||||
2. Runs the default offline test suite once, then builds both required Linux targets
|
||||
in a temporary directory using the same build script as CI.
|
||||
3. Checks that the candidate and worktree are unchanged and that it includes the
|
||||
current `origin/main` history.
|
||||
4. Creates the version tag at that commit and atomically pushes the exact source
|
||||
commit to `main` and the exact tag to `origin`.
|
||||
5. Reports release success and exits without polling CI or checking asset availability.
|
||||
|
||||
To validate a prepared candidate without remote access, creating a tag, or pushing:
|
||||
|
||||
```sh
|
||||
bash scripts/release.sh --check v0.2.0
|
||||
```
|
||||
|
||||
This is an optional diagnostic command, not a required step before `--publish`.
|
||||
Publication validates within its own invocation; it does not use persistent cached
|
||||
results. Both modes require the candidate note and a clean `main` checkout.
|
||||
|
||||
## Artifact Delivery
|
||||
|
||||
[Woodpecker](../.woodpecker/release.yml) runs on tag events. It invokes
|
||||
[scripts/build-release.sh](../scripts/build-release.sh), then publishes:
|
||||
|
||||
- `audita-<version>-linux-amd64`
|
||||
- `audita-<version>-linux-arm64`
|
||||
- `SHA256SUMS`
|
||||
|
||||
Builds disable CGO and use the shared script's release flags. The version is in the
|
||||
asset name and source tag; Audita has no CLI version command. Cross-compilation
|
||||
checks compilation, not native execution on each architecture.
|
||||
|
||||
The CI job status and logs show delivery progress or failure. Assets appear on
|
||||
the matching Gitea release page when delivery succeeds. A successful tag push
|
||||
must not be reported as confirmation that these files are available.
|
||||
|
||||
Consumers can download the binary for their Linux architecture and `SHA256SUMS`
|
||||
from that page, check its listed SHA-256 digest with `sha256sum`, and make it
|
||||
executable. Optional consumer checks are not maintainer release gates. Digests
|
||||
detect file mismatches; they are not an independent signature of authenticity.
|
||||
|
||||
## Failure and Retry
|
||||
|
||||
**Local validation fails:** correct the issue, commit any changes, and rerun the
|
||||
publication command. A failure before tag creation publishes nothing. Temporary
|
||||
local build files are removed on exit.
|
||||
|
||||
**Push fails or the connection is interrupted:** the script retains the local tag
|
||||
and reports failure rather than guessing whether publication succeeded. Inspect
|
||||
remote state with the actual version:
|
||||
|
||||
```sh
|
||||
git rev-parse 'v0.2.0^{commit}'
|
||||
git ls-remote origin refs/heads/main refs/tags/v0.2.0
|
||||
```
|
||||
|
||||
If the remote tag identifies the intended candidate and its source is on remote
|
||||
`main`, the release succeeded; no CI waiting is needed. An atomic push normally
|
||||
updates both refs or neither. If `main` has advanced since publication, fetch it
|
||||
and check candidate ancestry rather than requiring tip equality.
|
||||
|
||||
If the tag is absent remotely and was never published or used by a release
|
||||
channel, remove only that unpublished local tag (`git tag -d v0.2.0`), resolve
|
||||
any rejected branch update, and rerun the command. If the remote tag has a
|
||||
different target, do not overwrite it; choose a fresh version. Never force a
|
||||
branch or tag update to bypass the guards.
|
||||
|
||||
**CI delivery fails:** the release remains successful. Inspect the failed tag
|
||||
job's logs and retry that same tagged job for a transient failure. Publication is
|
||||
configured to skip existing filenames and not overwrite assets. Keep the same
|
||||
source and build environment when retrying; inspect any existing assets/checksum
|
||||
manifest before filling gaps. Do not regenerate and replace a published manifest
|
||||
or binary. If an existing manifest cannot describe the missing rebuilt assets,
|
||||
use a new version instead. A pipeline or source defect also requires a corrected
|
||||
commit and new version. An outage does not require moving the tag.
|
||||
|
||||
**A released source defect is discovered:** fix it on `main` and publish a new
|
||||
appropriate version with a note explaining the correction. Leave the old tag and
|
||||
published files intact.
|
||||
|
||||
## Maintaining the Release Workflow
|
||||
|
||||
The shared build script keeps local and CI build settings aligned. Tests in
|
||||
[scripts/release_test.go](../scripts/release_test.go) exercise publication against
|
||||
temporary local Git remotes, including guard failures and rejected pushes, with
|
||||
Go execution stubbed to avoid recursive builds/tests. They run in `go test ./...`.
|
||||
Changes to CI delivery should preserve explicit asset names, checksums, and
|
||||
non-overwriting publication. CI delivery is separate from the local success path.
|
||||
8
docs/releases/README.md
Normal file
8
docs/releases/README.md
Normal file
@@ -0,0 +1,8 @@
|
||||
# Release Notes
|
||||
|
||||
- [Audita v1.1.0](v1.1.0.md)
|
||||
|
||||
See the [release documentation requirements](../policy/release.md#release-documentation).
|
||||
Use the [release procedure](../release.md) to prepare and publish new releases.
|
||||
|
||||
This index does not reconstruct or replace records for existing tags.
|
||||
22
docs/releases/v1.1.0.md
Normal file
22
docs/releases/v1.1.0.md
Normal file
@@ -0,0 +1,22 @@
|
||||
# Audita v1.1.0
|
||||
|
||||
This release refreshes Audita's documentation and repository policies and adds a
|
||||
small, automated release workflow. Transcript-processing behavior, CLI syntax,
|
||||
configuration, and serialized output formats are unchanged from v1.0.0.
|
||||
|
||||
- Consolidated CLI, configuration, operations, and integration references, including
|
||||
a dedicated reference for output files and reports.
|
||||
- Added contributor navigation and agent, testing, and release policies; retired
|
||||
speculative troubleshooting documentation and the completed migration roadmap.
|
||||
- Added local release validation and publication scripts, with publication complete
|
||||
after the source and version tag are pushed. CI delivers binaries asynchronously.
|
||||
- Isolated CLI tests from host configuration for a reliable local test baseline.
|
||||
|
||||
Release binaries are now provided only for Linux amd64 and Linux arm64. Windows
|
||||
and Intel macOS are no longer supported. macOS arm64 remains a best-effort local
|
||||
build/development environment; macOS binaries are no longer published.
|
||||
|
||||
Linux users need no configuration or data migration. Users of previously published
|
||||
Windows or macOS binaries should account for the distribution changes above before
|
||||
upgrading. See the [release procedure](../release.md) for artifact delivery and the
|
||||
[contributor guide](../development.md) for local build instructions.
|
||||
17
docs/roadmap/runtime-behavior.md
Normal file
17
docs/roadmap/runtime-behavior.md
Normal file
@@ -0,0 +1,17 @@
|
||||
# Runtime Behavior Decisions
|
||||
|
||||
**Last updated:** 2026-09-23
|
||||
|
||||
The documentation migration identified these existing behaviors for separate
|
||||
product consideration. No behavior change is approved or implemented by this list.
|
||||
|
||||
- Decide whether the `never` retention selection should delete successful-run
|
||||
diagnostics; current behavior is documented in
|
||||
[operations](../operations.md#retention-behavior).
|
||||
- Decide whether an explicit zero retry count should disable retries; current
|
||||
adapter behavior is documented in [configuration](../config.md).
|
||||
- Decide whether missing or null input timestamps should be rejected; current
|
||||
decoding is documented in the [input contract](../integrations/transcript-glossary-files.md).
|
||||
|
||||
These decisions are independent of the completed documentation migration and
|
||||
release workflow.
|
||||
6
examples/minimal-config.yml
Normal file
6
examples/minimal-config.yml
Normal file
@@ -0,0 +1,6 @@
|
||||
version: 1
|
||||
output:
|
||||
schema: bare-segments
|
||||
llm:
|
||||
proposal:
|
||||
api_key_env: AUDITA_LLM_API_KEY
|
||||
42
examples/production-config.yml
Normal file
42
examples/production-config.yml
Normal file
@@ -0,0 +1,42 @@
|
||||
version: 1
|
||||
pipeline:
|
||||
modules: [glossary, homophones, glossary, spoken_word, grammar]
|
||||
output:
|
||||
schema: audita-v1
|
||||
llm:
|
||||
proposal:
|
||||
base_url: https://openrouter.ai/api/v1
|
||||
model: openrouter/google/gemma-4-31b-it
|
||||
api_key_env: AUDITA_LLM_API_KEY
|
||||
timeout: 120s
|
||||
max_retries: 3
|
||||
validation:
|
||||
base_url: https://openrouter.ai/api/v1
|
||||
model: openrouter/google/gemma-4-31b-it
|
||||
api_key_env: AUDITA_VALIDATION_LLM_API_KEY
|
||||
timeout: 120
|
||||
max_retries: 3
|
||||
concurrency:
|
||||
total_llm: 2
|
||||
proposal_llm: 2
|
||||
validation_llm: 1
|
||||
chunking:
|
||||
# Requires at least this many normalized segments; omit for automatic chunking.
|
||||
target_sections: 8
|
||||
max_section_tokens: 8192
|
||||
min_section_tokens: 2048
|
||||
normalization:
|
||||
max_segment_gap: 4s
|
||||
ellipsis_gap: 3.5s
|
||||
max_segment_duration: 60s
|
||||
max_segment_tokens: 2048
|
||||
thresholds:
|
||||
glossary: 0.8
|
||||
homophones: 0.8
|
||||
spoken_word: 0.8
|
||||
grammar: 0.8
|
||||
context:
|
||||
description: "General context for domain vocabulary and speaker names."
|
||||
diagnostics:
|
||||
work_dir: /tmp/audita
|
||||
retention: auto
|
||||
6
examples/tiny-glossary.yaml
Normal file
6
examples/tiny-glossary.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
glossary:
|
||||
- name: Audita
|
||||
aliases:
|
||||
- audita
|
||||
category: product
|
||||
summary: The Audita transcript correction CLI.
|
||||
9
examples/tiny-transcript.json
Normal file
9
examples/tiny-transcript.json
Normal file
@@ -0,0 +1,9 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "A",
|
||||
"start": 0.0,
|
||||
"end": 1.2,
|
||||
"text": "hello world"
|
||||
}
|
||||
]
|
||||
11
go.mod
Normal file
11
go.mod
Normal file
@@ -0,0 +1,11 @@
|
||||
module gitea.maximumdirect.net/eric/audita
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require gopkg.in/yaml.v3 v3.0.1
|
||||
|
||||
require (
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
)
|
||||
17
go.sum
Normal file
17
go.sum
Normal file
@@ -0,0 +1,17 @@
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
36
internal/cli/main_test.go
Normal file
36
internal/cli/main_test.go
Normal file
@@ -0,0 +1,36 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// Tests supply their own overrides; inherited settings must not affect them.
|
||||
for _, entry := range os.Environ() {
|
||||
key, _, _ := strings.Cut(entry, "=")
|
||||
if strings.HasPrefix(key, "AUDITA_") || key == "OPENROUTER_API_KEY" {
|
||||
if err := os.Unsetenv(key); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dir, err := os.MkdirTemp("", "audita-cli-test-config-")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
configPath := filepath.Join(dir, "config.yml")
|
||||
if err := os.WriteFile(configPath, []byte("version: 1\n"), 0600); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// An explicit environment path prevents discovery of installed system config.
|
||||
// Individual tests can still override it with t.Setenv or --config.
|
||||
if err := os.Setenv("AUDITA_CONFIG", configPath); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
m.Run()
|
||||
}
|
||||
338
internal/cli/parity_test.go
Normal file
338
internal/cli/parity_test.go
Normal file
@@ -0,0 +1,338 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
)
|
||||
|
||||
type parityFixtureCase struct {
|
||||
Name string `json:"name"`
|
||||
TranscriptFile string `json:"transcript_file"`
|
||||
GlossaryFile string `json:"glossary_file"`
|
||||
ModulesCSV string `json:"modules_csv,omitempty"`
|
||||
ProposalResponsesFile string `json:"proposal_responses_file,omitempty"`
|
||||
ValidationResponsesFile string `json:"validation_responses_file,omitempty"`
|
||||
Env map[string]string `json:"env,omitempty"`
|
||||
Expect parityExpectation `json:"expect"`
|
||||
}
|
||||
|
||||
type parityExpectation struct {
|
||||
ExitCode int `json:"exit_code"`
|
||||
Status string `json:"status,omitempty"`
|
||||
ErrorPhase string `json:"error_phase,omitempty"`
|
||||
StdoutMode string `json:"stdout_mode,omitempty"` // empty|json
|
||||
StderrContains string `json:"stderr_contains,omitempty"`
|
||||
OutputTranscriptFile string `json:"output_transcript_file,omitempty"`
|
||||
ModuleInstances []string `json:"module_instances,omitempty"`
|
||||
ModuleCount int `json:"module_count,omitempty"`
|
||||
TotalAppliedChanges int `json:"total_applied_changes,omitempty"`
|
||||
TotalSkippedChanges int `json:"total_skipped_changes,omitempty"`
|
||||
FailedModuleInstance string `json:"failed_module_instance,omitempty"`
|
||||
ValidatorRejectedReasonCodes []string `json:"validator_rejected_reason_codes,omitempty"`
|
||||
ApplicationSkipReasonCodes []string `json:"application_skip_reason_codes,omitempty"`
|
||||
RequireErrorLog bool `json:"require_error_log,omitempty"`
|
||||
SecretMarkers []string `json:"secret_markers,omitempty"`
|
||||
ExpectedProposalCalls []string `json:"expected_proposal_calls,omitempty"`
|
||||
ExpectedValidationCalls []string `json:"expected_validation_calls,omitempty"`
|
||||
ModuleAppliedCounts []int `json:"module_applied_counts,omitempty"`
|
||||
ModuleRejectedCounts []int `json:"module_rejected_counts,omitempty"`
|
||||
ModuleSkipCounts []int `json:"module_skip_counts,omitempty"`
|
||||
MinResponsePayloadArtifacts int `json:"min_response_payload_artifacts,omitempty"`
|
||||
}
|
||||
|
||||
func TestParityFixtures(t *testing.T) {
|
||||
casePaths, err := filepath.Glob(parityFixturePath("*.case.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("glob parity fixtures: %v", err)
|
||||
}
|
||||
if len(casePaths) == 0 {
|
||||
t.Fatal("expected at least one parity fixture case")
|
||||
}
|
||||
|
||||
for _, casePath := range casePaths {
|
||||
fx := loadParityFixtureCase(t, casePath)
|
||||
t.Run(fx.Name, func(t *testing.T) {
|
||||
runParityFixtureCase(t, filepath.Dir(casePath), fx)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func loadParityFixtureCase(t *testing.T, casePath string) parityFixtureCase {
|
||||
t.Helper()
|
||||
var fx parityFixtureCase
|
||||
raw := readFile(t, casePath)
|
||||
if err := json.Unmarshal(raw, &fx); err != nil {
|
||||
t.Fatalf("parse parity case %q: %v", casePath, err)
|
||||
}
|
||||
if strings.TrimSpace(fx.Name) == "" {
|
||||
t.Fatalf("parity case %q missing name", casePath)
|
||||
}
|
||||
return fx
|
||||
}
|
||||
|
||||
func runParityFixtureCase(t *testing.T, caseDir string, fx parityFixtureCase) {
|
||||
t.Helper()
|
||||
for k, v := range fx.Env {
|
||||
t.Setenv(k, v)
|
||||
}
|
||||
|
||||
proposalClient := &fakeStructuredLLMClient{}
|
||||
validationClient := &fakeStructuredLLMClient{}
|
||||
|
||||
if strings.TrimSpace(fx.ProposalResponsesFile) != "" {
|
||||
raw := readFile(t, filepath.Join(caseDir, fx.ProposalResponsesFile))
|
||||
if err := json.Unmarshal(raw, &proposalClient.proposalResponses); err != nil {
|
||||
t.Fatalf("parse proposal responses: %v", err)
|
||||
}
|
||||
processProposalLLMClient = proposalClient
|
||||
}
|
||||
if strings.TrimSpace(fx.ValidationResponsesFile) != "" {
|
||||
raw := readFile(t, filepath.Join(caseDir, fx.ValidationResponsesFile))
|
||||
if err := json.Unmarshal(raw, &validationClient.validationResponses); err != nil {
|
||||
t.Fatalf("parse validation responses: %v", err)
|
||||
}
|
||||
processValidationLLMClient = validationClient
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
processProposalLLMClient = nil
|
||||
processValidationLLMClient = nil
|
||||
})
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
workDir := t.TempDir()
|
||||
outputPath := filepath.Join(t.TempDir(), "out.json")
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
args := []string{
|
||||
"process",
|
||||
filepath.Join(caseDir, fx.TranscriptFile),
|
||||
"--glossary",
|
||||
filepath.Join(caseDir, fx.GlossaryFile),
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
}
|
||||
// Keep stdout shape deterministic for parity tests.
|
||||
if fx.Expect.StdoutMode != "json" {
|
||||
args = append(args, "--output", outputPath)
|
||||
}
|
||||
if strings.TrimSpace(fx.ModulesCSV) != "" {
|
||||
args = append(args, "--modules", fx.ModulesCSV)
|
||||
}
|
||||
|
||||
exitCode := Run(args, &stdout, &stderr)
|
||||
if exitCode != fx.Expect.ExitCode {
|
||||
t.Fatalf("expected exit code %d, got %d stderr=%q", fx.Expect.ExitCode, exitCode, stderr.String())
|
||||
}
|
||||
|
||||
switch fx.Expect.StdoutMode {
|
||||
case "json":
|
||||
if _, err := json.Marshal(stdout.String()); err != nil {
|
||||
t.Fatalf("unexpected stdout marshal error: %v", err)
|
||||
}
|
||||
if !json.Valid(stdout.Bytes()) {
|
||||
t.Fatalf("expected JSON stdout, got %q", stdout.String())
|
||||
}
|
||||
default:
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("expected empty stdout, got %q", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
if fx.Expect.StderrContains != "" && !strings.Contains(stderr.String(), fx.Expect.StderrContains) {
|
||||
t.Fatalf("expected stderr to contain %q, got %q", fx.Expect.StderrContains, stderr.String())
|
||||
}
|
||||
|
||||
report := readProcessReport(t, reportPath)
|
||||
assertParityReport(t, report, fx.Expect)
|
||||
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
runDirReport := readProcessReport(t, filepath.Join(runDir, "report.json"))
|
||||
assertParityReport(t, runDirReport, fx.Expect)
|
||||
|
||||
if fx.Expect.RequireErrorLog {
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if strings.TrimSpace(fx.Expect.OutputTranscriptFile) != "" && fx.Expect.ExitCode == 0 {
|
||||
got := readFile(t, outputPath)
|
||||
want := readFile(t, filepath.Join(caseDir, fx.Expect.OutputTranscriptFile))
|
||||
assertJSONSemanticEqual(t, want, got)
|
||||
}
|
||||
|
||||
if len(fx.Expect.ExpectedProposalCalls) > 0 && !reflect.DeepEqual(proposalClient.calls, fx.Expect.ExpectedProposalCalls) {
|
||||
t.Fatalf("unexpected proposal calls: got %v want %v", proposalClient.calls, fx.Expect.ExpectedProposalCalls)
|
||||
}
|
||||
if len(fx.Expect.ExpectedValidationCalls) > 0 && !reflect.DeepEqual(validationClient.calls, fx.Expect.ExpectedValidationCalls) {
|
||||
t.Fatalf("unexpected validation calls: got %v want %v", validationClient.calls, fx.Expect.ExpectedValidationCalls)
|
||||
}
|
||||
|
||||
if len(fx.Expect.SecretMarkers) > 0 {
|
||||
assertNoSecretMarkers(t, reportPath, fx.Expect.SecretMarkers)
|
||||
assertNoSecretMarkersInTree(t, runDir, fx.Expect.SecretMarkers)
|
||||
}
|
||||
if fx.Expect.MinResponsePayloadArtifacts > 0 {
|
||||
matches, err := filepath.Glob(filepath.Join(runDir, "*", "*response-payload.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("glob response payload artifacts: %v", err)
|
||||
}
|
||||
if len(matches) < fx.Expect.MinResponsePayloadArtifacts {
|
||||
t.Fatalf("expected at least %d response payload artifacts, got %d", fx.Expect.MinResponsePayloadArtifacts, len(matches))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertParityReport(t *testing.T, report reporting.ProcessReport, exp parityExpectation) {
|
||||
t.Helper()
|
||||
if exp.Status != "" && report.Status != exp.Status {
|
||||
t.Fatalf("expected report status %q, got %q", exp.Status, report.Status)
|
||||
}
|
||||
if exp.ErrorPhase != "" && report.ErrorPhase != exp.ErrorPhase {
|
||||
t.Fatalf("expected report error_phase %q, got %q", exp.ErrorPhase, report.ErrorPhase)
|
||||
}
|
||||
if len(exp.ModuleInstances) > 0 {
|
||||
got := make([]string, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, mr.ModuleInstance)
|
||||
}
|
||||
if !reflect.DeepEqual(got, exp.ModuleInstances) {
|
||||
t.Fatalf("unexpected module instances: got %v want %v", got, exp.ModuleInstances)
|
||||
}
|
||||
}
|
||||
if exp.ModuleCount > 0 {
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.ModuleCount != exp.ModuleCount {
|
||||
t.Fatalf("expected module_count=%d, got %+v", exp.ModuleCount, report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
if exp.TotalAppliedChanges > 0 {
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.TotalAppliedChanges != exp.TotalAppliedChanges {
|
||||
t.Fatalf("expected total_applied_changes=%d, got %+v", exp.TotalAppliedChanges, report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
if exp.TotalSkippedChanges > 0 {
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.TotalSkippedChanges != exp.TotalSkippedChanges {
|
||||
t.Fatalf("expected total_skipped_changes=%d, got %+v", exp.TotalSkippedChanges, report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
if exp.FailedModuleInstance != "" {
|
||||
if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != exp.FailedModuleInstance {
|
||||
t.Fatalf("expected failed_module_instance=%q, got %+v", exp.FailedModuleInstance, report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
|
||||
if len(exp.ValidatorRejectedReasonCodes) > 0 {
|
||||
got := collectValidatorRejectedReasonCodes(report.ModuleResults)
|
||||
if !reflect.DeepEqual(got, exp.ValidatorRejectedReasonCodes) {
|
||||
t.Fatalf("unexpected validator rejected reason codes: got %v want %v", got, exp.ValidatorRejectedReasonCodes)
|
||||
}
|
||||
}
|
||||
if len(exp.ApplicationSkipReasonCodes) > 0 {
|
||||
got := collectApplicationSkipReasonCodes(report.ModuleResults)
|
||||
if !reflect.DeepEqual(got, exp.ApplicationSkipReasonCodes) {
|
||||
t.Fatalf("unexpected application skip reason codes: got %v want %v", got, exp.ApplicationSkipReasonCodes)
|
||||
}
|
||||
}
|
||||
if len(exp.ModuleAppliedCounts) > 0 {
|
||||
got := make([]int, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, len(mr.AppliedChanges))
|
||||
}
|
||||
if !reflect.DeepEqual(got, exp.ModuleAppliedCounts) {
|
||||
t.Fatalf("unexpected per-module applied counts: got %v want %v", got, exp.ModuleAppliedCounts)
|
||||
}
|
||||
}
|
||||
if len(exp.ModuleRejectedCounts) > 0 {
|
||||
got := make([]int, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, len(mr.ValidatorRejected))
|
||||
}
|
||||
if !reflect.DeepEqual(got, exp.ModuleRejectedCounts) {
|
||||
t.Fatalf("unexpected per-module rejected counts: got %v want %v", got, exp.ModuleRejectedCounts)
|
||||
}
|
||||
}
|
||||
if len(exp.ModuleSkipCounts) > 0 {
|
||||
got := make([]int, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, len(mr.SkippedChanges))
|
||||
}
|
||||
if !reflect.DeepEqual(got, exp.ModuleSkipCounts) {
|
||||
t.Fatalf("unexpected per-module skip counts: got %v want %v", got, exp.ModuleSkipCounts)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func collectValidatorRejectedReasonCodes(results []reporting.ModuleReport) []string {
|
||||
out := make([]string, 0)
|
||||
for _, mr := range results {
|
||||
for _, vr := range mr.ValidatorRejected {
|
||||
out = append(out, vr.ReasonCode)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func collectApplicationSkipReasonCodes(results []reporting.ModuleReport) []string {
|
||||
out := make([]string, 0)
|
||||
for _, mr := range results {
|
||||
for _, sk := range mr.SkippedChanges {
|
||||
out = append(out, string(sk.SkipReason))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func assertJSONSemanticEqual(t *testing.T, expected []byte, actual []byte) {
|
||||
t.Helper()
|
||||
var exp any
|
||||
var act any
|
||||
if err := json.Unmarshal(expected, &exp); err != nil {
|
||||
t.Fatalf("unmarshal expected json: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(actual, &act); err != nil {
|
||||
t.Fatalf("unmarshal actual json: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(exp, act) {
|
||||
t.Fatalf("JSON mismatch\nexpected=%s\nactual=%s", string(expected), string(actual))
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoSecretMarkers(t *testing.T, filePath string, markers []string) {
|
||||
t.Helper()
|
||||
raw := string(readFile(t, filePath))
|
||||
for _, marker := range markers {
|
||||
if marker != "" && strings.Contains(raw, marker) {
|
||||
t.Fatalf("secret marker %q leaked in %s", marker, filePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoSecretMarkersInTree(t *testing.T, root string, markers []string) {
|
||||
t.Helper()
|
||||
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil || d == nil || d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
raw := string(readFile(t, path))
|
||||
for _, marker := range markers {
|
||||
if marker != "" && strings.Contains(raw, marker) {
|
||||
t.Fatalf("secret marker %q leaked in %s", marker, path)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func parityFixturePath(name string) string {
|
||||
return filepath.Join("testdata", "parity", name)
|
||||
}
|
||||
121
internal/cli/process_flags.go
Normal file
121
internal/cli/process_flags.go
Normal file
@@ -0,0 +1,121 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"flag"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
)
|
||||
|
||||
type processOverrideBinding func(*config.CLIOverrides, processFlags)
|
||||
|
||||
var processOverrideBindings = map[string]processOverrideBinding{
|
||||
"modules": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ModulesCSV = flags.modules
|
||||
},
|
||||
"output-schema": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.OutputSchema = flags.outputSchema
|
||||
},
|
||||
"llm-api-key": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.PrimaryLLMAPIKey = flags.llmAPIKey
|
||||
},
|
||||
"validation-llm-api-key": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationLLMAPIKey = flags.validationLLMAPIKey
|
||||
},
|
||||
"model": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.PrimaryModel = flags.model
|
||||
},
|
||||
"validation-model": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationModel = flags.validationModel
|
||||
},
|
||||
"base-url": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.PrimaryBaseURL = flags.baseURL
|
||||
},
|
||||
"validation-base-url": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationBaseURL = flags.validationBaseURL
|
||||
},
|
||||
"llm-timeout-seconds": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.PrimaryLLMTimeoutSeconds = flags.llmTimeoutSeconds
|
||||
},
|
||||
"total-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.TotalLLMConcurrency = flags.totalLLMConcurrency
|
||||
},
|
||||
"proposal-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ProposalLLMConcurrency = flags.proposalLLMConcurrency
|
||||
},
|
||||
"llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.PrimaryLLMConcurrency = flags.llmConcurrency
|
||||
},
|
||||
"validation-llm-timeout-seconds": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationLLMTimeoutSeconds = flags.validationLLMTimeoutSeconds
|
||||
},
|
||||
"max-retries": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.MaxRetries = flags.maxRetries
|
||||
},
|
||||
"validation-max-retries": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationMaxRetries = flags.validationMaxRetries
|
||||
},
|
||||
"validation-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationLLMConcurrency = flags.validationLLMConcurrency
|
||||
},
|
||||
"validation-max-prompt-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.ValidationMaxPromptTokens = flags.validationMaxPromptTokens
|
||||
},
|
||||
"max-section-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.MaxSectionTokens = flags.maxSectionTokens
|
||||
},
|
||||
"min-section-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.MinSectionTokens = flags.minSectionTokens
|
||||
},
|
||||
"target-sections": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.TargetSections = flags.targetSections
|
||||
},
|
||||
"glossary-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.GlossaryConfidenceThreshold = flags.glossaryConfidenceThreshold
|
||||
},
|
||||
"grammar-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.GrammarConfidenceThreshold = flags.grammarConfidenceThreshold
|
||||
},
|
||||
"homophones-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.HomophonesConfidenceThreshold = flags.homophonesConfidenceThreshold
|
||||
},
|
||||
"spoken-word-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.SpokenWordConfidenceThreshold = flags.spokenWordConfidenceThreshold
|
||||
},
|
||||
"normalize-max-segment-gap": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.NormalizeMaxSegmentGap = flags.normalizeMaxSegmentGap
|
||||
},
|
||||
"normalize-ellipsis-gap": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.NormalizeEllipsisGap = flags.normalizeEllipsisGap
|
||||
},
|
||||
"normalize-max-segment-duration": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.NormalizeMaxSegmentDuration = flags.normalizeMaxSegmentDuration
|
||||
},
|
||||
"normalize-max-segment-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.NormalizeMaxSegmentTokens = flags.normalizeMaxSegmentTokens
|
||||
},
|
||||
"transcript-description": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.TranscriptDescription = flags.transcriptDescription
|
||||
},
|
||||
"work-dir": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.WorkDir = flags.workDir
|
||||
},
|
||||
"work-dir-retention": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||
overrides.WorkDirRetention = flags.workDirRetention
|
||||
},
|
||||
}
|
||||
|
||||
func processCLIOverrides(fs *flag.FlagSet, flags processFlags) (config.CLIOverrides, bool) {
|
||||
overrides := config.CLIOverrides{}
|
||||
explicitModules := false
|
||||
fs.Visit(func(f *flag.Flag) {
|
||||
if f.Name == "modules" {
|
||||
explicitModules = true
|
||||
}
|
||||
binding, ok := processOverrideBindings[f.Name]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
binding(&overrides, flags)
|
||||
})
|
||||
return overrides, explicitModules
|
||||
}
|
||||
433
internal/cli/process_flags_test.go
Normal file
433
internal/cli/process_flags_test.go
Normal file
@@ -0,0 +1,433 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"io"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
)
|
||||
|
||||
func TestProcessCLIOverridesMapsEveryConfigMutatingFlag(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
flagName string
|
||||
value string
|
||||
wantExplicitModules bool
|
||||
assertOverrideFields func(t *testing.T, overrides config.CLIOverrides)
|
||||
}{
|
||||
{
|
||||
name: "modules",
|
||||
flagName: "modules",
|
||||
value: "grammar,glossary",
|
||||
wantExplicitModules: true,
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "ModulesCSV", overrides.ModulesCSV, "grammar,glossary")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "output schema",
|
||||
flagName: "output-schema",
|
||||
value: "audita-v1",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "OutputSchema", overrides.OutputSchema, "audita-v1")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "primary api key",
|
||||
flagName: "llm-api-key",
|
||||
value: "primary-key",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "PrimaryLLMAPIKey", overrides.PrimaryLLMAPIKey, "primary-key")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation api key",
|
||||
flagName: "validation-llm-api-key",
|
||||
value: "validation-key",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "ValidationLLMAPIKey", overrides.ValidationLLMAPIKey, "validation-key")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "primary model",
|
||||
flagName: "model",
|
||||
value: "primary-model",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "PrimaryModel", overrides.PrimaryModel, "primary-model")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation model",
|
||||
flagName: "validation-model",
|
||||
value: "validation-model",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "ValidationModel", overrides.ValidationModel, "validation-model")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "primary base url",
|
||||
flagName: "base-url",
|
||||
value: "https://primary.example.test",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "PrimaryBaseURL", overrides.PrimaryBaseURL, "https://primary.example.test")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation base url",
|
||||
flagName: "validation-base-url",
|
||||
value: "https://validation.example.test",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "ValidationBaseURL", overrides.ValidationBaseURL, "https://validation.example.test")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "primary timeout",
|
||||
flagName: "llm-timeout-seconds",
|
||||
value: "101",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "PrimaryLLMTimeoutSeconds", overrides.PrimaryLLMTimeoutSeconds, 101)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "total concurrency",
|
||||
flagName: "total-llm-concurrency",
|
||||
value: "5",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "TotalLLMConcurrency", overrides.TotalLLMConcurrency, 5)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "proposal concurrency",
|
||||
flagName: "proposal-llm-concurrency",
|
||||
value: "3",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "ProposalLLMConcurrency", overrides.ProposalLLMConcurrency, 3)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "legacy concurrency alias",
|
||||
flagName: "llm-concurrency",
|
||||
value: "4",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "PrimaryLLMConcurrency", overrides.PrimaryLLMConcurrency, 4)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation timeout",
|
||||
flagName: "validation-llm-timeout-seconds",
|
||||
value: "202",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "ValidationLLMTimeoutSeconds", overrides.ValidationLLMTimeoutSeconds, 202)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "max retries",
|
||||
flagName: "max-retries",
|
||||
value: "6",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "MaxRetries", overrides.MaxRetries, 6)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation max retries",
|
||||
flagName: "validation-max-retries",
|
||||
value: "7",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "ValidationMaxRetries", overrides.ValidationMaxRetries, 7)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation concurrency",
|
||||
flagName: "validation-llm-concurrency",
|
||||
value: "8",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "ValidationLLMConcurrency", overrides.ValidationLLMConcurrency, 8)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "validation max prompt tokens",
|
||||
flagName: "validation-max-prompt-tokens",
|
||||
value: "4096",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "ValidationMaxPromptTokens", overrides.ValidationMaxPromptTokens, 4096)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "max section tokens",
|
||||
flagName: "max-section-tokens",
|
||||
value: "9000",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "MaxSectionTokens", overrides.MaxSectionTokens, 9000)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "min section tokens",
|
||||
flagName: "min-section-tokens",
|
||||
value: "1000",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "MinSectionTokens", overrides.MinSectionTokens, 1000)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "target sections",
|
||||
flagName: "target-sections",
|
||||
value: "12",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "TargetSections", overrides.TargetSections, 12)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "glossary threshold",
|
||||
flagName: "glossary-confidence-threshold",
|
||||
value: "0.91",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "GlossaryConfidenceThreshold", overrides.GlossaryConfidenceThreshold, 0.91)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "grammar threshold",
|
||||
flagName: "grammar-confidence-threshold",
|
||||
value: "0.92",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "GrammarConfidenceThreshold", overrides.GrammarConfidenceThreshold, 0.92)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "homophones threshold",
|
||||
flagName: "homophones-confidence-threshold",
|
||||
value: "0.93",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "HomophonesConfidenceThreshold", overrides.HomophonesConfidenceThreshold, 0.93)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "spoken word threshold",
|
||||
flagName: "spoken-word-confidence-threshold",
|
||||
value: "0.94",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "SpokenWordConfidenceThreshold", overrides.SpokenWordConfidenceThreshold, 0.94)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "normalize max segment gap",
|
||||
flagName: "normalize-max-segment-gap",
|
||||
value: "1.2",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "NormalizeMaxSegmentGap", overrides.NormalizeMaxSegmentGap, 1.2)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "normalize ellipsis gap",
|
||||
flagName: "normalize-ellipsis-gap",
|
||||
value: "2.3",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "NormalizeEllipsisGap", overrides.NormalizeEllipsisGap, 2.3)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "normalize max segment duration",
|
||||
flagName: "normalize-max-segment-duration",
|
||||
value: "45.6",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertFloatOverride(t, "NormalizeMaxSegmentDuration", overrides.NormalizeMaxSegmentDuration, 45.6)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "normalize max segment tokens",
|
||||
flagName: "normalize-max-segment-tokens",
|
||||
value: "321",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertIntOverride(t, "NormalizeMaxSegmentTokens", overrides.NormalizeMaxSegmentTokens, 321)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "transcript description",
|
||||
flagName: "transcript-description",
|
||||
value: "podcast episode",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "TranscriptDescription", overrides.TranscriptDescription, "podcast episode")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "work dir",
|
||||
flagName: "work-dir",
|
||||
value: "/tmp/custom-audita",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "WorkDir", overrides.WorkDir, "/tmp/custom-audita")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "work dir retention",
|
||||
flagName: "work-dir-retention",
|
||||
value: "always",
|
||||
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||
assertStringOverride(t, "WorkDirRetention", overrides.WorkDirRetention, "always")
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fs, flags := newProcessFlagSet(config.Default(), io.Discard)
|
||||
if err := fs.Parse([]string{"--" + tc.flagName, tc.value}); err != nil {
|
||||
t.Fatalf("parse flag: %v", err)
|
||||
}
|
||||
|
||||
overrides, explicitModules := processCLIOverrides(fs, flags)
|
||||
if explicitModules != tc.wantExplicitModules {
|
||||
t.Fatalf("explicitModules=%v, want %v", explicitModules, tc.wantExplicitModules)
|
||||
}
|
||||
tc.assertOverrideFields(t, overrides)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessCLIOverridesIgnoresNonConfigFlags(t *testing.T) {
|
||||
fs, flags := newProcessFlagSet(config.Default(), io.Discard)
|
||||
if err := fs.Parse([]string{
|
||||
"--config", "/tmp/config.yml",
|
||||
"--glossary", "/tmp/glossary.yml",
|
||||
"--output", "/tmp/output.json",
|
||||
"--report-json", "/tmp/report.json",
|
||||
}); err != nil {
|
||||
t.Fatalf("parse flags: %v", err)
|
||||
}
|
||||
|
||||
overrides, explicitModules := processCLIOverrides(fs, flags)
|
||||
if explicitModules {
|
||||
t.Fatal("non-config flags should not mark modules explicit")
|
||||
}
|
||||
assertNoCLIOverrides(t, overrides)
|
||||
}
|
||||
|
||||
func TestNewProcessFlagSetDefaultsReflectEffectiveConfig(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
cfg.Modules = []string{"grammar", "glossary"}
|
||||
cfg.OutputSchema = "audita-v1"
|
||||
cfg.PrimaryLLM.APIKey = "primary-key"
|
||||
cfg.ValidationLLM.APIKey = "validation-key"
|
||||
cfg.PrimaryLLM.Model = "primary-model"
|
||||
cfg.ValidationLLM.Model = "validation-model"
|
||||
cfg.PrimaryLLM.BaseURL = "https://primary.example.test"
|
||||
cfg.ValidationLLM.BaseURL = "https://validation.example.test"
|
||||
cfg.PrimaryLLM.TimeoutSeconds = 101
|
||||
cfg.TotalLLMConcurrency = 5
|
||||
cfg.ProposalLLMConcurrency = 3
|
||||
cfg.PrimaryLLM.MaxRetries = 6
|
||||
cfg.ValidationMaxPromptTokens = 4096
|
||||
cfg.MaxSectionTokens = 9000
|
||||
cfg.MinSectionTokens = 1000
|
||||
cfg.Thresholds.Glossary = 0.91
|
||||
cfg.Thresholds.Grammar = 0.92
|
||||
cfg.Thresholds.Homophones = 0.93
|
||||
cfg.Thresholds.SpokenWord = 0.94
|
||||
cfg.Normalization.MaxSegmentGap = 1.2
|
||||
cfg.Normalization.EllipsisGap = 2.3
|
||||
cfg.Normalization.MaxSegmentDuration = 45.6
|
||||
cfg.Normalization.MaxSegmentTokens = 321
|
||||
cfg.TranscriptDescription = "podcast episode"
|
||||
cfg.WorkDir = "/tmp/custom-audita"
|
||||
cfg.WorkDirRetention = config.WorkDirRetentionAlways
|
||||
|
||||
validationTimeout := 202
|
||||
validationRetries := 7
|
||||
validationConcurrency := 8
|
||||
targetSections := 12
|
||||
cfg.ValidationLLM.TimeoutSeconds = &validationTimeout
|
||||
cfg.ValidationLLM.MaxRetries = &validationRetries
|
||||
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||
cfg.TargetSections = &targetSections
|
||||
|
||||
_, flags := newProcessFlagSet(cfg, io.Discard)
|
||||
|
||||
assertStringOverride(t, "modules default", flags.modules, "grammar,glossary")
|
||||
assertStringOverride(t, "output schema default", flags.outputSchema, "audita-v1")
|
||||
assertStringOverride(t, "primary api key default", flags.llmAPIKey, "primary-key")
|
||||
assertStringOverride(t, "validation api key default", flags.validationLLMAPIKey, "validation-key")
|
||||
assertStringOverride(t, "primary model default", flags.model, "primary-model")
|
||||
assertStringOverride(t, "validation model default", flags.validationModel, "validation-model")
|
||||
assertStringOverride(t, "primary base url default", flags.baseURL, "https://primary.example.test")
|
||||
assertStringOverride(t, "validation base url default", flags.validationBaseURL, "https://validation.example.test")
|
||||
assertIntOverride(t, "primary timeout default", flags.llmTimeoutSeconds, 101)
|
||||
assertIntOverride(t, "total concurrency default", flags.totalLLMConcurrency, 5)
|
||||
assertIntOverride(t, "proposal concurrency default", flags.proposalLLMConcurrency, 3)
|
||||
assertIntOverride(t, "legacy concurrency alias default", flags.llmConcurrency, 5)
|
||||
assertIntOverride(t, "validation timeout default", flags.validationLLMTimeoutSeconds, validationTimeout)
|
||||
assertIntOverride(t, "max retries default", flags.maxRetries, 6)
|
||||
assertIntOverride(t, "validation max retries default", flags.validationMaxRetries, validationRetries)
|
||||
assertIntOverride(t, "validation concurrency default", flags.validationLLMConcurrency, validationConcurrency)
|
||||
assertIntOverride(t, "validation max prompt tokens default", flags.validationMaxPromptTokens, 4096)
|
||||
assertIntOverride(t, "max section tokens default", flags.maxSectionTokens, 9000)
|
||||
assertIntOverride(t, "min section tokens default", flags.minSectionTokens, 1000)
|
||||
assertIntOverride(t, "target sections default", flags.targetSections, targetSections)
|
||||
assertFloatOverride(t, "glossary threshold default", flags.glossaryConfidenceThreshold, 0.91)
|
||||
assertFloatOverride(t, "grammar threshold default", flags.grammarConfidenceThreshold, 0.92)
|
||||
assertFloatOverride(t, "homophones threshold default", flags.homophonesConfidenceThreshold, 0.93)
|
||||
assertFloatOverride(t, "spoken word threshold default", flags.spokenWordConfidenceThreshold, 0.94)
|
||||
assertFloatOverride(t, "normalize max segment gap default", flags.normalizeMaxSegmentGap, 1.2)
|
||||
assertFloatOverride(t, "normalize ellipsis gap default", flags.normalizeEllipsisGap, 2.3)
|
||||
assertFloatOverride(t, "normalize max segment duration default", flags.normalizeMaxSegmentDuration, 45.6)
|
||||
assertIntOverride(t, "normalize max segment tokens default", flags.normalizeMaxSegmentTokens, 321)
|
||||
assertStringOverride(t, "transcript description default", flags.transcriptDescription, "podcast episode")
|
||||
assertStringOverride(t, "work dir default", flags.workDir, "/tmp/custom-audita")
|
||||
assertStringOverride(t, "work dir retention default", flags.workDirRetention, "always")
|
||||
}
|
||||
|
||||
func TestNewProcessFlagSetUsesFallbackDefaultsForUnsetOptionalConfig(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
|
||||
_, flags := newProcessFlagSet(cfg, io.Discard)
|
||||
|
||||
assertIntOverride(t, "validation timeout fallback", flags.validationLLMTimeoutSeconds, cfg.PrimaryLLM.TimeoutSeconds)
|
||||
assertIntOverride(t, "validation retries fallback", flags.validationMaxRetries, cfg.PrimaryLLM.MaxRetries)
|
||||
assertIntOverride(t, "validation concurrency fallback", flags.validationLLMConcurrency, cfg.TotalLLMConcurrency)
|
||||
assertIntOverride(t, "target sections fallback", flags.targetSections, 0)
|
||||
}
|
||||
|
||||
func assertStringOverride(t *testing.T, name string, got *string, want string) {
|
||||
t.Helper()
|
||||
if got == nil || *got != want {
|
||||
t.Fatalf("%s=%v, want %q", name, pointerValue(got), want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertIntOverride(t *testing.T, name string, got *int, want int) {
|
||||
t.Helper()
|
||||
if got == nil || *got != want {
|
||||
t.Fatalf("%s=%v, want %d", name, pointerValue(got), want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertFloatOverride(t *testing.T, name string, got *float64, want float64) {
|
||||
t.Helper()
|
||||
if got == nil || *got != want {
|
||||
t.Fatalf("%s=%v, want %v", name, pointerValue(got), want)
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoCLIOverrides(t *testing.T, overrides config.CLIOverrides) {
|
||||
t.Helper()
|
||||
value := reflect.ValueOf(overrides)
|
||||
typ := value.Type()
|
||||
for i := 0; i < value.NumField(); i++ {
|
||||
field := value.Field(i)
|
||||
if field.Kind() != reflect.Ptr {
|
||||
t.Fatalf("unexpected non-pointer CLIOverrides field %s", typ.Field(i).Name)
|
||||
}
|
||||
if !field.IsNil() {
|
||||
t.Fatalf("expected no CLI overrides, field %s was set", typ.Field(i).Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func pointerValue[T any](ptr *T) any {
|
||||
if ptr == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
if stringer, ok := any(*ptr).(interface{ String() string }); ok {
|
||||
return strings.TrimSpace(stringer.String())
|
||||
}
|
||||
return *ptr
|
||||
}
|
||||
469
internal/cli/release_fixtures_test.go
Normal file
469
internal/cli/release_fixtures_test.go
Normal file
@@ -0,0 +1,469 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
)
|
||||
|
||||
type releaseFixtureExpectations struct {
|
||||
MustApplyTexts []string `json:"must_apply_texts"`
|
||||
MustNotApplyTexts []string `json:"must_not_apply_texts"`
|
||||
ProtectedTerms []string `json:"protected_terms"`
|
||||
ExpectedModuleInstance []string `json:"expected_module_instances"`
|
||||
MinimumCounts struct {
|
||||
Applied int `json:"applied"`
|
||||
Rejected int `json:"rejected"`
|
||||
Skipped int `json:"skipped"`
|
||||
} `json:"minimum_counts"`
|
||||
}
|
||||
|
||||
func TestReleaseFixtureDefaultPipelineReadiness(t *testing.T) {
|
||||
base := fixturePath(filepath.Join("release", "default-release"))
|
||||
|
||||
var expectations releaseFixtureExpectations
|
||||
if err := json.Unmarshal(readFile(t, base+".expectations.json"), &expectations); err != nil {
|
||||
t.Fatalf("unmarshal release expectations: %v", err)
|
||||
}
|
||||
|
||||
proposalResponses := readProposalResponses(t, base+".proposals.json")
|
||||
validationResponses := readValidationResponses(t, base+".validations.json")
|
||||
|
||||
// First pass: default full pipeline with deterministic fake LLM responses.
|
||||
first := runReleaseFixturePass(t, releaseRunConfig{
|
||||
transcriptPath: base + ".transcript.json",
|
||||
glossaryPath: base + ".glossary.yaml",
|
||||
outputSchema: "bare-segments",
|
||||
proposalResponses: proposalResponses,
|
||||
validationResponses: validationResponses,
|
||||
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
|
||||
reportSchemaName: reporting.DefaultProcessReportSchemaName,
|
||||
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
||||
expectedOutputSchema: "bare-segments",
|
||||
expectModuleInstances: expectations.ExpectedModuleInstance,
|
||||
expectUtilizationPaths: true,
|
||||
})
|
||||
|
||||
gotTranscript := mustReadTranscript(t, first.outputPath)
|
||||
expectFinalTranscriptContains(t, gotTranscript, expectations.MustApplyTexts)
|
||||
expectFinalTranscriptDoesNotContain(t, gotTranscript, expectations.MustNotApplyTexts)
|
||||
expectFinalTranscriptContains(t, gotTranscript, expectations.ProtectedTerms)
|
||||
|
||||
assertReleaseCounts(t, first.report, expectations)
|
||||
assertPromptAndSchemaMetadataPresent(t, first.runDir)
|
||||
assertReleaseLedgerShape(t, first.report)
|
||||
assertReleaseUtilizationShape(t, first.report)
|
||||
assertStableValidatorKeysPresent(t, first.report)
|
||||
assertStdoutStderrContract(t, first.stdout, first.stderr)
|
||||
assertNoSecretMarkersInTree(t, first.runDir, []string{"release-secret"})
|
||||
assertNoSecretMarkers(t, first.reportPath, []string{"release-secret"})
|
||||
|
||||
// Output schema check: audita-v1 object payload.
|
||||
auditaV1 := runReleaseFixturePass(t, releaseRunConfig{
|
||||
transcriptPath: base + ".transcript.json",
|
||||
glossaryPath: base + ".glossary.yaml",
|
||||
outputSchema: "audita-v1",
|
||||
proposalResponses: proposalResponses,
|
||||
validationResponses: validationResponses,
|
||||
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
|
||||
reportSchemaName: reporting.DefaultProcessReportSchemaName,
|
||||
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
||||
expectedOutputSchema: "audita-v1",
|
||||
expectModuleInstances: expectations.ExpectedModuleInstance,
|
||||
expectUtilizationPaths: true,
|
||||
})
|
||||
assertAuditaV1OutputShape(t, auditaV1.outputPath)
|
||||
|
||||
// Idempotence-oriented second pass:
|
||||
// run again on first output with deterministic no-op responses.
|
||||
noOpProposals := make([]proposal_generation.StructuredCorrectionSet, 5)
|
||||
for i := range noOpProposals {
|
||||
noOpProposals[i] = proposal_generation.StructuredCorrectionSet{Corrections: nil}
|
||||
}
|
||||
second := runReleaseFixturePass(t, releaseRunConfig{
|
||||
transcriptPath: first.outputPath,
|
||||
glossaryPath: base + ".glossary.yaml",
|
||||
outputSchema: "bare-segments",
|
||||
proposalResponses: noOpProposals,
|
||||
validationResponses: nil,
|
||||
expectedProposalCalls: []string{"glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal"},
|
||||
reportSchemaName: reporting.DefaultProcessReportSchemaName,
|
||||
reportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
||||
expectedOutputSchema: "bare-segments",
|
||||
expectModuleInstances: expectations.ExpectedModuleInstance,
|
||||
expectUtilizationPaths: true,
|
||||
})
|
||||
firstSegments := mustReadTranscript(t, first.outputPath)
|
||||
secondSegments := mustReadTranscript(t, second.outputPath)
|
||||
if !reflect.DeepEqual(firstSegments, secondSegments) {
|
||||
t.Fatalf("expected idempotent second pass transcript; first=%+v second=%+v", firstSegments, secondSegments)
|
||||
}
|
||||
if second.report.ModulesSummary == nil {
|
||||
t.Fatalf("expected modules summary on second pass")
|
||||
}
|
||||
if second.report.ModulesSummary.TotalAppliedChanges != 0 {
|
||||
t.Fatalf("expected no-op second pass (0 applied), got %+v", second.report.ModulesSummary)
|
||||
}
|
||||
}
|
||||
|
||||
type releaseRunConfig struct {
|
||||
transcriptPath string
|
||||
glossaryPath string
|
||||
outputSchema string
|
||||
proposalResponses []proposal_generation.StructuredCorrectionSet
|
||||
validationResponses []validators.LLMValidationResponse
|
||||
expectedProposalCalls []string
|
||||
reportSchemaName string
|
||||
reportSchemaVersion string
|
||||
expectedOutputSchema string
|
||||
expectModuleInstances []string
|
||||
expectUtilizationPaths bool
|
||||
}
|
||||
|
||||
type releaseRunResult struct {
|
||||
stdout string
|
||||
stderr string
|
||||
outputPath string
|
||||
reportPath string
|
||||
report reporting.ProcessReport
|
||||
runDir string
|
||||
}
|
||||
|
||||
func runReleaseFixturePass(t *testing.T, cfg releaseRunConfig) releaseRunResult {
|
||||
t.Helper()
|
||||
|
||||
processProposalLLMClient = &fakeStructuredLLMClient{proposalResponses: append([]proposal_generation.StructuredCorrectionSet(nil), cfg.proposalResponses...)}
|
||||
processValidationLLMClient = &fakeStructuredLLMClient{validationResponses: append([]validators.LLMValidationResponse(nil), cfg.validationResponses...)}
|
||||
t.Cleanup(func() {
|
||||
processProposalLLMClient = nil
|
||||
processValidationLLMClient = nil
|
||||
})
|
||||
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
outputPath := filepath.Join(t.TempDir(), "out.json")
|
||||
configPath := writeFile(t, "release-config.yml", "version: 1\n")
|
||||
|
||||
args := []string{
|
||||
"process",
|
||||
cfg.transcriptPath,
|
||||
"--glossary",
|
||||
cfg.glossaryPath,
|
||||
"--config",
|
||||
configPath,
|
||||
"--output",
|
||||
outputPath,
|
||||
"--output-schema",
|
||||
cfg.outputSchema,
|
||||
"--report-json",
|
||||
reportPath,
|
||||
"--work-dir",
|
||||
workDir,
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
}
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
exitCode := Run(args, &stdout, &stderr)
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||
}
|
||||
|
||||
report := readProcessReport(t, reportPath)
|
||||
if report.ReportMetadata.ReportSchemaName != cfg.reportSchemaName || report.ReportMetadata.ReportSchemaVersion != cfg.reportSchemaVersion {
|
||||
t.Fatalf("unexpected report schema metadata: %+v", report.ReportMetadata)
|
||||
}
|
||||
if report.ReportMetadata.OutputSchema != cfg.expectedOutputSchema {
|
||||
t.Fatalf("unexpected output schema metadata: got %q want %q", report.ReportMetadata.OutputSchema, cfg.expectedOutputSchema)
|
||||
}
|
||||
if len(cfg.expectModuleInstances) > 0 {
|
||||
got := make([]string, 0, len(report.ModuleResults))
|
||||
for _, mr := range report.ModuleResults {
|
||||
got = append(got, mr.ModuleInstance)
|
||||
}
|
||||
if !reflect.DeepEqual(got, cfg.expectModuleInstances) {
|
||||
t.Fatalf("unexpected module instances: got %v want %v", got, cfg.expectModuleInstances)
|
||||
}
|
||||
}
|
||||
if report.Diagnostics == nil {
|
||||
t.Fatalf("expected diagnostics metadata")
|
||||
}
|
||||
if cfg.expectUtilizationPaths {
|
||||
if report.Diagnostics.UtilizationSummaryPath == "" || report.Diagnostics.CorrectionLedgerPath == "" {
|
||||
t.Fatalf("expected utilization/ledger artifact paths in report diagnostics: %+v", report.Diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "report.json")); err != nil {
|
||||
t.Fatalf("expected run-dir report: %v", err)
|
||||
}
|
||||
|
||||
if c, ok := processProposalLLMClient.(*fakeStructuredLLMClient); ok {
|
||||
if !reflect.DeepEqual(c.calls, cfg.expectedProposalCalls) {
|
||||
t.Fatalf("unexpected proposal call order: got %v want %v", c.calls, cfg.expectedProposalCalls)
|
||||
}
|
||||
}
|
||||
|
||||
return releaseRunResult{
|
||||
stdout: stdout.String(),
|
||||
stderr: stderr.String(),
|
||||
outputPath: outputPath,
|
||||
reportPath: reportPath,
|
||||
report: report,
|
||||
runDir: runDir,
|
||||
}
|
||||
}
|
||||
|
||||
func readProposalResponses(t *testing.T, path string) []proposal_generation.StructuredCorrectionSet {
|
||||
t.Helper()
|
||||
var out []proposal_generation.StructuredCorrectionSet
|
||||
if err := json.Unmarshal(readFile(t, path), &out); err != nil {
|
||||
t.Fatalf("unmarshal proposal responses: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func readValidationResponses(t *testing.T, path string) []validators.LLMValidationResponse {
|
||||
t.Helper()
|
||||
var out []validators.LLMValidationResponse
|
||||
if err := json.Unmarshal(readFile(t, path), &out); err != nil {
|
||||
t.Fatalf("unmarshal validation responses: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mustReadTranscript(t *testing.T, path string) []schema.Segment {
|
||||
t.Helper()
|
||||
transcript, err := schema.ParseTranscriptJSON(readFile(t, path))
|
||||
if err != nil {
|
||||
t.Fatalf("parse transcript output: %v", err)
|
||||
}
|
||||
return transcript.Segments
|
||||
}
|
||||
|
||||
func expectFinalTranscriptContains(t *testing.T, segments []schema.Segment, needles []string) {
|
||||
t.Helper()
|
||||
joined := flattenTranscriptText(segments)
|
||||
for _, needle := range needles {
|
||||
if !strings.Contains(joined, needle) {
|
||||
t.Fatalf("expected transcript to contain %q, got %q", needle, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func expectFinalTranscriptDoesNotContain(t *testing.T, segments []schema.Segment, needles []string) {
|
||||
t.Helper()
|
||||
joined := flattenTranscriptText(segments)
|
||||
for _, needle := range needles {
|
||||
if strings.Contains(joined, needle) {
|
||||
t.Fatalf("expected transcript to not contain %q, got %q", needle, joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func flattenTranscriptText(segments []schema.Segment) string {
|
||||
parts := make([]string, 0, len(segments))
|
||||
for _, s := range segments {
|
||||
parts = append(parts, s.Text)
|
||||
}
|
||||
return strings.Join(parts, "\n")
|
||||
}
|
||||
|
||||
func assertReleaseCounts(t *testing.T, report reporting.ProcessReport, exp releaseFixtureExpectations) {
|
||||
t.Helper()
|
||||
if report.ModulesSummary == nil {
|
||||
t.Fatalf("expected modules_summary")
|
||||
}
|
||||
if report.ModulesSummary.TotalAppliedChanges < exp.MinimumCounts.Applied {
|
||||
t.Fatalf("expected at least %d applied changes, got %+v", exp.MinimumCounts.Applied, report.ModulesSummary)
|
||||
}
|
||||
validatorRejected := 0
|
||||
skipped := 0
|
||||
for _, mr := range report.ModuleResults {
|
||||
validatorRejected += len(mr.ValidatorRejected)
|
||||
skipped += len(mr.SkippedChanges)
|
||||
}
|
||||
if validatorRejected < exp.MinimumCounts.Rejected {
|
||||
t.Fatalf("expected at least %d validator rejections, got %d", exp.MinimumCounts.Rejected, validatorRejected)
|
||||
}
|
||||
if skipped < exp.MinimumCounts.Skipped {
|
||||
t.Fatalf("expected at least %d application skips, got %d", exp.MinimumCounts.Skipped, skipped)
|
||||
}
|
||||
}
|
||||
|
||||
func assertReleaseUtilizationShape(t *testing.T, report reporting.ProcessReport) {
|
||||
t.Helper()
|
||||
var payload struct {
|
||||
EffectiveConcurrency struct {
|
||||
TotalLLM int `json:"total_llm"`
|
||||
} `json:"effective_concurrency"`
|
||||
RunTiming struct {
|
||||
SchedulerQueueWaitMS int64 `json:"scheduler_queue_wait_ms"`
|
||||
LLMExecutionTimeMS int64 `json:"llm_execution_time_ms"`
|
||||
DeterministicValidationMS int64 `json:"deterministic_validation_time_ms"`
|
||||
} `json:"run_timing"`
|
||||
Modules []map[string]any `json:"modules"`
|
||||
Validators []map[string]any `json:"validators"`
|
||||
}
|
||||
if err := json.Unmarshal(readFile(t, report.Diagnostics.UtilizationSummaryPath), &payload); err != nil {
|
||||
t.Fatalf("unmarshal utilization diagnostics: %v", err)
|
||||
}
|
||||
if payload.EffectiveConcurrency.TotalLLM <= 0 {
|
||||
t.Fatalf("expected positive total llm concurrency, got %+v", payload.EffectiveConcurrency)
|
||||
}
|
||||
if payload.RunTiming.SchedulerQueueWaitMS < 0 || payload.RunTiming.LLMExecutionTimeMS < 0 || payload.RunTiming.DeterministicValidationMS < 0 {
|
||||
t.Fatalf("expected non-negative run timing values, got %+v", payload.RunTiming)
|
||||
}
|
||||
if len(payload.Modules) == 0 {
|
||||
t.Fatalf("expected module timing summaries")
|
||||
}
|
||||
if len(payload.Validators) == 0 {
|
||||
t.Fatalf("expected validator timing summaries")
|
||||
}
|
||||
}
|
||||
|
||||
func assertReleaseLedgerShape(t *testing.T, report reporting.ProcessReport) {
|
||||
t.Helper()
|
||||
var entries []struct {
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ProposalIndex int `json:"proposal_index"`
|
||||
Disposition string `json:"disposition"`
|
||||
DispositionReason string `json:"disposition_reason_code"`
|
||||
OriginalText string `json:"original_text"`
|
||||
ProposedCorrected string `json:"proposed_corrected_text"`
|
||||
ReplacementPolicy string `json:"replacement_policy"`
|
||||
DeterministicResults []struct {
|
||||
ValidatorKey string `json:"validator_key"`
|
||||
} `json:"deterministic_validator_decisions"`
|
||||
LLMResults []struct {
|
||||
ValidatorKey string `json:"validator_key"`
|
||||
} `json:"llm_validator_decisions"`
|
||||
}
|
||||
if err := json.Unmarshal(readFile(t, report.Diagnostics.CorrectionLedgerPath), &entries); err != nil {
|
||||
t.Fatalf("unmarshal correction ledger: %v", err)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
t.Fatalf("expected correction ledger entries")
|
||||
}
|
||||
hasApplied := false
|
||||
hasRejected := false
|
||||
hasSkipped := false
|
||||
for _, entry := range entries {
|
||||
if entry.ModuleInstance == "" || entry.ModuleKey == "" {
|
||||
t.Fatalf("expected module identity in ledger entry: %+v", entry)
|
||||
}
|
||||
switch entry.Disposition {
|
||||
case "applied":
|
||||
hasApplied = true
|
||||
case "rejected":
|
||||
hasRejected = true
|
||||
case "skipped":
|
||||
hasSkipped = true
|
||||
}
|
||||
}
|
||||
if !hasApplied || !hasRejected {
|
||||
t.Fatalf("expected applied and rejected entries in correction ledger, got %+v", entries)
|
||||
}
|
||||
if !hasSkipped {
|
||||
// Some deterministic fixture paths do not trigger apply-time skips;
|
||||
// rejections are still captured separately from application skips.
|
||||
}
|
||||
}
|
||||
|
||||
func assertPromptAndSchemaMetadataPresent(t *testing.T, runDir string) {
|
||||
t.Helper()
|
||||
metadataPaths, err := filepath.Glob(filepath.Join(runDir, "*", "*request-metadata.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("glob request metadata artifacts: %v", err)
|
||||
}
|
||||
if len(metadataPaths) == 0 {
|
||||
t.Fatalf("expected request metadata artifacts with prompt metadata")
|
||||
}
|
||||
|
||||
foundPromptMetadata := false
|
||||
foundSchemaMetadata := false
|
||||
for _, path := range metadataPaths {
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(readFile(t, path), &payload); err != nil {
|
||||
t.Fatalf("unmarshal request metadata artifact %q: %v", path, err)
|
||||
}
|
||||
if pm, ok := payload["prompt_metadata"].(map[string]any); ok {
|
||||
if pm["prompt_id"] != nil && pm["prompt_version"] != nil && pm["sha256"] != nil {
|
||||
foundPromptMetadata = true
|
||||
}
|
||||
}
|
||||
if sm, ok := payload["response_schema"].(map[string]any); ok {
|
||||
if sm["id"] != nil && sm["version"] != nil && sm["name"] != nil && sm["sha256"] != nil {
|
||||
foundSchemaMetadata = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !foundPromptMetadata {
|
||||
t.Fatalf("expected prompt metadata in request metadata artifacts")
|
||||
}
|
||||
if !foundSchemaMetadata {
|
||||
t.Fatalf("expected structured response schema metadata in request metadata artifacts")
|
||||
}
|
||||
}
|
||||
|
||||
func assertStableValidatorKeysPresent(t *testing.T, report reporting.ProcessReport) {
|
||||
t.Helper()
|
||||
seen := map[string]bool{}
|
||||
for _, module := range report.ModuleResults {
|
||||
for _, decision := range module.ValidatorDecisions {
|
||||
seen[decision.ValidatorName] = true
|
||||
}
|
||||
for _, rejected := range module.ValidatorRejected {
|
||||
seen[rejected.ValidatorName] = true
|
||||
}
|
||||
}
|
||||
expectedAny := []string{
|
||||
"confidence_threshold",
|
||||
"original_text_presence",
|
||||
"no_effect",
|
||||
}
|
||||
for _, key := range expectedAny {
|
||||
if !seen[key] {
|
||||
t.Fatalf("expected stable validator key %q in report decisions/rejections; seen=%v", key, seen)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertStdoutStderrContract(t *testing.T, stdout, stderr string) {
|
||||
t.Helper()
|
||||
if stdout != "" {
|
||||
t.Fatalf("expected empty stdout with --output, got %q", stdout)
|
||||
}
|
||||
if strings.Contains(stderr, `"module_results"`) || strings.Contains(stderr, `"report_metadata"`) {
|
||||
t.Fatalf("stderr should remain human-readable, not report JSON: %q", stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func assertAuditaV1OutputShape(t *testing.T, outputPath string) {
|
||||
t.Helper()
|
||||
var payload struct {
|
||||
Schema string `json:"schema"`
|
||||
Version string `json:"version"`
|
||||
Segments []schema.Segment `json:"segments"`
|
||||
}
|
||||
if err := json.Unmarshal(readFile(t, outputPath), &payload); err != nil {
|
||||
t.Fatalf("unmarshal audita-v1 output: %v", err)
|
||||
}
|
||||
if payload.Schema != "audita-v1" || payload.Version != "v1" {
|
||||
t.Fatalf("unexpected audita-v1 metadata: %+v", payload)
|
||||
}
|
||||
if len(payload.Segments) == 0 {
|
||||
t.Fatalf("expected non-empty audita-v1 segments")
|
||||
}
|
||||
}
|
||||
867
internal/cli/run.go
Normal file
867
internal/cli/run.go
Normal file
@@ -0,0 +1,867 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
||||
coreio "gitea.maximumdirect.net/eric/audita/internal/core/io"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/processreport"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
)
|
||||
|
||||
type noOpStructuredLLMClient struct{}
|
||||
|
||||
func (c noOpStructuredLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
_ = ctx
|
||||
_ = req
|
||||
switch target := out.(type) {
|
||||
case *validators.LLMValidationResponse:
|
||||
*target = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{}}
|
||||
case *proposal_generation.StructuredCorrectionSet:
|
||||
*target = proposal_generation.StructuredCorrectionSet{Corrections: nil}
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
}
|
||||
|
||||
func shouldUseNoOpLLMClientForTests() bool {
|
||||
return strings.HasSuffix(filepath.Base(os.Args[0]), ".test") || os.Getenv("GO_WANT_HELPER_PROCESS") == "1"
|
||||
}
|
||||
|
||||
type processInvocation struct {
|
||||
TranscriptPath string
|
||||
GlossaryPath string
|
||||
OutputPath string
|
||||
ReportJSONPath string
|
||||
Config config.Config
|
||||
ConfigPath string
|
||||
ConfigSource string
|
||||
ConfigVersion *int
|
||||
ExplicitModules bool
|
||||
}
|
||||
|
||||
var processModuleFactory runner.ModuleFactory
|
||||
var processProposalLLMClient contracts.StructuredLLMClient
|
||||
var processProposalLLMScheduler runner.ValidationScheduler
|
||||
var processValidationLLMClient contracts.StructuredLLMClient
|
||||
var processValidationLLMScheduler runner.ValidationScheduler
|
||||
var processRunnerContext = func() (context.Context, context.CancelFunc) {
|
||||
return context.Background(), func() {}
|
||||
}
|
||||
|
||||
var processRunner = func(inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) {
|
||||
runDir, err := diagnostics.NewRunDirectory(inv.Config.WorkDir, string(inv.Config.WorkDirRetention))
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, fmt.Errorf("run_dir_creation: %w", err)
|
||||
}
|
||||
|
||||
fail := func(phase string, err error, runOutput *runner.RunOutput) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("%s: %v", phase, err))
|
||||
return nil, nil, runOutput, runDir, fmt.Errorf("%s: %w", phase, err)
|
||||
}
|
||||
|
||||
if err := runDir.WriteInvocationMetadata(diagnostics.InvocationMetadata{
|
||||
Operation: "process",
|
||||
TranscriptPath: inv.TranscriptPath,
|
||||
GlossaryPath: inv.GlossaryPath,
|
||||
OutputPath: inv.OutputPath,
|
||||
ReportJSONPath: inv.ReportJSONPath,
|
||||
ConfigPath: inv.ConfigPath,
|
||||
ConfigSource: inv.ConfigSource,
|
||||
ConfigVersion: inv.ConfigVersion,
|
||||
TranscriptDescription: inv.Config.TranscriptDescription,
|
||||
Modules: append([]string(nil), inv.Config.Modules...),
|
||||
}); err != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("invocation_metadata: %v", err))
|
||||
}
|
||||
if err := runDir.WriteEffectiveConfig(inv.Config); err != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("effective_config: %v", err))
|
||||
}
|
||||
|
||||
transcriptBytes, err := coreio.ReadRequiredFile(inv.TranscriptPath, "transcript")
|
||||
if err != nil {
|
||||
return fail("transcript_read", err, nil)
|
||||
}
|
||||
|
||||
glossaryBytes, err := coreio.ReadRequiredFile(inv.GlossaryPath, "glossary")
|
||||
if err != nil {
|
||||
return fail("glossary_read", err, nil)
|
||||
}
|
||||
|
||||
sourceTranscript, err := schema.ParseSourceTranscriptJSON(transcriptBytes)
|
||||
if err != nil {
|
||||
return fail("transcript_schema", err, nil)
|
||||
}
|
||||
|
||||
glossary, err := schema.ParseGlossaryYAML(glossaryBytes)
|
||||
if err != nil {
|
||||
return fail("glossary_schema", err, nil)
|
||||
}
|
||||
|
||||
if err := runDir.WriteSourceTranscript(sourceTranscript, transcriptBytes); err != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("source_artifact: %v", err))
|
||||
}
|
||||
|
||||
canonical := sourceToCanonicalTranscript(sourceTranscript)
|
||||
normalizer := normalization.NewNormalizer(normalization.NormalizationConfig{
|
||||
MaxSegmentGap: inv.Config.Normalization.MaxSegmentGap,
|
||||
EllipsisGap: inv.Config.Normalization.EllipsisGap,
|
||||
MaxSegmentDuration: inv.Config.Normalization.MaxSegmentDuration,
|
||||
MaxSegmentTokens: inv.Config.Normalization.MaxSegmentTokens,
|
||||
})
|
||||
|
||||
normalizedTranscript, normSummary := normalizer.Normalize(canonical)
|
||||
|
||||
if err := runDir.WriteNormalizedTranscript(normalizedTranscript); err != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("normalized_artifact: %v", err))
|
||||
}
|
||||
if err := runDir.WriteNormalizationSummary(normSummary); err != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("normalization_summary: %v", err))
|
||||
}
|
||||
|
||||
// Compute chunks after normalization
|
||||
chunker := chunking.NewChunker(chunking.ChunkingConfig{
|
||||
MaxSectionTokens: inv.Config.MaxSectionTokens,
|
||||
MinSectionTokens: inv.Config.MinSectionTokens,
|
||||
TargetSections: inv.Config.TargetSections,
|
||||
})
|
||||
|
||||
sections, chunkErr := chunker.ChunkTranscript(normalizedTranscript)
|
||||
if chunkErr != nil {
|
||||
return fail("chunking", chunkErr, nil)
|
||||
}
|
||||
|
||||
chunkConfig := chunking.ChunkingConfig{
|
||||
MaxSectionTokens: inv.Config.MaxSectionTokens,
|
||||
MinSectionTokens: inv.Config.MinSectionTokens,
|
||||
TargetSections: inv.Config.TargetSections,
|
||||
}
|
||||
chunkSummary := chunking.ComputeSummary(sections, chunkConfig)
|
||||
chunkDetailedSummary := chunking.ComputeDetailedSummary(sections, chunkConfig)
|
||||
|
||||
if err := runDir.WriteChunkingSummary(&chunkDetailedSummary); err != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("chunking_summary: %v", err))
|
||||
}
|
||||
|
||||
workingTranscript := normalizedTranscript
|
||||
var runOutput *runner.RunOutput
|
||||
moduleFactory := processModuleFactory
|
||||
if moduleFactory == nil {
|
||||
moduleFactory = modules.NewFactory(modules.Dependencies{
|
||||
Config: &inv.Config,
|
||||
Glossary: glossary,
|
||||
DiagnosticsDir: runDir.Path(),
|
||||
})
|
||||
}
|
||||
if moduleFactory != nil {
|
||||
proposalLLMClient := processProposalLLMClient
|
||||
validationLLMClient := processValidationLLMClient
|
||||
proposalScheduler := processProposalLLMScheduler
|
||||
validationScheduler := processValidationLLMScheduler
|
||||
|
||||
if processModuleFactory == nil {
|
||||
// Production runtime path: construct clients/schedulers from config.
|
||||
if proposalLLMClient == nil {
|
||||
if shouldUseNoOpLLMClientForTests() {
|
||||
proposalLLMClient = noOpStructuredLLMClient{}
|
||||
} else {
|
||||
primaryCfg := llm.ResolvePrimaryConfig(inv.Config)
|
||||
client, clientErr := llm.NewOpenAICompatibleClient(primaryCfg.ToOpenAICompatibleClientConfig(nil))
|
||||
if clientErr != nil {
|
||||
return fail("runner_setup", clientErr, nil)
|
||||
}
|
||||
proposalLLMClient = client
|
||||
}
|
||||
}
|
||||
if validationLLMClient == nil {
|
||||
if shouldUseNoOpLLMClientForTests() {
|
||||
validationLLMClient = noOpStructuredLLMClient{}
|
||||
} else {
|
||||
validationCfg := llm.ResolveValidationConfig(inv.Config)
|
||||
client, clientErr := llm.NewOpenAICompatibleClient(validationCfg.ToOpenAICompatibleClientConfig(nil))
|
||||
if clientErr != nil {
|
||||
return fail("runner_setup", clientErr, nil)
|
||||
}
|
||||
validationLLMClient = client
|
||||
}
|
||||
}
|
||||
globalScheduler := proposalScheduler
|
||||
if globalScheduler == nil {
|
||||
s, sErr := llm.NewScheduler(inv.Config.TotalLLMConcurrency)
|
||||
if sErr != nil {
|
||||
return fail("runner_setup", sErr, nil)
|
||||
}
|
||||
globalScheduler = s
|
||||
}
|
||||
if proposalScheduler == nil {
|
||||
proposalScheduler = globalScheduler
|
||||
if inv.Config.EffectiveProposalLLMConcurrency() < inv.Config.TotalLLMConcurrency {
|
||||
s, sErr := llm.NewScheduler(inv.Config.EffectiveProposalLLMConcurrency())
|
||||
if sErr != nil {
|
||||
return fail("runner_setup", sErr, nil)
|
||||
}
|
||||
proposalScheduler = composeSchedulers(globalScheduler, s)
|
||||
}
|
||||
}
|
||||
if validationScheduler == nil {
|
||||
validationScheduler = globalScheduler
|
||||
if inv.Config.ValidationLLMConcurrency != nil && inv.Config.EffectiveValidationLLMConcurrency() < inv.Config.TotalLLMConcurrency {
|
||||
s, sErr := llm.NewScheduler(inv.Config.EffectiveValidationLLMConcurrency())
|
||||
if sErr != nil {
|
||||
return fail("runner_setup", sErr, nil)
|
||||
}
|
||||
validationScheduler = composeSchedulers(globalScheduler, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
moduleSpecs, err := contracts.ResolveModuleRunSpecs(inv.Config.Modules)
|
||||
if err != nil {
|
||||
return fail("runner_setup", err, nil)
|
||||
}
|
||||
|
||||
runCtx, cancelRun := processRunnerContext()
|
||||
defer cancelRun()
|
||||
runnerResult, runErr := runner.New(moduleFactory).Run(runCtx, runner.RunInput{
|
||||
Config: &inv.Config,
|
||||
Transcript: normalizedTranscript,
|
||||
Glossary: glossary,
|
||||
ModuleSpecs: moduleSpecs,
|
||||
EffectiveConcurrency: runner.EffectiveConcurrencyLimits{
|
||||
TotalLLM: inv.Config.TotalLLMConcurrency,
|
||||
ProposalLLM: inv.Config.EffectiveProposalLLMConcurrency(),
|
||||
ValidationLLM: inv.Config.EffectiveValidationLLMConcurrency(),
|
||||
},
|
||||
ProposalLLMClient: proposalLLMClient,
|
||||
ProposalLLMScheduler: proposalScheduler,
|
||||
ProposalDiagnosticsDir: runDir.Path(),
|
||||
ValidationLLMClient: validationLLMClient,
|
||||
ValidationLLMScheduler: validationScheduler,
|
||||
ValidationDiagnosticsDir: runDir.Path(),
|
||||
})
|
||||
runOutput = &runnerResult
|
||||
if runErr != nil {
|
||||
return fail("runner_execution", runErr, runOutput)
|
||||
}
|
||||
workingTranscript = runnerResult.FinalTranscript
|
||||
}
|
||||
|
||||
encoderDef, err := outputschema.Resolve(inv.Config.OutputSchema)
|
||||
if err != nil {
|
||||
return fail("output_schema", err, runOutput)
|
||||
}
|
||||
outputBytes, err := encoderDef.Encoder(workingTranscript)
|
||||
if err != nil {
|
||||
return fail("serialization", err, runOutput)
|
||||
}
|
||||
|
||||
if strings.TrimSpace(inv.OutputPath) != "" {
|
||||
if err := coreio.WriteFile(inv.OutputPath, outputBytes); err != nil {
|
||||
return fail("output_write", err, runOutput)
|
||||
}
|
||||
return normSummary, &chunkSummary, runOutput, runDir, nil
|
||||
}
|
||||
|
||||
if _, err := stdout.Write(outputBytes); err != nil {
|
||||
return fail("stdout_write", err, runOutput)
|
||||
}
|
||||
|
||||
return normSummary, &chunkSummary, runOutput, runDir, nil
|
||||
}
|
||||
|
||||
func sourceToCanonicalTranscript(source *schema.SourceTranscript) *schema.Transcript {
|
||||
segments := make([]schema.Segment, len(source.Segments))
|
||||
for i, s := range source.Segments {
|
||||
id := i + 1
|
||||
if s.ID != nil {
|
||||
id = *s.ID
|
||||
}
|
||||
segments[i] = schema.Segment{
|
||||
ID: id,
|
||||
Speaker: s.Speaker,
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
Text: s.Text,
|
||||
Categories: s.Categories,
|
||||
}
|
||||
}
|
||||
return &schema.Transcript{Segments: segments}
|
||||
}
|
||||
|
||||
type chainedScheduler struct {
|
||||
schedulers []runner.ValidationScheduler
|
||||
}
|
||||
|
||||
func (s chainedScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
|
||||
if len(s.schedulers) == 0 {
|
||||
return fn(ctx)
|
||||
}
|
||||
run := fn
|
||||
for i := len(s.schedulers) - 1; i >= 0; i-- {
|
||||
scheduler := s.schedulers[i]
|
||||
next := run
|
||||
run = func(callCtx context.Context) error {
|
||||
return scheduler.Run(callCtx, next)
|
||||
}
|
||||
}
|
||||
return run(ctx)
|
||||
}
|
||||
|
||||
func composeSchedulers(schedulers ...runner.ValidationScheduler) runner.ValidationScheduler {
|
||||
filtered := make([]runner.ValidationScheduler, 0, len(schedulers))
|
||||
for _, scheduler := range schedulers {
|
||||
if scheduler != nil {
|
||||
filtered = append(filtered, scheduler)
|
||||
}
|
||||
}
|
||||
switch len(filtered) {
|
||||
case 0:
|
||||
return nil
|
||||
case 1:
|
||||
return filtered[0]
|
||||
default:
|
||||
return chainedScheduler{schedulers: filtered}
|
||||
}
|
||||
}
|
||||
|
||||
// Run executes the Audita CLI with the provided arguments and streams.
|
||||
func Run(args []string, stdout, stderr io.Writer) int {
|
||||
if len(args) == 0 {
|
||||
writeRootUsage(stdout)
|
||||
return 0
|
||||
}
|
||||
|
||||
if isHelpCommand(args) {
|
||||
writeRootUsage(stdout)
|
||||
return 0
|
||||
}
|
||||
|
||||
if args[0] == "process" {
|
||||
return runProcess(args[1:], stdout, stderr)
|
||||
}
|
||||
if args[0] == "config" {
|
||||
return runConfig(args[1:], stdout, stderr)
|
||||
}
|
||||
|
||||
fmt.Fprintf(stderr, "audita: unknown command %q\n\n", args[0])
|
||||
writeRootUsage(stderr)
|
||||
return 2
|
||||
}
|
||||
|
||||
func runProcess(args []string, stdout, stderr io.Writer) int {
|
||||
startedAt := time.Now().UTC()
|
||||
|
||||
configPathOverride, configPathOverrideSet, err := findConfigPathOverride(args)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
effectiveConfig, err := config.LoadEffectiveConfig(configPathOverride, configPathOverrideSet)
|
||||
if err != nil {
|
||||
var effectiveConfigErr *config.EffectiveConfigError
|
||||
if errors.As(err, &effectiveConfigErr) {
|
||||
switch effectiveConfigErr.Kind {
|
||||
case config.EffectiveConfigErrorLoadFile, config.EffectiveConfigErrorApplyFile:
|
||||
fmt.Fprintf(stderr, "audita process: invalid config file: %v\n", effectiveConfigErr)
|
||||
case config.EffectiveConfigErrorApplyEnv:
|
||||
fmt.Fprintf(stderr, "audita process: invalid environment configuration: %v\n", effectiveConfigErr)
|
||||
default:
|
||||
fmt.Fprintf(stderr, "audita process: %v\n", effectiveConfigErr)
|
||||
}
|
||||
} else {
|
||||
fmt.Fprintf(stderr, "audita process: %v\n", err)
|
||||
}
|
||||
return 2
|
||||
}
|
||||
|
||||
cfg := effectiveConfig.Config
|
||||
configPath := effectiveConfig.ConfigPath
|
||||
configSource := effectiveConfig.ConfigSource
|
||||
configVersion := effectiveConfig.ConfigVersion
|
||||
|
||||
fs, pFlags := newProcessFlagSet(cfg, stderr)
|
||||
|
||||
if isHelpCommand(args) || hasHelpFlag(args) {
|
||||
writeProcessUsage(stdout, fs)
|
||||
return 0
|
||||
}
|
||||
|
||||
parseArgs := args
|
||||
transcriptFromFront := ""
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
transcriptFromFront = args[0]
|
||||
parseArgs = args[1:]
|
||||
}
|
||||
|
||||
if err := fs.Parse(parseArgs); err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
writeProcessUsage(stdout, fs)
|
||||
return 0
|
||||
}
|
||||
return 2
|
||||
}
|
||||
|
||||
overrides, explicitModules := processCLIOverrides(fs, pFlags)
|
||||
|
||||
if err := cfg.ApplyCLIOverrides(overrides); err != nil {
|
||||
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
|
||||
remaining := fs.Args()
|
||||
positional := make([]string, 0, len(remaining)+1)
|
||||
if transcriptFromFront != "" {
|
||||
positional = append(positional, transcriptFromFront)
|
||||
}
|
||||
positional = append(positional, remaining...)
|
||||
|
||||
if len(positional) != 1 {
|
||||
fmt.Fprintln(stderr, "audita process: expected exactly 1 transcript JSON path argument")
|
||||
return 2
|
||||
}
|
||||
if strings.TrimSpace(*pFlags.glossaryPath) == "" {
|
||||
fmt.Fprintln(stderr, "audita process: --glossary is required")
|
||||
return 2
|
||||
}
|
||||
|
||||
inv := processInvocation{
|
||||
TranscriptPath: positional[0],
|
||||
GlossaryPath: *pFlags.glossaryPath,
|
||||
OutputPath: *pFlags.outputPath,
|
||||
ReportJSONPath: *pFlags.reportJSONPath,
|
||||
Config: cfg,
|
||||
ConfigPath: configPath,
|
||||
ConfigSource: configSource,
|
||||
ConfigVersion: configVersion,
|
||||
ExplicitModules: explicitModules,
|
||||
}
|
||||
|
||||
normSummary, chunkSummary, runOutput, runDir, runErr := processRunner(inv, stdout)
|
||||
completedAt := time.Now().UTC()
|
||||
|
||||
if runErr != nil {
|
||||
if runDir != nil && runOutput != nil {
|
||||
if runOutput.Utilization != nil {
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
||||
}
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
|
||||
RunDirectoryPath: runDir.Path(),
|
||||
RunOutput: runOutput,
|
||||
}))
|
||||
}
|
||||
errorPhase, errorMessage := extractErrorPhase(runErr)
|
||||
report := processreport.Build(processReportInput("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput))
|
||||
|
||||
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
||||
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
||||
fmt.Fprintf(stderr, "audita process: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
if runDir != nil {
|
||||
_ = runDir.WriteReport(report)
|
||||
_ = runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
||||
RunSucceeded: false,
|
||||
})
|
||||
}
|
||||
|
||||
fmt.Fprintf(stderr, "audita process: %v\n", runErr)
|
||||
if runDir != nil {
|
||||
fmt.Fprintf(stderr, "audita process: diagnostics: %s\n", runDir.Path())
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
if runDir != nil && runOutput != nil {
|
||||
if runOutput.Utilization != nil {
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
||||
}
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
|
||||
RunDirectoryPath: runDir.Path(),
|
||||
RunOutput: runOutput,
|
||||
}))
|
||||
}
|
||||
|
||||
report := processreport.Build(processReportInput("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput))
|
||||
|
||||
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
||||
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
||||
if runDir != nil {
|
||||
_ = runDir.WriteErrorLog(fmt.Sprintf("report_write: %v", err))
|
||||
_ = runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
||||
RunSucceeded: false,
|
||||
})
|
||||
}
|
||||
fmt.Fprintf(stderr, "audita process: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
if runDir != nil {
|
||||
_ = runDir.WriteReport(report)
|
||||
if err := runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
||||
RunSucceeded: true,
|
||||
HasSkippedCorrections: processreport.HasSkippedCorrections(runOutput),
|
||||
}); err != nil {
|
||||
fmt.Fprintf(stderr, "audita process: failed to apply work-dir retention: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
func runConfig(args []string, stdout, stderr io.Writer) int {
|
||||
if len(args) == 0 || isHelpCommand(args) || hasHelpFlag(args) {
|
||||
writeConfigUsage(stdout)
|
||||
return 0
|
||||
}
|
||||
|
||||
switch args[0] {
|
||||
case "validate":
|
||||
return runConfigValidate(args[1:], stdout, stderr)
|
||||
case "print-effective":
|
||||
return runConfigPrintEffective(args[1:], stdout, stderr)
|
||||
default:
|
||||
fmt.Fprintf(stderr, "audita config: unknown command %q\n\n", args[0])
|
||||
writeConfigUsage(stderr)
|
||||
return 2
|
||||
}
|
||||
}
|
||||
|
||||
func runConfigValidate(args []string, stdout, stderr io.Writer) int {
|
||||
fs := flag.NewFlagSet("config validate", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
configPath := fs.String("config", "", "Path to versioned YAML config file")
|
||||
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
writeConfigValidateUsage(stdout)
|
||||
return 0
|
||||
}
|
||||
return 2
|
||||
}
|
||||
if strings.TrimSpace(*configPath) == "" {
|
||||
fmt.Fprintln(stderr, "audita config validate: --config is required")
|
||||
return 2
|
||||
}
|
||||
if len(fs.Args()) != 0 {
|
||||
fmt.Fprintln(stderr, "audita config validate: unexpected positional arguments")
|
||||
return 2
|
||||
}
|
||||
|
||||
fileCfg, err := config.LoadFileConfig(strings.TrimSpace(*configPath))
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
cfg := config.Default()
|
||||
if err := cfg.ApplyFileConfig(fileCfg); err != nil {
|
||||
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
fmt.Fprintln(stdout, "config is valid")
|
||||
return 0
|
||||
}
|
||||
|
||||
func runConfigPrintEffective(args []string, stdout, stderr io.Writer) int {
|
||||
fs := flag.NewFlagSet("config print-effective", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
configPath := fs.String("config", "", "Path to versioned YAML config file")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
writeConfigPrintEffectiveUsage(stdout)
|
||||
return 0
|
||||
}
|
||||
return 2
|
||||
}
|
||||
if len(fs.Args()) != 0 {
|
||||
fmt.Fprintln(stderr, "audita config print-effective: unexpected positional arguments")
|
||||
return 2
|
||||
}
|
||||
|
||||
configPathValue := strings.TrimSpace(*configPath)
|
||||
configPathSet := configPathValue != ""
|
||||
effectiveConfig, err := config.LoadEffectiveConfig(configPathValue, configPathSet)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
|
||||
cfg := effectiveConfig.Config
|
||||
|
||||
redacted := cfg.Redacted()
|
||||
out, err := json.MarshalIndent(redacted, "", " ")
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
out = append(out, '\n')
|
||||
if _, err := stdout.Write(out); err != nil {
|
||||
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func extractErrorPhase(err error) (phase string, message string) {
|
||||
msg := err.Error()
|
||||
if strings.Contains(msg, ": ") {
|
||||
parts := strings.SplitN(msg, ": ", 2)
|
||||
if len(parts) == 2 {
|
||||
return parts[0], parts[1]
|
||||
}
|
||||
}
|
||||
return "", msg
|
||||
}
|
||||
|
||||
func processReportInput(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) processreport.BuildInput {
|
||||
runDirectoryPath := ""
|
||||
if runDir != nil {
|
||||
runDirectoryPath = runDir.Path()
|
||||
}
|
||||
return processreport.BuildInput{
|
||||
Status: status,
|
||||
TranscriptPath: inv.TranscriptPath,
|
||||
GlossaryPath: inv.GlossaryPath,
|
||||
OutputPath: inv.OutputPath,
|
||||
Modules: inv.Config.Modules,
|
||||
OutputSchema: inv.Config.OutputSchema,
|
||||
ConfigVersion: inv.ConfigVersion,
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: completedAt,
|
||||
ErrorMessage: errorMessage,
|
||||
ErrorPhase: errorPhase,
|
||||
RunDirectoryPath: runDirectoryPath,
|
||||
NormalizationSummary: normalizationSummary,
|
||||
ChunkingSummary: chunkingSummary,
|
||||
RunOutput: runOutput,
|
||||
}
|
||||
}
|
||||
|
||||
type processFlags struct {
|
||||
configPath *string
|
||||
glossaryPath *string
|
||||
outputPath *string
|
||||
reportJSONPath *string
|
||||
modules *string
|
||||
outputSchema *string
|
||||
llmAPIKey *string
|
||||
validationLLMAPIKey *string
|
||||
model *string
|
||||
validationModel *string
|
||||
baseURL *string
|
||||
validationBaseURL *string
|
||||
llmTimeoutSeconds *int
|
||||
totalLLMConcurrency *int
|
||||
proposalLLMConcurrency *int
|
||||
llmConcurrency *int
|
||||
validationLLMTimeoutSeconds *int
|
||||
validationMaxPromptTokens *int
|
||||
targetSections *int
|
||||
maxRetries *int
|
||||
validationMaxRetries *int
|
||||
validationLLMConcurrency *int
|
||||
maxSectionTokens *int
|
||||
minSectionTokens *int
|
||||
glossaryConfidenceThreshold *float64
|
||||
grammarConfidenceThreshold *float64
|
||||
homophonesConfidenceThreshold *float64
|
||||
spokenWordConfidenceThreshold *float64
|
||||
normalizeMaxSegmentGap *float64
|
||||
normalizeEllipsisGap *float64
|
||||
normalizeMaxSegmentDuration *float64
|
||||
normalizeMaxSegmentTokens *int
|
||||
transcriptDescription *string
|
||||
workDir *string
|
||||
workDirRetention *string
|
||||
}
|
||||
|
||||
func newProcessFlagSet(cfg config.Config, stderr io.Writer) (*flag.FlagSet, processFlags) {
|
||||
fs := flag.NewFlagSet("process", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
|
||||
validationTimeoutSecondsDefault := cfg.PrimaryLLM.TimeoutSeconds
|
||||
if cfg.ValidationLLM.TimeoutSeconds != nil {
|
||||
validationTimeoutSecondsDefault = *cfg.ValidationLLM.TimeoutSeconds
|
||||
}
|
||||
|
||||
validationMaxRetriesDefault := cfg.PrimaryLLM.MaxRetries
|
||||
if cfg.ValidationLLM.MaxRetries != nil {
|
||||
validationMaxRetriesDefault = *cfg.ValidationLLM.MaxRetries
|
||||
}
|
||||
|
||||
validationLLMConcurrencyDefault := cfg.TotalLLMConcurrency
|
||||
if cfg.ValidationLLMConcurrency != nil {
|
||||
validationLLMConcurrencyDefault = *cfg.ValidationLLMConcurrency
|
||||
}
|
||||
|
||||
targetSectionsDefault := 0
|
||||
if cfg.TargetSections != nil {
|
||||
targetSectionsDefault = *cfg.TargetSections
|
||||
}
|
||||
|
||||
pFlags := processFlags{
|
||||
configPath: fs.String("config", "", "Path to versioned YAML config file"),
|
||||
glossaryPath: fs.String("glossary", "", "Path to glossary YAML file"),
|
||||
outputPath: fs.String("output", "", "Path to corrected transcript JSON output file"),
|
||||
reportJSONPath: fs.String("report-json", "", "Path to machine-readable report JSON output file"),
|
||||
modules: fs.String("modules", strings.Join(cfg.Modules, ","), "Comma-separated module sequence override"),
|
||||
outputSchema: fs.String("output-schema", cfg.OutputSchema, "Output schema: bare-segments|audita-v1"),
|
||||
llmAPIKey: fs.String("llm-api-key", cfg.PrimaryLLM.APIKey, "Primary LLM API key"),
|
||||
validationLLMAPIKey: fs.String("validation-llm-api-key", cfg.ValidationLLM.APIKey, "Validation LLM API key"),
|
||||
model: fs.String("model", cfg.PrimaryLLM.Model, "Primary LLM model name"),
|
||||
validationModel: fs.String("validation-model", cfg.ValidationLLM.Model, "Validation LLM model name"),
|
||||
baseURL: fs.String("base-url", cfg.PrimaryLLM.BaseURL, "Primary OpenAI-compatible base URL"),
|
||||
validationBaseURL: fs.String("validation-base-url", cfg.ValidationLLM.BaseURL, "Validation OpenAI-compatible base URL"),
|
||||
llmTimeoutSeconds: fs.Int("llm-timeout-seconds", cfg.PrimaryLLM.TimeoutSeconds, "Primary LLM timeout in seconds"),
|
||||
totalLLMConcurrency: fs.Int("total-llm-concurrency", cfg.TotalLLMConcurrency, "Total concurrent LLM calls across proposal and validation"),
|
||||
proposalLLMConcurrency: fs.Int("proposal-llm-concurrency", cfg.EffectiveProposalLLMConcurrency(), "Concurrent proposal-generation LLM calls"),
|
||||
llmConcurrency: fs.Int("llm-concurrency", cfg.TotalLLMConcurrency, "Alias for --total-llm-concurrency"),
|
||||
validationLLMTimeoutSeconds: fs.Int("validation-llm-timeout-seconds", validationTimeoutSecondsDefault, "Validation LLM timeout in seconds"),
|
||||
validationMaxPromptTokens: fs.Int("validation-max-prompt-tokens", cfg.ValidationMaxPromptTokens, "Validation max prompt tokens"),
|
||||
targetSections: fs.Int("target-sections", targetSectionsDefault, "Target number of transcript sections"),
|
||||
maxRetries: fs.Int("max-retries", cfg.PrimaryLLM.MaxRetries, "Maximum structured-output retries"),
|
||||
validationMaxRetries: fs.Int("validation-max-retries", validationMaxRetriesDefault, "Validation structured-output retries"),
|
||||
validationLLMConcurrency: fs.Int("validation-llm-concurrency", validationLLMConcurrencyDefault, "Concurrent validation LLM calls (inherits total when unset)"),
|
||||
maxSectionTokens: fs.Int("max-section-tokens", cfg.MaxSectionTokens, "Maximum section tokens"),
|
||||
minSectionTokens: fs.Int("min-section-tokens", cfg.MinSectionTokens, "Minimum section tokens"),
|
||||
glossaryConfidenceThreshold: fs.Float64("glossary-confidence-threshold", cfg.Thresholds.Glossary, "Glossary confidence threshold"),
|
||||
grammarConfidenceThreshold: fs.Float64("grammar-confidence-threshold", cfg.Thresholds.Grammar, "Grammar confidence threshold"),
|
||||
homophonesConfidenceThreshold: fs.Float64("homophones-confidence-threshold", cfg.Thresholds.Homophones, "Homophones confidence threshold"),
|
||||
spokenWordConfidenceThreshold: fs.Float64("spoken-word-confidence-threshold", cfg.Thresholds.SpokenWord, "Spoken-word confidence threshold"),
|
||||
normalizeMaxSegmentGap: fs.Float64("normalize-max-segment-gap", cfg.Normalization.MaxSegmentGap, "Maximum same-speaker merge gap"),
|
||||
normalizeEllipsisGap: fs.Float64("normalize-ellipsis-gap", cfg.Normalization.EllipsisGap, "Gap threshold for ellipsis insertion"),
|
||||
normalizeMaxSegmentDuration: fs.Float64("normalize-max-segment-duration", cfg.Normalization.MaxSegmentDuration, "Maximum merged segment duration"),
|
||||
normalizeMaxSegmentTokens: fs.Int("normalize-max-segment-tokens", cfg.Normalization.MaxSegmentTokens, "Maximum merged segment token estimate"),
|
||||
transcriptDescription: fs.String("transcript-description", cfg.TranscriptDescription, "Brief background context for LLM prompts; does not override transcript content"),
|
||||
workDir: fs.String("work-dir", cfg.WorkDir, "Per-run work directory"),
|
||||
workDirRetention: fs.String("work-dir-retention", string(cfg.WorkDirRetention), "Work-dir retention policy: auto|always|never"),
|
||||
}
|
||||
|
||||
return fs, pFlags
|
||||
}
|
||||
|
||||
func findConfigPathOverride(args []string) (path string, set bool, err error) {
|
||||
for i := 0; i < len(args); i++ {
|
||||
arg := strings.TrimSpace(args[i])
|
||||
if arg == "" {
|
||||
continue
|
||||
}
|
||||
if arg == "--config" {
|
||||
if i+1 >= len(args) {
|
||||
return "", false, fmt.Errorf("--config requires a path")
|
||||
}
|
||||
return strings.TrimSpace(args[i+1]), true, nil
|
||||
}
|
||||
if strings.HasPrefix(arg, "--config=") {
|
||||
return strings.TrimSpace(strings.TrimPrefix(arg, "--config=")), true, nil
|
||||
}
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
func isHelpCommand(args []string) bool {
|
||||
if len(args) == 0 {
|
||||
return false
|
||||
}
|
||||
if len(args) == 1 {
|
||||
switch args[0] {
|
||||
case "help", "-h", "--help":
|
||||
return true
|
||||
}
|
||||
}
|
||||
if len(args) == 2 && args[0] == "help" {
|
||||
switch args[1] {
|
||||
case "process":
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasHelpFlag(args []string) bool {
|
||||
for _, arg := range args {
|
||||
if arg == "-h" || arg == "--help" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func writeRootUsage(w io.Writer) {
|
||||
fmt.Fprintln(w, "Audita is a transcript processing CLI.")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Usage:")
|
||||
fmt.Fprintln(w, " audita <command> [options]")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Commands:")
|
||||
fmt.Fprintln(w, " process Process a transcript JSON file")
|
||||
fmt.Fprintln(w, " config Validate and inspect config")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Example:")
|
||||
fmt.Fprintln(w, " audita process transcript.json --glossary glossary.yaml --output corrected.json")
|
||||
}
|
||||
|
||||
func writeConfigUsage(w io.Writer) {
|
||||
fmt.Fprintln(w, "Validate and inspect Audita config.")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Usage:")
|
||||
fmt.Fprintln(w, " audita config <command> [flags]")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Commands:")
|
||||
fmt.Fprintln(w, " validate Validate a versioned YAML config file")
|
||||
fmt.Fprintln(w, " print-effective Print redacted effective config JSON (defaults + config file + env)")
|
||||
}
|
||||
|
||||
func writeConfigValidateUsage(w io.Writer) {
|
||||
fmt.Fprintln(w, "Validate a versioned YAML config file.")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Usage:")
|
||||
fmt.Fprintln(w, " audita config validate --config <path>")
|
||||
}
|
||||
|
||||
func writeConfigPrintEffectiveUsage(w io.Writer) {
|
||||
fmt.Fprintln(w, "Print redacted effective config JSON.")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Usage:")
|
||||
fmt.Fprintln(w, " audita config print-effective [--config <path>]")
|
||||
}
|
||||
|
||||
func writeProcessUsage(w io.Writer, fs *flag.FlagSet) {
|
||||
fmt.Fprintln(w, "Process a transcript JSON file.")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Usage:")
|
||||
fmt.Fprintln(w, " audita process <transcript.json> [flags]")
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Flags:")
|
||||
fs.VisitAll(func(f *flag.Flag) {
|
||||
fmt.Fprintf(w, " --%s\n", f.Name)
|
||||
})
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "Example:")
|
||||
fmt.Fprintln(w, " audita process transcript.json --glossary glossary.yaml --output corrected.json")
|
||||
}
|
||||
4363
internal/cli/run_test.go
Normal file
4363
internal/cli/run_test.go
Normal file
File diff suppressed because it is too large
Load Diff
110
internal/cli/subprocess_test_hooks.go
Normal file
110
internal/cli/subprocess_test_hooks.go
Normal file
@@ -0,0 +1,110 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
)
|
||||
|
||||
const (
|
||||
subprocessTestLLMModeEnv = "AUDITA_SUBPROCESS_TEST_LLM_MODE"
|
||||
subprocessTestRunTimeoutMSEnv = "AUDITA_SUBPROCESS_TEST_RUN_TIMEOUT_MS"
|
||||
)
|
||||
|
||||
// ConfigureSubprocessTestHooksFromEnv enables deterministic test-only hooks for
|
||||
// subprocess integration tests that run through the Go test binary helper path.
|
||||
func ConfigureSubprocessTestHooksFromEnv() {
|
||||
mode := strings.TrimSpace(os.Getenv(subprocessTestLLMModeEnv))
|
||||
timeoutMSRaw := strings.TrimSpace(os.Getenv(subprocessTestRunTimeoutMSEnv))
|
||||
// Only activate in explicit subprocess test mode.
|
||||
if mode == "" && timeoutMSRaw == "" {
|
||||
return
|
||||
}
|
||||
|
||||
if mode != "" {
|
||||
client := &subprocessTestLLMClient{mode: mode}
|
||||
processProposalLLMClient = client
|
||||
processValidationLLMClient = client
|
||||
}
|
||||
|
||||
if timeoutMSRaw == "" {
|
||||
return
|
||||
}
|
||||
timeoutMS, err := strconv.Atoi(timeoutMSRaw)
|
||||
if err != nil || timeoutMS <= 0 {
|
||||
return
|
||||
}
|
||||
processRunnerContext = func() (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.Background(), time.Duration(timeoutMS)*time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
type subprocessTestLLMClient struct {
|
||||
mode string
|
||||
mu sync.Mutex
|
||||
proposals int
|
||||
}
|
||||
|
||||
func (c *subprocessTestLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
_ = req
|
||||
|
||||
switch c.mode {
|
||||
case "backend_error":
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("synthetic backend failure")
|
||||
case "block_until_cancel":
|
||||
<-ctx.Done()
|
||||
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
||||
case "malformed_structured":
|
||||
switch target := out.(type) {
|
||||
case *proposal_generation.StructuredCorrectionSet:
|
||||
*target = proposal_generation.StructuredCorrectionSet{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y", Confidence: 0.99},
|
||||
},
|
||||
}
|
||||
case *validators.LLMValidationResponse:
|
||||
*target = validators.LLMValidationResponse{
|
||||
Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 999, Approved: true, Confidence: 0.9, Reason: "bad index"},
|
||||
},
|
||||
}
|
||||
}
|
||||
case "mid_pipeline_fail":
|
||||
if _, ok := out.(*proposal_generation.StructuredCorrectionSet); ok {
|
||||
c.mu.Lock()
|
||||
c.proposals++
|
||||
proposalCall := c.proposals
|
||||
c.mu.Unlock()
|
||||
|
||||
if proposalCall >= 3 {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("synthetic mid-pipeline failure")
|
||||
}
|
||||
}
|
||||
switch target := out.(type) {
|
||||
case *proposal_generation.StructuredCorrectionSet:
|
||||
*target = proposal_generation.StructuredCorrectionSet{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99},
|
||||
},
|
||||
}
|
||||
case *validators.LLMValidationResponse:
|
||||
*target = validators.LLMValidationResponse{Validations: nil}
|
||||
}
|
||||
default:
|
||||
switch target := out.(type) {
|
||||
case *proposal_generation.StructuredCorrectionSet:
|
||||
*target = proposal_generation.StructuredCorrectionSet{Corrections: nil}
|
||||
case *validators.LLMValidationResponse:
|
||||
*target = validators.LLMValidationResponse{Validations: nil}
|
||||
}
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
}
|
||||
1
internal/cli/testdata/malformed_transcript.json
vendored
Normal file
1
internal/cli/testdata/malformed_transcript.json
vendored
Normal file
@@ -0,0 +1 @@
|
||||
{"segments":[{"id":1,"text":"oops"}
|
||||
22
internal/cli/testdata/parity/application-skip-ambiguous.case.json
vendored
Normal file
22
internal/cli/testdata/parity/application-skip-ambiguous.case.json
vendored
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"name": "ambiguous_match_rejected_before_application",
|
||||
"transcript_file": "application-skip-ambiguous.transcript.json",
|
||||
"glossary_file": "default-handoff.glossary.yaml",
|
||||
"modules_csv": "homophones",
|
||||
"proposal_responses_file": "application-skip-ambiguous.proposals.json",
|
||||
"validation_responses_file": "application-skip-ambiguous.validations.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "application-skip-ambiguous.expected-transcript.json",
|
||||
"module_instances": ["homophones"],
|
||||
"module_count": 1,
|
||||
"total_applied_changes": 0,
|
||||
"total_skipped_changes": 1,
|
||||
"module_applied_counts": [0],
|
||||
"module_rejected_counts": [1],
|
||||
"module_skip_counts": [0],
|
||||
"validator_rejected_reason_codes": ["ambiguous_original_text"],
|
||||
"expected_proposal_calls": ["homophones:proposal"]
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/application-skip-ambiguous.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/application-skip-ambiguous.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"the site near another site"}
|
||||
]
|
||||
3
internal/cli/testdata/parity/application-skip-ambiguous.proposals.json
vendored
Normal file
3
internal/cli/testdata/parity/application-skip-ambiguous.proposals.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]}
|
||||
]
|
||||
3
internal/cli/testdata/parity/application-skip-ambiguous.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/application-skip-ambiguous.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"the site near another site"}
|
||||
]
|
||||
4
internal/cli/testdata/parity/application-skip-ambiguous.validations.json
vendored
Normal file
4
internal/cli/testdata/parity/application-skip-ambiguous.validations.json
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
28
internal/cli/testdata/parity/default-full-pipeline.case.json
vendored
Normal file
28
internal/cli/testdata/parity/default-full-pipeline.case.json
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "default_full_pipeline_shape_and_reports",
|
||||
"transcript_file": "default-full-pipeline.transcript.json",
|
||||
"glossary_file": "default-full-pipeline.glossary.yaml",
|
||||
"proposal_responses_file": "default-full-pipeline.proposals.json",
|
||||
"validation_responses_file": "default-full-pipeline.validations.json",
|
||||
"env": {
|
||||
"AUDITA_LLM_API_KEY": "parity-secret",
|
||||
"AUDITA_VALIDATION_LLM_API_KEY": "parity-secret"
|
||||
},
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "default-full-pipeline.expected-transcript.json",
|
||||
"module_instances": ["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
|
||||
"module_count": 5,
|
||||
"total_applied_changes": 3,
|
||||
"total_skipped_changes": 3,
|
||||
"secret_markers": ["parity-secret"],
|
||||
"expected_proposal_calls": [
|
||||
"glossary_1:proposal",
|
||||
"homophones:proposal",
|
||||
"glossary_2:proposal",
|
||||
"spoken_word:proposal",
|
||||
"grammar:proposal"
|
||||
]
|
||||
}
|
||||
}
|
||||
9
internal/cli/testdata/parity/default-full-pipeline.expected-transcript.json
vendored
Normal file
9
internal/cli/testdata/parity/default-full-pipeline.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "Alice",
|
||||
"start": 0,
|
||||
"end": 1,
|
||||
"text": "Hello, there were Jesters hmm"
|
||||
}
|
||||
]
|
||||
7
internal/cli/testdata/parity/default-full-pipeline.glossary.yaml
vendored
Normal file
7
internal/cli/testdata/parity/default-full-pipeline.glossary.yaml
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
glossary:
|
||||
- name: Jesters
|
||||
aliases:
|
||||
- jester
|
||||
plural: jesters
|
||||
category: faction
|
||||
summary: A protected in-world faction term.
|
||||
28
internal/cli/testdata/parity/default-full-pipeline.proposals.json
vendored
Normal file
28
internal/cli/testdata/parity/default-full-pipeline.proposals.json
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
[
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "Jesters", "corrected_text": "jesters", "confidence": 0.99},
|
||||
{"id": 1, "original_text": "Jesters", "corrected_text": "JESTERX", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "jesters", "corrected_text": "JESTERS", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "uh", "corrected_text": "hmm", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
|
||||
]
|
||||
}
|
||||
]
|
||||
3
internal/cli/testdata/parity/default-full-pipeline.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/default-full-pipeline.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
|
||||
]
|
||||
12
internal/cli/testdata/parity/default-full-pipeline.validations.json
vendored
Normal file
12
internal/cli/testdata/parity/default-full-pipeline.validations.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject cleanup"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "parity-secret"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
27
internal/cli/testdata/parity/default-handoff.case.json
vendored
Normal file
27
internal/cli/testdata/parity/default-handoff.case.json
vendored
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"name": "default_pipeline_handoff_and_module_order",
|
||||
"transcript_file": "default-handoff.transcript.json",
|
||||
"glossary_file": "default-handoff.glossary.yaml",
|
||||
"proposal_responses_file": "default-handoff.proposals.json",
|
||||
"validation_responses_file": "default-handoff.validations.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "default-handoff.expected-transcript.json",
|
||||
"module_instances": ["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
|
||||
"module_count": 5,
|
||||
"total_applied_changes": 5,
|
||||
"total_skipped_changes": 0,
|
||||
"module_applied_counts": [1, 1, 1, 1, 1],
|
||||
"module_rejected_counts": [0, 0, 0, 0, 0],
|
||||
"module_skip_counts": [0, 0, 0, 0, 0],
|
||||
"expected_proposal_calls": [
|
||||
"glossary_1:proposal",
|
||||
"homophones:proposal",
|
||||
"glossary_2:proposal",
|
||||
"spoken_word:proposal",
|
||||
"grammar:proposal"
|
||||
],
|
||||
"min_response_payload_artifacts": 15
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/default-handoff.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/default-handoff.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"Hello, there were Jesters at the Sight um"}
|
||||
]
|
||||
6
internal/cli/testdata/parity/default-handoff.glossary.yaml
vendored
Normal file
6
internal/cli/testdata/parity/default-handoff.glossary.yaml
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
glossary:
|
||||
- name: Jesters
|
||||
aliases: [jester]
|
||||
plural: jesters
|
||||
category: faction
|
||||
summary: A protected in-world faction term.
|
||||
7
internal/cli/testdata/parity/default-handoff.proposals.json
vendored
Normal file
7
internal/cli/testdata/parity/default-handoff.proposals.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
[
|
||||
{"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "sight", "corrected_text": "Sight", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "um um", "corrected_text": "um", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}]}
|
||||
]
|
||||
3
internal/cli/testdata/parity/default-handoff.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/default-handoff.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures at the site um um"}
|
||||
]
|
||||
12
internal/cli/testdata/parity/default-handoff.validations.json
vendored
Normal file
12
internal/cli/testdata/parity/default-handoff.validations.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
17
internal/cli/testdata/parity/deterministic-validator-low-confidence.case.json
vendored
Normal file
17
internal/cli/testdata/parity/deterministic-validator-low-confidence.case.json
vendored
Normal file
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"name": "deterministic_validator_low_confidence",
|
||||
"transcript_file": "deterministic-validator-low-confidence.transcript.json",
|
||||
"glossary_file": "default-full-pipeline.glossary.yaml",
|
||||
"modules_csv": "grammar",
|
||||
"proposal_responses_file": "deterministic-validator-low-confidence.proposals.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "deterministic-validator-low-confidence.expected-transcript.json",
|
||||
"module_instances": ["grammar"],
|
||||
"module_count": 1,
|
||||
"total_skipped_changes": 1,
|
||||
"validator_rejected_reason_codes": ["low_confidence"],
|
||||
"expected_proposal_calls": ["grammar:proposal"]
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/deterministic-validator-low-confidence.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/deterministic-validator-low-confidence.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"hello , world"}
|
||||
]
|
||||
7
internal/cli/testdata/parity/deterministic-validator-low-confidence.proposals.json
vendored
Normal file
7
internal/cli/testdata/parity/deterministic-validator-low-confidence.proposals.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
[
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.1}
|
||||
]
|
||||
}
|
||||
]
|
||||
3
internal/cli/testdata/parity/deterministic-validator-low-confidence.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/deterministic-validator-low-confidence.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , world"}
|
||||
]
|
||||
12
internal/cli/testdata/parity/glossary-schema-error.case.json
vendored
Normal file
12
internal/cli/testdata/parity/glossary-schema-error.case.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"name": "glossary_schema_handling",
|
||||
"transcript_file": "default-full-pipeline.transcript.json",
|
||||
"glossary_file": "glossary-schema-error.glossary.yaml",
|
||||
"expect": {
|
||||
"exit_code": 1,
|
||||
"status": "failed",
|
||||
"error_phase": "glossary_schema",
|
||||
"stderr_contains": "glossary_schema",
|
||||
"require_error_log": true
|
||||
}
|
||||
}
|
||||
2
internal/cli/testdata/parity/glossary-schema-error.glossary.yaml
vendored
Normal file
2
internal/cli/testdata/parity/glossary-schema-error.glossary.yaml
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
glossary:
|
||||
- name: MissingCategoryAndSummary
|
||||
19
internal/cli/testdata/parity/llm-validator-rejection.case.json
vendored
Normal file
19
internal/cli/testdata/parity/llm-validator-rejection.case.json
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "llm_validator_decision_handling",
|
||||
"transcript_file": "llm-validator-rejection.transcript.json",
|
||||
"glossary_file": "default-full-pipeline.glossary.yaml",
|
||||
"modules_csv": "grammar",
|
||||
"proposal_responses_file": "llm-validator-rejection.proposals.json",
|
||||
"validation_responses_file": "llm-validator-rejection.validations.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "llm-validator-rejection.expected-transcript.json",
|
||||
"module_instances": ["grammar"],
|
||||
"module_count": 1,
|
||||
"total_skipped_changes": 1,
|
||||
"validator_rejected_reason_codes": ["llm_rejected"],
|
||||
"expected_proposal_calls": ["grammar:proposal"],
|
||||
"expected_validation_calls": ["grammar:section-0000:editorial_review:batch-0000"]
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/llm-validator-rejection.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/llm-validator-rejection.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"hello , world"}
|
||||
]
|
||||
7
internal/cli/testdata/parity/llm-validator-rejection.proposals.json
vendored
Normal file
7
internal/cli/testdata/parity/llm-validator-rejection.proposals.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
[
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
|
||||
]
|
||||
}
|
||||
]
|
||||
3
internal/cli/testdata/parity/llm-validator-rejection.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/llm-validator-rejection.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , world"}
|
||||
]
|
||||
7
internal/cli/testdata/parity/llm-validator-rejection.validations.json
vendored
Normal file
7
internal/cli/testdata/parity/llm-validator-rejection.validations.json
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
[
|
||||
{
|
||||
"validations": [
|
||||
{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject stylistic overreach"}
|
||||
]
|
||||
}
|
||||
]
|
||||
23
internal/cli/testdata/parity/mid-pipeline-failure.case.json
vendored
Normal file
23
internal/cli/testdata/parity/mid-pipeline-failure.case.json
vendored
Normal file
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "mid_pipeline_failure_partial_progress",
|
||||
"transcript_file": "default-handoff.transcript.json",
|
||||
"glossary_file": "default-handoff.glossary.yaml",
|
||||
"proposal_responses_file": "mid-pipeline-failure.proposals.json",
|
||||
"validation_responses_file": "mid-pipeline-failure.validations.json",
|
||||
"expect": {
|
||||
"exit_code": 1,
|
||||
"status": "failed",
|
||||
"error_phase": "runner_execution",
|
||||
"stderr_contains": "runner_execution",
|
||||
"module_instances": ["glossary_1", "homophones", "glossary_2"],
|
||||
"module_count": 3,
|
||||
"total_applied_changes": 2,
|
||||
"total_skipped_changes": 0,
|
||||
"failed_module_instance": "glossary_2",
|
||||
"module_applied_counts": [1, 1, 0],
|
||||
"module_rejected_counts": [0, 0, 0],
|
||||
"module_skip_counts": [0, 0, 0],
|
||||
"require_error_log": true,
|
||||
"expected_proposal_calls": ["glossary_1:proposal", "homophones:proposal", "glossary_2:proposal"]
|
||||
}
|
||||
}
|
||||
4
internal/cli/testdata/parity/mid-pipeline-failure.proposals.json
vendored
Normal file
4
internal/cli/testdata/parity/mid-pipeline-failure.proposals.json
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
[
|
||||
{"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}]},
|
||||
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]}
|
||||
]
|
||||
6
internal/cli/testdata/parity/mid-pipeline-failure.validations.json
vendored
Normal file
6
internal/cli/testdata/parity/mid-pipeline-failure.validations.json
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
21
internal/cli/testdata/parity/protected-term-rejection.case.json
vendored
Normal file
21
internal/cli/testdata/parity/protected-term-rejection.case.json
vendored
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "protected_glossary_term_behavior",
|
||||
"transcript_file": "protected-term-rejection.transcript.json",
|
||||
"glossary_file": "default-handoff.glossary.yaml",
|
||||
"modules_csv": "homophones",
|
||||
"proposal_responses_file": "protected-term-rejection.proposals.json",
|
||||
"expect": {
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
"output_transcript_file": "protected-term-rejection.expected-transcript.json",
|
||||
"module_instances": ["homophones"],
|
||||
"module_count": 1,
|
||||
"total_applied_changes": 0,
|
||||
"total_skipped_changes": 1,
|
||||
"module_applied_counts": [0],
|
||||
"module_rejected_counts": [1],
|
||||
"module_skip_counts": [0],
|
||||
"validator_rejected_reason_codes": ["protected_glossary_term"],
|
||||
"expected_proposal_calls": ["homophones:proposal"]
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/protected-term-rejection.expected-transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/protected-term-rejection.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"The Jesters entered the hall."}
|
||||
]
|
||||
3
internal/cli/testdata/parity/protected-term-rejection.proposals.json
vendored
Normal file
3
internal/cli/testdata/parity/protected-term-rejection.proposals.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"corrections": [{"id": 1, "original_text": "Jesters", "corrected_text": "Gestures", "confidence": 0.99}]}
|
||||
]
|
||||
3
internal/cli/testdata/parity/protected-term-rejection.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/protected-term-rejection.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"The Jesters entered the hall."}
|
||||
]
|
||||
12
internal/cli/testdata/parity/transcript-schema-error.case.json
vendored
Normal file
12
internal/cli/testdata/parity/transcript-schema-error.case.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"name": "transcript_schema_handling",
|
||||
"transcript_file": "transcript-schema-error.transcript.json",
|
||||
"glossary_file": "default-full-pipeline.glossary.yaml",
|
||||
"expect": {
|
||||
"exit_code": 1,
|
||||
"status": "failed",
|
||||
"error_phase": "transcript_schema",
|
||||
"stderr_contains": "transcript_schema",
|
||||
"require_error_log": true
|
||||
}
|
||||
}
|
||||
3
internal/cli/testdata/parity/transcript-schema-error.transcript.json
vendored
Normal file
3
internal/cli/testdata/parity/transcript-schema-error.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"","start":0.0,"end":1.0,"text":"bad"}
|
||||
]
|
||||
24
internal/cli/testdata/release/default-release.expectations.json
vendored
Normal file
24
internal/cli/testdata/release/default-release.expectations.json
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"must_apply_texts": [
|
||||
"Hello, there were Jesters hmm"
|
||||
],
|
||||
"must_not_apply_texts": [
|
||||
"JESTERX",
|
||||
"there were gestures"
|
||||
],
|
||||
"protected_terms": [
|
||||
"Jesters"
|
||||
],
|
||||
"expected_module_instances": [
|
||||
"glossary_1",
|
||||
"homophones",
|
||||
"glossary_2",
|
||||
"spoken_word",
|
||||
"grammar"
|
||||
],
|
||||
"minimum_counts": {
|
||||
"applied": 1,
|
||||
"rejected": 1,
|
||||
"skipped": 0
|
||||
}
|
||||
}
|
||||
9
internal/cli/testdata/release/default-release.expected-transcript.json
vendored
Normal file
9
internal/cli/testdata/release/default-release.expected-transcript.json
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "Alice",
|
||||
"start": 0,
|
||||
"end": 1,
|
||||
"text": "Hello, there were Jesters hmm"
|
||||
}
|
||||
]
|
||||
7
internal/cli/testdata/release/default-release.glossary.yaml
vendored
Normal file
7
internal/cli/testdata/release/default-release.glossary.yaml
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
glossary:
|
||||
- name: Jesters
|
||||
aliases:
|
||||
- jester
|
||||
plural: jesters
|
||||
category: faction
|
||||
summary: A protected in-world faction term.
|
||||
28
internal/cli/testdata/release/default-release.proposals.json
vendored
Normal file
28
internal/cli/testdata/release/default-release.proposals.json
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
[
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "Jesters", "corrected_text": "jesters", "confidence": 0.99},
|
||||
{"id": 1, "original_text": "Jesters", "corrected_text": "JESTERX", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "jesters", "corrected_text": "JESTERS", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "uh", "corrected_text": "hmm", "confidence": 0.99}
|
||||
]
|
||||
},
|
||||
{
|
||||
"corrections": [
|
||||
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
|
||||
]
|
||||
}
|
||||
]
|
||||
3
internal/cli/testdata/release/default-release.transcript.json
vendored
Normal file
3
internal/cli/testdata/release/default-release.transcript.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
|
||||
]
|
||||
12
internal/cli/testdata/release/default-release.validations.json
vendored
Normal file
12
internal/cli/testdata/release/default-release.validations.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
[
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject cleanup"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "release-secret"}]},
|
||||
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
|
||||
]
|
||||
6
internal/cli/testdata/tiny_glossary.yaml
vendored
Normal file
6
internal/cli/testdata/tiny_glossary.yaml
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
glossary:
|
||||
- name: Audita
|
||||
aliases:
|
||||
- audita
|
||||
category: product
|
||||
summary: The Audita transcript correction CLI.
|
||||
9
internal/cli/testdata/tiny_transcript.json
vendored
Normal file
9
internal/cli/testdata/tiny_transcript.json
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "A",
|
||||
"start": 0.0,
|
||||
"end": 1.2,
|
||||
"text": "hello world"
|
||||
}
|
||||
]
|
||||
312
internal/core/chunking/sections.go
Normal file
312
internal/core/chunking/sections.go
Normal file
@@ -0,0 +1,312 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
// Section represents a contiguous chunk of transcript segments with metadata.
|
||||
type Section struct {
|
||||
// Index is the 0-based section index within the chunked transcript
|
||||
Index int `json:"section_index"`
|
||||
|
||||
// StartSegmentID is the ID of the first segment in this section
|
||||
StartSegmentID int `json:"start_segment_id"`
|
||||
|
||||
// EndSegmentID is the ID of the last segment in this section
|
||||
EndSegmentID int `json:"end_segment_id"`
|
||||
|
||||
// EstimatedTokens is the approximate token count for this section
|
||||
EstimatedTokens int `json:"estimated_tokens"`
|
||||
|
||||
// Segments contains the segments in this section, in order
|
||||
Segments []schema.Segment `json:"segments"`
|
||||
}
|
||||
|
||||
// ChunkingConfig holds configuration for transcript chunking.
|
||||
type ChunkingConfig struct {
|
||||
// MaxSectionTokens is the maximum allowed tokens per section
|
||||
MaxSectionTokens int
|
||||
|
||||
// MinSectionTokens is a validated soft lower-bound setting retained for
|
||||
// configuration/reporting compatibility.
|
||||
MinSectionTokens int
|
||||
|
||||
// TargetSections is an optional target number of sections
|
||||
// If nil, section count is derived from total/max token budgeting.
|
||||
TargetSections *int
|
||||
}
|
||||
|
||||
// Chunker performs deterministic chunking of normalized transcript segments.
|
||||
type Chunker struct {
|
||||
config ChunkingConfig
|
||||
estimator TokenEstimator
|
||||
}
|
||||
|
||||
// NewChunker creates a new chunker with the given configuration.
|
||||
func NewChunker(config ChunkingConfig) *Chunker {
|
||||
return &Chunker{
|
||||
config: config,
|
||||
estimator: NewSimpleTokenEstimator(),
|
||||
}
|
||||
}
|
||||
|
||||
// NewChunkerWithEstimator creates a new chunker with a custom estimator.
|
||||
func NewChunkerWithEstimator(config ChunkingConfig, estimator TokenEstimator) *Chunker {
|
||||
return &Chunker{
|
||||
config: config,
|
||||
estimator: estimator,
|
||||
}
|
||||
}
|
||||
|
||||
// ChunkTranscript divides a normalized transcript into contiguous token-bounded
|
||||
// sections using a deterministic balanced forward pass.
|
||||
//
|
||||
// Behavior:
|
||||
// - preserve segment order and never split segments;
|
||||
// - estimate per-segment tokens once, then compute total;
|
||||
// - derive desired section count from ceil(total/max_section_tokens), unless
|
||||
// target_sections is explicitly set;
|
||||
// - prefer section sizes near ceil(total/section_count) while never exceeding
|
||||
// max_section_tokens unless a section consists of a single oversized segment.
|
||||
//
|
||||
// Returns an error if explicit target_sections is impossible under constraints.
|
||||
// The input transcript is never mutated.
|
||||
func (c *Chunker) ChunkTranscript(transcript *schema.Transcript) ([]Section, error) {
|
||||
if transcript == nil || len(transcript.Segments) == 0 {
|
||||
return []Section{}, nil
|
||||
}
|
||||
|
||||
// Validate configuration
|
||||
if c.config.MaxSectionTokens <= 0 {
|
||||
return nil, fmt.Errorf("max_section_tokens must be positive, got %d", c.config.MaxSectionTokens)
|
||||
}
|
||||
|
||||
if c.config.MinSectionTokens < 0 {
|
||||
return nil, fmt.Errorf("min_section_tokens must be non-negative, got %d", c.config.MinSectionTokens)
|
||||
}
|
||||
|
||||
if c.config.MinSectionTokens > c.config.MaxSectionTokens {
|
||||
return nil, fmt.Errorf("min_section_tokens (%d) cannot exceed max_section_tokens (%d)",
|
||||
c.config.MinSectionTokens, c.config.MaxSectionTokens)
|
||||
}
|
||||
|
||||
// Calculate token counts for each segment (deterministic).
|
||||
segmentTokens := make([]int, len(transcript.Segments))
|
||||
totalTokens := 0
|
||||
for i, seg := range transcript.Segments {
|
||||
segmentTokens[i] = c.estimator.EstimateTokens(seg.Text)
|
||||
totalTokens += segmentTokens[i]
|
||||
}
|
||||
|
||||
minPossibleSections := c.calculateMinPossibleSections(segmentTokens)
|
||||
|
||||
var desiredSections int
|
||||
useExplicitTarget := false
|
||||
if c.config.TargetSections != nil {
|
||||
desiredSections = *c.config.TargetSections
|
||||
useExplicitTarget = true
|
||||
if desiredSections <= 0 {
|
||||
return nil, fmt.Errorf("target_sections must be positive, got %d", desiredSections)
|
||||
}
|
||||
if err := c.validateTargetSections(desiredSections, segmentTokens); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
desiredSections = ceilDiv(totalTokens, c.config.MaxSectionTokens)
|
||||
if desiredSections < minPossibleSections {
|
||||
desiredSections = minPossibleSections
|
||||
}
|
||||
if desiredSections < 1 {
|
||||
desiredSections = 1
|
||||
}
|
||||
if desiredSections > len(transcript.Segments) {
|
||||
desiredSections = len(transcript.Segments)
|
||||
}
|
||||
}
|
||||
|
||||
targetTokensPerSection := ceilDiv(totalTokens, desiredSections)
|
||||
if useExplicitTarget {
|
||||
return c.buildSectionsWithExplicitTarget(
|
||||
transcript.Segments,
|
||||
segmentTokens,
|
||||
desiredSections,
|
||||
targetTokensPerSection,
|
||||
)
|
||||
}
|
||||
|
||||
return c.buildSectionsBalanced(transcript.Segments, segmentTokens, targetTokensPerSection), nil
|
||||
}
|
||||
|
||||
// validateTargetSections checks if the target section count is achievable.
|
||||
func (c *Chunker) validateTargetSections(target int, segmentTokens []int) error {
|
||||
// Maximum possible sections: limited by segment count
|
||||
maxPossibleSections := len(segmentTokens)
|
||||
|
||||
if target > maxPossibleSections {
|
||||
return fmt.Errorf(
|
||||
"target_sections (%d) is impossible: cannot have more sections than segments (%d)",
|
||||
target, maxPossibleSections)
|
||||
}
|
||||
|
||||
// Minimum possible sections: each segment must fit within max bounds
|
||||
minPossibleSections := c.calculateMinPossibleSections(segmentTokens)
|
||||
|
||||
if target < minPossibleSections {
|
||||
return fmt.Errorf(
|
||||
"target_sections (%d) is impossible: need at least %d sections to respect max_section_tokens (%d)",
|
||||
target, minPossibleSections, c.config.MaxSectionTokens)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildSectionsBalanced creates sections with a deterministic single-pass policy.
|
||||
func (c *Chunker) buildSectionsBalanced(segments []schema.Segment, segmentTokens []int, targetTokensPerSection int) []Section {
|
||||
var sections []Section
|
||||
var currentSegments []schema.Segment
|
||||
currentTokens := 0
|
||||
|
||||
for i, seg := range segments {
|
||||
tokens := segmentTokens[i]
|
||||
|
||||
// Empty section: always accept the next segment, including oversized.
|
||||
if len(currentSegments) == 0 {
|
||||
currentSegments = append(currentSegments, seg)
|
||||
currentTokens = tokens
|
||||
continue
|
||||
}
|
||||
|
||||
// If adding next segment would exceed max, close current section.
|
||||
if currentTokens+tokens > c.config.MaxSectionTokens {
|
||||
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
|
||||
currentSegments = []schema.Segment{seg}
|
||||
currentTokens = tokens
|
||||
continue
|
||||
}
|
||||
|
||||
// Prefer staying near target tokens per section.
|
||||
if targetTokensPerSection == 0 || currentTokens < targetTokensPerSection {
|
||||
currentSegments = append(currentSegments, seg)
|
||||
currentTokens += tokens
|
||||
continue
|
||||
}
|
||||
|
||||
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
|
||||
currentSegments = []schema.Segment{seg}
|
||||
currentTokens = tokens
|
||||
}
|
||||
|
||||
if len(currentSegments) > 0 {
|
||||
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
|
||||
}
|
||||
|
||||
return sections
|
||||
}
|
||||
|
||||
// buildSection creates a Section from segments.
|
||||
func (c *Chunker) buildSection(index int, segments []schema.Segment, tokens int) Section {
|
||||
return Section{
|
||||
Index: index,
|
||||
StartSegmentID: segments[0].ID,
|
||||
EndSegmentID: segments[len(segments)-1].ID,
|
||||
EstimatedTokens: tokens,
|
||||
Segments: segments,
|
||||
}
|
||||
}
|
||||
|
||||
// calculateMinPossibleSections calculates the minimum number of sections needed
|
||||
// to ensure no section exceeds max tokens.
|
||||
func (c *Chunker) calculateMinPossibleSections(segmentTokens []int) int {
|
||||
sections := 0
|
||||
currentTokens := 0
|
||||
|
||||
for _, tokens := range segmentTokens {
|
||||
if tokens > c.config.MaxSectionTokens {
|
||||
// Each oversized segment needs its own section
|
||||
if currentTokens > 0 {
|
||||
sections++
|
||||
currentTokens = 0
|
||||
}
|
||||
sections++
|
||||
} else if currentTokens+tokens > c.config.MaxSectionTokens {
|
||||
sections++
|
||||
currentTokens = tokens
|
||||
} else {
|
||||
currentTokens += tokens
|
||||
}
|
||||
}
|
||||
|
||||
if currentTokens > 0 {
|
||||
sections++
|
||||
}
|
||||
|
||||
return sections
|
||||
}
|
||||
|
||||
// buildSectionsWithExplicitTarget builds exactly desiredSections when feasible.
|
||||
func (c *Chunker) buildSectionsWithExplicitTarget(
|
||||
segments []schema.Segment,
|
||||
segmentTokens []int,
|
||||
desiredSections int,
|
||||
targetTokensPerSection int,
|
||||
) ([]Section, error) {
|
||||
n := len(segments)
|
||||
cursor := 0
|
||||
sections := make([]Section, 0, desiredSections)
|
||||
|
||||
for sectionIdx := 0; sectionIdx < desiredSections; sectionIdx++ {
|
||||
if cursor >= n {
|
||||
break
|
||||
}
|
||||
|
||||
remainingSectionsAfter := desiredSections - sectionIdx - 1
|
||||
currentSegments := []schema.Segment{segments[cursor]}
|
||||
currentTokens := segmentTokens[cursor]
|
||||
cursor++
|
||||
|
||||
for cursor < n {
|
||||
remainingSegments := n - cursor
|
||||
|
||||
// Reserve one segment per future section to avoid empty sections.
|
||||
if remainingSegments == remainingSectionsAfter {
|
||||
break
|
||||
}
|
||||
|
||||
nextTokens := segmentTokens[cursor]
|
||||
if currentTokens+nextTokens > c.config.MaxSectionTokens {
|
||||
break
|
||||
}
|
||||
if targetTokensPerSection == 0 || currentTokens < targetTokensPerSection {
|
||||
currentSegments = append(currentSegments, segments[cursor])
|
||||
currentTokens += nextTokens
|
||||
cursor++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
|
||||
}
|
||||
|
||||
if cursor != n || len(sections) != desiredSections {
|
||||
return nil, fmt.Errorf(
|
||||
"target_sections (%d) is impossible under current constraints (got %d sections)",
|
||||
desiredSections,
|
||||
len(sections),
|
||||
)
|
||||
}
|
||||
|
||||
return sections, nil
|
||||
}
|
||||
|
||||
func ceilDiv(numerator int, denominator int) int {
|
||||
if denominator <= 0 {
|
||||
return 0
|
||||
}
|
||||
if numerator <= 0 {
|
||||
return 0
|
||||
}
|
||||
return (numerator + denominator - 1) / denominator
|
||||
}
|
||||
518
internal/core/chunking/sections_test.go
Normal file
518
internal/core/chunking/sections_test.go
Normal file
@@ -0,0 +1,518 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
// mapTokenEstimator provides deterministic per-segment token counts for tests.
|
||||
type mapTokenEstimator struct {
|
||||
byText map[string]int
|
||||
}
|
||||
|
||||
func (e *mapTokenEstimator) EstimateTokens(text string) int {
|
||||
if e.byText == nil {
|
||||
return 0
|
||||
}
|
||||
if tokens, ok := e.byText[text]; ok {
|
||||
return tokens
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func makeSegments(texts []string) []schema.Segment {
|
||||
segments := make([]schema.Segment, len(texts))
|
||||
for i, text := range texts {
|
||||
segments[i] = schema.Segment{
|
||||
ID: i + 1,
|
||||
Speaker: "DM",
|
||||
Start: float64(i * 10),
|
||||
End: float64(i*10 + 5),
|
||||
Text: text,
|
||||
}
|
||||
}
|
||||
return segments
|
||||
}
|
||||
|
||||
func makeTranscript(segments []schema.Segment) *schema.Transcript {
|
||||
return &schema.Transcript{Segments: segments}
|
||||
}
|
||||
|
||||
func intPtr(i int) *int {
|
||||
return &i
|
||||
}
|
||||
|
||||
func assertSegmentCoverageAndOrder(t *testing.T, input []schema.Segment, sections []Section) {
|
||||
t.Helper()
|
||||
|
||||
seen := make([]schema.Segment, 0, len(input))
|
||||
for _, sec := range sections {
|
||||
seen = append(seen, sec.Segments...)
|
||||
}
|
||||
|
||||
if len(seen) != len(input) {
|
||||
t.Fatalf("expected %d total segment occurrences, got %d", len(input), len(seen))
|
||||
}
|
||||
|
||||
for i := range input {
|
||||
if seen[i].ID != input[i].ID {
|
||||
t.Fatalf("segment order mismatch at index %d: got id=%d want id=%d", i, seen[i].ID, input[i].ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertSectionMetadataConsistent(t *testing.T, sections []Section) {
|
||||
t.Helper()
|
||||
|
||||
for i, sec := range sections {
|
||||
if sec.Index != i {
|
||||
t.Fatalf("section %d: expected index=%d got=%d", i, i, sec.Index)
|
||||
}
|
||||
if len(sec.Segments) == 0 {
|
||||
t.Fatalf("section %d: section must not be empty", i)
|
||||
}
|
||||
if sec.StartSegmentID != sec.Segments[0].ID {
|
||||
t.Fatalf("section %d: start_segment_id mismatch", i)
|
||||
}
|
||||
if sec.EndSegmentID != sec.Segments[len(sec.Segments)-1].ID {
|
||||
t.Fatalf("section %d: end_segment_id mismatch", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertMaxBoundExceptSingletonOversized(t *testing.T, sections []Section, max int) {
|
||||
t.Helper()
|
||||
|
||||
for i, sec := range sections {
|
||||
if sec.EstimatedTokens <= max {
|
||||
continue
|
||||
}
|
||||
if len(sec.Segments) != 1 {
|
||||
t.Fatalf("section %d exceeds max tokens (%d>%d) with %d segments", i, sec.EstimatedTokens, max, len(sec.Segments))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func imbalance(sections []Section) int {
|
||||
if len(sections) == 0 {
|
||||
return 0
|
||||
}
|
||||
minTokens := sections[0].EstimatedTokens
|
||||
maxTokens := sections[0].EstimatedTokens
|
||||
for _, sec := range sections {
|
||||
if sec.EstimatedTokens < minTokens {
|
||||
minTokens = sec.EstimatedTokens
|
||||
}
|
||||
if sec.EstimatedTokens > maxTokens {
|
||||
maxTokens = sec.EstimatedTokens
|
||||
}
|
||||
}
|
||||
return maxTokens - minTokens
|
||||
}
|
||||
|
||||
func greedyMaxFillSections(segments []schema.Segment, tokens []int, max int) []Section {
|
||||
sections := make([]Section, 0)
|
||||
var current []schema.Segment
|
||||
currentTokens := 0
|
||||
|
||||
for i, seg := range segments {
|
||||
tok := tokens[i]
|
||||
if len(current) == 0 {
|
||||
current = append(current, seg)
|
||||
currentTokens = tok
|
||||
continue
|
||||
}
|
||||
if currentTokens+tok > max {
|
||||
sections = append(sections, Section{
|
||||
Index: len(sections),
|
||||
StartSegmentID: current[0].ID,
|
||||
EndSegmentID: current[len(current)-1].ID,
|
||||
EstimatedTokens: currentTokens,
|
||||
Segments: append([]schema.Segment(nil), current...),
|
||||
})
|
||||
current = []schema.Segment{seg}
|
||||
currentTokens = tok
|
||||
continue
|
||||
}
|
||||
current = append(current, seg)
|
||||
currentTokens += tok
|
||||
}
|
||||
|
||||
if len(current) > 0 {
|
||||
sections = append(sections, Section{
|
||||
Index: len(sections),
|
||||
StartSegmentID: current[0].ID,
|
||||
EndSegmentID: current[len(current)-1].ID,
|
||||
EstimatedTokens: currentTokens,
|
||||
Segments: append([]schema.Segment(nil), current...),
|
||||
})
|
||||
}
|
||||
|
||||
return sections
|
||||
}
|
||||
|
||||
func TestChunkEmptyTranscript(t *testing.T) {
|
||||
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 10})
|
||||
|
||||
sections, err := chunker.ChunkTranscript(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript(nil): %v", err)
|
||||
}
|
||||
if len(sections) != 0 {
|
||||
t.Fatalf("expected 0 sections for nil transcript, got %d", len(sections))
|
||||
}
|
||||
|
||||
sections, err = chunker.ChunkTranscript(makeTranscript(nil))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript(empty): %v", err)
|
||||
}
|
||||
if len(sections) != 0 {
|
||||
t.Fatalf("expected 0 sections for empty transcript, got %d", len(sections))
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkSingleSegment(t *testing.T) {
|
||||
segments := makeSegments([]string{"s1"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 10},
|
||||
&mapTokenEstimator{byText: map[string]int{"s1": 7}},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("expected 1 section, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 7 {
|
||||
t.Fatalf("expected estimated_tokens=7, got %d", sections[0].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertSectionMetadataConsistent(t, sections)
|
||||
}
|
||||
|
||||
func TestChunkSingleOversizedSegment(t *testing.T) {
|
||||
segments := makeSegments([]string{"big"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 5},
|
||||
&mapTokenEstimator{byText: map[string]int{"big": 120}},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("expected 1 section, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 120 {
|
||||
t.Fatalf("expected oversized singleton section, got %d", sections[0].EstimatedTokens)
|
||||
}
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 50)
|
||||
}
|
||||
|
||||
func TestChunkTotalBelowMaxSingleSection(t *testing.T) {
|
||||
segments := makeSegments([]string{"a", "b", "c"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 5},
|
||||
&ConstTokenEstimator{Tokens: 10},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 1 {
|
||||
t.Fatalf("expected 1 section, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 30 {
|
||||
t.Fatalf("expected 30 section tokens, got %d", sections[0].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
}
|
||||
|
||||
func TestChunkTotalExactlyDivisibleByMax(t *testing.T) {
|
||||
segments := makeSegments([]string{"a", "b", "c", "d"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1},
|
||||
&ConstTokenEstimator{Tokens: 5},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 2 {
|
||||
t.Fatalf("expected 2 sections, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 10 || sections[1].EstimatedTokens != 10 {
|
||||
t.Fatalf("expected [10,10] tokens, got [%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 10)
|
||||
}
|
||||
|
||||
func TestChunkTotalNotDivisibleByMax(t *testing.T) {
|
||||
segments := makeSegments([]string{"a", "b", "c", "d", "e"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1},
|
||||
&ConstTokenEstimator{Tokens: 5},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 3 {
|
||||
t.Fatalf("expected 3 sections, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 10 || sections[1].EstimatedTokens != 10 || sections[2].EstimatedTokens != 5 {
|
||||
t.Fatalf("expected [10,10,5] tokens, got [%d,%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens, sections[2].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 10)
|
||||
}
|
||||
|
||||
func TestChunkTargetSectionsPrecedenceAndSuccess(t *testing.T) {
|
||||
segments := makeSegments([]string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10"})
|
||||
target := 3
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 200, MinSectionTokens: 1, TargetSections: &target},
|
||||
&ConstTokenEstimator{Tokens: 10},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != target {
|
||||
t.Fatalf("expected %d sections from explicit target, got %d", target, len(sections))
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertSectionMetadataConsistent(t, sections)
|
||||
}
|
||||
|
||||
func TestChunkTargetSectionsImpossibleTooMany(t *testing.T) {
|
||||
segments := makeSegments([]string{"1", "2", "3", "4", "5"})
|
||||
target := 10
|
||||
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 1, TargetSections: &target})
|
||||
|
||||
_, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err == nil {
|
||||
t.Fatal("expected error for impossible target_sections")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "cannot have more sections than segments") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkTargetSectionsImpossibleTooFew(t *testing.T) {
|
||||
segments := makeSegments([]string{"1", "2", "3"})
|
||||
target := 1
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1, TargetSections: &target},
|
||||
&ConstTokenEstimator{Tokens: 30},
|
||||
)
|
||||
|
||||
_, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err == nil {
|
||||
t.Fatal("expected error for impossible target_sections")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "need at least 3 sections") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkManySmallSegmentsBalanced(t *testing.T) {
|
||||
texts := []string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"}
|
||||
segments := makeSegments(texts)
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1},
|
||||
&ConstTokenEstimator{Tokens: 10},
|
||||
)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 3 {
|
||||
t.Fatalf("expected 3 sections, got %d", len(sections))
|
||||
}
|
||||
if sections[0].EstimatedTokens != 40 || sections[1].EstimatedTokens != 40 || sections[2].EstimatedTokens != 30 {
|
||||
t.Fatalf("expected [40,40,30], got [%d,%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens, sections[2].EstimatedTokens)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 50)
|
||||
}
|
||||
|
||||
func TestChunkMixedLargeAndSmallSegments(t *testing.T) {
|
||||
segments := makeSegments([]string{"big1", "s1", "s2", "s3", "big2", "s4"})
|
||||
estimator := &mapTokenEstimator{byText: map[string]int{
|
||||
"big1": 120,
|
||||
"s1": 10,
|
||||
"s2": 10,
|
||||
"s3": 10,
|
||||
"big2": 120,
|
||||
"s4": 10,
|
||||
}}
|
||||
chunker := NewChunkerWithEstimator(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 1}, estimator)
|
||||
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
if len(sections) != 4 {
|
||||
t.Fatalf("expected 4 sections, got %d", len(sections))
|
||||
}
|
||||
if len(sections[0].Segments) != 1 || sections[0].Segments[0].Text != "big1" {
|
||||
t.Fatalf("expected first oversized segment in singleton section, got %+v", sections[0].Segments)
|
||||
}
|
||||
if len(sections[2].Segments) != 1 || sections[2].Segments[0].Text != "big2" {
|
||||
t.Fatalf("expected second oversized segment in singleton section, got %+v", sections[2].Segments)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertMaxBoundExceptSingletonOversized(t, sections, 100)
|
||||
}
|
||||
|
||||
func TestChunkDeterministicOrdering(t *testing.T) {
|
||||
segments := makeSegments([]string{"a", "b", "c", "d", "e", "f"})
|
||||
chunkerCfg := ChunkingConfig{MaxSectionTokens: 15, MinSectionTokens: 1}
|
||||
estimator := &ConstTokenEstimator{Tokens: 5}
|
||||
|
||||
var first []Section
|
||||
for i := 0; i < 5; i++ {
|
||||
chunker := NewChunkerWithEstimator(chunkerCfg, estimator)
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("iteration %d: %v", i, err)
|
||||
}
|
||||
if i == 0 {
|
||||
first = sections
|
||||
continue
|
||||
}
|
||||
if len(sections) != len(first) {
|
||||
t.Fatalf("iteration %d: section count mismatch (%d vs %d)", i, len(sections), len(first))
|
||||
}
|
||||
for j := range sections {
|
||||
if sections[j].Index != first[j].Index ||
|
||||
sections[j].StartSegmentID != first[j].StartSegmentID ||
|
||||
sections[j].EndSegmentID != first[j].EndSegmentID ||
|
||||
sections[j].EstimatedTokens != first[j].EstimatedTokens ||
|
||||
len(sections[j].Segments) != len(first[j].Segments) {
|
||||
t.Fatalf("iteration %d section %d mismatch", i, j)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkNoMutationOfInput(t *testing.T) {
|
||||
segments := makeSegments([]string{"original one", "original two"})
|
||||
transcript := makeTranscript(segments)
|
||||
original := make([]string, len(transcript.Segments))
|
||||
for i := range transcript.Segments {
|
||||
original[i] = transcript.Segments[i].Text
|
||||
}
|
||||
|
||||
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1})
|
||||
if _, err := chunker.ChunkTranscript(transcript); err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
for i := range transcript.Segments {
|
||||
if transcript.Segments[i].Text != original[i] {
|
||||
t.Fatalf("segment %d mutated", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkConfigValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config ChunkingConfig
|
||||
errContains string
|
||||
}{
|
||||
{
|
||||
name: "zero max tokens",
|
||||
config: ChunkingConfig{MaxSectionTokens: 0, MinSectionTokens: 1},
|
||||
errContains: "max_section_tokens must be positive",
|
||||
},
|
||||
{
|
||||
name: "negative max tokens",
|
||||
config: ChunkingConfig{MaxSectionTokens: -1, MinSectionTokens: 1},
|
||||
errContains: "max_section_tokens must be positive",
|
||||
},
|
||||
{
|
||||
name: "negative min tokens",
|
||||
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: -1},
|
||||
errContains: "min_section_tokens must be non-negative",
|
||||
},
|
||||
{
|
||||
name: "min exceeds max",
|
||||
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 11},
|
||||
errContains: "min_section_tokens (11) cannot exceed max_section_tokens (10)",
|
||||
},
|
||||
{
|
||||
name: "zero target sections",
|
||||
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1, TargetSections: intPtr(0)},
|
||||
errContains: "target_sections must be positive",
|
||||
},
|
||||
{
|
||||
name: "negative target sections",
|
||||
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1, TargetSections: intPtr(-1)},
|
||||
errContains: "target_sections must be positive",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
chunker := NewChunker(tt.config)
|
||||
_, err := chunker.ChunkTranscript(makeTranscript(makeSegments([]string{"x"})))
|
||||
if err == nil {
|
||||
t.Fatalf("expected error containing %q", tt.errContains)
|
||||
}
|
||||
if !strings.Contains(err.Error(), tt.errContains) {
|
||||
t.Fatalf("expected error containing %q, got %q", tt.errContains, err.Error())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkBalancedAlgorithmBeatsGreedyMaxFillOnUnevenTranscript(t *testing.T) {
|
||||
segments := makeSegments([]string{"s1", "s2", "s3", "s4", "s5", "s6"})
|
||||
tokenMap := map[string]int{
|
||||
"s1": 50,
|
||||
"s2": 10,
|
||||
"s3": 10,
|
||||
"s4": 10,
|
||||
"s5": 10,
|
||||
"s6": 10,
|
||||
}
|
||||
estimator := &mapTokenEstimator{byText: tokenMap}
|
||||
chunker := NewChunkerWithEstimator(ChunkingConfig{MaxSectionTokens: 80, MinSectionTokens: 1}, estimator)
|
||||
|
||||
balancedSections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil {
|
||||
t.Fatalf("ChunkTranscript: %v", err)
|
||||
}
|
||||
|
||||
tokens := make([]int, 0, len(segments))
|
||||
for _, seg := range segments {
|
||||
tokens = append(tokens, tokenMap[seg.Text])
|
||||
}
|
||||
greedySections := greedyMaxFillSections(segments, tokens, 80)
|
||||
|
||||
balancedImbalance := imbalance(balancedSections)
|
||||
greedyImbalance := imbalance(greedySections)
|
||||
if balancedImbalance >= greedyImbalance {
|
||||
t.Fatalf(
|
||||
"expected balanced chunking to improve over greedy max-fill; balanced=%d greedy=%d",
|
||||
balancedImbalance,
|
||||
greedyImbalance,
|
||||
)
|
||||
}
|
||||
|
||||
assertSegmentCoverageAndOrder(t, segments, balancedSections)
|
||||
assertMaxBoundExceptSingletonOversized(t, balancedSections, 80)
|
||||
}
|
||||
84
internal/core/chunking/summary.go
Normal file
84
internal/core/chunking/summary.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package chunking
|
||||
|
||||
// Summary provides a concise overview of chunking results for reports
|
||||
type Summary struct {
|
||||
ChunkCount int `json:"chunk_count"`
|
||||
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
|
||||
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
|
||||
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
|
||||
TargetSections *int `json:"target_sections,omitempty"`
|
||||
MaxSectionTokens int `json:"max_section_tokens"`
|
||||
MinSectionTokens int `json:"min_section_tokens"`
|
||||
}
|
||||
|
||||
// ChunkSummary represents a single chunk's metadata for diagnostics
|
||||
type ChunkSummary struct {
|
||||
Index int `json:"index"`
|
||||
StartSegmentID int `json:"start_segment_id"`
|
||||
EndSegmentID int `json:"end_segment_id"`
|
||||
EstimatedTokens int `json:"estimated_tokens"`
|
||||
SegmentCount int `json:"segment_count"`
|
||||
}
|
||||
|
||||
// DetailedSummary provides per-chunk details for diagnostics
|
||||
type DetailedSummary struct {
|
||||
Summary `json:",inline"`
|
||||
Chunks []ChunkSummary `json:"chunks"`
|
||||
}
|
||||
|
||||
// ComputeSummary creates a Summary from sections and config
|
||||
func ComputeSummary(sections []Section, config ChunkingConfig) Summary {
|
||||
if len(sections) == 0 {
|
||||
return Summary{
|
||||
ChunkCount: 0,
|
||||
MaxSectionTokens: config.MaxSectionTokens,
|
||||
MinSectionTokens: config.MinSectionTokens,
|
||||
TargetSections: config.TargetSections,
|
||||
}
|
||||
}
|
||||
|
||||
minTokens := sections[0].EstimatedTokens
|
||||
maxTokens := sections[0].EstimatedTokens
|
||||
totalTokens := 0
|
||||
|
||||
for _, sec := range sections {
|
||||
if sec.EstimatedTokens < minTokens {
|
||||
minTokens = sec.EstimatedTokens
|
||||
}
|
||||
if sec.EstimatedTokens > maxTokens {
|
||||
maxTokens = sec.EstimatedTokens
|
||||
}
|
||||
totalTokens += sec.EstimatedTokens
|
||||
}
|
||||
|
||||
return Summary{
|
||||
ChunkCount: len(sections),
|
||||
MinEstimatedTokens: minTokens,
|
||||
MaxEstimatedTokens: maxTokens,
|
||||
TotalEstimatedTokens: totalTokens,
|
||||
TargetSections: config.TargetSections,
|
||||
MaxSectionTokens: config.MaxSectionTokens,
|
||||
MinSectionTokens: config.MinSectionTokens,
|
||||
}
|
||||
}
|
||||
|
||||
// ComputeDetailedSummary creates a DetailedSummary from sections and config
|
||||
func ComputeDetailedSummary(sections []Section, config ChunkingConfig) DetailedSummary {
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
chunks := make([]ChunkSummary, len(sections))
|
||||
for i, sec := range sections {
|
||||
chunks[i] = ChunkSummary{
|
||||
Index: sec.Index,
|
||||
StartSegmentID: sec.StartSegmentID,
|
||||
EndSegmentID: sec.EndSegmentID,
|
||||
EstimatedTokens: sec.EstimatedTokens,
|
||||
SegmentCount: len(sec.Segments),
|
||||
}
|
||||
}
|
||||
|
||||
return DetailedSummary{
|
||||
Summary: summary,
|
||||
Chunks: chunks,
|
||||
}
|
||||
}
|
||||
148
internal/core/chunking/summary_test.go
Normal file
148
internal/core/chunking/summary_test.go
Normal file
@@ -0,0 +1,148 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
func TestComputeSummary(t *testing.T) {
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
}
|
||||
|
||||
sections := []Section{
|
||||
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
|
||||
{Index: 1, EstimatedTokens: 50, StartSegmentID: 3, EndSegmentID: 4},
|
||||
{Index: 2, EstimatedTokens: 20, StartSegmentID: 5, EndSegmentID: 5},
|
||||
}
|
||||
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
if summary.ChunkCount != 3 {
|
||||
t.Errorf("expected chunk_count=3, got %d", summary.ChunkCount)
|
||||
}
|
||||
if summary.MinEstimatedTokens != 20 {
|
||||
t.Errorf("expected min_estimated_tokens=20, got %d", summary.MinEstimatedTokens)
|
||||
}
|
||||
if summary.MaxEstimatedTokens != 50 {
|
||||
t.Errorf("expected max_estimated_tokens=50, got %d", summary.MaxEstimatedTokens)
|
||||
}
|
||||
if summary.TotalEstimatedTokens != 100 {
|
||||
t.Errorf("expected total_estimated_tokens=100, got %d", summary.TotalEstimatedTokens)
|
||||
}
|
||||
if summary.MaxSectionTokens != 100 {
|
||||
t.Errorf("expected max_section_tokens=100, got %d", summary.MaxSectionTokens)
|
||||
}
|
||||
if summary.MinSectionTokens != 10 {
|
||||
t.Errorf("expected min_section_tokens=10, got %d", summary.MinSectionTokens)
|
||||
}
|
||||
if summary.TargetSections != nil {
|
||||
t.Errorf("expected target_sections=nil, got %v", summary.TargetSections)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeSummaryWithTarget(t *testing.T) {
|
||||
target := 5
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
TargetSections: &target,
|
||||
}
|
||||
|
||||
sections := []Section{
|
||||
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
|
||||
}
|
||||
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
if summary.TargetSections == nil || *summary.TargetSections != 5 {
|
||||
t.Errorf("expected target_sections=5, got %v", summary.TargetSections)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeSummaryEmptySections(t *testing.T) {
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
}
|
||||
|
||||
sections := []Section{}
|
||||
|
||||
summary := ComputeSummary(sections, config)
|
||||
|
||||
if summary.ChunkCount != 0 {
|
||||
t.Errorf("expected chunk_count=0, got %d", summary.ChunkCount)
|
||||
}
|
||||
if summary.MinEstimatedTokens != 0 {
|
||||
t.Errorf("expected min_estimated_tokens=0 for empty, got %d", summary.MinEstimatedTokens)
|
||||
}
|
||||
if summary.MaxSectionTokens != 100 {
|
||||
t.Errorf("expected max_section_tokens preserved, got %d", summary.MaxSectionTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeDetailedSummary(t *testing.T) {
|
||||
config := ChunkingConfig{
|
||||
MaxSectionTokens: 100,
|
||||
MinSectionTokens: 10,
|
||||
}
|
||||
|
||||
sections := []Section{
|
||||
{
|
||||
Index: 0,
|
||||
EstimatedTokens: 30,
|
||||
StartSegmentID: 1,
|
||||
EndSegmentID: 2,
|
||||
Segments: make([]schema.Segment, 2), // 2 segments
|
||||
},
|
||||
{
|
||||
Index: 1,
|
||||
EstimatedTokens: 50,
|
||||
StartSegmentID: 3,
|
||||
EndSegmentID: 5,
|
||||
Segments: make([]schema.Segment, 3), // 3 segments
|
||||
},
|
||||
}
|
||||
|
||||
detailed := ComputeDetailedSummary(sections, config)
|
||||
|
||||
if detailed.ChunkCount != 2 {
|
||||
t.Errorf("expected chunk_count=2, got %d", detailed.ChunkCount)
|
||||
}
|
||||
if len(detailed.Chunks) != 2 {
|
||||
t.Fatalf("expected 2 chunk entries, got %d", len(detailed.Chunks))
|
||||
}
|
||||
|
||||
// Check first chunk
|
||||
if detailed.Chunks[0].Index != 0 {
|
||||
t.Errorf("expected chunk[0].index=0, got %d", detailed.Chunks[0].Index)
|
||||
}
|
||||
if detailed.Chunks[0].StartSegmentID != 1 {
|
||||
t.Errorf("expected chunk[0].start_segment_id=1, got %d", detailed.Chunks[0].StartSegmentID)
|
||||
}
|
||||
if detailed.Chunks[0].EndSegmentID != 2 {
|
||||
t.Errorf("expected chunk[0].end_segment_id=2, got %d", detailed.Chunks[0].EndSegmentID)
|
||||
}
|
||||
if detailed.Chunks[0].EstimatedTokens != 30 {
|
||||
t.Errorf("expected chunk[0].estimated_tokens=30, got %d", detailed.Chunks[0].EstimatedTokens)
|
||||
}
|
||||
if detailed.Chunks[0].SegmentCount != 2 {
|
||||
t.Errorf("expected chunk[0].segment_count=2, got %d", detailed.Chunks[0].SegmentCount)
|
||||
}
|
||||
|
||||
// Check second chunk
|
||||
if detailed.Chunks[1].Index != 1 {
|
||||
t.Errorf("expected chunk[1].index=1, got %d", detailed.Chunks[1].Index)
|
||||
}
|
||||
if detailed.Chunks[1].StartSegmentID != 3 {
|
||||
t.Errorf("expected chunk[1].start_segment_id=3, got %d", detailed.Chunks[1].StartSegmentID)
|
||||
}
|
||||
if detailed.Chunks[1].EndSegmentID != 5 {
|
||||
t.Errorf("expected chunk[1].end_segment_id=5, got %d", detailed.Chunks[1].EndSegmentID)
|
||||
}
|
||||
if detailed.Chunks[1].SegmentCount != 3 {
|
||||
t.Errorf("expected chunk[1].segment_count=3, got %d", detailed.Chunks[1].SegmentCount)
|
||||
}
|
||||
}
|
||||
55
internal/core/chunking/tokens.go
Normal file
55
internal/core/chunking/tokens.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// TokenEstimator provides a deterministic token estimation suitable for prompt budgeting.
|
||||
// The estimator is approximate but stable, isolated, and replaceable.
|
||||
type TokenEstimator interface {
|
||||
EstimateTokens(text string) int
|
||||
}
|
||||
|
||||
// SimpleTokenEstimator provides a basic deterministic token estimation.
|
||||
// This uses a simple heuristic based on word count and punctuation.
|
||||
type SimpleTokenEstimator struct{}
|
||||
|
||||
// NewSimpleTokenEstimator creates a new simple token estimator.
|
||||
func NewSimpleTokenEstimator() *SimpleTokenEstimator {
|
||||
return &SimpleTokenEstimator{}
|
||||
}
|
||||
|
||||
// EstimateTokens provides a rough estimate of the number of tokens in the given text.
|
||||
// This implementation uses a simple heuristic: count words and punctuation as tokens.
|
||||
// The estimate is deterministic and stable for the same input text.
|
||||
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
|
||||
if text == "" {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Simple heuristic: split on whitespace and count non-empty segments
|
||||
words := strings.Fields(text)
|
||||
tokenCount := len(words)
|
||||
|
||||
// Add some estimate for punctuation that might be separate tokens
|
||||
punctuationCount := 0
|
||||
for _, r := range text {
|
||||
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
|
||||
punctuationCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Rough estimate: each word is a token, plus half the punctuation as separate tokens
|
||||
return tokenCount + (punctuationCount / 2)
|
||||
}
|
||||
|
||||
// ConstTokenEstimator returns a constant token count for testing purposes.
|
||||
type ConstTokenEstimator struct {
|
||||
Tokens int
|
||||
}
|
||||
|
||||
// EstimateTokens returns the configured constant token count.
|
||||
func (e *ConstTokenEstimator) EstimateTokens(text string) int {
|
||||
return e.Tokens
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user