Compare commits
19 Commits
f790c1441c
...
v1.1.0
| Author | SHA1 | Date | |
|---|---|---|---|
| c5627460d0 | |||
| 3c68ec9d05 | |||
| e8b8fbd509 | |||
| 588ca36dfc | |||
| 018d08fd4c | |||
| 3a0a0b3940 | |||
| 99ab2f181b | |||
| 4e4801dc98 | |||
| 900ad74958 | |||
| 28d5201a69 | |||
| e5944f9875 | |||
| 6344fc91ba | |||
| 7f3a91cc9e | |||
| 72fb021453 | |||
| 40e8b54d3b | |||
| 76651333b1 | |||
| 0b01c3a83d | |||
| 0630d36734 | |||
| 52c2697040 |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -12,7 +12,6 @@ build/
|
|||||||
|
|
||||||
# ---> Codex
|
# ---> Codex
|
||||||
.codex
|
.codex
|
||||||
AGENTS.md
|
|
||||||
|
|
||||||
# ---> Go
|
# ---> Go
|
||||||
# If you prefer the allow list template instead of the deny list, see community template:
|
# If you prefer the allow list template instead of the deny list, see community template:
|
||||||
|
|||||||
@@ -5,33 +5,7 @@ steps:
|
|||||||
- name: build-release-assets
|
- name: build-release-assets
|
||||||
image: golang:1.25
|
image: golang:1.25
|
||||||
commands:
|
commands:
|
||||||
- |
|
- bash scripts/build-release.sh "$CI_COMMIT_TAG" dist
|
||||||
set -eu
|
|
||||||
|
|
||||||
version="$CI_COMMIT_TAG"
|
|
||||||
dist="dist"
|
|
||||||
pkg="gitea.maximumdirect.net/eric/audita/cmd/audita"
|
|
||||||
|
|
||||||
rm -rf "$dist"
|
|
||||||
mkdir -p "$dist"
|
|
||||||
|
|
||||||
build_binary() {
|
|
||||||
goos="$1"
|
|
||||||
goarch="$2"
|
|
||||||
suffix="$3"
|
|
||||||
output="$dist/audita-$version-$goos-$goarch$suffix"
|
|
||||||
|
|
||||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \
|
|
||||||
go build -trimpath -ldflags "-s -w -X gitea.maximumdirect.net/eric/audita/internal/buildinfo.Version=$version" \
|
|
||||||
-o "$output" "$pkg"
|
|
||||||
}
|
|
||||||
|
|
||||||
build_binary linux amd64 ""
|
|
||||||
build_binary linux arm64 ""
|
|
||||||
build_binary darwin amd64 ""
|
|
||||||
build_binary darwin arm64 ""
|
|
||||||
build_binary windows amd64 ".exe"
|
|
||||||
build_binary windows arm64 ".exe"
|
|
||||||
|
|
||||||
- name: publish-release
|
- name: publish-release
|
||||||
image: woodpeckerci/plugin-release
|
image: woodpeckerci/plugin-release
|
||||||
@@ -41,7 +15,8 @@ steps:
|
|||||||
api_key:
|
api_key:
|
||||||
from_secret: GITEA_RELEASE_TOKEN
|
from_secret: GITEA_RELEASE_TOKEN
|
||||||
files:
|
files:
|
||||||
- dist/audita-*
|
- dist/audita-*-linux-amd64
|
||||||
|
- dist/audita-*-linux-arm64
|
||||||
checksum: sha256
|
checksum: sha256
|
||||||
checksum-file: SHA256SUMS
|
checksum-file: SHA256SUMS
|
||||||
checksum-flatten: true
|
checksum-flatten: true
|
||||||
|
|||||||
4
AGENTS.md
Normal file
4
AGENTS.md
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
Please review `docs/development.md` for initial orientation in this repository
|
||||||
|
and follow its task-specific reading guide.
|
||||||
|
|
||||||
|
Review and follow `docs/policy/agents.md` before recommending, planning, or implementing any changes to this repository.
|
||||||
2
LICENSE
2
LICENSE
@@ -1,4 +1,4 @@
|
|||||||
Copyright (c) 2026 eric.
|
Copyright (c) 2026 Eric Rakestraw.
|
||||||
|
|
||||||
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
||||||
|
|
||||||
|
|||||||
315
README.md
315
README.md
@@ -1,305 +1,44 @@
|
|||||||
# Audita
|
# Audita
|
||||||
|
|
||||||
Audita is a transcript polishing CLI.
|
Audita polishes transcript JSON using glossary-aware and LLM-backed correction
|
||||||
|
modules. It validates proposed corrections and produces corrected text plus
|
||||||
|
reports for review and subprocess integration.
|
||||||
|
|
||||||
`audita process` validates transcript/glossary input, normalizes and chunks transcript segments, runs the default correction pipeline, and emits corrected transcript output plus machine-readable diagnostics and reports.
|
## Quickstart
|
||||||
|
|
||||||
## What Audita Does
|
From a checkout, build with Go 1.24 or newer:
|
||||||
|
|
||||||
Default module sequence:
|
|
||||||
- `glossary`
|
|
||||||
- `homophones`
|
|
||||||
- `glossary`
|
|
||||||
- `spoken_word`
|
|
||||||
- `grammar`
|
|
||||||
|
|
||||||
Pipeline behavior includes:
|
|
||||||
- glossary-backed domain/acoustic corrections
|
|
||||||
- conservative homophone and mistranscription corrections
|
|
||||||
- conservative spoken-word dysfluency cleanup with semantic guardrails
|
|
||||||
- grammar/punctuation/capitalization/formatting cleanup
|
|
||||||
- validator-chain enforcement before application
|
|
||||||
- malformed module-stage LLM payloads degrade to warnings/rejections instead of failing the run
|
|
||||||
- run reports and diagnostics artifacts with secret redaction
|
|
||||||
|
|
||||||
## Build and Install
|
|
||||||
|
|
||||||
Build a local binary:
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
go build -o ./bin/audita ./cmd/audita
|
go build -o ./bin/audita ./cmd/audita
|
||||||
```
|
```
|
||||||
|
|
||||||
Install into your Go bin directory:
|
Processing requires a reachable [compatible LLM endpoint](docs/integrations/openai-compatible-llm.md)
|
||||||
|
and access to its configured model. Supply its credentials through the
|
||||||
|
[configuration mechanisms](docs/config.md#secrets); the
|
||||||
|
[minimal config](examples/minimal-config.yml) reads `AUDITA_LLM_API_KEY` and uses
|
||||||
|
the [default model and endpoint](docs/config.md#llm-settings). Set that variable
|
||||||
|
in your environment before using an endpoint that requires authentication.
|
||||||
|
For another endpoint or model, edit a copy of the config.
|
||||||
|
|
||||||
|
Run the included small input with the minimal config:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
go install ./cmd/audita
|
./bin/audita process examples/tiny-transcript.json \
|
||||||
```
|
--config examples/minimal-config.yml \
|
||||||
|
--glossary examples/tiny-glossary.yaml \
|
||||||
CLI help:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
audita --help
|
|
||||||
audita process --help
|
|
||||||
audita config --help
|
|
||||||
```
|
|
||||||
|
|
||||||
## Test
|
|
||||||
|
|
||||||
Run all tests:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./...
|
|
||||||
```
|
|
||||||
|
|
||||||
## Basic Usage
|
|
||||||
|
|
||||||
Required inputs:
|
|
||||||
- transcript JSON path (positional argument)
|
|
||||||
- `--glossary <glossary.yaml>`
|
|
||||||
|
|
||||||
Recommended run:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
audita process transcript.json \
|
|
||||||
--glossary glossary.yaml \
|
|
||||||
--output corrected.json \
|
|
||||||
--report-json report.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Select an explicit output schema (default is `bare-segments`):
|
|
||||||
|
|
||||||
```sh
|
|
||||||
audita process transcript.json \
|
|
||||||
--glossary glossary.yaml \
|
|
||||||
--output-schema audita-v1 \
|
|
||||||
--output corrected.json \
|
|
||||||
--report-json report.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Recommended config-based run:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
audita process transcript.json \
|
|
||||||
--glossary glossary.yaml \
|
|
||||||
--config audita.yml \
|
|
||||||
--output corrected.json \
|
|
||||||
--report-json report.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Explicit module override:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
audita process transcript.json \
|
|
||||||
--glossary glossary.yaml \
|
|
||||||
--modules glossary,homophones,grammar \
|
|
||||||
--output corrected.json \
|
|
||||||
--report-json report.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Optional transcript background context:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
audita process transcript.json \
|
|
||||||
--glossary glossary.yaml \
|
|
||||||
--transcript-description "Brief context that may help resolve ambiguous terms." \
|
|
||||||
--output corrected.json
|
--output corrected.json
|
||||||
```
|
```
|
||||||
|
|
||||||
The transcript description is background context only and does not override transcript content.
|
This makes LLM requests. Inherited configuration environment overrides still
|
||||||
|
apply; see [precedence](docs/config.md#effective-precedence).
|
||||||
Write transcript JSON to stdout (no `--output`):
|
|
||||||
|
|
||||||
```sh
|
|
||||||
audita process transcript.json --glossary glossary.yaml
|
|
||||||
```
|
|
||||||
|
|
||||||
Control diagnostics location/retention:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
audita process transcript.json \
|
|
||||||
--glossary glossary.yaml \
|
|
||||||
--work-dir /tmp/audita \
|
|
||||||
--work-dir-retention auto \
|
|
||||||
--output corrected.json \
|
|
||||||
--report-json report.json
|
|
||||||
```
|
|
||||||
|
|
||||||
## Stdout/Stderr Contract
|
|
||||||
|
|
||||||
- With `--output`, stdout is expected to be empty on success.
|
|
||||||
- Without `--output`, stdout contains transcript JSON only on success.
|
|
||||||
- `--report-json` writes a file and is never printed to stdout.
|
|
||||||
- stderr is human-readable diagnostics/errors.
|
|
||||||
- successful runs remain quiet on stderr even when module warnings are recorded in report/diagnostics artifacts.
|
|
||||||
|
|
||||||
For subprocess orchestration guidance, see [`docs/subprocess-operations.md`](docs/subprocess-operations.md).
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
Precedence:
|
|
||||||
1. defaults
|
|
||||||
2. config file (`--config`, `AUDITA_CONFIG`, or default search paths when present: `/usr/local/etc/audita/config.yml`, then `/etc/audita/config.yml`)
|
|
||||||
3. environment (`AUDITA_*`)
|
|
||||||
4. CLI flags
|
|
||||||
|
|
||||||
Config commands:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
audita config validate --config audita.yml
|
|
||||||
audita config print-effective --config audita.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
For full config-file schema and examples, see [`docs/configuration.md`](docs/configuration.md).
|
|
||||||
For output-schema details, see [`docs/architecture/output-schemas.md`](docs/architecture/output-schemas.md).
|
|
||||||
For built-in validator keys and chain definitions, see [`docs/architecture/validators.md`](docs/architecture/validators.md).
|
|
||||||
For embedded prompt assets and prompt metadata behavior, see [`docs/architecture/prompts.md`](docs/architecture/prompts.md).
|
|
||||||
For CLI/process compatibility guarantees, see [`docs/architecture/public-contract.md`](docs/architecture/public-contract.md).
|
|
||||||
|
|
||||||
### Modules
|
|
||||||
|
|
||||||
- `AUDITA_MODULES` (CSV)
|
|
||||||
- CLI: `--modules`
|
|
||||||
|
|
||||||
### Transcript Description
|
|
||||||
|
|
||||||
CLI:
|
|
||||||
- `--transcript-description`
|
|
||||||
|
|
||||||
Behavior:
|
|
||||||
- optional background context for proposal and LLM-validator prompts;
|
|
||||||
- trimmed and length-limited by CLI validation;
|
|
||||||
- does not override transcript content;
|
|
||||||
- no `AUDITA_*` environment variable is currently defined for this setting.
|
|
||||||
|
|
||||||
### Primary LLM
|
|
||||||
|
|
||||||
Environment:
|
|
||||||
- `AUDITA_LLM_API_KEY` (or `OPENROUTER_API_KEY` fallback)
|
|
||||||
- `AUDITA_MODEL`
|
|
||||||
- `AUDITA_BASE_URL`
|
|
||||||
- `AUDITA_LLM_TIMEOUT_SECONDS`
|
|
||||||
- `AUDITA_MAX_RETRIES`
|
|
||||||
|
|
||||||
CLI:
|
|
||||||
- `--llm-api-key`
|
|
||||||
- `--model`
|
|
||||||
- `--base-url`
|
|
||||||
- `--llm-timeout-seconds`
|
|
||||||
- `--max-retries`
|
|
||||||
|
|
||||||
### Validation LLM
|
|
||||||
|
|
||||||
Environment:
|
|
||||||
- `AUDITA_VALIDATION_LLM_API_KEY`
|
|
||||||
- `AUDITA_VALIDATION_MODEL`
|
|
||||||
- `AUDITA_VALIDATION_BASE_URL`
|
|
||||||
- `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS`
|
|
||||||
- `AUDITA_VALIDATION_MAX_RETRIES`
|
|
||||||
- `AUDITA_VALIDATION_LLM_CONCURRENCY`
|
|
||||||
- `AUDITA_VALIDATION_MAX_PROMPT_TOKENS`
|
|
||||||
|
|
||||||
CLI:
|
|
||||||
- `--validation-llm-api-key`
|
|
||||||
- `--validation-model`
|
|
||||||
- `--validation-base-url`
|
|
||||||
- `--validation-llm-timeout-seconds`
|
|
||||||
- `--validation-max-retries`
|
|
||||||
- `--validation-llm-concurrency`
|
|
||||||
- `--validation-max-prompt-tokens`
|
|
||||||
|
|
||||||
### LLM Concurrency
|
|
||||||
|
|
||||||
Environment:
|
|
||||||
- `AUDITA_TOTAL_LLM_CONCURRENCY`
|
|
||||||
- `AUDITA_PROPOSAL_LLM_CONCURRENCY`
|
|
||||||
- `AUDITA_VALIDATION_LLM_CONCURRENCY`
|
|
||||||
- `AUDITA_LLM_CONCURRENCY` (legacy alias for `AUDITA_TOTAL_LLM_CONCURRENCY`)
|
|
||||||
|
|
||||||
CLI:
|
|
||||||
- `--total-llm-concurrency`
|
|
||||||
- `--proposal-llm-concurrency`
|
|
||||||
- `--validation-llm-concurrency`
|
|
||||||
- `--llm-concurrency` (legacy alias for `--total-llm-concurrency`)
|
|
||||||
|
|
||||||
Behavior:
|
|
||||||
- all proposal and validation LLM calls are bounded by total LLM concurrency
|
|
||||||
- proposal LLM calls are additionally bounded by proposal LLM concurrency
|
|
||||||
- when validation concurrency is unset, it inherits total LLM concurrency
|
|
||||||
- when explicitly set, proposal and validation concurrency must each be `<= total-llm-concurrency`
|
|
||||||
- canonical total settings win when both canonical and legacy alias settings are provided at the same precedence layer
|
|
||||||
|
|
||||||
### Confidence Thresholds
|
|
||||||
|
|
||||||
Environment:
|
|
||||||
- `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD`
|
|
||||||
- `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD`
|
|
||||||
- `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD`
|
|
||||||
- `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD`
|
|
||||||
|
|
||||||
CLI:
|
|
||||||
- `--glossary-confidence-threshold`
|
|
||||||
- `--homophones-confidence-threshold`
|
|
||||||
- `--spoken-word-confidence-threshold`
|
|
||||||
- `--grammar-confidence-threshold`
|
|
||||||
|
|
||||||
### Normalization and Chunking
|
|
||||||
|
|
||||||
Environment:
|
|
||||||
- `AUDITA_NORMALIZE_MAX_SEGMENT_GAP`
|
|
||||||
- `AUDITA_NORMALIZE_ELLIPSIS_GAP`
|
|
||||||
- `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION`
|
|
||||||
- `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS`
|
|
||||||
- `AUDITA_MAX_SECTION_TOKENS`
|
|
||||||
- `AUDITA_MIN_SECTION_TOKENS`
|
|
||||||
- `AUDITA_TARGET_SECTIONS`
|
|
||||||
|
|
||||||
CLI:
|
|
||||||
- `--normalize-max-segment-gap`
|
|
||||||
- `--normalize-ellipsis-gap`
|
|
||||||
- `--normalize-max-segment-duration`
|
|
||||||
- `--normalize-max-segment-tokens`
|
|
||||||
- `--max-section-tokens`
|
|
||||||
- `--min-section-tokens`
|
|
||||||
- `--target-sections`
|
|
||||||
|
|
||||||
### Work Directory
|
|
||||||
|
|
||||||
Environment:
|
|
||||||
- `AUDITA_WORK_DIR`
|
|
||||||
- `AUDITA_WORK_DIR_RETENTION` (`auto`, `always`, `never`)
|
|
||||||
|
|
||||||
CLI:
|
|
||||||
- `--work-dir`
|
|
||||||
- `--work-dir-retention`
|
|
||||||
|
|
||||||
Retention behavior:
|
|
||||||
- `always`: keep all run directories
|
|
||||||
- `never`: keep successful run directories
|
|
||||||
- `auto`: keep failed runs and successful runs with skipped/rejected corrections
|
|
||||||
|
|
||||||
## Reports and Diagnostics
|
|
||||||
|
|
||||||
Per-run diagnostics include:
|
|
||||||
- source transcript artifacts
|
|
||||||
- normalized transcript artifact
|
|
||||||
- normalization summary
|
|
||||||
- chunking summary
|
|
||||||
- utilization diagnostics summary
|
|
||||||
- correction ledger
|
|
||||||
- invocation metadata
|
|
||||||
- redacted effective config
|
|
||||||
- module/validator prompt-response diagnostics
|
|
||||||
- `report.json`
|
|
||||||
- `error.log` on failure
|
|
||||||
|
|
||||||
Optional external report output:
|
|
||||||
- `--report-json <path>`
|
|
||||||
|
|
||||||
## Documentation
|
## Documentation
|
||||||
|
|
||||||
- Architecture: [`docs/architecture.md`](docs/architecture.md)
|
- [CLI reference](docs/cli.md)
|
||||||
- Diagnostics: [`docs/diagnostics.md`](docs/diagnostics.md)
|
- [Configuration and maintained examples](docs/config.md)
|
||||||
- Structured LLM adapter: [`docs/structured-llm.md`](docs/structured-llm.md)
|
- [Operations and artifact lifecycle](docs/operations.md)
|
||||||
- Subprocess operations: [`docs/subprocess-operations.md`](docs/subprocess-operations.md)
|
- [Transcript and glossary inputs](docs/integrations/transcript-glossary-files.md)
|
||||||
- Release checklist: [`docs/release-checklist.md`](docs/release-checklist.md)
|
- [Output files and report formats](docs/integrations/output-files.md)
|
||||||
|
- [Subprocess integration](docs/integrations/subprocess.md)
|
||||||
|
- [LLM endpoint contract](docs/integrations/openai-compatible-llm.md)
|
||||||
|
- [Contributor guide](docs/development.md)
|
||||||
|
|||||||
@@ -571,6 +571,15 @@ func runCLISubprocessContext(t *testing.T, ctx context.Context, extraEnv map[str
|
|||||||
cmdArgs := append([]string{"-test.run=TestHelperProcess", "--"}, args...)
|
cmdArgs := append([]string{"-test.run=TestHelperProcess", "--"}, args...)
|
||||||
cmd := exec.CommandContext(ctx, os.Args[0], cmdArgs...)
|
cmd := exec.CommandContext(ctx, os.Args[0], cmdArgs...)
|
||||||
env := append(filterAuditaEnv(os.Environ()), "GO_WANT_HELPER_PROCESS=1")
|
env := append(filterAuditaEnv(os.Environ()), "GO_WANT_HELPER_PROCESS=1")
|
||||||
|
// Use defaults without discovering the host's installed configuration.
|
||||||
|
// Explicit test environment overrides and --config retain their precedence.
|
||||||
|
if _, supplied := extraEnv["AUDITA_CONFIG"]; !supplied {
|
||||||
|
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||||
|
if err := os.WriteFile(configPath, []byte("version: 1\n"), 0600); err != nil {
|
||||||
|
t.Fatalf("write baseline config: %v", err)
|
||||||
|
}
|
||||||
|
env = append(env, "AUDITA_CONFIG="+configPath)
|
||||||
|
}
|
||||||
for k, v := range extraEnv {
|
for k, v := range extraEnv {
|
||||||
env = append(env, fmt.Sprintf("%s=%s", k, v))
|
env = append(env, fmt.Sprintf("%s=%s", k, v))
|
||||||
}
|
}
|
||||||
|
|||||||
7
docs/adr/README.md
Normal file
7
docs/adr/README.md
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
# Architectural Decision Records
|
||||||
|
|
||||||
|
No ADRs have been recorded here. Use the
|
||||||
|
[documentation policy's ADR lifecycle](../policy/documentation.md#architecture-decision-records)
|
||||||
|
for significant decisions. Current architecture belongs in the
|
||||||
|
[architecture policy](../policy/architecture.md); implementation status belongs in
|
||||||
|
[roadmap documents](../roadmap/).
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
# Audita Architecture Index
|
|
||||||
|
|
||||||
This file is the entrypoint for architecture documentation.
|
|
||||||
|
|
||||||
Core architecture overview:
|
|
||||||
- [Architecture Overview](./architecture/architecture.md)
|
|
||||||
|
|
||||||
Focused architecture contracts:
|
|
||||||
- [Public Contract](./architecture/public-contract.md)
|
|
||||||
- [Diagnostics](./architecture/diagnostics.md)
|
|
||||||
- [Structured LLM](./architecture/structured-llm.md)
|
|
||||||
- [Validators](./architecture/validators.md)
|
|
||||||
- [Prompts](./architecture/prompts.md)
|
|
||||||
- [Output Schemas](./architecture/output-schemas.md)
|
|
||||||
@@ -1,153 +0,0 @@
|
|||||||
# Audita Architecture
|
|
||||||
|
|
||||||
## Scope
|
|
||||||
This document describes the production architecture implemented in this repository today.
|
|
||||||
|
|
||||||
Audita is a single-process Go CLI that:
|
|
||||||
- loads effective runtime configuration;
|
|
||||||
- reads transcript and glossary inputs;
|
|
||||||
- normalizes and sections transcripts;
|
|
||||||
- runs a built-in module pipeline with validator chains;
|
|
||||||
- writes transcript output and run diagnostics.
|
|
||||||
|
|
||||||
## Runtime entrypoints
|
|
||||||
Primary CLI commands:
|
|
||||||
- `audita process <transcript.json> --glossary <glossary.yaml> [flags]`
|
|
||||||
- `audita config validate --config <config.yml>`
|
|
||||||
- `audita config print-effective [--config <config.yml>]`
|
|
||||||
|
|
||||||
Command ownership lives in `internal/cli/run.go`.
|
|
||||||
|
|
||||||
## Configuration model
|
|
||||||
`internal/core/config` owns defaults, file parsing, environment overrides, CLI overrides, and validation.
|
|
||||||
|
|
||||||
Effective-config loading for `process` and `config print-effective` is centralized in:
|
|
||||||
- `ResolveConfigPath`
|
|
||||||
- `LoadEffectiveConfig`
|
|
||||||
|
|
||||||
Effective precedence for `audita process`:
|
|
||||||
1. defaults
|
|
||||||
2. config file
|
|
||||||
3. environment overrides
|
|
||||||
4. CLI overrides
|
|
||||||
|
|
||||||
`audita config validate` is intentionally file-only validation:
|
|
||||||
- load versioned file;
|
|
||||||
- apply onto defaults;
|
|
||||||
- validate;
|
|
||||||
- do not apply environment overrides.
|
|
||||||
|
|
||||||
Supported module and output-schema keys are validated through shared catalogs:
|
|
||||||
- module keys: `internal/core/modulecatalog`
|
|
||||||
- output schemas: `internal/core/outputschema`
|
|
||||||
|
|
||||||
## Pipeline and module orchestration
|
|
||||||
The built-in module sequence is configured in runtime config and executed by `internal/framework/runner` through resolved module specs.
|
|
||||||
|
|
||||||
Current default sequence:
|
|
||||||
- `glossary`
|
|
||||||
- `homophones`
|
|
||||||
- `glossary`
|
|
||||||
- `spoken_word`
|
|
||||||
- `grammar`
|
|
||||||
|
|
||||||
Execution behavior:
|
|
||||||
- modules execute serially over the working transcript;
|
|
||||||
- section proposal work can run concurrently within a module;
|
|
||||||
- validator execution happens on generated proposals before application;
|
|
||||||
- approved proposals are applied once per module in deterministic proposal-index order.
|
|
||||||
|
|
||||||
Production modules remain separate packages:
|
|
||||||
- `internal/modules/glossary`
|
|
||||||
- `internal/modules/homophones`
|
|
||||||
- `internal/modules/spoken_word`
|
|
||||||
- `internal/modules/grammar`
|
|
||||||
|
|
||||||
## Proposal generation and prompt context
|
|
||||||
Shared proposal plumbing is centralized in `internal/framework/proposal_generation`.
|
|
||||||
|
|
||||||
Module packages provide:
|
|
||||||
- module identity and replacement policy;
|
|
||||||
- module-specific prompt message building;
|
|
||||||
- built-in validator chain selection.
|
|
||||||
|
|
||||||
Shared prompt payload helpers are in `internal/framework/promptcontext`.
|
|
||||||
|
|
||||||
## Validator architecture
|
|
||||||
Built-in validator construction and chain composition live in `internal/validators`.
|
|
||||||
|
|
||||||
Shared validator runtime mechanics live in `internal/framework/validators`.
|
|
||||||
|
|
||||||
Execution class metadata (deterministic vs LLM-backed) is centralized in `internal/validators/metadata` and used for ordering and reporting classification.
|
|
||||||
|
|
||||||
## Structured LLM boundary
|
|
||||||
All production LLM calls go through the internal contract:
|
|
||||||
- `contracts.StructuredLLMClient`
|
|
||||||
- `CompleteStructured(ctx, req, out)`
|
|
||||||
|
|
||||||
The OpenAI-compatible HTTP adapter is implemented in `internal/framework/llm`.
|
|
||||||
|
|
||||||
Structured response schemas are registered in `internal/framework/responseschema` and attached to requests via `response_format` metadata.
|
|
||||||
|
|
||||||
Malformed structured-output detection is centralized in `internal/framework/structuredoutput` and reused by proposal generation and validator execution so downgrade behavior stays consistent.
|
|
||||||
|
|
||||||
## Stage naming and diagnostics metadata
|
|
||||||
Diagnostics stage naming is centralized in `internal/framework/stagename`:
|
|
||||||
- module proposal stage names;
|
|
||||||
- proposal-generation stage names;
|
|
||||||
- validator batch stage names.
|
|
||||||
|
|
||||||
Prompt metadata and response-schema metadata each expose canonical diagnostics maps via:
|
|
||||||
- `prompts.Metadata.DiagnosticsMap()`
|
|
||||||
- `responseschema.Schema.DiagnosticsMap()`
|
|
||||||
|
|
||||||
## Diagnostics and reporting
|
|
||||||
Run-directory artifacts are owned by `internal/core/diagnostics`.
|
|
||||||
|
|
||||||
Stable artifact names are centralized constants (for example transcript artifacts, `invocation.json`, `effective-config.json`, `utilization-diagnostics.json`, `correction-ledger.json`, `report.json`, `error.log`).
|
|
||||||
|
|
||||||
Report diagnostics path metadata is constructed through `BuildDiagnosticsMetadata`, which keeps run-directory artifact references consistent between success and failure reports.
|
|
||||||
|
|
||||||
## Secret redaction
|
|
||||||
Redaction responsibilities are split by concern:
|
|
||||||
- structural config redaction: `config.Config.Redacted()`
|
|
||||||
- byte/string payload redaction for diagnostics and surfaced errors: framework redaction utilities.
|
|
||||||
|
|
||||||
Configured LLM secret extraction is centralized in `llm.ConfiguredSecrets(cfg)` and reused across proposal and validator diagnostics paths.
|
|
||||||
|
|
||||||
## Output contracts
|
|
||||||
Transcript output schema selection is owned by `internal/core/outputschema`.
|
|
||||||
|
|
||||||
Supported schemas:
|
|
||||||
- `bare-segments`
|
|
||||||
- `audita-v1`
|
|
||||||
|
|
||||||
Unknown schema keys fail validation and runtime resolution.
|
|
||||||
|
|
||||||
## Key package map
|
|
||||||
Core packages:
|
|
||||||
- `internal/core/config`
|
|
||||||
- `internal/core/schema`
|
|
||||||
- `internal/core/normalization`
|
|
||||||
- `internal/core/chunking`
|
|
||||||
- `internal/core/diagnostics`
|
|
||||||
- `internal/core/reporting`
|
|
||||||
- `internal/core/modulecatalog`
|
|
||||||
- `internal/core/outputschema`
|
|
||||||
|
|
||||||
Framework packages:
|
|
||||||
- `internal/framework/contracts`
|
|
||||||
- `internal/framework/proposals`
|
|
||||||
- `internal/framework/proposal_generation`
|
|
||||||
- `internal/framework/promptcontext`
|
|
||||||
- `internal/framework/runner`
|
|
||||||
- `internal/framework/validators`
|
|
||||||
- `internal/framework/llm`
|
|
||||||
- `internal/framework/responseschema`
|
|
||||||
- `internal/framework/stagename`
|
|
||||||
- `internal/framework/structuredoutput`
|
|
||||||
|
|
||||||
Domain packages:
|
|
||||||
- `internal/modules/*`
|
|
||||||
- `internal/validators/*`
|
|
||||||
- `internal/prompts`
|
|
||||||
@@ -1,104 +0,0 @@
|
|||||||
# Audita Diagnostics
|
|
||||||
|
|
||||||
This document describes the run-directory diagnostics artifacts produced by `audita process`.
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Diagnostics provide machine-readable run context and execution artifacts for:
|
|
||||||
- failure debugging;
|
|
||||||
- validator/correction review;
|
|
||||||
- post-run performance analysis.
|
|
||||||
|
|
||||||
Diagnostics are written under the configured work directory (`--work-dir`) when run-directory initialization succeeds.
|
|
||||||
|
|
||||||
## Core artifacts
|
|
||||||
|
|
||||||
Typical artifacts in each run directory:
|
|
||||||
- `source-transcript.json`
|
|
||||||
- `source-transcript-parsed.json`
|
|
||||||
- `normalized-transcript.json`
|
|
||||||
- `normalization-summary.json`
|
|
||||||
- `chunking-summary.json`
|
|
||||||
- `invocation.json`
|
|
||||||
- `effective-config.json` (redacted)
|
|
||||||
- module/validator LLM interaction artifacts
|
|
||||||
- `report.json`
|
|
||||||
- `error.log` on failure
|
|
||||||
|
|
||||||
## Utilization diagnostics artifact
|
|
||||||
|
|
||||||
Artifact:
|
|
||||||
- `utilization-diagnostics.json`
|
|
||||||
|
|
||||||
High-level fields:
|
|
||||||
- `effective_concurrency`:
|
|
||||||
- total/proposal/validation LLM concurrency limits in effect.
|
|
||||||
- `run_timing`:
|
|
||||||
- run wall time;
|
|
||||||
- scheduler queue wait time;
|
|
||||||
- LLM execution time;
|
|
||||||
- deterministic validator time;
|
|
||||||
- max/average in-flight LLM calls.
|
|
||||||
- `llm_calls`:
|
|
||||||
- total proposal and validation LLM call counts.
|
|
||||||
- `modules`:
|
|
||||||
- module-level timing summaries.
|
|
||||||
- `validators`:
|
|
||||||
- per-validator timing summaries keyed by stable validator key.
|
|
||||||
|
|
||||||
## Correction ledger artifact
|
|
||||||
|
|
||||||
Artifact:
|
|
||||||
- `correction-ledger.json`
|
|
||||||
|
|
||||||
Ledger records are flattened review entries derived from module results and include:
|
|
||||||
- module/proposal identity (`module_key`, `module_instance`, `proposal_index`, `segment_id`);
|
|
||||||
- correction text fields and replacement policy when available;
|
|
||||||
- disposition:
|
|
||||||
- `applied`
|
|
||||||
- `rejected`
|
|
||||||
- `skipped`
|
|
||||||
- `failed`
|
|
||||||
- stable reason codes/messages;
|
|
||||||
- deterministic and LLM validator decision snapshots using stable validator keys.
|
|
||||||
|
|
||||||
Validator rejection and proposal-application skip are distinct dispositions.
|
|
||||||
Module warnings are reported in module results and diagnostics metadata, but do not create standalone correction-ledger rows.
|
|
||||||
|
|
||||||
## Report references
|
|
||||||
|
|
||||||
`report.json` and optional `--report-json` output include diagnostics metadata paths for:
|
|
||||||
- utilization diagnostics artifact;
|
|
||||||
- correction ledger artifact;
|
|
||||||
- existing transcript/normalization/chunking/invocation/effective-config artifacts.
|
|
||||||
|
|
||||||
Module report entries also include warning records for malformed proposal-generation payloads and malformed validator batches.
|
|
||||||
|
|
||||||
## Retention behavior
|
|
||||||
|
|
||||||
Run-directory retention follows configured policy:
|
|
||||||
- `always`: keep all run directories;
|
|
||||||
- `never`: keep successful run directories;
|
|
||||||
- `auto`: keep failed runs and successful runs with skipped/rejected corrections.
|
|
||||||
|
|
||||||
## Redaction guarantees
|
|
||||||
|
|
||||||
API keys and other configured secrets are redacted from:
|
|
||||||
- `effective-config.json`;
|
|
||||||
- LLM interaction diagnostics artifacts;
|
|
||||||
- reports and surfaced errors.
|
|
||||||
|
|
||||||
## Debugging guide
|
|
||||||
|
|
||||||
When debugging:
|
|
||||||
- slow runs:
|
|
||||||
- inspect `utilization-diagnostics.json` (`run_timing`, `modules`, `validators`, in-flight metrics).
|
|
||||||
- validator rejections:
|
|
||||||
- inspect `correction-ledger.json` rejected entries and matching validator decisions;
|
|
||||||
- inspect validator response diagnostics payloads.
|
|
||||||
- module warnings:
|
|
||||||
- inspect module `warnings` entries in `report.json` or `--report-json`;
|
|
||||||
- follow any diagnostic artifact path on the warning to the recorded error/response payload.
|
|
||||||
- application skips:
|
|
||||||
- inspect `correction-ledger.json` skipped entries and skip reason codes;
|
|
||||||
- compare with validator decisions to distinguish validation rejection vs apply-time skip.
|
|
||||||
@@ -1,88 +0,0 @@
|
|||||||
# Audita Output Schemas
|
|
||||||
|
|
||||||
This document describes the built-in transcript output schema registry used by `audita process`.
|
|
||||||
|
|
||||||
## Supported schema names
|
|
||||||
|
|
||||||
### `bare-segments`
|
|
||||||
|
|
||||||
Status:
|
|
||||||
- implemented
|
|
||||||
- default output schema
|
|
||||||
|
|
||||||
Shape:
|
|
||||||
- top-level JSON array of transcript segments
|
|
||||||
|
|
||||||
Segment fields:
|
|
||||||
- `id`
|
|
||||||
- `speaker`
|
|
||||||
- `start`
|
|
||||||
- `end`
|
|
||||||
- `text`
|
|
||||||
- optional `categories`
|
|
||||||
|
|
||||||
Compatibility:
|
|
||||||
- this preserves the long-standing output shape used by existing consumers.
|
|
||||||
|
|
||||||
### `audita-v1`
|
|
||||||
|
|
||||||
Status:
|
|
||||||
- implemented
|
|
||||||
|
|
||||||
Shape:
|
|
||||||
- top-level JSON object:
|
|
||||||
- `schema`: `"audita-v1"`
|
|
||||||
- `version`: `"v1"`
|
|
||||||
- `segments`: transcript segment array
|
|
||||||
|
|
||||||
Segment fields inside `segments` match `bare-segments` segment fields.
|
|
||||||
|
|
||||||
Compatibility:
|
|
||||||
- this is the Audita-native object format with explicit schema/version metadata.
|
|
||||||
|
|
||||||
### `seriatim-intermediate`
|
|
||||||
|
|
||||||
Status:
|
|
||||||
- deferred / not implemented
|
|
||||||
|
|
||||||
Current behavior:
|
|
||||||
- selecting `seriatim-intermediate` fails clearly as an unsupported output schema.
|
|
||||||
|
|
||||||
Reason:
|
|
||||||
- a concrete, repository-backed contract for this schema has not been finalized yet.
|
|
||||||
|
|
||||||
## Selection
|
|
||||||
|
|
||||||
Choose output schema with CLI:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
audita process <transcript.json> --glossary <glossary.yaml> --output-schema audita-v1
|
|
||||||
```
|
|
||||||
|
|
||||||
Or in file config:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
version: 1
|
|
||||||
output:
|
|
||||||
schema: audita-v1
|
|
||||||
```
|
|
||||||
|
|
||||||
Precedence remains:
|
|
||||||
1. defaults
|
|
||||||
2. file config
|
|
||||||
3. environment overrides
|
|
||||||
4. CLI overrides
|
|
||||||
|
|
||||||
`--output-schema` overrides `output.schema` when both are supplied.
|
|
||||||
|
|
||||||
## Output routing behavior
|
|
||||||
|
|
||||||
- With `--output`, transcript JSON is written to file using the selected schema and stdout stays empty on success.
|
|
||||||
- Without `--output`, stdout contains transcript JSON only, using the selected schema.
|
|
||||||
- `--report-json` writes report JSON to file and does not write report payloads to stdout.
|
|
||||||
|
|
||||||
## Backward-compatibility expectations
|
|
||||||
|
|
||||||
- default schema stays `bare-segments` for compatibility unless explicitly changed in a future breaking release;
|
|
||||||
- supported schema names are treated as stable public contract values;
|
|
||||||
- unsupported schema names fail before output write.
|
|
||||||
@@ -1,118 +0,0 @@
|
|||||||
# Audita Prompts
|
|
||||||
|
|
||||||
This document describes Audita's built-in embedded prompt assets and prompt registry behavior.
|
|
||||||
|
|
||||||
## Why embedded prompt assets
|
|
||||||
|
|
||||||
Audita embeds production prompt text into the binary so runtime behavior is:
|
|
||||||
- deterministic;
|
|
||||||
- auditable;
|
|
||||||
- dependency-light;
|
|
||||||
- not dependent on external prompt files at execution time.
|
|
||||||
|
|
||||||
Prompt text is authored as Markdown assets and rendered by Go code using typed template data.
|
|
||||||
|
|
||||||
## Built-in prompt registry
|
|
||||||
|
|
||||||
The prompt registry lives in `internal/prompts` and is responsible for:
|
|
||||||
- loading embedded prompt assets;
|
|
||||||
- registering stable prompt IDs and versions;
|
|
||||||
- recording prompt source metadata;
|
|
||||||
- computing deterministic SHA-256 source hashes;
|
|
||||||
- rendering system/user prompts with strict missing-key failures.
|
|
||||||
|
|
||||||
Current prompt source behavior:
|
|
||||||
- built-in embedded prompts only (`prompt_source = builtin`).
|
|
||||||
- filesystem prompt overrides are not supported.
|
|
||||||
|
|
||||||
## Built-in prompt IDs
|
|
||||||
|
|
||||||
Module proposal prompts:
|
|
||||||
- `modules.glossary.proposal`
|
|
||||||
- `modules.homophones.proposal`
|
|
||||||
- `modules.spoken_word.proposal`
|
|
||||||
- `modules.grammar.proposal`
|
|
||||||
|
|
||||||
LLM-backed validator prompts:
|
|
||||||
- `validators.spoken_form_plausibility`
|
|
||||||
- `validators.meaning_reversal_review`
|
|
||||||
- `validators.editorial_review`
|
|
||||||
- `validators.grammar_review`
|
|
||||||
- `validators.spoken_word_review`
|
|
||||||
|
|
||||||
## Prompt version semantics
|
|
||||||
|
|
||||||
Current built-in prompt version value is `v1`.
|
|
||||||
|
|
||||||
Version is a stable metadata identifier for diagnostics and debugging. It is not a dynamic prompt-selection mechanism.
|
|
||||||
|
|
||||||
## Prompt hash semantics
|
|
||||||
|
|
||||||
Each registered prompt includes a deterministic SHA-256 hash of embedded source text.
|
|
||||||
|
|
||||||
Hash purpose:
|
|
||||||
- identify exact prompt source used in a run;
|
|
||||||
- support diagnostics reproducibility and change auditing.
|
|
||||||
|
|
||||||
Current hash scope:
|
|
||||||
- source prompt text (system + user assets for a registered prompt), not a runtime secret-bearing payload.
|
|
||||||
|
|
||||||
## Template rendering behavior
|
|
||||||
|
|
||||||
Prompt rendering uses Go `text/template` with typed template data from module/validator builders.
|
|
||||||
|
|
||||||
Missing-key behavior:
|
|
||||||
- rendering uses missing-key errors;
|
|
||||||
- missing/renamed template fields fail quickly instead of silently producing incomplete prompts.
|
|
||||||
|
|
||||||
Go code still owns:
|
|
||||||
- structured request/response models;
|
|
||||||
- response schema selection;
|
|
||||||
- transcript/glossary/payload formatting;
|
|
||||||
- module and validator selection;
|
|
||||||
- diagnostics wiring.
|
|
||||||
|
|
||||||
## Shared prompt hardening policy
|
|
||||||
|
|
||||||
A shared hardening fragment is embedded once and included in every module proposal prompt and every LLM-validator prompt.
|
|
||||||
|
|
||||||
Hardening policy includes:
|
|
||||||
- transcript text is untrusted data;
|
|
||||||
- glossary entries and transcript descriptions are reference data, not instructions;
|
|
||||||
- instructions found inside transcript text must not be obeyed;
|
|
||||||
- model must perform only the requested correction/validation task;
|
|
||||||
- no invention of facts, names, events, motivations, speaker intent, or corrections;
|
|
||||||
- transcript remains the source of truth.
|
|
||||||
|
|
||||||
## Transcript description behavior
|
|
||||||
|
|
||||||
Transcript description remains background-only prompt context:
|
|
||||||
- it may help interpret ambiguous terms;
|
|
||||||
- it is explicitly non-authoritative and must not override transcript content;
|
|
||||||
- empty descriptions do not render awkward blank context sections.
|
|
||||||
|
|
||||||
Generated transcript descriptions are not implemented in this workstream.
|
|
||||||
|
|
||||||
## Diagnostics and report metadata boundaries
|
|
||||||
|
|
||||||
Current metadata flow:
|
|
||||||
- proposal-generation diagnostics request metadata includes prompt metadata;
|
|
||||||
- LLM-validator diagnostics request metadata includes prompt metadata.
|
|
||||||
|
|
||||||
Prompt metadata fields used in diagnostics:
|
|
||||||
- `prompt_id`
|
|
||||||
- `prompt_version`
|
|
||||||
- `prompt_source`
|
|
||||||
- `embedded_path`
|
|
||||||
- `sha256`
|
|
||||||
|
|
||||||
Current boundary:
|
|
||||||
- detailed prompt metadata is diagnostics-first;
|
|
||||||
- broad report-level prompt registries/ledgers are deferred.
|
|
||||||
|
|
||||||
## 1.0 boundary
|
|
||||||
|
|
||||||
Not implemented for 1.0 in this workstream:
|
|
||||||
- filesystem prompt overrides;
|
|
||||||
- user-configurable prompt selection;
|
|
||||||
- external prompt directories.
|
|
||||||
@@ -1,121 +0,0 @@
|
|||||||
# Audita Public Contract
|
|
||||||
|
|
||||||
## Scope
|
|
||||||
This document defines stability expectations for Audita's external runtime interfaces.
|
|
||||||
|
|
||||||
Covered interfaces:
|
|
||||||
- CLI commands and major flags;
|
|
||||||
- versioned config behavior and precedence;
|
|
||||||
- transcript/glossary input forms;
|
|
||||||
- output schema selection;
|
|
||||||
- report schema metadata;
|
|
||||||
- diagnostics artifact path metadata;
|
|
||||||
- stdout/stderr and exit-code behavior;
|
|
||||||
- redaction guarantees.
|
|
||||||
|
|
||||||
## CLI contract
|
|
||||||
Stable commands:
|
|
||||||
- `audita process`
|
|
||||||
- `audita config validate`
|
|
||||||
- `audita config print-effective`
|
|
||||||
|
|
||||||
Stable high-value `process` flags:
|
|
||||||
- `--config`
|
|
||||||
- `--glossary`
|
|
||||||
- `--output`
|
|
||||||
- `--report-json`
|
|
||||||
- `--modules`
|
|
||||||
- `--output-schema`
|
|
||||||
|
|
||||||
## Config contract
|
|
||||||
Supported config format:
|
|
||||||
- YAML;
|
|
||||||
- `version: 1`;
|
|
||||||
- strict unknown-field rejection.
|
|
||||||
|
|
||||||
Path resolution for `process` and `config print-effective`:
|
|
||||||
1. `--config`
|
|
||||||
2. `AUDITA_CONFIG`
|
|
||||||
3. `/usr/local/etc/audita/config.yml`
|
|
||||||
4. `/etc/audita/config.yml`
|
|
||||||
|
|
||||||
Missing explicit path is an error. Missing default paths is non-fatal.
|
|
||||||
|
|
||||||
Precedence for `process`:
|
|
||||||
1. defaults
|
|
||||||
2. file config
|
|
||||||
3. environment overrides
|
|
||||||
4. CLI overrides
|
|
||||||
|
|
||||||
`config validate` remains file-only validation (defaults + file config; no env overrides).
|
|
||||||
|
|
||||||
Module and output-schema keys are validated against built-in catalogs. Unknown keys fail validation.
|
|
||||||
|
|
||||||
## Input contract
|
|
||||||
Supported transcript JSON top-level forms:
|
|
||||||
- array of segments
|
|
||||||
- object with `segments` array
|
|
||||||
|
|
||||||
Supported glossary YAML form:
|
|
||||||
- top-level `glossary` list with required entry fields validated by schema parsing.
|
|
||||||
|
|
||||||
## Output schema contract
|
|
||||||
Supported transcript output schemas:
|
|
||||||
- `bare-segments` (default)
|
|
||||||
- `audita-v1`
|
|
||||||
|
|
||||||
Unknown schema keys fail before output write.
|
|
||||||
|
|
||||||
## Report metadata contract
|
|
||||||
Process reports include stable report metadata fields:
|
|
||||||
- `report_schema_name`
|
|
||||||
- `report_schema_version`
|
|
||||||
- `output_schema`
|
|
||||||
- `config_version` (when file config is loaded)
|
|
||||||
|
|
||||||
Current values:
|
|
||||||
- `report_schema_name = audita-process-report`
|
|
||||||
- `report_schema_version = v1`
|
|
||||||
|
|
||||||
`--report-json` output and run-directory `report.json` use the same report schema metadata.
|
|
||||||
|
|
||||||
Validator decision/rejection records use stable validator keys via `validator_name`.
|
|
||||||
|
|
||||||
## Diagnostics metadata contract
|
|
||||||
When run-directory initialization succeeds, diagnostics metadata paths reference stable artifacts, including:
|
|
||||||
- transcript and normalization artifacts;
|
|
||||||
- chunking summary;
|
|
||||||
- invocation metadata;
|
|
||||||
- redacted effective config;
|
|
||||||
- utilization diagnostics;
|
|
||||||
- correction ledger;
|
|
||||||
- `error.log` on failures.
|
|
||||||
|
|
||||||
LLM interaction diagnostics include stable prompt and structured-schema identifiers where applicable.
|
|
||||||
|
|
||||||
## Stdout/stderr and exit codes
|
|
||||||
Success:
|
|
||||||
- with `--output`, stdout is empty;
|
|
||||||
- without `--output`, stdout contains transcript JSON only;
|
|
||||||
- report JSON is not written to stdout.
|
|
||||||
|
|
||||||
Failures:
|
|
||||||
- nonzero exit;
|
|
||||||
- human-readable stderr summary;
|
|
||||||
- diagnostics directory path on stderr when available.
|
|
||||||
|
|
||||||
Exit codes:
|
|
||||||
- `0` success
|
|
||||||
- nonzero failure
|
|
||||||
|
|
||||||
## Redaction contract
|
|
||||||
Configured secrets are redacted from:
|
|
||||||
- effective config outputs;
|
|
||||||
- diagnostics artifacts;
|
|
||||||
- report artifacts;
|
|
||||||
- surfaced adapter/runtime errors.
|
|
||||||
|
|
||||||
## Compatibility policy
|
|
||||||
Stable command behavior, schema names, report metadata keys, diagnostics-path field semantics, and validator key identities are treated as public contract.
|
|
||||||
|
|
||||||
Additive fields are acceptable when existing fields and behavior remain compatible.
|
|
||||||
@@ -1,72 +0,0 @@
|
|||||||
# Structured LLM Architecture
|
|
||||||
|
|
||||||
## Scope
|
|
||||||
This document describes Audita's structured LLM runtime boundary and adapter behavior.
|
|
||||||
|
|
||||||
## Runtime boundary
|
|
||||||
Production LLM integration depends on the internal contract only:
|
|
||||||
- `contracts.StructuredLLMClient`
|
|
||||||
- `CompleteStructured(ctx, req, out)`
|
|
||||||
|
|
||||||
Provider SDK types do not leak past this boundary.
|
|
||||||
|
|
||||||
## Adapter ownership
|
|
||||||
`internal/framework/llm` owns the OpenAI-compatible HTTP adapter and shared LLM runtime utilities.
|
|
||||||
|
|
||||||
Key responsibilities:
|
|
||||||
- request assembly;
|
|
||||||
- timeout/cancellation propagation;
|
|
||||||
- bounded retry behavior;
|
|
||||||
- scheduler integration;
|
|
||||||
- provider response decoding;
|
|
||||||
- error redaction.
|
|
||||||
|
|
||||||
## Structured schema registry
|
|
||||||
Structured response schemas are registered in `internal/framework/responseschema` and include stable metadata:
|
|
||||||
- `id`
|
|
||||||
- `version`
|
|
||||||
- `name`
|
|
||||||
- `json_schema`
|
|
||||||
- `sha256`
|
|
||||||
|
|
||||||
Current schema keys:
|
|
||||||
- `correction_set`
|
|
||||||
- `validator_decision_set`
|
|
||||||
|
|
||||||
Schema metadata is attached to diagnostics through `Schema.DiagnosticsMap()`.
|
|
||||||
|
|
||||||
## Request shape assumptions
|
|
||||||
Audita targets OpenAI-compatible chat-completions endpoints and sends structured requests with:
|
|
||||||
- model;
|
|
||||||
- chat messages;
|
|
||||||
- `response_format.type = json_schema`;
|
|
||||||
- schema name and JSON schema payload.
|
|
||||||
|
|
||||||
## Local validation remains mandatory
|
|
||||||
Provider schema enforcement is treated as transport-level guardrails.
|
|
||||||
|
|
||||||
Audita still validates output locally before applying behavior changes:
|
|
||||||
- proposal decoding and proposal invariants;
|
|
||||||
- validator decision decoding and cardinality checks;
|
|
||||||
- deterministic validation and apply-time rules.
|
|
||||||
|
|
||||||
## Shared malformed-output policy
|
|
||||||
Malformed structured-output classification is centralized in `internal/framework/structuredoutput`.
|
|
||||||
|
|
||||||
Proposal generation and validator execution both use this shared classifier so downgrade behavior cannot drift between the two paths.
|
|
||||||
|
|
||||||
## Secrets and redaction
|
|
||||||
Secret extraction for LLM redaction is centralized in `llm.ConfiguredSecrets(cfg)` and reused by proposal and validator diagnostics writers.
|
|
||||||
|
|
||||||
Secrets are redacted from:
|
|
||||||
- diagnostics artifacts;
|
|
||||||
- report artifacts;
|
|
||||||
- surfaced adapter/runtime errors.
|
|
||||||
|
|
||||||
## Concurrency and scheduling
|
|
||||||
LLM execution is constrained by composed scheduler limits:
|
|
||||||
- total LLM concurrency;
|
|
||||||
- proposal LLM concurrency;
|
|
||||||
- validation LLM concurrency.
|
|
||||||
|
|
||||||
The scheduler is FIFO and context-aware so permits are released on success, failure, and cancellation.
|
|
||||||
@@ -1,96 +0,0 @@
|
|||||||
# Audita Validators
|
|
||||||
|
|
||||||
## Scope
|
|
||||||
This document defines the built-in validator system used by production module runs.
|
|
||||||
|
|
||||||
## Ownership boundaries
|
|
||||||
Built-in validator keys, constructors, and module chains are owned by `internal/validators`.
|
|
||||||
|
|
||||||
Shared runtime execution mechanics are owned by `internal/framework/validators`, including:
|
|
||||||
- validator request/result models;
|
|
||||||
- deterministic proposal checks;
|
|
||||||
- LLM validator batching and execution;
|
|
||||||
- decision-cardinality enforcement;
|
|
||||||
- diagnostics integration.
|
|
||||||
|
|
||||||
Execution class metadata is owned by `internal/validators/metadata`.
|
|
||||||
|
|
||||||
## Stable validator keys
|
|
||||||
Deterministic:
|
|
||||||
- `proposal_shape`
|
|
||||||
- `confidence_threshold`
|
|
||||||
- `original_text_presence`
|
|
||||||
- `non_empty_corrected_text`
|
|
||||||
- `no_effect`
|
|
||||||
- `protected_terms`
|
|
||||||
|
|
||||||
LLM-backed:
|
|
||||||
- `spoken_form_plausibility`
|
|
||||||
- `meaning_reversal_review`
|
|
||||||
- `editorial_review`
|
|
||||||
|
|
||||||
## Built-in module chains
|
|
||||||
`glossary`:
|
|
||||||
- `proposal_shape`
|
|
||||||
- `no_effect`
|
|
||||||
- `original_text_presence`
|
|
||||||
- `confidence_threshold`
|
|
||||||
- `protected_terms`
|
|
||||||
- `non_empty_corrected_text`
|
|
||||||
- `spoken_form_plausibility`
|
|
||||||
- `meaning_reversal_review`
|
|
||||||
|
|
||||||
`homophones`:
|
|
||||||
- `proposal_shape`
|
|
||||||
- `no_effect`
|
|
||||||
- `original_text_presence`
|
|
||||||
- `confidence_threshold`
|
|
||||||
- `protected_terms`
|
|
||||||
- `non_empty_corrected_text`
|
|
||||||
- `spoken_form_plausibility`
|
|
||||||
- `meaning_reversal_review`
|
|
||||||
|
|
||||||
`spoken_word`:
|
|
||||||
- `proposal_shape`
|
|
||||||
- `no_effect`
|
|
||||||
- `original_text_presence`
|
|
||||||
- `confidence_threshold`
|
|
||||||
- `protected_terms`
|
|
||||||
- `non_empty_corrected_text`
|
|
||||||
- `editorial_review`
|
|
||||||
- `meaning_reversal_review`
|
|
||||||
|
|
||||||
`grammar`:
|
|
||||||
- `proposal_shape`
|
|
||||||
- `no_effect`
|
|
||||||
- `original_text_presence`
|
|
||||||
- `confidence_threshold`
|
|
||||||
- `protected_terms`
|
|
||||||
- `non_empty_corrected_text`
|
|
||||||
- `editorial_review`
|
|
||||||
- `meaning_reversal_review`
|
|
||||||
|
|
||||||
## Ordering and execution semantics
|
|
||||||
Validator ordering is based on canonical metadata:
|
|
||||||
- deterministic validators run before LLM-backed validators.
|
|
||||||
|
|
||||||
Within each module stage:
|
|
||||||
- proposals are generated per section;
|
|
||||||
- validator chains execute on those proposals;
|
|
||||||
- approved proposals are applied once after section work settles.
|
|
||||||
|
|
||||||
## Malformed payload behavior
|
|
||||||
Malformed structured-output from proposal generation and LLM validator calls is downgraded, not treated as a process-fatal transport error.
|
|
||||||
|
|
||||||
Current outcomes:
|
|
||||||
- malformed proposal-generation payloads produce section/module warnings and zero proposals for the affected section;
|
|
||||||
- malformed validator decision payloads reject the affected validator batch with warnings;
|
|
||||||
- deterministic validator behavior and runner order remain unchanged.
|
|
||||||
|
|
||||||
## Reporting identity
|
|
||||||
Reports and diagnostics use stable validator keys as identifiers.
|
|
||||||
|
|
||||||
Correction-ledger deterministic-vs-LLM classification is derived from canonical validator metadata, not package-local hardcoded maps.
|
|
||||||
|
|
||||||
## Prompt assets
|
|
||||||
LLM validator prompt assets and prompt metadata are documented in [Prompts](./prompts.md).
|
|
||||||
198
docs/cli.md
Normal file
198
docs/cli.md
Normal file
@@ -0,0 +1,198 @@
|
|||||||
|
# Audita CLI Reference
|
||||||
|
|
||||||
|
## Shortest Useful Command
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||||
|
```
|
||||||
|
|
||||||
|
This command runs the core workflow with your input files. For a copyable example
|
||||||
|
using maintained inputs and explicit prerequisites, see the [README](../README.md).
|
||||||
|
Commands here assume `audita` is on `PATH`; use `./bin/audita` after a local build.
|
||||||
|
|
||||||
|
## Command Overview
|
||||||
|
|
||||||
|
- `audita process`: process one transcript JSON file.
|
||||||
|
- `audita config validate`: validate a versioned YAML config file.
|
||||||
|
- `audita config print-effective`: print redacted effective config JSON.
|
||||||
|
|
||||||
|
General help:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita --help
|
||||||
|
audita process --help
|
||||||
|
audita config --help
|
||||||
|
```
|
||||||
|
|
||||||
|
## `process`
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita process <transcript.json> [flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
Input requirements:
|
||||||
|
- exactly one transcript JSON positional argument is required;
|
||||||
|
- `--glossary <path>` is required.
|
||||||
|
|
||||||
|
Place the transcript path first, followed by flags, as shown above. Flag values
|
||||||
|
may use `--name value` or `--name=value`. Unknown flags or extra positional
|
||||||
|
arguments are errors. Flags before a final transcript path are also accepted;
|
||||||
|
do not interleave further flags after a positional argument consumed by the flag
|
||||||
|
parser.
|
||||||
|
|
||||||
|
[Configuration discovery and precedence](config.md#loading-model) determine the
|
||||||
|
initial settings; explicit process flags supply overrides. Configuration is loaded
|
||||||
|
before process help is printed, so invalid discovered configuration can also make
|
||||||
|
process help fail. Input formats belong to the
|
||||||
|
[input-file contract](integrations/transcript-glossary-files.md).
|
||||||
|
|
||||||
|
### `process` Flag Reference
|
||||||
|
|
||||||
|
Core I/O flags:
|
||||||
|
- `--config <path>`: path to versioned YAML config file.
|
||||||
|
- `--glossary <path>`: glossary YAML input path (required).
|
||||||
|
- `--output <path>`: corrected transcript JSON output file path.
|
||||||
|
- `--report-json <path>`: machine-readable report JSON output path.
|
||||||
|
- `--output-schema <key>`: output encoder selection; see [configuration](config.md#pipeline-and-output).
|
||||||
|
- `--modules <csv>`: comma-separated module sequence override.
|
||||||
|
|
||||||
|
Primary LLM flags:
|
||||||
|
- `--llm-api-key <value>`: primary LLM API key.
|
||||||
|
- `--model <name>`: primary LLM model name.
|
||||||
|
- `--base-url <url>`: primary OpenAI-compatible base URL.
|
||||||
|
- `--llm-timeout-seconds <int>`: primary timeout in seconds.
|
||||||
|
- `--max-retries <int>`: primary structured-output retries.
|
||||||
|
|
||||||
|
Validation LLM flags:
|
||||||
|
- `--validation-llm-api-key <value>`: validation LLM API key.
|
||||||
|
- `--validation-model <name>`: validation LLM model name.
|
||||||
|
- `--validation-base-url <url>`: validation OpenAI-compatible base URL.
|
||||||
|
- `--validation-llm-timeout-seconds <int>`: validation timeout in seconds.
|
||||||
|
- `--validation-max-retries <int>`: validation structured-output retries.
|
||||||
|
- `--validation-max-prompt-tokens <int>`: validation max prompt tokens.
|
||||||
|
|
||||||
|
Concurrency flags:
|
||||||
|
- `--total-llm-concurrency <int>`: total concurrent proposal+validation LLM calls.
|
||||||
|
- `--proposal-llm-concurrency <int>`: concurrent proposal-generation LLM calls.
|
||||||
|
- `--validation-llm-concurrency <int>`: concurrent validation LLM calls.
|
||||||
|
- `--llm-concurrency <int>`: alias for `--total-llm-concurrency`.
|
||||||
|
|
||||||
|
Chunking and normalization flags:
|
||||||
|
- `--target-sections <int>`: target number of transcript sections.
|
||||||
|
- `--max-section-tokens <int>`: maximum section tokens.
|
||||||
|
- `--min-section-tokens <int>`: minimum section tokens.
|
||||||
|
- `--normalize-max-segment-gap <float>`: maximum same-speaker merge gap.
|
||||||
|
- `--normalize-ellipsis-gap <float>`: gap threshold for ellipsis insertion.
|
||||||
|
- `--normalize-max-segment-duration <float>`: maximum merged segment duration.
|
||||||
|
- `--normalize-max-segment-tokens <int>`: maximum merged segment token estimate.
|
||||||
|
|
||||||
|
Threshold flags:
|
||||||
|
- `--glossary-confidence-threshold <float>`
|
||||||
|
- `--homophones-confidence-threshold <float>`
|
||||||
|
- `--spoken-word-confidence-threshold <float>`
|
||||||
|
- `--grammar-confidence-threshold <float>`
|
||||||
|
|
||||||
|
Context and diagnostics flags:
|
||||||
|
- `--transcript-description <text>`: background context for prompts; does not override transcript content.
|
||||||
|
- `--work-dir <path>`: per-run diagnostics work directory.
|
||||||
|
- `--work-dir-retention <value>`: retention selection; see [configuration](config.md#thresholds-context-and-diagnostics).
|
||||||
|
|
||||||
|
Flag descriptions identify their settings; defaults, accepted keys, numeric
|
||||||
|
constraints, and inheritance rules are owned by [configuration](config.md).
|
||||||
|
Normalization time flags use numeric seconds; LLM timeout flags use integer seconds.
|
||||||
|
|
||||||
|
### `process` Output and Exit Behavior
|
||||||
|
|
||||||
|
- With `--output`: stdout is expected to be empty on success.
|
||||||
|
- Without `--output`: stdout contains transcript JSON only on success.
|
||||||
|
- `--report-json` writes a file and is never printed to stdout.
|
||||||
|
- Stderr is human-readable diagnostics/errors.
|
||||||
|
- Processing failures include the diagnostics directory path when initialization
|
||||||
|
succeeded. Later report-write or retention errors can omit that path.
|
||||||
|
- Output files may already exist when a later reporting or cleanup failure occurs.
|
||||||
|
- Rejected/skipped corrections and warnings can occur in a successful run; inspect
|
||||||
|
the [report format](integrations/output-files.md#process-report) for details.
|
||||||
|
- Diagnostic wording is human-readable and is not a machine-stable JSON channel.
|
||||||
|
|
||||||
|
Exit behavior:
|
||||||
|
- `0`: success.
|
||||||
|
- `1`: runtime failure during processing/reporting/output or retention handling.
|
||||||
|
- `2`: CLI usage or configuration input error.
|
||||||
|
|
||||||
|
Integration references:
|
||||||
|
- subprocess contract: [`docs/integrations/subprocess.md`](integrations/subprocess.md)
|
||||||
|
- transcript/glossary file contract: [`docs/integrations/transcript-glossary-files.md`](integrations/transcript-glossary-files.md)
|
||||||
|
|
||||||
|
### `process` Examples
|
||||||
|
|
||||||
|
Write corrected transcript to a file:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita process transcript.json \
|
||||||
|
--glossary glossary.yaml \
|
||||||
|
--output corrected.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Emit transcript JSON to stdout:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita process transcript.json --glossary glossary.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
Use explicit config and write report JSON:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita process transcript.json \
|
||||||
|
--glossary glossary.yaml \
|
||||||
|
--config audita.yml \
|
||||||
|
--output corrected.json \
|
||||||
|
--report-json report.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Override the module sequence:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita process transcript.json \
|
||||||
|
--glossary glossary.yaml \
|
||||||
|
--modules glossary,homophones,grammar \
|
||||||
|
--output corrected.json
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configuration Commands
|
||||||
|
|
||||||
|
### `config validate`
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita config validate --config examples/minimal-config.yml
|
||||||
|
audita config validate --config examples/production-config.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
The only command-specific flag is `--config <path>`, which is required.
|
||||||
|
Positional arguments are rejected. Success prints `config is valid` to stdout
|
||||||
|
and returns zero; usage, file-read, parse, and validation errors go to stderr and
|
||||||
|
return `2`. See [applied configuration sources](config.md#effective-precedence)
|
||||||
|
and [validation boundaries](config.md#examples).
|
||||||
|
|
||||||
|
### `config print-effective`
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita config print-effective --config examples/minimal-config.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
The only command-specific flag is optional `--config <path>`. Positional
|
||||||
|
arguments are rejected. Without an explicit file, [discovery](config.md#loading-model)
|
||||||
|
applies. Success prints redacted configuration JSON to stdout and returns zero.
|
||||||
|
Usage and configuration errors return `2`; serialization or stdout-write failures
|
||||||
|
return `1`.
|
||||||
|
|
||||||
|
The printed JSON reflects the effective in-memory configuration; it is not a YAML
|
||||||
|
configuration file and is not the process-report schema. Use it to inspect merged
|
||||||
|
settings, not as a file to pass back to the YAML loader.
|
||||||
|
|
||||||
|
## Help
|
||||||
|
|
||||||
|
Root invocation without arguments and root help print usage successfully. Use
|
||||||
|
`--help` or `-h` for subcommand help. Help normally returns zero; process help has
|
||||||
|
the configuration-loading caveat described above. Unknown commands return `2`.
|
||||||
188
docs/config.md
Normal file
188
docs/config.md
Normal file
@@ -0,0 +1,188 @@
|
|||||||
|
# Audita Configuration
|
||||||
|
|
||||||
|
This is the canonical reference for discovery, settings, defaults, environment
|
||||||
|
mapping, and validation. [CLI syntax](cli.md), [artifact lifecycle](operations.md),
|
||||||
|
and [file formats](integrations/output-files.md) have separate owners.
|
||||||
|
|
||||||
|
## Loading Model
|
||||||
|
|
||||||
|
For processing and effective-config inspection, the first applicable source
|
||||||
|
selects one YAML file:
|
||||||
|
|
||||||
|
1. Explicit CLI config path.
|
||||||
|
2. `AUDITA_CONFIG`.
|
||||||
|
3. `/usr/local/etc/audita/config.yml`, if present.
|
||||||
|
4. `/etc/audita/config.yml`, if present.
|
||||||
|
|
||||||
|
These files are alternatives, not layers merged together. Missing system files
|
||||||
|
are skipped. An empty or missing explicitly selected path is an error, as is an
|
||||||
|
inaccessible selected file. Without a selected file, built-in defaults apply.
|
||||||
|
The validation command requires an explicit file and does not use discovery.
|
||||||
|
|
||||||
|
## Effective Precedence
|
||||||
|
|
||||||
|
| Operation | Applied sources, lowest to highest precedence |
|
||||||
|
| --- | --- |
|
||||||
|
| Process | Built-in defaults, selected file, environment overrides, explicit CLI overrides |
|
||||||
|
| Print effective configuration | Built-in defaults, selected file, environment overrides |
|
||||||
|
| Validate configuration file | Built-in defaults, explicit file |
|
||||||
|
|
||||||
|
File validation does not apply configuration environment overrides, but it still
|
||||||
|
resolves environment variables named by `api_key_env` fields. Each configuration
|
||||||
|
layer is validated as it is applied: a later override cannot repair an invalid
|
||||||
|
earlier layer. [Config commands](cli.md#configuration-commands) provide inspection
|
||||||
|
and validation without processing a transcript or contacting an LLM.
|
||||||
|
|
||||||
|
## File Schema
|
||||||
|
|
||||||
|
The file is YAML with required `version: 1`. Unknown fields are rejected.
|
||||||
|
The remaining sections and fields below are optional. Omitted values preserve
|
||||||
|
lower-precedence settings, except for the documented inheritance rules.
|
||||||
|
Complete copyable files are in [examples](#examples).
|
||||||
|
|
||||||
|
### Pipeline and Output
|
||||||
|
|
||||||
|
| YAML field | Default | Meaning |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `pipeline.modules` | `[glossary, homophones, glossary, spoken_word, grammar]` | Ordered module sequence. Supported keys are `glossary`, `homophones`, `spoken_word`, and `grammar`; repeated keys are allowed. An omitted or empty YAML list leaves the sequence unchanged. |
|
||||||
|
| `output.schema` | `bare-segments` | Output encoder selection: `bare-segments` or `audita-v1`. See [output formats](integrations/output-files.md#corrected-transcript). |
|
||||||
|
|
||||||
|
Module keys select correction stages. Validator chains are built into each module;
|
||||||
|
there is no configuration field for selecting individual validator keys.
|
||||||
|
|
||||||
|
### LLM Settings
|
||||||
|
|
||||||
|
The same fields are supported under `llm.proposal` and `llm.validation`.
|
||||||
|
|
||||||
|
| Field suffix | Proposal default | Meaning |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `base_url` | `https://openrouter.ai/api/v1` | Endpoint base URL; the [LLM contract](integrations/openai-compatible-llm.md#endpoint-and-authentication) defines how it is used. |
|
||||||
|
| `model` | `openrouter/google/gemma-4-31b-it` | Model identifier sent to the endpoint. Availability is determined by that endpoint. |
|
||||||
|
| `api_key_env` | Unset | Name of an environment variable containing the API key; see [secrets](#secrets). |
|
||||||
|
| `timeout` | `600` seconds | Positive integer seconds or a duration string resolving to whole seconds, such as `120s`. |
|
||||||
|
| `max_retries` | `3` | Non-negative retry count after the initial attempt; retry conditions belong to the [LLM contract](integrations/openai-compatible-llm.md#retries-and-timeouts). |
|
||||||
|
|
||||||
|
Validation settings inherit the effective proposal settings individually. Empty
|
||||||
|
validation model, base URL, or API key values inherit the proposal value; omitted
|
||||||
|
validation timeout and retry count inherit, while an explicit retry count of zero
|
||||||
|
is retained in configuration. The current HTTP adapter interprets a retry count
|
||||||
|
of zero as its default of three retries, so zero does not disable retries.
|
||||||
|
No API key is required by local config validation. Model and base-URL
|
||||||
|
usability are checked during client setup/requests, not by an online config check.
|
||||||
|
|
||||||
|
### Concurrency
|
||||||
|
|
||||||
|
| YAML field | Default | Meaning |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `concurrency.total_llm` | `1` | Positive total concurrent LLM-call limit. |
|
||||||
|
| `concurrency.proposal_llm` | `1` | Positive proposal-call limit, no greater than total. |
|
||||||
|
| `concurrency.validation_llm` | Inherits total | Positive validation-call limit, no greater than total. |
|
||||||
|
|
||||||
|
In a YAML file, setting total alone does not change proposal concurrency. At the
|
||||||
|
environment or CLI layer, setting total without setting proposal at that same
|
||||||
|
layer also sets proposal to that total, even if a lower layer supplied a proposal
|
||||||
|
value. Unset validation concurrency inherits the final total. The canonical total
|
||||||
|
setting wins over its legacy alias when both appear at the same layer.
|
||||||
|
|
||||||
|
Validation prompt budgeting is available through environment or CLI only:
|
||||||
|
`AUDITA_VALIDATION_MAX_PROMPT_TOKENS` has default `2048` and must be positive.
|
||||||
|
There is no YAML field for it.
|
||||||
|
|
||||||
|
### Chunking and Normalization
|
||||||
|
|
||||||
|
| YAML field | Default | Meaning and validation |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `chunking.target_sections` | Unset | Optional positive target count. Processing rejects targets impossible for the normalized transcript and token constraints. |
|
||||||
|
| `chunking.max_section_tokens` | `8192` | Positive maximum section token estimate. |
|
||||||
|
| `chunking.min_section_tokens` | `2048` | Positive minimum used in chunk balancing; no greater than maximum. |
|
||||||
|
| `normalization.max_segment_gap` | `4.0` seconds | Non-negative gap limit for same-speaker merging. |
|
||||||
|
| `normalization.ellipsis_gap` | `3.5` seconds | Non-negative gap threshold for inserting an ellipsis during merging. |
|
||||||
|
| `normalization.max_segment_duration` | `60.0` seconds | Positive merged-segment duration limit. |
|
||||||
|
| `normalization.max_segment_tokens` | `2048` | Positive merged-segment token estimate limit. |
|
||||||
|
|
||||||
|
The three normalization time fields accept numeric seconds or YAML duration
|
||||||
|
strings. Their environment and CLI overrides use numeric seconds. Section token
|
||||||
|
counts are estimates; config validation does not test whether a particular input
|
||||||
|
can satisfy the selected chunking target.
|
||||||
|
|
||||||
|
### Thresholds, Context, and Diagnostics
|
||||||
|
|
||||||
|
| YAML field | Default | Meaning and validation |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `thresholds.glossary` | `0.8` | Glossary correction confidence threshold, in `[0, 1]`. |
|
||||||
|
| `thresholds.homophones` | `0.8` | Homophone correction confidence threshold, in `[0, 1]`. |
|
||||||
|
| `thresholds.spoken_word` | `0.8` | Spoken-word correction confidence threshold, in `[0, 1]`. |
|
||||||
|
| `thresholds.grammar` | `0.8` | Grammar correction confidence threshold, in `[0, 1]`. |
|
||||||
|
| `context.description` | Empty | Trimmed background context for prompts; limited to 500 bytes of UTF-8 text after trimming. It does not replace transcript content. |
|
||||||
|
| `diagnostics.work_dir` | `/tmp/audita` | Non-empty parent directory for run artifacts. |
|
||||||
|
| `diagnostics.retention` | `auto` | Accepted values: `auto`, `always`, `never`. Exact lifecycle behavior, including the current behavior of `never`, belongs to [operations](operations.md#retention-behavior). |
|
||||||
|
|
||||||
|
## Environment Overrides
|
||||||
|
|
||||||
|
Values are read when the variable is set. Empty values are not generally equivalent
|
||||||
|
to unset values: numeric overrides must parse, and empty strings may clear a
|
||||||
|
setting or fail validation. Numeric environment values use integers or floating
|
||||||
|
point seconds as indicated above, not duration strings.
|
||||||
|
|
||||||
|
| Environment variable | Setting |
|
||||||
|
| --- | --- |
|
||||||
|
| `AUDITA_CONFIG` | Config file selection |
|
||||||
|
| `AUDITA_MODULES` | Module sequence as comma-separated keys |
|
||||||
|
| `AUDITA_LLM_API_KEY` | Direct proposal API key |
|
||||||
|
| `OPENROUTER_API_KEY` | Proposal API key fallback when `AUDITA_LLM_API_KEY` is unset |
|
||||||
|
| `AUDITA_MODEL` | Proposal model |
|
||||||
|
| `AUDITA_BASE_URL` | Proposal endpoint |
|
||||||
|
| `AUDITA_LLM_TIMEOUT_SECONDS` | Proposal timeout |
|
||||||
|
| `AUDITA_MAX_RETRIES` | Proposal retries |
|
||||||
|
| `AUDITA_VALIDATION_LLM_API_KEY` | Direct validation API key |
|
||||||
|
| `AUDITA_VALIDATION_MODEL` | Validation model |
|
||||||
|
| `AUDITA_VALIDATION_BASE_URL` | Validation endpoint |
|
||||||
|
| `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | Validation timeout |
|
||||||
|
| `AUDITA_VALIDATION_MAX_RETRIES` | Validation retries |
|
||||||
|
| `AUDITA_TOTAL_LLM_CONCURRENCY` | Total concurrency |
|
||||||
|
| `AUDITA_LLM_CONCURRENCY` | Legacy total-concurrency alias |
|
||||||
|
| `AUDITA_PROPOSAL_LLM_CONCURRENCY` | Proposal concurrency |
|
||||||
|
| `AUDITA_VALIDATION_LLM_CONCURRENCY` | Validation concurrency |
|
||||||
|
| `AUDITA_VALIDATION_MAX_PROMPT_TOKENS` | Validation prompt budget |
|
||||||
|
| `AUDITA_TARGET_SECTIONS` | Section target |
|
||||||
|
| `AUDITA_MAX_SECTION_TOKENS` | Section maximum |
|
||||||
|
| `AUDITA_MIN_SECTION_TOKENS` | Section minimum |
|
||||||
|
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | Merge gap |
|
||||||
|
| `AUDITA_NORMALIZE_ELLIPSIS_GAP` | Ellipsis gap |
|
||||||
|
| `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | Merged duration limit |
|
||||||
|
| `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | Merged token estimate limit |
|
||||||
|
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | Glossary threshold |
|
||||||
|
| `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | Homophone threshold |
|
||||||
|
| `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | Spoken-word threshold |
|
||||||
|
| `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | Grammar threshold |
|
||||||
|
| `AUDITA_WORK_DIR` | Diagnostics directory |
|
||||||
|
| `AUDITA_WORK_DIR_RETENTION` | Retention selection |
|
||||||
|
|
||||||
|
There is no environment override for output schema or transcript description.
|
||||||
|
Use file configuration or the corresponding [CLI flags](cli.md#process-flag-reference).
|
||||||
|
|
||||||
|
## Secrets
|
||||||
|
|
||||||
|
File config accepts environment variable names through `api_key_env`, not literal
|
||||||
|
API-key fields. Names must be non-empty and match an environment-variable
|
||||||
|
identifier; an unset named variable resolves to an empty value. Runtime environment
|
||||||
|
key overrides take precedence over keys resolved from a file. CLI key flags supply
|
||||||
|
direct values and take precedence over environment settings.
|
||||||
|
|
||||||
|
Effective-config inspection redacts API keys. Configured LLM secrets are also
|
||||||
|
redacted by the runtime's error and diagnostic handling, but transcript content
|
||||||
|
is still sensitive: see [artifact handling](operations.md#artifact-handling).
|
||||||
|
|
||||||
|
## Examples
|
||||||
|
|
||||||
|
- [Minimal configuration](../examples/minimal-config.yml): starts from defaults
|
||||||
|
and obtains the proposal key from the environment.
|
||||||
|
- [Production-oriented configuration](../examples/production-config.yml): illustrates
|
||||||
|
both LLM targets, concurrency, chunking, normalization, thresholds, and diagnostics.
|
||||||
|
Its explicit section target is intended for larger inputs, not the tiny sample.
|
||||||
|
- [Tiny transcript](../examples/tiny-transcript.json) and
|
||||||
|
[tiny glossary](../examples/tiny-glossary.yaml): inputs for the README quickstart.
|
||||||
|
|
||||||
|
Use the [CLI configuration commands](cli.md#configuration-commands) to validate
|
||||||
|
files and inspect effective settings. Validation checks local structure and
|
||||||
|
constraints; it does not verify credentials, model availability, or endpoint access.
|
||||||
@@ -1,149 +0,0 @@
|
|||||||
# Audita Configuration
|
|
||||||
|
|
||||||
## Scope
|
|
||||||
This document defines the supported versioned YAML configuration model and runtime precedence behavior.
|
|
||||||
|
|
||||||
## Supported file version
|
|
||||||
Current supported config file version:
|
|
||||||
- `version: 1`
|
|
||||||
|
|
||||||
Validation rules:
|
|
||||||
- missing `version` fails;
|
|
||||||
- unsupported version fails;
|
|
||||||
- unknown YAML fields fail (strict decoding).
|
|
||||||
|
|
||||||
## Config path resolution
|
|
||||||
For `audita process` and `audita config print-effective`, path resolution order is:
|
|
||||||
1. `--config <path>`
|
|
||||||
2. `AUDITA_CONFIG`
|
|
||||||
3. `/usr/local/etc/audita/config.yml` (if present)
|
|
||||||
4. `/etc/audita/config.yml` (if present)
|
|
||||||
|
|
||||||
Missing-path behavior:
|
|
||||||
- missing `--config` path is an error;
|
|
||||||
- missing `AUDITA_CONFIG` path is an error;
|
|
||||||
- missing both default paths is non-fatal.
|
|
||||||
|
|
||||||
## Effective precedence
|
|
||||||
`audita process` effective precedence:
|
|
||||||
1. defaults
|
|
||||||
2. file config
|
|
||||||
3. environment overrides
|
|
||||||
4. CLI overrides
|
|
||||||
|
|
||||||
`audita config print-effective` uses:
|
|
||||||
1. defaults
|
|
||||||
2. file config
|
|
||||||
3. environment overrides
|
|
||||||
|
|
||||||
`audita config validate` intentionally uses file-only validation:
|
|
||||||
1. defaults
|
|
||||||
2. file config
|
|
||||||
|
|
||||||
Environment overrides are not applied in `config validate`.
|
|
||||||
|
|
||||||
## Supported top-level YAML fields
|
|
||||||
```yaml
|
|
||||||
version: 1
|
|
||||||
|
|
||||||
pipeline:
|
|
||||||
modules: [glossary, homophones, glossary, spoken_word, grammar]
|
|
||||||
|
|
||||||
output:
|
|
||||||
schema: bare-segments
|
|
||||||
|
|
||||||
llm:
|
|
||||||
proposal:
|
|
||||||
base_url: https://openrouter.ai/api/v1
|
|
||||||
model: openrouter/google/gemma-4-31b-it
|
|
||||||
api_key_env: AUDITA_LLM_API_KEY
|
|
||||||
timeout: 120s
|
|
||||||
max_retries: 3
|
|
||||||
validation:
|
|
||||||
base_url: https://openrouter.ai/api/v1
|
|
||||||
model: openrouter/google/gemma-4-31b-it
|
|
||||||
api_key_env: AUDITA_VALIDATION_LLM_API_KEY
|
|
||||||
timeout: 120s
|
|
||||||
max_retries: 3
|
|
||||||
|
|
||||||
concurrency:
|
|
||||||
total_llm: 2
|
|
||||||
proposal_llm: 2
|
|
||||||
validation_llm: 1
|
|
||||||
|
|
||||||
chunking:
|
|
||||||
target_sections: 8
|
|
||||||
max_section_tokens: 8192
|
|
||||||
min_section_tokens: 2048
|
|
||||||
|
|
||||||
normalization:
|
|
||||||
max_segment_gap: 4s
|
|
||||||
ellipsis_gap: 3.5s
|
|
||||||
max_segment_duration: 60s
|
|
||||||
max_segment_tokens: 2048
|
|
||||||
|
|
||||||
thresholds:
|
|
||||||
glossary: 0.8
|
|
||||||
homophones: 0.8
|
|
||||||
spoken_word: 0.8
|
|
||||||
grammar: 0.8
|
|
||||||
|
|
||||||
context:
|
|
||||||
description: "optional transcript background context"
|
|
||||||
|
|
||||||
diagnostics:
|
|
||||||
work_dir: /tmp/audita
|
|
||||||
retention: auto
|
|
||||||
```
|
|
||||||
|
|
||||||
## Module and output-schema validation
|
|
||||||
`pipeline.modules` keys are validated against the built-in supported module catalog.
|
|
||||||
|
|
||||||
Supported module keys:
|
|
||||||
- `glossary`
|
|
||||||
- `homophones`
|
|
||||||
- `spoken_word`
|
|
||||||
- `grammar`
|
|
||||||
|
|
||||||
Repeated supported module keys are allowed.
|
|
||||||
|
|
||||||
`output.schema` is validated against the built-in output schema catalog.
|
|
||||||
|
|
||||||
Supported output schema keys:
|
|
||||||
- `bare-segments`
|
|
||||||
- `audita-v1`
|
|
||||||
|
|
||||||
Unknown module keys and unknown output schema keys fail validation.
|
|
||||||
|
|
||||||
## Duration field parsing
|
|
||||||
Duration-like fields support:
|
|
||||||
- numeric seconds (for example `120`, `3.5`)
|
|
||||||
- duration strings (for example `120s`, `2m`)
|
|
||||||
|
|
||||||
LLM timeout duration strings must resolve to whole seconds.
|
|
||||||
|
|
||||||
## Secret handling
|
|
||||||
Use `api_key_env` fields for secrets:
|
|
||||||
- `llm.proposal.api_key_env`
|
|
||||||
- `llm.validation.api_key_env`
|
|
||||||
|
|
||||||
These fields store environment variable names, not secret values.
|
|
||||||
|
|
||||||
Resolved secret values are redacted from:
|
|
||||||
- `audita config print-effective` output;
|
|
||||||
- diagnostics `effective-config.json`;
|
|
||||||
- report and diagnostics payloads.
|
|
||||||
|
|
||||||
## Commands
|
|
||||||
Validate a file config:
|
|
||||||
```sh
|
|
||||||
audita config validate --config ./audita.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
Print redacted effective config:
|
|
||||||
```sh
|
|
||||||
audita config print-effective --config ./audita.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
## Compatibility notes
|
|
||||||
Legacy compatibility flags and environment aliases remain available where implemented, but the stable configuration surface is the versioned YAML model described above.
|
|
||||||
@@ -1,33 +1,49 @@
|
|||||||
# Audita Development Workflow
|
# Developing Audita
|
||||||
|
|
||||||
## Scope
|
Audita is a Go CLI for transcript polishing. Start with the
|
||||||
This document defines the canonical contributor workflow and engineering conventions for this repository.
|
[README](../README.md) for product orientation and the
|
||||||
|
[internal overview](internal/overview.md) for the implemented component map.
|
||||||
|
|
||||||
## Workflow
|
## Before Changing the Repository
|
||||||
1. Start from a clean understanding of scope and constraints.
|
|
||||||
2. Make focused changes that preserve existing public behavior unless behavior change is explicitly intended.
|
|
||||||
3. Run targeted tests for touched packages.
|
|
||||||
4. Run `go test ./...` before finalizing substantial changes.
|
|
||||||
5. Update affected documentation so it describes current behavior only.
|
|
||||||
|
|
||||||
## Engineering conventions
|
Coding agents must read and follow the [agent policy](policy/agents.md) before
|
||||||
- Keep module packages separate: `glossary`, `homophones`, `spoken_word`, `grammar`.
|
recommending, planning, or implementing changes. Read the
|
||||||
- Prefer narrow shared helpers and catalogs over broad abstractions.
|
[architecture policy](policy/architecture.md) for boundaries and invariants,
|
||||||
- Preserve diagnostics artifact naming and report field contracts unless intentionally changed.
|
then use the task guide below rather than reading every reference indiscriminately.
|
||||||
- Preserve CLI/config precedence semantics unless intentionally changed.
|
|
||||||
- Treat stable validator keys, prompt identifiers, and output-schema keys as contract surfaces.
|
|
||||||
|
|
||||||
## Configuration and runtime expectations
|
Confirm the requested scope and behavior contract, make focused changes in the
|
||||||
- `audita process` precedence is defaults -> file -> env -> CLI.
|
owning component, and update the relevant tests and canonical documentation
|
||||||
- `audita config validate` validates file config merged onto defaults only.
|
when behavior changes. The [testing policy](policy/testing.md) governs test
|
||||||
- `audita config print-effective` includes environment overrides and prints redacted JSON.
|
sufficiency; the [documentation policy](policy/documentation.md) governs content
|
||||||
|
ownership. Commit discipline and agent Git restrictions are in the
|
||||||
|
[agent policy](policy/agents.md#git-commit-policy).
|
||||||
|
|
||||||
## Testing expectations
|
## Baseline Validation
|
||||||
- Add tests for new behavior and for bug fixes.
|
|
||||||
- Keep deterministic fixtures stable.
|
|
||||||
- Do not reduce existing parity, release-fixture, subprocess, or module-specific coverage without equivalent replacement.
|
|
||||||
|
|
||||||
## Commit discipline
|
Use Go 1.24 or newer. From the repository root:
|
||||||
- Keep commits scoped and reviewable.
|
|
||||||
- Avoid mixing unrelated refactors with behavior changes.
|
```sh
|
||||||
- Use clear plain-English commit messages.
|
go build -o ./bin/audita ./cmd/audita
|
||||||
|
go test ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
Run targeted package tests for touched behavior and the complete suite for
|
||||||
|
substantial changes. Documentation-only work normally needs affected-link,
|
||||||
|
example, and whitespace checks; use `git diff --check`. Detailed component
|
||||||
|
validation belongs with the change recipes below.
|
||||||
|
|
||||||
|
## Task-Specific Reading Guide
|
||||||
|
|
||||||
|
| Task | Read |
|
||||||
|
| --- | --- |
|
||||||
|
| Locate components or change orchestration | [Internal overview](internal/overview.md), then [pipeline](internal/pipeline.md) |
|
||||||
|
| Change configuration or CLI behavior | [Configuration contract](config.md), [CLI contract](cli.md), and [configuration/CLI change recipes](internal/cli-config.md) |
|
||||||
|
| Change modules or validators | [Modules](internal/modules.md) or [validators](internal/validators.md); follow their prompt, registry, and test references |
|
||||||
|
| Change LLM transport or concurrency | [LLM integration contract](integrations/openai-compatible-llm.md), [LLM runtime](internal/llm-runtime.md), and [pipeline](internal/pipeline.md) |
|
||||||
|
| Change prompts | [Prompt registry](internal/prompts.md) and the consuming [module](internal/modules.md) or [validator](internal/validators.md) |
|
||||||
|
| Change input/output formats | [Input file contracts](integrations/transcript-glossary-files.md), [output-file contract](integrations/output-files.md), [encoder internals](internal/output-schemas.md), and [configuration](config.md) |
|
||||||
|
| Change reports or diagnostics | [Operations](operations.md), [output-file contract](integrations/output-files.md), [subprocess integration](integrations/subprocess.md), and [diagnostics/reporting internals](internal/diagnostics-reporting.md) |
|
||||||
|
| Change tests | [Testing policy](policy/testing.md) and the affected component's internal documentation |
|
||||||
|
| Change docs or examples | [Documentation policy](policy/documentation.md), the canonical topic owner, and [example validation](internal/cli-config.md#validating-examples) |
|
||||||
|
| Prepare a release | [Release policy](policy/release.md) and [release procedure](release.md) |
|
||||||
|
| Record a significant architectural decision | [ADR lifecycle](policy/documentation.md#architecture-decision-records) and [ADR index](adr/README.md) |
|
||||||
|
|||||||
@@ -1,27 +0,0 @@
|
|||||||
# Documentation Policy
|
|
||||||
|
|
||||||
## Scope
|
|
||||||
This policy defines how project documentation should be authored and maintained.
|
|
||||||
|
|
||||||
## Core rules
|
|
||||||
- Document the current behavior of the codebase.
|
|
||||||
- Remove stale behavior descriptions promptly when code changes.
|
|
||||||
- Do not describe development history in architecture or behavior docs unless a document is explicitly historical.
|
|
||||||
- Do not use architecture or behavior docs as changelogs.
|
|
||||||
- Prefer rewriting stale sections from scratch when substantial behavior or ownership changes occur.
|
|
||||||
|
|
||||||
## Consistency requirements
|
|
||||||
- Keep command examples aligned with current CLI surfaces.
|
|
||||||
- Keep configuration examples aligned with supported fields and precedence.
|
|
||||||
- Keep architecture package ownership descriptions aligned with current code layout.
|
|
||||||
- Keep stable contract identifiers accurate (module keys, validator keys, output-schema keys, report metadata fields).
|
|
||||||
|
|
||||||
## Cross-document expectations
|
|
||||||
- `docs/architecture/*` documents runtime behavior and package ownership.
|
|
||||||
- `docs/configuration.md` documents config schema and precedence.
|
|
||||||
- `docs/development.md` documents contributor workflow and engineering conventions.
|
|
||||||
|
|
||||||
## Review expectations for documentation changes
|
|
||||||
- Verify referenced files and links exist.
|
|
||||||
- Verify examples match current behavior.
|
|
||||||
- Prefer concise, direct language and avoid speculative future claims.
|
|
||||||
@@ -1,96 +0,0 @@
|
|||||||
# Audita Subprocess Operations
|
|
||||||
|
|
||||||
This document describes how parent processes should invoke `audita process` safely in production orchestration.
|
|
||||||
|
|
||||||
## Recommended command form
|
|
||||||
|
|
||||||
Use explicit file outputs for orchestrated runs:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
audita process <transcript.json> \
|
|
||||||
--transcript-description "Brief context that may help resolve ambiguous terms." \
|
|
||||||
--glossary <glossary.yaml> \
|
|
||||||
--output <output-transcript.json> \
|
|
||||||
--report-json <report.json>
|
|
||||||
```
|
|
||||||
|
|
||||||
Additional flags that may be situationally appropriate:
|
|
||||||
- `--config <path>` to select an explicit versioned config file.
|
|
||||||
- `--output-schema <bare-segments|audita-v1>` to select transcript output shape.
|
|
||||||
- `--work-dir <dir>` to control diagnostics location.
|
|
||||||
- `--work-dir-retention <always|auto|never>` to control retained run directories.
|
|
||||||
- `--total-llm-concurrency`, `--proposal-llm-concurrency`, and `--validation-llm-concurrency` when orchestration needs to set explicit LLM throughput controls.
|
|
||||||
- `--modules ...` only when intentionally overriding the default sequence.
|
|
||||||
|
|
||||||
For config-driven orchestration, validate config files in CI/preflight:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
audita config validate --config <path>
|
|
||||||
```
|
|
||||||
|
|
||||||
## Stdout behavior
|
|
||||||
|
|
||||||
- With `--output`: stdout is expected to be empty on success.
|
|
||||||
- Without `--output`: stdout contains transcript JSON only on success.
|
|
||||||
- Report JSON is never written to stdout.
|
|
||||||
|
|
||||||
## Stderr behavior
|
|
||||||
|
|
||||||
- Success path should be quiet or minimal human-readable logs.
|
|
||||||
- Failure path writes concise human-readable errors.
|
|
||||||
- When a diagnostics run directory exists, failure stderr includes its path.
|
|
||||||
- Prompt/response diagnostic payloads are not streamed to stderr.
|
|
||||||
|
|
||||||
## Output file behavior
|
|
||||||
|
|
||||||
- `--output` writes transcript JSON in the selected output schema to the provided path.
|
|
||||||
- Output write failures return nonzero and surface actionable errors.
|
|
||||||
- The command does not silently ignore output write errors.
|
|
||||||
|
|
||||||
## Report JSON behavior
|
|
||||||
|
|
||||||
- `--report-json` writes a machine-readable process report to the requested path.
|
|
||||||
- Run-directory `report.json` is written independently under diagnostics.
|
|
||||||
- Best-effort failure reports are emitted when possible without masking the primary failure.
|
|
||||||
- Report write failures return nonzero with clear stderr messaging.
|
|
||||||
- Report diagnostics metadata references run-directory artifacts including utilization diagnostics and correction ledger paths when available.
|
|
||||||
|
|
||||||
## Diagnostics directory behavior
|
|
||||||
|
|
||||||
- Each run creates (when possible) a per-run diagnostics directory.
|
|
||||||
- Typical artifacts include transcript, normalization, chunking, invocation, effective config, LLM diagnostics, `utilization-diagnostics.json`, `correction-ledger.json`, `report.json`, and `error.log` on failure.
|
|
||||||
- Failed runs retain diagnostics.
|
|
||||||
- Under `auto` retention, successful runs with skipped/rejected corrections are retained; clean successful runs may be removed.
|
|
||||||
|
|
||||||
## Exit codes
|
|
||||||
|
|
||||||
- `0`: success.
|
|
||||||
- Nonzero: failure (input/schema/config/module/LLM/runtime/output/report/diagnostics errors).
|
|
||||||
|
|
||||||
Treat any nonzero as a failed subprocess invocation.
|
|
||||||
|
|
||||||
## Timeout and cancellation
|
|
||||||
|
|
||||||
- Runtime operations propagate context cancellation and request timeouts through LLM/scheduler paths.
|
|
||||||
- On cancellation or timeout, the process exits nonzero and should not hang.
|
|
||||||
- If diagnostics were initialized before failure, failure artifacts remain available for debugging.
|
|
||||||
|
|
||||||
## Secret redaction expectations
|
|
||||||
|
|
||||||
API keys and configured secret values are redacted from:
|
|
||||||
- reports (`--report-json` and run-dir `report.json`);
|
|
||||||
- diagnostics artifacts (including effective config and LLM interaction artifacts);
|
|
||||||
- surfaced adapter/runtime errors;
|
|
||||||
- test fixtures and regression outputs.
|
|
||||||
|
|
||||||
Parent-process logs should still avoid printing raw environment variables.
|
|
||||||
|
|
||||||
## Parent-process pipe guidance
|
|
||||||
|
|
||||||
To avoid deadlocks in orchestrators:
|
|
||||||
- always read both stdout and stderr concurrently when invoking as a subprocess;
|
|
||||||
- prefer file outputs (`--output`, `--report-json`) for machine workflows;
|
|
||||||
- treat stderr as human-readable diagnostics, not structured data;
|
|
||||||
- parse structured results from output/report files.
|
|
||||||
|
|
||||||
For Go callers, prefer `exec.CommandContext` with explicit timeout/cancellation and buffered/streamed readers for both pipes.
|
|
||||||
76
docs/integrations/openai-compatible-llm.md
Normal file
76
docs/integrations/openai-compatible-llm.md
Normal file
@@ -0,0 +1,76 @@
|
|||||||
|
# OpenAI-Compatible LLM Integration
|
||||||
|
|
||||||
|
This document owns the external protocol used by Audita's structured LLM adapter.
|
||||||
|
Settings, inheritance, defaults, and credential supply belong to
|
||||||
|
[configuration](../config.md#llm-settings); implementation and scheduling belong to
|
||||||
|
[LLM runtime internals](../internal/llm-runtime.md).
|
||||||
|
|
||||||
|
## Endpoint and Authentication
|
||||||
|
|
||||||
|
Audita sends an HTTP `POST` to the configured base URL with trailing slashes
|
||||||
|
removed and `/chat/completions` appended. Supply an HTTP or HTTPS base URL that
|
||||||
|
makes this the intended endpoint; the adapter does not require HTTPS in code.
|
||||||
|
Requests have `Content-Type: application/json`. A non-empty configured API key
|
||||||
|
adds `Authorization: Bearer <api_key>`; otherwise that header is omitted.
|
||||||
|
|
||||||
|
The endpoint must support the configured model and structured JSON-schema
|
||||||
|
responses. Local configuration validation does not probe that support.
|
||||||
|
|
||||||
|
## Request Shape
|
||||||
|
|
||||||
|
The payload contains `model`, an array of `messages` with string `role`/`content`,
|
||||||
|
and `response_format`. The structured-output portion has this illustrative shape:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "json_schema",
|
||||||
|
"json_schema": {
|
||||||
|
"name": "correction_set",
|
||||||
|
"strict": true,
|
||||||
|
"schema": {"type": "object"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The actual schema is supplied for the call; the fragment above is not a complete
|
||||||
|
request schema. Runtime calls use `correction_set` and `validator_decision_set`
|
||||||
|
identities. The adapter requires a non-empty resolved model, non-empty message
|
||||||
|
roles/content, a schema name, and valid schema JSON. It sends strict mode to the
|
||||||
|
provider; local typed decoding and proposal/decision checks still apply afterward.
|
||||||
|
|
||||||
|
## Response Handling Contract
|
||||||
|
|
||||||
|
A successful response must be a JSON envelope with at least one `choices` item.
|
||||||
|
Audita reads the first choice's `message.content`, accepting a non-empty string
|
||||||
|
containing JSON or a non-null raw JSON value that can decode into the expected
|
||||||
|
output type. Empty content, missing choices, and incompatible payloads fail.
|
||||||
|
The adapter also reads optional `model` and token-usage metadata.
|
||||||
|
|
||||||
|
There is no streaming protocol or provider SDK dependency in this integration.
|
||||||
|
Audita uses this chat-completions contract, not other provider API shapes.
|
||||||
|
|
||||||
|
## Retries and Timeouts
|
||||||
|
|
||||||
|
The effective adapter retry count is additional attempts after the initial request;
|
||||||
|
see [configuration](../config.md#llm-settings) for the current zero-value behavior.
|
||||||
|
Transport/read errors, malformed response envelopes/content, typed decode failures,
|
||||||
|
HTTP `429`, and server-error responses are retryable. Other non-success HTTP
|
||||||
|
responses and a response missing choices are not retried by the adapter.
|
||||||
|
Cancellation or an expired context stops retrying.
|
||||||
|
|
||||||
|
The configured HTTP timeout applies per request; it is not a whole-process time
|
||||||
|
budget. A parent process can impose its own overall timeout. See
|
||||||
|
[configuration](../config.md#llm-settings) for timeout and retry values and
|
||||||
|
[subprocess guidance](subprocess.md) for parent responsibilities.
|
||||||
|
|
||||||
|
## Error Behavior
|
||||||
|
|
||||||
|
For non-success responses, error detail comes from provider `error.message`, then
|
||||||
|
a top-level `message`, otherwise the HTTP status and response body. Configured
|
||||||
|
keys are redacted from surfaced errors and LLM diagnostic payloads.
|
||||||
|
|
||||||
|
Malformed structured responses can become section/batch warnings and rejected
|
||||||
|
proposals rather than fatal run errors, depending on the call site. Other adapter
|
||||||
|
failures can terminate processing with partial results. Consult the
|
||||||
|
[report contract](output-files.md#module-outcomes) for those outcomes and
|
||||||
|
[operations](../operations.md#artifact-handling) for handling diagnostic contents.
|
||||||
186
docs/integrations/output-files.md
Normal file
186
docs/integrations/output-files.md
Normal file
@@ -0,0 +1,186 @@
|
|||||||
|
# Output Files and Report Formats
|
||||||
|
|
||||||
|
This document owns externally consumed corrected-transcript and report formats,
|
||||||
|
and describes the correction ledger and utilization diagnostics. It documents
|
||||||
|
current serialized shapes without making every debugging artifact a versioned
|
||||||
|
compatibility promise. Physical filenames, placement, retention, and permissions
|
||||||
|
belong to [operations](../operations.md); invocation and output channels belong to
|
||||||
|
the [CLI](../cli.md#process-output-and-exit-behavior).
|
||||||
|
|
||||||
|
## Corrected Transcript
|
||||||
|
|
||||||
|
The [output setting](../config.md#pipeline-and-output) selects the envelope:
|
||||||
|
|
||||||
|
| Encoder | JSON shape |
|
||||||
|
| --- | --- |
|
||||||
|
| `bare-segments` | Top-level array of segment objects |
|
||||||
|
| `audita-v1` | Object with `schema: "audita-v1"`, `version: "v1"`, and `segments` containing the segment array |
|
||||||
|
|
||||||
|
Both emit these segment fields:
|
||||||
|
|
||||||
|
| Field | Type | Meaning |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `id` | Integer | Identifier in the normalized transcript |
|
||||||
|
| `speaker` | String | Speaker label |
|
||||||
|
| `start`, `end` | Numbers | Start/end times in seconds |
|
||||||
|
| `text` | String | Text after normalization and accepted corrections |
|
||||||
|
| `categories` | Array of strings, optional | Segment categories; omitted when empty |
|
||||||
|
|
||||||
|
Input IDs and segment boundaries need not survive normalization unchanged;
|
||||||
|
normalization can merge segments and reassign IDs. The output describes the
|
||||||
|
resulting transcript, not a lossless copy of the source JSON. Input acceptance
|
||||||
|
rules belong to the [input-file contract](transcript-glossary-files.md).
|
||||||
|
|
||||||
|
## Process Report
|
||||||
|
|
||||||
|
The report is a JSON object. `report_metadata` identifies its schema separately
|
||||||
|
from the transcript output envelope:
|
||||||
|
|
||||||
|
| Metadata field | Type / current value |
|
||||||
|
| --- | --- |
|
||||||
|
| `report_schema_name` | String: `audita-process-report` |
|
||||||
|
| `report_schema_version` | String: `v1` |
|
||||||
|
| `output_schema` | String identifying the selected transcript encoder |
|
||||||
|
| `config_version` | Optional integer, present when file configuration was loaded |
|
||||||
|
|
||||||
|
The remaining top-level fields are:
|
||||||
|
|
||||||
|
| Field | Type and meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `phase` | String; currently `default_pipeline` |
|
||||||
|
| `status` | String: `success` or `failed` |
|
||||||
|
| `operation` | String; currently `process` |
|
||||||
|
| `transcript_path`, `glossary_path` | Input path strings |
|
||||||
|
| `output_path` | Optional transcript destination string |
|
||||||
|
| `modules` | Array of configured module keys, including repeats |
|
||||||
|
| `started_at`, `completed_at` | RFC 3339 timestamps, possibly fractional; completion is an optional field populated by completed process runs |
|
||||||
|
| `error_phase`, `error_message` | Optional failure classification and human-readable detail |
|
||||||
|
| `input_segment_count`, `normalized_segment_count` | Optional integer normalization counts |
|
||||||
|
| `normalization_merges`, `normalization_id_reassignments` | Optional integer transformation counts |
|
||||||
|
| `normalization_skipped` | Object of optional integer counters: `different_speakers`, `gap_too_large`, `duration_exceeded`, `token_limit_exceeded`; can be empty |
|
||||||
|
| `chunking` | Optional section summary described below |
|
||||||
|
| `diagnostics` | Optional artifact-path metadata described below |
|
||||||
|
| `modules_summary` | Optional aggregate module outcome |
|
||||||
|
| `module_results` | Optional array of module outcomes |
|
||||||
|
|
||||||
|
Failures can preserve partial module results. Missing summaries indicate unavailable
|
||||||
|
information, not necessarily zero work. Early configuration failures occur before
|
||||||
|
report construction. Report success describes processing results; a later file-write
|
||||||
|
or cleanup error can still cause a nonzero CLI exit. Use the exit result and report
|
||||||
|
together as explained in [subprocess guidance](subprocess.md).
|
||||||
|
|
||||||
|
Error phases identify where processing failed, for example `transcript_read`,
|
||||||
|
`glossary_read`, `transcript_schema`, `glossary_schema`, `chunking`, `runner_setup`,
|
||||||
|
`runner_execution`, `output_schema`, `serialization`, `output_write`, and
|
||||||
|
`stdout_write`. This is not an exhaustive enumeration or a promise that every
|
||||||
|
late failure is represented in a successfully written report. Error messages are
|
||||||
|
human-readable, not structured fields to parse for control flow.
|
||||||
|
|
||||||
|
### Chunking and Diagnostics Metadata
|
||||||
|
|
||||||
|
`chunking` contains integer fields `chunk_count`, `min_estimated_chunk_tokens`,
|
||||||
|
`max_estimated_chunk_tokens`, `total_estimated_transcript_tokens`,
|
||||||
|
`max_section_tokens`, and `min_section_tokens`, plus optional integer
|
||||||
|
`target_sections`. Token counts are estimates.
|
||||||
|
|
||||||
|
`diagnostics` contains optional path strings:
|
||||||
|
|
||||||
|
- `directory_path`
|
||||||
|
- `source_transcript_path`, `parsed_source_transcript_path`, `normalized_transcript_path`
|
||||||
|
- `normalization_summary_path`, `chunking_summary_path`
|
||||||
|
- `utilization_summary_path`, `correction_ledger_path`
|
||||||
|
- `invocation_metadata_path`, `redacted_effective_config_path`, `error_log_path`
|
||||||
|
|
||||||
|
The [operations guide](../operations.md#diagnostics-directory) defines the physical
|
||||||
|
artifacts and path-lifetime caveats.
|
||||||
|
|
||||||
|
### Module Outcomes
|
||||||
|
|
||||||
|
`modules_summary` contains integer `module_count`, `total_applied_changes`, and
|
||||||
|
`total_skipped_changes`, plus optional string `failed_module_instance`.
|
||||||
|
`total_skipped_changes` counts both apply-time skipped changes and validator
|
||||||
|
rejections. Their detailed records are stored in separate arrays.
|
||||||
|
|
||||||
|
Each `module_results` object contains:
|
||||||
|
|
||||||
|
| Field | Type and meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `module_key`, `module_instance` | Strings; the instance distinguishes repeated occurrences of a configured module |
|
||||||
|
| `replacement_policy` | Optional string describing the module's replacement policy |
|
||||||
|
| `status` | String describing module outcome (`success` or `failed`) |
|
||||||
|
| `proposal_count` | Integer |
|
||||||
|
| `started_at`, `completed_at` | Optional timestamps |
|
||||||
|
| `error_message` | Optional failure detail |
|
||||||
|
| `warnings` | Optional array of warning records |
|
||||||
|
| `validator_decisions` | Optional array of decision records |
|
||||||
|
| `validator_rejected` | Optional array of rejected-correction records |
|
||||||
|
| `applied_changes`, `skipped_changes` | Optional arrays of apply outcomes |
|
||||||
|
|
||||||
|
A validator decision has `validator_name` (string), `proposal_index` (integer),
|
||||||
|
`approved` (boolean), `reason_code` and `message` (strings), and optional
|
||||||
|
`diagnostic_artifact_path` (string).
|
||||||
|
|
||||||
|
Rejected corrections include `validator_name`, `proposal_index`, `module_key`,
|
||||||
|
`module_instance`, `target_segment_id`, `original_text`, `corrected_text`,
|
||||||
|
`reason_code`, and `message`. Applied changes include the same correction identity
|
||||||
|
and text fields without validator/reason/message fields, plus integer
|
||||||
|
`replacement_count`. Skipped changes include correction identity/text fields,
|
||||||
|
`skip_reason`, and `message`.
|
||||||
|
|
||||||
|
A warning has string `scope`, `reason_code`, and `message`; optional integer
|
||||||
|
`section_index` and `batch_index`; and optional string `validator_name` and
|
||||||
|
`diagnostic_artifact_path`. Warnings, skipped changes, and rejected corrections
|
||||||
|
can occur without a failed run. Proposal indexes associate outcomes with their
|
||||||
|
proposal within a module instance; they are not transcript segment IDs.
|
||||||
|
|
||||||
|
## Correction Ledger
|
||||||
|
|
||||||
|
The ledger is an array of correction entries; it can be `null` when no module
|
||||||
|
results are available and empty when modules produced no entries. Unlike the
|
||||||
|
process report, it has no separate schema-version envelope.
|
||||||
|
|
||||||
|
Each entry has required `module_key`, `module_instance`, integer `proposal_index`,
|
||||||
|
and `disposition`. The current dispositions are:
|
||||||
|
|
||||||
|
| Disposition | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `applied` | The proposed change was applied |
|
||||||
|
| `skipped` | The proposal reached application but was skipped |
|
||||||
|
| `rejected` | A validator rejected the proposal |
|
||||||
|
| `failed` | Module-level failure record, not necessarily an individual proposal |
|
||||||
|
|
||||||
|
Optional fields are `run_id`, integer `segment_id`, `original_text`,
|
||||||
|
`proposed_corrected_text`, `applied_corrected_text`, `replacement_policy`,
|
||||||
|
`disposition_reason_code`, and `disposition_message`. Optional validator arrays
|
||||||
|
are `deterministic_validator_decisions` and `llm_validator_decisions`; each record
|
||||||
|
contains string `validator_key`, boolean `approved`, string `reason_code`, and
|
||||||
|
optional string `message`.
|
||||||
|
|
||||||
|
Module failure entries can have the zero-value proposal index and no correction
|
||||||
|
text or segment. Do not interpret them as evidence that proposal zero failed.
|
||||||
|
Empty optional values are omitted. The ledger records disposition, not a replay
|
||||||
|
or resume protocol.
|
||||||
|
|
||||||
|
## Utilization Diagnostics
|
||||||
|
|
||||||
|
This unversioned JSON object describes observations from the runtime, not service
|
||||||
|
limits or performance guarantees:
|
||||||
|
|
||||||
|
- `effective_concurrency`: integer `total_llm`, `proposal_llm`, `validation_llm`.
|
||||||
|
- `run_timing`: millisecond integers `run_wall_time_ms`, `scheduler_queue_wait_ms`,
|
||||||
|
`llm_execution_time_ms`, `deterministic_validation_time_ms`; integer
|
||||||
|
`max_in_flight_llm_calls` and `average_in_flight_llm_calls`.
|
||||||
|
- `llm_calls`: integer `total_proposal_calls` and `total_validation_calls`.
|
||||||
|
- Optional `modules`: records with `module_key`, `module_instance`, integer
|
||||||
|
`proposal_llm_calls` and `validation_llm_calls`, and millisecond integer
|
||||||
|
`scheduler_queue_wait_ms`, `llm_execution_time_ms`,
|
||||||
|
`deterministic_validation_time_ms`, and `module_wall_time_ms`.
|
||||||
|
- Optional `validators`: records with `module_key`, `module_instance`,
|
||||||
|
`validator_key`, boolean `llm_backed`, integer `calls`, and millisecond integer
|
||||||
|
`elapsed_ms`.
|
||||||
|
|
||||||
|
Aggregated execution/wait durations can overlap in concurrent work; they need
|
||||||
|
not sum to wall time. The observed average concurrency is serialized as an integer.
|
||||||
|
Other source snapshots, prompt payloads, and diagnostic files are debugging
|
||||||
|
material; their placement is documented in operations without defining a separate
|
||||||
|
public schema for every file.
|
||||||
40
docs/integrations/subprocess.md
Normal file
40
docs/integrations/subprocess.md
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
# Subprocess Integration
|
||||||
|
|
||||||
|
This guide explains how a parent process can invoke Audita and consume its results.
|
||||||
|
The [CLI reference](../cli.md) owns command syntax, streams, and exit semantics;
|
||||||
|
[configuration](../config.md) owns discovery and precedence.
|
||||||
|
|
||||||
|
## Invocation
|
||||||
|
|
||||||
|
Use an argument array with the transcript path and required glossary argument,
|
||||||
|
following the [process examples](../cli.md#process-examples). For machine workflows,
|
||||||
|
select explicit transcript and report destinations rather than depending on the
|
||||||
|
working directory or run-directory retention. Supplying a configuration path also
|
||||||
|
makes file selection predictable; inherited environment overrides still apply.
|
||||||
|
|
||||||
|
Resolve the executable deliberately and set the child's working directory so
|
||||||
|
relative input/output paths and reported artifact paths have a known base.
|
||||||
|
Read stdout and stderr concurrently to avoid pipe blocking, and impose timeout
|
||||||
|
or cancellation in the parent process when needed.
|
||||||
|
|
||||||
|
## Result Handling
|
||||||
|
|
||||||
|
Interpret the exit result according to the
|
||||||
|
[CLI output and exit contract](../cli.md#process-output-and-exit-behavior).
|
||||||
|
Consume transcript/report data through the
|
||||||
|
[output-file contract](output-files.md), and treat stderr as human-readable detail.
|
||||||
|
The report supplements the exit result: it may be absent after early failures,
|
||||||
|
and output may have been written before a later reporting or cleanup failure.
|
||||||
|
|
||||||
|
If processing fails, preserve available evidence and follow
|
||||||
|
[operations recovery](../operations.md#failure-inspection-and-recovery).
|
||||||
|
For reports retained outside the run directory, account for the
|
||||||
|
[artifact lifecycle](../operations.md#retention-behavior) before dereferencing
|
||||||
|
reported paths. Neither the existence of an output file nor a diagnostics path
|
||||||
|
alone establishes successful completion.
|
||||||
|
|
||||||
|
## Related Contracts
|
||||||
|
|
||||||
|
- [Transcript and glossary inputs](transcript-glossary-files.md)
|
||||||
|
- [LLM endpoint compatibility](openai-compatible-llm.md)
|
||||||
|
- [Configuration inspection commands](../cli.md#configuration-commands)
|
||||||
50
docs/integrations/transcript-glossary-files.md
Normal file
50
docs/integrations/transcript-glossary-files.md
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
# Transcript and Glossary Input Files
|
||||||
|
|
||||||
|
This document owns accepted input formats. Use the [CLI reference](../cli.md) for
|
||||||
|
invocation and error exit semantics, [operations](../operations.md) for artifact
|
||||||
|
handling, and [output files](output-files.md) for corrected-transcript serialization.
|
||||||
|
|
||||||
|
## Transcript JSON Contract
|
||||||
|
|
||||||
|
Accepted top-level shapes are an array of segments or an object containing a
|
||||||
|
`segments` array. The array must contain at least one segment. Unknown object
|
||||||
|
fields are ignored by the current parser; an input envelope does not require a
|
||||||
|
schema/version marker.
|
||||||
|
|
||||||
|
| Segment field | Type and validation |
|
||||||
|
| --- | --- |
|
||||||
|
| `id` | Optional integer (or null). Supplied IDs must be unique; missing IDs are accepted. |
|
||||||
|
| `speaker` | Non-empty string |
|
||||||
|
| `start`, `end` | Numeric seconds, finite and non-negative; end must be at least start |
|
||||||
|
| `text` | Non-empty string |
|
||||||
|
| `categories` | Optional array of non-empty strings |
|
||||||
|
|
||||||
|
Supply explicit start and end times for meaningful timing. The current decoder
|
||||||
|
uses zero for missing or null timestamps rather than enforcing their presence.
|
||||||
|
String validation checks empty values, not whitespace trimming. Normalization can
|
||||||
|
merge segments and reassign IDs, so source IDs need not match output IDs.
|
||||||
|
|
||||||
|
The [tiny transcript](../../examples/tiny-transcript.json) is a complete copyable
|
||||||
|
input. Invalid JSON, unsupported top-level shapes, empty transcripts, invalid field
|
||||||
|
types, duplicate supplied IDs, and the validation failures above are rejected.
|
||||||
|
|
||||||
|
## Glossary YAML Contract
|
||||||
|
|
||||||
|
The top-level object contains a `glossary` list with at least one entry.
|
||||||
|
|
||||||
|
| Entry field | Type and validation |
|
||||||
|
| --- | --- |
|
||||||
|
| `name` | Non-empty string |
|
||||||
|
| `category` | Non-empty string |
|
||||||
|
| `summary` | Non-empty string |
|
||||||
|
| `aliases` | Optional list of non-empty strings |
|
||||||
|
| `plural` | Optional string |
|
||||||
|
|
||||||
|
The parser checks empty strings without trimming them. Unknown YAML fields are
|
||||||
|
ignored by the glossary parser, unlike the stricter
|
||||||
|
[configuration-file schema](../config.md#file-schema).
|
||||||
|
|
||||||
|
The [tiny glossary](../../examples/tiny-glossary.yaml) is a complete copyable input.
|
||||||
|
Malformed YAML, an empty/missing glossary, invalid shapes, missing required text,
|
||||||
|
and empty aliases are rejected. These formats describe input acceptance, not
|
||||||
|
additional vocabulary restrictions.
|
||||||
64
docs/internal/cli-config.md
Normal file
64
docs/internal/cli-config.md
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
# Configuration and CLI Change Recipes
|
||||||
|
|
||||||
|
This document owns contributor editing and validation steps for configuration,
|
||||||
|
CLI wiring, and maintained examples. Definitions of behavior belong to the
|
||||||
|
[configuration](../config.md) and [CLI](../cli.md) contracts.
|
||||||
|
|
||||||
|
## Changing Configuration
|
||||||
|
|
||||||
|
1. Update fields, defaults, loading, and validation in `internal/core/config`.
|
||||||
|
2. Update file, environment, and CLI mapping as applicable, including CLI flag
|
||||||
|
extraction. Preserve the source-specific behavior defined in the
|
||||||
|
[configuration contract](../config.md#effective-precedence), including the
|
||||||
|
distinct validation and effective-config commands.
|
||||||
|
3. Update redaction when secrets are involved.
|
||||||
|
4. Review configuration tests and related CLI tests for precedence, validation,
|
||||||
|
and source-specific behavior. Add or adjust tests for the changed contract
|
||||||
|
according to the [testing policy](../policy/testing.md).
|
||||||
|
5. Update the configuration reference and affected [examples](../../examples/).
|
||||||
|
|
||||||
|
## Changing CLI Behavior
|
||||||
|
|
||||||
|
1. Implement parsing and wiring in `internal/cli`.
|
||||||
|
2. Preserve [stream and exit contracts](../cli.md) unless a change is intentional
|
||||||
|
and documented. Review [subprocess integration](../integrations/subprocess.md)
|
||||||
|
when changing output, error, or report handling.
|
||||||
|
3. Review `internal/cli` tests and `cmd/audita` subprocess tests; update coverage
|
||||||
|
for the changed behavior.
|
||||||
|
4. Update the CLI reference and affected integration guidance.
|
||||||
|
|
||||||
|
## Validating Examples
|
||||||
|
|
||||||
|
Keep complete copyable files in [examples](../../examples/) and link to them from
|
||||||
|
their canonical contract documentation. Keep them valid and secret-free.
|
||||||
|
From the repository root, validate the maintained configuration examples:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go run ./cmd/audita config validate --config examples/minimal-config.yml
|
||||||
|
go run ./cmd/audita config validate --config examples/production-config.yml
|
||||||
|
go run ./cmd/audita config print-effective --config examples/minimal-config.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
The effective-config command incorporates environment settings; use a controlled
|
||||||
|
environment when comparing its results. Check affected CLI examples against the
|
||||||
|
parser and use schema tests for maintained transcript/glossary inputs. Example
|
||||||
|
validation should not invoke a live LLM merely to check documentation.
|
||||||
|
|
||||||
|
## Runtime Change Checks
|
||||||
|
|
||||||
|
Select checks according to the [testing policy](../policy/testing.md):
|
||||||
|
|
||||||
|
- Configuration or CLI changes: inspect `internal/core/config` tests,
|
||||||
|
`internal/cli` tests, and `cmd/audita` integration tests.
|
||||||
|
- Process/output changes: verify stdout, stderr, exit, output-file, and report
|
||||||
|
behavior against the [CLI contract](../cli.md) and
|
||||||
|
[subprocess guidance](../integrations/subprocess.md), including failures.
|
||||||
|
- Report or diagnostics changes: review [report assembly tests and redaction
|
||||||
|
checks](diagnostics-reporting.md#key-tests), metadata fields, and artifact paths.
|
||||||
|
- Schema or serialization changes: review [output-schema tests](output-schemas.md#key-tests)
|
||||||
|
and `internal/core/schema` tests, including supported formats and rejection of
|
||||||
|
unsupported schemas.
|
||||||
|
- Changes affecting assembled output: inspect CLI parity and release fixtures.
|
||||||
|
|
||||||
|
Use the [contributor guide](../development.md#baseline-validation) for baseline
|
||||||
|
commands. These recipes supplement, rather than duplicate, public contracts.
|
||||||
74
docs/internal/diagnostics-reporting.md
Normal file
74
docs/internal/diagnostics-reporting.md
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
# Diagnostics and Report Assembly
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This document owns implementation flow and collaborators. Physical artifacts and
|
||||||
|
retention belong to [operations](../operations.md). Serialized reports, ledger
|
||||||
|
records, and utilization shapes belong to the
|
||||||
|
[output-file contract](../integrations/output-files.md).
|
||||||
|
|
||||||
|
## Run Directory and Artifact Writers
|
||||||
|
|
||||||
|
`internal/core/diagnostics` creates run directories, writes snapshots and metadata,
|
||||||
|
and evaluates retention. CLI orchestration supplies the selected configuration,
|
||||||
|
paths, run outcome, and correction-presence information. Artifact names are
|
||||||
|
centralized in diagnostics; their operator-facing inventory is in
|
||||||
|
[operations](../operations.md#diagnostics-directory).
|
||||||
|
|
||||||
|
Snapshot writes are attempted at the relevant processing phases. Several diagnostic
|
||||||
|
writes are best-effort, so report metadata can identify a location without proving
|
||||||
|
that a file was written. The CLI distinguishes primary processing errors from
|
||||||
|
secondary diagnostic/report write errors.
|
||||||
|
|
||||||
|
## Process Report Mapping
|
||||||
|
|
||||||
|
`internal/framework/processreport.Build` maps invocation facts, normalization and
|
||||||
|
chunking summaries, and runner output into `reporting.ProcessReport`. Core reporting
|
||||||
|
owns the Go data structures and JSON writer. The builder attaches diagnostic
|
||||||
|
metadata when a run-directory path is available.
|
||||||
|
|
||||||
|
Module mapping copies outcomes from runner results, collects summary counts, and
|
||||||
|
preserves available partial module results on failure. The CLI failure path does
|
||||||
|
not pass normalization or chunking summaries to report construction even when
|
||||||
|
some earlier processing completed. Consumer interpretation belongs to the
|
||||||
|
[process-report contract](../integrations/output-files.md#process-report).
|
||||||
|
|
||||||
|
## Correction Ledger and Utilization
|
||||||
|
|
||||||
|
`BuildCorrectionLedger` flattens module changes, rejections, and failures into
|
||||||
|
ledger entries. It associates validator records by proposal index within each
|
||||||
|
module, separates execution classes through validator metadata, and sorts entries
|
||||||
|
by module instance, proposal index, and disposition.
|
||||||
|
|
||||||
|
Runner instrumentation collects utilization; CLI orchestration writes it when
|
||||||
|
run output is available. Report assembly does not independently measure calls.
|
||||||
|
For the serialized forms see [ledger](../integrations/output-files.md#correction-ledger)
|
||||||
|
and [utilization](../integrations/output-files.md#utilization-diagnostics).
|
||||||
|
|
||||||
|
## CLI Coordination and Retention
|
||||||
|
|
||||||
|
CLI orchestration writes caller-selected reports and run-local artifacts before
|
||||||
|
asking diagnostics to apply retention. `HasSkippedCorrections` derives the
|
||||||
|
retention input from runner outcomes. Keep the exact lifecycle definition in
|
||||||
|
[operations](../operations.md#retention-behavior), not in another internal table.
|
||||||
|
|
||||||
|
On a processing failure, secondary report-write errors do not replace the original
|
||||||
|
error. On the success path, an external report-write failure returns before the
|
||||||
|
run-local report write; retention errors occur later. This ordering explains why
|
||||||
|
artifacts and process exit status must be assessed together.
|
||||||
|
|
||||||
|
## Redaction
|
||||||
|
|
||||||
|
Effective configuration uses config redaction. LLM diagnostic and error paths
|
||||||
|
use `llm.ConfiguredSecrets` and the runtime's redaction helpers. Keep these separate
|
||||||
|
mechanisms aligned with the [architecture requirement](../policy/architecture.md#errors-and-secret-handling).
|
||||||
|
Operational handling of transcript-bearing artifacts belongs to
|
||||||
|
[operations](../operations.md#artifact-handling).
|
||||||
|
|
||||||
|
## Key Tests
|
||||||
|
|
||||||
|
- `internal/core/diagnostics/*_test.go`
|
||||||
|
- `internal/framework/processreport/*_test.go`
|
||||||
|
- `internal/core/reporting/report_test.go`
|
||||||
|
- `internal/cli/run_test.go`
|
||||||
|
- `cmd/audita/main_integration_test.go`
|
||||||
68
docs/internal/llm-runtime.md
Normal file
68
docs/internal/llm-runtime.md
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
# Audita LLM Runtime
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This document describes the structured LLM runtime and scheduler behavior.
|
||||||
|
|
||||||
|
## Client Boundary
|
||||||
|
|
||||||
|
All runtime LLM calls go through `contracts.StructuredLLMClient`.
|
||||||
|
|
||||||
|
Primary adapter:
|
||||||
|
- `internal/framework/llm/OpenAICompatibleClient`
|
||||||
|
|
||||||
|
## Adapter and Response Schema Registry
|
||||||
|
|
||||||
|
The adapter implements the [LLM wire contract](../integrations/openai-compatible-llm.md).
|
||||||
|
It maps internal structured requests to HTTP messages and decodes provider content
|
||||||
|
into the supplied Go output target. Endpoint, authentication, response shapes, and
|
||||||
|
retry conditions are defined in that contract rather than repeated here.
|
||||||
|
|
||||||
|
`internal/framework/responseschema` stores the per-call response schema definitions
|
||||||
|
and diagnostic metadata. Proposal generation and validation choose their schema
|
||||||
|
before invoking the client. Provider strict mode does not replace local checks.
|
||||||
|
|
||||||
|
## Error Classification
|
||||||
|
|
||||||
|
The adapter marks retryable errors internally and uses context state and attempt
|
||||||
|
count to decide whether to retry. It sanitizes errors before returning them.
|
||||||
|
`internal/framework/structuredoutput` classifies malformed output for proposal
|
||||||
|
and validator call sites, which can turn those errors into warnings and rejected
|
||||||
|
section/batch work. Other errors propagate through the runner.
|
||||||
|
|
||||||
|
## Scheduling and Concurrency
|
||||||
|
|
||||||
|
`internal/framework/llm/Scheduler` provides FIFO, context-aware permit gating.
|
||||||
|
|
||||||
|
CLI orchestration composes global and stage schedulers; runner instrumentation
|
||||||
|
measures them. Limit definitions and inheritance belong to
|
||||||
|
[configuration](../config.md#concurrency).
|
||||||
|
|
||||||
|
Scheduler release is guarded to avoid permit leaks on cancellation/error. Runner
|
||||||
|
collects utilization observations; their serialized shape is defined by the
|
||||||
|
[output-file contract](../integrations/output-files.md#utilization-diagnostics).
|
||||||
|
|
||||||
|
## Diagnostics and Redaction
|
||||||
|
|
||||||
|
`internal/framework/llm/DiagnosticsWriter` writes request/response/error artifacts.
|
||||||
|
|
||||||
|
Configured secrets are derived from `llm.ConfiguredSecrets(cfg)` and redacted from:
|
||||||
|
- diagnostics payloads;
|
||||||
|
- surfaced runtime/adapter errors.
|
||||||
|
|
||||||
|
## Changing LLM Integration or Scheduling
|
||||||
|
|
||||||
|
Review client, scheduler, redaction, and structured-output malformed-response
|
||||||
|
coverage alongside runner orchestration and utilization tests. Use the
|
||||||
|
[testing policy](../policy/testing.md) to select sufficient checks; preserve the
|
||||||
|
[LLM integration contract](../integrations/openai-compatible-llm.md) and
|
||||||
|
[architecture invariants](../policy/architecture.md#correction-and-concurrency-invariants).
|
||||||
|
|
||||||
|
## Key Tests
|
||||||
|
|
||||||
|
- `internal/framework/llm/openai_compatible_client_test.go`
|
||||||
|
- `internal/framework/llm/scheduler_test.go`
|
||||||
|
- `internal/framework/llm/diagnostics_test.go`
|
||||||
|
- `internal/framework/responseschema/registry_test.go`
|
||||||
|
- `internal/framework/structuredoutput/malformed_test.go`
|
||||||
|
- `internal/framework/runner/*_test.go` (orchestration and utilization)
|
||||||
76
docs/internal/modules.md
Normal file
76
docs/internal/modules.md
Normal file
@@ -0,0 +1,76 @@
|
|||||||
|
# Audita Modules
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This document covers internal module contracts and packages. User-selectable keys
|
||||||
|
and the default sequence belong to [configuration](../config.md#pipeline-and-output).
|
||||||
|
|
||||||
|
## Module Contract
|
||||||
|
|
||||||
|
Modules implement `contracts.TranscriptModule`:
|
||||||
|
- `Key()`
|
||||||
|
- `ReplacementPolicy()`
|
||||||
|
- `Validators()`
|
||||||
|
- `Propose(ctx, req)`
|
||||||
|
|
||||||
|
Runner resolves configured module specs to module instances through `internal/framework/modules`.
|
||||||
|
|
||||||
|
## Built-In Modules
|
||||||
|
|
||||||
|
Current module packages:
|
||||||
|
- `internal/modules/glossary`
|
||||||
|
- `internal/modules/homophones`
|
||||||
|
- `internal/modules/spoken_word`
|
||||||
|
- `internal/modules/grammar`
|
||||||
|
|
||||||
|
Current replacement policies:
|
||||||
|
- `glossary`: `replace_all`
|
||||||
|
- `homophones`: `require_unique`
|
||||||
|
- `spoken_word`: `require_unique`
|
||||||
|
- `grammar`: `require_unique`
|
||||||
|
|
||||||
|
## Proposal Generation Ownership
|
||||||
|
|
||||||
|
Shared proposal-generation plumbing is centralized in:
|
||||||
|
- `internal/framework/proposal_generation`
|
||||||
|
|
||||||
|
Module packages own:
|
||||||
|
- prompt selection (`internal/prompts` prompt IDs);
|
||||||
|
- module-specific prompt payload construction.
|
||||||
|
|
||||||
|
Shared prompt helpers live in `internal/framework/promptcontext`.
|
||||||
|
|
||||||
|
## Validator Chain Ownership
|
||||||
|
|
||||||
|
Built-in chains are resolved in `internal/validators` per module key.
|
||||||
|
Module packages call the built-in chain resolver at construction.
|
||||||
|
|
||||||
|
## Failure and Warning Behavior
|
||||||
|
|
||||||
|
Module construction failures propagate to orchestration; execution failures can
|
||||||
|
preserve partial module results. Malformed structured proposal payloads become
|
||||||
|
warnings and section-level rejection. Consumer-visible outcomes are defined by
|
||||||
|
the [report contract](../integrations/output-files.md#module-outcomes).
|
||||||
|
|
||||||
|
## Changing Modules
|
||||||
|
|
||||||
|
1. Add or update the narrowly scoped package under `internal/modules/<module_key>`.
|
||||||
|
2. Keep module-specific prompt ownership in the module and `internal/prompts`;
|
||||||
|
see [prompt rendering and metadata](prompts.md).
|
||||||
|
3. Wire module registration and catalog resolution through framework module
|
||||||
|
registration and the core module catalog. Update selectable keys in the
|
||||||
|
[configuration contract](../config.md) when they change.
|
||||||
|
4. Verify replacement policy and validator-chain selection against the
|
||||||
|
[architecture requirements](../policy/architecture.md#correction-and-concurrency-invariants).
|
||||||
|
5. Review module, prompt-rendering, and proposal-generation tests. Inspect parity
|
||||||
|
and release fixtures when public output changes, choosing sufficient coverage
|
||||||
|
under the [testing policy](../policy/testing.md).
|
||||||
|
6. Update this document for implementation changes and the appropriate public
|
||||||
|
contract owner for externally visible changes.
|
||||||
|
|
||||||
|
## Key Tests
|
||||||
|
|
||||||
|
- `internal/modules/*/module_test.go`
|
||||||
|
- `internal/framework/modules/registry_test.go`
|
||||||
|
- `internal/framework/proposal_generation/*_test.go`
|
||||||
|
- `internal/cli/run_test.go` (pipeline/report integration)
|
||||||
34
docs/internal/output-schemas.md
Normal file
34
docs/internal/output-schemas.md
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
# Output Encoding Internals
|
||||||
|
|
||||||
|
## Scope and Ownership
|
||||||
|
|
||||||
|
`internal/core/outputschema` owns encoder registration and resolution. External
|
||||||
|
JSON envelopes and segment fields are defined by the
|
||||||
|
[output-file contract](../integrations/output-files.md#corrected-transcript);
|
||||||
|
selectable values and defaults belong to
|
||||||
|
[configuration](../config.md#pipeline-and-output).
|
||||||
|
|
||||||
|
## Resolution and Encoding
|
||||||
|
|
||||||
|
Configuration validation checks registry support. CLI orchestration resolves the
|
||||||
|
selected encoder, passes the final transcript to it, and routes the encoded bytes
|
||||||
|
to the destination defined by the [CLI contract](../cli.md#process-output-and-exit-behavior).
|
||||||
|
An unsupported key fails validation or resolution rather than selecting an
|
||||||
|
implicit fallback.
|
||||||
|
|
||||||
|
The registry delegates bare segment serialization to the transcript schema
|
||||||
|
package and uses an envelope encoder for the versioned form. Report construction
|
||||||
|
receives the selected key separately; see [report assembly](diagnostics-reporting.md).
|
||||||
|
|
||||||
|
## Changing an Encoder
|
||||||
|
|
||||||
|
Update registration, configuration validation, and serialization together. Inspect
|
||||||
|
round-trip and shape tests, then update the external output-file contract if the
|
||||||
|
serialized form changes. Keep internal helper changes separate from decisions
|
||||||
|
about externally visible compatibility.
|
||||||
|
|
||||||
|
## Key Tests
|
||||||
|
|
||||||
|
- `internal/core/outputschema/registry_test.go`
|
||||||
|
- `internal/core/config/config_test.go`
|
||||||
|
- `internal/cli/run_test.go`
|
||||||
94
docs/internal/overview.md
Normal file
94
docs/internal/overview.md
Normal file
@@ -0,0 +1,94 @@
|
|||||||
|
# Audita Internal Overview
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This document owns the implemented component map. For normative boundaries, see
|
||||||
|
[architecture policy](../policy/architecture.md); for task-specific reading and
|
||||||
|
baseline commands, see the [contributor guide](../development.md).
|
||||||
|
|
||||||
|
## Package Map
|
||||||
|
|
||||||
|
Executable and command orchestration:
|
||||||
|
- `cmd/audita`: executable entry point.
|
||||||
|
- `internal/cli`: argument parsing, config commands, streams, filesystem wiring, and process orchestration.
|
||||||
|
|
||||||
|
Core deterministic components:
|
||||||
|
- `internal/core/config`
|
||||||
|
- `internal/core/schema`
|
||||||
|
- `internal/core/normalization`
|
||||||
|
- `internal/core/chunking`
|
||||||
|
- `internal/core/outputschema`
|
||||||
|
- `internal/core/diagnostics`
|
||||||
|
- `internal/core/reporting`
|
||||||
|
- `internal/core/modulecatalog`
|
||||||
|
|
||||||
|
Framework orchestration and contracts:
|
||||||
|
- `internal/framework/contracts`
|
||||||
|
- `internal/framework/modules`
|
||||||
|
- `internal/framework/proposal_generation`
|
||||||
|
- `internal/framework/proposals`
|
||||||
|
- `internal/framework/runner`
|
||||||
|
- `internal/framework/validators`
|
||||||
|
- `internal/framework/llm`
|
||||||
|
- `internal/framework/responseschema`
|
||||||
|
- `internal/framework/structuredoutput`
|
||||||
|
- `internal/framework/processreport`
|
||||||
|
- `internal/framework/promptcontext`
|
||||||
|
- `internal/framework/stagename`
|
||||||
|
|
||||||
|
Domain implementations:
|
||||||
|
- `internal/modules/*`
|
||||||
|
- `internal/validators/*`
|
||||||
|
- `internal/prompts`
|
||||||
|
|
||||||
|
Supporting areas:
|
||||||
|
- `docs/`: canonical documentation.
|
||||||
|
- `examples/`: maintained copyable inputs and configuration.
|
||||||
|
|
||||||
|
## Current Scope
|
||||||
|
|
||||||
|
Runtime state is held in memory plus per-run diagnostics artifacts. Audita does
|
||||||
|
not currently implement a plugin framework, generic workflow engine,
|
||||||
|
resume/checkpoint/manifest system, remote storage, or multi-process service mode.
|
||||||
|
Its provider adapter is OpenAI-compatible HTTP behind the structured LLM client
|
||||||
|
contract. These are current implementation boundaries, not bans on future work.
|
||||||
|
|
||||||
|
## Main Execution Path (`audita process`)
|
||||||
|
|
||||||
|
The CLI prepares input and configuration, invokes the runner, and writes results.
|
||||||
|
The [pipeline](pipeline.md) owns detailed state transitions; the
|
||||||
|
[operations guide](../operations.md) owns the operator workflow and physical
|
||||||
|
artifact lifecycle.
|
||||||
|
|
||||||
|
## Component Responsibilities
|
||||||
|
|
||||||
|
Core components implement configuration defaults/loading/validation/redaction,
|
||||||
|
transcript and glossary schemas, normalization and chunking, output encoding,
|
||||||
|
diagnostics artifact helpers, and public report structures.
|
||||||
|
|
||||||
|
Framework components implement module and validator contracts, proposal
|
||||||
|
generation and application, prompt context, runner orchestration, LLM scheduling
|
||||||
|
and transport, response schemas, and process-report/correction-ledger assembly.
|
||||||
|
Domain packages provide correction modules, built-in validator chains and
|
||||||
|
execution metadata, and embedded prompt assets.
|
||||||
|
|
||||||
|
These describe the implementation; dependency and ownership requirements belong
|
||||||
|
to the [architecture policy](../policy/architecture.md#ownership-and-dependency-direction).
|
||||||
|
|
||||||
|
## Change Recipes
|
||||||
|
|
||||||
|
- [Configuration, CLI, and example changes](cli-config.md)
|
||||||
|
- [Module changes](modules.md#changing-modules)
|
||||||
|
- [Validator changes](validators.md#changing-validators)
|
||||||
|
- Runtime orchestration changes: use [pipeline](pipeline.md), [LLM runtime](llm-runtime.md),
|
||||||
|
or [diagnostics/reporting](diagnostics-reporting.md) and their test references.
|
||||||
|
|
||||||
|
## Related Internal Docs
|
||||||
|
|
||||||
|
- [`docs/internal/pipeline.md`](pipeline.md)
|
||||||
|
- [`docs/internal/modules.md`](modules.md)
|
||||||
|
- [`docs/internal/validators.md`](validators.md)
|
||||||
|
- [`docs/internal/llm-runtime.md`](llm-runtime.md)
|
||||||
|
- [`docs/internal/diagnostics-reporting.md`](diagnostics-reporting.md)
|
||||||
|
- [`docs/internal/prompts.md`](prompts.md)
|
||||||
|
- [`docs/internal/output-schemas.md`](output-schemas.md)
|
||||||
66
docs/internal/pipeline.md
Normal file
66
docs/internal/pipeline.md
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
# Audita Internal Pipeline
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This document describes processing collaborators and state transitions. External
|
||||||
|
input definitions belong to the [input-file contract](../integrations/transcript-glossary-files.md),
|
||||||
|
settings to [configuration](../config.md), and serialized results to the
|
||||||
|
[output-file contract](../integrations/output-files.md).
|
||||||
|
|
||||||
|
## Inputs
|
||||||
|
|
||||||
|
Pipeline inputs are:
|
||||||
|
- effective config (`internal/core/config`);
|
||||||
|
- transcript JSON (`internal/core/schema`);
|
||||||
|
- glossary YAML (`internal/core/schema`).
|
||||||
|
|
||||||
|
## Pipeline Phases
|
||||||
|
|
||||||
|
1. **Input loading:** CLI filesystem adapters read inputs; schema parsing produces
|
||||||
|
source transcript and glossary objects. Source segments are converted to the
|
||||||
|
canonical transcript shape before normalization.
|
||||||
|
2. **Normalization:** the normalizer sorts and merges segments using its configured
|
||||||
|
time/token rules, reassigns IDs, and produces summary counters.
|
||||||
|
3. **Chunking:** the chunker partitions the normalized transcript and returns
|
||||||
|
sections and token estimates, or fails when constraints are impossible.
|
||||||
|
4. **Proposal generation:** the runner executes configured module instances in
|
||||||
|
sequence. Sections can generate proposals concurrently; results are collected
|
||||||
|
for deterministic indexing.
|
||||||
|
5. **Validation:** deterministic validators run before LLM-backed validators.
|
||||||
|
LLM validation is batched by prompt budget; candidate decisions feed application.
|
||||||
|
6. **Application:** `internal/framework/proposals` clones transcript state and
|
||||||
|
applies approved proposals in index order through replacement-policy checks.
|
||||||
|
7. **Serialization and reporting:** the CLI encodes the final transcript, maps
|
||||||
|
runner output into reports, writes artifacts, and delegates retention to
|
||||||
|
diagnostics. See [report assembly](diagnostics-reporting.md).
|
||||||
|
|
||||||
|
## Runner Outputs
|
||||||
|
|
||||||
|
`runner.Run` returns:
|
||||||
|
- final transcript;
|
||||||
|
- per-module results;
|
||||||
|
- utilization diagnostics.
|
||||||
|
|
||||||
|
CLI/reporting then map this into process report and diagnostics artifacts.
|
||||||
|
|
||||||
|
## Failure Behavior
|
||||||
|
|
||||||
|
CLI orchestration labels failures at the phase boundary. Runner execution failures
|
||||||
|
can return completed or partial module results, which report assembly preserves.
|
||||||
|
[Report fields](../integrations/output-files.md#process-report) define how consumers
|
||||||
|
see these outcomes; [CLI semantics](../cli.md#process-output-and-exit-behavior)
|
||||||
|
define streams and exits. Diagnostic write failures do not always abort processing.
|
||||||
|
|
||||||
|
## Ordering and State
|
||||||
|
|
||||||
|
The runner mutates its working transcript between module instances, after collecting
|
||||||
|
section results and validating proposals. Proposal indexes follow section order;
|
||||||
|
reports are derived from actual run results. Global requirements are owned by the
|
||||||
|
[architecture policy](../policy/architecture.md#correction-and-concurrency-invariants).
|
||||||
|
|
||||||
|
## Key Tests
|
||||||
|
|
||||||
|
- `internal/framework/runner/runner_test.go`
|
||||||
|
- `internal/framework/proposal_generation/*_test.go`
|
||||||
|
- `internal/cli/run_test.go`
|
||||||
|
- `cmd/audita/main_integration_test.go`
|
||||||
62
docs/internal/prompts.md
Normal file
62
docs/internal/prompts.md
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
# Audita Prompt Registry
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This document describes embedded prompt assets, prompt metadata, and rendering behavior.
|
||||||
|
|
||||||
|
## Registry Ownership
|
||||||
|
|
||||||
|
Prompt registry lives in `internal/prompts` and embeds assets under `internal/prompts/assets/**`.
|
||||||
|
|
||||||
|
Registered prompt IDs:
|
||||||
|
- `modules.glossary.proposal`
|
||||||
|
- `modules.homophones.proposal`
|
||||||
|
- `modules.spoken_word.proposal`
|
||||||
|
- `modules.grammar.proposal`
|
||||||
|
- `validators.spoken_form_plausibility`
|
||||||
|
- `validators.meaning_reversal_review`
|
||||||
|
- `validators.editorial_review`
|
||||||
|
- `validators.grammar_review`
|
||||||
|
- `validators.spoken_word_review`
|
||||||
|
|
||||||
|
## Metadata Model
|
||||||
|
|
||||||
|
Each prompt has metadata:
|
||||||
|
- `prompt_id`
|
||||||
|
- `prompt_version`
|
||||||
|
- `prompt_source`
|
||||||
|
- `embedded_path`
|
||||||
|
- `sha256`
|
||||||
|
|
||||||
|
Current source/version values:
|
||||||
|
- `prompt_source = builtin`
|
||||||
|
- `prompt_version = v1`
|
||||||
|
|
||||||
|
## Rendering
|
||||||
|
|
||||||
|
`prompts.RenderUserSystem(promptID, data)` renders system/user templates.
|
||||||
|
|
||||||
|
Template behavior:
|
||||||
|
- uses Go `text/template`;
|
||||||
|
- `missingkey=error` is enabled;
|
||||||
|
- output is trimmed.
|
||||||
|
|
||||||
|
A shared hardening fragment is embedded once and referenced by prompt templates.
|
||||||
|
|
||||||
|
## Prompt Context Inputs
|
||||||
|
|
||||||
|
Shared prompt payload helpers:
|
||||||
|
- transcript section JSON (`internal/framework/promptcontext/MarshalTranscriptSectionJSON`)
|
||||||
|
- transcript description block (`TranscriptDescriptionBlock`)
|
||||||
|
|
||||||
|
Modules and LLM validators provide typed data maps to render prompt assets.
|
||||||
|
|
||||||
|
## Diagnostics Integration
|
||||||
|
|
||||||
|
Prompt metadata is attached to proposal/validator diagnostics request metadata using `Metadata.DiagnosticsMap()`.
|
||||||
|
|
||||||
|
## Key Tests
|
||||||
|
|
||||||
|
- `internal/prompts/registry_test.go`
|
||||||
|
- `internal/framework/promptcontext/*_test.go`
|
||||||
|
- module and validator prompt builder tests
|
||||||
87
docs/internal/validators.md
Normal file
87
docs/internal/validators.md
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
# Audita Validators
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This document describes internal validator composition, execution order, and
|
||||||
|
decision handling. These registry keys identify implementation components; they
|
||||||
|
are not independently selectable through [configuration](../config.md#pipeline-and-output).
|
||||||
|
|
||||||
|
## Ownership
|
||||||
|
|
||||||
|
Built-in validator keys and chains:
|
||||||
|
- `internal/validators`
|
||||||
|
|
||||||
|
Shared validator runtime mechanics:
|
||||||
|
- `internal/framework/validators`
|
||||||
|
|
||||||
|
Execution-class metadata:
|
||||||
|
- `internal/validators/metadata`
|
||||||
|
|
||||||
|
## Built-In Validator Keys
|
||||||
|
|
||||||
|
Deterministic:
|
||||||
|
- `proposal_shape`
|
||||||
|
- `confidence_threshold`
|
||||||
|
- `original_text_presence`
|
||||||
|
- `non_empty_corrected_text`
|
||||||
|
- `no_effect`
|
||||||
|
- `protected_terms`
|
||||||
|
|
||||||
|
LLM-backed:
|
||||||
|
- `spoken_form_plausibility`
|
||||||
|
- `meaning_reversal_review`
|
||||||
|
- `editorial_review`
|
||||||
|
|
||||||
|
## Built-In Chains
|
||||||
|
|
||||||
|
Module chains are defined in `internal/validators/chains.go`.
|
||||||
|
Glossary, homophones, spoken_word, and grammar each resolve a fixed ordered chain.
|
||||||
|
|
||||||
|
## Runtime Execution
|
||||||
|
|
||||||
|
For each module section:
|
||||||
|
1. run deterministic validators;
|
||||||
|
2. run LLM-backed validators;
|
||||||
|
3. record decisions and warnings;
|
||||||
|
4. carry only approved proposals forward.
|
||||||
|
|
||||||
|
Decision cardinality is enforced: each candidate proposal must receive exactly one decision per validator.
|
||||||
|
|
||||||
|
## LLM Validator Batching
|
||||||
|
|
||||||
|
LLM validators:
|
||||||
|
- build canonical validation request payloads;
|
||||||
|
- batch using the [configured prompt budget](../config.md#concurrency);
|
||||||
|
- call structured LLM client using response schema registry.
|
||||||
|
|
||||||
|
Oversized single proposals are rejected with `validator_input_too_large`.
|
||||||
|
Malformed LLM validator responses are downgraded to warnings and rejected batch decisions.
|
||||||
|
|
||||||
|
## Decision and Rejection Reporting
|
||||||
|
|
||||||
|
Runner collects decisions, rejected candidates, and warning records. Ledger
|
||||||
|
assembly groups decisions by deterministic versus LLM-backed execution metadata.
|
||||||
|
The serialized names and fields belong to the
|
||||||
|
[report and ledger contract](../integrations/output-files.md); implementation mapping
|
||||||
|
is covered by [report assembly](diagnostics-reporting.md).
|
||||||
|
|
||||||
|
## Changing Validators
|
||||||
|
|
||||||
|
1. Implement validator behavior under `internal/validators/<validator_key>`;
|
||||||
|
update registry and chain wiring in `internal/validators`.
|
||||||
|
2. Put shared runtime mechanics in `internal/framework/validators` only when
|
||||||
|
needed. Keep deterministic versus LLM-backed execution metadata explicit.
|
||||||
|
3. Preserve exposed identities and decision semantics under the
|
||||||
|
[architecture policy](../policy/architecture.md#correction-and-concurrency-invariants).
|
||||||
|
4. Review validator, registry, chain, batching, malformed-output, diagnostics,
|
||||||
|
and decision-cardinality tests; update meaningful coverage under the
|
||||||
|
[testing policy](../policy/testing.md).
|
||||||
|
5. Update implementation guidance here and any affected external contract in its
|
||||||
|
[canonical owner](../policy/documentation.md#canonical-ownership).
|
||||||
|
|
||||||
|
## Key Tests
|
||||||
|
|
||||||
|
- `internal/validators/*_test.go`
|
||||||
|
- `internal/framework/validators/*_test.go`
|
||||||
|
- `internal/framework/processreport/correction_ledger_test.go`
|
||||||
|
- `internal/cli/run_test.go`
|
||||||
110
docs/operations.md
Normal file
110
docs/operations.md
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
# Audita Operations
|
||||||
|
|
||||||
|
This document owns runtime workflow, physical artifact placement, retention, and
|
||||||
|
recovery. Use the [CLI reference](cli.md) for invocation and stream/exit semantics,
|
||||||
|
[configuration](config.md) for settings, and [output-file contracts](integrations/output-files.md)
|
||||||
|
for logical formats.
|
||||||
|
|
||||||
|
## Normal Workflow
|
||||||
|
|
||||||
|
1. Choose a configuration and validate it using the
|
||||||
|
[configuration commands](cli.md#configuration-commands).
|
||||||
|
2. Prepare input files conforming to the [input contract](integrations/transcript-glossary-files.md).
|
||||||
|
3. Run the [processing command](cli.md#process-examples), choosing explicit output
|
||||||
|
and report destinations when those files must survive run-directory cleanup.
|
||||||
|
4. Inspect the corrected transcript and process report. Review rejected or skipped
|
||||||
|
corrections when the report indicates them; successful execution does not mean
|
||||||
|
every proposed correction was applied.
|
||||||
|
|
||||||
|
Processing creates a run directory, loads and normalizes the inputs, chunks the
|
||||||
|
transcript, executes the correction pipeline, writes results, and applies
|
||||||
|
retention. Early configuration failures happen before run-directory creation.
|
||||||
|
Failures after initialization attempt to preserve diagnostic evidence; individual
|
||||||
|
artifact writes can fail, so a run directory need not contain every listed file.
|
||||||
|
|
||||||
|
## Output Files
|
||||||
|
|
||||||
|
The caller chooses transcript and optional external report destinations using the
|
||||||
|
[CLI](cli.md#process-output-and-exit-behavior). These are separate from the per-run
|
||||||
|
diagnostic directory unless the caller deliberately places them inside it.
|
||||||
|
Choose existing, writable parent directories. Output/report files can be overwritten
|
||||||
|
by subsequent runs using the same destinations.
|
||||||
|
|
||||||
|
A later report or cleanup failure can occur after transcript output was written.
|
||||||
|
Inspect the process result and available report before treating the files as a
|
||||||
|
completed run. Retention of the diagnostic directory does not archive caller-selected
|
||||||
|
output files elsewhere.
|
||||||
|
|
||||||
|
## Diagnostics Directory
|
||||||
|
|
||||||
|
The [diagnostics configuration](config.md#thresholds-context-and-diagnostics)
|
||||||
|
selects the parent work directory. Runs create children named like
|
||||||
|
`run-<unix-nanoseconds>`. The runtime creates the work directory if needed.
|
||||||
|
Report paths may be relative when the supplied work directory is relative;
|
||||||
|
resolve them using the invocation's working directory.
|
||||||
|
|
||||||
|
| Artifact in the run directory | Purpose |
|
||||||
|
| --- | --- |
|
||||||
|
| `source-transcript.json` | Original transcript input |
|
||||||
|
| `source-transcript-parsed.json` | Parsed source representation |
|
||||||
|
| `normalized-transcript.json` | Transcript after normalization |
|
||||||
|
| `normalization-summary.json` | Normalization counters |
|
||||||
|
| `chunking-summary.json` | Section/token summary |
|
||||||
|
| `invocation.json` | Invocation metadata |
|
||||||
|
| `effective-config.json` | Redacted configuration snapshot |
|
||||||
|
| `report.json` | Run's [process report](integrations/output-files.md#process-report) |
|
||||||
|
| `correction-ledger.json` | [Correction dispositions](integrations/output-files.md#correction-ledger) |
|
||||||
|
| `utilization-diagnostics.json` | [Concurrency and timing observations](integrations/output-files.md#utilization-diagnostics) |
|
||||||
|
| `error.log` | Failure details when written |
|
||||||
|
|
||||||
|
LLM request, response, and error diagnostics are also written within the run
|
||||||
|
directory. Their nested naming and payloads are debugging details. Prefer paths
|
||||||
|
provided by the report and its decision/warning records over reconstructing those
|
||||||
|
paths. The report describes intended artifact locations, not proof that every
|
||||||
|
write succeeded or that the files still exist after retention.
|
||||||
|
|
||||||
|
## Retention Behavior
|
||||||
|
|
||||||
|
[Configuration](config.md#thresholds-context-and-diagnostics) owns accepted
|
||||||
|
retention values and the default. Current lifecycle behavior is:
|
||||||
|
|
||||||
|
| Run outcome / selection | Directory outcome |
|
||||||
|
| --- | --- |
|
||||||
|
| Failed run | Retained regardless of selection |
|
||||||
|
| Successful run, `always` | Retained |
|
||||||
|
| Successful run, `auto` | Retained if corrections were skipped or rejected; otherwise removed |
|
||||||
|
| Successful run, `never` | Currently retained, the same outcome as `always` |
|
||||||
|
|
||||||
|
The name `never` does not currently mean deletion. Warnings alone do not trigger
|
||||||
|
retention under `auto` unless they also produce skipped or rejected corrections.
|
||||||
|
An external report outside the run directory survives automatic cleanup, but its
|
||||||
|
artifact paths can then point to removed files.
|
||||||
|
|
||||||
|
## Artifact Handling
|
||||||
|
|
||||||
|
Run artifacts can contain full transcripts, glossary-related context, and LLM
|
||||||
|
prompts/responses. Configured-key redaction does not anonymize those contents.
|
||||||
|
Choose a work directory whose ownership and permissions fit the transcript data,
|
||||||
|
and retain or share artifacts intentionally. Credential-supply mechanisms are
|
||||||
|
owned by [configuration](config.md#secrets).
|
||||||
|
|
||||||
|
For a debugging run, select retained diagnostics using the
|
||||||
|
[retention flag](cli.md#process-flag-reference). Copy needed artifacts before
|
||||||
|
removing completed run directories manually. Avoid deleting a directory while
|
||||||
|
its run is active. Audita does not provide a separate archival/backup or remote
|
||||||
|
artifact storage facility.
|
||||||
|
|
||||||
|
## Failure Inspection and Recovery
|
||||||
|
|
||||||
|
1. Read stderr and the exit classification defined by the [CLI](cli.md#process-output-and-exit-behavior).
|
||||||
|
2. Locate the external report, or the run directory identified by the error when
|
||||||
|
available. If a late error omits the path, inspect the selected work directory.
|
||||||
|
3. Read the available report and error log. Use
|
||||||
|
[report status and error fields](integrations/output-files.md#process-report)
|
||||||
|
to identify the failed phase, then inspect referenced artifacts.
|
||||||
|
4. Correct the concrete input, configuration, endpoint, or filesystem issue and
|
||||||
|
rerun the command with diagnostics retained if further investigation is needed.
|
||||||
|
|
||||||
|
Each invocation starts a new run. There is no implemented resume/checkpoint API;
|
||||||
|
recovery is a rerun, not continuation from an artifact. Failures during early
|
||||||
|
configuration or directory creation may leave no report or diagnostic files.
|
||||||
103
docs/policy/agents.md
Normal file
103
docs/policy/agents.md
Normal file
@@ -0,0 +1,103 @@
|
|||||||
|
# Agent Policy
|
||||||
|
|
||||||
|
**Base Policy Version:** 1.0
|
||||||
|
|
||||||
|
**Last Updated**: 2026-09-23
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Agents working in this repository must read and adhere to the following house rules.
|
||||||
|
|
||||||
|
## Implementation Discipline
|
||||||
|
|
||||||
|
**KEY PRINCIPLES:** Write the simplest idiomatic code that satisfies the user's
|
||||||
|
request. Do not implicitly strengthen requirements, restrictions, or guarantees
|
||||||
|
beyond what the user actually requested.
|
||||||
|
|
||||||
|
**Keep implementation machinery proportionate.** Use straightforward,
|
||||||
|
idiomatic code and introduce only the machinery needed to support the requested
|
||||||
|
behavior and flexibility. Optimize for conceptual simplicity and
|
||||||
|
maintainability, not for the fewest lines, fewest options, or narrowest product
|
||||||
|
surface.
|
||||||
|
|
||||||
|
**Do not invent restrictions.** Do not reject, prohibit, constrain, cap,
|
||||||
|
harden, or otherwise narrow behavior unless required by the user, an explicit
|
||||||
|
repository contract, or an unavoidable correctness constraint. Preserve
|
||||||
|
operator and caller choice where policy has not been established. If an
|
||||||
|
additional restriction appears advisable, present it as a decision rather than
|
||||||
|
silently implementing it.
|
||||||
|
|
||||||
|
**Do not invent guarantees.** Do not strengthen public contracts, guarantees,
|
||||||
|
or invariants beyond those expressly requested by the user.
|
||||||
|
|
||||||
|
**Use the standard library.** For example: in almost every case, the best
|
||||||
|
approach to copying files is to use the standard library. Writing custom code to
|
||||||
|
implement power-loss protection at the application level, or to guard against
|
||||||
|
bad actors with privileged and concurrent access to the host's filesystem,
|
||||||
|
is wasteful, unnecessary, and not idiomatic. Stick to the simple, standard, and
|
||||||
|
idiomatic approach unless another path is expressly requested by the user.
|
||||||
|
|
||||||
|
**An implementation detail is not repository policy.** Tests and documentation
|
||||||
|
must not elevate incidental behavior into a durable guarantee or restriction
|
||||||
|
unless that behavior was expressly selected as part of the contract.
|
||||||
|
|
||||||
|
## Codebase Discovery
|
||||||
|
|
||||||
|
This project uses codebase-memory-mcp to maintain a knowledge graph of the
|
||||||
|
codebase. Always prefer its graph tools over grep, glob, or file search for code discovery.
|
||||||
|
If the project is not indexed, run `index_repository` first.
|
||||||
|
|
||||||
|
Use this order:
|
||||||
|
|
||||||
|
1. `search_graph` to find functions, classes, routes, and variables by pattern.
|
||||||
|
2. `trace_path` to trace callers and callees.
|
||||||
|
3. `get_code_snippet` to read a specific function or class.
|
||||||
|
4. `query_graph` for complex graph queries.
|
||||||
|
5. `get_architecture` for the high-level project summary.
|
||||||
|
|
||||||
|
Fall back to grep or glob for string literals, error messages, configuration
|
||||||
|
values, non-code files, or when the graph tools return insufficient results.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
- Find a handler: `search_graph(name_pattern=".*OrderHandler.*")`.
|
||||||
|
- Find callers: `trace_path(function_name="OrderHandler", direction="inbound")`.
|
||||||
|
- Read source: `get_code_snippet(qualified_name="pkg/orders.OrderHandler")`.
|
||||||
|
|
||||||
|
## Git Commit Policy
|
||||||
|
|
||||||
|
This policy applies to all coding agents and agent harnesses working in this
|
||||||
|
repository, not only Codex.
|
||||||
|
|
||||||
|
By default, create a commit after completing each coherent unit of work, using a
|
||||||
|
concise, plain-English message describing what changed. This policy is standing
|
||||||
|
authorization to create those local commits; a separate user request is not
|
||||||
|
required. An explicit user instruction to leave work uncommitted takes precedence.
|
||||||
|
|
||||||
|
A unit of work is a focused, reviewable outcome, including its relevant tests and
|
||||||
|
documentation. Do not commit after every intermediate edit or create empty commits.
|
||||||
|
Complete the appropriate validation before committing. If work is incomplete or
|
||||||
|
validation is blocked, report that state rather than presenting it as completed.
|
||||||
|
|
||||||
|
Before committing:
|
||||||
|
|
||||||
|
1. Run `git status --short` and review the relevant diff, including any staged
|
||||||
|
changes. Identify changes that predate the current unit of work.
|
||||||
|
2. Exclude secrets, local environment files, generated junk, and unrelated changes.
|
||||||
|
3. Stage only the files or hunks belonging to the completed unit. Use `git add .`
|
||||||
|
only when every included change belongs to that unit. Do not disturb unrelated
|
||||||
|
staged or unstaged work.
|
||||||
|
4. Review `git diff --staged` and run appropriate checks for the change. Do not
|
||||||
|
repeat checks already passed for unchanged content merely to create a commit.
|
||||||
|
5. Review `git log --oneline -n 10` and use
|
||||||
|
`git commit -m "<concise plain-English commit message>"`.
|
||||||
|
6. Report the commit identifier and any remaining uncommitted work.
|
||||||
|
|
||||||
|
If the unit cannot be safely separated from pre-existing work, leave it uncommitted
|
||||||
|
and explain the overlap rather than silently including that work. If a commit
|
||||||
|
fails, preserve the changes and report the failure; do not bypass hooks.
|
||||||
|
|
||||||
|
Default commit authorization does not authorize pushing, publishing, amending,
|
||||||
|
resetting, reverting, or rewriting history. Those actions require a separate user
|
||||||
|
instruction. Do not use `git commit -a` or `git commit --no-verify` to bypass the
|
||||||
|
staging and validation discipline above.
|
||||||
105
docs/policy/architecture.md
Normal file
105
docs/policy/architecture.md
Normal file
@@ -0,0 +1,105 @@
|
|||||||
|
# Architecture Policy
|
||||||
|
|
||||||
|
**Base Policy Version:** 1.0
|
||||||
|
|
||||||
|
**Last Updated**: 2026-09-23
|
||||||
|
|
||||||
|
## Purpose and System Shape
|
||||||
|
|
||||||
|
Audita is a single-process Go CLI for transcript polishing. Its architecture
|
||||||
|
separates deterministic transcript processing from external LLM effects through
|
||||||
|
narrow contracts and adapters. Correction modules and validators remain explicit,
|
||||||
|
composable components in an ordered processing pipeline.
|
||||||
|
|
||||||
|
This policy owns architectural requirements. The [internal overview](../internal/overview.md)
|
||||||
|
owns the implemented component inventory and links to runtime mechanics.
|
||||||
|
[CLI](../cli.md), [configuration](../config.md), [operations](../operations.md),
|
||||||
|
and [integration documentation](../integrations/subprocess.md) own external contracts.
|
||||||
|
|
||||||
|
## Ownership and Dependency Direction
|
||||||
|
|
||||||
|
- **Core:** owns domain data handling and reusable configuration, schema,
|
||||||
|
normalization, chunking, output, diagnostics, and reporting contracts. Domain
|
||||||
|
behavior must not require CLI or provider-transport knowledge.
|
||||||
|
- **Framework:** owns orchestration contracts and shared runtime mechanics for
|
||||||
|
modules, validators, proposals, LLM adapters, scheduling, and report assembly.
|
||||||
|
It composes domain operations through explicit inputs and outputs.
|
||||||
|
- **Modules:** own narrowly scoped correction behavior and module-specific prompt
|
||||||
|
choices. Keep module packages separate rather than collapsing their distinct
|
||||||
|
responsibilities into a broad generic stage abstraction.
|
||||||
|
- **Validators:** own validation behavior and composition. Deterministic and
|
||||||
|
LLM-backed validators remain explicit, independently composable elements.
|
||||||
|
- **Prompts:** own embedded prompt assets and metadata used by the runtime.
|
||||||
|
- **CLI and adapters:** own command parsing, streams, filesystem wiring, and
|
||||||
|
transport. Keep these concerns outside module, validator, and schema logic.
|
||||||
|
|
||||||
|
LLM transport belongs behind `StructuredLLMClient` and framework adapter
|
||||||
|
boundaries. Prefer small shared helpers over broad rewrites. Keep module and
|
||||||
|
validator inputs, outputs, and configuration dependencies explicit, without
|
||||||
|
hidden global state.
|
||||||
|
|
||||||
|
## Correction and Concurrency Invariants
|
||||||
|
|
||||||
|
- Treat LLM responses as nondeterministic inputs. Preserve deterministic proposal
|
||||||
|
indexing, validator ordering, transcript mutation, reports, and serialization.
|
||||||
|
- Validate corrections before final mutation, including local proposal checks,
|
||||||
|
decision cardinality, and apply-time safety. Provider-side schema enforcement
|
||||||
|
does not replace these checks.
|
||||||
|
- Keep concurrency bounded, configurable, context-aware, and observable through
|
||||||
|
the existing scheduler and utilization diagnostics. Preserve permit release on
|
||||||
|
success, failure, and cancellation.
|
||||||
|
- Keep module and validator identities, reason codes, and execution-class
|
||||||
|
distinctions stable where exposed as contracts.
|
||||||
|
|
||||||
|
The [pipeline](../internal/pipeline.md), [modules](../internal/modules.md),
|
||||||
|
[validators](../internal/validators.md), and [LLM runtime](../internal/llm-runtime.md)
|
||||||
|
describe how these requirements are implemented.
|
||||||
|
|
||||||
|
## State and Compatibility
|
||||||
|
|
||||||
|
Keep transcript mutation and output handling deterministic around external
|
||||||
|
calls. Reports and correction ledgers are derived from processing results.
|
||||||
|
Preserve the separation between transient processing state and filesystem
|
||||||
|
adapters; physical artifact placement and retention belong to
|
||||||
|
[operations](../operations.md).
|
||||||
|
|
||||||
|
Public CLI, configuration, output-schema, diagnostics, report, prompt, module,
|
||||||
|
and validator contracts should remain stable unless a change is deliberate and
|
||||||
|
tested. Prefer additive, compatible report changes. Current implementation
|
||||||
|
limits are documented in the [internal overview](../internal/overview.md#current-scope);
|
||||||
|
they are not additional prohibitions on future development.
|
||||||
|
|
||||||
|
## Errors and Secret Handling
|
||||||
|
|
||||||
|
Errors should identify the failed processing phase sufficiently for users and
|
||||||
|
subprocess callers. Preserve the stream and exit behavior defined by the
|
||||||
|
[CLI contract](../cli.md) and the artifact handling defined by
|
||||||
|
[operations](../operations.md).
|
||||||
|
|
||||||
|
Diagnostics, reports, and surfaced errors must not leak configured LLM secrets.
|
||||||
|
Configuration redaction and LLM payload/error redaction have separate
|
||||||
|
responsibilities and should remain separate. Implementation details belong in
|
||||||
|
[diagnostics and reporting](../internal/diagnostics-reporting.md).
|
||||||
|
|
||||||
|
## Dependencies
|
||||||
|
|
||||||
|
Keep Audita dependency-light and prefer the Go standard library for CLI parsing,
|
||||||
|
HTTP, JSON, filesystems, synchronization, and tests. Third-party dependencies
|
||||||
|
should be narrow, justified, and preferably established for their purpose.
|
||||||
|
YAML parsing is the existing direct dependency exception.
|
||||||
|
|
||||||
|
Do not introduce broad CLI, dependency-injection, workflow, logging, or plugin
|
||||||
|
frameworks without a concrete implemented need and focused tests.
|
||||||
|
|
||||||
|
## Testing and Documentation
|
||||||
|
|
||||||
|
Follow the [testing policy](testing.md) for risk-based sufficiency and stable test
|
||||||
|
boundaries. Keep tests close to the behavior they protect; shared helpers should
|
||||||
|
remove duplication without obscuring component-specific behavior. Relevant
|
||||||
|
internal docs identify component tests and change recipes; the
|
||||||
|
[contributor guide](../development.md) provides baseline validation commands.
|
||||||
|
|
||||||
|
Follow the [documentation policy](documentation.md) for ownership, current versus
|
||||||
|
future behavior, and the ADR lifecycle. Keep architecture concise and normative;
|
||||||
|
use the roadmap for implementation plans rather than maintaining a changelog or
|
||||||
|
future-feature specification here.
|
||||||
164
docs/policy/documentation.md
Normal file
164
docs/policy/documentation.md
Normal file
@@ -0,0 +1,164 @@
|
|||||||
|
# Documentation Policy
|
||||||
|
|
||||||
|
**Base Policy Version:** 1.1
|
||||||
|
|
||||||
|
**Last Updated**: 2026-09-23
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This policy assigns each documentation topic to one canonical owner. Its goal is
|
||||||
|
to keep this application's documentation accurate, concise, discoverable, and resistant
|
||||||
|
to drift for users, operators, developers, integrators, and LLM coding agents.
|
||||||
|
|
||||||
|
## Core Rules
|
||||||
|
|
||||||
|
### One Canonical Owner
|
||||||
|
|
||||||
|
Each authoritative fact belongs in one document. A non-owning document may give
|
||||||
|
a short, stable summary for orientation, but it must link to the canonical owner
|
||||||
|
instead of repeating volatile details.
|
||||||
|
|
||||||
|
Volatile details include commands, flags, configuration fields and defaults,
|
||||||
|
module keys, schemas, file names, paths, status codes, retry behavior, and
|
||||||
|
runtime guarantees. If readers could reasonably treat a statement as a
|
||||||
|
contract, maintain it only in the owning document.
|
||||||
|
|
||||||
|
### Current And Future Behavior
|
||||||
|
|
||||||
|
Outside `docs/roadmap/`, documentation describes implemented behavior only.
|
||||||
|
Partial features may be described only to their implemented boundary.
|
||||||
|
|
||||||
|
ADRs are the narrow exception: an ADR may record an accepted architectural
|
||||||
|
decision before implementation, but acceptance must not be presented as proof
|
||||||
|
that the behavior exists. The roadmap owns implementation status and sequencing
|
||||||
|
until the decision is implemented. Current architecture, user, operator,
|
||||||
|
integration, and internal documentation are updated when the behavior lands.
|
||||||
|
|
||||||
|
### Audience And Detail
|
||||||
|
|
||||||
|
Write for the document's stated audience and include only the detail needed for
|
||||||
|
its owned topic. User and operator docs should not expose implementation detail.
|
||||||
|
Developer docs should link to user-facing and external contracts rather than
|
||||||
|
restate them.
|
||||||
|
|
||||||
|
### Examples
|
||||||
|
|
||||||
|
Complete copyable files belong in `examples/`. Documentation may use the
|
||||||
|
smallest illustrative snippet needed to explain its owned topic, but should link
|
||||||
|
to maintained examples instead of embedding a second complete copy.
|
||||||
|
|
||||||
|
Examples must be valid, secret-free, and tested where practical. Commands and
|
||||||
|
configuration used in documentation should match the application.
|
||||||
|
|
||||||
|
### Security And Privacy
|
||||||
|
|
||||||
|
Documentation and examples must not contain real credentials, private keys,
|
||||||
|
private environment dumps, sensitive source material, or private infrastructure
|
||||||
|
details unless intentionally public. Document secret-handling mechanisms, not
|
||||||
|
secret values.
|
||||||
|
|
||||||
|
## Canonical Ownership
|
||||||
|
|
||||||
|
| Topic | Canonical owner | Owned content | Content owned elsewhere |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Product orientation and minimal end-to-end quickstart | `README.md` | What this application is, why it is useful, one shortest successful invocation, and links onward. | Complete command reference, configuration reference, operational procedures, implementation detail. |
|
||||||
|
| Contributor entry point | `docs/development.md` | Task-oriented reading guide, minimal contributor orientation, baseline validation commands, and links to canonical docs. | Package inventory, architecture rules, subsystem behavior, detailed change recipes. |
|
||||||
|
| Application architecture policy | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, architectural boundaries, invariants, safety properties, and non-goals. | Concrete package inventory, implementation mechanics, contributor procedures, decision history, future work. |
|
||||||
|
| Agent policy | `docs/policy/agents.md` | Implementation discipline for coding agents, including proportionality, idiomatic simplicity, code discovery, Git commit restrictions, and rules against inventing requirements, restrictions, guarantees, or repository policy. | Application contracts, architecture, testing policy, and contributor procedures. |
|
||||||
|
| Documentation policy | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
|
||||||
|
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk-based sufficiency, test boundaries, doubles, coverage guidance, regression-test policy, and criteria for adding, rewriting, or deleting tests. | Subsystem behavior, application contracts, subsystem-specific test inventories, and implementation plans. |
|
||||||
|
| Release policy | `docs/policy/release.md` | Versioning and compatibility rules, release integrity and immutability requirements, release-documentation policy, required gates, publication controls, verification requirements, and exception policy. | Repository-specific commands, product installation guidance, CLI version semantics, historical release summaries, and CI implementation detail. |
|
||||||
|
| Repository-specific release procedure | `docs/release.md` | Maintainer prerequisites, candidate preparation, concrete validation and publication commands, repository and registry guards, deliverable publication, consumer-facing verification, and failure recovery. | General release requirements, product installation guidance, historical release summaries, and current compatibility contracts. |
|
||||||
|
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
|
||||||
|
| Configuration contract | `docs/config.md` | Discovery and precedence, file schema, fields, defaults, environment overrides, validation rules, and user-selectable module or validator keys. | Complete example files, CLI syntax, runtime state lifecycle, module implementation details. |
|
||||||
|
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
|
||||||
|
| Integration contracts for external components | `docs/integrations/`, including `docs/integrations/output-files.md` for serialized outputs | External file formats and protocols, upstream and downstream contracts, logical output bundle paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
||||||
|
| Inventory of implemented internal packages | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
||||||
|
| Behavior and architecture of internal packages | Other files under `docs/internal/` | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
||||||
|
| Contributor change recipes | Relevant files under `docs/internal/`, including `docs/internal/cli-config.md`, `docs/internal/modules.md`, and `docs/internal/validators.md` | Concrete editing and validation steps for each component. | Contributor orientation, global policy, and definitions of external contracts. |
|
||||||
|
| Architectural decision records | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
|
||||||
|
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status; sequencing; and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
||||||
|
| Release-note history | `docs/releases/` | One checked-in historical summary for each source release made under the procedure. The note at the immutable tag is that release's record. | Current commands, behavior, contracts, and compatibility definitions. |
|
||||||
|
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other files intended to be copied or run. | Field-by-field reference, command reference, prose explanation. |
|
||||||
|
|
||||||
|
Documents that do not exist are required only when the corresponding interface
|
||||||
|
or responsibility exists. Do not create placeholder API, consumer, integration,
|
||||||
|
or operations documents for behavior the application does not have. Audita has
|
||||||
|
retired its standalone troubleshooting guide; concrete operating and recovery
|
||||||
|
procedures belong in `docs/operations.md`.
|
||||||
|
|
||||||
|
## Boundary Rules
|
||||||
|
|
||||||
|
### Orientation
|
||||||
|
|
||||||
|
The README owns product orientation. The developer guide routes contributors.
|
||||||
|
Architecture owns normative structure. Internal overview owns the current
|
||||||
|
concrete component map. These documents may link to one another but should not
|
||||||
|
maintain parallel package or behavior descriptions.
|
||||||
|
|
||||||
|
### Commands, Configuration, And Operations
|
||||||
|
|
||||||
|
CLI documentation answers how to invoke the application. Configuration
|
||||||
|
documentation answers what settings mean. Operations answers what happens to
|
||||||
|
runtime state and how to operate or recover the application. When a workflow
|
||||||
|
crosses these topics, choose the document that owns the task and link to the
|
||||||
|
other contracts.
|
||||||
|
|
||||||
|
### Releases
|
||||||
|
|
||||||
|
The release policy defines requirements and invariants that every release must
|
||||||
|
satisfy. The repository-specific release procedure defines the commands and
|
||||||
|
workflow maintainers use to satisfy that policy. Release notes are immutable
|
||||||
|
historical summaries, not current-state contract owners: the checked-in note at
|
||||||
|
an immutable tag records that release, while current canonical documentation
|
||||||
|
must change with the behavior it describes. Do not use a release procedure or
|
||||||
|
release note to weaken policy, redefine current behavior, or defer required
|
||||||
|
documentation updates.
|
||||||
|
|
||||||
|
### Contracts And Implementation
|
||||||
|
|
||||||
|
Integration and API documents define externally observable shapes and
|
||||||
|
semantics. Internal documents explain how this application implements or consumes those
|
||||||
|
contracts. Internal docs may name a field, file, or protocol to identify a
|
||||||
|
dependency, but must link to its canonical contract for the definition.
|
||||||
|
|
||||||
|
### Security Topics
|
||||||
|
|
||||||
|
This policy owns what documentation and examples may contain. Architecture owns
|
||||||
|
application security invariants. Configuration owns credential-supply
|
||||||
|
mechanisms. Operations owns permissions and handling of sensitive runtime
|
||||||
|
artifacts. Internal docs own implementation mechanisms only.
|
||||||
|
|
||||||
|
## Architecture Decision Records
|
||||||
|
|
||||||
|
Use sequentially numbered ADR filenames such as
|
||||||
|
`0001-record-architecture-decisions.md`. Follow the lightweight Nygard format:
|
||||||
|
|
||||||
|
1. title;
|
||||||
|
2. status;
|
||||||
|
3. date;
|
||||||
|
4. context;
|
||||||
|
5. decision;
|
||||||
|
6. alternatives considered;
|
||||||
|
7. consequences.
|
||||||
|
|
||||||
|
Treat the decision content of an accepted ADR as immutable. When a decision
|
||||||
|
changes, create a new ADR and update the earlier ADR's status to superseded.
|
||||||
|
Rejected architectural alternatives belong in the ADR; rejected product ideas
|
||||||
|
belong in the roadmap.
|
||||||
|
|
||||||
|
## Maintenance
|
||||||
|
|
||||||
|
When behavior changes, update its canonical owner in the same change. If
|
||||||
|
ownership moves, remove the old definition and replace it with a link where
|
||||||
|
navigation remains useful.
|
||||||
|
|
||||||
|
Before completing documentation work:
|
||||||
|
|
||||||
|
- verify affected behavior and examples;
|
||||||
|
- check commands, flags, fields, defaults, schemas, and paths against their
|
||||||
|
implementation;
|
||||||
|
- keep unimplemented behavior in the roadmap, subject to the ADR exception;
|
||||||
|
- remove stale references and validate links;
|
||||||
|
- confirm that non-owning documents summarize and link rather than redefine;
|
||||||
|
- confirm that no secrets or sensitive private data were added.
|
||||||
168
docs/policy/release.md
Normal file
168
docs/policy/release.md
Normal file
@@ -0,0 +1,168 @@
|
|||||||
|
# Release Policy
|
||||||
|
|
||||||
|
**Base Policy Version:** 1.0
|
||||||
|
|
||||||
|
**Last Updated**: 2026-09-23
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This policy defines the minimum requirements for selecting, validating,
|
||||||
|
publishing, verifying, and correcting releases. Its goal is to make every
|
||||||
|
release intentional, reproducible from an exact source revision, documented
|
||||||
|
for consumers, and safe from retrospective replacement.
|
||||||
|
|
||||||
|
The repository-specific commands that implement these requirements belong in
|
||||||
|
`docs/release.md`. Historical release summaries belong in `docs/releases/`.
|
||||||
|
This is an Audita-specific adaptation of the base policy, not a requirement to
|
||||||
|
implement the standard unchanged. Adapt it further when Audita's needs warrant;
|
||||||
|
the canonical procedure must agree with the adopted policy.
|
||||||
|
|
||||||
|
## Release Model
|
||||||
|
|
||||||
|
A release identifies one exact source commit with an immutable version tag and
|
||||||
|
its checked-in release note. The primary release branch is `main`.
|
||||||
|
|
||||||
|
Release success means that the intended version tag has been created and
|
||||||
|
successfully pushed to the authoritative remote with its source commit. The
|
||||||
|
release procedure ends at that point. It must never wait for CI to complete or
|
||||||
|
require successful CI status, artifact publication, or consumer installation as
|
||||||
|
a completion gate.
|
||||||
|
|
||||||
|
CI builds and publishes binaries and checksums asynchronously from the tagged
|
||||||
|
source. Delayed or failed automation does not reverse release success. Artifact
|
||||||
|
availability is a separate status and must not be claimed merely because the tag
|
||||||
|
was pushed. Published artifacts must be traceable to the tagged source.
|
||||||
|
|
||||||
|
Automation must not silently change the source candidate, move a published tag,
|
||||||
|
or replace an existing published artifact.
|
||||||
|
|
||||||
|
## Supported Platforms
|
||||||
|
|
||||||
|
| Platform | Support |
|
||||||
|
| --- | --- |
|
||||||
|
| Linux amd64 | Required production target |
|
||||||
|
| Linux arm64 | Required production target |
|
||||||
|
| macOS arm64 | Best effort for local development; failures do not block a release or Linux artifact publication |
|
||||||
|
|
||||||
|
Windows and macOS amd64 are unsupported. Neither their builds nor tests are
|
||||||
|
release requirements. Native macOS testing and access to a Mac are not release
|
||||||
|
prerequisites.
|
||||||
|
|
||||||
|
## Versioning And Compatibility
|
||||||
|
|
||||||
|
Stable releases use Semantic Versioning in the form `vMAJOR.MINOR.PATCH`, with
|
||||||
|
no leading zeroes. Each version must be unused locally and on the authoritative
|
||||||
|
remote before publication begins.
|
||||||
|
|
||||||
|
Unless an adopted policy explicitly supports and defines prereleases,
|
||||||
|
prerelease versions are not permitted.
|
||||||
|
|
||||||
|
For releases at or after `v1.0.0`:
|
||||||
|
|
||||||
|
- a major release may make intentional incompatible changes;
|
||||||
|
- a minor release may add backward-compatible behavior; and
|
||||||
|
- a patch release may make backward-compatible fixes.
|
||||||
|
|
||||||
|
Before `v1.0.0`, a minor release may change a documented public, operational,
|
||||||
|
configuration, data, integration, or compatibility contract when the release
|
||||||
|
note explains the impact and required consumer action. A patch release must
|
||||||
|
not intentionally break those contracts within its minor line.
|
||||||
|
|
||||||
|
A repository with additional compatibility promises must define them in its
|
||||||
|
adopted policy or canonical contract documentation. A release note may explain
|
||||||
|
a compatibility change, but it must not redefine the current compatibility
|
||||||
|
contract.
|
||||||
|
|
||||||
|
## Release Documentation
|
||||||
|
|
||||||
|
Every release made under this policy must include
|
||||||
|
`docs/releases/<version>.md` in the tagged source tree. The filename must use
|
||||||
|
the complete version tag, and the document must begin with a title containing
|
||||||
|
the product name and that exact version.
|
||||||
|
|
||||||
|
Release notes must contain a short summary and any relevant compatibility or
|
||||||
|
upgrade instructions. Fixed sections and explicit no-action boilerplate are not
|
||||||
|
required. Keep notes proportionate to the change.
|
||||||
|
|
||||||
|
Release notes are immutable historical summaries. They should link to current
|
||||||
|
canonical documentation for exact behavior. Every affected canonical document
|
||||||
|
must be updated in the same candidate; a release note must not substitute for
|
||||||
|
current documentation or defer required documentation changes.
|
||||||
|
|
||||||
|
## Candidate Preparation And Validation
|
||||||
|
|
||||||
|
Use the minimum gates needed for confidence in Audita's release. Provide one
|
||||||
|
local command for the selected automated checks, following the
|
||||||
|
[testing policy](testing.md). Run the deterministic, offline default test suite
|
||||||
|
once for the candidate and verify that both required Linux targets build.
|
||||||
|
Avoid duplicate test runs solely to satisfy separate release phases.
|
||||||
|
|
||||||
|
Additional checks must address a concrete risk not already covered. Do not
|
||||||
|
inherit a mandatory linting, documentation audit, platform matrix, or live-service
|
||||||
|
checklist solely from the base policy. Documentation migration validation is
|
||||||
|
separate work, not automatically a recurring release gate.
|
||||||
|
|
||||||
|
Automate mechanical guards. Do not require manual approvals, sign-offs, laptop
|
||||||
|
testing, or post-publication inspection as release gates. Version selection and
|
||||||
|
a concise release summary remain preparation inputs, not separate approval rounds.
|
||||||
|
|
||||||
|
## Publication Guards
|
||||||
|
|
||||||
|
The procedure must automate the small set of checks needed to publish the
|
||||||
|
intended source under a fresh version:
|
||||||
|
|
||||||
|
- valid, unused version on the authoritative remote;
|
||||||
|
- clean worktree and index on the primary release branch;
|
||||||
|
- the exact candidate that passed the local checks, including its matching note;
|
||||||
|
- publication of that source commit and the exact intended tag.
|
||||||
|
|
||||||
|
Do not require remote CI success. Avoid an additional manual confirmation for
|
||||||
|
each mechanical guard. Publish explicit refs rather than all local tags.
|
||||||
|
A successful push is the completion signal; a failed or ambiguous push must be
|
||||||
|
resolved against remote ref state before declaring success or retrying.
|
||||||
|
|
||||||
|
## Immutability And Correction
|
||||||
|
|
||||||
|
Published version tags and published deliverables are immutable. Do not move,
|
||||||
|
overwrite, recreate, or delete a published tag. Do not replace an artifact,
|
||||||
|
image, package, checksum, signature, provenance statement, or release note
|
||||||
|
under an already published version.
|
||||||
|
|
||||||
|
If validation fails before publication, correct and commit the candidate, then
|
||||||
|
rerun the release command. Validation must apply to the exact candidate. An
|
||||||
|
unpublished local tag may be removed after confirming that it was never published
|
||||||
|
or used by a release channel.
|
||||||
|
|
||||||
|
If a defect is discovered after publication, leave the published release intact. Correct the defect on the primary branch, choose a
|
||||||
|
new patch version or other semantically appropriate new version, write a new
|
||||||
|
matching release note, and perform the complete release procedure again.
|
||||||
|
|
||||||
|
A distribution channel's supported yank, deprecation, or advisory mechanism
|
||||||
|
may warn consumers away from a defective release, but it must not replace that
|
||||||
|
release's contents or reuse its version.
|
||||||
|
|
||||||
|
## Repository Release Procedure
|
||||||
|
|
||||||
|
Maintain [docs/release.md](../release.md) as a short, executable maintainer runbook.
|
||||||
|
It must identify prerequisites, authoritative remote, version and note preparation,
|
||||||
|
one local validation command, automated source/tag guards, exact publication
|
||||||
|
commands, and failure handling. It must name the asynchronous automation and link
|
||||||
|
to this policy for supported platforms and release-success semantics.
|
||||||
|
|
||||||
|
Describe how to diagnose or retry failed artifact delivery separately from release
|
||||||
|
completion. Missing artifacts may be published later from the same immutable tag;
|
||||||
|
existing published artifacts must not be overwritten. A source defect requires a
|
||||||
|
new version. No polling or CI waiting belongs in the release command.
|
||||||
|
|
||||||
|
## Audita Adaptations
|
||||||
|
|
||||||
|
The maintainer approved these deviations from the base policy on 2026-09-23:
|
||||||
|
|
||||||
|
- Audita's needs govern policy adaptation rather than strict template compliance.
|
||||||
|
- Only Linux amd64 and arm64 are required; macOS arm64 is best effort.
|
||||||
|
- A successful tag push completes the release; asynchronous CI cannot block it.
|
||||||
|
- Gates are minimal, automated where practical, and avoid redundant validation
|
||||||
|
and human intervention.
|
||||||
|
|
||||||
|
The [release procedure](../release.md) and its linked scripts implement these
|
||||||
|
requirements. CI delivery remains asynchronous.
|
||||||
300
docs/policy/testing.md
Normal file
300
docs/policy/testing.md
Normal file
@@ -0,0 +1,300 @@
|
|||||||
|
# Testing Policy
|
||||||
|
|
||||||
|
**Base Policy Version:** 1.0
|
||||||
|
|
||||||
|
**Last Updated**: 2026-09-23
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Our tests exist to make **incorrect changes expensive and correct changes cheap**.
|
||||||
|
|
||||||
|
We do not optimize for test count, line coverage, exhaustive isolation, or the fewest possible tests. We optimize for sufficient confidence in important behavior while imposing as little unnecessary friction as possible on future development.
|
||||||
|
|
||||||
|
## Every test has a cost
|
||||||
|
|
||||||
|
Testing is not an unqualified good. Every test imposes both an immediate cost and a continuing lifetime cost.
|
||||||
|
|
||||||
|
A test must be:
|
||||||
|
|
||||||
|
- written and reviewed;
|
||||||
|
- understood by future maintainers and coding agents;
|
||||||
|
- executed in local and CI workflows;
|
||||||
|
- diagnosed when it fails;
|
||||||
|
- updated when legitimate behavior changes;
|
||||||
|
- maintained as fixtures, APIs, and dependencies evolve; and
|
||||||
|
- removed or rewritten when it becomes redundant, brittle, misleading, or obsolete.
|
||||||
|
|
||||||
|
Tests also create cognitive and architectural friction. They can constrain refactoring, duplicate policy, slow feedback loops, add noise to failures, and cause harmless implementation changes to require unrelated edits across the suite.
|
||||||
|
|
||||||
|
A test is warranted only when the confidence it provides justifies these costs.
|
||||||
|
|
||||||
|
Apply this cost-benefit analysis at two levels:
|
||||||
|
|
||||||
|
1. **Per test:** What realistic defect does this test detect, how consequential would that defect be, and is that protection worth the test's lifetime cost?
|
||||||
|
2. **Across the suite:** Does this collection provide materially more confidence than a smaller, simpler suite would?
|
||||||
|
|
||||||
|
The preferred test suite is a **lean suite that provides sufficient confidence in the risks that matter, without redundant or low-value tests**. We seek sufficient confidence with the least unnecessary testing friction, not the fewest possible tests.
|
||||||
|
|
||||||
|
Some friction is intentional. Tests should make dangerous changes—such as breaking compatibility, corrupting data, violating security boundaries, or reintroducing subtle bugs—require deliberate review. They should not make ordinary internal changes needlessly expensive.
|
||||||
|
|
||||||
|
The cost of a test is not a reason to omit testing by default. Do not cite maintenance cost abstractly. When omitting a plausible test, be able to state why the protected failure is low-risk, already covered, obvious, reversible, or cheaper to detect elsewhere. For consequential, subtle, or difficult-to-observe behavior, the presumption should favor testing.
|
||||||
|
|
||||||
|
## Default testing style
|
||||||
|
|
||||||
|
Use a **classical/Detroit-style** approach:
|
||||||
|
|
||||||
|
- Test observable behavior, resulting state, contracts, and invariants.
|
||||||
|
- Use real internal collaborators when they are fast and deterministic.
|
||||||
|
- Use fakes, stubs, or mocks primarily at expensive, nondeterministic, destructive, or external boundaries.
|
||||||
|
- Prefer package-level behavioral tests over tests coupled to private helpers or internal call sequences.
|
||||||
|
- Treat exact collaborator interactions as testable behavior only when the interaction itself is a requirement.
|
||||||
|
|
||||||
|
Examples of appropriate seams include clocks, randomness, subprocesses, remote APIs, object storage, email, and paid LLM calls.
|
||||||
|
|
||||||
|
## Test execution requirements
|
||||||
|
|
||||||
|
Tests in the default suite must be deterministic, offline, and independent of real credentials. They must not invoke paid APIs or depend on mutable external services. Tests that require live infrastructure must be explicitly opt-in and clearly separated from the default suite.
|
||||||
|
|
||||||
|
Control clocks, randomness, environment variables, and other process-global or machine-specific state when they affect behavior. Tests should be safe to run repeatedly and alongside other tests without depending on execution order or state left by an earlier test.
|
||||||
|
|
||||||
|
## What deserves tests
|
||||||
|
|
||||||
|
To the extent the following features are present in the application, prioritize tests for:
|
||||||
|
|
||||||
|
1. Public and package-level contracts.
|
||||||
|
2. Domain rules and important invariants.
|
||||||
|
3. Boundary conditions and malformed input.
|
||||||
|
4. Failure handling, cancellation, retries, recovery, and partial success.
|
||||||
|
5. Serialization, schemas, compatibility, and round trips.
|
||||||
|
6. Previously observed or plausible regressions.
|
||||||
|
7. Representative integration and end-to-end workflows.
|
||||||
|
|
||||||
|
A package-level contract is behavior relied upon by another package or major collaborator, not every observable detail of a package implementation.
|
||||||
|
|
||||||
|
For behavior involving **data integrity, destructive operations, compatibility, security, concurrency, idempotency, or recovery**, presume that durable tests are required unless the behavior is already credibly protected at another layer.
|
||||||
|
|
||||||
|
Do not add tests merely because a function, branch, or line exists. Do not add a test when the same meaningful risk is already adequately protected elsewhere.
|
||||||
|
|
||||||
|
## Choose the right test boundary
|
||||||
|
|
||||||
|
Test through the narrowest stable boundary that expresses the behavior clearly.
|
||||||
|
|
||||||
|
This is often the package API, but it may instead be:
|
||||||
|
|
||||||
|
- a smaller pure function when dense domain logic is most clearly isolated there;
|
||||||
|
- a package-level operation when several internal collaborators jointly produce the behavior; or
|
||||||
|
- a larger integration boundary when correctness emerges from interaction with a real dependency.
|
||||||
|
|
||||||
|
Do not force all behavior through oversized end-to-end tests. Do not test every private helper merely because it exists. Choose the boundary that gives durable confidence with the least incidental coupling.
|
||||||
|
|
||||||
|
## Test behavior, not implementation
|
||||||
|
|
||||||
|
A test should protect a decision, contract, or invariant. A test must not memorialize the current implementation.
|
||||||
|
|
||||||
|
Before adding or retaining a test, ask:
|
||||||
|
|
||||||
|
> What realistic defect would this test catch?
|
||||||
|
|
||||||
|
A test is suspect when its main purpose is to detect that someone:
|
||||||
|
|
||||||
|
- changed an internal constant;
|
||||||
|
- renamed or split a private helper;
|
||||||
|
- reordered equivalent internal operations;
|
||||||
|
- changed incidental formatting;
|
||||||
|
- replaced one correct algorithm with another; or
|
||||||
|
- refactored internal object structure without changing behavior.
|
||||||
|
|
||||||
|
Refactoring should normally require no test edits unless the refactored structure is itself part of the contract.
|
||||||
|
|
||||||
|
A test can be factually correct and still have negative value. Accurately describing current behavior is not sufficient; the protected behavior must be important enough to justify the future friction.
|
||||||
|
|
||||||
|
## Expected effects of different changes
|
||||||
|
|
||||||
|
Use the following expectations when evaluating test failures and test maintenance:
|
||||||
|
|
||||||
|
| Change | Expected effect on tests |
|
||||||
|
|---|---|
|
||||||
|
| Internal refactor that preserves behavior | Existing tests should normally remain unchanged and continue to pass. |
|
||||||
|
| Change to an internal default with no contractual significance | Behavioral tests should normally remain unchanged; tests should derive expectations from configuration or relationships rather than duplicate the old value. |
|
||||||
|
| Intentional change to public behavior, policy, schema, or compatibility guarantees | The relevant tests should be reviewed and changed deliberately. |
|
||||||
|
| Accidental violation of a contract or invariant | Tests should fail; fix the production code rather than rewriting the tests to accept the defect. |
|
||||||
|
|
||||||
|
A test failing is not the same as a test needing to be edited. Many tests may correctly fail because of one production defect. The maintenance smell is a correct internal change that requires unrelated expectation updates throughout the suite.
|
||||||
|
|
||||||
|
## Separate mechanism from policy
|
||||||
|
|
||||||
|
Configurable thresholds and defaults must not be duplicated throughout the test suite.
|
||||||
|
|
||||||
|
For example, do not encode an internal concurrency limit indirectly:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// Production policy:
|
||||||
|
const maxConcurrency = 4
|
||||||
|
|
||||||
|
// Brittle test:
|
||||||
|
err := startProcesses(5)
|
||||||
|
require.Error(t, err)
|
||||||
|
```
|
||||||
|
|
||||||
|
Instead, test the mechanism relationally:
|
||||||
|
|
||||||
|
```go
|
||||||
|
const limit = 2
|
||||||
|
runner := NewRunner(limit)
|
||||||
|
|
||||||
|
require.NoError(t, runner.Start(limit))
|
||||||
|
require.ErrorIs(t, runner.Start(limit+1), ErrTooMuchConcurrency)
|
||||||
|
```
|
||||||
|
|
||||||
|
The test should prove:
|
||||||
|
|
||||||
|
- the configured limit is accepted; and
|
||||||
|
- one beyond the configured limit is rejected.
|
||||||
|
|
||||||
|
The production default should be tested exactly only when its literal value is itself a public, operational, safety, protocol, or compatibility requirement.
|
||||||
|
|
||||||
|
Apply the same rule to limits, timeouts, capacities, retry counts, and ranges: test relationships and behavior, not duplicated literals.
|
||||||
|
|
||||||
|
For concurrency limits, test both kinds of behavior when relevant:
|
||||||
|
|
||||||
|
1. **Configuration enforcement:** invalid or excessive requested values are handled correctly.
|
||||||
|
2. **Runtime enforcement:** observed peak concurrency never exceeds the configured limit.
|
||||||
|
|
||||||
|
Use a test-controlled limit and measure the behavior relative to that limit. Do not merely assert today's default value.
|
||||||
|
|
||||||
|
## Avoid semantic duplication across layers
|
||||||
|
|
||||||
|
Each behavior should have a clear test owner.
|
||||||
|
|
||||||
|
- Parser tests own parsing cases.
|
||||||
|
- Validator tests own validation rules.
|
||||||
|
- Domain tests own transformations and invariants.
|
||||||
|
- Adapter tests own external integration behavior.
|
||||||
|
- Orchestrator tests own coordination and failure propagation.
|
||||||
|
- CLI tests own argument and configuration mapping.
|
||||||
|
- End-to-end tests prove that representative assembled workflows work.
|
||||||
|
|
||||||
|
Higher-level tests should not repeat every lower-level case. A single intentional policy change should not require unrelated edits across many test files.
|
||||||
|
|
||||||
|
Tests that are individually reasonable may still be collectively redundant. Evaluate the marginal value of each additional test in light of the protection already provided by the rest of the suite.
|
||||||
|
|
||||||
|
## Use test doubles deliberately
|
||||||
|
|
||||||
|
Choose the least elaborate test double that provides the required control or observation.
|
||||||
|
|
||||||
|
As a default:
|
||||||
|
|
||||||
|
1. Prefer real collaborators when they are fast and deterministic.
|
||||||
|
2. Use small in-memory fakes when realistic stateful behavior is helpful.
|
||||||
|
3. Use stubs when a dependency only needs to provide controlled responses.
|
||||||
|
4. Use mocks when the interaction itself is contractual.
|
||||||
|
|
||||||
|
Mocks are appropriate when the contract includes facts such as:
|
||||||
|
|
||||||
|
- a notification is sent exactly once;
|
||||||
|
- a transaction is committed only after successful writes;
|
||||||
|
- cancellation reaches a subprocess;
|
||||||
|
- an expensive API is called no more than once; or
|
||||||
|
- a security audit event is emitted.
|
||||||
|
|
||||||
|
Do not use mocks merely to isolate every object or reproduce the implementation's call graph.
|
||||||
|
|
||||||
|
## Go-specific guidance
|
||||||
|
|
||||||
|
Use:
|
||||||
|
|
||||||
|
- table-driven tests for meaningful behavioral categories and boundaries;
|
||||||
|
- `t.TempDir()` for real filesystem behavior;
|
||||||
|
- `httptest.Server` for realistic HTTP interactions;
|
||||||
|
- fuzz tests for parsers, normalization, path handling, and broad input spaces;
|
||||||
|
- golden files only when the complete output is intentionally stable;
|
||||||
|
- integration tests where correctness depends on component interaction; and
|
||||||
|
- a small number of representative end-to-end tests.
|
||||||
|
|
||||||
|
Avoid exact error-string assertions unless the wording is itself contractual. Prefer `errors.Is`, `errors.As`, typed errors, or structured error fields.
|
||||||
|
|
||||||
|
At CLI boundaries, prefer exit classifications, structured output, and the smallest stable semantic fragment needed to identify the error. Do not snapshot complete diagnostic wording unless it is contractual.
|
||||||
|
|
||||||
|
Golden-file updates must require an explicit local flag. CI must not update golden files automatically, and reviewers must inspect the semantic diff before accepting an update.
|
||||||
|
|
||||||
|
Keep tests readable and direct. Test helpers and fixture frameworks must earn their own maintenance cost; do not build elaborate test infrastructure for small or isolated needs.
|
||||||
|
|
||||||
|
## Coverage
|
||||||
|
|
||||||
|
Coverage is a diagnostic, not a target.
|
||||||
|
|
||||||
|
Use it to find untested critical branches and unexpectedly weak packages. Do not write low-value tests solely to increase a percentage, and do not infer test quality from coverage alone.
|
||||||
|
|
||||||
|
Pure domain logic will often warrant higher coverage than CLI wiring or external adapters. Uneven coverage is acceptable when it reflects risk.
|
||||||
|
|
||||||
|
Increasing coverage is valuable only when the newly covered behavior protects a meaningful risk at an acceptable cost.
|
||||||
|
|
||||||
|
## Regression tests
|
||||||
|
|
||||||
|
A bug fix should normally include a regression test that fails before the fix and passes afterward.
|
||||||
|
|
||||||
|
Retain the test when the defect could realistically recur and its consequences justify the ongoing cost. Prefer the narrowest durable test of the violated contract or invariant; do not preserve accidental implementation details from the original bug.
|
||||||
|
|
||||||
|
Not every historical bug requires a permanent test. If the underlying design has made recurrence impossible, the test has become redundant, or a stronger invariant test now subsumes it, remove or consolidate it.
|
||||||
|
|
||||||
|
## Deleting or rewriting tests
|
||||||
|
|
||||||
|
Tests are maintained code, not permanent historical artifacts.
|
||||||
|
|
||||||
|
Delete or rewrite a test when its maintenance cost exceeds the confidence it provides.
|
||||||
|
|
||||||
|
Strong candidates include tests that:
|
||||||
|
|
||||||
|
- require updates after harmless internal changes;
|
||||||
|
- directly assert private constants without protecting a real contract;
|
||||||
|
- duplicate the same policy across several layers;
|
||||||
|
- verify mock choreography rather than outcomes;
|
||||||
|
- snapshot large amounts of incidental output;
|
||||||
|
- test trivial private helpers already exercised through stable package behavior;
|
||||||
|
- protect risks already covered more effectively elsewhere;
|
||||||
|
- are flaky, misleading, obsolete, or disproportionately expensive to diagnose; or
|
||||||
|
- no longer correspond to a plausible failure mode.
|
||||||
|
|
||||||
|
Several brittle tests may encode one genuine requirement. Replace them with one durable behavior-level or invariant test rather than preserving all of them.
|
||||||
|
|
||||||
|
Deleting a low-value test can improve the quality of the suite by reducing noise, maintenance burden, and friction around legitimate change.
|
||||||
|
|
||||||
|
## Reviewing a proposed test
|
||||||
|
|
||||||
|
Use the following questions when the value, boundary, or durability of a proposed test is not self-evident. Significant test additions should be reviewable against them, but written answers are not required for every routine test.
|
||||||
|
|
||||||
|
1. What realistic defect would it catch?
|
||||||
|
2. How likely is that defect?
|
||||||
|
3. How consequential would it be?
|
||||||
|
4. Is the behavior already protected elsewhere?
|
||||||
|
5. At which layer should this behavior be owned?
|
||||||
|
6. Does the test assert a durable contract or an incidental implementation detail?
|
||||||
|
7. Could the implementation be refactored without changing the behavior and without editing this test?
|
||||||
|
8. What should cause this test to fail?
|
||||||
|
9. What legitimate changes should not cause this test to fail?
|
||||||
|
10. What ongoing maintenance, execution, and diagnostic cost will the test impose?
|
||||||
|
11. Is there a smaller or more direct test that protects the same risk?
|
||||||
|
|
||||||
|
Do not add the test when its expected lifetime cost exceeds its expected protective value.
|
||||||
|
|
||||||
|
When deciding not to test plausible behavior, record or be able to explain why the risk is low, already protected, obvious, reversible, or cheaper to detect elsewhere.
|
||||||
|
|
||||||
|
## Definition of sufficient
|
||||||
|
|
||||||
|
A test suite is sufficient when:
|
||||||
|
|
||||||
|
- important contracts and invariants are protected;
|
||||||
|
- meaningful boundaries and failure modes are exercised;
|
||||||
|
- realistic and consequential regressions are credibly protected against silent recurrence;
|
||||||
|
- behavior involving data integrity, destructive operations, compatibility, security, concurrency, idempotency, and recovery is credibly protected;
|
||||||
|
- important external boundaries have realistic integration coverage;
|
||||||
|
- representative complete workflows are tested;
|
||||||
|
- failures provide useful signal rather than redundant noise;
|
||||||
|
- legitimate internal changes usually do not require test edits; and
|
||||||
|
- additional tests would mostly repeat existing protection or preserve inconsequential implementation details.
|
||||||
|
|
||||||
|
Sufficiency is a risk judgment, not a coverage percentage or test count. Reassess it as the application, its users, and the consequences of failure evolve.
|
||||||
|
|
||||||
|
The governing rule is:
|
||||||
|
|
||||||
|
> Test heavily where failure is consequential, subtle, or difficult to detect after the fact. Test lightly where failure is obvious, reversible, and inexpensive—and retain no test whose lifetime cost exceeds the confidence it provides.
|
||||||
@@ -1,128 +0,0 @@
|
|||||||
# Audita Release Checklist
|
|
||||||
|
|
||||||
Use this checklist before cutting a pre-1.0 or 1.0 release candidate.
|
|
||||||
|
|
||||||
## Core test pass
|
|
||||||
|
|
||||||
- Run:
|
|
||||||
- `go test ./...`
|
|
||||||
- Confirm tests pass without live LLM credentials and without Python dependencies.
|
|
||||||
|
|
||||||
## Config validation and precedence
|
|
||||||
|
|
||||||
- Validate a representative config:
|
|
||||||
- `audita config validate --config <path>`
|
|
||||||
- Inspect redacted effective config:
|
|
||||||
- `audita config print-effective --config <path>`
|
|
||||||
- Confirm precedence behavior:
|
|
||||||
- defaults -> file config -> environment -> CLI.
|
|
||||||
- Confirm default config search order:
|
|
||||||
- `/usr/local/etc/audita/config.yml` first, then `/etc/audita/config.yml`.
|
|
||||||
- Confirm missing both default-path config files is non-fatal when `--config`/`AUDITA_CONFIG` are unset.
|
|
||||||
|
|
||||||
## Output schema checks
|
|
||||||
|
|
||||||
- Verify default output schema remains `bare-segments`.
|
|
||||||
- Verify `--output-schema audita-v1` emits object payload with `schema` and `version`.
|
|
||||||
- Verify unknown schema (for example `seriatim-intermediate`) fails clearly.
|
|
||||||
|
|
||||||
## Subprocess contract checks
|
|
||||||
|
|
||||||
- With `--output`, verify stdout is empty on success.
|
|
||||||
- Without `--output`, verify stdout contains transcript JSON only.
|
|
||||||
- Verify `--report-json` writes file output and does not write report JSON to stdout.
|
|
||||||
- Verify failure stderr remains human-readable and includes diagnostics path when available.
|
|
||||||
- Verify nonzero exit on failures.
|
|
||||||
|
|
||||||
## Structured LLM checks
|
|
||||||
|
|
||||||
- Verify runtime uses the Audita-owned OpenAI-compatible adapter.
|
|
||||||
- Verify structured response schemas are attached via `response_format.type=json_schema`.
|
|
||||||
- Verify diagnostics metadata includes structured schema `id/version/name/sha256`.
|
|
||||||
- Verify provider output is still locally decoded/validated before use.
|
|
||||||
- Verify malformed module-stage structured payloads degrade to warnings/rejections instead of failing the run.
|
|
||||||
|
|
||||||
## Report and diagnostics schema checks
|
|
||||||
|
|
||||||
- Verify report metadata fields:
|
|
||||||
- `report_schema_name`
|
|
||||||
- `report_schema_version`
|
|
||||||
- `output_schema`
|
|
||||||
- `config_version` when file config is used.
|
|
||||||
- Verify diagnostics artifact references exist in reports:
|
|
||||||
- transcript/normalization/chunking/invocation/effective-config artifacts
|
|
||||||
- utilization diagnostics artifact
|
|
||||||
- correction ledger artifact
|
|
||||||
- error log on failures.
|
|
||||||
|
|
||||||
## Redaction checks
|
|
||||||
|
|
||||||
- Verify secrets are redacted from:
|
|
||||||
- `effective-config.json`
|
|
||||||
- run-dir and `--report-json` reports
|
|
||||||
- LLM request/response/error diagnostics payloads.
|
|
||||||
- Verify no API keys/bearer tokens leak into fixtures or outputs.
|
|
||||||
|
|
||||||
## Prompt and validator metadata checks
|
|
||||||
|
|
||||||
- Verify prompt metadata appears in LLM request metadata diagnostics:
|
|
||||||
- `prompt_id`, `prompt_version`, `prompt_source`, `embedded_path`, `sha256`.
|
|
||||||
- Verify stable validator keys appear in report decisions/rejections.
|
|
||||||
- Verify module warning records appear in reports for malformed proposal-generation payloads and malformed validator batches.
|
|
||||||
- Verify built-in validator chains resolve and execute for default and explicit module runs.
|
|
||||||
|
|
||||||
## Utilization diagnostics checks
|
|
||||||
|
|
||||||
- Verify `utilization-diagnostics.json` exists on successful runs.
|
|
||||||
- Verify partial utilization artifact behavior on controlled failure paths.
|
|
||||||
- Verify utilization fields are structurally present and nonnegative:
|
|
||||||
- effective concurrency
|
|
||||||
- run timing
|
|
||||||
- module timing summaries
|
|
||||||
- per-validator timing summaries.
|
|
||||||
|
|
||||||
## Correction ledger checks
|
|
||||||
|
|
||||||
- Verify `correction-ledger.json` exists on successful runs.
|
|
||||||
- Verify report references ledger artifact path.
|
|
||||||
- Verify ledger dispositions include applied/rejected and skipped/failed where exercised.
|
|
||||||
- Verify validator rejection and proposal-application skip remain distinct.
|
|
||||||
|
|
||||||
## Pipeline behavior checks
|
|
||||||
|
|
||||||
- Verify default full pipeline run remains:
|
|
||||||
- `glossary`, `homophones`, `glossary`, `spoken_word`, `grammar`
|
|
||||||
- with deterministic repeated instance naming (`glossary_1`, `glossary_2`).
|
|
||||||
- Verify explicit module runs (`--modules`) still work.
|
|
||||||
|
|
||||||
## Failure and cancellation checks
|
|
||||||
|
|
||||||
- Verify controlled failure paths retain diagnostics and produce best-effort failure reports.
|
|
||||||
- Verify malformed proposal-generation payloads keep exit code `0`, keep stderr empty on success, and record warnings in reports/diagnostics.
|
|
||||||
- Verify malformed validator payloads reject only the affected batch and do not fail the module.
|
|
||||||
- Verify timeout/cancellation paths exit nonzero, do not hang, and retain failure diagnostics when initialized.
|
|
||||||
|
|
||||||
## Release fixture/idempotence checks
|
|
||||||
|
|
||||||
- Run release fixtures (`internal/cli/testdata/release`) through `go test ./...`.
|
|
||||||
- Confirm fixture checks cover:
|
|
||||||
- must-apply and must-not-apply expectations
|
|
||||||
- protected-term survival
|
|
||||||
- report and diagnostics contracts
|
|
||||||
- output-schema checks
|
|
||||||
- prompt/schema metadata diagnostics
|
|
||||||
- utilization/ledger artifacts
|
|
||||||
- idempotence-oriented second pass no-op behavior with deterministic fake responses.
|
|
||||||
|
|
||||||
## Deferred-feature guardrail
|
|
||||||
|
|
||||||
- Confirm release docs do not claim support for deferred items:
|
|
||||||
- filesystem prompt overrides
|
|
||||||
- user-configurable validator chains
|
|
||||||
- arbitrary user-supplied output schemas
|
|
||||||
- resume/start-at/stop-after execution
|
|
||||||
- diff/check/propose-only modes
|
|
||||||
- generated transcript descriptions enabled by default
|
|
||||||
- interactive review UI
|
|
||||||
- UI/server wrapper
|
|
||||||
- provider benchmarking harness.
|
|
||||||
130
docs/release.md
Normal file
130
docs/release.md
Normal file
@@ -0,0 +1,130 @@
|
|||||||
|
# Releasing Audita
|
||||||
|
|
||||||
|
The [release policy](policy/release.md) defines versioning, supported platforms,
|
||||||
|
immutability, and success. This runbook implements it with
|
||||||
|
[scripts/release.sh](../scripts/release.sh). Publication ends after a successful
|
||||||
|
source/tag push; binary delivery runs asynchronously.
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
Use Bash, Git, and Go 1.25 or newer (CI uses Go 1.25), with dependencies available
|
||||||
|
locally or downloadable. Tests themselves are offline. Work from a
|
||||||
|
checkout of `main` with permission to push that branch and tags to `origin`.
|
||||||
|
For this repository, `origin` is `https://gitea.maximumdirect.net/eric/audita.git`.
|
||||||
|
The script uses the checkout's configured `origin`; set it correctly before use.
|
||||||
|
The remote must support atomic pushes. No CI credentials or Mac are needed locally.
|
||||||
|
|
||||||
|
CI uses the existing `GITEA_RELEASE_TOKEN` secret for asynchronous asset delivery.
|
||||||
|
macOS arm64 is a best-effort development environment; no macOS binary is published.
|
||||||
|
|
||||||
|
## Prepare the Candidate
|
||||||
|
|
||||||
|
Choose an unused version following the policy. Write
|
||||||
|
`docs/releases/<version>.md`, beginning with `# Audita <version>`, followed by a
|
||||||
|
short summary and any compatibility or upgrade instructions relevant to the
|
||||||
|
release. There are no mandatory boilerplate sections. Update affected canonical
|
||||||
|
documentation with behavior changes.
|
||||||
|
|
||||||
|
Commit the intended changes, including the note, on `main`. The release script
|
||||||
|
requires a clean index and worktree and does not create commits. There is no
|
||||||
|
separate approval round. Published notes describe their tagged release; existing
|
||||||
|
release history does not need to be reconstructed.
|
||||||
|
|
||||||
|
## Validate and Publish
|
||||||
|
|
||||||
|
From the repository root, substitute the chosen version:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
bash scripts/release.sh --publish v0.2.0
|
||||||
|
```
|
||||||
|
|
||||||
|
This single invocation:
|
||||||
|
|
||||||
|
1. Checks the version, branch, clean candidate, matching note title, and tag absence.
|
||||||
|
2. Runs the default offline test suite once, then builds both required Linux targets
|
||||||
|
in a temporary directory using the same build script as CI.
|
||||||
|
3. Checks that the candidate and worktree are unchanged and that it includes the
|
||||||
|
current `origin/main` history.
|
||||||
|
4. Creates the version tag at that commit and atomically pushes the exact source
|
||||||
|
commit to `main` and the exact tag to `origin`.
|
||||||
|
5. Reports release success and exits without polling CI or checking asset availability.
|
||||||
|
|
||||||
|
To validate a prepared candidate without remote access, creating a tag, or pushing:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
bash scripts/release.sh --check v0.2.0
|
||||||
|
```
|
||||||
|
|
||||||
|
This is an optional diagnostic command, not a required step before `--publish`.
|
||||||
|
Publication validates within its own invocation; it does not use persistent cached
|
||||||
|
results. Both modes require the candidate note and a clean `main` checkout.
|
||||||
|
|
||||||
|
## Artifact Delivery
|
||||||
|
|
||||||
|
[Woodpecker](../.woodpecker/release.yml) runs on tag events. It invokes
|
||||||
|
[scripts/build-release.sh](../scripts/build-release.sh), then publishes:
|
||||||
|
|
||||||
|
- `audita-<version>-linux-amd64`
|
||||||
|
- `audita-<version>-linux-arm64`
|
||||||
|
- `SHA256SUMS`
|
||||||
|
|
||||||
|
Builds disable CGO and use the shared script's release flags. The version is in the
|
||||||
|
asset name and source tag; Audita has no CLI version command. Cross-compilation
|
||||||
|
checks compilation, not native execution on each architecture.
|
||||||
|
|
||||||
|
The CI job status and logs show delivery progress or failure. Assets appear on
|
||||||
|
the matching Gitea release page when delivery succeeds. A successful tag push
|
||||||
|
must not be reported as confirmation that these files are available.
|
||||||
|
|
||||||
|
Consumers can download the binary for their Linux architecture and `SHA256SUMS`
|
||||||
|
from that page, check its listed SHA-256 digest with `sha256sum`, and make it
|
||||||
|
executable. Optional consumer checks are not maintainer release gates. Digests
|
||||||
|
detect file mismatches; they are not an independent signature of authenticity.
|
||||||
|
|
||||||
|
## Failure and Retry
|
||||||
|
|
||||||
|
**Local validation fails:** correct the issue, commit any changes, and rerun the
|
||||||
|
publication command. A failure before tag creation publishes nothing. Temporary
|
||||||
|
local build files are removed on exit.
|
||||||
|
|
||||||
|
**Push fails or the connection is interrupted:** the script retains the local tag
|
||||||
|
and reports failure rather than guessing whether publication succeeded. Inspect
|
||||||
|
remote state with the actual version:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git rev-parse 'v0.2.0^{commit}'
|
||||||
|
git ls-remote origin refs/heads/main refs/tags/v0.2.0
|
||||||
|
```
|
||||||
|
|
||||||
|
If the remote tag identifies the intended candidate and its source is on remote
|
||||||
|
`main`, the release succeeded; no CI waiting is needed. An atomic push normally
|
||||||
|
updates both refs or neither. If `main` has advanced since publication, fetch it
|
||||||
|
and check candidate ancestry rather than requiring tip equality.
|
||||||
|
|
||||||
|
If the tag is absent remotely and was never published or used by a release
|
||||||
|
channel, remove only that unpublished local tag (`git tag -d v0.2.0`), resolve
|
||||||
|
any rejected branch update, and rerun the command. If the remote tag has a
|
||||||
|
different target, do not overwrite it; choose a fresh version. Never force a
|
||||||
|
branch or tag update to bypass the guards.
|
||||||
|
|
||||||
|
**CI delivery fails:** the release remains successful. Inspect the failed tag
|
||||||
|
job's logs and retry that same tagged job for a transient failure. Publication is
|
||||||
|
configured to skip existing filenames and not overwrite assets. Keep the same
|
||||||
|
source and build environment when retrying; inspect any existing assets/checksum
|
||||||
|
manifest before filling gaps. Do not regenerate and replace a published manifest
|
||||||
|
or binary. If an existing manifest cannot describe the missing rebuilt assets,
|
||||||
|
use a new version instead. A pipeline or source defect also requires a corrected
|
||||||
|
commit and new version. An outage does not require moving the tag.
|
||||||
|
|
||||||
|
**A released source defect is discovered:** fix it on `main` and publish a new
|
||||||
|
appropriate version with a note explaining the correction. Leave the old tag and
|
||||||
|
published files intact.
|
||||||
|
|
||||||
|
## Maintaining the Release Workflow
|
||||||
|
|
||||||
|
The shared build script keeps local and CI build settings aligned. Tests in
|
||||||
|
[scripts/release_test.go](../scripts/release_test.go) exercise publication against
|
||||||
|
temporary local Git remotes, including guard failures and rejected pushes, with
|
||||||
|
Go execution stubbed to avoid recursive builds/tests. They run in `go test ./...`.
|
||||||
|
Changes to CI delivery should preserve explicit asset names, checksums, and
|
||||||
|
non-overwriting publication. CI delivery is separate from the local success path.
|
||||||
8
docs/releases/README.md
Normal file
8
docs/releases/README.md
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
# Release Notes
|
||||||
|
|
||||||
|
- [Audita v1.1.0](v1.1.0.md)
|
||||||
|
|
||||||
|
See the [release documentation requirements](../policy/release.md#release-documentation).
|
||||||
|
Use the [release procedure](../release.md) to prepare and publish new releases.
|
||||||
|
|
||||||
|
This index does not reconstruct or replace records for existing tags.
|
||||||
22
docs/releases/v1.1.0.md
Normal file
22
docs/releases/v1.1.0.md
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
# Audita v1.1.0
|
||||||
|
|
||||||
|
This release refreshes Audita's documentation and repository policies and adds a
|
||||||
|
small, automated release workflow. Transcript-processing behavior, CLI syntax,
|
||||||
|
configuration, and serialized output formats are unchanged from v1.0.0.
|
||||||
|
|
||||||
|
- Consolidated CLI, configuration, operations, and integration references, including
|
||||||
|
a dedicated reference for output files and reports.
|
||||||
|
- Added contributor navigation and agent, testing, and release policies; retired
|
||||||
|
speculative troubleshooting documentation and the completed migration roadmap.
|
||||||
|
- Added local release validation and publication scripts, with publication complete
|
||||||
|
after the source and version tag are pushed. CI delivers binaries asynchronously.
|
||||||
|
- Isolated CLI tests from host configuration for a reliable local test baseline.
|
||||||
|
|
||||||
|
Release binaries are now provided only for Linux amd64 and Linux arm64. Windows
|
||||||
|
and Intel macOS are no longer supported. macOS arm64 remains a best-effort local
|
||||||
|
build/development environment; macOS binaries are no longer published.
|
||||||
|
|
||||||
|
Linux users need no configuration or data migration. Users of previously published
|
||||||
|
Windows or macOS binaries should account for the distribution changes above before
|
||||||
|
upgrading. See the [release procedure](../release.md) for artifact delivery and the
|
||||||
|
[contributor guide](../development.md) for local build instructions.
|
||||||
@@ -1,791 +0,0 @@
|
|||||||
# Pre-1.0 Code Quality and Deduplication Audit
|
|
||||||
|
|
||||||
## 1. Executive summary
|
|
||||||
|
|
||||||
Audita is in good shape for a limited pre-1.0 cleanup pass. The repository is small, package boundaries are mostly explicit, and the core public contract is already documented around `audita process`, config loading, output schemas, diagnostics, reports, embedded prompts, modules, and validators. The highest-value improvements are targeted centralization, not a rewrite.
|
|
||||||
|
|
||||||
Top three refactoring targets before 1.0:
|
|
||||||
|
|
||||||
1. Centralize module proposal plumbing and prompt payload construction across the four production modules.
|
|
||||||
2. Centralize effective config loading plus schema/module catalog validation so `process`, `config print-effective`, and `config validate` cannot drift.
|
|
||||||
3. Centralize diagnostics artifact names, stage names, and validator classification metadata used by reports and the correction ledger.
|
|
||||||
|
|
||||||
No major architectural risk is apparent. The main pre-1.0 risk is public-behavior drift from repeated policy strings, catalog values, artifact paths, and nearly identical command/module scaffolding.
|
|
||||||
|
|
||||||
This report was written to `docs/roadmap/audit.md`. `docs/roadmap/` already exists in the repository, although its previous `publish.md` file is currently deleted in the worktree by an unrelated change.
|
|
||||||
|
|
||||||
## 2. Repository map reviewed
|
|
||||||
|
|
||||||
Reviewed documentation:
|
|
||||||
|
|
||||||
- `README.md`
|
|
||||||
- `docs/configuration.md`
|
|
||||||
- `docs/architecture/architecture.md`
|
|
||||||
- `docs/architecture/public-contract.md`
|
|
||||||
- `docs/architecture/diagnostics.md`
|
|
||||||
- `docs/architecture/output-schemas.md`
|
|
||||||
- `docs/architecture/prompts.md`
|
|
||||||
- `docs/architecture/validators.md`
|
|
||||||
- `docs/architecture/structured-llm.md`
|
|
||||||
- `docs/integration/subprocess-operations.md`
|
|
||||||
- `docs/release-checklist.md`
|
|
||||||
|
|
||||||
Reviewed implementation areas:
|
|
||||||
|
|
||||||
- `cmd/audita`
|
|
||||||
- `internal/cli`
|
|
||||||
- `internal/core/config`
|
|
||||||
- `internal/core/schema`
|
|
||||||
- `internal/core/io`
|
|
||||||
- `internal/core/normalization`
|
|
||||||
- `internal/core/chunking`
|
|
||||||
- `internal/core/diagnostics`
|
|
||||||
- `internal/core/outputschema`
|
|
||||||
- `internal/core/reporting`
|
|
||||||
- `internal/framework/contracts`
|
|
||||||
- `internal/framework/modules`
|
|
||||||
- `internal/framework/proposal_generation`
|
|
||||||
- `internal/framework/proposals`
|
|
||||||
- `internal/framework/runner`
|
|
||||||
- `internal/framework/validators`
|
|
||||||
- `internal/framework/llm`
|
|
||||||
- `internal/framework/responseschema`
|
|
||||||
- `internal/framework/promptcontext`
|
|
||||||
- `internal/framework/warnings`
|
|
||||||
- `internal/modules/glossary`
|
|
||||||
- `internal/modules/homophones`
|
|
||||||
- `internal/modules/spoken_word`
|
|
||||||
- `internal/modules/grammar`
|
|
||||||
- `internal/prompts`
|
|
||||||
- `internal/validators`
|
|
||||||
- package tests and CLI parity/release fixtures under `internal/cli/testdata`
|
|
||||||
|
|
||||||
Major execution paths reviewed:
|
|
||||||
|
|
||||||
- `audita process <transcript.json> --glossary <glossary.yaml>`
|
|
||||||
- `audita config validate --config <path>`
|
|
||||||
- `audita config print-effective [--config <path>]`
|
|
||||||
- default module sequence resolution and repeated glossary instance naming
|
|
||||||
- proposal generation, validator execution, proposal application, report writing, diagnostics writing, and retention
|
|
||||||
|
|
||||||
Important absent or not-applicable areas:
|
|
||||||
|
|
||||||
- No `pkg/` directory exists.
|
|
||||||
- No `examples/` directory exists.
|
|
||||||
- No `docs/internal/` directory exists.
|
|
||||||
- No `internal/app`, `internal/stage`, `internal/storage`, `internal/artifacts`, or `internal/manifest` packages exist. Their closest equivalents are `internal/cli`, `internal/framework/runner`, `internal/core/diagnostics`, and `internal/core/reporting`.
|
|
||||||
|
|
||||||
## 3. High-confidence deduplication opportunities
|
|
||||||
|
|
||||||
### 3.1 Module proposal plumbing is duplicated across all production modules
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/modules/glossary/module.go`
|
|
||||||
- `internal/modules/homophones/module.go`
|
|
||||||
- `internal/modules/spoken_word/module.go`
|
|
||||||
- `internal/modules/grammar/module.go`
|
|
||||||
- `internal/modules/*/prompt.go`
|
|
||||||
- `internal/framework/proposal_generation`
|
|
||||||
- `internal/framework/promptcontext`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Each module has the same `Module` struct shape, `Validators` copy behavior, `Propose` flow, section transcript extraction, transcript description extraction, `proposal_generation.GenerateCandidates` request construction, prompt metadata map construction, and stage-name formatting.
|
|
||||||
- Each module also has a near-identical prompt payload builder with local `promptSegment` and `promptTranscriptSection` types, glossary JSON marshaling, transcript section JSON marshaling, transcript description block rendering, and two-message return shape.
|
|
||||||
- `collectSectionProposals` already passes a section transcript to each module, but each module then filters that transcript again by section metadata.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- A diagnostics or prompt-context bug fix would need to be repeated in four modules.
|
|
||||||
- Prompt metadata fields and stage names are diagnostics-visible and could drift by module.
|
|
||||||
- The double section filtering is currently harmless, but it obscures the runner/module contract.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Add a small shared helper for module proposal execution, likely in `internal/framework/proposal_generation` or a narrow `internal/modules/modulekit` package.
|
|
||||||
- Keep domain-specific prompt IDs and prompt text local to each module.
|
|
||||||
- Move transcript section prompt payload construction into a shared prompt-context helper, for example `promptcontext.MarshalTranscriptSection`.
|
|
||||||
- Provide one helper for prompt metadata maps instead of manually expanding `prompt_id`, `prompt_version`, `prompt_source`, `embedded_path`, and `sha256` in every module.
|
|
||||||
- Preserve current module `Key`, replacement policy, and validator chain ownership.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Keep one golden or table-driven prompt payload test per module for domain-specific wording.
|
|
||||||
- Add shared tests for transcript section JSON shape, empty transcript handling, categories copy behavior, and prompt metadata fields.
|
|
||||||
- Add a parity test that all four module `Propose` methods still write diagnostics under the same module instance directory and produce the same correction mapping.
|
|
||||||
|
|
||||||
Risk level:
|
|
||||||
|
|
||||||
- Low to medium. The behavior is highly duplicated, but prompt and diagnostics behavior is sensitive. Refactor behind existing module tests and CLI parity fixtures.
|
|
||||||
|
|
||||||
### 3.2 Effective config loading is repeated between commands
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/cli/run.go`
|
|
||||||
- `internal/core/config`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- `runProcess` and `runConfigPrintEffective` both resolve config path, start from defaults, optionally load/apply file config, then apply environment overrides.
|
|
||||||
- `runConfigValidate` separately loads a file, applies it to defaults, and validates it.
|
|
||||||
- Path source metadata is computed in `internal/cli`, not `internal/core/config`, even though the precedence contract is documented as config behavior.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Config precedence is part of the public contract. If a future setting is added, three command paths may need coordinated updates.
|
|
||||||
- `config print-effective` is the user-visible diagnostic for effective config. It should use the same loader as `process`, except for intentionally omitted CLI overrides.
|
|
||||||
- The current code is understandable, but the behavior is repeated in a way that makes drift likely as config grows.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Add a narrow effective-config loader in `internal/core/config`, returning `Config`, source path, source type, and version metadata.
|
|
||||||
- Keep command-specific CLI flag parsing in `internal/cli`.
|
|
||||||
- Model the intentional differences explicitly:
|
|
||||||
- `process`: defaults + file + env + CLI overrides
|
|
||||||
- `config print-effective`: defaults + file + env
|
|
||||||
- `config validate`: file schema + default-backed config validation, no env
|
|
||||||
- Move `resolveConfigPath` or an equivalent path resolver into `internal/core/config`.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- One table-driven config loader test covering explicit `--config`, `AUDITA_CONFIG`, default search paths, missing explicit paths, and missing default paths.
|
|
||||||
- CLI tests asserting `process` and `print-effective` share file+env behavior.
|
|
||||||
- A regression test that `config validate` remains file-only and does not read environment overrides.
|
|
||||||
|
|
||||||
Risk level:
|
|
||||||
|
|
||||||
- Low. Behavior is already explicit and well tested; the refactor can be done by moving code without changing precedence.
|
|
||||||
|
|
||||||
### 3.3 Module catalog validation is split across config, contracts, and module factory
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/core/config/validation.go`
|
|
||||||
- `internal/framework/contracts/contracts.go`
|
|
||||||
- `internal/framework/modules/registry.go`
|
|
||||||
- `internal/validators/chains.go`
|
|
||||||
- `internal/framework/validators/models.go`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Module keys appear in multiple places:
|
|
||||||
- config default CSV: `glossary,homophones,glossary,spoken_word,grammar`
|
|
||||||
- module factory constants and known-key map
|
|
||||||
- built-in validator chains
|
|
||||||
- confidence threshold lookup
|
|
||||||
- individual module `Key()` methods
|
|
||||||
- `Config.Validate` checks only that module names are non-empty. An unsupported configured module can pass `audita config validate` and fail later in `process` runner setup.
|
|
||||||
- `contracts.ResolveModuleRunSpecs` only assigns instance names; it does not validate production module support.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- `audita config validate` is documented as a CI/preflight command. Letting unsupported modules pass weakens that preflight.
|
|
||||||
- Module key drift could affect thresholds, validator chains, reports, and unsupported-module errors.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Introduce a small canonical module catalog or key package that can be imported by config validation, module factory construction, validator chain resolution, and threshold lookup without creating a cycle.
|
|
||||||
- Keep module construction in `internal/framework/modules`; the catalog should expose keys and validation only.
|
|
||||||
- Make `Config.Validate` reject unknown built-in module keys through that catalog.
|
|
||||||
- Keep repeated module instances valid.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- `internal/core/config` test: unknown `pipeline.modules` fails validation.
|
|
||||||
- `internal/cli` test: `audita config validate --config` rejects an unsupported module before runtime.
|
|
||||||
- Existing `internal/framework/modules` unknown-module tests should continue to pass.
|
|
||||||
- Validator chain tests should assert every catalog module has a built-in chain.
|
|
||||||
|
|
||||||
Risk level:
|
|
||||||
|
|
||||||
- Medium. This tightens validation behavior. It is desirable before 1.0, but if unknown modules were intentionally allowed for future extension, document that explicitly instead.
|
|
||||||
|
|
||||||
### 3.4 Output schema support is hardcoded in config validation and registry
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/core/config/validation.go`
|
|
||||||
- `internal/core/outputschema/registry.go`
|
|
||||||
- `docs/architecture/output-schemas.md`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- `Config.Validate` hardcodes `bare-segments` and `audita-v1`.
|
|
||||||
- `outputschema.Resolve` owns the actual output schema registry and returns the runtime error for unsupported schema names.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Adding or deferring a schema requires updating multiple places.
|
|
||||||
- Public behavior could drift: a schema might validate in config but fail at output time, or vice versa.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Make `internal/core/outputschema` expose `IsSupported`, `SupportedKeys`, or a validation function.
|
|
||||||
- Have config validation call that helper or consume shared constants.
|
|
||||||
- Keep actual encoding logic in `outputschema`; config should not know encoder details.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Config validation test for every output schema returned by the registry.
|
|
||||||
- Output schema registry test that unsupported `seriatim-intermediate` still fails clearly until implemented.
|
|
||||||
- CLI test that unsupported `--output-schema` fails before output write.
|
|
||||||
|
|
||||||
Risk level:
|
|
||||||
|
|
||||||
- Low. This is a straightforward catalog centralization.
|
|
||||||
|
|
||||||
### 3.5 Diagnostics artifact names and report metadata paths are repeated
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/core/diagnostics/run_dir.go`
|
|
||||||
- `internal/cli/run.go`
|
|
||||||
- `internal/core/reporting/report.go`
|
|
||||||
- docs under `docs/architecture` and `docs/integration`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Artifact filenames such as `source-transcript.json`, `source-transcript-parsed.json`, `normalized-transcript.json`, `normalization-summary.json`, `chunking-summary.json`, `utilization-diagnostics.json`, `correction-ledger.json`, `invocation.json`, `effective-config.json`, `report.json`, and `error.log` are repeated between run-directory writers and `buildProcessReport`.
|
|
||||||
- `runProcess` writes `utilization-diagnostics.json` and `correction-ledger.json` by raw string on both success and failure paths.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- These names are part of the documented diagnostics contract.
|
|
||||||
- A filename change would need to be made in multiple places, and report metadata could point at files that are no longer written.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Define diagnostics artifact name constants in `internal/core/diagnostics`.
|
|
||||||
- Add a helper that returns `reporting.DiagnosticsMetadata` for a run directory and status.
|
|
||||||
- Add named methods for utilization diagnostics and correction ledger writes, or at least constants used by `WriteJSONArtifact`.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Unit test that `diagnostics.MetadataForRunDirectory` matches files written by `RunDirectory`.
|
|
||||||
- CLI success/failure tests should continue to assert report metadata paths and actual file existence.
|
|
||||||
- Add a test for failure report metadata including `error.log`.
|
|
||||||
|
|
||||||
Risk level:
|
|
||||||
|
|
||||||
- Low. This is mostly string centralization, with high public-contract value.
|
|
||||||
|
|
||||||
### 3.6 Validator execution class is duplicated and partially hardcoded
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/validators/registry.go`
|
|
||||||
- `internal/validators/metadata/metadata.go`
|
|
||||||
- `internal/validators/*/validator.go`
|
|
||||||
- `internal/framework/runner/runner.go`
|
|
||||||
- `internal/cli/review_artifacts.go`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Validator constructors wrap validators with execution class metadata.
|
|
||||||
- `BuiltInValidatorDefinition` also has an `LLMBacked` field.
|
|
||||||
- Runner uses `metadata.ClassOf` to order deterministic validators before LLM-backed validators.
|
|
||||||
- Correction ledger classification uses a local hardcoded map of LLM-backed validator names.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Adding a new LLM-backed validator could be ordered correctly by runner metadata but appear in the wrong correction-ledger section.
|
|
||||||
- Validator class is domain metadata, not report-building policy. It should have one source of truth.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Make validator classification resolvable by validator instance or stable key from a single metadata source.
|
|
||||||
- Remove the unused or redundant `LLMBacked` field, or make it the canonical source used by constructors, runner ordering, and ledger formatting.
|
|
||||||
- Replace the local ledger map with `metadata.ClassOf` when possible, or a registry lookup by stable key.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Correction ledger test that LLM-backed decisions are classified from validator metadata, not a local string map.
|
|
||||||
- Registry test that every registered LLM-backed validator reports the same class through every public metadata path.
|
|
||||||
- Runner ordering test should remain in place.
|
|
||||||
|
|
||||||
Risk level:
|
|
||||||
|
|
||||||
- Low to medium. The implementation is small, but correction-ledger shape is diagnostics-visible.
|
|
||||||
|
|
||||||
### 3.7 Malformed structured-output classification is duplicated
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/framework/proposal_generation/generate.go`
|
|
||||||
- `internal/framework/validators/llm_validators.go`
|
|
||||||
- `internal/framework/llm/openai_compatible_client.go`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Proposal generation and LLM validators both classify malformed structured-output errors by scanning error message substrings.
|
|
||||||
- The marker lists are currently the same, but they are maintained independently.
|
|
||||||
- The actual errors originate in the LLM adapter.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Proposal-generation malformed payloads become warnings with zero proposals, while validator malformed payloads reject affected batches with warnings. If classifiers drift, similar adapter failures could be downgraded in one workflow and hard-fail in another.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Prefer a typed error or exported classifier from `internal/framework/llm`.
|
|
||||||
- If typed errors are too invasive, create one shared classifier function in a lower framework package used by both proposal generation and validators.
|
|
||||||
- Preserve the different handling semantics at each call site.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Shared classifier table for all adapter malformed-output errors.
|
|
||||||
- Proposal-generation test and validator test should assert the same representative malformed adapter errors are downgraded.
|
|
||||||
- Adapter tests should assert typed/classified errors wrap useful context and still redact secrets.
|
|
||||||
|
|
||||||
Risk level:
|
|
||||||
|
|
||||||
- Medium. Error typing can accidentally affect retry and wrapping behavior; do this with focused tests.
|
|
||||||
|
|
||||||
## 4. Medium-confidence opportunities
|
|
||||||
|
|
||||||
### 4.1 CLI flag registration and override extraction are large and repetitive
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/cli/run.go`
|
|
||||||
- `internal/core/config/flags.go`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Each process flag has a field in `processFlags`, a registration entry in `newProcessFlagSet`, a case in `fs.Visit`, and an assignment in `config.ApplyCLIOverrides`.
|
|
||||||
- File config and environment config also set many of the same effective config fields.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Adding a new config option requires multiple edits. Missing one edit could create a flag that displays but does not override, or a config field with no CLI override.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Avoid a generic reflection-heavy flag system before 1.0.
|
|
||||||
- Consider a small metadata table only for simple scalar flags, or a focused helper that maps visited flags to `CLIOverrides`.
|
|
||||||
- Keep nontrivial semantics, such as legacy concurrency alias precedence, explicit in code.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- CLI override parity test for every stable flag that mutates config.
|
|
||||||
- A test that default flag values reflect file+env effective config before CLI overrides.
|
|
||||||
|
|
||||||
Risk level:
|
|
||||||
|
|
||||||
- Medium. A broad flag abstraction would be riskier than the current duplication. Do only a small helper if it clearly reduces missed updates.
|
|
||||||
|
|
||||||
### 4.2 Config source application repeats field-level assignments
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/core/config/file_config.go`
|
|
||||||
- `internal/core/config/env.go`
|
|
||||||
- `internal/core/config/flags.go`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- The same effective fields are assigned from file config, env vars, and CLI overrides.
|
|
||||||
- Some semantics differ intentionally: file config supports `api_key_env`, env supports `OPENROUTER_API_KEY` fallback, CLI uses direct values.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Field additions are easy to miss in one source.
|
|
||||||
- Error messages and trimming behavior can drift.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Do not force all config sources through one generic mapper.
|
|
||||||
- Add small setter helpers for repeated config subdomains such as LLM target, concurrency, thresholds, normalization, and diagnostics.
|
|
||||||
- Keep source-specific parsing and error labels local.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Cross-source table proving file, env, and CLI all reach the same effective fields where they are meant to.
|
|
||||||
- Tests for intentional differences: API key env resolution, `OPENROUTER_API_KEY` fallback, CLI direct API key, and transcript description trimming.
|
|
||||||
|
|
||||||
Risk level:
|
|
||||||
|
|
||||||
- Medium. Useful, but only after the effective loader and catalog cleanup.
|
|
||||||
|
|
||||||
### 4.3 Prompt metadata and response schema metadata map construction repeats
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/modules/*/module.go`
|
|
||||||
- `internal/framework/proposal_generation/generate.go`
|
|
||||||
- `internal/framework/validators/llm_validators.go`
|
|
||||||
- `internal/prompts`
|
|
||||||
- `internal/framework/responseschema`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Prompt metadata maps are manually expanded in module proposal generation and validator diagnostics.
|
|
||||||
- Response schema metadata maps are built independently in proposal generation and validator diagnostics.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Metadata fields are diagnostics-visible and useful for reproducibility.
|
|
||||||
- Adding a metadata field requires updating multiple call sites.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Add `Metadata.Map()` or a typed diagnostics metadata struct in `internal/prompts`.
|
|
||||||
- Add `responseschema.Metadata()` or a method returning a stable diagnostics shape.
|
|
||||||
- Prefer typed structs over `map[string]any` where possible.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Prompt metadata rendering test should assert all registered prompts expose stable metadata.
|
|
||||||
- Proposal and validator diagnostics tests should assert the shared metadata helper is used.
|
|
||||||
|
|
||||||
Risk level:
|
|
||||||
|
|
||||||
- Low.
|
|
||||||
|
|
||||||
### 4.4 Secret redaction logic is split across config, LLM diagnostics, and adapter errors
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/core/config/redaction.go`
|
|
||||||
- `internal/framework/llm/diagnostics.go`
|
|
||||||
- `internal/framework/llm/client_common.go`
|
|
||||||
- `internal/framework/proposal_generation/generate.go`
|
|
||||||
- `internal/framework/runner/runner.go`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Config redaction replaces non-empty API keys with `[REDACTED]`.
|
|
||||||
- LLM diagnostics replace configured secret values and `Bearer <secret>`.
|
|
||||||
- Adapter error sanitization separately replaces secrets and bearer values.
|
|
||||||
- Proposal and validator paths separately assemble secret lists.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Secret redaction is a public guarantee.
|
|
||||||
- New secret-bearing config fields could be missed in one path.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Add a small redaction helper package or keep it in `internal/framework/llm` only if it remains LLM-specific.
|
|
||||||
- Centralize `[]string` secret extraction from `config.Config`.
|
|
||||||
- Keep config structural redaction separate from byte/string payload redaction, but share the redaction token and value replacement behavior.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- One test that a proposal-generation error, validator diagnostic artifact, effective config artifact, and surfaced provider error all redact the same configured secrets.
|
|
||||||
- Existing subprocess no-secret-leak test should remain as an end-to-end guard.
|
|
||||||
|
|
||||||
Risk level:
|
|
||||||
|
|
||||||
- Medium. The current coverage appears strong; change carefully.
|
|
||||||
|
|
||||||
### 4.5 Test fakes and fixture helpers are duplicated across packages
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/modules/*/module_test.go`
|
|
||||||
- `internal/framework/proposal_generation/generate_test.go`
|
|
||||||
- `internal/framework/validators/llm_validators_test.go`
|
|
||||||
- `internal/cli/run_test.go`
|
|
||||||
- `cmd/audita/main_integration_test.go`
|
|
||||||
- `internal/cli/release_fixtures_test.go`
|
|
||||||
- `internal/cli/parity_test.go`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Several packages define fake structured LLM clients, fixture path helpers, read/write helpers, diagnostics glob assertions, and run-directory helpers.
|
|
||||||
- The four module test files have particularly similar fake clients and proposal-diagnostics assertions.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Refactors in LLM or diagnostics behavior require updating many tests.
|
|
||||||
- Some duplicated tests are valuable because they preserve per-module public behavior; the issue is helper duplication, not coverage volume.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Add package-local helper files where duplication is within a package.
|
|
||||||
- For cross-package fakes, prefer a small internal test support package only if it does not create import cycles or hide test intent.
|
|
||||||
- Keep module-specific assertions local.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- This is test infrastructure cleanup. Existing tests should remain semantically equivalent.
|
|
||||||
- Add helper tests only if helpers contain nontrivial behavior, such as fake response sequencing.
|
|
||||||
|
|
||||||
Risk level:
|
|
||||||
|
|
||||||
- Low.
|
|
||||||
|
|
||||||
### 4.6 Stage-name construction is inconsistent enough to centralize, but not enough to redesign
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/modules/*/module.go`
|
|
||||||
- `internal/framework/proposal_generation/generate.go`
|
|
||||||
- `internal/framework/validators/llm_validators.go`
|
|
||||||
- `internal/framework/runner/observability.go`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Modules pass stage names like `<module_instance>:proposal:section-0001`.
|
|
||||||
- `proposal_generation` has a default builder using `<module_instance>:proposal-generation:section-0001`, but production modules bypass it.
|
|
||||||
- Validators build `<module_instance>:<validator>:batch-0001`.
|
|
||||||
- Utilization extracts module instance by splitting stage names on `:`.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Stage names affect diagnostics filenames and observability grouping.
|
|
||||||
- Current behavior works, but the naming grammar is implicit.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Add narrow helpers for proposal and validator stage names.
|
|
||||||
- Preserve current production stage names unless there is a deliberate pre-1.0 diagnostics compatibility decision.
|
|
||||||
- Keep filename sanitization in `internal/framework/llm`.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Unit tests for stage-name helper output.
|
|
||||||
- Utilization test that module instance extraction still works for proposal and validator stage names.
|
|
||||||
|
|
||||||
Risk level:
|
|
||||||
|
|
||||||
- Medium. Renaming stages can change diagnostics filenames, so avoid unnecessary churn.
|
|
||||||
|
|
||||||
## 5. Boundary and responsibility concerns
|
|
||||||
|
|
||||||
### CLI owns too much report and diagnostics metadata assembly
|
|
||||||
|
|
||||||
`internal/cli/run.go` is doing orchestration, command parsing, config loading, output routing, report assembly, diagnostics metadata path assembly, and correction-ledger construction. This is acceptable for a small CLI, but two pieces are drifting beyond command responsibility:
|
|
||||||
|
|
||||||
- diagnostics artifact path metadata belongs closer to `internal/core/diagnostics`;
|
|
||||||
- report assembly and correction-ledger mapping belong closer to `internal/core/reporting` or a narrow reporting adapter package.
|
|
||||||
|
|
||||||
Recommended home:
|
|
||||||
|
|
||||||
- `internal/core/diagnostics`: artifact constants and diagnostics metadata path construction.
|
|
||||||
- `internal/core/reporting`: pure mapping from runner/config/diagnostics state into report payloads.
|
|
||||||
- `internal/cli`: command parsing, invocation wiring, exit codes, stdout/stderr behavior.
|
|
||||||
|
|
||||||
### Config validation lacks catalog ownership
|
|
||||||
|
|
||||||
`internal/core/config` currently validates only generic module list shape and hardcodes output schema keys. Because modules and output schemas are public contract values, config validation should use a catalog owned by the relevant domain.
|
|
||||||
|
|
||||||
Recommended home:
|
|
||||||
|
|
||||||
- output schema validation: `internal/core/outputschema`;
|
|
||||||
- module key validation: a small catalog package or lower-level constants package importable by config, module factory, validator chains, and threshold lookup.
|
|
||||||
|
|
||||||
### Runner owns adapter shims between contracts and validator framework
|
|
||||||
|
|
||||||
`internal/framework/runner` contains `validationLLMClientAdapter` and `llmDiagnosticsWriterAdapter`. This is not a serious problem today because runner wires proposal and validation workflows. If these adapters grow, move them to `internal/framework/validators` or a small integration package so runner remains focused on orchestration.
|
|
||||||
|
|
||||||
### LLM malformed-output policy is spread across callers
|
|
||||||
|
|
||||||
The LLM adapter emits the errors, while proposal generation and validators classify them by message text. The policy decision is caller-specific, but the classification should live with the LLM/framework error type.
|
|
||||||
|
|
||||||
## 6. Path, key, and naming construction review
|
|
||||||
|
|
||||||
Centralized enough:
|
|
||||||
|
|
||||||
- LLM diagnostics artifact suffixes and stage sanitization are centralized in `internal/framework/llm/diagnostics.go`.
|
|
||||||
- Output file writing is routed through `internal/core/io.WriteFile`.
|
|
||||||
- Run directories are created in `internal/core/diagnostics.NewRunDirectory`.
|
|
||||||
|
|
||||||
Needs cleanup:
|
|
||||||
|
|
||||||
- Core diagnostics artifact names are repeated between `RunDirectory` writer methods and `buildProcessReport`.
|
|
||||||
- `utilization-diagnostics.json` and `correction-ledger.json` are raw strings in both success and failure paths.
|
|
||||||
- Proposal and validator diagnostics subdirectory construction repeats `filepath.Join(diagnosticsDir, moduleInstance)`.
|
|
||||||
- Proposal and validator stage names are manually formatted in multiple packages.
|
|
||||||
- Module keys are repeated across config defaults, module factory, validator chains, confidence threshold lookup, and module implementations.
|
|
||||||
- Output schema names are repeated between config validation and `outputschema`.
|
|
||||||
|
|
||||||
Recommendation:
|
|
||||||
|
|
||||||
- Start with artifact constants and metadata helpers because that is the lowest-risk path/key cleanup.
|
|
||||||
- Then centralize stage-name helpers without changing current production naming.
|
|
||||||
- Defer any broader "path manager" abstraction.
|
|
||||||
|
|
||||||
## 7. Resolution and catalog review
|
|
||||||
|
|
||||||
Modules:
|
|
||||||
|
|
||||||
- Runtime module construction has a production registry in `internal/framework/modules`.
|
|
||||||
- Instance naming for repeated modules is centralized in `contracts.ResolveModuleRunSpecs`.
|
|
||||||
- Unknown module failure exists in the factory, but config validation does not catch unknown modules.
|
|
||||||
- Built-in validator chain resolution separately maps module key to validator keys.
|
|
||||||
|
|
||||||
Output schemas:
|
|
||||||
|
|
||||||
- Encoding is centralized in `internal/core/outputschema`.
|
|
||||||
- Validation is duplicated in config.
|
|
||||||
|
|
||||||
Prompts:
|
|
||||||
|
|
||||||
- Prompt asset lookup and metadata are centralized in `internal/prompts`.
|
|
||||||
- Prompt metadata map construction is repeated at call sites.
|
|
||||||
- Prompt source selection is intentionally built-in only and should remain that way for 1.0.
|
|
||||||
|
|
||||||
Validators:
|
|
||||||
|
|
||||||
- Validator construction is package-owned under `internal/validators`.
|
|
||||||
- Chains are centralized in `internal/validators/chains.go`.
|
|
||||||
- Execution class metadata exists, but reporting/correction-ledger classification does not fully use it.
|
|
||||||
|
|
||||||
Schemas:
|
|
||||||
|
|
||||||
- Transcript and glossary parsing/validation are centralized in `internal/core/schema`.
|
|
||||||
- Structured LLM response schemas are centralized in `internal/framework/responseschema`.
|
|
||||||
- Output schema registry and response schema registry are appropriately separate.
|
|
||||||
|
|
||||||
Recommendation:
|
|
||||||
|
|
||||||
- Introduce only small catalog helpers for module keys, output schema keys, prompt metadata maps, response schema metadata maps, and validator execution class.
|
|
||||||
- Avoid user-configurable modules, validators, prompts, or schemas before 1.0 unless already planned elsewhere.
|
|
||||||
|
|
||||||
## 8. Config and command-loading review
|
|
||||||
|
|
||||||
Consistent behavior:
|
|
||||||
|
|
||||||
- The documented precedence for `process` is implemented: defaults, file config, environment, CLI.
|
|
||||||
- `config print-effective` intentionally omits CLI process flags and uses defaults, file config, and environment.
|
|
||||||
- `config validate` intentionally requires `--config` and does not require transcript/glossary inputs.
|
|
||||||
- Missing explicit config paths are hard failures; missing default paths are non-fatal.
|
|
||||||
- Environment parsing and CLI parsing both preserve legacy total-concurrency alias behavior.
|
|
||||||
|
|
||||||
Likely accidental or high-risk differences:
|
|
||||||
|
|
||||||
- Unsupported module names pass `Config.Validate` and `audita config validate`.
|
|
||||||
- Output schema support is duplicated instead of delegated to the output schema registry.
|
|
||||||
- Config path resolution lives in CLI even though it is part of config behavior.
|
|
||||||
|
|
||||||
Intentional differences:
|
|
||||||
|
|
||||||
- File config resolves `api_key_env`; env and CLI set direct API key values.
|
|
||||||
- `OPENROUTER_API_KEY` is an environment fallback only for the primary LLM.
|
|
||||||
- `transcript-description` has CLI/config support but no `AUDITA_*` environment variable, matching documentation.
|
|
||||||
|
|
||||||
Recommendation:
|
|
||||||
|
|
||||||
- Build a shared effective config context helper and keep source-specific parsing semantics explicit.
|
|
||||||
- Tighten catalog validation before 1.0 if unknown modules are not meant to be accepted.
|
|
||||||
|
|
||||||
## 9. State, manifest, or progress handling review
|
|
||||||
|
|
||||||
Audita does not currently have a manifest/checkpoint/resume model. State is per-run diagnostics and report artifacts.
|
|
||||||
|
|
||||||
Consistent behavior:
|
|
||||||
|
|
||||||
- `process` creates one diagnostics run directory when diagnostics initialization succeeds.
|
|
||||||
- Failures after run-dir creation write `error.log`, best-effort report artifacts, and retain diagnostics.
|
|
||||||
- Success writes optional `--report-json`, run-dir `report.json`, utilization diagnostics, and correction ledger.
|
|
||||||
- Retention is centralized in `diagnostics.ShouldRetainRunDirectory`.
|
|
||||||
- There is no resume/retry/force behavior to preserve.
|
|
||||||
|
|
||||||
Drift risks:
|
|
||||||
|
|
||||||
- Success and failure paths both write utilization and correction-ledger artifacts with duplicated raw filenames.
|
|
||||||
- Report diagnostics metadata is assembled independently from the run-directory writer methods.
|
|
||||||
- Retention mode `never` currently still retains successful run directories in `ShouldRetainRunDirectory`, which may be intentional per tests or a naming/documentation mismatch. Do not change it in a dedup pass without first confirming semantics.
|
|
||||||
|
|
||||||
Recommendation:
|
|
||||||
|
|
||||||
- Centralize artifact names and report metadata path construction.
|
|
||||||
- Keep retention behavior unchanged unless a separate bug review confirms the intended meaning of `never`.
|
|
||||||
|
|
||||||
## 10. Refactors to avoid before 1.0
|
|
||||||
|
|
||||||
- Do not introduce a generic workflow engine. The current sequential runner is clear and explicit.
|
|
||||||
- Do not add a plugin architecture for modules, validators, prompts, or schemas before 1.0.
|
|
||||||
- Do not redesign the CLI or replace `flag` with a larger framework only for deduplication.
|
|
||||||
- Do not collapse all config source parsing into a reflection-based mapper; source semantics differ intentionally.
|
|
||||||
- Do not merge module packages into one generic module type. Keep domain-specific prompt assets, keys, validator chains, and replacement policies visible.
|
|
||||||
- Do not rewrite diagnostics or reporting schemas broadly. Centralize names and mapping helpers first.
|
|
||||||
- Do not change diagnostics stage names casually; they affect artifact filenames and debugging workflows.
|
|
||||||
- Do not consolidate deterministic and LLM validator behavior just because both return decisions. Their failure and batching semantics differ.
|
|
||||||
- Do not generalize transcript/glossary schema parsing into a broad schema framework.
|
|
||||||
- Do not reduce duplicated tests where the duplication protects distinct public command/module behavior.
|
|
||||||
|
|
||||||
## 11. Recommended implementation sequence
|
|
||||||
|
|
||||||
1. Centralize diagnostics artifact constants and diagnostics metadata path construction.
|
|
||||||
2. Centralize output schema validation through `internal/core/outputschema`.
|
|
||||||
3. Introduce a small module key catalog and use it in config validation, module factory, validator chains, and threshold lookup.
|
|
||||||
4. Add an effective config loading context helper for defaults + file + env, then update `process` and `config print-effective`.
|
|
||||||
5. Extract shared module proposal plumbing and prompt transcript-section payload construction.
|
|
||||||
6. Centralize prompt metadata and response schema metadata map construction.
|
|
||||||
7. Centralize validator execution-class lookup and update correction-ledger classification.
|
|
||||||
8. Centralize malformed structured-output classification through a typed/shared LLM error helper.
|
|
||||||
9. Add or consolidate focused test helpers for module LLM fakes, diagnostics assertions, and fixture paths.
|
|
||||||
10. Do a final dead-code and legacy sweep for redundant helper fields such as unused validator definition metadata.
|
|
||||||
|
|
||||||
Each item can be a separate commit with package-level tests and at least one CLI regression where public behavior is involved.
|
|
||||||
|
|
||||||
## 12. Test strategy
|
|
||||||
|
|
||||||
Tests to add before refactoring:
|
|
||||||
|
|
||||||
- `internal/core/config`: unknown module key fails validation, if unsupported modules are not intended to be accepted.
|
|
||||||
- `internal/core/config`: every output schema registry key validates through config.
|
|
||||||
- `internal/core/diagnostics`: report metadata paths match run-directory artifact names.
|
|
||||||
- `internal/validators`: validator class by key/instance is consistent for all registered validators.
|
|
||||||
- `internal/framework/llm`: shared malformed structured-output classifier covers all current adapter malformed errors.
|
|
||||||
|
|
||||||
Tests to add during refactoring:
|
|
||||||
|
|
||||||
- `internal/framework/promptcontext`: transcript section prompt payload preserves IDs, speaker, timestamps, text, and categories.
|
|
||||||
- `internal/framework/proposal_generation`: shared module proposal helper preserves current stage name, diagnostics dir, schema metadata, and malformed-output warning behavior.
|
|
||||||
- `internal/cli`: `process` and `config print-effective` share defaults+file+env behavior.
|
|
||||||
- `internal/cli`: `config validate` remains file-only and does not read env overrides.
|
|
||||||
- `internal/cli`: correction ledger classifies deterministic and LLM validator decisions through canonical metadata.
|
|
||||||
|
|
||||||
Existing tests to run after each cleanup:
|
|
||||||
|
|
||||||
- `go test ./internal/core/config ./internal/core/outputschema`
|
|
||||||
- `go test ./internal/core/diagnostics ./internal/core/reporting`
|
|
||||||
- `go test ./internal/framework/proposal_generation ./internal/framework/validators ./internal/framework/runner`
|
|
||||||
- `go test ./internal/validators/...`
|
|
||||||
- `go test ./internal/modules/...`
|
|
||||||
- `go test ./internal/cli ./cmd/audita`
|
|
||||||
- Run `go test ./...` before merging a multi-package cleanup.
|
|
||||||
|
|
||||||
Validation note:
|
|
||||||
|
|
||||||
- During this report-only pass, no full test suite was run. A lightweight `go list ./...` completed package listing but emitted a sandbox warning while trying to write the Go module stat cache outside the repository.
|
|
||||||
|
|
||||||
## 13. Appendix: findings not worth acting on
|
|
||||||
|
|
||||||
### Separate module packages
|
|
||||||
|
|
||||||
The four production module packages contain visible repetition, but keeping separate packages is useful. The module domains, prompt assets, validator chains, and tests are distinct enough that a single generic module package would hide important behavior.
|
|
||||||
|
|
||||||
Do not refactor now beyond shared proposal/prompt plumbing.
|
|
||||||
|
|
||||||
### Report type duplication between runner and reporting
|
|
||||||
|
|
||||||
`runner.ModuleResult` and `reporting.ModuleReport` look similar. Keeping separate runtime and public report shapes is reasonable because runner owns execution state and reporting owns serialized public schema.
|
|
||||||
|
|
||||||
Only centralize mapping helpers; do not merge the types.
|
|
||||||
|
|
||||||
### Transcript and glossary parsing stay separate
|
|
||||||
|
|
||||||
Transcript JSON and glossary YAML parsing have different formats, validation rules, and error messages. There is no useful shared parser abstraction to extract.
|
|
||||||
|
|
||||||
### Response schema registry and output schema registry stay separate
|
|
||||||
|
|
||||||
Structured LLM response schemas and transcript output schemas are both "schemas", but they serve different users and have different lifecycles. Do not combine their registries.
|
|
||||||
|
|
||||||
### `flag` package usage
|
|
||||||
|
|
||||||
The CLI command surface is small. Replacing `flag` with a larger CLI framework would not pay for itself before 1.0.
|
|
||||||
|
|
||||||
### Local test duplication that protects public behavior
|
|
||||||
|
|
||||||
Some test duplication in CLI, subprocess, parity, and release fixtures is intentional. These tests exercise different public surfaces and should remain explicit even if helpers are shared.
|
|
||||||
|
|
||||||
### Filesystem state as diagnostics state
|
|
||||||
|
|
||||||
Audita has no resume/checkpoint semantics. Treating diagnostics artifacts as filesystem outputs is currently acceptable. A manifest system would be speculative before there is a resume or audit workflow that needs it.
|
|
||||||
@@ -1,329 +0,0 @@
|
|||||||
# Pre-1.0 Deduplication Implementation Plan
|
|
||||||
|
|
||||||
This plan turns `docs/roadmap/audit.md` into staged, prompt-sized cleanup work for an LLM coding agent. Each stage should be implemented in order and kept small enough to review as an independent commit.
|
|
||||||
|
|
||||||
## Operating rules
|
|
||||||
|
|
||||||
- Read `docs/roadmap/audit.md` before starting any stage.
|
|
||||||
- Preserve public CLI, report, diagnostics, config precedence, prompt metadata, and output-schema behavior unless a stage explicitly calls out an intended behavior change.
|
|
||||||
- Keep the four production module packages separate: `glossary`, `homophones`, `spoken_word`, and `grammar`.
|
|
||||||
- Do not introduce plugin systems, generic workflow engines, broad CLI framework rewrites, reflection-heavy config mappers, or merged module packages.
|
|
||||||
- Prefer narrow helpers, catalogs, constants, and pure mapping functions over broad abstractions.
|
|
||||||
- Run the targeted tests listed in each stage before moving to the next stage.
|
|
||||||
- Run `go test ./...` before declaring the full sequence complete.
|
|
||||||
- Ignore unrelated worktree changes, including the existing deletion of `docs/roadmap/publish.md`, unless the user explicitly asks to handle them.
|
|
||||||
- Do not reduce parity, release-fixture, subprocess, or module-specific behavior coverage while consolidating helpers.
|
|
||||||
|
|
||||||
## Stages
|
|
||||||
|
|
||||||
### Stage 1: Diagnostics artifact constants and metadata paths
|
|
||||||
|
|
||||||
Goal:
|
|
||||||
|
|
||||||
- Centralize diagnostics artifact names and report diagnostics metadata path construction without changing any filenames or report fields.
|
|
||||||
|
|
||||||
Key edits:
|
|
||||||
|
|
||||||
- Define constants in `internal/core/diagnostics` for:
|
|
||||||
- `source-transcript.json`
|
|
||||||
- `source-transcript-parsed.json`
|
|
||||||
- `normalized-transcript.json`
|
|
||||||
- `normalization-summary.json`
|
|
||||||
- `chunking-summary.json`
|
|
||||||
- `utilization-diagnostics.json`
|
|
||||||
- `correction-ledger.json`
|
|
||||||
- `invocation.json`
|
|
||||||
- `effective-config.json`
|
|
||||||
- `report.json`
|
|
||||||
- `error.log`
|
|
||||||
- Add a diagnostics helper that builds `reporting.DiagnosticsMetadata` from a run directory path and failure/success status.
|
|
||||||
- Update `RunDirectory` methods to use the constants.
|
|
||||||
- Update CLI report assembly and utilization/correction-ledger writes to use the constants/helper instead of raw strings.
|
|
||||||
|
|
||||||
Behavior changes:
|
|
||||||
|
|
||||||
- None. All artifact names, report JSON keys, and path values must remain byte-for-byte compatible except for normal timestamp/order differences in existing outputs.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- Add or update `internal/core/diagnostics` tests proving metadata helper paths match the artifact constants.
|
|
||||||
- Run `go test ./internal/core/diagnostics ./internal/core/reporting ./internal/cli`.
|
|
||||||
- Run any existing CLI report/diagnostics tests touched by this stage.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- No raw core diagnostics artifact filename strings remain in CLI report metadata assembly.
|
|
||||||
- Existing success and failure reports still point to files that are actually written.
|
|
||||||
- Retention behavior is unchanged.
|
|
||||||
|
|
||||||
### Stage 2: Output schema validation and module catalog
|
|
||||||
|
|
||||||
Goal:
|
|
||||||
|
|
||||||
- Move public key validation to small canonical catalogs so config validation, runtime resolution, and factory behavior cannot drift.
|
|
||||||
|
|
||||||
Key edits:
|
|
||||||
|
|
||||||
- Add `SupportedKeys`, `IsSupported`, or an equivalent validation helper to `internal/core/outputschema`.
|
|
||||||
- Update `config.Validate` to use `internal/core/outputschema` for output schema validation.
|
|
||||||
- Add a small canonical module key catalog that is importable by:
|
|
||||||
- `internal/core/config`
|
|
||||||
- `internal/framework/modules`
|
|
||||||
- `internal/validators`
|
|
||||||
- `internal/framework/validators`
|
|
||||||
- Use the module catalog for default module key constants, known-key checks, validator chain keys, and confidence-threshold lookup.
|
|
||||||
- Keep module construction in `internal/framework/modules`; the catalog must not construct modules.
|
|
||||||
|
|
||||||
Behavior changes:
|
|
||||||
|
|
||||||
- Intended behavior change: unsupported configured module keys should fail during config validation, including `audita config validate`.
|
|
||||||
- Repeated supported module keys remain valid.
|
|
||||||
- Output schema behavior remains unchanged for `bare-segments`, `audita-v1`, and unsupported names.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- Add `internal/core/config` tests for unsupported module keys and repeated supported module keys.
|
|
||||||
- Add config validation tests that every supported output schema validates.
|
|
||||||
- Add or update output schema registry tests for supported and unsupported schemas.
|
|
||||||
- Update module registry and validator chain tests to use the shared catalog where appropriate.
|
|
||||||
- Run `go test ./internal/core/config ./internal/core/outputschema ./internal/framework/modules ./internal/framework/validators ./internal/validators/... ./internal/cli`.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- Unknown modules fail before runner setup in config validation paths.
|
|
||||||
- No duplicated hardcoded output schema support list remains in config validation.
|
|
||||||
- No import cycle is introduced.
|
|
||||||
|
|
||||||
### Stage 3: Effective config loading context
|
|
||||||
|
|
||||||
Goal:
|
|
||||||
|
|
||||||
- Centralize config path resolution and defaults+file+env loading while keeping command-specific CLI overrides explicit.
|
|
||||||
|
|
||||||
Key edits:
|
|
||||||
|
|
||||||
- Move config path resolution from `internal/cli` into `internal/core/config` or add an equivalent exported helper there.
|
|
||||||
- Add an effective config loader that returns:
|
|
||||||
- effective `config.Config`
|
|
||||||
- config path
|
|
||||||
- config source (`flag`, `env`, `default`, or empty)
|
|
||||||
- config version pointer when a file was loaded
|
|
||||||
- Use the shared loader in `audita process` before applying CLI overrides.
|
|
||||||
- Use the shared loader in `audita config print-effective`.
|
|
||||||
- Keep `audita config validate` as file-only: load file, apply to defaults, validate, and do not apply environment overrides.
|
|
||||||
|
|
||||||
Behavior changes:
|
|
||||||
|
|
||||||
- None. Preserve existing precedence:
|
|
||||||
- `process`: defaults, file config, environment, CLI flags
|
|
||||||
- `config print-effective`: defaults, file config, environment
|
|
||||||
- `config validate`: file config applied to defaults only
|
|
||||||
- Preserve explicit config path failure behavior and missing default path non-fatal behavior.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- Add table-driven config loader tests for:
|
|
||||||
- explicit `--config`
|
|
||||||
- `AUDITA_CONFIG`
|
|
||||||
- default search paths
|
|
||||||
- missing explicit path
|
|
||||||
- missing env path
|
|
||||||
- missing default paths
|
|
||||||
- Add or update CLI tests proving `process` and `config print-effective` share file+env behavior.
|
|
||||||
- Add or update CLI tests proving `config validate` ignores environment overrides.
|
|
||||||
- Run `go test ./internal/core/config ./internal/cli ./cmd/audita`.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- Config precedence is unchanged.
|
|
||||||
- Config source/path/version metadata in invocation and reports is unchanged.
|
|
||||||
- Config command stdout/stderr and exit-code behavior is unchanged except for the intended unknown-module validation from Stage 2.
|
|
||||||
|
|
||||||
### Stage 4: Prompt/schema metadata and stage-name helpers
|
|
||||||
|
|
||||||
Goal:
|
|
||||||
|
|
||||||
- Centralize diagnostics-visible metadata and stage-name construction without changing production diagnostics names.
|
|
||||||
|
|
||||||
Key edits:
|
|
||||||
|
|
||||||
- Add a helper or method in `internal/prompts` that returns the stable prompt metadata diagnostics shape currently expanded by call sites.
|
|
||||||
- Add a helper or method in `internal/framework/responseschema` that returns the stable response schema metadata diagnostics shape currently expanded by call sites.
|
|
||||||
- Add shared proposal and validator stage-name helpers in the lowest package that avoids import cycles.
|
|
||||||
- Use the helpers in proposal generation, LLM validators, and production modules.
|
|
||||||
|
|
||||||
Behavior changes:
|
|
||||||
|
|
||||||
- None. Preserve current production stage names:
|
|
||||||
- module proposal stages keep their existing `proposal` naming form;
|
|
||||||
- validator batch stages keep their existing validator/batch naming form.
|
|
||||||
- Preserve all prompt metadata and response schema metadata field names and values.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- Add prompt metadata helper tests covering every registered prompt.
|
|
||||||
- Add response schema metadata helper tests covering every registered response schema.
|
|
||||||
- Add stage-name helper tests for no-section, section, and validator batch cases.
|
|
||||||
- Run `go test ./internal/prompts ./internal/framework/responseschema ./internal/framework/proposal_generation ./internal/framework/validators ./internal/modules/...`.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- No manual prompt metadata map expansion remains in production module proposal plumbing.
|
|
||||||
- No duplicated response schema metadata map construction remains in proposal generation and LLM validators.
|
|
||||||
- Existing diagnostics fixture/path assertions still pass.
|
|
||||||
|
|
||||||
### Stage 5: Shared module proposal and prompt payload plumbing
|
|
||||||
|
|
||||||
Goal:
|
|
||||||
|
|
||||||
- Remove duplicated proposal execution and transcript-section prompt payload construction while preserving module-specific domain behavior.
|
|
||||||
|
|
||||||
Key edits:
|
|
||||||
|
|
||||||
- Add a narrow shared proposal execution helper, preferably in `internal/framework/proposal_generation` unless import cycles require a small module helper package.
|
|
||||||
- The helper should own:
|
|
||||||
- transcript description extraction from config;
|
|
||||||
- `GenerateCandidates` request construction;
|
|
||||||
- prompt metadata attachment;
|
|
||||||
- stage-name selection;
|
|
||||||
- conversion from generated corrections/warnings to `contracts.ProposalResult`.
|
|
||||||
- Add shared transcript-section prompt payload construction in `internal/framework/promptcontext`.
|
|
||||||
- Update each production module to provide only:
|
|
||||||
- module key;
|
|
||||||
- replacement policy;
|
|
||||||
- validator chain;
|
|
||||||
- prompt ID;
|
|
||||||
- domain-specific `BuildProposalMessages` call or message builder.
|
|
||||||
- Remove each module's redundant section transcript filtering if the runner already passes section-limited transcripts.
|
|
||||||
|
|
||||||
Behavior changes:
|
|
||||||
|
|
||||||
- None. Preserve module keys, replacement policies, validator chains, prompt IDs, diagnostics directories, proposal indexes, warning behavior, and correction mapping.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- Add promptcontext tests for transcript section payload shape, empty transcript handling, section index, and category copying.
|
|
||||||
- Keep one module-specific prompt test per production module for domain wording and constraints.
|
|
||||||
- Add or update module proposal tests proving diagnostics are still written under the same module instance directory.
|
|
||||||
- Run `go test ./internal/framework/promptcontext ./internal/framework/proposal_generation ./internal/modules/... ./internal/cli`.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- Four production modules share proposal execution plumbing.
|
|
||||||
- Module packages remain separate and readable.
|
|
||||||
- CLI parity and release fixture behavior is unchanged.
|
|
||||||
|
|
||||||
### Stage 6: Validator classification and malformed LLM output policy
|
|
||||||
|
|
||||||
Goal:
|
|
||||||
|
|
||||||
- Use one source of truth for validator execution class and one shared classifier for malformed structured-output errors.
|
|
||||||
|
|
||||||
Key edits:
|
|
||||||
|
|
||||||
- Make validator execution class resolvable by stable validator key and by validator instance.
|
|
||||||
- Replace the correction-ledger hardcoded LLM-backed validator map with the canonical metadata source.
|
|
||||||
- Remove redundant validator metadata fields only after all call sites use the canonical source.
|
|
||||||
- Add a shared malformed structured-output classifier in `internal/framework/llm` or another low-level framework package.
|
|
||||||
- Update proposal generation and LLM validators to use the shared classifier while preserving their different handling outcomes.
|
|
||||||
|
|
||||||
Behavior changes:
|
|
||||||
|
|
||||||
- None. Proposal-generation malformed payloads still downgrade to warnings with zero proposals for affected sections.
|
|
||||||
- Validator malformed payloads still reject affected batches with warnings.
|
|
||||||
- Correction-ledger deterministic vs LLM validator sections should be unchanged for current validators.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- Add validator metadata tests proving every registered validator has the expected execution class by key and instance.
|
|
||||||
- Add correction-ledger tests proving deterministic and LLM-backed decisions are classified through canonical metadata.
|
|
||||||
- Add shared malformed-output classifier tests covering current adapter malformed-output messages.
|
|
||||||
- Update proposal-generation and validator tests to assert representative malformed adapter errors are still downgraded.
|
|
||||||
- Run `go test ./internal/validators/... ./internal/framework/validators ./internal/framework/proposal_generation ./internal/framework/llm ./internal/cli`.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- No local hardcoded LLM-backed validator map remains in correction-ledger construction.
|
|
||||||
- Proposal-generation and validator malformed-output classifier lists cannot drift.
|
|
||||||
- Existing runner validator ordering is unchanged.
|
|
||||||
|
|
||||||
### Stage 7: Redaction and adapter workflow cleanup
|
|
||||||
|
|
||||||
Goal:
|
|
||||||
|
|
||||||
- Reduce duplicated secret extraction/redaction setup while preserving all no-secret-leak guarantees.
|
|
||||||
|
|
||||||
Key edits:
|
|
||||||
|
|
||||||
- Add a shared helper that extracts all configured LLM secret values from `config.Config`.
|
|
||||||
- Use the helper in proposal-generation diagnostics and validator diagnostics setup.
|
|
||||||
- Keep config structural redaction (`Config.Redacted`) separate from byte/string payload redaction.
|
|
||||||
- Keep adapter error redaction behavior compatible with current surfaced errors.
|
|
||||||
- Move runner adapter shims only if Stage 6 or this stage makes them materially larger; otherwise leave them in runner.
|
|
||||||
|
|
||||||
Behavior changes:
|
|
||||||
|
|
||||||
- None. Redaction token and no-secret-leak behavior remain unchanged.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- Add or update tests proving proposal diagnostics, validator diagnostics, effective config artifacts, and surfaced adapter errors redact the same configured secrets.
|
|
||||||
- Keep existing subprocess no-secret-leak tests.
|
|
||||||
- Run `go test ./internal/core/config ./internal/framework/llm ./internal/framework/proposal_generation ./internal/framework/validators ./internal/cli ./cmd/audita`.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- Secret-list assembly is no longer duplicated between proposal and validator paths.
|
|
||||||
- No plaintext configured API key appears in diagnostics, reports, stdout, or stderr in existing redaction tests.
|
|
||||||
- No unrelated adapter behavior changes.
|
|
||||||
|
|
||||||
### Stage 8: Test helper cleanup and dead-code sweep
|
|
||||||
|
|
||||||
Goal:
|
|
||||||
|
|
||||||
- Consolidate test-only duplication and remove dead/redundant code left by prior stages.
|
|
||||||
|
|
||||||
Key edits:
|
|
||||||
|
|
||||||
- Consolidate package-local fake LLM clients, fixture readers, diagnostics glob helpers, and run-directory helpers where duplication is clear.
|
|
||||||
- Use cross-package test support only if it does not obscure test intent or introduce awkward imports.
|
|
||||||
- Remove redundant metadata fields, constants, or helper functions made obsolete by earlier stages.
|
|
||||||
- Keep module-specific prompt and behavior assertions local to each module package.
|
|
||||||
|
|
||||||
Behavior changes:
|
|
||||||
|
|
||||||
- None.
|
|
||||||
|
|
||||||
Tests:
|
|
||||||
|
|
||||||
- Run all package tests touched by helper cleanup.
|
|
||||||
- Run `go test ./internal/modules/... ./internal/framework/... ./internal/cli ./cmd/audita`.
|
|
||||||
- Run `go test ./...` before completing the full sequence.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- Test helpers are simpler without reducing coverage.
|
|
||||||
- No parity or release fixture assertions are removed unless replaced by equivalent or stronger assertions.
|
|
||||||
- No production behavior changes.
|
|
||||||
|
|
||||||
## Final verification
|
|
||||||
|
|
||||||
Before declaring the staged cleanup complete:
|
|
||||||
|
|
||||||
- Run:
|
|
||||||
- `go test ./internal/core/config ./internal/core/outputschema`
|
|
||||||
- `go test ./internal/core/diagnostics ./internal/core/reporting`
|
|
||||||
- `go test ./internal/framework/proposal_generation ./internal/framework/validators ./internal/framework/runner`
|
|
||||||
- `go test ./internal/validators/...`
|
|
||||||
- `go test ./internal/modules/...`
|
|
||||||
- `go test ./internal/cli ./cmd/audita`
|
|
||||||
- `go test ./...`
|
|
||||||
- Inspect `git diff` for accidental public CLI, config, report, diagnostics, prompt metadata, stage-name, or output-schema changes.
|
|
||||||
- Update docs only when behavior intentionally changes, especially the intended Stage 2 unknown-module validation change.
|
|
||||||
- Keep commits stage-sized and mention behavior-preservation tests in each commit message or PR description.
|
|
||||||
|
|
||||||
## Assumptions
|
|
||||||
|
|
||||||
- Unknown configured module keys should become config-validation failures before 1.0.
|
|
||||||
- Diagnostics filenames and stage names are public enough to preserve unless a stage explicitly says otherwise.
|
|
||||||
- Each stage should be implemented and reviewed separately.
|
|
||||||
17
docs/roadmap/runtime-behavior.md
Normal file
17
docs/roadmap/runtime-behavior.md
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
# Runtime Behavior Decisions
|
||||||
|
|
||||||
|
**Last updated:** 2026-09-23
|
||||||
|
|
||||||
|
The documentation migration identified these existing behaviors for separate
|
||||||
|
product consideration. No behavior change is approved or implemented by this list.
|
||||||
|
|
||||||
|
- Decide whether the `never` retention selection should delete successful-run
|
||||||
|
diagnostics; current behavior is documented in
|
||||||
|
[operations](../operations.md#retention-behavior).
|
||||||
|
- Decide whether an explicit zero retry count should disable retries; current
|
||||||
|
adapter behavior is documented in [configuration](../config.md).
|
||||||
|
- Decide whether missing or null input timestamps should be rejected; current
|
||||||
|
decoding is documented in the [input contract](../integrations/transcript-glossary-files.md).
|
||||||
|
|
||||||
|
These decisions are independent of the completed documentation migration and
|
||||||
|
release workflow.
|
||||||
6
examples/minimal-config.yml
Normal file
6
examples/minimal-config.yml
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
version: 1
|
||||||
|
output:
|
||||||
|
schema: bare-segments
|
||||||
|
llm:
|
||||||
|
proposal:
|
||||||
|
api_key_env: AUDITA_LLM_API_KEY
|
||||||
42
examples/production-config.yml
Normal file
42
examples/production-config.yml
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
version: 1
|
||||||
|
pipeline:
|
||||||
|
modules: [glossary, homophones, glossary, spoken_word, grammar]
|
||||||
|
output:
|
||||||
|
schema: audita-v1
|
||||||
|
llm:
|
||||||
|
proposal:
|
||||||
|
base_url: https://openrouter.ai/api/v1
|
||||||
|
model: openrouter/google/gemma-4-31b-it
|
||||||
|
api_key_env: AUDITA_LLM_API_KEY
|
||||||
|
timeout: 120s
|
||||||
|
max_retries: 3
|
||||||
|
validation:
|
||||||
|
base_url: https://openrouter.ai/api/v1
|
||||||
|
model: openrouter/google/gemma-4-31b-it
|
||||||
|
api_key_env: AUDITA_VALIDATION_LLM_API_KEY
|
||||||
|
timeout: 120
|
||||||
|
max_retries: 3
|
||||||
|
concurrency:
|
||||||
|
total_llm: 2
|
||||||
|
proposal_llm: 2
|
||||||
|
validation_llm: 1
|
||||||
|
chunking:
|
||||||
|
# Requires at least this many normalized segments; omit for automatic chunking.
|
||||||
|
target_sections: 8
|
||||||
|
max_section_tokens: 8192
|
||||||
|
min_section_tokens: 2048
|
||||||
|
normalization:
|
||||||
|
max_segment_gap: 4s
|
||||||
|
ellipsis_gap: 3.5s
|
||||||
|
max_segment_duration: 60s
|
||||||
|
max_segment_tokens: 2048
|
||||||
|
thresholds:
|
||||||
|
glossary: 0.8
|
||||||
|
homophones: 0.8
|
||||||
|
spoken_word: 0.8
|
||||||
|
grammar: 0.8
|
||||||
|
context:
|
||||||
|
description: "General context for domain vocabulary and speaker names."
|
||||||
|
diagnostics:
|
||||||
|
work_dir: /tmp/audita
|
||||||
|
retention: auto
|
||||||
6
examples/tiny-glossary.yaml
Normal file
6
examples/tiny-glossary.yaml
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
glossary:
|
||||||
|
- name: Audita
|
||||||
|
aliases:
|
||||||
|
- audita
|
||||||
|
category: product
|
||||||
|
summary: The Audita transcript correction CLI.
|
||||||
9
examples/tiny-transcript.json
Normal file
9
examples/tiny-transcript.json
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"speaker": "A",
|
||||||
|
"start": 0.0,
|
||||||
|
"end": 1.2,
|
||||||
|
"text": "hello world"
|
||||||
|
}
|
||||||
|
]
|
||||||
36
internal/cli/main_test.go
Normal file
36
internal/cli/main_test.go
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMain(m *testing.M) {
|
||||||
|
// Tests supply their own overrides; inherited settings must not affect them.
|
||||||
|
for _, entry := range os.Environ() {
|
||||||
|
key, _, _ := strings.Cut(entry, "=")
|
||||||
|
if strings.HasPrefix(key, "AUDITA_") || key == "OPENROUTER_API_KEY" {
|
||||||
|
if err := os.Unsetenv(key); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dir, err := os.MkdirTemp("", "audita-cli-test-config-")
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(dir)
|
||||||
|
configPath := filepath.Join(dir, "config.yml")
|
||||||
|
if err := os.WriteFile(configPath, []byte("version: 1\n"), 0600); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
// An explicit environment path prevents discovery of installed system config.
|
||||||
|
// Individual tests can still override it with t.Setenv or --config.
|
||||||
|
if err := os.Setenv("AUDITA_CONFIG", configPath); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
m.Run()
|
||||||
|
}
|
||||||
121
internal/cli/process_flags.go
Normal file
121
internal/cli/process_flags.go
Normal file
@@ -0,0 +1,121 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"flag"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
type processOverrideBinding func(*config.CLIOverrides, processFlags)
|
||||||
|
|
||||||
|
var processOverrideBindings = map[string]processOverrideBinding{
|
||||||
|
"modules": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.ModulesCSV = flags.modules
|
||||||
|
},
|
||||||
|
"output-schema": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.OutputSchema = flags.outputSchema
|
||||||
|
},
|
||||||
|
"llm-api-key": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.PrimaryLLMAPIKey = flags.llmAPIKey
|
||||||
|
},
|
||||||
|
"validation-llm-api-key": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.ValidationLLMAPIKey = flags.validationLLMAPIKey
|
||||||
|
},
|
||||||
|
"model": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.PrimaryModel = flags.model
|
||||||
|
},
|
||||||
|
"validation-model": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.ValidationModel = flags.validationModel
|
||||||
|
},
|
||||||
|
"base-url": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.PrimaryBaseURL = flags.baseURL
|
||||||
|
},
|
||||||
|
"validation-base-url": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.ValidationBaseURL = flags.validationBaseURL
|
||||||
|
},
|
||||||
|
"llm-timeout-seconds": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.PrimaryLLMTimeoutSeconds = flags.llmTimeoutSeconds
|
||||||
|
},
|
||||||
|
"total-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.TotalLLMConcurrency = flags.totalLLMConcurrency
|
||||||
|
},
|
||||||
|
"proposal-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.ProposalLLMConcurrency = flags.proposalLLMConcurrency
|
||||||
|
},
|
||||||
|
"llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.PrimaryLLMConcurrency = flags.llmConcurrency
|
||||||
|
},
|
||||||
|
"validation-llm-timeout-seconds": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.ValidationLLMTimeoutSeconds = flags.validationLLMTimeoutSeconds
|
||||||
|
},
|
||||||
|
"max-retries": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.MaxRetries = flags.maxRetries
|
||||||
|
},
|
||||||
|
"validation-max-retries": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.ValidationMaxRetries = flags.validationMaxRetries
|
||||||
|
},
|
||||||
|
"validation-llm-concurrency": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.ValidationLLMConcurrency = flags.validationLLMConcurrency
|
||||||
|
},
|
||||||
|
"validation-max-prompt-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.ValidationMaxPromptTokens = flags.validationMaxPromptTokens
|
||||||
|
},
|
||||||
|
"max-section-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.MaxSectionTokens = flags.maxSectionTokens
|
||||||
|
},
|
||||||
|
"min-section-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.MinSectionTokens = flags.minSectionTokens
|
||||||
|
},
|
||||||
|
"target-sections": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.TargetSections = flags.targetSections
|
||||||
|
},
|
||||||
|
"glossary-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.GlossaryConfidenceThreshold = flags.glossaryConfidenceThreshold
|
||||||
|
},
|
||||||
|
"grammar-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.GrammarConfidenceThreshold = flags.grammarConfidenceThreshold
|
||||||
|
},
|
||||||
|
"homophones-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.HomophonesConfidenceThreshold = flags.homophonesConfidenceThreshold
|
||||||
|
},
|
||||||
|
"spoken-word-confidence-threshold": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.SpokenWordConfidenceThreshold = flags.spokenWordConfidenceThreshold
|
||||||
|
},
|
||||||
|
"normalize-max-segment-gap": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.NormalizeMaxSegmentGap = flags.normalizeMaxSegmentGap
|
||||||
|
},
|
||||||
|
"normalize-ellipsis-gap": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.NormalizeEllipsisGap = flags.normalizeEllipsisGap
|
||||||
|
},
|
||||||
|
"normalize-max-segment-duration": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.NormalizeMaxSegmentDuration = flags.normalizeMaxSegmentDuration
|
||||||
|
},
|
||||||
|
"normalize-max-segment-tokens": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.NormalizeMaxSegmentTokens = flags.normalizeMaxSegmentTokens
|
||||||
|
},
|
||||||
|
"transcript-description": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.TranscriptDescription = flags.transcriptDescription
|
||||||
|
},
|
||||||
|
"work-dir": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.WorkDir = flags.workDir
|
||||||
|
},
|
||||||
|
"work-dir-retention": func(overrides *config.CLIOverrides, flags processFlags) {
|
||||||
|
overrides.WorkDirRetention = flags.workDirRetention
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func processCLIOverrides(fs *flag.FlagSet, flags processFlags) (config.CLIOverrides, bool) {
|
||||||
|
overrides := config.CLIOverrides{}
|
||||||
|
explicitModules := false
|
||||||
|
fs.Visit(func(f *flag.Flag) {
|
||||||
|
if f.Name == "modules" {
|
||||||
|
explicitModules = true
|
||||||
|
}
|
||||||
|
binding, ok := processOverrideBindings[f.Name]
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
binding(&overrides, flags)
|
||||||
|
})
|
||||||
|
return overrides, explicitModules
|
||||||
|
}
|
||||||
433
internal/cli/process_flags_test.go
Normal file
433
internal/cli/process_flags_test.go
Normal file
@@ -0,0 +1,433 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProcessCLIOverridesMapsEveryConfigMutatingFlag(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
flagName string
|
||||||
|
value string
|
||||||
|
wantExplicitModules bool
|
||||||
|
assertOverrideFields func(t *testing.T, overrides config.CLIOverrides)
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "modules",
|
||||||
|
flagName: "modules",
|
||||||
|
value: "grammar,glossary",
|
||||||
|
wantExplicitModules: true,
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "ModulesCSV", overrides.ModulesCSV, "grammar,glossary")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "output schema",
|
||||||
|
flagName: "output-schema",
|
||||||
|
value: "audita-v1",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "OutputSchema", overrides.OutputSchema, "audita-v1")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "primary api key",
|
||||||
|
flagName: "llm-api-key",
|
||||||
|
value: "primary-key",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "PrimaryLLMAPIKey", overrides.PrimaryLLMAPIKey, "primary-key")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation api key",
|
||||||
|
flagName: "validation-llm-api-key",
|
||||||
|
value: "validation-key",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "ValidationLLMAPIKey", overrides.ValidationLLMAPIKey, "validation-key")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "primary model",
|
||||||
|
flagName: "model",
|
||||||
|
value: "primary-model",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "PrimaryModel", overrides.PrimaryModel, "primary-model")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation model",
|
||||||
|
flagName: "validation-model",
|
||||||
|
value: "validation-model",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "ValidationModel", overrides.ValidationModel, "validation-model")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "primary base url",
|
||||||
|
flagName: "base-url",
|
||||||
|
value: "https://primary.example.test",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "PrimaryBaseURL", overrides.PrimaryBaseURL, "https://primary.example.test")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation base url",
|
||||||
|
flagName: "validation-base-url",
|
||||||
|
value: "https://validation.example.test",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "ValidationBaseURL", overrides.ValidationBaseURL, "https://validation.example.test")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "primary timeout",
|
||||||
|
flagName: "llm-timeout-seconds",
|
||||||
|
value: "101",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "PrimaryLLMTimeoutSeconds", overrides.PrimaryLLMTimeoutSeconds, 101)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "total concurrency",
|
||||||
|
flagName: "total-llm-concurrency",
|
||||||
|
value: "5",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "TotalLLMConcurrency", overrides.TotalLLMConcurrency, 5)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "proposal concurrency",
|
||||||
|
flagName: "proposal-llm-concurrency",
|
||||||
|
value: "3",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "ProposalLLMConcurrency", overrides.ProposalLLMConcurrency, 3)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "legacy concurrency alias",
|
||||||
|
flagName: "llm-concurrency",
|
||||||
|
value: "4",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "PrimaryLLMConcurrency", overrides.PrimaryLLMConcurrency, 4)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation timeout",
|
||||||
|
flagName: "validation-llm-timeout-seconds",
|
||||||
|
value: "202",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "ValidationLLMTimeoutSeconds", overrides.ValidationLLMTimeoutSeconds, 202)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "max retries",
|
||||||
|
flagName: "max-retries",
|
||||||
|
value: "6",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "MaxRetries", overrides.MaxRetries, 6)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation max retries",
|
||||||
|
flagName: "validation-max-retries",
|
||||||
|
value: "7",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "ValidationMaxRetries", overrides.ValidationMaxRetries, 7)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation concurrency",
|
||||||
|
flagName: "validation-llm-concurrency",
|
||||||
|
value: "8",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "ValidationLLMConcurrency", overrides.ValidationLLMConcurrency, 8)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation max prompt tokens",
|
||||||
|
flagName: "validation-max-prompt-tokens",
|
||||||
|
value: "4096",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "ValidationMaxPromptTokens", overrides.ValidationMaxPromptTokens, 4096)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "max section tokens",
|
||||||
|
flagName: "max-section-tokens",
|
||||||
|
value: "9000",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "MaxSectionTokens", overrides.MaxSectionTokens, 9000)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "min section tokens",
|
||||||
|
flagName: "min-section-tokens",
|
||||||
|
value: "1000",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "MinSectionTokens", overrides.MinSectionTokens, 1000)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "target sections",
|
||||||
|
flagName: "target-sections",
|
||||||
|
value: "12",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "TargetSections", overrides.TargetSections, 12)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "glossary threshold",
|
||||||
|
flagName: "glossary-confidence-threshold",
|
||||||
|
value: "0.91",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertFloatOverride(t, "GlossaryConfidenceThreshold", overrides.GlossaryConfidenceThreshold, 0.91)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "grammar threshold",
|
||||||
|
flagName: "grammar-confidence-threshold",
|
||||||
|
value: "0.92",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertFloatOverride(t, "GrammarConfidenceThreshold", overrides.GrammarConfidenceThreshold, 0.92)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "homophones threshold",
|
||||||
|
flagName: "homophones-confidence-threshold",
|
||||||
|
value: "0.93",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertFloatOverride(t, "HomophonesConfidenceThreshold", overrides.HomophonesConfidenceThreshold, 0.93)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "spoken word threshold",
|
||||||
|
flagName: "spoken-word-confidence-threshold",
|
||||||
|
value: "0.94",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertFloatOverride(t, "SpokenWordConfidenceThreshold", overrides.SpokenWordConfidenceThreshold, 0.94)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "normalize max segment gap",
|
||||||
|
flagName: "normalize-max-segment-gap",
|
||||||
|
value: "1.2",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertFloatOverride(t, "NormalizeMaxSegmentGap", overrides.NormalizeMaxSegmentGap, 1.2)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "normalize ellipsis gap",
|
||||||
|
flagName: "normalize-ellipsis-gap",
|
||||||
|
value: "2.3",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertFloatOverride(t, "NormalizeEllipsisGap", overrides.NormalizeEllipsisGap, 2.3)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "normalize max segment duration",
|
||||||
|
flagName: "normalize-max-segment-duration",
|
||||||
|
value: "45.6",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertFloatOverride(t, "NormalizeMaxSegmentDuration", overrides.NormalizeMaxSegmentDuration, 45.6)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "normalize max segment tokens",
|
||||||
|
flagName: "normalize-max-segment-tokens",
|
||||||
|
value: "321",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertIntOverride(t, "NormalizeMaxSegmentTokens", overrides.NormalizeMaxSegmentTokens, 321)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "transcript description",
|
||||||
|
flagName: "transcript-description",
|
||||||
|
value: "podcast episode",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "TranscriptDescription", overrides.TranscriptDescription, "podcast episode")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "work dir",
|
||||||
|
flagName: "work-dir",
|
||||||
|
value: "/tmp/custom-audita",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "WorkDir", overrides.WorkDir, "/tmp/custom-audita")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "work dir retention",
|
||||||
|
flagName: "work-dir-retention",
|
||||||
|
value: "always",
|
||||||
|
assertOverrideFields: func(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
assertStringOverride(t, "WorkDirRetention", overrides.WorkDirRetention, "always")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
fs, flags := newProcessFlagSet(config.Default(), io.Discard)
|
||||||
|
if err := fs.Parse([]string{"--" + tc.flagName, tc.value}); err != nil {
|
||||||
|
t.Fatalf("parse flag: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
overrides, explicitModules := processCLIOverrides(fs, flags)
|
||||||
|
if explicitModules != tc.wantExplicitModules {
|
||||||
|
t.Fatalf("explicitModules=%v, want %v", explicitModules, tc.wantExplicitModules)
|
||||||
|
}
|
||||||
|
tc.assertOverrideFields(t, overrides)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessCLIOverridesIgnoresNonConfigFlags(t *testing.T) {
|
||||||
|
fs, flags := newProcessFlagSet(config.Default(), io.Discard)
|
||||||
|
if err := fs.Parse([]string{
|
||||||
|
"--config", "/tmp/config.yml",
|
||||||
|
"--glossary", "/tmp/glossary.yml",
|
||||||
|
"--output", "/tmp/output.json",
|
||||||
|
"--report-json", "/tmp/report.json",
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("parse flags: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
overrides, explicitModules := processCLIOverrides(fs, flags)
|
||||||
|
if explicitModules {
|
||||||
|
t.Fatal("non-config flags should not mark modules explicit")
|
||||||
|
}
|
||||||
|
assertNoCLIOverrides(t, overrides)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewProcessFlagSetDefaultsReflectEffectiveConfig(t *testing.T) {
|
||||||
|
cfg := config.Default()
|
||||||
|
cfg.Modules = []string{"grammar", "glossary"}
|
||||||
|
cfg.OutputSchema = "audita-v1"
|
||||||
|
cfg.PrimaryLLM.APIKey = "primary-key"
|
||||||
|
cfg.ValidationLLM.APIKey = "validation-key"
|
||||||
|
cfg.PrimaryLLM.Model = "primary-model"
|
||||||
|
cfg.ValidationLLM.Model = "validation-model"
|
||||||
|
cfg.PrimaryLLM.BaseURL = "https://primary.example.test"
|
||||||
|
cfg.ValidationLLM.BaseURL = "https://validation.example.test"
|
||||||
|
cfg.PrimaryLLM.TimeoutSeconds = 101
|
||||||
|
cfg.TotalLLMConcurrency = 5
|
||||||
|
cfg.ProposalLLMConcurrency = 3
|
||||||
|
cfg.PrimaryLLM.MaxRetries = 6
|
||||||
|
cfg.ValidationMaxPromptTokens = 4096
|
||||||
|
cfg.MaxSectionTokens = 9000
|
||||||
|
cfg.MinSectionTokens = 1000
|
||||||
|
cfg.Thresholds.Glossary = 0.91
|
||||||
|
cfg.Thresholds.Grammar = 0.92
|
||||||
|
cfg.Thresholds.Homophones = 0.93
|
||||||
|
cfg.Thresholds.SpokenWord = 0.94
|
||||||
|
cfg.Normalization.MaxSegmentGap = 1.2
|
||||||
|
cfg.Normalization.EllipsisGap = 2.3
|
||||||
|
cfg.Normalization.MaxSegmentDuration = 45.6
|
||||||
|
cfg.Normalization.MaxSegmentTokens = 321
|
||||||
|
cfg.TranscriptDescription = "podcast episode"
|
||||||
|
cfg.WorkDir = "/tmp/custom-audita"
|
||||||
|
cfg.WorkDirRetention = config.WorkDirRetentionAlways
|
||||||
|
|
||||||
|
validationTimeout := 202
|
||||||
|
validationRetries := 7
|
||||||
|
validationConcurrency := 8
|
||||||
|
targetSections := 12
|
||||||
|
cfg.ValidationLLM.TimeoutSeconds = &validationTimeout
|
||||||
|
cfg.ValidationLLM.MaxRetries = &validationRetries
|
||||||
|
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||||
|
cfg.TargetSections = &targetSections
|
||||||
|
|
||||||
|
_, flags := newProcessFlagSet(cfg, io.Discard)
|
||||||
|
|
||||||
|
assertStringOverride(t, "modules default", flags.modules, "grammar,glossary")
|
||||||
|
assertStringOverride(t, "output schema default", flags.outputSchema, "audita-v1")
|
||||||
|
assertStringOverride(t, "primary api key default", flags.llmAPIKey, "primary-key")
|
||||||
|
assertStringOverride(t, "validation api key default", flags.validationLLMAPIKey, "validation-key")
|
||||||
|
assertStringOverride(t, "primary model default", flags.model, "primary-model")
|
||||||
|
assertStringOverride(t, "validation model default", flags.validationModel, "validation-model")
|
||||||
|
assertStringOverride(t, "primary base url default", flags.baseURL, "https://primary.example.test")
|
||||||
|
assertStringOverride(t, "validation base url default", flags.validationBaseURL, "https://validation.example.test")
|
||||||
|
assertIntOverride(t, "primary timeout default", flags.llmTimeoutSeconds, 101)
|
||||||
|
assertIntOverride(t, "total concurrency default", flags.totalLLMConcurrency, 5)
|
||||||
|
assertIntOverride(t, "proposal concurrency default", flags.proposalLLMConcurrency, 3)
|
||||||
|
assertIntOverride(t, "legacy concurrency alias default", flags.llmConcurrency, 5)
|
||||||
|
assertIntOverride(t, "validation timeout default", flags.validationLLMTimeoutSeconds, validationTimeout)
|
||||||
|
assertIntOverride(t, "max retries default", flags.maxRetries, 6)
|
||||||
|
assertIntOverride(t, "validation max retries default", flags.validationMaxRetries, validationRetries)
|
||||||
|
assertIntOverride(t, "validation concurrency default", flags.validationLLMConcurrency, validationConcurrency)
|
||||||
|
assertIntOverride(t, "validation max prompt tokens default", flags.validationMaxPromptTokens, 4096)
|
||||||
|
assertIntOverride(t, "max section tokens default", flags.maxSectionTokens, 9000)
|
||||||
|
assertIntOverride(t, "min section tokens default", flags.minSectionTokens, 1000)
|
||||||
|
assertIntOverride(t, "target sections default", flags.targetSections, targetSections)
|
||||||
|
assertFloatOverride(t, "glossary threshold default", flags.glossaryConfidenceThreshold, 0.91)
|
||||||
|
assertFloatOverride(t, "grammar threshold default", flags.grammarConfidenceThreshold, 0.92)
|
||||||
|
assertFloatOverride(t, "homophones threshold default", flags.homophonesConfidenceThreshold, 0.93)
|
||||||
|
assertFloatOverride(t, "spoken word threshold default", flags.spokenWordConfidenceThreshold, 0.94)
|
||||||
|
assertFloatOverride(t, "normalize max segment gap default", flags.normalizeMaxSegmentGap, 1.2)
|
||||||
|
assertFloatOverride(t, "normalize ellipsis gap default", flags.normalizeEllipsisGap, 2.3)
|
||||||
|
assertFloatOverride(t, "normalize max segment duration default", flags.normalizeMaxSegmentDuration, 45.6)
|
||||||
|
assertIntOverride(t, "normalize max segment tokens default", flags.normalizeMaxSegmentTokens, 321)
|
||||||
|
assertStringOverride(t, "transcript description default", flags.transcriptDescription, "podcast episode")
|
||||||
|
assertStringOverride(t, "work dir default", flags.workDir, "/tmp/custom-audita")
|
||||||
|
assertStringOverride(t, "work dir retention default", flags.workDirRetention, "always")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewProcessFlagSetUsesFallbackDefaultsForUnsetOptionalConfig(t *testing.T) {
|
||||||
|
cfg := config.Default()
|
||||||
|
|
||||||
|
_, flags := newProcessFlagSet(cfg, io.Discard)
|
||||||
|
|
||||||
|
assertIntOverride(t, "validation timeout fallback", flags.validationLLMTimeoutSeconds, cfg.PrimaryLLM.TimeoutSeconds)
|
||||||
|
assertIntOverride(t, "validation retries fallback", flags.validationMaxRetries, cfg.PrimaryLLM.MaxRetries)
|
||||||
|
assertIntOverride(t, "validation concurrency fallback", flags.validationLLMConcurrency, cfg.TotalLLMConcurrency)
|
||||||
|
assertIntOverride(t, "target sections fallback", flags.targetSections, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertStringOverride(t *testing.T, name string, got *string, want string) {
|
||||||
|
t.Helper()
|
||||||
|
if got == nil || *got != want {
|
||||||
|
t.Fatalf("%s=%v, want %q", name, pointerValue(got), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertIntOverride(t *testing.T, name string, got *int, want int) {
|
||||||
|
t.Helper()
|
||||||
|
if got == nil || *got != want {
|
||||||
|
t.Fatalf("%s=%v, want %d", name, pointerValue(got), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertFloatOverride(t *testing.T, name string, got *float64, want float64) {
|
||||||
|
t.Helper()
|
||||||
|
if got == nil || *got != want {
|
||||||
|
t.Fatalf("%s=%v, want %v", name, pointerValue(got), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertNoCLIOverrides(t *testing.T, overrides config.CLIOverrides) {
|
||||||
|
t.Helper()
|
||||||
|
value := reflect.ValueOf(overrides)
|
||||||
|
typ := value.Type()
|
||||||
|
for i := 0; i < value.NumField(); i++ {
|
||||||
|
field := value.Field(i)
|
||||||
|
if field.Kind() != reflect.Ptr {
|
||||||
|
t.Fatalf("unexpected non-pointer CLIOverrides field %s", typ.Field(i).Name)
|
||||||
|
}
|
||||||
|
if !field.IsNil() {
|
||||||
|
t.Fatalf("expected no CLI overrides, field %s was set", typ.Field(i).Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func pointerValue[T any](ptr *T) any {
|
||||||
|
if ptr == nil {
|
||||||
|
return "<nil>"
|
||||||
|
}
|
||||||
|
if stringer, ok := any(*ptr).(interface{ String() string }); ok {
|
||||||
|
return strings.TrimSpace(stringer.String())
|
||||||
|
}
|
||||||
|
return *ptr
|
||||||
|
}
|
||||||
@@ -1,46 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(t *testing.T) {
|
|
||||||
output := &runner.RunOutput{
|
|
||||||
ModuleResults: []runner.ModuleResult{
|
|
||||||
{
|
|
||||||
ModuleKey: "glossary",
|
|
||||||
ModuleInstance: "glossary",
|
|
||||||
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
|
|
||||||
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
|
||||||
{ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
|
||||||
{ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
|
||||||
},
|
|
||||||
AppliedChanges: []proposals.AppliedChange{
|
|
||||||
{
|
|
||||||
ProposalIndex: 3,
|
|
||||||
ModuleKey: "glossary",
|
|
||||||
ModuleInstance: "glossary",
|
|
||||||
TargetSegmentID: 1,
|
|
||||||
OriginalText: "gestures",
|
|
||||||
CorrectedText: "Jesters",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
ledger := buildCorrectionLedger("/tmp/audita-run-id", output)
|
|
||||||
if len(ledger) != 1 {
|
|
||||||
t.Fatalf("expected one ledger entry, got %d", len(ledger))
|
|
||||||
}
|
|
||||||
entry := ledger[0]
|
|
||||||
if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "proposal_shape" {
|
|
||||||
t.Fatalf("unexpected deterministic decision split: %+v", entry.DeterministicValidatorResults)
|
|
||||||
}
|
|
||||||
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "spoken_form_plausibility" {
|
|
||||||
t.Fatalf("unexpected llm-backed decision split: %+v", entry.LLMValidatorResults)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -23,10 +23,10 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/processreport"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type noOpStructuredLLMClient struct{}
|
type noOpStructuredLLMClient struct{}
|
||||||
@@ -420,75 +420,7 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
|||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
|
|
||||||
overrides := config.CLIOverrides{}
|
overrides, explicitModules := processCLIOverrides(fs, pFlags)
|
||||||
explicitModules := false
|
|
||||||
fs.Visit(func(f *flag.Flag) {
|
|
||||||
switch f.Name {
|
|
||||||
case "modules":
|
|
||||||
explicitModules = true
|
|
||||||
overrides.ModulesCSV = pFlags.modules
|
|
||||||
case "output-schema":
|
|
||||||
overrides.OutputSchema = pFlags.outputSchema
|
|
||||||
case "llm-api-key":
|
|
||||||
overrides.PrimaryLLMAPIKey = pFlags.llmAPIKey
|
|
||||||
case "validation-llm-api-key":
|
|
||||||
overrides.ValidationLLMAPIKey = pFlags.validationLLMAPIKey
|
|
||||||
case "model":
|
|
||||||
overrides.PrimaryModel = pFlags.model
|
|
||||||
case "validation-model":
|
|
||||||
overrides.ValidationModel = pFlags.validationModel
|
|
||||||
case "base-url":
|
|
||||||
overrides.PrimaryBaseURL = pFlags.baseURL
|
|
||||||
case "validation-base-url":
|
|
||||||
overrides.ValidationBaseURL = pFlags.validationBaseURL
|
|
||||||
case "llm-timeout-seconds":
|
|
||||||
overrides.PrimaryLLMTimeoutSeconds = pFlags.llmTimeoutSeconds
|
|
||||||
case "total-llm-concurrency":
|
|
||||||
overrides.TotalLLMConcurrency = pFlags.totalLLMConcurrency
|
|
||||||
case "proposal-llm-concurrency":
|
|
||||||
overrides.ProposalLLMConcurrency = pFlags.proposalLLMConcurrency
|
|
||||||
case "llm-concurrency":
|
|
||||||
overrides.PrimaryLLMConcurrency = pFlags.llmConcurrency
|
|
||||||
case "validation-llm-timeout-seconds":
|
|
||||||
overrides.ValidationLLMTimeoutSeconds = pFlags.validationLLMTimeoutSeconds
|
|
||||||
case "max-retries":
|
|
||||||
overrides.MaxRetries = pFlags.maxRetries
|
|
||||||
case "validation-max-retries":
|
|
||||||
overrides.ValidationMaxRetries = pFlags.validationMaxRetries
|
|
||||||
case "validation-llm-concurrency":
|
|
||||||
overrides.ValidationLLMConcurrency = pFlags.validationLLMConcurrency
|
|
||||||
case "validation-max-prompt-tokens":
|
|
||||||
overrides.ValidationMaxPromptTokens = pFlags.validationMaxPromptTokens
|
|
||||||
case "max-section-tokens":
|
|
||||||
overrides.MaxSectionTokens = pFlags.maxSectionTokens
|
|
||||||
case "min-section-tokens":
|
|
||||||
overrides.MinSectionTokens = pFlags.minSectionTokens
|
|
||||||
case "target-sections":
|
|
||||||
overrides.TargetSections = pFlags.targetSections
|
|
||||||
case "glossary-confidence-threshold":
|
|
||||||
overrides.GlossaryConfidenceThreshold = pFlags.glossaryConfidenceThreshold
|
|
||||||
case "grammar-confidence-threshold":
|
|
||||||
overrides.GrammarConfidenceThreshold = pFlags.grammarConfidenceThreshold
|
|
||||||
case "homophones-confidence-threshold":
|
|
||||||
overrides.HomophonesConfidenceThreshold = pFlags.homophonesConfidenceThreshold
|
|
||||||
case "spoken-word-confidence-threshold":
|
|
||||||
overrides.SpokenWordConfidenceThreshold = pFlags.spokenWordConfidenceThreshold
|
|
||||||
case "normalize-max-segment-gap":
|
|
||||||
overrides.NormalizeMaxSegmentGap = pFlags.normalizeMaxSegmentGap
|
|
||||||
case "normalize-ellipsis-gap":
|
|
||||||
overrides.NormalizeEllipsisGap = pFlags.normalizeEllipsisGap
|
|
||||||
case "normalize-max-segment-duration":
|
|
||||||
overrides.NormalizeMaxSegmentDuration = pFlags.normalizeMaxSegmentDuration
|
|
||||||
case "normalize-max-segment-tokens":
|
|
||||||
overrides.NormalizeMaxSegmentTokens = pFlags.normalizeMaxSegmentTokens
|
|
||||||
case "transcript-description":
|
|
||||||
overrides.TranscriptDescription = pFlags.transcriptDescription
|
|
||||||
case "work-dir":
|
|
||||||
overrides.WorkDir = pFlags.workDir
|
|
||||||
case "work-dir-retention":
|
|
||||||
overrides.WorkDirRetention = pFlags.workDirRetention
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
if err := cfg.ApplyCLIOverrides(overrides); err != nil {
|
if err := cfg.ApplyCLIOverrides(overrides); err != nil {
|
||||||
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
|
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
|
||||||
@@ -531,10 +463,13 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
|||||||
if runOutput.Utilization != nil {
|
if runOutput.Utilization != nil {
|
||||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
||||||
}
|
}
|
||||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, buildCorrectionLedger(runDir.Path(), runOutput))
|
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
|
||||||
|
RunDirectoryPath: runDir.Path(),
|
||||||
|
RunOutput: runOutput,
|
||||||
|
}))
|
||||||
}
|
}
|
||||||
errorPhase, errorMessage := extractErrorPhase(runErr)
|
errorPhase, errorMessage := extractErrorPhase(runErr)
|
||||||
report := buildProcessReport("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput)
|
report := processreport.Build(processReportInput("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput))
|
||||||
|
|
||||||
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
||||||
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
||||||
@@ -560,10 +495,13 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
|||||||
if runOutput.Utilization != nil {
|
if runOutput.Utilization != nil {
|
||||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
||||||
}
|
}
|
||||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, buildCorrectionLedger(runDir.Path(), runOutput))
|
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
|
||||||
|
RunDirectoryPath: runDir.Path(),
|
||||||
|
RunOutput: runOutput,
|
||||||
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
report := buildProcessReport("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput)
|
report := processreport.Build(processReportInput("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput))
|
||||||
|
|
||||||
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
if strings.TrimSpace(inv.ReportJSONPath) != "" {
|
||||||
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
if err := reporting.WriteProcessReport(inv.ReportJSONPath, report); err != nil {
|
||||||
@@ -578,21 +516,11 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
hasSkippedCorrections := false
|
|
||||||
if runOutput != nil {
|
|
||||||
for _, mr := range runOutput.ModuleResults {
|
|
||||||
if len(mr.SkippedChanges) > 0 || len(mr.ValidatorRejected) > 0 {
|
|
||||||
hasSkippedCorrections = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if runDir != nil {
|
if runDir != nil {
|
||||||
_ = runDir.WriteReport(report)
|
_ = runDir.WriteReport(report)
|
||||||
if err := runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
if err := runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
||||||
RunSucceeded: true,
|
RunSucceeded: true,
|
||||||
HasSkippedCorrections: hasSkippedCorrections,
|
HasSkippedCorrections: processreport.HasSkippedCorrections(runOutput),
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
fmt.Fprintf(stderr, "audita process: failed to apply work-dir retention: %v\n", err)
|
fmt.Fprintf(stderr, "audita process: failed to apply work-dir retention: %v\n", err)
|
||||||
return 1
|
return 1
|
||||||
@@ -710,130 +638,28 @@ func extractErrorPhase(err error) (phase string, message string) {
|
|||||||
return "", msg
|
return "", msg
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildProcessReport(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) reporting.ProcessReport {
|
func processReportInput(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) processreport.BuildInput {
|
||||||
report := reporting.ProcessReport{
|
runDirectoryPath := ""
|
||||||
ReportMetadata: reporting.ReportMetadata{
|
|
||||||
ReportSchemaName: reporting.DefaultProcessReportSchemaName,
|
|
||||||
ReportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
|
||||||
OutputSchema: inv.Config.OutputSchema,
|
|
||||||
ConfigVersion: inv.ConfigVersion,
|
|
||||||
},
|
|
||||||
Phase: "default_pipeline",
|
|
||||||
Status: status,
|
|
||||||
Operation: "process",
|
|
||||||
TranscriptPath: inv.TranscriptPath,
|
|
||||||
GlossaryPath: inv.GlossaryPath,
|
|
||||||
OutputPath: inv.OutputPath,
|
|
||||||
Modules: append([]string(nil), inv.Config.Modules...),
|
|
||||||
StartedAt: startedAt,
|
|
||||||
CompletedAt: &completedAt,
|
|
||||||
ErrorPhase: errorPhase,
|
|
||||||
}
|
|
||||||
if runDir != nil {
|
if runDir != nil {
|
||||||
runSucceeded := status == "success"
|
runDirectoryPath = runDir.Path()
|
||||||
metadata := diagnostics.BuildDiagnosticsMetadata(runDir.Path(), runSucceeded)
|
|
||||||
report.Diagnostics = &metadata
|
|
||||||
}
|
}
|
||||||
if errorMessage != "" {
|
return processreport.BuildInput{
|
||||||
report.ErrorMessage = errorMessage
|
Status: status,
|
||||||
|
TranscriptPath: inv.TranscriptPath,
|
||||||
|
GlossaryPath: inv.GlossaryPath,
|
||||||
|
OutputPath: inv.OutputPath,
|
||||||
|
Modules: inv.Config.Modules,
|
||||||
|
OutputSchema: inv.Config.OutputSchema,
|
||||||
|
ConfigVersion: inv.ConfigVersion,
|
||||||
|
StartedAt: startedAt,
|
||||||
|
CompletedAt: completedAt,
|
||||||
|
ErrorMessage: errorMessage,
|
||||||
|
ErrorPhase: errorPhase,
|
||||||
|
RunDirectoryPath: runDirectoryPath,
|
||||||
|
NormalizationSummary: normalizationSummary,
|
||||||
|
ChunkingSummary: chunkingSummary,
|
||||||
|
RunOutput: runOutput,
|
||||||
}
|
}
|
||||||
if normalizationSummary != nil {
|
|
||||||
report.InputSegmentCount = &normalizationSummary.InputSegmentCount
|
|
||||||
report.NormalizedSegmentCount = &normalizationSummary.OutputSegmentCount
|
|
||||||
report.NormalizationMerges = &normalizationSummary.MergesPerformed
|
|
||||||
report.NormalizationIDReassignments = &normalizationSummary.IDsReassigned
|
|
||||||
report.NormalizationSkipped.DifferentSpeakers = &normalizationSummary.SkippedMerges.DifferentSpeakers
|
|
||||||
report.NormalizationSkipped.GapTooLarge = &normalizationSummary.SkippedMerges.GapTooLarge
|
|
||||||
report.NormalizationSkipped.DurationExceeded = &normalizationSummary.SkippedMerges.DurationExceeded
|
|
||||||
report.NormalizationSkipped.TokenLimitExceeded = &normalizationSummary.SkippedMerges.TokenLimitExceeded
|
|
||||||
}
|
|
||||||
if chunkingSummary != nil {
|
|
||||||
report.Chunking = &reporting.ChunkingSummary{
|
|
||||||
ChunkCount: chunkingSummary.ChunkCount,
|
|
||||||
MinEstimatedTokens: chunkingSummary.MinEstimatedTokens,
|
|
||||||
MaxEstimatedTokens: chunkingSummary.MaxEstimatedTokens,
|
|
||||||
TotalEstimatedTokens: chunkingSummary.TotalEstimatedTokens,
|
|
||||||
TargetSections: chunkingSummary.TargetSections,
|
|
||||||
MaxSectionTokens: chunkingSummary.MaxSectionTokens,
|
|
||||||
MinSectionTokens: chunkingSummary.MinSectionTokens,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
report.ModulesSummary, report.ModuleResults = buildModuleReporting(runOutput)
|
|
||||||
return report
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummary, []reporting.ModuleReport) {
|
|
||||||
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
moduleReports := make([]reporting.ModuleReport, 0, len(runOutput.ModuleResults))
|
|
||||||
summary := &reporting.ModulesSummary{ModuleCount: len(runOutput.ModuleResults)}
|
|
||||||
for _, r := range runOutput.ModuleResults {
|
|
||||||
startedAt := r.StartedAt
|
|
||||||
completedAt := r.CompletedAt
|
|
||||||
moduleReports = append(moduleReports, reporting.ModuleReport{
|
|
||||||
ModuleKey: r.ModuleKey,
|
|
||||||
ModuleInstance: r.ModuleInstance,
|
|
||||||
ReplacementPolicy: string(r.ReplacementPolicy),
|
|
||||||
Status: r.Status,
|
|
||||||
ProposalCount: r.ProposalCount,
|
|
||||||
Warnings: append([]stagewarnings.StageWarning(nil), r.Warnings...),
|
|
||||||
ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions),
|
|
||||||
ValidatorRejected: mapValidatorRejected(r.ValidatorRejected),
|
|
||||||
AppliedChanges: r.AppliedChanges,
|
|
||||||
SkippedChanges: r.SkippedChanges,
|
|
||||||
ErrorMessage: r.ErrorMessage,
|
|
||||||
StartedAt: &startedAt,
|
|
||||||
CompletedAt: &completedAt,
|
|
||||||
})
|
|
||||||
summary.TotalAppliedChanges += len(r.AppliedChanges)
|
|
||||||
summary.TotalSkippedChanges += len(r.SkippedChanges) + len(r.ValidatorRejected)
|
|
||||||
if r.Status == runner.ModuleStatusFailed && summary.FailedModuleInstance == "" {
|
|
||||||
summary.FailedModuleInstance = r.ModuleInstance
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return summary, moduleReports
|
|
||||||
}
|
|
||||||
|
|
||||||
func mapValidatorDecisions(in []runner.ValidatorDecisionRecord) []reporting.ValidatorDecisionReport {
|
|
||||||
if len(in) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := make([]reporting.ValidatorDecisionReport, len(in))
|
|
||||||
for i, d := range in {
|
|
||||||
out[i] = reporting.ValidatorDecisionReport{
|
|
||||||
ValidatorName: d.ValidatorName,
|
|
||||||
ProposalIndex: d.ProposalIndex,
|
|
||||||
Approved: d.Approved,
|
|
||||||
ReasonCode: d.ReasonCode,
|
|
||||||
Message: d.Message,
|
|
||||||
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func mapValidatorRejected(in []runner.ValidatorRejectedChange) []reporting.ValidatorRejectedReport {
|
|
||||||
if len(in) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
out := make([]reporting.ValidatorRejectedReport, len(in))
|
|
||||||
for i, d := range in {
|
|
||||||
out[i] = reporting.ValidatorRejectedReport{
|
|
||||||
ValidatorName: d.ValidatorName,
|
|
||||||
ProposalIndex: d.ProposalIndex,
|
|
||||||
ModuleKey: d.ModuleKey,
|
|
||||||
ModuleInstance: d.ModuleInstance,
|
|
||||||
TargetSegmentID: d.TargetSegmentID,
|
|
||||||
OriginalText: d.OriginalText,
|
|
||||||
CorrectedText: d.CorrectedText,
|
|
||||||
ReasonCode: d.ReasonCode,
|
|
||||||
Message: d.Message,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type processFlags struct {
|
type processFlags struct {
|
||||||
|
|||||||
@@ -1173,6 +1173,15 @@ func waitForAtomicAtLeast(t *testing.T, value *int32, want int32) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunProcessReportJSONSuccessIncludesNormalizationSummary(t *testing.T) {
|
func TestRunProcessReportJSONSuccessIncludesNormalizationSummary(t *testing.T) {
|
||||||
|
// Exercise the no-config metadata contract without consulting system files.
|
||||||
|
t.Setenv("AUDITA_CONFIG", "")
|
||||||
|
if err := os.Unsetenv("AUDITA_CONFIG"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
searchPaths := config.DefaultConfigSearchPaths
|
||||||
|
config.DefaultConfigSearchPaths = nil
|
||||||
|
t.Cleanup(func() { config.DefaultConfigSearchPaths = searchPaths })
|
||||||
|
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
@@ -1350,7 +1359,7 @@ func TestRunProcessReportJSONAndRunDirReportShareDiagnosticsMetadata(t *testing.
|
|||||||
if externalReport.Diagnostics == nil || runDirReport.Diagnostics == nil {
|
if externalReport.Diagnostics == nil || runDirReport.Diagnostics == nil {
|
||||||
t.Fatalf("expected diagnostics metadata in both reports")
|
t.Fatalf("expected diagnostics metadata in both reports")
|
||||||
}
|
}
|
||||||
if externalReport.ReportMetadata != runDirReport.ReportMetadata {
|
if !reflect.DeepEqual(externalReport.ReportMetadata, runDirReport.ReportMetadata) {
|
||||||
t.Fatalf("expected same report metadata in --report-json and run-dir report\nexternal=%+v\nrun-dir=%+v",
|
t.Fatalf("expected same report metadata in --report-json and run-dir report\nexternal=%+v\nrun-dir=%+v",
|
||||||
externalReport.ReportMetadata, runDirReport.ReportMetadata)
|
externalReport.ReportMetadata, runDirReport.ReportMetadata)
|
||||||
}
|
}
|
||||||
|
|||||||
171
internal/core/config/apply_helpers.go
Normal file
171
internal/core/config/apply_helpers.go
Normal file
@@ -0,0 +1,171 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
|
type llmTargetPatch struct {
|
||||||
|
apiKey *string
|
||||||
|
model *string
|
||||||
|
baseURL *string
|
||||||
|
timeoutSeconds *int
|
||||||
|
maxRetries *int
|
||||||
|
}
|
||||||
|
|
||||||
|
type concurrencyPatch struct {
|
||||||
|
totalLLM *int
|
||||||
|
legacyTotalLLM *int
|
||||||
|
proposalLLM *int
|
||||||
|
validationLLM *int
|
||||||
|
inheritProposal bool
|
||||||
|
allowLegacyAlias bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type chunkingPatch struct {
|
||||||
|
targetSections *int
|
||||||
|
maxSectionTokens *int
|
||||||
|
minSectionTokens *int
|
||||||
|
}
|
||||||
|
|
||||||
|
type thresholdsPatch struct {
|
||||||
|
glossary *float64
|
||||||
|
grammar *float64
|
||||||
|
homophones *float64
|
||||||
|
spokenWord *float64
|
||||||
|
}
|
||||||
|
|
||||||
|
type normalizationPatch struct {
|
||||||
|
maxSegmentGap *float64
|
||||||
|
ellipsisGap *float64
|
||||||
|
maxSegmentDuration *float64
|
||||||
|
maxSegmentTokens *int
|
||||||
|
}
|
||||||
|
|
||||||
|
type contextPatch struct {
|
||||||
|
transcriptDescription *string
|
||||||
|
}
|
||||||
|
|
||||||
|
type diagnosticsPatch struct {
|
||||||
|
workDir *string
|
||||||
|
workDirRetention *string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Config) applyPrimaryLLMTargetPatch(patch llmTargetPatch) {
|
||||||
|
if patch.apiKey != nil {
|
||||||
|
c.PrimaryLLM.APIKey = *patch.apiKey
|
||||||
|
}
|
||||||
|
if patch.model != nil {
|
||||||
|
c.PrimaryLLM.Model = *patch.model
|
||||||
|
}
|
||||||
|
if patch.baseURL != nil {
|
||||||
|
c.PrimaryLLM.BaseURL = *patch.baseURL
|
||||||
|
}
|
||||||
|
if patch.timeoutSeconds != nil {
|
||||||
|
c.PrimaryLLM.TimeoutSeconds = *patch.timeoutSeconds
|
||||||
|
}
|
||||||
|
if patch.maxRetries != nil {
|
||||||
|
c.PrimaryLLM.MaxRetries = *patch.maxRetries
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Config) applyValidationLLMTargetPatch(patch llmTargetPatch) {
|
||||||
|
if patch.apiKey != nil {
|
||||||
|
c.ValidationLLM.APIKey = *patch.apiKey
|
||||||
|
}
|
||||||
|
if patch.model != nil {
|
||||||
|
c.ValidationLLM.Model = *patch.model
|
||||||
|
}
|
||||||
|
if patch.baseURL != nil {
|
||||||
|
c.ValidationLLM.BaseURL = *patch.baseURL
|
||||||
|
}
|
||||||
|
if patch.timeoutSeconds != nil {
|
||||||
|
value := *patch.timeoutSeconds
|
||||||
|
c.ValidationLLM.TimeoutSeconds = &value
|
||||||
|
}
|
||||||
|
if patch.maxRetries != nil {
|
||||||
|
value := *patch.maxRetries
|
||||||
|
c.ValidationLLM.MaxRetries = &value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Config) applyConcurrencyPatch(patch concurrencyPatch) {
|
||||||
|
totalSet := false
|
||||||
|
if patch.totalLLM != nil {
|
||||||
|
c.TotalLLMConcurrency = *patch.totalLLM
|
||||||
|
totalSet = true
|
||||||
|
}
|
||||||
|
if patch.allowLegacyAlias && patch.legacyTotalLLM != nil && !totalSet {
|
||||||
|
c.TotalLLMConcurrency = *patch.legacyTotalLLM
|
||||||
|
totalSet = true
|
||||||
|
}
|
||||||
|
|
||||||
|
proposalSet := false
|
||||||
|
if patch.proposalLLM != nil {
|
||||||
|
c.ProposalLLMConcurrency = *patch.proposalLLM
|
||||||
|
proposalSet = true
|
||||||
|
}
|
||||||
|
if patch.inheritProposal && totalSet && !proposalSet {
|
||||||
|
c.ProposalLLMConcurrency = c.TotalLLMConcurrency
|
||||||
|
}
|
||||||
|
|
||||||
|
if patch.validationLLM != nil {
|
||||||
|
value := *patch.validationLLM
|
||||||
|
c.ValidationLLMConcurrency = &value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Config) applyChunkingPatch(patch chunkingPatch) {
|
||||||
|
if patch.targetSections != nil {
|
||||||
|
value := *patch.targetSections
|
||||||
|
c.TargetSections = &value
|
||||||
|
}
|
||||||
|
if patch.maxSectionTokens != nil {
|
||||||
|
c.MaxSectionTokens = *patch.maxSectionTokens
|
||||||
|
}
|
||||||
|
if patch.minSectionTokens != nil {
|
||||||
|
c.MinSectionTokens = *patch.minSectionTokens
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Config) applyThresholdsPatch(patch thresholdsPatch) {
|
||||||
|
if patch.glossary != nil {
|
||||||
|
c.Thresholds.Glossary = *patch.glossary
|
||||||
|
}
|
||||||
|
if patch.grammar != nil {
|
||||||
|
c.Thresholds.Grammar = *patch.grammar
|
||||||
|
}
|
||||||
|
if patch.homophones != nil {
|
||||||
|
c.Thresholds.Homophones = *patch.homophones
|
||||||
|
}
|
||||||
|
if patch.spokenWord != nil {
|
||||||
|
c.Thresholds.SpokenWord = *patch.spokenWord
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Config) applyNormalizationPatch(patch normalizationPatch) {
|
||||||
|
if patch.maxSegmentGap != nil {
|
||||||
|
c.Normalization.MaxSegmentGap = *patch.maxSegmentGap
|
||||||
|
}
|
||||||
|
if patch.ellipsisGap != nil {
|
||||||
|
c.Normalization.EllipsisGap = *patch.ellipsisGap
|
||||||
|
}
|
||||||
|
if patch.maxSegmentDuration != nil {
|
||||||
|
c.Normalization.MaxSegmentDuration = *patch.maxSegmentDuration
|
||||||
|
}
|
||||||
|
if patch.maxSegmentTokens != nil {
|
||||||
|
c.Normalization.MaxSegmentTokens = *patch.maxSegmentTokens
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Config) applyContextPatch(patch contextPatch) {
|
||||||
|
if patch.transcriptDescription != nil {
|
||||||
|
c.TranscriptDescription = strings.TrimSpace(*patch.transcriptDescription)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Config) applyDiagnosticsPatch(patch diagnosticsPatch) {
|
||||||
|
if patch.workDir != nil {
|
||||||
|
c.WorkDir = *patch.workDir
|
||||||
|
}
|
||||||
|
if patch.workDirRetention != nil {
|
||||||
|
c.WorkDirRetention = WorkDirRetention(*patch.workDirRetention)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -239,6 +239,186 @@ func TestApplyCLIOverridesTrimsTranscriptDescription(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConfigSourcesApplySharedEffectiveFieldsConsistently(t *testing.T) {
|
||||||
|
fileCfg := mustParseFileConfigYAML(t, `
|
||||||
|
version: 1
|
||||||
|
output:
|
||||||
|
schema: " audita-v1 "
|
||||||
|
llm:
|
||||||
|
proposal:
|
||||||
|
base_url: https://proposal.example.test/v1
|
||||||
|
model: provider/proposal
|
||||||
|
timeout: 101
|
||||||
|
max_retries: 5
|
||||||
|
validation:
|
||||||
|
base_url: https://validation.example.test/v1
|
||||||
|
model: provider/validation
|
||||||
|
timeout: 202
|
||||||
|
max_retries: 6
|
||||||
|
chunking:
|
||||||
|
target_sections: 7
|
||||||
|
max_section_tokens: 9000
|
||||||
|
min_section_tokens: 1000
|
||||||
|
thresholds:
|
||||||
|
glossary: 0.91
|
||||||
|
grammar: 0.92
|
||||||
|
homophones: 0.93
|
||||||
|
spoken_word: 0.94
|
||||||
|
normalization:
|
||||||
|
max_segment_gap: 1.2
|
||||||
|
ellipsis_gap: 2.3
|
||||||
|
max_segment_duration: 45.6
|
||||||
|
max_segment_tokens: 321
|
||||||
|
context:
|
||||||
|
description: " shared context "
|
||||||
|
diagnostics:
|
||||||
|
work_dir: /tmp/audita-shared
|
||||||
|
retention: always
|
||||||
|
`)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
apply func(*Config) error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "file",
|
||||||
|
apply: func(cfg *Config) error {
|
||||||
|
return cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{}))
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "env",
|
||||||
|
apply: func(cfg *Config) error {
|
||||||
|
return cfg.applyEnvOverrides(mapLookup(map[string]string{
|
||||||
|
"AUDITA_MODEL": "provider/proposal",
|
||||||
|
"AUDITA_BASE_URL": "https://proposal.example.test/v1",
|
||||||
|
"AUDITA_LLM_TIMEOUT_SECONDS": "101",
|
||||||
|
"AUDITA_MAX_RETRIES": "5",
|
||||||
|
"AUDITA_VALIDATION_MODEL": "provider/validation",
|
||||||
|
"AUDITA_VALIDATION_BASE_URL": "https://validation.example.test/v1",
|
||||||
|
"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "202",
|
||||||
|
"AUDITA_VALIDATION_MAX_RETRIES": "6",
|
||||||
|
"AUDITA_TARGET_SECTIONS": "7",
|
||||||
|
"AUDITA_MAX_SECTION_TOKENS": "9000",
|
||||||
|
"AUDITA_MIN_SECTION_TOKENS": "1000",
|
||||||
|
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.91",
|
||||||
|
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.92",
|
||||||
|
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.93",
|
||||||
|
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.94",
|
||||||
|
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "1.2",
|
||||||
|
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "2.3",
|
||||||
|
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.6",
|
||||||
|
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "321",
|
||||||
|
"AUDITA_WORK_DIR": "/tmp/audita-shared",
|
||||||
|
"AUDITA_WORK_DIR_RETENTION": "always",
|
||||||
|
}))
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "cli",
|
||||||
|
apply: func(cfg *Config) error {
|
||||||
|
outputSchema := " audita-v1 "
|
||||||
|
proposalModel := "provider/proposal"
|
||||||
|
proposalBaseURL := "https://proposal.example.test/v1"
|
||||||
|
proposalTimeout := 101
|
||||||
|
proposalMaxRetries := 5
|
||||||
|
validationModel := "provider/validation"
|
||||||
|
validationBaseURL := "https://validation.example.test/v1"
|
||||||
|
validationTimeout := 202
|
||||||
|
validationMaxRetries := 6
|
||||||
|
targetSections := 7
|
||||||
|
maxSectionTokens := 9000
|
||||||
|
minSectionTokens := 1000
|
||||||
|
glossaryThreshold := 0.91
|
||||||
|
grammarThreshold := 0.92
|
||||||
|
homophonesThreshold := 0.93
|
||||||
|
spokenWordThreshold := 0.94
|
||||||
|
normalizeMaxSegmentGap := 1.2
|
||||||
|
normalizeEllipsisGap := 2.3
|
||||||
|
normalizeMaxSegmentDuration := 45.6
|
||||||
|
normalizeMaxSegmentTokens := 321
|
||||||
|
description := " shared context "
|
||||||
|
workDir := "/tmp/audita-shared"
|
||||||
|
workDirRetention := "always"
|
||||||
|
return cfg.ApplyCLIOverrides(CLIOverrides{
|
||||||
|
OutputSchema: &outputSchema,
|
||||||
|
PrimaryModel: &proposalModel,
|
||||||
|
PrimaryBaseURL: &proposalBaseURL,
|
||||||
|
PrimaryLLMTimeoutSeconds: &proposalTimeout,
|
||||||
|
MaxRetries: &proposalMaxRetries,
|
||||||
|
ValidationModel: &validationModel,
|
||||||
|
ValidationBaseURL: &validationBaseURL,
|
||||||
|
ValidationLLMTimeoutSeconds: &validationTimeout,
|
||||||
|
ValidationMaxRetries: &validationMaxRetries,
|
||||||
|
TargetSections: &targetSections,
|
||||||
|
MaxSectionTokens: &maxSectionTokens,
|
||||||
|
MinSectionTokens: &minSectionTokens,
|
||||||
|
GlossaryConfidenceThreshold: &glossaryThreshold,
|
||||||
|
GrammarConfidenceThreshold: &grammarThreshold,
|
||||||
|
HomophonesConfidenceThreshold: &homophonesThreshold,
|
||||||
|
SpokenWordConfidenceThreshold: &spokenWordThreshold,
|
||||||
|
NormalizeMaxSegmentGap: &normalizeMaxSegmentGap,
|
||||||
|
NormalizeEllipsisGap: &normalizeEllipsisGap,
|
||||||
|
NormalizeMaxSegmentDuration: &normalizeMaxSegmentDuration,
|
||||||
|
NormalizeMaxSegmentTokens: &normalizeMaxSegmentTokens,
|
||||||
|
TranscriptDescription: &description,
|
||||||
|
WorkDir: &workDir,
|
||||||
|
WorkDirRetention: &workDirRetention,
|
||||||
|
})
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
cfg := Default()
|
||||||
|
if err := tc.apply(&cfg); err != nil {
|
||||||
|
t.Fatalf("apply config source: %v", err)
|
||||||
|
}
|
||||||
|
assertSharedEffectiveFields(t, cfg, sharedEffectiveFieldOptions{
|
||||||
|
wantOutputSchemaOverride: tc.name != "env",
|
||||||
|
wantTranscriptDescriptionPatch: tc.name != "env",
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCLIAPIKeyOverrideIsDirectValue(t *testing.T) {
|
||||||
|
cfg := Default()
|
||||||
|
apiKey := "NOT_AN_ENV_VAR_NAME"
|
||||||
|
validationAPIKey := "also direct"
|
||||||
|
|
||||||
|
if err := cfg.ApplyCLIOverrides(CLIOverrides{PrimaryLLMAPIKey: &apiKey, ValidationLLMAPIKey: &validationAPIKey}); err != nil {
|
||||||
|
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.PrimaryLLM.APIKey != apiKey {
|
||||||
|
t.Fatalf("expected direct primary api key, got %q", cfg.PrimaryLLM.APIKey)
|
||||||
|
}
|
||||||
|
if cfg.ValidationLLM.APIKey != validationAPIKey {
|
||||||
|
t.Fatalf("expected direct validation api key, got %q", cfg.ValidationLLM.APIKey)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyFileConfigTotalConcurrencyDoesNotChangeProposalWhenProposalUnset(t *testing.T) {
|
||||||
|
fileCfg := mustParseFileConfigYAML(t, `
|
||||||
|
version: 1
|
||||||
|
concurrency:
|
||||||
|
total_llm: 4
|
||||||
|
`)
|
||||||
|
cfg := Default()
|
||||||
|
|
||||||
|
if err := cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{})); err != nil {
|
||||||
|
t.Fatalf("applyFileConfigWithLookup failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.TotalLLMConcurrency != 4 {
|
||||||
|
t.Fatalf("expected file total concurrency 4, got %d", cfg.TotalLLMConcurrency)
|
||||||
|
}
|
||||||
|
if cfg.ProposalLLMConcurrency != DefaultLLMConcurrency {
|
||||||
|
t.Fatalf("expected file config to preserve proposal concurrency when unset, got %d", cfg.ProposalLLMConcurrency)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestValidationRejectsOverlyLongTranscriptDescription(t *testing.T) {
|
func TestValidationRejectsOverlyLongTranscriptDescription(t *testing.T) {
|
||||||
cfg := Default()
|
cfg := Default()
|
||||||
cfg.TranscriptDescription = strings.Repeat("a", DefaultTranscriptDescriptionMaxChars+1)
|
cfg.TranscriptDescription = strings.Repeat("a", DefaultTranscriptDescriptionMaxChars+1)
|
||||||
@@ -456,3 +636,70 @@ func mapLookup(values map[string]string) func(string) (string, bool) {
|
|||||||
return value, ok
|
return value, ok
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func mustParseFileConfigYAML(t *testing.T, raw string) FileConfig {
|
||||||
|
t.Helper()
|
||||||
|
fileCfg, err := ParseFileConfigYAML([]byte(raw))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseFileConfigYAML failed: %v", err)
|
||||||
|
}
|
||||||
|
return fileCfg
|
||||||
|
}
|
||||||
|
|
||||||
|
type sharedEffectiveFieldOptions struct {
|
||||||
|
wantOutputSchemaOverride bool
|
||||||
|
wantTranscriptDescriptionPatch bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertSharedEffectiveFields(t *testing.T, cfg Config, opts sharedEffectiveFieldOptions) {
|
||||||
|
t.Helper()
|
||||||
|
wantOutputSchema := DefaultOutputSchema
|
||||||
|
if opts.wantOutputSchemaOverride {
|
||||||
|
wantOutputSchema = "audita-v1"
|
||||||
|
}
|
||||||
|
if cfg.OutputSchema != wantOutputSchema {
|
||||||
|
t.Fatalf("unexpected output schema: %q", cfg.OutputSchema)
|
||||||
|
}
|
||||||
|
if cfg.PrimaryLLM.Model != "provider/proposal" ||
|
||||||
|
cfg.PrimaryLLM.BaseURL != "https://proposal.example.test/v1" ||
|
||||||
|
cfg.PrimaryLLM.TimeoutSeconds != 101 ||
|
||||||
|
cfg.PrimaryLLM.MaxRetries != 5 {
|
||||||
|
t.Fatalf("unexpected primary llm config: %+v", cfg.PrimaryLLM)
|
||||||
|
}
|
||||||
|
if cfg.ValidationLLM.Model != "provider/validation" ||
|
||||||
|
cfg.ValidationLLM.BaseURL != "https://validation.example.test/v1" ||
|
||||||
|
cfg.ValidationLLM.TimeoutSeconds == nil ||
|
||||||
|
*cfg.ValidationLLM.TimeoutSeconds != 202 ||
|
||||||
|
cfg.ValidationLLM.MaxRetries == nil ||
|
||||||
|
*cfg.ValidationLLM.MaxRetries != 6 {
|
||||||
|
t.Fatalf("unexpected validation llm config: %+v", cfg.ValidationLLM)
|
||||||
|
}
|
||||||
|
if cfg.TargetSections == nil || *cfg.TargetSections != 7 ||
|
||||||
|
cfg.MaxSectionTokens != 9000 ||
|
||||||
|
cfg.MinSectionTokens != 1000 {
|
||||||
|
t.Fatalf("unexpected chunking config: target=%v max=%d min=%d", cfg.TargetSections, cfg.MaxSectionTokens, cfg.MinSectionTokens)
|
||||||
|
}
|
||||||
|
if cfg.Thresholds.Glossary != 0.91 ||
|
||||||
|
cfg.Thresholds.Grammar != 0.92 ||
|
||||||
|
cfg.Thresholds.Homophones != 0.93 ||
|
||||||
|
cfg.Thresholds.SpokenWord != 0.94 {
|
||||||
|
t.Fatalf("unexpected thresholds: %+v", cfg.Thresholds)
|
||||||
|
}
|
||||||
|
if cfg.Normalization.MaxSegmentGap != 1.2 ||
|
||||||
|
cfg.Normalization.EllipsisGap != 2.3 ||
|
||||||
|
cfg.Normalization.MaxSegmentDuration != 45.6 ||
|
||||||
|
cfg.Normalization.MaxSegmentTokens != 321 {
|
||||||
|
t.Fatalf("unexpected normalization: %+v", cfg.Normalization)
|
||||||
|
}
|
||||||
|
wantDescription := ""
|
||||||
|
if opts.wantTranscriptDescriptionPatch {
|
||||||
|
wantDescription = "shared context"
|
||||||
|
}
|
||||||
|
if cfg.TranscriptDescription != wantDescription {
|
||||||
|
t.Fatalf("unexpected transcript description: %q", cfg.TranscriptDescription)
|
||||||
|
}
|
||||||
|
if cfg.WorkDir != "/tmp/audita-shared" ||
|
||||||
|
cfg.WorkDirRetention != WorkDirRetentionAlways {
|
||||||
|
t.Fatalf("unexpected diagnostics config: work_dir=%q retention=%q", cfg.WorkDir, cfg.WorkDirRetention)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,8 +7,8 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
DefaultConfigPath = "/etc/audita/config.yml"
|
DefaultConfigPath = "/etc/audita/config.yml"
|
||||||
DefaultConfigPathUsrLocal = "/usr/local/etc/audita/config.yml"
|
DefaultConfigPathUsrLocal = "/usr/local/etc/audita/config.yml"
|
||||||
)
|
)
|
||||||
|
|
||||||
var DefaultConfigSearchPaths = []string{
|
var DefaultConfigSearchPaths = []string{
|
||||||
@@ -50,100 +50,93 @@ func (c *Config) applyEnvOverrides(lookup func(string) (string, bool)) error {
|
|||||||
cfg.Modules = modules
|
cfg.Modules = modules
|
||||||
}
|
}
|
||||||
|
|
||||||
|
primaryLLM := llmTargetPatch{}
|
||||||
if raw, ok := lookup("AUDITA_LLM_API_KEY"); ok {
|
if raw, ok := lookup("AUDITA_LLM_API_KEY"); ok {
|
||||||
cfg.PrimaryLLM.APIKey = raw
|
primaryLLM.apiKey = &raw
|
||||||
} else if raw, ok := lookup("OPENROUTER_API_KEY"); ok {
|
} else if raw, ok := lookup("OPENROUTER_API_KEY"); ok {
|
||||||
cfg.PrimaryLLM.APIKey = raw
|
primaryLLM.apiKey = &raw
|
||||||
}
|
}
|
||||||
|
|
||||||
if raw, ok := lookup("AUDITA_VALIDATION_LLM_API_KEY"); ok {
|
|
||||||
cfg.ValidationLLM.APIKey = raw
|
|
||||||
}
|
|
||||||
|
|
||||||
if raw, ok := lookup("AUDITA_MODEL"); ok {
|
if raw, ok := lookup("AUDITA_MODEL"); ok {
|
||||||
cfg.PrimaryLLM.Model = raw
|
primaryLLM.model = &raw
|
||||||
}
|
}
|
||||||
if raw, ok := lookup("AUDITA_VALIDATION_MODEL"); ok {
|
|
||||||
cfg.ValidationLLM.Model = raw
|
|
||||||
}
|
|
||||||
|
|
||||||
if raw, ok := lookup("AUDITA_BASE_URL"); ok {
|
if raw, ok := lookup("AUDITA_BASE_URL"); ok {
|
||||||
cfg.PrimaryLLM.BaseURL = raw
|
primaryLLM.baseURL = &raw
|
||||||
}
|
}
|
||||||
if raw, ok := lookup("AUDITA_VALIDATION_BASE_URL"); ok {
|
|
||||||
cfg.ValidationLLM.BaseURL = raw
|
|
||||||
}
|
|
||||||
|
|
||||||
if raw, ok := lookup("AUDITA_LLM_TIMEOUT_SECONDS"); ok {
|
if raw, ok := lookup("AUDITA_LLM_TIMEOUT_SECONDS"); ok {
|
||||||
value, err := parseInt(raw)
|
value, err := parseInt(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("AUDITA_LLM_TIMEOUT_SECONDS: %w", err)
|
return fmt.Errorf("AUDITA_LLM_TIMEOUT_SECONDS: %w", err)
|
||||||
}
|
}
|
||||||
cfg.PrimaryLLM.TimeoutSeconds = value
|
primaryLLM.timeoutSeconds = &value
|
||||||
}
|
}
|
||||||
|
|
||||||
if raw, ok := lookup("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"); ok {
|
|
||||||
value, err := parseInt(raw)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS: %w", err)
|
|
||||||
}
|
|
||||||
cfg.ValidationLLM.TimeoutSeconds = &value
|
|
||||||
}
|
|
||||||
|
|
||||||
if raw, ok := lookup("AUDITA_MAX_RETRIES"); ok {
|
if raw, ok := lookup("AUDITA_MAX_RETRIES"); ok {
|
||||||
value, err := parseInt(raw)
|
value, err := parseInt(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("AUDITA_MAX_RETRIES: %w", err)
|
return fmt.Errorf("AUDITA_MAX_RETRIES: %w", err)
|
||||||
}
|
}
|
||||||
cfg.PrimaryLLM.MaxRetries = value
|
primaryLLM.maxRetries = &value
|
||||||
|
}
|
||||||
|
cfg.applyPrimaryLLMTargetPatch(primaryLLM)
|
||||||
|
|
||||||
|
validationLLM := llmTargetPatch{}
|
||||||
|
if raw, ok := lookup("AUDITA_VALIDATION_LLM_API_KEY"); ok {
|
||||||
|
validationLLM.apiKey = &raw
|
||||||
|
}
|
||||||
|
if raw, ok := lookup("AUDITA_VALIDATION_MODEL"); ok {
|
||||||
|
validationLLM.model = &raw
|
||||||
|
}
|
||||||
|
if raw, ok := lookup("AUDITA_VALIDATION_BASE_URL"); ok {
|
||||||
|
validationLLM.baseURL = &raw
|
||||||
|
}
|
||||||
|
if raw, ok := lookup("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"); ok {
|
||||||
|
value, err := parseInt(raw)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS: %w", err)
|
||||||
|
}
|
||||||
|
validationLLM.timeoutSeconds = &value
|
||||||
|
}
|
||||||
|
if raw, ok := lookup("AUDITA_VALIDATION_MAX_RETRIES"); ok {
|
||||||
|
value, err := parseInt(raw)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("AUDITA_VALIDATION_MAX_RETRIES: %w", err)
|
||||||
|
}
|
||||||
|
validationLLM.maxRetries = &value
|
||||||
|
}
|
||||||
|
cfg.applyValidationLLMTargetPatch(validationLLM)
|
||||||
|
|
||||||
|
concurrency := concurrencyPatch{
|
||||||
|
inheritProposal: true,
|
||||||
|
allowLegacyAlias: true,
|
||||||
}
|
}
|
||||||
totalConcurrencySet := false
|
|
||||||
if raw, ok := lookup("AUDITA_TOTAL_LLM_CONCURRENCY"); ok {
|
if raw, ok := lookup("AUDITA_TOTAL_LLM_CONCURRENCY"); ok {
|
||||||
value, err := parseInt(raw)
|
value, err := parseInt(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("AUDITA_TOTAL_LLM_CONCURRENCY: %w", err)
|
return fmt.Errorf("AUDITA_TOTAL_LLM_CONCURRENCY: %w", err)
|
||||||
}
|
}
|
||||||
cfg.TotalLLMConcurrency = value
|
concurrency.totalLLM = &value
|
||||||
totalConcurrencySet = true
|
|
||||||
}
|
}
|
||||||
if raw, ok := lookup("AUDITA_LLM_CONCURRENCY"); ok {
|
if raw, ok := lookup("AUDITA_LLM_CONCURRENCY"); ok {
|
||||||
value, err := parseInt(raw)
|
value, err := parseInt(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("AUDITA_LLM_CONCURRENCY: %w", err)
|
return fmt.Errorf("AUDITA_LLM_CONCURRENCY: %w", err)
|
||||||
}
|
}
|
||||||
if !totalConcurrencySet {
|
concurrency.legacyTotalLLM = &value
|
||||||
cfg.TotalLLMConcurrency = value
|
|
||||||
totalConcurrencySet = true
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
proposalConcurrencySet := false
|
|
||||||
if raw, ok := lookup("AUDITA_PROPOSAL_LLM_CONCURRENCY"); ok {
|
if raw, ok := lookup("AUDITA_PROPOSAL_LLM_CONCURRENCY"); ok {
|
||||||
value, err := parseInt(raw)
|
value, err := parseInt(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("AUDITA_PROPOSAL_LLM_CONCURRENCY: %w", err)
|
return fmt.Errorf("AUDITA_PROPOSAL_LLM_CONCURRENCY: %w", err)
|
||||||
}
|
}
|
||||||
cfg.ProposalLLMConcurrency = value
|
concurrency.proposalLLM = &value
|
||||||
proposalConcurrencySet = true
|
|
||||||
}
|
|
||||||
if totalConcurrencySet && !proposalConcurrencySet {
|
|
||||||
cfg.ProposalLLMConcurrency = cfg.TotalLLMConcurrency
|
|
||||||
}
|
|
||||||
|
|
||||||
if raw, ok := lookup("AUDITA_VALIDATION_MAX_RETRIES"); ok {
|
|
||||||
value, err := parseInt(raw)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("AUDITA_VALIDATION_MAX_RETRIES: %w", err)
|
|
||||||
}
|
|
||||||
cfg.ValidationLLM.MaxRetries = &value
|
|
||||||
}
|
}
|
||||||
if raw, ok := lookup("AUDITA_VALIDATION_LLM_CONCURRENCY"); ok {
|
if raw, ok := lookup("AUDITA_VALIDATION_LLM_CONCURRENCY"); ok {
|
||||||
value, err := parseInt(raw)
|
value, err := parseInt(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("AUDITA_VALIDATION_LLM_CONCURRENCY: %w", err)
|
return fmt.Errorf("AUDITA_VALIDATION_LLM_CONCURRENCY: %w", err)
|
||||||
}
|
}
|
||||||
cfg.ValidationLLMConcurrency = &value
|
concurrency.validationLLM = &value
|
||||||
}
|
}
|
||||||
|
cfg.applyConcurrencyPatch(concurrency)
|
||||||
|
|
||||||
if raw, ok := lookup("AUDITA_VALIDATION_MAX_PROMPT_TOKENS"); ok {
|
if raw, ok := lookup("AUDITA_VALIDATION_MAX_PROMPT_TOKENS"); ok {
|
||||||
value, err := parseInt(raw)
|
value, err := parseInt(raw)
|
||||||
@@ -153,12 +146,13 @@ func (c *Config) applyEnvOverrides(lookup func(string) (string, bool)) error {
|
|||||||
cfg.ValidationMaxPromptTokens = value
|
cfg.ValidationMaxPromptTokens = value
|
||||||
}
|
}
|
||||||
|
|
||||||
|
chunking := chunkingPatch{}
|
||||||
if raw, ok := lookup("AUDITA_MAX_SECTION_TOKENS"); ok {
|
if raw, ok := lookup("AUDITA_MAX_SECTION_TOKENS"); ok {
|
||||||
value, err := parseInt(raw)
|
value, err := parseInt(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("AUDITA_MAX_SECTION_TOKENS: %w", err)
|
return fmt.Errorf("AUDITA_MAX_SECTION_TOKENS: %w", err)
|
||||||
}
|
}
|
||||||
cfg.MaxSectionTokens = value
|
chunking.maxSectionTokens = &value
|
||||||
}
|
}
|
||||||
|
|
||||||
if raw, ok := lookup("AUDITA_MIN_SECTION_TOKENS"); ok {
|
if raw, ok := lookup("AUDITA_MIN_SECTION_TOKENS"); ok {
|
||||||
@@ -166,7 +160,7 @@ func (c *Config) applyEnvOverrides(lookup func(string) (string, bool)) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("AUDITA_MIN_SECTION_TOKENS: %w", err)
|
return fmt.Errorf("AUDITA_MIN_SECTION_TOKENS: %w", err)
|
||||||
}
|
}
|
||||||
cfg.MinSectionTokens = value
|
chunking.minSectionTokens = &value
|
||||||
}
|
}
|
||||||
|
|
||||||
if raw, ok := lookup("AUDITA_TARGET_SECTIONS"); ok {
|
if raw, ok := lookup("AUDITA_TARGET_SECTIONS"); ok {
|
||||||
@@ -174,73 +168,80 @@ func (c *Config) applyEnvOverrides(lookup func(string) (string, bool)) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("AUDITA_TARGET_SECTIONS: %w", err)
|
return fmt.Errorf("AUDITA_TARGET_SECTIONS: %w", err)
|
||||||
}
|
}
|
||||||
cfg.TargetSections = &value
|
chunking.targetSections = &value
|
||||||
}
|
}
|
||||||
|
cfg.applyChunkingPatch(chunking)
|
||||||
|
|
||||||
|
thresholds := thresholdsPatch{}
|
||||||
if raw, ok := lookup("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD"); ok {
|
if raw, ok := lookup("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD"); ok {
|
||||||
value, err := parseFloat(raw)
|
value, err := parseFloat(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD: %w", err)
|
return fmt.Errorf("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD: %w", err)
|
||||||
}
|
}
|
||||||
cfg.Thresholds.Glossary = value
|
thresholds.glossary = &value
|
||||||
}
|
}
|
||||||
if raw, ok := lookup("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD"); ok {
|
if raw, ok := lookup("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD"); ok {
|
||||||
value, err := parseFloat(raw)
|
value, err := parseFloat(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD: %w", err)
|
return fmt.Errorf("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD: %w", err)
|
||||||
}
|
}
|
||||||
cfg.Thresholds.Grammar = value
|
thresholds.grammar = &value
|
||||||
}
|
}
|
||||||
if raw, ok := lookup("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD"); ok {
|
if raw, ok := lookup("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD"); ok {
|
||||||
value, err := parseFloat(raw)
|
value, err := parseFloat(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD: %w", err)
|
return fmt.Errorf("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD: %w", err)
|
||||||
}
|
}
|
||||||
cfg.Thresholds.Homophones = value
|
thresholds.homophones = &value
|
||||||
}
|
}
|
||||||
if raw, ok := lookup("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD"); ok {
|
if raw, ok := lookup("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD"); ok {
|
||||||
value, err := parseFloat(raw)
|
value, err := parseFloat(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD: %w", err)
|
return fmt.Errorf("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD: %w", err)
|
||||||
}
|
}
|
||||||
cfg.Thresholds.SpokenWord = value
|
thresholds.spokenWord = &value
|
||||||
}
|
}
|
||||||
|
cfg.applyThresholdsPatch(thresholds)
|
||||||
|
|
||||||
|
normalization := normalizationPatch{}
|
||||||
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_GAP"); ok {
|
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_GAP"); ok {
|
||||||
value, err := parseFloat(raw)
|
value, err := parseFloat(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_GAP: %w", err)
|
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_GAP: %w", err)
|
||||||
}
|
}
|
||||||
cfg.Normalization.MaxSegmentGap = value
|
normalization.maxSegmentGap = &value
|
||||||
}
|
}
|
||||||
if raw, ok := lookup("AUDITA_NORMALIZE_ELLIPSIS_GAP"); ok {
|
if raw, ok := lookup("AUDITA_NORMALIZE_ELLIPSIS_GAP"); ok {
|
||||||
value, err := parseFloat(raw)
|
value, err := parseFloat(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("AUDITA_NORMALIZE_ELLIPSIS_GAP: %w", err)
|
return fmt.Errorf("AUDITA_NORMALIZE_ELLIPSIS_GAP: %w", err)
|
||||||
}
|
}
|
||||||
cfg.Normalization.EllipsisGap = value
|
normalization.ellipsisGap = &value
|
||||||
}
|
}
|
||||||
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION"); ok {
|
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION"); ok {
|
||||||
value, err := parseFloat(raw)
|
value, err := parseFloat(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION: %w", err)
|
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_DURATION: %w", err)
|
||||||
}
|
}
|
||||||
cfg.Normalization.MaxSegmentDuration = value
|
normalization.maxSegmentDuration = &value
|
||||||
}
|
}
|
||||||
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS"); ok {
|
if raw, ok := lookup("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS"); ok {
|
||||||
value, err := parseInt(raw)
|
value, err := parseInt(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS: %w", err)
|
return fmt.Errorf("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS: %w", err)
|
||||||
}
|
}
|
||||||
cfg.Normalization.MaxSegmentTokens = value
|
normalization.maxSegmentTokens = &value
|
||||||
}
|
}
|
||||||
|
cfg.applyNormalizationPatch(normalization)
|
||||||
|
|
||||||
|
diagnostics := diagnosticsPatch{}
|
||||||
if raw, ok := lookup("AUDITA_WORK_DIR"); ok {
|
if raw, ok := lookup("AUDITA_WORK_DIR"); ok {
|
||||||
cfg.WorkDir = raw
|
diagnostics.workDir = &raw
|
||||||
}
|
}
|
||||||
if raw, ok := lookup("AUDITA_WORK_DIR_RETENTION"); ok {
|
if raw, ok := lookup("AUDITA_WORK_DIR_RETENTION"); ok {
|
||||||
cfg.WorkDirRetention = WorkDirRetention(raw)
|
diagnostics.workDirRetention = &raw
|
||||||
}
|
}
|
||||||
|
cfg.applyDiagnosticsPatch(diagnostics)
|
||||||
|
|
||||||
cfg.syncLegacyConcurrencyAliases()
|
cfg.syncLegacyConcurrencyAliases()
|
||||||
|
|
||||||
|
|||||||
@@ -205,118 +205,98 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
|||||||
|
|
||||||
if fileCfg.LLM != nil {
|
if fileCfg.LLM != nil {
|
||||||
if fileCfg.LLM.Proposal != nil {
|
if fileCfg.LLM.Proposal != nil {
|
||||||
if fileCfg.LLM.Proposal.BaseURL != nil {
|
patch := llmTargetPatch{
|
||||||
c.PrimaryLLM.BaseURL = *fileCfg.LLM.Proposal.BaseURL
|
model: fileCfg.LLM.Proposal.Model,
|
||||||
}
|
baseURL: fileCfg.LLM.Proposal.BaseURL,
|
||||||
if fileCfg.LLM.Proposal.Model != nil {
|
maxRetries: fileCfg.LLM.Proposal.MaxRetries,
|
||||||
c.PrimaryLLM.Model = *fileCfg.LLM.Proposal.Model
|
|
||||||
}
|
}
|
||||||
if fileCfg.LLM.Proposal.Timeout != nil {
|
if fileCfg.LLM.Proposal.Timeout != nil {
|
||||||
c.PrimaryLLM.TimeoutSeconds = fileCfg.LLM.Proposal.Timeout.Seconds()
|
timeoutSeconds := fileCfg.LLM.Proposal.Timeout.Seconds()
|
||||||
}
|
patch.timeoutSeconds = &timeoutSeconds
|
||||||
if fileCfg.LLM.Proposal.MaxRetries != nil {
|
|
||||||
c.PrimaryLLM.MaxRetries = *fileCfg.LLM.Proposal.MaxRetries
|
|
||||||
}
|
}
|
||||||
if fileCfg.LLM.Proposal.APIKeyEnv != nil {
|
if fileCfg.LLM.Proposal.APIKeyEnv != nil {
|
||||||
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Proposal.APIKeyEnv, lookup)
|
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Proposal.APIKeyEnv, lookup)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("llm.proposal.api_key_env: %w", err)
|
return fmt.Errorf("llm.proposal.api_key_env: %w", err)
|
||||||
}
|
}
|
||||||
c.PrimaryLLM.APIKey = apiKey
|
patch.apiKey = &apiKey
|
||||||
}
|
}
|
||||||
|
c.applyPrimaryLLMTargetPatch(patch)
|
||||||
}
|
}
|
||||||
if fileCfg.LLM.Validation != nil {
|
if fileCfg.LLM.Validation != nil {
|
||||||
if fileCfg.LLM.Validation.BaseURL != nil {
|
patch := llmTargetPatch{
|
||||||
c.ValidationLLM.BaseURL = *fileCfg.LLM.Validation.BaseURL
|
model: fileCfg.LLM.Validation.Model,
|
||||||
}
|
baseURL: fileCfg.LLM.Validation.BaseURL,
|
||||||
if fileCfg.LLM.Validation.Model != nil {
|
maxRetries: fileCfg.LLM.Validation.MaxRetries,
|
||||||
c.ValidationLLM.Model = *fileCfg.LLM.Validation.Model
|
|
||||||
}
|
}
|
||||||
if fileCfg.LLM.Validation.Timeout != nil {
|
if fileCfg.LLM.Validation.Timeout != nil {
|
||||||
v := fileCfg.LLM.Validation.Timeout.Seconds()
|
timeoutSeconds := fileCfg.LLM.Validation.Timeout.Seconds()
|
||||||
c.ValidationLLM.TimeoutSeconds = &v
|
patch.timeoutSeconds = &timeoutSeconds
|
||||||
}
|
|
||||||
if fileCfg.LLM.Validation.MaxRetries != nil {
|
|
||||||
v := *fileCfg.LLM.Validation.MaxRetries
|
|
||||||
c.ValidationLLM.MaxRetries = &v
|
|
||||||
}
|
}
|
||||||
if fileCfg.LLM.Validation.APIKeyEnv != nil {
|
if fileCfg.LLM.Validation.APIKeyEnv != nil {
|
||||||
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Validation.APIKeyEnv, lookup)
|
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Validation.APIKeyEnv, lookup)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("llm.validation.api_key_env: %w", err)
|
return fmt.Errorf("llm.validation.api_key_env: %w", err)
|
||||||
}
|
}
|
||||||
c.ValidationLLM.APIKey = apiKey
|
patch.apiKey = &apiKey
|
||||||
}
|
}
|
||||||
|
c.applyValidationLLMTargetPatch(patch)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if fileCfg.Concurrency != nil {
|
if fileCfg.Concurrency != nil {
|
||||||
if fileCfg.Concurrency.TotalLLM != nil {
|
c.applyConcurrencyPatch(concurrencyPatch{
|
||||||
c.TotalLLMConcurrency = *fileCfg.Concurrency.TotalLLM
|
totalLLM: fileCfg.Concurrency.TotalLLM,
|
||||||
}
|
proposalLLM: fileCfg.Concurrency.ProposalLLM,
|
||||||
if fileCfg.Concurrency.ProposalLLM != nil {
|
validationLLM: fileCfg.Concurrency.ValidationLLM,
|
||||||
c.ProposalLLMConcurrency = *fileCfg.Concurrency.ProposalLLM
|
})
|
||||||
}
|
|
||||||
if fileCfg.Concurrency.ValidationLLM != nil {
|
|
||||||
v := *fileCfg.Concurrency.ValidationLLM
|
|
||||||
c.ValidationLLMConcurrency = &v
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if fileCfg.Chunking != nil {
|
if fileCfg.Chunking != nil {
|
||||||
if fileCfg.Chunking.TargetSections != nil {
|
c.applyChunkingPatch(chunkingPatch{
|
||||||
v := *fileCfg.Chunking.TargetSections
|
targetSections: fileCfg.Chunking.TargetSections,
|
||||||
c.TargetSections = &v
|
maxSectionTokens: fileCfg.Chunking.MaxSectionTokens,
|
||||||
}
|
minSectionTokens: fileCfg.Chunking.MinSectionTokens,
|
||||||
if fileCfg.Chunking.MaxSectionTokens != nil {
|
})
|
||||||
c.MaxSectionTokens = *fileCfg.Chunking.MaxSectionTokens
|
|
||||||
}
|
|
||||||
if fileCfg.Chunking.MinSectionTokens != nil {
|
|
||||||
c.MinSectionTokens = *fileCfg.Chunking.MinSectionTokens
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if fileCfg.Normalization != nil {
|
if fileCfg.Normalization != nil {
|
||||||
|
patch := normalizationPatch{
|
||||||
|
maxSegmentTokens: fileCfg.Normalization.MaxSegmentTokens,
|
||||||
|
}
|
||||||
if fileCfg.Normalization.MaxSegmentGap != nil {
|
if fileCfg.Normalization.MaxSegmentGap != nil {
|
||||||
c.Normalization.MaxSegmentGap = fileCfg.Normalization.MaxSegmentGap.Seconds()
|
maxSegmentGap := fileCfg.Normalization.MaxSegmentGap.Seconds()
|
||||||
|
patch.maxSegmentGap = &maxSegmentGap
|
||||||
}
|
}
|
||||||
if fileCfg.Normalization.EllipsisGap != nil {
|
if fileCfg.Normalization.EllipsisGap != nil {
|
||||||
c.Normalization.EllipsisGap = fileCfg.Normalization.EllipsisGap.Seconds()
|
ellipsisGap := fileCfg.Normalization.EllipsisGap.Seconds()
|
||||||
|
patch.ellipsisGap = &ellipsisGap
|
||||||
}
|
}
|
||||||
if fileCfg.Normalization.MaxSegmentDuration != nil {
|
if fileCfg.Normalization.MaxSegmentDuration != nil {
|
||||||
c.Normalization.MaxSegmentDuration = fileCfg.Normalization.MaxSegmentDuration.Seconds()
|
maxSegmentDuration := fileCfg.Normalization.MaxSegmentDuration.Seconds()
|
||||||
}
|
patch.maxSegmentDuration = &maxSegmentDuration
|
||||||
if fileCfg.Normalization.MaxSegmentTokens != nil {
|
|
||||||
c.Normalization.MaxSegmentTokens = *fileCfg.Normalization.MaxSegmentTokens
|
|
||||||
}
|
}
|
||||||
|
c.applyNormalizationPatch(patch)
|
||||||
}
|
}
|
||||||
|
|
||||||
if fileCfg.Thresholds != nil {
|
if fileCfg.Thresholds != nil {
|
||||||
if fileCfg.Thresholds.Glossary != nil {
|
c.applyThresholdsPatch(thresholdsPatch{
|
||||||
c.Thresholds.Glossary = *fileCfg.Thresholds.Glossary
|
glossary: fileCfg.Thresholds.Glossary,
|
||||||
}
|
grammar: fileCfg.Thresholds.Grammar,
|
||||||
if fileCfg.Thresholds.Homophones != nil {
|
homophones: fileCfg.Thresholds.Homophones,
|
||||||
c.Thresholds.Homophones = *fileCfg.Thresholds.Homophones
|
spokenWord: fileCfg.Thresholds.SpokenWord,
|
||||||
}
|
})
|
||||||
if fileCfg.Thresholds.SpokenWord != nil {
|
|
||||||
c.Thresholds.SpokenWord = *fileCfg.Thresholds.SpokenWord
|
|
||||||
}
|
|
||||||
if fileCfg.Thresholds.Grammar != nil {
|
|
||||||
c.Thresholds.Grammar = *fileCfg.Thresholds.Grammar
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if fileCfg.Context != nil && fileCfg.Context.Description != nil {
|
if fileCfg.Context != nil && fileCfg.Context.Description != nil {
|
||||||
c.TranscriptDescription = strings.TrimSpace(*fileCfg.Context.Description)
|
c.applyContextPatch(contextPatch{transcriptDescription: fileCfg.Context.Description})
|
||||||
}
|
}
|
||||||
|
|
||||||
if fileCfg.Diagnostics != nil {
|
if fileCfg.Diagnostics != nil {
|
||||||
if fileCfg.Diagnostics.WorkDir != nil {
|
c.applyDiagnosticsPatch(diagnosticsPatch{
|
||||||
c.WorkDir = *fileCfg.Diagnostics.WorkDir
|
workDir: fileCfg.Diagnostics.WorkDir,
|
||||||
}
|
workDirRetention: fileCfg.Diagnostics.Retention,
|
||||||
if fileCfg.Diagnostics.Retention != nil {
|
})
|
||||||
c.WorkDirRetention = WorkDirRetention(*fileCfg.Diagnostics.Retention)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
c.syncLegacyConcurrencyAliases()
|
c.syncLegacyConcurrencyAliases()
|
||||||
|
|||||||
@@ -51,107 +51,54 @@ func (c *Config) ApplyCLIOverrides(overrides CLIOverrides) error {
|
|||||||
c.OutputSchema = strings.TrimSpace(*overrides.OutputSchema)
|
c.OutputSchema = strings.TrimSpace(*overrides.OutputSchema)
|
||||||
}
|
}
|
||||||
|
|
||||||
if overrides.PrimaryLLMAPIKey != nil {
|
c.applyPrimaryLLMTargetPatch(llmTargetPatch{
|
||||||
c.PrimaryLLM.APIKey = *overrides.PrimaryLLMAPIKey
|
apiKey: overrides.PrimaryLLMAPIKey,
|
||||||
}
|
model: overrides.PrimaryModel,
|
||||||
if overrides.ValidationLLMAPIKey != nil {
|
baseURL: overrides.PrimaryBaseURL,
|
||||||
c.ValidationLLM.APIKey = *overrides.ValidationLLMAPIKey
|
timeoutSeconds: overrides.PrimaryLLMTimeoutSeconds,
|
||||||
}
|
maxRetries: overrides.MaxRetries,
|
||||||
if overrides.PrimaryModel != nil {
|
})
|
||||||
c.PrimaryLLM.Model = *overrides.PrimaryModel
|
c.applyValidationLLMTargetPatch(llmTargetPatch{
|
||||||
}
|
apiKey: overrides.ValidationLLMAPIKey,
|
||||||
if overrides.ValidationModel != nil {
|
model: overrides.ValidationModel,
|
||||||
c.ValidationLLM.Model = *overrides.ValidationModel
|
baseURL: overrides.ValidationBaseURL,
|
||||||
}
|
timeoutSeconds: overrides.ValidationLLMTimeoutSeconds,
|
||||||
if overrides.PrimaryBaseURL != nil {
|
maxRetries: overrides.ValidationMaxRetries,
|
||||||
c.PrimaryLLM.BaseURL = *overrides.PrimaryBaseURL
|
})
|
||||||
}
|
c.applyConcurrencyPatch(concurrencyPatch{
|
||||||
if overrides.ValidationBaseURL != nil {
|
totalLLM: overrides.TotalLLMConcurrency,
|
||||||
c.ValidationLLM.BaseURL = *overrides.ValidationBaseURL
|
legacyTotalLLM: overrides.PrimaryLLMConcurrency,
|
||||||
}
|
proposalLLM: overrides.ProposalLLMConcurrency,
|
||||||
if overrides.PrimaryLLMTimeoutSeconds != nil {
|
validationLLM: overrides.ValidationLLMConcurrency,
|
||||||
c.PrimaryLLM.TimeoutSeconds = *overrides.PrimaryLLMTimeoutSeconds
|
inheritProposal: true,
|
||||||
}
|
allowLegacyAlias: true,
|
||||||
totalConcurrencySet := false
|
})
|
||||||
if overrides.TotalLLMConcurrency != nil {
|
|
||||||
c.TotalLLMConcurrency = *overrides.TotalLLMConcurrency
|
|
||||||
totalConcurrencySet = true
|
|
||||||
}
|
|
||||||
// Backward-compatible alias: --llm-concurrency maps to total concurrency
|
|
||||||
// only when --total-llm-concurrency is not set in the same CLI invocation.
|
|
||||||
if overrides.PrimaryLLMConcurrency != nil && !totalConcurrencySet {
|
|
||||||
c.TotalLLMConcurrency = *overrides.PrimaryLLMConcurrency
|
|
||||||
totalConcurrencySet = true
|
|
||||||
}
|
|
||||||
proposalConcurrencySet := false
|
|
||||||
if overrides.ProposalLLMConcurrency != nil {
|
|
||||||
c.ProposalLLMConcurrency = *overrides.ProposalLLMConcurrency
|
|
||||||
proposalConcurrencySet = true
|
|
||||||
}
|
|
||||||
if totalConcurrencySet && !proposalConcurrencySet {
|
|
||||||
c.ProposalLLMConcurrency = c.TotalLLMConcurrency
|
|
||||||
}
|
|
||||||
if overrides.ValidationLLMTimeoutSeconds != nil {
|
|
||||||
value := *overrides.ValidationLLMTimeoutSeconds
|
|
||||||
c.ValidationLLM.TimeoutSeconds = &value
|
|
||||||
}
|
|
||||||
if overrides.MaxRetries != nil {
|
|
||||||
c.PrimaryLLM.MaxRetries = *overrides.MaxRetries
|
|
||||||
}
|
|
||||||
if overrides.ValidationMaxRetries != nil {
|
|
||||||
value := *overrides.ValidationMaxRetries
|
|
||||||
c.ValidationLLM.MaxRetries = &value
|
|
||||||
}
|
|
||||||
if overrides.ValidationLLMConcurrency != nil {
|
|
||||||
value := *overrides.ValidationLLMConcurrency
|
|
||||||
c.ValidationLLMConcurrency = &value
|
|
||||||
}
|
|
||||||
if overrides.ValidationMaxPromptTokens != nil {
|
if overrides.ValidationMaxPromptTokens != nil {
|
||||||
c.ValidationMaxPromptTokens = *overrides.ValidationMaxPromptTokens
|
c.ValidationMaxPromptTokens = *overrides.ValidationMaxPromptTokens
|
||||||
}
|
}
|
||||||
if overrides.MaxSectionTokens != nil {
|
c.applyChunkingPatch(chunkingPatch{
|
||||||
c.MaxSectionTokens = *overrides.MaxSectionTokens
|
targetSections: overrides.TargetSections,
|
||||||
}
|
maxSectionTokens: overrides.MaxSectionTokens,
|
||||||
if overrides.MinSectionTokens != nil {
|
minSectionTokens: overrides.MinSectionTokens,
|
||||||
c.MinSectionTokens = *overrides.MinSectionTokens
|
})
|
||||||
}
|
c.applyThresholdsPatch(thresholdsPatch{
|
||||||
if overrides.TargetSections != nil {
|
glossary: overrides.GlossaryConfidenceThreshold,
|
||||||
value := *overrides.TargetSections
|
grammar: overrides.GrammarConfidenceThreshold,
|
||||||
c.TargetSections = &value
|
homophones: overrides.HomophonesConfidenceThreshold,
|
||||||
}
|
spokenWord: overrides.SpokenWordConfidenceThreshold,
|
||||||
if overrides.GlossaryConfidenceThreshold != nil {
|
})
|
||||||
c.Thresholds.Glossary = *overrides.GlossaryConfidenceThreshold
|
c.applyNormalizationPatch(normalizationPatch{
|
||||||
}
|
maxSegmentGap: overrides.NormalizeMaxSegmentGap,
|
||||||
if overrides.GrammarConfidenceThreshold != nil {
|
ellipsisGap: overrides.NormalizeEllipsisGap,
|
||||||
c.Thresholds.Grammar = *overrides.GrammarConfidenceThreshold
|
maxSegmentDuration: overrides.NormalizeMaxSegmentDuration,
|
||||||
}
|
maxSegmentTokens: overrides.NormalizeMaxSegmentTokens,
|
||||||
if overrides.HomophonesConfidenceThreshold != nil {
|
})
|
||||||
c.Thresholds.Homophones = *overrides.HomophonesConfidenceThreshold
|
c.applyContextPatch(contextPatch{transcriptDescription: overrides.TranscriptDescription})
|
||||||
}
|
c.applyDiagnosticsPatch(diagnosticsPatch{
|
||||||
if overrides.SpokenWordConfidenceThreshold != nil {
|
workDir: overrides.WorkDir,
|
||||||
c.Thresholds.SpokenWord = *overrides.SpokenWordConfidenceThreshold
|
workDirRetention: overrides.WorkDirRetention,
|
||||||
}
|
})
|
||||||
if overrides.NormalizeMaxSegmentGap != nil {
|
|
||||||
c.Normalization.MaxSegmentGap = *overrides.NormalizeMaxSegmentGap
|
|
||||||
}
|
|
||||||
if overrides.NormalizeEllipsisGap != nil {
|
|
||||||
c.Normalization.EllipsisGap = *overrides.NormalizeEllipsisGap
|
|
||||||
}
|
|
||||||
if overrides.NormalizeMaxSegmentDuration != nil {
|
|
||||||
c.Normalization.MaxSegmentDuration = *overrides.NormalizeMaxSegmentDuration
|
|
||||||
}
|
|
||||||
if overrides.NormalizeMaxSegmentTokens != nil {
|
|
||||||
c.Normalization.MaxSegmentTokens = *overrides.NormalizeMaxSegmentTokens
|
|
||||||
}
|
|
||||||
if overrides.TranscriptDescription != nil {
|
|
||||||
c.TranscriptDescription = strings.TrimSpace(*overrides.TranscriptDescription)
|
|
||||||
}
|
|
||||||
if overrides.WorkDir != nil {
|
|
||||||
c.WorkDir = *overrides.WorkDir
|
|
||||||
}
|
|
||||||
if overrides.WorkDirRetention != nil {
|
|
||||||
c.WorkDirRetention = WorkDirRetention(*overrides.WorkDirRetention)
|
|
||||||
}
|
|
||||||
|
|
||||||
c.syncLegacyConcurrencyAliases()
|
c.syncLegacyConcurrencyAliases()
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
package cli
|
package processreport
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -15,7 +15,12 @@ const (
|
|||||||
correctionDispositionFailed = "failed"
|
correctionDispositionFailed = "failed"
|
||||||
)
|
)
|
||||||
|
|
||||||
type correctionLedgerEntry struct {
|
type CorrectionLedgerInput struct {
|
||||||
|
RunDirectoryPath string
|
||||||
|
RunOutput *runner.RunOutput
|
||||||
|
}
|
||||||
|
|
||||||
|
type CorrectionLedgerEntry struct {
|
||||||
RunID string `json:"run_id,omitempty"`
|
RunID string `json:"run_id,omitempty"`
|
||||||
ModuleKey string `json:"module_key"`
|
ModuleKey string `json:"module_key"`
|
||||||
ModuleInstance string `json:"module_instance"`
|
ModuleInstance string `json:"module_instance"`
|
||||||
@@ -28,27 +33,28 @@ type correctionLedgerEntry struct {
|
|||||||
Disposition string `json:"disposition"`
|
Disposition string `json:"disposition"`
|
||||||
DispositionReasonCode string `json:"disposition_reason_code,omitempty"`
|
DispositionReasonCode string `json:"disposition_reason_code,omitempty"`
|
||||||
DispositionMessage string `json:"disposition_message,omitempty"`
|
DispositionMessage string `json:"disposition_message,omitempty"`
|
||||||
DeterministicValidatorResults []ledgerValidatorDecisionRecord `json:"deterministic_validator_decisions,omitempty"`
|
DeterministicValidatorResults []LedgerValidatorDecisionRecord `json:"deterministic_validator_decisions,omitempty"`
|
||||||
LLMValidatorResults []ledgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
|
LLMValidatorResults []LedgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ledgerValidatorDecisionRecord struct {
|
type LedgerValidatorDecisionRecord struct {
|
||||||
ValidatorKey string `json:"validator_key"`
|
ValidatorKey string `json:"validator_key"`
|
||||||
Approved bool `json:"approved"`
|
Approved bool `json:"approved"`
|
||||||
ReasonCode string `json:"reason_code"`
|
ReasonCode string `json:"reason_code"`
|
||||||
Message string `json:"message,omitempty"`
|
Message string `json:"message,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []correctionLedgerEntry {
|
func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry {
|
||||||
|
runOutput := input.RunOutput
|
||||||
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
runID := ""
|
runID := ""
|
||||||
if runDirPath != "" {
|
if input.RunDirectoryPath != "" {
|
||||||
runID = filepath.Base(runDirPath)
|
runID = filepath.Base(input.RunDirectoryPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
entries := make([]correctionLedgerEntry, 0)
|
entries := make([]CorrectionLedgerEntry, 0)
|
||||||
for _, module := range runOutput.ModuleResults {
|
for _, module := range runOutput.ModuleResults {
|
||||||
decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord)
|
decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord)
|
||||||
for _, decision := range module.ValidatorDecisions {
|
for _, decision := range module.ValidatorDecisions {
|
||||||
@@ -56,7 +62,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, change := range module.AppliedChanges {
|
for _, change := range module.AppliedChanges {
|
||||||
entries = append(entries, correctionLedgerEntry{
|
entries = append(entries, CorrectionLedgerEntry{
|
||||||
RunID: runID,
|
RunID: runID,
|
||||||
ModuleKey: module.ModuleKey,
|
ModuleKey: module.ModuleKey,
|
||||||
ModuleInstance: module.ModuleInstance,
|
ModuleInstance: module.ModuleInstance,
|
||||||
@@ -72,7 +78,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
for _, change := range module.SkippedChanges {
|
for _, change := range module.SkippedChanges {
|
||||||
entries = append(entries, correctionLedgerEntry{
|
entries = append(entries, CorrectionLedgerEntry{
|
||||||
RunID: runID,
|
RunID: runID,
|
||||||
ModuleKey: module.ModuleKey,
|
ModuleKey: module.ModuleKey,
|
||||||
ModuleInstance: module.ModuleInstance,
|
ModuleInstance: module.ModuleInstance,
|
||||||
@@ -89,7 +95,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
for _, rejection := range module.ValidatorRejected {
|
for _, rejection := range module.ValidatorRejected {
|
||||||
entries = append(entries, correctionLedgerEntry{
|
entries = append(entries, CorrectionLedgerEntry{
|
||||||
RunID: runID,
|
RunID: runID,
|
||||||
ModuleKey: module.ModuleKey,
|
ModuleKey: module.ModuleKey,
|
||||||
ModuleInstance: module.ModuleInstance,
|
ModuleInstance: module.ModuleInstance,
|
||||||
@@ -106,7 +112,7 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
if module.Status == runner.ModuleStatusFailed {
|
if module.Status == runner.ModuleStatusFailed {
|
||||||
entries = append(entries, correctionLedgerEntry{
|
entries = append(entries, CorrectionLedgerEntry{
|
||||||
RunID: runID,
|
RunID: runID,
|
||||||
ModuleKey: module.ModuleKey,
|
ModuleKey: module.ModuleKey,
|
||||||
ModuleInstance: module.ModuleInstance,
|
ModuleInstance: module.ModuleInstance,
|
||||||
@@ -130,17 +136,29 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
return entries
|
return entries
|
||||||
}
|
}
|
||||||
|
|
||||||
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []ledgerValidatorDecisionRecord {
|
func HasSkippedCorrections(runOutput *runner.RunOutput) bool {
|
||||||
|
if runOutput == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, mr := range runOutput.ModuleResults {
|
||||||
|
if len(mr.SkippedChanges) > 0 || len(mr.ValidatorRejected) > 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []LedgerValidatorDecisionRecord {
|
||||||
if len(in) == 0 {
|
if len(in) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
out := make([]ledgerValidatorDecisionRecord, 0, len(in))
|
out := make([]LedgerValidatorDecisionRecord, 0, len(in))
|
||||||
for _, decision := range in {
|
for _, decision := range in {
|
||||||
isLLMBacked := validatormetadata.ClassForKey(decision.ValidatorName) == validatormetadata.ExecutionClassLLMBacked
|
isLLMBacked := validatormetadata.ClassForKey(decision.ValidatorName) == validatormetadata.ExecutionClassLLMBacked
|
||||||
if isLLMBacked != wantLLM {
|
if isLLMBacked != wantLLM {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
out = append(out, ledgerValidatorDecisionRecord{
|
out = append(out, LedgerValidatorDecisionRecord{
|
||||||
ValidatorKey: decision.ValidatorName,
|
ValidatorKey: decision.ValidatorName,
|
||||||
Approved: decision.Approved,
|
Approved: decision.Approved,
|
||||||
ReasonCode: decision.ReasonCode,
|
ReasonCode: decision.ReasonCode,
|
||||||
128
internal/framework/processreport/correction_ledger_test.go
Normal file
128
internal/framework/processreport/correction_ledger_test.go
Normal file
@@ -0,0 +1,128 @@
|
|||||||
|
package processreport
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(t *testing.T) {
|
||||||
|
output := &runner.RunOutput{
|
||||||
|
ModuleResults: []runner.ModuleResult{
|
||||||
|
{
|
||||||
|
ModuleKey: "glossary",
|
||||||
|
ModuleInstance: "glossary",
|
||||||
|
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
|
||||||
|
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
||||||
|
{ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
||||||
|
{ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
||||||
|
},
|
||||||
|
AppliedChanges: []proposals.AppliedChange{
|
||||||
|
{
|
||||||
|
ProposalIndex: 3,
|
||||||
|
ModuleKey: "glossary",
|
||||||
|
ModuleInstance: "glossary",
|
||||||
|
TargetSegmentID: 1,
|
||||||
|
OriginalText: "gestures",
|
||||||
|
CorrectedText: "Jesters",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
ledger := BuildCorrectionLedger(CorrectionLedgerInput{
|
||||||
|
RunDirectoryPath: "/tmp/audita-run-id",
|
||||||
|
RunOutput: output,
|
||||||
|
})
|
||||||
|
if len(ledger) != 1 {
|
||||||
|
t.Fatalf("expected one ledger entry, got %d", len(ledger))
|
||||||
|
}
|
||||||
|
entry := ledger[0]
|
||||||
|
if entry.RunID != "audita-run-id" || entry.Disposition != "applied" || entry.AppliedCorrectedText != "Jesters" {
|
||||||
|
t.Fatalf("unexpected applied ledger entry: %+v", entry)
|
||||||
|
}
|
||||||
|
if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "proposal_shape" {
|
||||||
|
t.Fatalf("unexpected deterministic decision split: %+v", entry.DeterministicValidatorResults)
|
||||||
|
}
|
||||||
|
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "spoken_form_plausibility" {
|
||||||
|
t.Fatalf("unexpected llm-backed decision split: %+v", entry.LLMValidatorResults)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildCorrectionLedgerPreservesDispositionPolicy(t *testing.T) {
|
||||||
|
output := &runner.RunOutput{
|
||||||
|
ModuleResults: []runner.ModuleResult{
|
||||||
|
{
|
||||||
|
ModuleKey: "grammar",
|
||||||
|
ModuleInstance: "grammar",
|
||||||
|
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||||
|
Status: runner.ModuleStatusSuccess,
|
||||||
|
SkippedChanges: []proposals.SkippedChange{
|
||||||
|
{
|
||||||
|
ProposalIndex: 2,
|
||||||
|
TargetSegmentID: 7,
|
||||||
|
OriginalText: "old",
|
||||||
|
CorrectedText: "new",
|
||||||
|
SkipReason: proposals.SkipReasonAmbiguousOriginal,
|
||||||
|
Message: "ambiguous",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ValidatorRejected: []runner.ValidatorRejectedChange{
|
||||||
|
{
|
||||||
|
ProposalIndex: 3,
|
||||||
|
TargetSegmentID: 8,
|
||||||
|
OriginalText: "before",
|
||||||
|
CorrectedText: "after",
|
||||||
|
ReasonCode: "protected_term",
|
||||||
|
Message: "blocked",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ModuleKey: "capitalization",
|
||||||
|
ModuleInstance: "capitalization",
|
||||||
|
Status: runner.ModuleStatusFailed,
|
||||||
|
ErrorMessage: "failed",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
|
||||||
|
if len(ledger) != 3 {
|
||||||
|
t.Fatalf("expected skipped, rejected, and failed entries, got %+v", ledger)
|
||||||
|
}
|
||||||
|
|
||||||
|
byDisposition := make(map[string]CorrectionLedgerEntry)
|
||||||
|
for _, entry := range ledger {
|
||||||
|
byDisposition[entry.Disposition] = entry
|
||||||
|
}
|
||||||
|
if byDisposition["skipped"].DispositionReasonCode != string(proposals.SkipReasonAmbiguousOriginal) ||
|
||||||
|
byDisposition["skipped"].DispositionMessage != "ambiguous" {
|
||||||
|
t.Fatalf("unexpected skipped ledger entry: %+v", byDisposition["skipped"])
|
||||||
|
}
|
||||||
|
if byDisposition["rejected"].DispositionReasonCode != "protected_term" ||
|
||||||
|
byDisposition["rejected"].ProposedCorrectedText != "after" {
|
||||||
|
t.Fatalf("unexpected rejected ledger entry: %+v", byDisposition["rejected"])
|
||||||
|
}
|
||||||
|
if byDisposition["failed"].DispositionReasonCode != "module_failed" ||
|
||||||
|
byDisposition["failed"].DispositionMessage != "failed" {
|
||||||
|
t.Fatalf("unexpected failed ledger entry: %+v", byDisposition["failed"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHasSkippedCorrectionsIncludesApplicationSkipsAndValidatorRejections(t *testing.T) {
|
||||||
|
if HasSkippedCorrections(nil) {
|
||||||
|
t.Fatal("nil output should not have skipped corrections")
|
||||||
|
}
|
||||||
|
if HasSkippedCorrections(&runner.RunOutput{ModuleResults: []runner.ModuleResult{{AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 1}}}}}) {
|
||||||
|
t.Fatal("applied-only output should not have skipped corrections")
|
||||||
|
}
|
||||||
|
if !HasSkippedCorrections(&runner.RunOutput{ModuleResults: []runner.ModuleResult{{SkippedChanges: []proposals.SkippedChange{{ProposalIndex: 1}}}}}) {
|
||||||
|
t.Fatal("application skips should count as skipped corrections")
|
||||||
|
}
|
||||||
|
if !HasSkippedCorrections(&runner.RunOutput{ModuleResults: []runner.ModuleResult{{ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 1}}}}}) {
|
||||||
|
t.Fatal("validator rejections should count as skipped corrections")
|
||||||
|
}
|
||||||
|
}
|
||||||
158
internal/framework/processreport/report.go
Normal file
158
internal/framework/processreport/report.go
Normal file
@@ -0,0 +1,158 @@
|
|||||||
|
package processreport
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||||
|
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// BuildInput contains already-computed process execution facts for report assembly.
|
||||||
|
type BuildInput struct {
|
||||||
|
Status string
|
||||||
|
TranscriptPath string
|
||||||
|
GlossaryPath string
|
||||||
|
OutputPath string
|
||||||
|
Modules []string
|
||||||
|
OutputSchema string
|
||||||
|
ConfigVersion *int
|
||||||
|
StartedAt time.Time
|
||||||
|
CompletedAt time.Time
|
||||||
|
ErrorMessage string
|
||||||
|
ErrorPhase string
|
||||||
|
RunDirectoryPath string
|
||||||
|
NormalizationSummary *normalization.NormalizationSummary
|
||||||
|
ChunkingSummary *chunking.Summary
|
||||||
|
RunOutput *runner.RunOutput
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build creates the public process report without owning command parsing or config loading.
|
||||||
|
func Build(input BuildInput) reporting.ProcessReport {
|
||||||
|
report := reporting.ProcessReport{
|
||||||
|
ReportMetadata: reporting.ReportMetadata{
|
||||||
|
ReportSchemaName: reporting.DefaultProcessReportSchemaName,
|
||||||
|
ReportSchemaVersion: reporting.DefaultProcessReportSchemaVersion,
|
||||||
|
OutputSchema: input.OutputSchema,
|
||||||
|
ConfigVersion: input.ConfigVersion,
|
||||||
|
},
|
||||||
|
Phase: "default_pipeline",
|
||||||
|
Status: input.Status,
|
||||||
|
Operation: "process",
|
||||||
|
TranscriptPath: input.TranscriptPath,
|
||||||
|
GlossaryPath: input.GlossaryPath,
|
||||||
|
OutputPath: input.OutputPath,
|
||||||
|
Modules: append([]string(nil), input.Modules...),
|
||||||
|
StartedAt: input.StartedAt,
|
||||||
|
CompletedAt: &input.CompletedAt,
|
||||||
|
ErrorPhase: input.ErrorPhase,
|
||||||
|
}
|
||||||
|
if input.RunDirectoryPath != "" {
|
||||||
|
runSucceeded := input.Status == "success"
|
||||||
|
metadata := diagnostics.BuildDiagnosticsMetadata(input.RunDirectoryPath, runSucceeded)
|
||||||
|
report.Diagnostics = &metadata
|
||||||
|
}
|
||||||
|
if input.ErrorMessage != "" {
|
||||||
|
report.ErrorMessage = input.ErrorMessage
|
||||||
|
}
|
||||||
|
if input.NormalizationSummary != nil {
|
||||||
|
report.InputSegmentCount = &input.NormalizationSummary.InputSegmentCount
|
||||||
|
report.NormalizedSegmentCount = &input.NormalizationSummary.OutputSegmentCount
|
||||||
|
report.NormalizationMerges = &input.NormalizationSummary.MergesPerformed
|
||||||
|
report.NormalizationIDReassignments = &input.NormalizationSummary.IDsReassigned
|
||||||
|
report.NormalizationSkipped.DifferentSpeakers = &input.NormalizationSummary.SkippedMerges.DifferentSpeakers
|
||||||
|
report.NormalizationSkipped.GapTooLarge = &input.NormalizationSummary.SkippedMerges.GapTooLarge
|
||||||
|
report.NormalizationSkipped.DurationExceeded = &input.NormalizationSummary.SkippedMerges.DurationExceeded
|
||||||
|
report.NormalizationSkipped.TokenLimitExceeded = &input.NormalizationSummary.SkippedMerges.TokenLimitExceeded
|
||||||
|
}
|
||||||
|
if input.ChunkingSummary != nil {
|
||||||
|
report.Chunking = &reporting.ChunkingSummary{
|
||||||
|
ChunkCount: input.ChunkingSummary.ChunkCount,
|
||||||
|
MinEstimatedTokens: input.ChunkingSummary.MinEstimatedTokens,
|
||||||
|
MaxEstimatedTokens: input.ChunkingSummary.MaxEstimatedTokens,
|
||||||
|
TotalEstimatedTokens: input.ChunkingSummary.TotalEstimatedTokens,
|
||||||
|
TargetSections: input.ChunkingSummary.TargetSections,
|
||||||
|
MaxSectionTokens: input.ChunkingSummary.MaxSectionTokens,
|
||||||
|
MinSectionTokens: input.ChunkingSummary.MinSectionTokens,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
report.ModulesSummary, report.ModuleResults = buildModuleReporting(input.RunOutput)
|
||||||
|
return report
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummary, []reporting.ModuleReport) {
|
||||||
|
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
moduleReports := make([]reporting.ModuleReport, 0, len(runOutput.ModuleResults))
|
||||||
|
summary := &reporting.ModulesSummary{ModuleCount: len(runOutput.ModuleResults)}
|
||||||
|
for _, r := range runOutput.ModuleResults {
|
||||||
|
startedAt := r.StartedAt
|
||||||
|
completedAt := r.CompletedAt
|
||||||
|
moduleReports = append(moduleReports, reporting.ModuleReport{
|
||||||
|
ModuleKey: r.ModuleKey,
|
||||||
|
ModuleInstance: r.ModuleInstance,
|
||||||
|
ReplacementPolicy: string(r.ReplacementPolicy),
|
||||||
|
Status: r.Status,
|
||||||
|
ProposalCount: r.ProposalCount,
|
||||||
|
Warnings: append([]stagewarnings.StageWarning(nil), r.Warnings...),
|
||||||
|
ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions),
|
||||||
|
ValidatorRejected: mapValidatorRejected(r.ValidatorRejected),
|
||||||
|
AppliedChanges: r.AppliedChanges,
|
||||||
|
SkippedChanges: r.SkippedChanges,
|
||||||
|
ErrorMessage: r.ErrorMessage,
|
||||||
|
StartedAt: &startedAt,
|
||||||
|
CompletedAt: &completedAt,
|
||||||
|
})
|
||||||
|
summary.TotalAppliedChanges += len(r.AppliedChanges)
|
||||||
|
summary.TotalSkippedChanges += len(r.SkippedChanges) + len(r.ValidatorRejected)
|
||||||
|
if r.Status == runner.ModuleStatusFailed && summary.FailedModuleInstance == "" {
|
||||||
|
summary.FailedModuleInstance = r.ModuleInstance
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return summary, moduleReports
|
||||||
|
}
|
||||||
|
|
||||||
|
func mapValidatorDecisions(in []runner.ValidatorDecisionRecord) []reporting.ValidatorDecisionReport {
|
||||||
|
if len(in) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]reporting.ValidatorDecisionReport, len(in))
|
||||||
|
for i, d := range in {
|
||||||
|
out[i] = reporting.ValidatorDecisionReport{
|
||||||
|
ValidatorName: d.ValidatorName,
|
||||||
|
ProposalIndex: d.ProposalIndex,
|
||||||
|
Approved: d.Approved,
|
||||||
|
ReasonCode: d.ReasonCode,
|
||||||
|
Message: d.Message,
|
||||||
|
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func mapValidatorRejected(in []runner.ValidatorRejectedChange) []reporting.ValidatorRejectedReport {
|
||||||
|
if len(in) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]reporting.ValidatorRejectedReport, len(in))
|
||||||
|
for i, d := range in {
|
||||||
|
out[i] = reporting.ValidatorRejectedReport{
|
||||||
|
ValidatorName: d.ValidatorName,
|
||||||
|
ProposalIndex: d.ProposalIndex,
|
||||||
|
ModuleKey: d.ModuleKey,
|
||||||
|
ModuleInstance: d.ModuleInstance,
|
||||||
|
TargetSegmentID: d.TargetSegmentID,
|
||||||
|
OriginalText: d.OriginalText,
|
||||||
|
CorrectedText: d.CorrectedText,
|
||||||
|
ReasonCode: d.ReasonCode,
|
||||||
|
Message: d.Message,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
180
internal/framework/processreport/report_test.go
Normal file
180
internal/framework/processreport/report_test.go
Normal file
@@ -0,0 +1,180 @@
|
|||||||
|
package processreport
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBuildSuccessReportMapsExecutionFacts(t *testing.T) {
|
||||||
|
startedAt := time.Date(2026, 5, 23, 10, 0, 0, 0, time.UTC)
|
||||||
|
completedAt := startedAt.Add(time.Second)
|
||||||
|
configVersion := 4
|
||||||
|
targetSections := 2
|
||||||
|
inputSegments := 5
|
||||||
|
outputSegments := 4
|
||||||
|
merges := 1
|
||||||
|
reassigned := 2
|
||||||
|
differentSpeakers := 3
|
||||||
|
|
||||||
|
report := Build(BuildInput{
|
||||||
|
Status: "success",
|
||||||
|
TranscriptPath: "transcript.json",
|
||||||
|
GlossaryPath: "glossary.yaml",
|
||||||
|
OutputPath: "out.json",
|
||||||
|
Modules: []string{"grammar"},
|
||||||
|
OutputSchema: "default",
|
||||||
|
ConfigVersion: &configVersion,
|
||||||
|
StartedAt: startedAt,
|
||||||
|
CompletedAt: completedAt,
|
||||||
|
RunDirectoryPath: filepath.Join(
|
||||||
|
"tmp",
|
||||||
|
"audita-run",
|
||||||
|
),
|
||||||
|
NormalizationSummary: &normalization.NormalizationSummary{
|
||||||
|
InputSegmentCount: inputSegments,
|
||||||
|
OutputSegmentCount: outputSegments,
|
||||||
|
MergesPerformed: merges,
|
||||||
|
IDsReassigned: reassigned,
|
||||||
|
SkippedMerges: struct {
|
||||||
|
DifferentSpeakers int `json:"different_speakers"`
|
||||||
|
GapTooLarge int `json:"gap_too_large"`
|
||||||
|
DurationExceeded int `json:"duration_exceeded"`
|
||||||
|
TokenLimitExceeded int `json:"token_limit_exceeded"`
|
||||||
|
}{
|
||||||
|
DifferentSpeakers: differentSpeakers,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ChunkingSummary: &chunking.Summary{
|
||||||
|
ChunkCount: 3,
|
||||||
|
MinEstimatedTokens: 10,
|
||||||
|
MaxEstimatedTokens: 20,
|
||||||
|
TotalEstimatedTokens: 45,
|
||||||
|
TargetSections: &targetSections,
|
||||||
|
MaxSectionTokens: 200,
|
||||||
|
MinSectionTokens: 50,
|
||||||
|
},
|
||||||
|
RunOutput: &runner.RunOutput{
|
||||||
|
ModuleResults: []runner.ModuleResult{
|
||||||
|
{
|
||||||
|
ModuleKey: "grammar",
|
||||||
|
ModuleInstance: "grammar",
|
||||||
|
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||||
|
Status: runner.ModuleStatusSuccess,
|
||||||
|
ProposalCount: 2,
|
||||||
|
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
||||||
|
{
|
||||||
|
ValidatorName: "proposal_shape",
|
||||||
|
ProposalIndex: 1,
|
||||||
|
Approved: true,
|
||||||
|
ReasonCode: "approved",
|
||||||
|
Message: "ok",
|
||||||
|
DiagnosticArtifactPath: "diagnostics/validator.json",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ValidatorRejected: []runner.ValidatorRejectedChange{
|
||||||
|
{
|
||||||
|
ValidatorName: "protected_term",
|
||||||
|
ProposalIndex: 2,
|
||||||
|
ModuleKey: "grammar",
|
||||||
|
ModuleInstance: "grammar",
|
||||||
|
TargetSegmentID: 7,
|
||||||
|
OriginalText: "old",
|
||||||
|
CorrectedText: "new",
|
||||||
|
ReasonCode: "protected_term",
|
||||||
|
Message: "blocked",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
AppliedChanges: []proposals.AppliedChange{
|
||||||
|
{ProposalIndex: 1, TargetSegmentID: 7, OriginalText: "old", CorrectedText: "new"},
|
||||||
|
},
|
||||||
|
SkippedChanges: []proposals.SkippedChange{
|
||||||
|
{ProposalIndex: 3, TargetSegmentID: 8, SkipReason: proposals.SkipReasonMissingSegment},
|
||||||
|
},
|
||||||
|
StartedAt: startedAt,
|
||||||
|
CompletedAt: completedAt,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
if report.ReportMetadata.ReportSchemaName != reporting.DefaultProcessReportSchemaName ||
|
||||||
|
report.ReportMetadata.ReportSchemaVersion != reporting.DefaultProcessReportSchemaVersion ||
|
||||||
|
report.ReportMetadata.OutputSchema != "default" ||
|
||||||
|
report.ReportMetadata.ConfigVersion == nil ||
|
||||||
|
*report.ReportMetadata.ConfigVersion != configVersion {
|
||||||
|
t.Fatalf("unexpected report metadata: %+v", report.ReportMetadata)
|
||||||
|
}
|
||||||
|
if report.Phase != "default_pipeline" || report.Operation != "process" || report.Status != "success" {
|
||||||
|
t.Fatalf("unexpected process identity fields: phase=%q operation=%q status=%q", report.Phase, report.Operation, report.Status)
|
||||||
|
}
|
||||||
|
if report.Diagnostics == nil || report.Diagnostics.CorrectionLedgerPath != filepath.Join("tmp", "audita-run", diagnostics.ArtifactCorrectionLedger) {
|
||||||
|
t.Fatalf("unexpected diagnostics metadata: %+v", report.Diagnostics)
|
||||||
|
}
|
||||||
|
if report.InputSegmentCount == nil || *report.InputSegmentCount != inputSegments ||
|
||||||
|
report.NormalizationSkipped.DifferentSpeakers == nil ||
|
||||||
|
*report.NormalizationSkipped.DifferentSpeakers != differentSpeakers {
|
||||||
|
t.Fatalf("unexpected normalization summary: %+v", report)
|
||||||
|
}
|
||||||
|
if report.Chunking == nil || report.Chunking.ChunkCount != 3 || report.Chunking.TargetSections == nil || *report.Chunking.TargetSections != targetSections {
|
||||||
|
t.Fatalf("unexpected chunking summary: %+v", report.Chunking)
|
||||||
|
}
|
||||||
|
if report.ModulesSummary == nil ||
|
||||||
|
report.ModulesSummary.ModuleCount != 1 ||
|
||||||
|
report.ModulesSummary.TotalAppliedChanges != 1 ||
|
||||||
|
report.ModulesSummary.TotalSkippedChanges != 2 {
|
||||||
|
t.Fatalf("unexpected modules summary: %+v", report.ModulesSummary)
|
||||||
|
}
|
||||||
|
if len(report.ModuleResults) != 1 ||
|
||||||
|
len(report.ModuleResults[0].ValidatorDecisions) != 1 ||
|
||||||
|
len(report.ModuleResults[0].ValidatorRejected) != 1 {
|
||||||
|
t.Fatalf("unexpected module reports: %+v", report.ModuleResults)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildFailedReportPreservesErrorAndFailureSummary(t *testing.T) {
|
||||||
|
startedAt := time.Date(2026, 5, 23, 10, 0, 0, 0, time.UTC)
|
||||||
|
completedAt := startedAt.Add(time.Second)
|
||||||
|
|
||||||
|
report := Build(BuildInput{
|
||||||
|
Status: "failed",
|
||||||
|
TranscriptPath: "transcript.json",
|
||||||
|
GlossaryPath: "glossary.yaml",
|
||||||
|
Modules: []string{"grammar"},
|
||||||
|
OutputSchema: "default",
|
||||||
|
StartedAt: startedAt,
|
||||||
|
CompletedAt: completedAt,
|
||||||
|
ErrorPhase: "module",
|
||||||
|
ErrorMessage: "module failed",
|
||||||
|
RunDirectoryPath: "run-dir",
|
||||||
|
RunOutput: &runner.RunOutput{
|
||||||
|
ModuleResults: []runner.ModuleResult{
|
||||||
|
{
|
||||||
|
ModuleKey: "grammar",
|
||||||
|
ModuleInstance: "grammar",
|
||||||
|
Status: runner.ModuleStatusFailed,
|
||||||
|
ErrorMessage: "module failed",
|
||||||
|
StartedAt: startedAt,
|
||||||
|
CompletedAt: completedAt,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
if report.Status != "failed" || report.ErrorPhase != "module" || report.ErrorMessage != "module failed" {
|
||||||
|
t.Fatalf("unexpected failure fields: %+v", report)
|
||||||
|
}
|
||||||
|
if report.Diagnostics == nil || report.Diagnostics.ErrorLogPath == "" {
|
||||||
|
t.Fatalf("expected failure diagnostics metadata, got %+v", report.Diagnostics)
|
||||||
|
}
|
||||||
|
if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != "grammar" {
|
||||||
|
t.Fatalf("unexpected failed module summary: %+v", report.ModulesSummary)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": [
|
||||||
|
"corrections"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"corrections": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": [
|
||||||
|
"id",
|
||||||
|
"original_text",
|
||||||
|
"corrected_text",
|
||||||
|
"confidence"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"id": {
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 1
|
||||||
|
},
|
||||||
|
"original_text": {
|
||||||
|
"type": "string",
|
||||||
|
"minLength": 1
|
||||||
|
},
|
||||||
|
"corrected_text": {
|
||||||
|
"type": "string",
|
||||||
|
"minLength": 1
|
||||||
|
},
|
||||||
|
"confidence": {
|
||||||
|
"type": "number",
|
||||||
|
"minimum": 0,
|
||||||
|
"maximum": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": [
|
||||||
|
"validations"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"validations": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": [
|
||||||
|
"correction_index",
|
||||||
|
"approved",
|
||||||
|
"confidence",
|
||||||
|
"reason"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"correction_index": {
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 0
|
||||||
|
},
|
||||||
|
"approved": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"confidence": {
|
||||||
|
"type": "number",
|
||||||
|
"minimum": 0,
|
||||||
|
"maximum": 1
|
||||||
|
},
|
||||||
|
"reason": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@ package responseschema
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
|
"embed"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -20,6 +21,9 @@ const (
|
|||||||
schemaVersionV1 = "v1"
|
schemaVersionV1 = "v1"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
//go:embed assets/*.json
|
||||||
|
var schemaAssets embed.FS
|
||||||
|
|
||||||
// Schema describes one registered structured response schema.
|
// Schema describes one registered structured response schema.
|
||||||
type Schema struct {
|
type Schema struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
@@ -39,17 +43,17 @@ func (s Schema) DiagnosticsMap() map[string]any {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var registry = map[Key]Schema{
|
var registry = map[Key]Schema{
|
||||||
CorrectionSetKey: mustBuildSchema(
|
CorrectionSetKey: mustBuildSchemaFromAsset(
|
||||||
correctionSetSchemaID,
|
correctionSetSchemaID,
|
||||||
schemaVersionV1,
|
schemaVersionV1,
|
||||||
"audita_correction_set_v1",
|
"audita_correction_set_v1",
|
||||||
[]byte(`{"type":"object","additionalProperties":false,"required":["corrections"],"properties":{"corrections":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["id","original_text","corrected_text","confidence"],"properties":{"id":{"type":"integer","minimum":1},"original_text":{"type":"string","minLength":1},"corrected_text":{"type":"string","minLength":1},"confidence":{"type":"number","minimum":0,"maximum":1}}}}}}`),
|
"assets/correction_set.v1.json",
|
||||||
),
|
),
|
||||||
ValidatorDecisionSetKey: mustBuildSchema(
|
ValidatorDecisionSetKey: mustBuildSchemaFromAsset(
|
||||||
validatorDecisionSchemaID,
|
validatorDecisionSchemaID,
|
||||||
schemaVersionV1,
|
schemaVersionV1,
|
||||||
"audita_validator_decision_set_v1",
|
"audita_validator_decision_set_v1",
|
||||||
[]byte(`{"type":"object","additionalProperties":false,"required":["validations"],"properties":{"validations":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["correction_index","approved","confidence","reason"],"properties":{"correction_index":{"type":"integer","minimum":0},"approved":{"type":"boolean"},"confidence":{"type":"number","minimum":0,"maximum":1},"reason":{"type":"string"}}}}}}`),
|
"assets/validator_decision_set.v1.json",
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -93,6 +97,14 @@ func cloneSchema(in Schema) Schema {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func mustBuildSchemaFromAsset(id string, version string, name string, assetPath string) Schema {
|
||||||
|
rawSchema, err := schemaAssets.ReadFile(assetPath)
|
||||||
|
if err != nil {
|
||||||
|
panic(fmt.Sprintf("read response schema asset %q: %v", assetPath, err))
|
||||||
|
}
|
||||||
|
return mustBuildSchema(id, version, name, rawSchema)
|
||||||
|
}
|
||||||
|
|
||||||
func mustBuildSchema(id string, version string, name string, rawSchema []byte) Schema {
|
func mustBuildSchema(id string, version string, name string, rawSchema []byte) Schema {
|
||||||
id = strings.TrimSpace(id)
|
id = strings.TrimSpace(id)
|
||||||
version = strings.TrimSpace(version)
|
version = strings.TrimSpace(version)
|
||||||
|
|||||||
@@ -1,14 +1,16 @@
|
|||||||
package responseschema
|
package responseschema
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
expectedCorrectionSetSHA256 = "05f8ff3fa04f68115c0cb1859d2656f51aa5c0bae8ff2470b2d4f6f531953195"
|
expectedCorrectionSetSHA256 = "b86a2dde38d7f440d26470fa8830167512bb0aa1b35e5fee5547057be583c388"
|
||||||
expectedValidatorDecisionSetSHA256 = "b73f4790b98fbb955f0aec5496dd8ce9a8fe14aa2f35c700b4b4e5634f106fd5"
|
expectedValidatorDecisionSetSHA256 = "2fe90d450e2595b57885aba91c8cc5cedf783dd36758ab430309e3eace401f54"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestLookupKnownSchemas(t *testing.T) {
|
func TestLookupKnownSchemas(t *testing.T) {
|
||||||
@@ -47,6 +49,20 @@ func TestLookupUnknownSchema(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRegisteredSchemasLoadReadableEmbeddedJSON(t *testing.T) {
|
||||||
|
for _, schema := range Registered() {
|
||||||
|
if len(schema.JSONSchema) == 0 {
|
||||||
|
t.Fatalf("expected non-empty JSON schema for %q", schema.ID)
|
||||||
|
}
|
||||||
|
if !json.Valid(schema.JSONSchema) {
|
||||||
|
t.Fatalf("expected valid JSON schema for %q", schema.ID)
|
||||||
|
}
|
||||||
|
if !bytes.Contains(schema.JSONSchema, []byte("\n ")) {
|
||||||
|
t.Fatalf("expected readable formatted JSON schema for %q", schema.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSchemaHashesMatchRegisteredJSON(t *testing.T) {
|
func TestSchemaHashesMatchRegisteredJSON(t *testing.T) {
|
||||||
expectedByKey := map[Key]string{
|
expectedByKey := map[Key]string{
|
||||||
CorrectionSetKey: expectedCorrectionSetSHA256,
|
CorrectionSetKey: expectedCorrectionSetSHA256,
|
||||||
|
|||||||
17
scripts/build-release.sh
Executable file
17
scripts/build-release.sh
Executable file
@@ -0,0 +1,17 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
if [[ $# != 2 || ! $1 =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then
|
||||||
|
echo 'Usage: scripts/build-release.sh vMAJOR.MINOR.PATCH OUTPUT_DIRECTORY' >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
version=$1
|
||||||
|
output=$2
|
||||||
|
mkdir -p "$output"
|
||||||
|
output=$(cd "$output" && pwd)
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
for arch in amd64 arm64; do
|
||||||
|
CGO_ENABLED=0 GOOS=linux GOARCH=$arch GOWORK=off \
|
||||||
|
go build -mod=readonly -trimpath -ldflags '-s -w' \
|
||||||
|
-o "$output/audita-$version-linux-$arch" ./cmd/audita
|
||||||
|
done
|
||||||
48
scripts/release.sh
Executable file
48
scripts/release.sh
Executable file
@@ -0,0 +1,48 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
fail() { echo "Release stopped: $*" >&2; exit 1; }
|
||||||
|
if [[ $# != 2 || ( $1 != --check && $1 != --publish ) || ! $2 =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then
|
||||||
|
echo 'Usage: scripts/release.sh --check|--publish vMAJOR.MINOR.PATCH' >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
mode=$1
|
||||||
|
version=$2
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
[[ $(git symbolic-ref --quiet --short HEAD) == main ]] || fail 'switch to main'
|
||||||
|
[[ -z $(git status --porcelain) ]] || fail 'commit or remove outstanding changes first'
|
||||||
|
candidate=$(git rev-parse HEAD)
|
||||||
|
note="docs/releases/$version.md"
|
||||||
|
[[ -s $note ]] || fail "missing release note: $note"
|
||||||
|
IFS= read -r title < "$note" || true
|
||||||
|
[[ $title == "# Audita $version" ]] || fail "note must begin with # Audita $version"
|
||||||
|
git show-ref --verify --quiet "refs/tags/$version" && fail 'version already exists locally'
|
||||||
|
|
||||||
|
# Check mode is local and never contacts a remote or creates a tag.
|
||||||
|
if [[ $mode == --publish ]]; then
|
||||||
|
existing=$(git ls-remote --tags origin "refs/tags/$version")
|
||||||
|
[[ -z $existing ]] || fail 'version already exists on origin'
|
||||||
|
fi
|
||||||
|
build_dir=$(mktemp -d)
|
||||||
|
trap 'rm -rf "$build_dir"' EXIT
|
||||||
|
GOWORK=off go test -mod=readonly ./...
|
||||||
|
bash scripts/build-release.sh "$version" "$build_dir"
|
||||||
|
[[ $(git rev-parse HEAD) == "$candidate" ]] || fail 'HEAD changed during validation'
|
||||||
|
[[ -z $(git status --porcelain) ]] || fail 'worktree changed during validation'
|
||||||
|
[[ $(git symbolic-ref --quiet --short HEAD) == main ]] || fail 'branch changed during validation'
|
||||||
|
if [[ $mode == --check ]]; then
|
||||||
|
echo "Validated $version at $candidate; nothing published."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
# Fetch the authoritative branch before checking that this is a forward update.
|
||||||
|
git fetch --no-tags origin refs/heads/main
|
||||||
|
git merge-base --is-ancestor FETCH_HEAD "$candidate" || fail 'candidate does not include origin/main'
|
||||||
|
existing=$(git ls-remote --tags origin "refs/tags/$version")
|
||||||
|
[[ -z $existing ]] || fail 'version already exists on origin'
|
||||||
|
git tag "$version" "$candidate"
|
||||||
|
[[ $(git rev-parse "refs/tags/$version") == "$candidate" ]] || fail 'tag target changed'
|
||||||
|
# Atomic publication prevents a rejected tag/branch update from publishing only half.
|
||||||
|
if ! git push --atomic origin "$candidate:refs/heads/main" "refs/tags/$version:refs/tags/$version"; then
|
||||||
|
fail "push failed or its outcome is uncertain; inspect origin/main and refs/tags/$version before retrying (local tag retained)"
|
||||||
|
fi
|
||||||
|
echo "Released $version at $candidate. Binary publication runs asynchronously; CI is not awaited."
|
||||||
142
scripts/release_test.go
Normal file
142
scripts/release_test.go
Normal file
@@ -0,0 +1,142 @@
|
|||||||
|
package scripts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Real local Git repositories exercise publication; Go is stubbed at the slow
|
||||||
|
// build/test boundary so these tests remain offline and cannot recurse.
|
||||||
|
func TestReleaseWorkflow(t *testing.T) {
|
||||||
|
for _, scenario := range []string{"check", "publish", "dirty", "tests-fail", "build-fail", "existing-remote-tag", "remote-ahead", "rejected-push", "source-changed", "missing-note", "invalid-version", "existing-local-tag", "wrong-branch", "head-changed"} {
|
||||||
|
t.Run(scenario, func(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
repo, remote := filepath.Join(root, "repo"), filepath.Join(root, "remote.git")
|
||||||
|
if err := os.MkdirAll(repo, 0755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
env := append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL="+os.DevNull, "GIT_AUTHOR_NAME=Release Test", "GIT_AUTHOR_EMAIL=test@example.invalid", "GIT_COMMITTER_NAME=Release Test", "GIT_COMMITTER_EMAIL=test@example.invalid", "GIT_TERMINAL_PROMPT=0")
|
||||||
|
run := func(dir string, args ...string) string {
|
||||||
|
t.Helper()
|
||||||
|
c := exec.Command("git", args...)
|
||||||
|
c.Dir = dir
|
||||||
|
c.Env = env
|
||||||
|
b, e := c.CombinedOutput()
|
||||||
|
if e != nil {
|
||||||
|
t.Fatalf("git %v: %v\n%s", args, e, b)
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(string(b))
|
||||||
|
}
|
||||||
|
write := func(path, data string, mode os.FileMode) {
|
||||||
|
t.Helper()
|
||||||
|
if e := os.MkdirAll(filepath.Dir(path), 0755); e != nil {
|
||||||
|
t.Fatal(e)
|
||||||
|
}
|
||||||
|
if e := os.WriteFile(path, []byte(data), mode); e != nil {
|
||||||
|
t.Fatal(e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
run(root, "init", "--bare", "--initial-branch=main", remote)
|
||||||
|
run(repo, "init", "--initial-branch=main")
|
||||||
|
for _, name := range []string{"release.sh", "build-release.sh"} {
|
||||||
|
b, e := os.ReadFile(name)
|
||||||
|
if e != nil {
|
||||||
|
t.Fatal(e)
|
||||||
|
}
|
||||||
|
write(filepath.Join(repo, "scripts", name), string(b), 0755)
|
||||||
|
}
|
||||||
|
write(filepath.Join(repo, "docs/releases/v1.2.3.md"), "# Audita v1.2.3\n\nA test release.\n", 0644)
|
||||||
|
run(repo, "add", ".")
|
||||||
|
run(repo, "commit", "-m", "Initial fixture")
|
||||||
|
run(repo, "remote", "add", "origin", remote)
|
||||||
|
run(repo, "push", "origin", "main")
|
||||||
|
write(filepath.Join(repo, "change.txt"), "candidate\n", 0644)
|
||||||
|
run(repo, "add", ".")
|
||||||
|
run(repo, "commit", "-m", "Candidate fixture")
|
||||||
|
candidate := run(repo, "rev-parse", "HEAD")
|
||||||
|
log := filepath.Join(root, "go.log")
|
||||||
|
stub := "#!/bin/sh\necho \"$* GOOS=$GOOS GOARCH=$GOARCH CGO_ENABLED=$CGO_ENABLED GOWORK=$GOWORK\" >> \"$GO_LOG\"\n"
|
||||||
|
switch scenario {
|
||||||
|
case "existing-local-tag":
|
||||||
|
run(repo, "tag", "v1.2.3")
|
||||||
|
case "wrong-branch":
|
||||||
|
run(repo, "checkout", "-b", "feature")
|
||||||
|
case "head-changed":
|
||||||
|
stub += "if [ \"$1\" = test ]; then echo changed > change.txt; git add change.txt; git commit -m Changed; fi\n"
|
||||||
|
case "dirty":
|
||||||
|
write(filepath.Join(repo, "untracked"), "dirty", 0644)
|
||||||
|
case "tests-fail":
|
||||||
|
stub += "[ \"$1\" != test ] || exit 1\n"
|
||||||
|
case "build-fail":
|
||||||
|
stub += "[ \"$GOARCH\" != arm64 ] || exit 1\n"
|
||||||
|
case "existing-remote-tag":
|
||||||
|
run(remote, "tag", "v1.2.3", "main")
|
||||||
|
case "remote-ahead":
|
||||||
|
other := filepath.Join(root, "other")
|
||||||
|
run(root, "clone", remote, other)
|
||||||
|
write(filepath.Join(other, "other.txt"), "new remote commit", 0644)
|
||||||
|
run(other, "add", ".")
|
||||||
|
run(other, "commit", "-m", "Remote fixture")
|
||||||
|
run(other, "push", "origin", "main")
|
||||||
|
case "rejected-push":
|
||||||
|
write(filepath.Join(remote, "hooks/pre-receive"), "#!/bin/sh\nexit 1\n", 0755)
|
||||||
|
case "source-changed":
|
||||||
|
stub += "[ \"$1\" != test ] || echo changed > change.txt\n"
|
||||||
|
case "missing-note":
|
||||||
|
run(repo, "rm", "docs/releases/v1.2.3.md")
|
||||||
|
run(repo, "commit", "-m", "Remove fixture note")
|
||||||
|
}
|
||||||
|
stub += "exit 0\n"
|
||||||
|
write(filepath.Join(root, "bin/go"), stub, 0755)
|
||||||
|
mode, version := "--publish", "v1.2.3"
|
||||||
|
if scenario == "check" {
|
||||||
|
mode = "--check"
|
||||||
|
run(repo, "remote", "remove", "origin")
|
||||||
|
}
|
||||||
|
if scenario == "invalid-version" {
|
||||||
|
version = "v01.2.3"
|
||||||
|
}
|
||||||
|
c := exec.Command("bash", "scripts/release.sh", mode, version)
|
||||||
|
c.Dir = repo
|
||||||
|
c.Env = append(env, "PATH="+filepath.Join(root, "bin")+":"+os.Getenv("PATH"), "GO_LOG="+log)
|
||||||
|
b, e := c.CombinedOutput()
|
||||||
|
success := scenario == "check" || scenario == "publish"
|
||||||
|
if (e == nil) != success {
|
||||||
|
t.Fatalf("success=%v: %v\n%s", success, e, b)
|
||||||
|
}
|
||||||
|
tags := run(remote, "tag", "--list")
|
||||||
|
if scenario == "publish" {
|
||||||
|
if run(remote, "rev-parse", "refs/tags/v1.2.3") != candidate || run(remote, "rev-parse", "main") != candidate {
|
||||||
|
t.Fatal("wrong source published")
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(b), "CI is not awaited") {
|
||||||
|
t.Fatal("missing asynchronous completion message")
|
||||||
|
}
|
||||||
|
} else if scenario != "existing-remote-tag" && tags != "" {
|
||||||
|
t.Fatalf("unexpected remote tags: %s", tags)
|
||||||
|
}
|
||||||
|
if scenario == "rejected-push" && run(remote, "rev-parse", "main") == candidate {
|
||||||
|
t.Fatal("rejected push published candidate")
|
||||||
|
}
|
||||||
|
if scenario == "check" && run(repo, "tag", "--list") != "" {
|
||||||
|
t.Fatal("check created a tag")
|
||||||
|
}
|
||||||
|
if success {
|
||||||
|
calls, e := os.ReadFile(log)
|
||||||
|
if e != nil {
|
||||||
|
t.Fatal(e)
|
||||||
|
}
|
||||||
|
lines := strings.Split(strings.TrimSpace(string(calls)), "\n")
|
||||||
|
if len(lines) != 3 {
|
||||||
|
t.Fatalf("expected one test and two builds: %s", calls)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(lines[0], "test -mod=readonly ./...") || !strings.Contains(lines[1], "GOOS=linux GOARCH=amd64 CGO_ENABLED=0 GOWORK=off") || !strings.Contains(lines[2], "GOOS=linux GOARCH=arm64 CGO_ENABLED=0 GOWORK=off") {
|
||||||
|
t.Fatalf("unexpected validation: %s", calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user