Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c5627460d0 | |||
| 3c68ec9d05 | |||
| e8b8fbd509 | |||
| 588ca36dfc |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -12,7 +12,6 @@ build/
|
||||
|
||||
# ---> Codex
|
||||
.codex
|
||||
AGENTS.md
|
||||
|
||||
# ---> Go
|
||||
# If you prefer the allow list template instead of the deny list, see community template:
|
||||
|
||||
@@ -5,33 +5,7 @@ steps:
|
||||
- name: build-release-assets
|
||||
image: golang:1.25
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
|
||||
version="$CI_COMMIT_TAG"
|
||||
dist="dist"
|
||||
pkg="gitea.maximumdirect.net/eric/audita/cmd/audita"
|
||||
|
||||
rm -rf "$dist"
|
||||
mkdir -p "$dist"
|
||||
|
||||
build_binary() {
|
||||
goos="$1"
|
||||
goarch="$2"
|
||||
suffix="$3"
|
||||
output="$dist/audita-$version-$goos-$goarch$suffix"
|
||||
|
||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \
|
||||
go build -trimpath -ldflags "-s -w -X gitea.maximumdirect.net/eric/audita/internal/buildinfo.Version=$version" \
|
||||
-o "$output" "$pkg"
|
||||
}
|
||||
|
||||
build_binary linux amd64 ""
|
||||
build_binary linux arm64 ""
|
||||
build_binary darwin amd64 ""
|
||||
build_binary darwin arm64 ""
|
||||
build_binary windows amd64 ".exe"
|
||||
build_binary windows arm64 ".exe"
|
||||
- bash scripts/build-release.sh "$CI_COMMIT_TAG" dist
|
||||
|
||||
- name: publish-release
|
||||
image: woodpeckerci/plugin-release
|
||||
@@ -41,7 +15,8 @@ steps:
|
||||
api_key:
|
||||
from_secret: GITEA_RELEASE_TOKEN
|
||||
files:
|
||||
- dist/audita-*
|
||||
- dist/audita-*-linux-amd64
|
||||
- dist/audita-*-linux-arm64
|
||||
checksum: sha256
|
||||
checksum-file: SHA256SUMS
|
||||
checksum-flatten: true
|
||||
|
||||
4
AGENTS.md
Normal file
4
AGENTS.md
Normal file
@@ -0,0 +1,4 @@
|
||||
Please review `docs/development.md` for initial orientation in this repository
|
||||
and follow its task-specific reading guide.
|
||||
|
||||
Review and follow `docs/policy/agents.md` before recommending, planning, or implementing any changes to this repository.
|
||||
56
README.md
56
README.md
@@ -1,46 +1,44 @@
|
||||
# Audita
|
||||
|
||||
Audita is a CLI that polishes transcript JSON using glossary-aware and LLM-backed correction modules.
|
||||
Audita polishes transcript JSON using glossary-aware and LLM-backed correction
|
||||
modules. It validates proposed corrections and produces corrected text plus
|
||||
reports for review and subprocess integration.
|
||||
|
||||
## Quickstart
|
||||
|
||||
Build:
|
||||
From a checkout, build with Go 1.24 or newer:
|
||||
|
||||
```sh
|
||||
go build -o ./bin/audita ./cmd/audita
|
||||
```
|
||||
|
||||
Run the shortest useful command:
|
||||
Processing requires a reachable [compatible LLM endpoint](docs/integrations/openai-compatible-llm.md)
|
||||
and access to its configured model. Supply its credentials through the
|
||||
[configuration mechanisms](docs/config.md#secrets); the
|
||||
[minimal config](examples/minimal-config.yml) reads `AUDITA_LLM_API_KEY` and uses
|
||||
the [default model and endpoint](docs/config.md#llm-settings). Set that variable
|
||||
in your environment before using an endpoint that requires authentication.
|
||||
For another endpoint or model, edit a copy of the config.
|
||||
|
||||
Run the included small input with the minimal config:
|
||||
|
||||
```sh
|
||||
audita process ./transcript.json --glossary ./glossary.yaml --output ./corrected.json
|
||||
./bin/audita process examples/tiny-transcript.json \
|
||||
--config examples/minimal-config.yml \
|
||||
--glossary examples/tiny-glossary.yaml \
|
||||
--output corrected.json
|
||||
```
|
||||
|
||||
Notes:
|
||||
- the transcript JSON path is required as a positional argument;
|
||||
- `--glossary` is required;
|
||||
- without `--output`, corrected transcript JSON is written to stdout.
|
||||
|
||||
## Configuration
|
||||
|
||||
Audita loads defaults, optional file config, environment overrides, then CLI overrides.
|
||||
|
||||
Use these commands to validate and inspect config:
|
||||
|
||||
```sh
|
||||
audita config validate --config ./audita.yml
|
||||
audita config print-effective --config ./audita.yml
|
||||
```
|
||||
This makes LLM requests. Inherited configuration environment overrides still
|
||||
apply; see [precedence](docs/config.md#effective-precedence).
|
||||
|
||||
## Documentation
|
||||
|
||||
- CLI reference: [`docs/cli.md`](docs/cli.md)
|
||||
- Configuration reference: [`docs/config.md`](docs/config.md)
|
||||
- Operations guide: [`docs/operations.md`](docs/operations.md)
|
||||
- Troubleshooting: [`docs/troubleshooting.md`](docs/troubleshooting.md)
|
||||
- Subprocess integration: [`docs/integrations/subprocess.md`](docs/integrations/subprocess.md)
|
||||
- OpenAI-compatible LLM integration: [`docs/integrations/openai-compatible-llm.md`](docs/integrations/openai-compatible-llm.md)
|
||||
- Transcript and glossary file integration: [`docs/integrations/transcript-glossary-files.md`](docs/integrations/transcript-glossary-files.md)
|
||||
- Development workflow: [`docs/policy/development.md`](docs/policy/development.md)
|
||||
- Architecture policy: [`docs/policy/architecture.md`](docs/policy/architecture.md)
|
||||
- Documentation policy: [`docs/policy/documentation.md`](docs/policy/documentation.md)
|
||||
- [CLI reference](docs/cli.md)
|
||||
- [Configuration and maintained examples](docs/config.md)
|
||||
- [Operations and artifact lifecycle](docs/operations.md)
|
||||
- [Transcript and glossary inputs](docs/integrations/transcript-glossary-files.md)
|
||||
- [Output files and report formats](docs/integrations/output-files.md)
|
||||
- [Subprocess integration](docs/integrations/subprocess.md)
|
||||
- [LLM endpoint contract](docs/integrations/openai-compatible-llm.md)
|
||||
- [Contributor guide](docs/development.md)
|
||||
|
||||
@@ -571,6 +571,15 @@ func runCLISubprocessContext(t *testing.T, ctx context.Context, extraEnv map[str
|
||||
cmdArgs := append([]string{"-test.run=TestHelperProcess", "--"}, args...)
|
||||
cmd := exec.CommandContext(ctx, os.Args[0], cmdArgs...)
|
||||
env := append(filterAuditaEnv(os.Environ()), "GO_WANT_HELPER_PROCESS=1")
|
||||
// Use defaults without discovering the host's installed configuration.
|
||||
// Explicit test environment overrides and --config retain their precedence.
|
||||
if _, supplied := extraEnv["AUDITA_CONFIG"]; !supplied {
|
||||
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||
if err := os.WriteFile(configPath, []byte("version: 1\n"), 0600); err != nil {
|
||||
t.Fatalf("write baseline config: %v", err)
|
||||
}
|
||||
env = append(env, "AUDITA_CONFIG="+configPath)
|
||||
}
|
||||
for k, v := range extraEnv {
|
||||
env = append(env, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
|
||||
7
docs/adr/README.md
Normal file
7
docs/adr/README.md
Normal file
@@ -0,0 +1,7 @@
|
||||
# Architectural Decision Records
|
||||
|
||||
No ADRs have been recorded here. Use the
|
||||
[documentation policy's ADR lifecycle](../policy/documentation.md#architecture-decision-records)
|
||||
for significant decisions. Current architecture belongs in the
|
||||
[architecture policy](../policy/architecture.md); implementation status belongs in
|
||||
[roadmap documents](../roadmap/).
|
||||
90
docs/cli.md
90
docs/cli.md
@@ -3,10 +3,12 @@
|
||||
## Shortest Useful Command
|
||||
|
||||
```sh
|
||||
audita process <transcript.json> --glossary <glossary.yaml> --output <corrected.json>
|
||||
audita process transcript.json --glossary glossary.yaml --output corrected.json
|
||||
```
|
||||
|
||||
This command validates input files, runs the configured correction pipeline, and writes corrected transcript JSON.
|
||||
This command runs the core workflow with your input files. For a copyable example
|
||||
using maintained inputs and explicit prerequisites, see the [README](../README.md).
|
||||
Commands here assume `audita` is on `PATH`; use `./bin/audita` after a local build.
|
||||
|
||||
## Command Overview
|
||||
|
||||
@@ -34,13 +36,17 @@ Input requirements:
|
||||
- exactly one transcript JSON positional argument is required;
|
||||
- `--glossary <path>` is required.
|
||||
|
||||
Config path selection for `process`:
|
||||
1. `--config <path>`
|
||||
2. `AUDITA_CONFIG`
|
||||
3. `/usr/local/etc/audita/config.yml` (if present)
|
||||
4. `/etc/audita/config.yml` (if present)
|
||||
Place the transcript path first, followed by flags, as shown above. Flag values
|
||||
may use `--name value` or `--name=value`. Unknown flags or extra positional
|
||||
arguments are errors. Flags before a final transcript path are also accepted;
|
||||
do not interleave further flags after a positional argument consumed by the flag
|
||||
parser.
|
||||
|
||||
For precedence and full config schema, see [`docs/config.md`](config.md).
|
||||
[Configuration discovery and precedence](config.md#loading-model) determine the
|
||||
initial settings; explicit process flags supply overrides. Configuration is loaded
|
||||
before process help is printed, so invalid discovered configuration can also make
|
||||
process help fail. Input formats belong to the
|
||||
[input-file contract](integrations/transcript-glossary-files.md).
|
||||
|
||||
### `process` Flag Reference
|
||||
|
||||
@@ -49,7 +55,7 @@ Core I/O flags:
|
||||
- `--glossary <path>`: glossary YAML input path (required).
|
||||
- `--output <path>`: corrected transcript JSON output file path.
|
||||
- `--report-json <path>`: machine-readable report JSON output path.
|
||||
- `--output-schema <key>`: output schema key (`bare-segments` or `audita-v1`).
|
||||
- `--output-schema <key>`: output encoder selection; see [configuration](config.md#pipeline-and-output).
|
||||
- `--modules <csv>`: comma-separated module sequence override.
|
||||
|
||||
Primary LLM flags:
|
||||
@@ -91,7 +97,11 @@ Threshold flags:
|
||||
Context and diagnostics flags:
|
||||
- `--transcript-description <text>`: background context for prompts; does not override transcript content.
|
||||
- `--work-dir <path>`: per-run diagnostics work directory.
|
||||
- `--work-dir-retention <auto|always|never>`: run-directory retention policy.
|
||||
- `--work-dir-retention <value>`: retention selection; see [configuration](config.md#thresholds-context-and-diagnostics).
|
||||
|
||||
Flag descriptions identify their settings; defaults, accepted keys, numeric
|
||||
constraints, and inheritance rules are owned by [configuration](config.md).
|
||||
Normalization time flags use numeric seconds; LLM timeout flags use integer seconds.
|
||||
|
||||
### `process` Output and Exit Behavior
|
||||
|
||||
@@ -99,11 +109,16 @@ Context and diagnostics flags:
|
||||
- Without `--output`: stdout contains transcript JSON only on success.
|
||||
- `--report-json` writes a file and is never printed to stdout.
|
||||
- Stderr is human-readable diagnostics/errors.
|
||||
- On failures after diagnostics initialization, stderr includes the diagnostics directory path.
|
||||
- Processing failures include the diagnostics directory path when initialization
|
||||
succeeded. Later report-write or retention errors can omit that path.
|
||||
- Output files may already exist when a later reporting or cleanup failure occurs.
|
||||
- Rejected/skipped corrections and warnings can occur in a successful run; inspect
|
||||
the [report format](integrations/output-files.md#process-report) for details.
|
||||
- Diagnostic wording is human-readable and is not a machine-stable JSON channel.
|
||||
|
||||
Exit behavior:
|
||||
- `0`: success.
|
||||
- `1`: runtime failure during processing/reporting/output paths.
|
||||
- `1`: runtime failure during processing/reporting/output or retention handling.
|
||||
- `2`: CLI usage or configuration input error.
|
||||
|
||||
Integration references:
|
||||
@@ -145,42 +160,39 @@ audita process transcript.json \
|
||||
--output corrected.json
|
||||
```
|
||||
|
||||
## `config validate`
|
||||
## Configuration Commands
|
||||
|
||||
Usage:
|
||||
### `config validate`
|
||||
|
||||
```sh
|
||||
audita config validate --config <path>
|
||||
audita config validate --config examples/minimal-config.yml
|
||||
audita config validate --config examples/production-config.yml
|
||||
```
|
||||
|
||||
Behavior:
|
||||
- validates defaults merged with file config;
|
||||
- does not apply environment overrides;
|
||||
- prints `config is valid` on success.
|
||||
The only command-specific flag is `--config <path>`, which is required.
|
||||
Positional arguments are rejected. Success prints `config is valid` to stdout
|
||||
and returns zero; usage, file-read, parse, and validation errors go to stderr and
|
||||
return `2`. See [applied configuration sources](config.md#effective-precedence)
|
||||
and [validation boundaries](config.md#examples).
|
||||
|
||||
Errors:
|
||||
- `--config` is required;
|
||||
- positional arguments are rejected;
|
||||
- validation failures are printed to stderr.
|
||||
|
||||
## `config print-effective`
|
||||
|
||||
Usage:
|
||||
### `config print-effective`
|
||||
|
||||
```sh
|
||||
audita config print-effective [--config <path>]
|
||||
audita config print-effective --config examples/minimal-config.yml
|
||||
```
|
||||
|
||||
Config path selection:
|
||||
1. `--config <path>` when provided
|
||||
2. `AUDITA_CONFIG`
|
||||
3. `/usr/local/etc/audita/config.yml` (if present)
|
||||
4. `/etc/audita/config.yml` (if present)
|
||||
The only command-specific flag is optional `--config <path>`. Positional
|
||||
arguments are rejected. Without an explicit file, [discovery](config.md#loading-model)
|
||||
applies. Success prints redacted configuration JSON to stdout and returns zero.
|
||||
Usage and configuration errors return `2`; serialization or stdout-write failures
|
||||
return `1`.
|
||||
|
||||
Behavior:
|
||||
- merges defaults, optional config file, and environment overrides;
|
||||
- prints redacted JSON to stdout.
|
||||
The printed JSON reflects the effective in-memory configuration; it is not a YAML
|
||||
configuration file and is not the process-report schema. Use it to inspect merged
|
||||
settings, not as a file to pass back to the YAML loader.
|
||||
|
||||
Errors:
|
||||
- positional arguments are rejected;
|
||||
- resolution or parse failures are printed to stderr.
|
||||
## Help
|
||||
|
||||
Root invocation without arguments and root help print usage successfully. Use
|
||||
`--help` or `-h` for subcommand help. Help normally returns zero; process help has
|
||||
the configuration-loading caveat described above. Unknown commands return `2`.
|
||||
|
||||
342
docs/config.md
342
docs/config.md
@@ -1,238 +1,188 @@
|
||||
# Audita Configuration
|
||||
|
||||
## Scope
|
||||
|
||||
This is the canonical configuration reference for Audita.
|
||||
|
||||
It documents:
|
||||
- config path resolution;
|
||||
- effective precedence across defaults, file config, environment, and CLI;
|
||||
- supported `version: 1` YAML schema;
|
||||
- environment overrides;
|
||||
- CLI override relationship;
|
||||
- validation and secrets behavior.
|
||||
|
||||
For CLI command syntax, see [`docs/cli.md`](cli.md).
|
||||
For OpenAI-compatible endpoint behavior, see [`docs/integrations/openai-compatible-llm.md`](integrations/openai-compatible-llm.md).
|
||||
For transcript/glossary input file contracts, see [`docs/integrations/transcript-glossary-files.md`](integrations/transcript-glossary-files.md).
|
||||
This is the canonical reference for discovery, settings, defaults, environment
|
||||
mapping, and validation. [CLI syntax](cli.md), [artifact lifecycle](operations.md),
|
||||
and [file formats](integrations/output-files.md) have separate owners.
|
||||
|
||||
## Loading Model
|
||||
|
||||
Path resolution for `audita process` and `audita config print-effective`:
|
||||
1. `--config <path>`
|
||||
2. `AUDITA_CONFIG`
|
||||
3. `/usr/local/etc/audita/config.yml` (if present)
|
||||
4. `/etc/audita/config.yml` (if present)
|
||||
For processing and effective-config inspection, the first applicable source
|
||||
selects one YAML file:
|
||||
|
||||
Missing explicit path behavior:
|
||||
- missing `--config` target is an error;
|
||||
- missing `AUDITA_CONFIG` target is an error.
|
||||
1. Explicit CLI config path.
|
||||
2. `AUDITA_CONFIG`.
|
||||
3. `/usr/local/etc/audita/config.yml`, if present.
|
||||
4. `/etc/audita/config.yml`, if present.
|
||||
|
||||
Missing default-path files are non-fatal.
|
||||
These files are alternatives, not layers merged together. Missing system files
|
||||
are skipped. An empty or missing explicitly selected path is an error, as is an
|
||||
inaccessible selected file. Without a selected file, built-in defaults apply.
|
||||
The validation command requires an explicit file and does not use discovery.
|
||||
|
||||
## Effective Precedence
|
||||
|
||||
`audita process`:
|
||||
1. defaults
|
||||
2. file config
|
||||
3. environment overrides
|
||||
4. CLI overrides
|
||||
| Operation | Applied sources, lowest to highest precedence |
|
||||
| --- | --- |
|
||||
| Process | Built-in defaults, selected file, environment overrides, explicit CLI overrides |
|
||||
| Print effective configuration | Built-in defaults, selected file, environment overrides |
|
||||
| Validate configuration file | Built-in defaults, explicit file |
|
||||
|
||||
`audita config print-effective`:
|
||||
1. defaults
|
||||
2. file config
|
||||
3. environment overrides
|
||||
File validation does not apply configuration environment overrides, but it still
|
||||
resolves environment variables named by `api_key_env` fields. Each configuration
|
||||
layer is validated as it is applied: a later override cannot repair an invalid
|
||||
earlier layer. [Config commands](cli.md#configuration-commands) provide inspection
|
||||
and validation without processing a transcript or contacting an LLM.
|
||||
|
||||
`audita config validate`:
|
||||
1. defaults
|
||||
2. file config
|
||||
## File Schema
|
||||
|
||||
`config validate` is intentionally file-only (no environment overrides).
|
||||
The file is YAML with required `version: 1`. Unknown fields are rejected.
|
||||
The remaining sections and fields below are optional. Omitted values preserve
|
||||
lower-precedence settings, except for the documented inheritance rules.
|
||||
Complete copyable files are in [examples](#examples).
|
||||
|
||||
## Defaults
|
||||
### Pipeline and Output
|
||||
|
||||
Current defaults:
|
||||
- modules: `glossary,homophones,glossary,spoken_word,grammar`
|
||||
- output schema: `bare-segments`
|
||||
- primary model: `openrouter/google/gemma-4-31b-it`
|
||||
- primary base URL: `https://openrouter.ai/api/v1`
|
||||
- primary timeout: `600` seconds
|
||||
- max retries: `3`
|
||||
- total/proposal LLM concurrency: `1`
|
||||
- validation max prompt tokens: `2048`
|
||||
- max section tokens: `8192`
|
||||
- min section tokens: `2048`
|
||||
- confidence thresholds: `0.8`
|
||||
- normalization max segment gap: `4.0`
|
||||
- normalization ellipsis gap: `3.5`
|
||||
- normalization max segment duration: `60.0`
|
||||
- normalization max segment tokens: `2048`
|
||||
- transcript description: empty
|
||||
- work dir: `/tmp/audita`
|
||||
- work dir retention: `auto`
|
||||
| YAML field | Default | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `pipeline.modules` | `[glossary, homophones, glossary, spoken_word, grammar]` | Ordered module sequence. Supported keys are `glossary`, `homophones`, `spoken_word`, and `grammar`; repeated keys are allowed. An omitted or empty YAML list leaves the sequence unchanged. |
|
||||
| `output.schema` | `bare-segments` | Output encoder selection: `bare-segments` or `audita-v1`. See [output formats](integrations/output-files.md#corrected-transcript). |
|
||||
|
||||
## YAML Schema (`version: 1`)
|
||||
Module keys select correction stages. Validator chains are built into each module;
|
||||
there is no configuration field for selecting individual validator keys.
|
||||
|
||||
Supported file version:
|
||||
- `version: 1` (required)
|
||||
### LLM Settings
|
||||
|
||||
Unknown YAML fields are rejected.
|
||||
The same fields are supported under `llm.proposal` and `llm.validation`.
|
||||
|
||||
```yaml
|
||||
version: 1
|
||||
| Field suffix | Proposal default | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `base_url` | `https://openrouter.ai/api/v1` | Endpoint base URL; the [LLM contract](integrations/openai-compatible-llm.md#endpoint-and-authentication) defines how it is used. |
|
||||
| `model` | `openrouter/google/gemma-4-31b-it` | Model identifier sent to the endpoint. Availability is determined by that endpoint. |
|
||||
| `api_key_env` | Unset | Name of an environment variable containing the API key; see [secrets](#secrets). |
|
||||
| `timeout` | `600` seconds | Positive integer seconds or a duration string resolving to whole seconds, such as `120s`. |
|
||||
| `max_retries` | `3` | Non-negative retry count after the initial attempt; retry conditions belong to the [LLM contract](integrations/openai-compatible-llm.md#retries-and-timeouts). |
|
||||
|
||||
pipeline:
|
||||
modules: [glossary, homophones, glossary, spoken_word, grammar]
|
||||
Validation settings inherit the effective proposal settings individually. Empty
|
||||
validation model, base URL, or API key values inherit the proposal value; omitted
|
||||
validation timeout and retry count inherit, while an explicit retry count of zero
|
||||
is retained in configuration. The current HTTP adapter interprets a retry count
|
||||
of zero as its default of three retries, so zero does not disable retries.
|
||||
No API key is required by local config validation. Model and base-URL
|
||||
usability are checked during client setup/requests, not by an online config check.
|
||||
|
||||
output:
|
||||
schema: bare-segments
|
||||
### Concurrency
|
||||
|
||||
llm:
|
||||
proposal:
|
||||
base_url: https://openrouter.ai/api/v1
|
||||
model: openrouter/google/gemma-4-31b-it
|
||||
api_key_env: AUDITA_LLM_API_KEY
|
||||
timeout: 600s
|
||||
max_retries: 3
|
||||
validation:
|
||||
base_url: https://openrouter.ai/api/v1
|
||||
model: openrouter/google/gemma-4-31b-it
|
||||
api_key_env: AUDITA_VALIDATION_LLM_API_KEY
|
||||
timeout: 600
|
||||
max_retries: 3
|
||||
| YAML field | Default | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `concurrency.total_llm` | `1` | Positive total concurrent LLM-call limit. |
|
||||
| `concurrency.proposal_llm` | `1` | Positive proposal-call limit, no greater than total. |
|
||||
| `concurrency.validation_llm` | Inherits total | Positive validation-call limit, no greater than total. |
|
||||
|
||||
concurrency:
|
||||
total_llm: 1
|
||||
proposal_llm: 1
|
||||
validation_llm: 1
|
||||
In a YAML file, setting total alone does not change proposal concurrency. At the
|
||||
environment or CLI layer, setting total without setting proposal at that same
|
||||
layer also sets proposal to that total, even if a lower layer supplied a proposal
|
||||
value. Unset validation concurrency inherits the final total. The canonical total
|
||||
setting wins over its legacy alias when both appear at the same layer.
|
||||
|
||||
chunking:
|
||||
target_sections: 8
|
||||
max_section_tokens: 8192
|
||||
min_section_tokens: 2048
|
||||
Validation prompt budgeting is available through environment or CLI only:
|
||||
`AUDITA_VALIDATION_MAX_PROMPT_TOKENS` has default `2048` and must be positive.
|
||||
There is no YAML field for it.
|
||||
|
||||
normalization:
|
||||
max_segment_gap: 4s
|
||||
ellipsis_gap: 3.5s
|
||||
max_segment_duration: 60s
|
||||
max_segment_tokens: 2048
|
||||
### Chunking and Normalization
|
||||
|
||||
thresholds:
|
||||
glossary: 0.8
|
||||
homophones: 0.8
|
||||
spoken_word: 0.8
|
||||
grammar: 0.8
|
||||
| YAML field | Default | Meaning and validation |
|
||||
| --- | --- | --- |
|
||||
| `chunking.target_sections` | Unset | Optional positive target count. Processing rejects targets impossible for the normalized transcript and token constraints. |
|
||||
| `chunking.max_section_tokens` | `8192` | Positive maximum section token estimate. |
|
||||
| `chunking.min_section_tokens` | `2048` | Positive minimum used in chunk balancing; no greater than maximum. |
|
||||
| `normalization.max_segment_gap` | `4.0` seconds | Non-negative gap limit for same-speaker merging. |
|
||||
| `normalization.ellipsis_gap` | `3.5` seconds | Non-negative gap threshold for inserting an ellipsis during merging. |
|
||||
| `normalization.max_segment_duration` | `60.0` seconds | Positive merged-segment duration limit. |
|
||||
| `normalization.max_segment_tokens` | `2048` | Positive merged-segment token estimate limit. |
|
||||
|
||||
context:
|
||||
description: optional background context
|
||||
The three normalization time fields accept numeric seconds or YAML duration
|
||||
strings. Their environment and CLI overrides use numeric seconds. Section token
|
||||
counts are estimates; config validation does not test whether a particular input
|
||||
can satisfy the selected chunking target.
|
||||
|
||||
diagnostics:
|
||||
work_dir: /tmp/audita
|
||||
retention: auto
|
||||
```
|
||||
### Thresholds, Context, and Diagnostics
|
||||
|
||||
Duration-parsing behavior:
|
||||
- `llm.*.timeout`: integer seconds or duration string; duration strings must resolve to whole seconds.
|
||||
- `normalization.*` duration-like fields: numeric seconds or duration string.
|
||||
| YAML field | Default | Meaning and validation |
|
||||
| --- | --- | --- |
|
||||
| `thresholds.glossary` | `0.8` | Glossary correction confidence threshold, in `[0, 1]`. |
|
||||
| `thresholds.homophones` | `0.8` | Homophone correction confidence threshold, in `[0, 1]`. |
|
||||
| `thresholds.spoken_word` | `0.8` | Spoken-word correction confidence threshold, in `[0, 1]`. |
|
||||
| `thresholds.grammar` | `0.8` | Grammar correction confidence threshold, in `[0, 1]`. |
|
||||
| `context.description` | Empty | Trimmed background context for prompts; limited to 500 bytes of UTF-8 text after trimming. It does not replace transcript content. |
|
||||
| `diagnostics.work_dir` | `/tmp/audita` | Non-empty parent directory for run artifacts. |
|
||||
| `diagnostics.retention` | `auto` | Accepted values: `auto`, `always`, `never`. Exact lifecycle behavior, including the current behavior of `never`, belongs to [operations](operations.md#retention-behavior). |
|
||||
|
||||
## Environment Overrides
|
||||
|
||||
Modules:
|
||||
- `AUDITA_MODULES`
|
||||
Values are read when the variable is set. Empty values are not generally equivalent
|
||||
to unset values: numeric overrides must parse, and empty strings may clear a
|
||||
setting or fail validation. Numeric environment values use integers or floating
|
||||
point seconds as indicated above, not duration strings.
|
||||
|
||||
Config path:
|
||||
- `AUDITA_CONFIG`
|
||||
| Environment variable | Setting |
|
||||
| --- | --- |
|
||||
| `AUDITA_CONFIG` | Config file selection |
|
||||
| `AUDITA_MODULES` | Module sequence as comma-separated keys |
|
||||
| `AUDITA_LLM_API_KEY` | Direct proposal API key |
|
||||
| `OPENROUTER_API_KEY` | Proposal API key fallback when `AUDITA_LLM_API_KEY` is unset |
|
||||
| `AUDITA_MODEL` | Proposal model |
|
||||
| `AUDITA_BASE_URL` | Proposal endpoint |
|
||||
| `AUDITA_LLM_TIMEOUT_SECONDS` | Proposal timeout |
|
||||
| `AUDITA_MAX_RETRIES` | Proposal retries |
|
||||
| `AUDITA_VALIDATION_LLM_API_KEY` | Direct validation API key |
|
||||
| `AUDITA_VALIDATION_MODEL` | Validation model |
|
||||
| `AUDITA_VALIDATION_BASE_URL` | Validation endpoint |
|
||||
| `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | Validation timeout |
|
||||
| `AUDITA_VALIDATION_MAX_RETRIES` | Validation retries |
|
||||
| `AUDITA_TOTAL_LLM_CONCURRENCY` | Total concurrency |
|
||||
| `AUDITA_LLM_CONCURRENCY` | Legacy total-concurrency alias |
|
||||
| `AUDITA_PROPOSAL_LLM_CONCURRENCY` | Proposal concurrency |
|
||||
| `AUDITA_VALIDATION_LLM_CONCURRENCY` | Validation concurrency |
|
||||
| `AUDITA_VALIDATION_MAX_PROMPT_TOKENS` | Validation prompt budget |
|
||||
| `AUDITA_TARGET_SECTIONS` | Section target |
|
||||
| `AUDITA_MAX_SECTION_TOKENS` | Section maximum |
|
||||
| `AUDITA_MIN_SECTION_TOKENS` | Section minimum |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | Merge gap |
|
||||
| `AUDITA_NORMALIZE_ELLIPSIS_GAP` | Ellipsis gap |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | Merged duration limit |
|
||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | Merged token estimate limit |
|
||||
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | Glossary threshold |
|
||||
| `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | Homophone threshold |
|
||||
| `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | Spoken-word threshold |
|
||||
| `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | Grammar threshold |
|
||||
| `AUDITA_WORK_DIR` | Diagnostics directory |
|
||||
| `AUDITA_WORK_DIR_RETENTION` | Retention selection |
|
||||
|
||||
Primary LLM:
|
||||
- `AUDITA_LLM_API_KEY` (falls back to `OPENROUTER_API_KEY` when unset)
|
||||
- `AUDITA_MODEL`
|
||||
- `AUDITA_BASE_URL`
|
||||
- `AUDITA_LLM_TIMEOUT_SECONDS`
|
||||
- `AUDITA_MAX_RETRIES`
|
||||
|
||||
Validation LLM:
|
||||
- `AUDITA_VALIDATION_LLM_API_KEY`
|
||||
- `AUDITA_VALIDATION_MODEL`
|
||||
- `AUDITA_VALIDATION_BASE_URL`
|
||||
- `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS`
|
||||
- `AUDITA_VALIDATION_MAX_RETRIES`
|
||||
- `AUDITA_VALIDATION_MAX_PROMPT_TOKENS`
|
||||
|
||||
Concurrency:
|
||||
- `AUDITA_TOTAL_LLM_CONCURRENCY`
|
||||
- `AUDITA_PROPOSAL_LLM_CONCURRENCY`
|
||||
- `AUDITA_VALIDATION_LLM_CONCURRENCY`
|
||||
- `AUDITA_LLM_CONCURRENCY` (legacy alias for total)
|
||||
|
||||
Chunking:
|
||||
- `AUDITA_MAX_SECTION_TOKENS`
|
||||
- `AUDITA_MIN_SECTION_TOKENS`
|
||||
- `AUDITA_TARGET_SECTIONS`
|
||||
|
||||
Thresholds:
|
||||
- `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD`
|
||||
- `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD`
|
||||
- `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD`
|
||||
- `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD`
|
||||
|
||||
Normalization:
|
||||
- `AUDITA_NORMALIZE_MAX_SEGMENT_GAP`
|
||||
- `AUDITA_NORMALIZE_ELLIPSIS_GAP`
|
||||
- `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION`
|
||||
- `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS`
|
||||
|
||||
Diagnostics:
|
||||
- `AUDITA_WORK_DIR`
|
||||
- `AUDITA_WORK_DIR_RETENTION` (`auto`, `always`, `never`)
|
||||
|
||||
Transcript description:
|
||||
- no `AUDITA_*` environment variable is currently defined.
|
||||
|
||||
## CLI Override Relationship
|
||||
|
||||
CLI flags override file and environment values for `audita process`.
|
||||
|
||||
The CLI supports canonical total concurrency (`--total-llm-concurrency`) and legacy alias (`--llm-concurrency`):
|
||||
- when both are provided at the same precedence layer, canonical total wins;
|
||||
- if proposal concurrency is not explicitly set and total is set via environment or CLI, proposal concurrency inherits that total;
|
||||
- validation concurrency inherits total only when validation concurrency is unset.
|
||||
|
||||
For full flag syntax, see [`docs/cli.md`](cli.md).
|
||||
|
||||
## Validation Rules
|
||||
|
||||
Validation includes:
|
||||
- supported module keys only;
|
||||
- supported output schema keys only (`bare-segments`, `audita-v1`);
|
||||
- positive timeout/concurrency/token constraints;
|
||||
- `proposal_llm <= total_llm` and `validation_llm <= total_llm` when validation is set;
|
||||
- confidence thresholds in `[0.0, 1.0]`;
|
||||
- transcript description length `<= 500` characters;
|
||||
- non-empty work dir;
|
||||
- work-dir retention in `auto|always|never`.
|
||||
There is no environment override for output schema or transcript description.
|
||||
Use file configuration or the corresponding [CLI flags](cli.md#process-flag-reference).
|
||||
|
||||
## Secrets
|
||||
|
||||
Recommended secret handling:
|
||||
- use `llm.proposal.api_key_env` and `llm.validation.api_key_env` in file config;
|
||||
- use `AUDITA_*_API_KEY` environment overrides or CLI key flags when needed.
|
||||
File config accepts environment variable names through `api_key_env`, not literal
|
||||
API-key fields. Names must be non-empty and match an environment-variable
|
||||
identifier; an unset named variable resolves to an empty value. Runtime environment
|
||||
key overrides take precedence over keys resolved from a file. CLI key flags supply
|
||||
direct values and take precedence over environment settings.
|
||||
|
||||
`api_key_env` fields contain environment variable names, not secret values.
|
||||
|
||||
Redaction behavior:
|
||||
- `audita config print-effective` redacts resolved API keys.
|
||||
- diagnostics and report paths redact configured secret values.
|
||||
Effective-config inspection redacts API keys. Configured LLM secrets are also
|
||||
redacted by the runtime's error and diagnostic handling, but transcript content
|
||||
is still sensitive: see [artifact handling](operations.md#artifact-handling).
|
||||
|
||||
## Examples
|
||||
|
||||
- Minimal config: [`examples/minimal-config.yml`](../examples/minimal-config.yml)
|
||||
- Production-style config: [`examples/production-config.yml`](../examples/production-config.yml)
|
||||
- Tiny transcript input: [`examples/tiny-transcript.json`](../examples/tiny-transcript.json)
|
||||
- Tiny glossary input: [`examples/tiny-glossary.yaml`](../examples/tiny-glossary.yaml)
|
||||
- [Minimal configuration](../examples/minimal-config.yml): starts from defaults
|
||||
and obtains the proposal key from the environment.
|
||||
- [Production-oriented configuration](../examples/production-config.yml): illustrates
|
||||
both LLM targets, concurrency, chunking, normalization, thresholds, and diagnostics.
|
||||
Its explicit section target is intended for larger inputs, not the tiny sample.
|
||||
- [Tiny transcript](../examples/tiny-transcript.json) and
|
||||
[tiny glossary](../examples/tiny-glossary.yaml): inputs for the README quickstart.
|
||||
|
||||
Validate the config examples:
|
||||
|
||||
```sh
|
||||
audita config validate --config examples/minimal-config.yml
|
||||
audita config validate --config examples/production-config.yml
|
||||
```
|
||||
Use the [CLI configuration commands](cli.md#configuration-commands) to validate
|
||||
files and inspect effective settings. Validation checks local structure and
|
||||
constraints; it does not verify credentials, model availability, or endpoint access.
|
||||
|
||||
49
docs/development.md
Normal file
49
docs/development.md
Normal file
@@ -0,0 +1,49 @@
|
||||
# Developing Audita
|
||||
|
||||
Audita is a Go CLI for transcript polishing. Start with the
|
||||
[README](../README.md) for product orientation and the
|
||||
[internal overview](internal/overview.md) for the implemented component map.
|
||||
|
||||
## Before Changing the Repository
|
||||
|
||||
Coding agents must read and follow the [agent policy](policy/agents.md) before
|
||||
recommending, planning, or implementing changes. Read the
|
||||
[architecture policy](policy/architecture.md) for boundaries and invariants,
|
||||
then use the task guide below rather than reading every reference indiscriminately.
|
||||
|
||||
Confirm the requested scope and behavior contract, make focused changes in the
|
||||
owning component, and update the relevant tests and canonical documentation
|
||||
when behavior changes. The [testing policy](policy/testing.md) governs test
|
||||
sufficiency; the [documentation policy](policy/documentation.md) governs content
|
||||
ownership. Commit discipline and agent Git restrictions are in the
|
||||
[agent policy](policy/agents.md#git-commit-policy).
|
||||
|
||||
## Baseline Validation
|
||||
|
||||
Use Go 1.24 or newer. From the repository root:
|
||||
|
||||
```sh
|
||||
go build -o ./bin/audita ./cmd/audita
|
||||
go test ./...
|
||||
```
|
||||
|
||||
Run targeted package tests for touched behavior and the complete suite for
|
||||
substantial changes. Documentation-only work normally needs affected-link,
|
||||
example, and whitespace checks; use `git diff --check`. Detailed component
|
||||
validation belongs with the change recipes below.
|
||||
|
||||
## Task-Specific Reading Guide
|
||||
|
||||
| Task | Read |
|
||||
| --- | --- |
|
||||
| Locate components or change orchestration | [Internal overview](internal/overview.md), then [pipeline](internal/pipeline.md) |
|
||||
| Change configuration or CLI behavior | [Configuration contract](config.md), [CLI contract](cli.md), and [configuration/CLI change recipes](internal/cli-config.md) |
|
||||
| Change modules or validators | [Modules](internal/modules.md) or [validators](internal/validators.md); follow their prompt, registry, and test references |
|
||||
| Change LLM transport or concurrency | [LLM integration contract](integrations/openai-compatible-llm.md), [LLM runtime](internal/llm-runtime.md), and [pipeline](internal/pipeline.md) |
|
||||
| Change prompts | [Prompt registry](internal/prompts.md) and the consuming [module](internal/modules.md) or [validator](internal/validators.md) |
|
||||
| Change input/output formats | [Input file contracts](integrations/transcript-glossary-files.md), [output-file contract](integrations/output-files.md), [encoder internals](internal/output-schemas.md), and [configuration](config.md) |
|
||||
| Change reports or diagnostics | [Operations](operations.md), [output-file contract](integrations/output-files.md), [subprocess integration](integrations/subprocess.md), and [diagnostics/reporting internals](internal/diagnostics-reporting.md) |
|
||||
| Change tests | [Testing policy](policy/testing.md) and the affected component's internal documentation |
|
||||
| Change docs or examples | [Documentation policy](policy/documentation.md), the canonical topic owner, and [example validation](internal/cli-config.md#validating-examples) |
|
||||
| Prepare a release | [Release policy](policy/release.md) and [release procedure](release.md) |
|
||||
| Record a significant architectural decision | [ADR lifecycle](policy/documentation.md#architecture-decision-records) and [ADR index](adr/README.md) |
|
||||
@@ -1,119 +1,76 @@
|
||||
# OpenAI-Compatible LLM Integration
|
||||
|
||||
## Scope
|
||||
This document owns the external protocol used by Audita's structured LLM adapter.
|
||||
Settings, inheritance, defaults, and credential supply belong to
|
||||
[configuration](../config.md#llm-settings); implementation and scheduling belong to
|
||||
[LLM runtime internals](../internal/llm-runtime.md).
|
||||
|
||||
This document defines the external LLM endpoint contract Audita currently uses.
|
||||
## Endpoint and Authentication
|
||||
|
||||
It covers:
|
||||
- endpoint and auth expectations;
|
||||
- structured request and response shape;
|
||||
- retry and timeout behavior;
|
||||
- diagnostics and secret redaction.
|
||||
Audita sends an HTTP `POST` to the configured base URL with trailing slashes
|
||||
removed and `/chat/completions` appended. Supply an HTTP or HTTPS base URL that
|
||||
makes this the intended endpoint; the adapter does not require HTTPS in code.
|
||||
Requests have `Content-Type: application/json`. A non-empty configured API key
|
||||
adds `Authorization: Bearer <api_key>`; otherwise that header is omitted.
|
||||
|
||||
For user-facing CLI flags and config keys, see [`docs/cli.md`](../cli.md) and [`docs/config.md`](../config.md).
|
||||
|
||||
## Endpoint Contract
|
||||
|
||||
Audita sends HTTPS `POST` requests to:
|
||||
|
||||
- `<base_url>/chat/completions`
|
||||
|
||||
`base_url` comes from primary or validation LLM config and is required.
|
||||
|
||||
## Authentication Contract
|
||||
|
||||
When an API key is configured, Audita sends:
|
||||
|
||||
- `Authorization: Bearer <api_key>`
|
||||
|
||||
When no API key is configured, the `Authorization` header is omitted.
|
||||
The endpoint must support the configured model and structured JSON-schema
|
||||
responses. Local configuration validation does not probe that support.
|
||||
|
||||
## Request Shape
|
||||
|
||||
Audita sends a chat-completions payload with:
|
||||
- `model`;
|
||||
- `messages` (role/content pairs);
|
||||
- `response_format` using JSON Schema strict mode.
|
||||
|
||||
Representative shape:
|
||||
The payload contains `model`, an array of `messages` with string `role`/`content`,
|
||||
and `response_format`. The structured-output portion has this illustrative shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "example-model",
|
||||
"messages": [
|
||||
{"role": "system", "content": "..."},
|
||||
{"role": "user", "content": "..."}
|
||||
],
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "correction_set",
|
||||
"strict": true,
|
||||
"schema": {"type": "object"}
|
||||
}
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "correction_set",
|
||||
"strict": true,
|
||||
"schema": {"type": "object"}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Behavioral requirements enforced by Audita:
|
||||
- `model` must resolve to a non-empty value;
|
||||
- each message must have non-empty `role` and `content`;
|
||||
- `response_format.type` is always `json_schema`;
|
||||
- `response_format.json_schema.name` and `schema` must be present;
|
||||
- request schema JSON must be valid JSON.
|
||||
The actual schema is supplied for the call; the fragment above is not a complete
|
||||
request schema. Runtime calls use `correction_set` and `validator_decision_set`
|
||||
identities. The adapter requires a non-empty resolved model, non-empty message
|
||||
roles/content, a schema name, and valid schema JSON. It sends strict mode to the
|
||||
provider; local typed decoding and proposal/decision checks still apply afterward.
|
||||
|
||||
## Response Handling Contract
|
||||
|
||||
Audita expects a successful JSON response with at least one choice and assistant content that can be interpreted as JSON.
|
||||
A successful response must be a JSON envelope with at least one `choices` item.
|
||||
Audita reads the first choice's `message.content`, accepting a non-empty string
|
||||
containing JSON or a non-null raw JSON value that can decode into the expected
|
||||
output type. Empty content, missing choices, and incompatible payloads fail.
|
||||
The adapter also reads optional `model` and token-usage metadata.
|
||||
|
||||
Supported assistant content forms:
|
||||
- string containing JSON;
|
||||
- raw JSON value.
|
||||
|
||||
Audita then decodes the JSON against the expected structured output type.
|
||||
|
||||
Current structured schema identities used by Audita runtime:
|
||||
- `correction_set`
|
||||
- `validator_decision_set`
|
||||
There is no streaming protocol or provider SDK dependency in this integration.
|
||||
Audita uses this chat-completions contract, not other provider API shapes.
|
||||
|
||||
## Retries and Timeouts
|
||||
|
||||
Retry behavior:
|
||||
- default max retries is `3` when unset;
|
||||
- retries apply to retryable transport/decode/server-side errors;
|
||||
- HTTP `429` and `5xx` responses are retryable;
|
||||
- retry stops immediately when context is canceled or deadline expires.
|
||||
The effective adapter retry count is additional attempts after the initial request;
|
||||
see [configuration](../config.md#llm-settings) for the current zero-value behavior.
|
||||
Transport/read errors, malformed response envelopes/content, typed decode failures,
|
||||
HTTP `429`, and server-error responses are retryable. Other non-success HTTP
|
||||
responses and a response missing choices are not retried by the adapter.
|
||||
Cancellation or an expired context stops retrying.
|
||||
|
||||
Timeout behavior:
|
||||
- request timeout is derived from configured LLM timeout settings;
|
||||
- timeout/cancellation propagate through HTTP requests and return nonzero process failures.
|
||||
The configured HTTP timeout applies per request; it is not a whole-process time
|
||||
budget. A parent process can impose its own overall timeout. See
|
||||
[configuration](../config.md#llm-settings) for timeout and retry values and
|
||||
[subprocess guidance](subprocess.md) for parent responsibilities.
|
||||
|
||||
## Error Behavior
|
||||
|
||||
Non-2xx responses fail the request.
|
||||
For non-success responses, error detail comes from provider `error.message`, then
|
||||
a top-level `message`, otherwise the HTTP status and response body. Configured
|
||||
keys are redacted from surfaced errors and LLM diagnostic payloads.
|
||||
|
||||
Error message extraction behavior:
|
||||
- if provider JSON includes `error.message`, Audita surfaces that message;
|
||||
- else if provider JSON includes top-level `message`, Audita surfaces that;
|
||||
- otherwise Audita surfaces status code plus response body text.
|
||||
|
||||
Malformed or incompatible structured responses fail safely and are surfaced as runtime errors or validator/proposal warnings depending on call site.
|
||||
|
||||
## Secret Redaction
|
||||
|
||||
Configured LLM secrets are redacted from:
|
||||
- surfaced adapter/runtime errors;
|
||||
- LLM diagnostics request/response/error artifacts;
|
||||
- effective config/report artifacts that include LLM configuration material.
|
||||
|
||||
Redaction marker:
|
||||
- `[REDACTED]`
|
||||
|
||||
## Compatibility Boundaries
|
||||
|
||||
This integration documentation applies only to the implemented OpenAI-compatible chat completions flow.
|
||||
|
||||
Not part of current behavior:
|
||||
- provider SDK integration;
|
||||
- non-OpenAI-compatible API contracts;
|
||||
- server-side model routing features beyond explicitly configured model/base URL.
|
||||
Malformed structured responses can become section/batch warnings and rejected
|
||||
proposals rather than fatal run errors, depending on the call site. Other adapter
|
||||
failures can terminate processing with partial results. Consult the
|
||||
[report contract](output-files.md#module-outcomes) for those outcomes and
|
||||
[operations](../operations.md#artifact-handling) for handling diagnostic contents.
|
||||
|
||||
186
docs/integrations/output-files.md
Normal file
186
docs/integrations/output-files.md
Normal file
@@ -0,0 +1,186 @@
|
||||
# Output Files and Report Formats
|
||||
|
||||
This document owns externally consumed corrected-transcript and report formats,
|
||||
and describes the correction ledger and utilization diagnostics. It documents
|
||||
current serialized shapes without making every debugging artifact a versioned
|
||||
compatibility promise. Physical filenames, placement, retention, and permissions
|
||||
belong to [operations](../operations.md); invocation and output channels belong to
|
||||
the [CLI](../cli.md#process-output-and-exit-behavior).
|
||||
|
||||
## Corrected Transcript
|
||||
|
||||
The [output setting](../config.md#pipeline-and-output) selects the envelope:
|
||||
|
||||
| Encoder | JSON shape |
|
||||
| --- | --- |
|
||||
| `bare-segments` | Top-level array of segment objects |
|
||||
| `audita-v1` | Object with `schema: "audita-v1"`, `version: "v1"`, and `segments` containing the segment array |
|
||||
|
||||
Both emit these segment fields:
|
||||
|
||||
| Field | Type | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `id` | Integer | Identifier in the normalized transcript |
|
||||
| `speaker` | String | Speaker label |
|
||||
| `start`, `end` | Numbers | Start/end times in seconds |
|
||||
| `text` | String | Text after normalization and accepted corrections |
|
||||
| `categories` | Array of strings, optional | Segment categories; omitted when empty |
|
||||
|
||||
Input IDs and segment boundaries need not survive normalization unchanged;
|
||||
normalization can merge segments and reassign IDs. The output describes the
|
||||
resulting transcript, not a lossless copy of the source JSON. Input acceptance
|
||||
rules belong to the [input-file contract](transcript-glossary-files.md).
|
||||
|
||||
## Process Report
|
||||
|
||||
The report is a JSON object. `report_metadata` identifies its schema separately
|
||||
from the transcript output envelope:
|
||||
|
||||
| Metadata field | Type / current value |
|
||||
| --- | --- |
|
||||
| `report_schema_name` | String: `audita-process-report` |
|
||||
| `report_schema_version` | String: `v1` |
|
||||
| `output_schema` | String identifying the selected transcript encoder |
|
||||
| `config_version` | Optional integer, present when file configuration was loaded |
|
||||
|
||||
The remaining top-level fields are:
|
||||
|
||||
| Field | Type and meaning |
|
||||
| --- | --- |
|
||||
| `phase` | String; currently `default_pipeline` |
|
||||
| `status` | String: `success` or `failed` |
|
||||
| `operation` | String; currently `process` |
|
||||
| `transcript_path`, `glossary_path` | Input path strings |
|
||||
| `output_path` | Optional transcript destination string |
|
||||
| `modules` | Array of configured module keys, including repeats |
|
||||
| `started_at`, `completed_at` | RFC 3339 timestamps, possibly fractional; completion is an optional field populated by completed process runs |
|
||||
| `error_phase`, `error_message` | Optional failure classification and human-readable detail |
|
||||
| `input_segment_count`, `normalized_segment_count` | Optional integer normalization counts |
|
||||
| `normalization_merges`, `normalization_id_reassignments` | Optional integer transformation counts |
|
||||
| `normalization_skipped` | Object of optional integer counters: `different_speakers`, `gap_too_large`, `duration_exceeded`, `token_limit_exceeded`; can be empty |
|
||||
| `chunking` | Optional section summary described below |
|
||||
| `diagnostics` | Optional artifact-path metadata described below |
|
||||
| `modules_summary` | Optional aggregate module outcome |
|
||||
| `module_results` | Optional array of module outcomes |
|
||||
|
||||
Failures can preserve partial module results. Missing summaries indicate unavailable
|
||||
information, not necessarily zero work. Early configuration failures occur before
|
||||
report construction. Report success describes processing results; a later file-write
|
||||
or cleanup error can still cause a nonzero CLI exit. Use the exit result and report
|
||||
together as explained in [subprocess guidance](subprocess.md).
|
||||
|
||||
Error phases identify where processing failed, for example `transcript_read`,
|
||||
`glossary_read`, `transcript_schema`, `glossary_schema`, `chunking`, `runner_setup`,
|
||||
`runner_execution`, `output_schema`, `serialization`, `output_write`, and
|
||||
`stdout_write`. This is not an exhaustive enumeration or a promise that every
|
||||
late failure is represented in a successfully written report. Error messages are
|
||||
human-readable, not structured fields to parse for control flow.
|
||||
|
||||
### Chunking and Diagnostics Metadata
|
||||
|
||||
`chunking` contains integer fields `chunk_count`, `min_estimated_chunk_tokens`,
|
||||
`max_estimated_chunk_tokens`, `total_estimated_transcript_tokens`,
|
||||
`max_section_tokens`, and `min_section_tokens`, plus optional integer
|
||||
`target_sections`. Token counts are estimates.
|
||||
|
||||
`diagnostics` contains optional path strings:
|
||||
|
||||
- `directory_path`
|
||||
- `source_transcript_path`, `parsed_source_transcript_path`, `normalized_transcript_path`
|
||||
- `normalization_summary_path`, `chunking_summary_path`
|
||||
- `utilization_summary_path`, `correction_ledger_path`
|
||||
- `invocation_metadata_path`, `redacted_effective_config_path`, `error_log_path`
|
||||
|
||||
The [operations guide](../operations.md#diagnostics-directory) defines the physical
|
||||
artifacts and path-lifetime caveats.
|
||||
|
||||
### Module Outcomes
|
||||
|
||||
`modules_summary` contains integer `module_count`, `total_applied_changes`, and
|
||||
`total_skipped_changes`, plus optional string `failed_module_instance`.
|
||||
`total_skipped_changes` counts both apply-time skipped changes and validator
|
||||
rejections. Their detailed records are stored in separate arrays.
|
||||
|
||||
Each `module_results` object contains:
|
||||
|
||||
| Field | Type and meaning |
|
||||
| --- | --- |
|
||||
| `module_key`, `module_instance` | Strings; the instance distinguishes repeated occurrences of a configured module |
|
||||
| `replacement_policy` | Optional string describing the module's replacement policy |
|
||||
| `status` | String describing module outcome (`success` or `failed`) |
|
||||
| `proposal_count` | Integer |
|
||||
| `started_at`, `completed_at` | Optional timestamps |
|
||||
| `error_message` | Optional failure detail |
|
||||
| `warnings` | Optional array of warning records |
|
||||
| `validator_decisions` | Optional array of decision records |
|
||||
| `validator_rejected` | Optional array of rejected-correction records |
|
||||
| `applied_changes`, `skipped_changes` | Optional arrays of apply outcomes |
|
||||
|
||||
A validator decision has `validator_name` (string), `proposal_index` (integer),
|
||||
`approved` (boolean), `reason_code` and `message` (strings), and optional
|
||||
`diagnostic_artifact_path` (string).
|
||||
|
||||
Rejected corrections include `validator_name`, `proposal_index`, `module_key`,
|
||||
`module_instance`, `target_segment_id`, `original_text`, `corrected_text`,
|
||||
`reason_code`, and `message`. Applied changes include the same correction identity
|
||||
and text fields without validator/reason/message fields, plus integer
|
||||
`replacement_count`. Skipped changes include correction identity/text fields,
|
||||
`skip_reason`, and `message`.
|
||||
|
||||
A warning has string `scope`, `reason_code`, and `message`; optional integer
|
||||
`section_index` and `batch_index`; and optional string `validator_name` and
|
||||
`diagnostic_artifact_path`. Warnings, skipped changes, and rejected corrections
|
||||
can occur without a failed run. Proposal indexes associate outcomes with their
|
||||
proposal within a module instance; they are not transcript segment IDs.
|
||||
|
||||
## Correction Ledger
|
||||
|
||||
The ledger is an array of correction entries; it can be `null` when no module
|
||||
results are available and empty when modules produced no entries. Unlike the
|
||||
process report, it has no separate schema-version envelope.
|
||||
|
||||
Each entry has required `module_key`, `module_instance`, integer `proposal_index`,
|
||||
and `disposition`. The current dispositions are:
|
||||
|
||||
| Disposition | Meaning |
|
||||
| --- | --- |
|
||||
| `applied` | The proposed change was applied |
|
||||
| `skipped` | The proposal reached application but was skipped |
|
||||
| `rejected` | A validator rejected the proposal |
|
||||
| `failed` | Module-level failure record, not necessarily an individual proposal |
|
||||
|
||||
Optional fields are `run_id`, integer `segment_id`, `original_text`,
|
||||
`proposed_corrected_text`, `applied_corrected_text`, `replacement_policy`,
|
||||
`disposition_reason_code`, and `disposition_message`. Optional validator arrays
|
||||
are `deterministic_validator_decisions` and `llm_validator_decisions`; each record
|
||||
contains string `validator_key`, boolean `approved`, string `reason_code`, and
|
||||
optional string `message`.
|
||||
|
||||
Module failure entries can have the zero-value proposal index and no correction
|
||||
text or segment. Do not interpret them as evidence that proposal zero failed.
|
||||
Empty optional values are omitted. The ledger records disposition, not a replay
|
||||
or resume protocol.
|
||||
|
||||
## Utilization Diagnostics
|
||||
|
||||
This unversioned JSON object describes observations from the runtime, not service
|
||||
limits or performance guarantees:
|
||||
|
||||
- `effective_concurrency`: integer `total_llm`, `proposal_llm`, `validation_llm`.
|
||||
- `run_timing`: millisecond integers `run_wall_time_ms`, `scheduler_queue_wait_ms`,
|
||||
`llm_execution_time_ms`, `deterministic_validation_time_ms`; integer
|
||||
`max_in_flight_llm_calls` and `average_in_flight_llm_calls`.
|
||||
- `llm_calls`: integer `total_proposal_calls` and `total_validation_calls`.
|
||||
- Optional `modules`: records with `module_key`, `module_instance`, integer
|
||||
`proposal_llm_calls` and `validation_llm_calls`, and millisecond integer
|
||||
`scheduler_queue_wait_ms`, `llm_execution_time_ms`,
|
||||
`deterministic_validation_time_ms`, and `module_wall_time_ms`.
|
||||
- Optional `validators`: records with `module_key`, `module_instance`,
|
||||
`validator_key`, boolean `llm_backed`, integer `calls`, and millisecond integer
|
||||
`elapsed_ms`.
|
||||
|
||||
Aggregated execution/wait durations can overlap in concurrent work; they need
|
||||
not sum to wall time. The observed average concurrency is serialized as an integer.
|
||||
Other source snapshots, prompt payloads, and diagnostic files are debugging
|
||||
material; their placement is documented in operations without defining a separate
|
||||
public schema for every file.
|
||||
@@ -1,99 +1,40 @@
|
||||
# Subprocess Integration
|
||||
|
||||
## Scope
|
||||
This guide explains how a parent process can invoke Audita and consume its results.
|
||||
The [CLI reference](../cli.md) owns command syntax, streams, and exit semantics;
|
||||
[configuration](../config.md) owns discovery and precedence.
|
||||
|
||||
This document describes how a parent process should invoke Audita as a subprocess.
|
||||
## Invocation
|
||||
|
||||
It covers:
|
||||
- invocation shape;
|
||||
- stdout/stderr behavior;
|
||||
- output/report file behavior;
|
||||
- diagnostics and exit behavior.
|
||||
Use an argument array with the transcript path and required glossary argument,
|
||||
following the [process examples](../cli.md#process-examples). For machine workflows,
|
||||
select explicit transcript and report destinations rather than depending on the
|
||||
working directory or run-directory retention. Supplying a configuration path also
|
||||
makes file selection predictable; inherited environment overrides still apply.
|
||||
|
||||
For full CLI and config references, see [`docs/cli.md`](../cli.md) and [`docs/config.md`](../config.md).
|
||||
Resolve the executable deliberately and set the child's working directory so
|
||||
relative input/output paths and reported artifact paths have a known base.
|
||||
Read stdout and stderr concurrently to avoid pipe blocking, and impose timeout
|
||||
or cancellation in the parent process when needed.
|
||||
|
||||
## Recommended Invocation
|
||||
## Result Handling
|
||||
|
||||
Use explicit output and report paths for machine workflows:
|
||||
Interpret the exit result according to the
|
||||
[CLI output and exit contract](../cli.md#process-output-and-exit-behavior).
|
||||
Consume transcript/report data through the
|
||||
[output-file contract](output-files.md), and treat stderr as human-readable detail.
|
||||
The report supplements the exit result: it may be absent after early failures,
|
||||
and output may have been written before a later reporting or cleanup failure.
|
||||
|
||||
```sh
|
||||
audita process <transcript.json> \
|
||||
--glossary <glossary.yaml> \
|
||||
--output <output-transcript.json> \
|
||||
--report-json <report.json>
|
||||
```
|
||||
If processing fails, preserve available evidence and follow
|
||||
[operations recovery](../operations.md#failure-inspection-and-recovery).
|
||||
For reports retained outside the run directory, account for the
|
||||
[artifact lifecycle](../operations.md#retention-behavior) before dereferencing
|
||||
reported paths. Neither the existence of an output file nor a diagnostics path
|
||||
alone establishes successful completion.
|
||||
|
||||
Optional commonly used flags:
|
||||
- `--config <path>`
|
||||
- `--output-schema <bare-segments|audita-v1>`
|
||||
- `--work-dir <dir>`
|
||||
- `--work-dir-retention <always|auto|never>`
|
||||
- `--transcript-description <text>`
|
||||
## Related Contracts
|
||||
|
||||
## Stdout Contract
|
||||
|
||||
On success:
|
||||
- with `--output`: stdout is expected to be empty;
|
||||
- without `--output`: stdout contains transcript JSON only.
|
||||
|
||||
`--report-json` output is never written to stdout.
|
||||
|
||||
## Stderr Contract
|
||||
|
||||
Stderr is human-readable status/error output.
|
||||
|
||||
On failures:
|
||||
- stderr includes a concise top-level error;
|
||||
- when diagnostics are initialized, stderr includes diagnostics directory path.
|
||||
|
||||
Do not treat stderr as a machine-stable JSON channel.
|
||||
|
||||
## Output and Report File Contract
|
||||
|
||||
Transcript output:
|
||||
- `--output` writes corrected transcript JSON to the provided path;
|
||||
- output write failures return nonzero.
|
||||
|
||||
Report output:
|
||||
- `--report-json` writes machine-readable process report JSON to the provided path;
|
||||
- run diagnostics also attempt to write their own `report.json`;
|
||||
- report write failures return nonzero;
|
||||
- on failure paths, report writing is best-effort and does not mask the primary run error.
|
||||
|
||||
## Diagnostics Contract
|
||||
|
||||
When run-directory initialization succeeds, per-run diagnostics artifacts are written under the configured work directory.
|
||||
|
||||
Typical artifacts include:
|
||||
- `source-transcript.json`
|
||||
- `source-transcript-parsed.json`
|
||||
- `normalized-transcript.json`
|
||||
- `normalization-summary.json`
|
||||
- `chunking-summary.json`
|
||||
- `invocation.json`
|
||||
- `effective-config.json`
|
||||
- `utilization-diagnostics.json`
|
||||
- `correction-ledger.json`
|
||||
- `report.json`
|
||||
- `error.log` (failure)
|
||||
|
||||
Retention behavior is controlled by `--work-dir-retention` / config.
|
||||
|
||||
## Exit Behavior
|
||||
|
||||
Exit codes:
|
||||
- `0`: success;
|
||||
- `1`: runtime processing/output/report failure;
|
||||
- `2`: CLI usage or configuration input error.
|
||||
|
||||
Treat any nonzero as subprocess failure.
|
||||
|
||||
## Parent-Process Guidance
|
||||
|
||||
For reliable orchestration:
|
||||
- read stdout and stderr concurrently to avoid pipe blocking;
|
||||
- prefer `--output` and `--report-json` for machine parsing;
|
||||
- use timeout/cancellation in the parent process;
|
||||
- inspect diagnostics path and `report.json`/`error.log` on failure.
|
||||
|
||||
For input file contracts, see [`docs/integrations/transcript-glossary-files.md`](transcript-glossary-files.md).
|
||||
- [Transcript and glossary inputs](transcript-glossary-files.md)
|
||||
- [LLM endpoint compatibility](openai-compatible-llm.md)
|
||||
- [Configuration inspection commands](../cli.md#configuration-commands)
|
||||
|
||||
@@ -1,98 +1,50 @@
|
||||
# Transcript and Glossary File Integration
|
||||
# Transcript and Glossary Input Files
|
||||
|
||||
## Scope
|
||||
|
||||
This document defines the input file contracts for:
|
||||
- transcript JSON;
|
||||
- glossary YAML.
|
||||
|
||||
These files are loaded and validated before processing begins.
|
||||
This document owns accepted input formats. Use the [CLI reference](../cli.md) for
|
||||
invocation and error exit semantics, [operations](../operations.md) for artifact
|
||||
handling, and [output files](output-files.md) for corrected-transcript serialization.
|
||||
|
||||
## Transcript JSON Contract
|
||||
|
||||
Audita accepts either top-level shape:
|
||||
- JSON array of segments; or
|
||||
- JSON object with a `segments` array.
|
||||
Accepted top-level shapes are an array of segments or an object containing a
|
||||
`segments` array. The array must contain at least one segment. Unknown object
|
||||
fields are ignored by the current parser; an input envelope does not require a
|
||||
schema/version marker.
|
||||
|
||||
Segment fields:
|
||||
- `id` (optional integer in source form);
|
||||
- `speaker` (required non-empty string);
|
||||
- `start` (required finite non-negative number);
|
||||
- `end` (required finite non-negative number, `>= start`);
|
||||
- `text` (required non-empty string);
|
||||
- `categories` (optional string array; entries must be non-empty).
|
||||
| Segment field | Type and validation |
|
||||
| --- | --- |
|
||||
| `id` | Optional integer (or null). Supplied IDs must be unique; missing IDs are accepted. |
|
||||
| `speaker` | Non-empty string |
|
||||
| `start`, `end` | Numeric seconds, finite and non-negative; end must be at least start |
|
||||
| `text` | Non-empty string |
|
||||
| `categories` | Optional array of non-empty strings |
|
||||
|
||||
Additional rules:
|
||||
- transcript must contain at least one segment;
|
||||
- duplicate segment IDs are rejected when IDs are present.
|
||||
Supply explicit start and end times for meaningful timing. The current decoder
|
||||
uses zero for missing or null timestamps rather than enforcing their presence.
|
||||
String validation checks empty values, not whitespace trimming. Normalization can
|
||||
merge segments and reassign IDs, so source IDs need not match output IDs.
|
||||
|
||||
Example (`examples/tiny-transcript.json`):
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"speaker": "A",
|
||||
"start": 0.0,
|
||||
"end": 1.2,
|
||||
"text": "hello world"
|
||||
}
|
||||
]
|
||||
```
|
||||
The [tiny transcript](../../examples/tiny-transcript.json) is a complete copyable
|
||||
input. Invalid JSON, unsupported top-level shapes, empty transcripts, invalid field
|
||||
types, duplicate supplied IDs, and the validation failures above are rejected.
|
||||
|
||||
## Glossary YAML Contract
|
||||
|
||||
Audita expects top-level `glossary` list entries.
|
||||
The top-level object contains a `glossary` list with at least one entry.
|
||||
|
||||
Entry fields:
|
||||
- `name` (required non-empty string);
|
||||
- `category` (required non-empty string);
|
||||
- `summary` (required non-empty string);
|
||||
- `aliases` (optional list of strings; entries must be non-empty);
|
||||
- `plural` (optional string).
|
||||
| Entry field | Type and validation |
|
||||
| --- | --- |
|
||||
| `name` | Non-empty string |
|
||||
| `category` | Non-empty string |
|
||||
| `summary` | Non-empty string |
|
||||
| `aliases` | Optional list of non-empty strings |
|
||||
| `plural` | Optional string |
|
||||
|
||||
Additional rules:
|
||||
- glossary must contain at least one entry.
|
||||
The parser checks empty strings without trimming them. Unknown YAML fields are
|
||||
ignored by the glossary parser, unlike the stricter
|
||||
[configuration-file schema](../config.md#file-schema).
|
||||
|
||||
Example (`examples/tiny-glossary.yaml`):
|
||||
|
||||
```yaml
|
||||
glossary:
|
||||
- name: Audita
|
||||
aliases:
|
||||
- audita
|
||||
category: product
|
||||
summary: The Audita transcript correction CLI.
|
||||
```
|
||||
|
||||
## Validation Failure Behavior
|
||||
|
||||
Representative transcript validation failures:
|
||||
- invalid JSON;
|
||||
- unsupported top-level shape;
|
||||
- empty `speaker` or `text`;
|
||||
- invalid times (`NaN`, `Inf`, negative, or `end < start`);
|
||||
- duplicate IDs;
|
||||
- empty transcript array.
|
||||
|
||||
Representative glossary validation failures:
|
||||
- invalid YAML;
|
||||
- empty or missing glossary entries;
|
||||
- missing required entry fields;
|
||||
- empty alias values.
|
||||
|
||||
These failures surface as schema errors and the process exits nonzero.
|
||||
|
||||
## CLI Usage
|
||||
|
||||
Minimal invocation:
|
||||
|
||||
```sh
|
||||
audita process ./transcript.json --glossary ./glossary.yaml --output ./corrected.json
|
||||
```
|
||||
|
||||
See also:
|
||||
- [`docs/cli.md`](../cli.md)
|
||||
- [`docs/config.md`](../config.md)
|
||||
- [`examples/tiny-transcript.json`](../../examples/tiny-transcript.json)
|
||||
- [`examples/tiny-glossary.yaml`](../../examples/tiny-glossary.yaml)
|
||||
The [tiny glossary](../../examples/tiny-glossary.yaml) is a complete copyable input.
|
||||
Malformed YAML, an empty/missing glossary, invalid shapes, missing required text,
|
||||
and empty aliases are rejected. These formats describe input acceptance, not
|
||||
additional vocabulary restrictions.
|
||||
|
||||
64
docs/internal/cli-config.md
Normal file
64
docs/internal/cli-config.md
Normal file
@@ -0,0 +1,64 @@
|
||||
# Configuration and CLI Change Recipes
|
||||
|
||||
This document owns contributor editing and validation steps for configuration,
|
||||
CLI wiring, and maintained examples. Definitions of behavior belong to the
|
||||
[configuration](../config.md) and [CLI](../cli.md) contracts.
|
||||
|
||||
## Changing Configuration
|
||||
|
||||
1. Update fields, defaults, loading, and validation in `internal/core/config`.
|
||||
2. Update file, environment, and CLI mapping as applicable, including CLI flag
|
||||
extraction. Preserve the source-specific behavior defined in the
|
||||
[configuration contract](../config.md#effective-precedence), including the
|
||||
distinct validation and effective-config commands.
|
||||
3. Update redaction when secrets are involved.
|
||||
4. Review configuration tests and related CLI tests for precedence, validation,
|
||||
and source-specific behavior. Add or adjust tests for the changed contract
|
||||
according to the [testing policy](../policy/testing.md).
|
||||
5. Update the configuration reference and affected [examples](../../examples/).
|
||||
|
||||
## Changing CLI Behavior
|
||||
|
||||
1. Implement parsing and wiring in `internal/cli`.
|
||||
2. Preserve [stream and exit contracts](../cli.md) unless a change is intentional
|
||||
and documented. Review [subprocess integration](../integrations/subprocess.md)
|
||||
when changing output, error, or report handling.
|
||||
3. Review `internal/cli` tests and `cmd/audita` subprocess tests; update coverage
|
||||
for the changed behavior.
|
||||
4. Update the CLI reference and affected integration guidance.
|
||||
|
||||
## Validating Examples
|
||||
|
||||
Keep complete copyable files in [examples](../../examples/) and link to them from
|
||||
their canonical contract documentation. Keep them valid and secret-free.
|
||||
From the repository root, validate the maintained configuration examples:
|
||||
|
||||
```sh
|
||||
go run ./cmd/audita config validate --config examples/minimal-config.yml
|
||||
go run ./cmd/audita config validate --config examples/production-config.yml
|
||||
go run ./cmd/audita config print-effective --config examples/minimal-config.yml
|
||||
```
|
||||
|
||||
The effective-config command incorporates environment settings; use a controlled
|
||||
environment when comparing its results. Check affected CLI examples against the
|
||||
parser and use schema tests for maintained transcript/glossary inputs. Example
|
||||
validation should not invoke a live LLM merely to check documentation.
|
||||
|
||||
## Runtime Change Checks
|
||||
|
||||
Select checks according to the [testing policy](../policy/testing.md):
|
||||
|
||||
- Configuration or CLI changes: inspect `internal/core/config` tests,
|
||||
`internal/cli` tests, and `cmd/audita` integration tests.
|
||||
- Process/output changes: verify stdout, stderr, exit, output-file, and report
|
||||
behavior against the [CLI contract](../cli.md) and
|
||||
[subprocess guidance](../integrations/subprocess.md), including failures.
|
||||
- Report or diagnostics changes: review [report assembly tests and redaction
|
||||
checks](diagnostics-reporting.md#key-tests), metadata fields, and artifact paths.
|
||||
- Schema or serialization changes: review [output-schema tests](output-schemas.md#key-tests)
|
||||
and `internal/core/schema` tests, including supported formats and rejection of
|
||||
unsupported schemas.
|
||||
- Changes affecting assembled output: inspect CLI parity and release fixtures.
|
||||
|
||||
Use the [contributor guide](../development.md#baseline-validation) for baseline
|
||||
commands. These recipes supplement, rather than duplicate, public contracts.
|
||||
@@ -1,74 +1,69 @@
|
||||
# Audita Diagnostics and Reporting
|
||||
# Diagnostics and Report Assembly
|
||||
|
||||
## Scope
|
||||
|
||||
This document describes diagnostics artifacts, process report mapping, and correction ledger generation.
|
||||
This document owns implementation flow and collaborators. Physical artifacts and
|
||||
retention belong to [operations](../operations.md). Serialized reports, ledger
|
||||
records, and utilization shapes belong to the
|
||||
[output-file contract](../integrations/output-files.md).
|
||||
|
||||
## Run Directory Ownership
|
||||
## Run Directory and Artifact Writers
|
||||
|
||||
`internal/core/diagnostics` owns run-directory creation, artifact writes, and retention decisions.
|
||||
`internal/core/diagnostics` creates run directories, writes snapshots and metadata,
|
||||
and evaluates retention. CLI orchestration supplies the selected configuration,
|
||||
paths, run outcome, and correction-presence information. Artifact names are
|
||||
centralized in diagnostics; their operator-facing inventory is in
|
||||
[operations](../operations.md#diagnostics-directory).
|
||||
|
||||
Stable artifact names include:
|
||||
- `source-transcript.json`
|
||||
- `source-transcript-parsed.json`
|
||||
- `normalized-transcript.json`
|
||||
- `normalization-summary.json`
|
||||
- `chunking-summary.json`
|
||||
- `utilization-diagnostics.json`
|
||||
- `correction-ledger.json`
|
||||
- `invocation.json`
|
||||
- `effective-config.json`
|
||||
- `report.json`
|
||||
- `error.log` (failure)
|
||||
Snapshot writes are attempted at the relevant processing phases. Several diagnostic
|
||||
writes are best-effort, so report metadata can identify a location without proving
|
||||
that a file was written. The CLI distinguishes primary processing errors from
|
||||
secondary diagnostic/report write errors.
|
||||
|
||||
## Process Report Mapping
|
||||
|
||||
`internal/framework/processreport` maps runner/CLI execution facts into `reporting.ProcessReport`.
|
||||
`internal/framework/processreport.Build` maps invocation facts, normalization and
|
||||
chunking summaries, and runner output into `reporting.ProcessReport`. Core reporting
|
||||
owns the Go data structures and JSON writer. The builder attaches diagnostic
|
||||
metadata when a run-directory path is available.
|
||||
|
||||
Report metadata fields include:
|
||||
- `report_schema_name` (`audita-process-report`)
|
||||
- `report_schema_version` (`v1`)
|
||||
- `output_schema`
|
||||
- `config_version` (when file config exists)
|
||||
Module mapping copies outcomes from runner results, collects summary counts, and
|
||||
preserves available partial module results on failure. The CLI failure path does
|
||||
not pass normalization or chunking summaries to report construction even when
|
||||
some earlier processing completed. Consumer interpretation belongs to the
|
||||
[process-report contract](../integrations/output-files.md#process-report).
|
||||
|
||||
The report includes:
|
||||
- top-level status/error phase/error message;
|
||||
- normalization/chunking summaries;
|
||||
- diagnostics metadata paths;
|
||||
- per-module results and module summary.
|
||||
## Correction Ledger and Utilization
|
||||
|
||||
## Correction Ledger
|
||||
`BuildCorrectionLedger` flattens module changes, rejections, and failures into
|
||||
ledger entries. It associates validator records by proposal index within each
|
||||
module, separates execution classes through validator metadata, and sorts entries
|
||||
by module instance, proposal index, and disposition.
|
||||
|
||||
`internal/framework/processreport/BuildCorrectionLedger` flattens run results into `correction-ledger.json` entries.
|
||||
Runner instrumentation collects utilization; CLI orchestration writes it when
|
||||
run output is available. Report assembly does not independently measure calls.
|
||||
For the serialized forms see [ledger](../integrations/output-files.md#correction-ledger)
|
||||
and [utilization](../integrations/output-files.md#utilization-diagnostics).
|
||||
|
||||
Dispositions:
|
||||
- `applied`
|
||||
- `skipped`
|
||||
- `rejected`
|
||||
- `failed`
|
||||
## CLI Coordination and Retention
|
||||
|
||||
Validator decisions are split into deterministic and LLM-backed groups using validator metadata classification.
|
||||
CLI orchestration writes caller-selected reports and run-local artifacts before
|
||||
asking diagnostics to apply retention. `HasSkippedCorrections` derives the
|
||||
retention input from runner outcomes. Keep the exact lifecycle definition in
|
||||
[operations](../operations.md#retention-behavior), not in another internal table.
|
||||
|
||||
## Report Write Paths
|
||||
|
||||
- run directory always attempts to write `report.json` when possible;
|
||||
- optional `--report-json` writes an external report file;
|
||||
- on failure paths, report writing is best-effort and does not mask primary run errors.
|
||||
|
||||
## Retention Interaction
|
||||
|
||||
Current retention behavior:
|
||||
- failed runs are retained;
|
||||
- `always` keeps successful runs;
|
||||
- `auto` removes only clean successful runs;
|
||||
- `never` currently retains successful runs in current implementation.
|
||||
On a processing failure, secondary report-write errors do not replace the original
|
||||
error. On the success path, an external report-write failure returns before the
|
||||
run-local report write; retention errors occur later. This ordering explains why
|
||||
artifacts and process exit status must be assessed together.
|
||||
|
||||
## Redaction
|
||||
|
||||
Redacted data expectations:
|
||||
- effective config artifact uses config redaction;
|
||||
- diagnostics payloads and surfaced errors use LLM secret redaction;
|
||||
- reports should not include raw API key values.
|
||||
Effective configuration uses config redaction. LLM diagnostic and error paths
|
||||
use `llm.ConfiguredSecrets` and the runtime's redaction helpers. Keep these separate
|
||||
mechanisms aligned with the [architecture requirement](../policy/architecture.md#errors-and-secret-handling).
|
||||
Operational handling of transcript-bearing artifacts belongs to
|
||||
[operations](../operations.md#artifact-handling).
|
||||
|
||||
## Key Tests
|
||||
|
||||
|
||||
@@ -11,44 +11,36 @@ All runtime LLM calls go through `contracts.StructuredLLMClient`.
|
||||
Primary adapter:
|
||||
- `internal/framework/llm/OpenAICompatibleClient`
|
||||
|
||||
## Request/Response Behavior
|
||||
## Adapter and Response Schema Registry
|
||||
|
||||
The OpenAI-compatible adapter sends chat completions requests with:
|
||||
- model;
|
||||
- messages;
|
||||
- `response_format.type = json_schema`;
|
||||
- strict schema envelope (`name`, `schema`, `strict=true`).
|
||||
The adapter implements the [LLM wire contract](../integrations/openai-compatible-llm.md).
|
||||
It maps internal structured requests to HTTP messages and decodes provider content
|
||||
into the supplied Go output target. Endpoint, authentication, response shapes, and
|
||||
retry conditions are defined in that contract rather than repeated here.
|
||||
|
||||
The response is decoded into the requested structured output target.
|
||||
`internal/framework/responseschema` stores the per-call response schema definitions
|
||||
and diagnostic metadata. Proposal generation and validation choose their schema
|
||||
before invoking the client. Provider strict mode does not replace local checks.
|
||||
|
||||
## Response Schema Registry
|
||||
## Error Classification
|
||||
|
||||
Structured response schemas are registered in `internal/framework/responseschema`:
|
||||
- `correction_set`
|
||||
- `validator_decision_set`
|
||||
|
||||
Each schema includes stable diagnostics metadata (`id`, `version`, `name`, `sha256`).
|
||||
|
||||
## Retries and Error Handling
|
||||
|
||||
Adapter retries apply to retryable conditions (for example transport/decoding/retryable status classes) up to configured `max_retries`.
|
||||
|
||||
Errors are sanitized to redact configured API-key values before surfacing.
|
||||
|
||||
Malformed structured output detection is shared through `internal/framework/structuredoutput` and is used by:
|
||||
- proposal generation;
|
||||
- LLM-backed validators.
|
||||
The adapter marks retryable errors internally and uses context state and attempt
|
||||
count to decide whether to retry. It sanitizes errors before returning them.
|
||||
`internal/framework/structuredoutput` classifies malformed output for proposal
|
||||
and validator call sites, which can turn those errors into warnings and rejected
|
||||
section/batch work. Other errors propagate through the runner.
|
||||
|
||||
## Scheduling and Concurrency
|
||||
|
||||
`internal/framework/llm/Scheduler` provides FIFO, context-aware permit gating.
|
||||
|
||||
Runner composes scheduler limits across:
|
||||
- total LLM concurrency;
|
||||
- proposal LLM concurrency;
|
||||
- validation LLM concurrency.
|
||||
CLI orchestration composes global and stage schedulers; runner instrumentation
|
||||
measures them. Limit definitions and inheritance belong to
|
||||
[configuration](../config.md#concurrency).
|
||||
|
||||
Scheduler release is guarded to avoid permit leaks on cancellation/error.
|
||||
Scheduler release is guarded to avoid permit leaks on cancellation/error. Runner
|
||||
collects utilization observations; their serialized shape is defined by the
|
||||
[output-file contract](../integrations/output-files.md#utilization-diagnostics).
|
||||
|
||||
## Diagnostics and Redaction
|
||||
|
||||
@@ -58,6 +50,14 @@ Configured secrets are derived from `llm.ConfiguredSecrets(cfg)` and redacted fr
|
||||
- diagnostics payloads;
|
||||
- surfaced runtime/adapter errors.
|
||||
|
||||
## Changing LLM Integration or Scheduling
|
||||
|
||||
Review client, scheduler, redaction, and structured-output malformed-response
|
||||
coverage alongside runner orchestration and utilization tests. Use the
|
||||
[testing policy](../policy/testing.md) to select sufficient checks; preserve the
|
||||
[LLM integration contract](../integrations/openai-compatible-llm.md) and
|
||||
[architecture invariants](../policy/architecture.md#correction-and-concurrency-invariants).
|
||||
|
||||
## Key Tests
|
||||
|
||||
- `internal/framework/llm/openai_compatible_client_test.go`
|
||||
@@ -65,3 +65,4 @@ Configured secrets are derived from `llm.ConfiguredSecrets(cfg)` and redacted fr
|
||||
- `internal/framework/llm/diagnostics_test.go`
|
||||
- `internal/framework/responseschema/registry_test.go`
|
||||
- `internal/framework/structuredoutput/malformed_test.go`
|
||||
- `internal/framework/runner/*_test.go` (orchestration and utilization)
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
|
||||
## Scope
|
||||
|
||||
This document covers module contracts and built-in module packages.
|
||||
This document covers internal module contracts and packages. User-selectable keys
|
||||
and the default sequence belong to [configuration](../config.md#pipeline-and-output).
|
||||
|
||||
## Module Contract
|
||||
|
||||
@@ -46,9 +47,26 @@ Module packages call the built-in chain resolver at construction.
|
||||
|
||||
## Failure and Warning Behavior
|
||||
|
||||
- module setup failures surface as `runner_setup` or module setup errors;
|
||||
- module runtime failures surface as `runner_execution` with partial module results preserved;
|
||||
- malformed structured proposal payloads are downgraded to warnings and section-level proposal rejection.
|
||||
Module construction failures propagate to orchestration; execution failures can
|
||||
preserve partial module results. Malformed structured proposal payloads become
|
||||
warnings and section-level rejection. Consumer-visible outcomes are defined by
|
||||
the [report contract](../integrations/output-files.md#module-outcomes).
|
||||
|
||||
## Changing Modules
|
||||
|
||||
1. Add or update the narrowly scoped package under `internal/modules/<module_key>`.
|
||||
2. Keep module-specific prompt ownership in the module and `internal/prompts`;
|
||||
see [prompt rendering and metadata](prompts.md).
|
||||
3. Wire module registration and catalog resolution through framework module
|
||||
registration and the core module catalog. Update selectable keys in the
|
||||
[configuration contract](../config.md) when they change.
|
||||
4. Verify replacement policy and validator-chain selection against the
|
||||
[architecture requirements](../policy/architecture.md#correction-and-concurrency-invariants).
|
||||
5. Review module, prompt-rendering, and proposal-generation tests. Inspect parity
|
||||
and release fixtures when public output changes, choosing sufficient coverage
|
||||
under the [testing policy](../policy/testing.md).
|
||||
6. Update this document for implementation changes and the appropriate public
|
||||
contract owner for externally visible changes.
|
||||
|
||||
## Key Tests
|
||||
|
||||
|
||||
@@ -1,43 +1,31 @@
|
||||
# Audita Output Schemas
|
||||
# Output Encoding Internals
|
||||
|
||||
## Scope
|
||||
## Scope and Ownership
|
||||
|
||||
This document describes the implemented transcript output schema registry.
|
||||
`internal/core/outputschema` owns encoder registration and resolution. External
|
||||
JSON envelopes and segment fields are defined by the
|
||||
[output-file contract](../integrations/output-files.md#corrected-transcript);
|
||||
selectable values and defaults belong to
|
||||
[configuration](../config.md#pipeline-and-output).
|
||||
|
||||
## Registry Ownership
|
||||
## Resolution and Encoding
|
||||
|
||||
Output schema registry is owned by `internal/core/outputschema`.
|
||||
Configuration validation checks registry support. CLI orchestration resolves the
|
||||
selected encoder, passes the final transcript to it, and routes the encoded bytes
|
||||
to the destination defined by the [CLI contract](../cli.md#process-output-and-exit-behavior).
|
||||
An unsupported key fails validation or resolution rather than selecting an
|
||||
implicit fallback.
|
||||
|
||||
Supported schema keys:
|
||||
- `bare-segments`
|
||||
- `audita-v1`
|
||||
The registry delegates bare segment serialization to the transcript schema
|
||||
package and uses an envelope encoder for the versioned form. Report construction
|
||||
receives the selected key separately; see [report assembly](diagnostics-reporting.md).
|
||||
|
||||
## Schemas
|
||||
## Changing an Encoder
|
||||
|
||||
`bare-segments`:
|
||||
- top-level JSON array of transcript segments.
|
||||
|
||||
`audita-v1`:
|
||||
- top-level JSON object with:
|
||||
- `schema: "audita-v1"`
|
||||
- `version: "v1"`
|
||||
- `segments: [...]`
|
||||
|
||||
Segment fields include `id`, `speaker`, `start`, `end`, `text`, and optional `categories`.
|
||||
|
||||
## Validation and Resolution
|
||||
|
||||
Config validation and runtime resolution both reject unsupported schema keys.
|
||||
|
||||
Unknown schema keys fail with `unsupported output schema` before output emission.
|
||||
|
||||
## Output Emission
|
||||
|
||||
The selected schema is used by `audita process` when writing:
|
||||
- output file (`--output`) or
|
||||
- stdout (when no `--output`).
|
||||
|
||||
Report metadata records selected `output_schema`.
|
||||
Update registration, configuration validation, and serialization together. Inspect
|
||||
round-trip and shape tests, then update the external output-file contract if the
|
||||
serialized form changes. Keep internal helper changes separate from decisions
|
||||
about externally visible compatibility.
|
||||
|
||||
## Key Tests
|
||||
|
||||
|
||||
@@ -2,17 +2,15 @@
|
||||
|
||||
## Scope
|
||||
|
||||
This document is the internal architecture entry point for developers and coding agents.
|
||||
|
||||
It summarizes:
|
||||
- package boundaries;
|
||||
- the main `process` execution path;
|
||||
- where to add new code safely.
|
||||
This document owns the implemented component map. For normative boundaries, see
|
||||
[architecture policy](../policy/architecture.md); for task-specific reading and
|
||||
baseline commands, see the [contributor guide](../development.md).
|
||||
|
||||
## Package Map
|
||||
|
||||
CLI and command orchestration:
|
||||
- `internal/cli`
|
||||
Executable and command orchestration:
|
||||
- `cmd/audita`: executable entry point.
|
||||
- `internal/cli`: argument parsing, config commands, streams, filesystem wiring, and process orchestration.
|
||||
|
||||
Core deterministic components:
|
||||
- `internal/core/config`
|
||||
@@ -43,43 +41,47 @@ Domain implementations:
|
||||
- `internal/validators/*`
|
||||
- `internal/prompts`
|
||||
|
||||
Supporting areas:
|
||||
- `docs/`: canonical documentation.
|
||||
- `examples/`: maintained copyable inputs and configuration.
|
||||
|
||||
## Current Scope
|
||||
|
||||
Runtime state is held in memory plus per-run diagnostics artifacts. Audita does
|
||||
not currently implement a plugin framework, generic workflow engine,
|
||||
resume/checkpoint/manifest system, remote storage, or multi-process service mode.
|
||||
Its provider adapter is OpenAI-compatible HTTP behind the structured LLM client
|
||||
contract. These are current implementation boundaries, not bans on future work.
|
||||
|
||||
## Main Execution Path (`audita process`)
|
||||
|
||||
High-level flow:
|
||||
1. CLI loads effective config and validates CLI requirements.
|
||||
2. Run directory is created and invocation/effective config artifacts are written.
|
||||
3. Transcript/glossary files are loaded and parsed.
|
||||
4. Transcript is normalized and chunked.
|
||||
5. `runner.Run` executes configured module instances.
|
||||
6. Proposals are validated, applied deterministically, and serialized in selected output schema.
|
||||
7. Process report, utilization diagnostics, correction ledger, and retention decisions are finalized.
|
||||
The CLI prepares input and configuration, invokes the runner, and writes results.
|
||||
The [pipeline](pipeline.md) owns detailed state transitions; the
|
||||
[operations guide](../operations.md) owns the operator workflow and physical
|
||||
artifact lifecycle.
|
||||
|
||||
## Boundary Summary
|
||||
## Component Responsibilities
|
||||
|
||||
- `internal/core/*` owns deterministic, reusable logic and persistence-independent rules.
|
||||
- `internal/framework/*` owns orchestration contracts and reusable runtime plumbing.
|
||||
- `internal/modules/*` owns module-specific proposal behavior and prompt usage.
|
||||
- `internal/validators/*` owns validator composition and built-in chain assembly.
|
||||
- `internal/prompts` owns embedded prompt assets and metadata registry.
|
||||
Core components implement configuration defaults/loading/validation/redaction,
|
||||
transcript and glossary schemas, normalization and chunking, output encoding,
|
||||
diagnostics artifact helpers, and public report structures.
|
||||
|
||||
## Where To Add New Code
|
||||
Framework components implement module and validator contracts, proposal
|
||||
generation and application, prompt context, runner orchestration, LLM scheduling
|
||||
and transport, response schemas, and process-report/correction-ledger assembly.
|
||||
Domain packages provide correction modules, built-in validator chains and
|
||||
execution metadata, and embedded prompt assets.
|
||||
|
||||
Add config fields:
|
||||
- `internal/core/config`
|
||||
These describe the implementation; dependency and ownership requirements belong
|
||||
to the [architecture policy](../policy/architecture.md#ownership-and-dependency-direction).
|
||||
|
||||
Add module behavior:
|
||||
- one package under `internal/modules/<module_key>`
|
||||
- registration/wiring through `internal/framework/modules` and config module list
|
||||
## Change Recipes
|
||||
|
||||
Add validators:
|
||||
- implementation under `internal/validators/<validator_key>`
|
||||
- registry/chain wiring in `internal/validators`
|
||||
|
||||
Add runtime orchestration behavior:
|
||||
- `internal/framework/*` (runner/proposal/validator/LLM plumbing)
|
||||
|
||||
Add CLI surface:
|
||||
- `internal/cli`
|
||||
- [Configuration, CLI, and example changes](cli-config.md)
|
||||
- [Module changes](modules.md#changing-modules)
|
||||
- [Validator changes](validators.md#changing-validators)
|
||||
- Runtime orchestration changes: use [pipeline](pipeline.md), [LLM runtime](llm-runtime.md),
|
||||
or [diagnostics/reporting](diagnostics-reporting.md) and their test references.
|
||||
|
||||
## Related Internal Docs
|
||||
|
||||
|
||||
@@ -2,7 +2,10 @@
|
||||
|
||||
## Scope
|
||||
|
||||
This document describes the implemented `audita process` pipeline.
|
||||
This document describes processing collaborators and state transitions. External
|
||||
input definitions belong to the [input-file contract](../integrations/transcript-glossary-files.md),
|
||||
settings to [configuration](../config.md), and serialized results to the
|
||||
[output-file contract](../integrations/output-files.md).
|
||||
|
||||
## Inputs
|
||||
|
||||
@@ -13,35 +16,23 @@ Pipeline inputs are:
|
||||
|
||||
## Pipeline Phases
|
||||
|
||||
1. Input loading and schema validation
|
||||
- transcript and glossary files are read and parsed.
|
||||
- schema failures stop the run with `transcript_schema` or `glossary_schema`.
|
||||
|
||||
2. Normalization
|
||||
- canonical transcript segments are normalized by configured gap/duration/token settings.
|
||||
- normalization summary artifacts are written.
|
||||
|
||||
3. Chunking
|
||||
- normalized transcript is chunked with configured max/min tokens and target sections.
|
||||
|
||||
4. Module proposal generation
|
||||
- runner executes configured module instances in sequence.
|
||||
- each module proposes corrections per section.
|
||||
- per-section proposal generation can run concurrently.
|
||||
|
||||
5. Validator filtering
|
||||
- validators run on candidate proposals before apply.
|
||||
- deterministic validators run before LLM-backed validators.
|
||||
- LLM validator inputs are batched by max prompt token limit.
|
||||
|
||||
6. Deterministic apply
|
||||
- approved proposals are applied via replacement policy.
|
||||
- applied/skipped/rejected outcomes are recorded.
|
||||
|
||||
7. Output and reporting
|
||||
- final transcript is serialized with selected output schema.
|
||||
- report, utilization diagnostics, and correction ledger are written.
|
||||
- retention policy is applied to run directory.
|
||||
1. **Input loading:** CLI filesystem adapters read inputs; schema parsing produces
|
||||
source transcript and glossary objects. Source segments are converted to the
|
||||
canonical transcript shape before normalization.
|
||||
2. **Normalization:** the normalizer sorts and merges segments using its configured
|
||||
time/token rules, reassigns IDs, and produces summary counters.
|
||||
3. **Chunking:** the chunker partitions the normalized transcript and returns
|
||||
sections and token estimates, or fails when constraints are impossible.
|
||||
4. **Proposal generation:** the runner executes configured module instances in
|
||||
sequence. Sections can generate proposals concurrently; results are collected
|
||||
for deterministic indexing.
|
||||
5. **Validation:** deterministic validators run before LLM-backed validators.
|
||||
LLM validation is batched by prompt budget; candidate decisions feed application.
|
||||
6. **Application:** `internal/framework/proposals` clones transcript state and
|
||||
applies approved proposals in index order through replacement-policy checks.
|
||||
7. **Serialization and reporting:** the CLI encodes the final transcript, maps
|
||||
runner output into reports, writes artifacts, and delegates retention to
|
||||
diagnostics. See [report assembly](diagnostics-reporting.md).
|
||||
|
||||
## Runner Outputs
|
||||
|
||||
@@ -54,22 +45,18 @@ CLI/reporting then map this into process report and diagnostics artifacts.
|
||||
|
||||
## Failure Behavior
|
||||
|
||||
Representative failure phases include:
|
||||
- `run_dir_creation`
|
||||
- `transcript_read`, `glossary_read`
|
||||
- `transcript_schema`, `glossary_schema`
|
||||
- `chunking`
|
||||
- `runner_setup`, `runner_execution`
|
||||
- `output_schema`, `serialization`, `output_write`, `stdout_write`
|
||||
CLI orchestration labels failures at the phase boundary. Runner execution failures
|
||||
can return completed or partial module results, which report assembly preserves.
|
||||
[Report fields](../integrations/output-files.md#process-report) define how consumers
|
||||
see these outcomes; [CLI semantics](../cli.md#process-output-and-exit-behavior)
|
||||
define streams and exits. Diagnostic write failures do not always abort processing.
|
||||
|
||||
When diagnostics are available, failure stderr includes diagnostics path.
|
||||
## Ordering and State
|
||||
|
||||
## Invariants
|
||||
|
||||
- module execution order follows configured module sequence;
|
||||
- proposal/validator nondeterminism is isolated before deterministic apply;
|
||||
- proposal indices are assigned deterministically by section order;
|
||||
- output/report artifacts are generated from run results, not speculative state.
|
||||
The runner mutates its working transcript between module instances, after collecting
|
||||
section results and validating proposals. Proposal indexes follow section order;
|
||||
reports are derived from actual run results. Global requirements are owned by the
|
||||
[architecture policy](../policy/architecture.md#correction-and-concurrency-invariants).
|
||||
|
||||
## Key Tests
|
||||
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
## Scope
|
||||
|
||||
This document describes validator composition, execution order, and decision handling.
|
||||
This document describes internal validator composition, execution order, and
|
||||
decision handling. These registry keys identify implementation components; they
|
||||
are not independently selectable through [configuration](../config.md#pipeline-and-output).
|
||||
|
||||
## Ownership
|
||||
|
||||
@@ -49,7 +51,7 @@ Decision cardinality is enforced: each candidate proposal must receive exactly o
|
||||
|
||||
LLM validators:
|
||||
- build canonical validation request payloads;
|
||||
- batch by `validation_max_prompt_tokens`;
|
||||
- batch using the [configured prompt budget](../config.md#concurrency);
|
||||
- call structured LLM client using response schema registry.
|
||||
|
||||
Oversized single proposals are rejected with `validator_input_too_large`.
|
||||
@@ -57,12 +59,25 @@ Malformed LLM validator responses are downgraded to warnings and rejected batch
|
||||
|
||||
## Decision and Rejection Reporting
|
||||
|
||||
Runner records:
|
||||
- `validator_decisions`
|
||||
- `validator_rejected`
|
||||
- warning records (including malformed response warnings)
|
||||
Runner collects decisions, rejected candidates, and warning records. Ledger
|
||||
assembly groups decisions by deterministic versus LLM-backed execution metadata.
|
||||
The serialized names and fields belong to the
|
||||
[report and ledger contract](../integrations/output-files.md); implementation mapping
|
||||
is covered by [report assembly](diagnostics-reporting.md).
|
||||
|
||||
Correction ledger classifies deterministic vs LLM validator decisions using canonical metadata classes.
|
||||
## Changing Validators
|
||||
|
||||
1. Implement validator behavior under `internal/validators/<validator_key>`;
|
||||
update registry and chain wiring in `internal/validators`.
|
||||
2. Put shared runtime mechanics in `internal/framework/validators` only when
|
||||
needed. Keep deterministic versus LLM-backed execution metadata explicit.
|
||||
3. Preserve exposed identities and decision semantics under the
|
||||
[architecture policy](../policy/architecture.md#correction-and-concurrency-invariants).
|
||||
4. Review validator, registry, chain, batching, malformed-output, diagnostics,
|
||||
and decision-cardinality tests; update meaningful coverage under the
|
||||
[testing policy](../policy/testing.md).
|
||||
5. Update implementation guidance here and any affected external contract in its
|
||||
[canonical owner](../policy/documentation.md#canonical-ownership).
|
||||
|
||||
## Key Tests
|
||||
|
||||
|
||||
@@ -1,113 +1,110 @@
|
||||
# Audita Operations
|
||||
|
||||
## Scope
|
||||
This document owns runtime workflow, physical artifact placement, retention, and
|
||||
recovery. Use the [CLI reference](cli.md) for invocation and stream/exit semantics,
|
||||
[configuration](config.md) for settings, and [output-file contracts](integrations/output-files.md)
|
||||
for logical formats.
|
||||
|
||||
This document covers operational behavior for `audita process` as currently implemented:
|
||||
- run lifecycle;
|
||||
- output and report files;
|
||||
- diagnostics artifacts;
|
||||
- run-directory retention behavior;
|
||||
- failure inspection and recovery.
|
||||
## Normal Workflow
|
||||
|
||||
For command syntax, see [`docs/cli.md`](cli.md).
|
||||
1. Choose a configuration and validate it using the
|
||||
[configuration commands](cli.md#configuration-commands).
|
||||
2. Prepare input files conforming to the [input contract](integrations/transcript-glossary-files.md).
|
||||
3. Run the [processing command](cli.md#process-examples), choosing explicit output
|
||||
and report destinations when those files must survive run-directory cleanup.
|
||||
4. Inspect the corrected transcript and process report. Review rejected or skipped
|
||||
corrections when the report indicates them; successful execution does not mean
|
||||
every proposed correction was applied.
|
||||
|
||||
## Process Run Lifecycle
|
||||
|
||||
A `process` run performs these high-level steps:
|
||||
1. load effective config (defaults + optional file + env + CLI);
|
||||
2. create a per-run diagnostics directory;
|
||||
3. load transcript JSON and glossary YAML;
|
||||
4. parse/validate input schemas;
|
||||
5. normalize transcript and compute chunking;
|
||||
6. run configured modules/validators;
|
||||
7. serialize output schema and write transcript output;
|
||||
8. build and write process report;
|
||||
9. apply run-directory retention.
|
||||
|
||||
If a failure happens after diagnostics initialization, the run writes failure details and returns nonzero.
|
||||
Processing creates a run directory, loads and normalizes the inputs, chunks the
|
||||
transcript, executes the correction pipeline, writes results, and applies
|
||||
retention. Early configuration failures happen before run-directory creation.
|
||||
Failures after initialization attempt to preserve diagnostic evidence; individual
|
||||
artifact writes can fail, so a run directory need not contain every listed file.
|
||||
|
||||
## Output Files
|
||||
|
||||
Transcript output:
|
||||
- when `--output <path>` is set, corrected transcript JSON is written to that file;
|
||||
- when `--output` is omitted, corrected transcript JSON is written to stdout.
|
||||
The caller chooses transcript and optional external report destinations using the
|
||||
[CLI](cli.md#process-output-and-exit-behavior). These are separate from the per-run
|
||||
diagnostic directory unless the caller deliberately places them inside it.
|
||||
Choose existing, writable parent directories. Output/report files can be overwritten
|
||||
by subsequent runs using the same destinations.
|
||||
|
||||
Report output:
|
||||
- when `--report-json <path>` is set, Audita writes a process report JSON file;
|
||||
- the run directory also writes its own `report.json` artifact.
|
||||
|
||||
On success with `--output`, stdout is expected to be empty.
|
||||
A later report or cleanup failure can occur after transcript output was written.
|
||||
Inspect the process result and available report before treating the files as a
|
||||
completed run. Retention of the diagnostic directory does not archive caller-selected
|
||||
output files elsewhere.
|
||||
|
||||
## Diagnostics Directory
|
||||
|
||||
By default, runs use `work_dir` from effective config (default `/tmp/audita`).
|
||||
Each run directory is created under the work dir using a generated ID like `run-<unix-nanos>`.
|
||||
The [diagnostics configuration](config.md#thresholds-context-and-diagnostics)
|
||||
selects the parent work directory. Runs create children named like
|
||||
`run-<unix-nanoseconds>`. The runtime creates the work directory if needed.
|
||||
Report paths may be relative when the supplied work directory is relative;
|
||||
resolve them using the invocation's working directory.
|
||||
|
||||
Top-level diagnostics artifacts:
|
||||
- `source-transcript.json`
|
||||
- `source-transcript-parsed.json`
|
||||
- `normalized-transcript.json`
|
||||
- `normalization-summary.json`
|
||||
- `chunking-summary.json`
|
||||
- `utilization-diagnostics.json`
|
||||
- `correction-ledger.json`
|
||||
- `invocation.json`
|
||||
- `effective-config.json` (redacted)
|
||||
- `report.json`
|
||||
- `error.log` (failure runs)
|
||||
| Artifact in the run directory | Purpose |
|
||||
| --- | --- |
|
||||
| `source-transcript.json` | Original transcript input |
|
||||
| `source-transcript-parsed.json` | Parsed source representation |
|
||||
| `normalized-transcript.json` | Transcript after normalization |
|
||||
| `normalization-summary.json` | Normalization counters |
|
||||
| `chunking-summary.json` | Section/token summary |
|
||||
| `invocation.json` | Invocation metadata |
|
||||
| `effective-config.json` | Redacted configuration snapshot |
|
||||
| `report.json` | Run's [process report](integrations/output-files.md#process-report) |
|
||||
| `correction-ledger.json` | [Correction dispositions](integrations/output-files.md#correction-ledger) |
|
||||
| `utilization-diagnostics.json` | [Concurrency and timing observations](integrations/output-files.md#utilization-diagnostics) |
|
||||
| `error.log` | Failure details when written |
|
||||
|
||||
Report diagnostics metadata includes resolved paths to these artifacts.
|
||||
|
||||
## Correction Ledger and Utilization Diagnostics
|
||||
|
||||
`correction-ledger.json` records correction dispositions:
|
||||
- `applied`
|
||||
- `skipped`
|
||||
- `rejected`
|
||||
- `failed`
|
||||
|
||||
`utilization-diagnostics.json` records effective concurrency and execution timing summaries for run/module/validator activity.
|
||||
LLM request, response, and error diagnostics are also written within the run
|
||||
directory. Their nested naming and payloads are debugging details. Prefer paths
|
||||
provided by the report and its decision/warning records over reconstructing those
|
||||
paths. The report describes intended artifact locations, not proof that every
|
||||
write succeeded or that the files still exist after retention.
|
||||
|
||||
## Retention Behavior
|
||||
|
||||
Retention is controlled by `work_dir_retention` (`auto|always|never`).
|
||||
[Configuration](config.md#thresholds-context-and-diagnostics) owns accepted
|
||||
retention values and the default. Current lifecycle behavior is:
|
||||
|
||||
Current behavior:
|
||||
- failed runs are always retained;
|
||||
- `always`: successful runs are retained;
|
||||
- `auto`: successful runs are retained only when skipped/rejected corrections occurred; clean successful runs are removed;
|
||||
- `never`: successful runs are currently retained (same net retention outcome as `always` in current implementation).
|
||||
| Run outcome / selection | Directory outcome |
|
||||
| --- | --- |
|
||||
| Failed run | Retained regardless of selection |
|
||||
| Successful run, `always` | Retained |
|
||||
| Successful run, `auto` | Retained if corrections were skipped or rejected; otherwise removed |
|
||||
| Successful run, `never` | Currently retained, the same outcome as `always` |
|
||||
|
||||
Even when a successful run directory is removed under `auto`, an explicit `--report-json` file is still preserved at its target path.
|
||||
The name `never` does not currently mean deletion. Warnings alone do not trigger
|
||||
retention under `auto` unless they also produce skipped or rejected corrections.
|
||||
An external report outside the run directory survives automatic cleanup, but its
|
||||
artifact paths can then point to removed files.
|
||||
|
||||
## Failure Inspection
|
||||
## Artifact Handling
|
||||
|
||||
For failed runs:
|
||||
1. read stderr for the top-level failure and diagnostics path;
|
||||
2. open `error.log` in the reported run directory;
|
||||
3. inspect run `report.json` (`status`, `error_phase`, `error_message`);
|
||||
4. inspect related artifacts referenced by report diagnostics metadata.
|
||||
Run artifacts can contain full transcripts, glossary-related context, and LLM
|
||||
prompts/responses. Configured-key redaction does not anonymize those contents.
|
||||
Choose a work directory whose ownership and permissions fit the transcript data,
|
||||
and retain or share artifacts intentionally. Credential-supply mechanisms are
|
||||
owned by [configuration](config.md#secrets).
|
||||
|
||||
Typical `error_phase` values include:
|
||||
- `transcript_read`
|
||||
- `glossary_read`
|
||||
- `transcript_schema`
|
||||
- `glossary_schema`
|
||||
- `chunking`
|
||||
- `runner_setup`
|
||||
- `runner_execution`
|
||||
- `output_schema`
|
||||
- `serialization`
|
||||
- `output_write`
|
||||
- `stdout_write`
|
||||
For a debugging run, select retained diagnostics using the
|
||||
[retention flag](cli.md#process-flag-reference). Copy needed artifacts before
|
||||
removing completed run directories manually. Avoid deleting a directory while
|
||||
its run is active. Audita does not provide a separate archival/backup or remote
|
||||
artifact storage facility.
|
||||
|
||||
## Recovery Guidance
|
||||
## Failure Inspection and Recovery
|
||||
|
||||
Safe recovery pattern:
|
||||
1. correct the immediate input/config/output-path problem;
|
||||
2. rerun with `--work-dir-retention always` during debugging;
|
||||
3. once stable, restore your normal retention mode.
|
||||
1. Read stderr and the exit classification defined by the [CLI](cli.md#process-output-and-exit-behavior).
|
||||
2. Locate the external report, or the run directory identified by the error when
|
||||
available. If a late error omits the path, inspect the selected work directory.
|
||||
3. Read the available report and error log. Use
|
||||
[report status and error fields](integrations/output-files.md#process-report)
|
||||
to identify the failed phase, then inspect referenced artifacts.
|
||||
4. Correct the concrete input, configuration, endpoint, or filesystem issue and
|
||||
rerun the command with diagnostics retained if further investigation is needed.
|
||||
|
||||
Not implemented:
|
||||
- resume/checkpoint APIs
|
||||
- remote diagnostics/report storage
|
||||
Each invocation starts a new run. There is no implemented resume/checkpoint API;
|
||||
recovery is a rerun, not continuation from an artifact. Failures during early
|
||||
configuration or directory creation may leave no report or diagnostic files.
|
||||
|
||||
103
docs/policy/agents.md
Normal file
103
docs/policy/agents.md
Normal file
@@ -0,0 +1,103 @@
|
||||
# Agent Policy
|
||||
|
||||
**Base Policy Version:** 1.0
|
||||
|
||||
**Last Updated**: 2026-09-23
|
||||
|
||||
## Purpose
|
||||
|
||||
Agents working in this repository must read and adhere to the following house rules.
|
||||
|
||||
## Implementation Discipline
|
||||
|
||||
**KEY PRINCIPLES:** Write the simplest idiomatic code that satisfies the user's
|
||||
request. Do not implicitly strengthen requirements, restrictions, or guarantees
|
||||
beyond what the user actually requested.
|
||||
|
||||
**Keep implementation machinery proportionate.** Use straightforward,
|
||||
idiomatic code and introduce only the machinery needed to support the requested
|
||||
behavior and flexibility. Optimize for conceptual simplicity and
|
||||
maintainability, not for the fewest lines, fewest options, or narrowest product
|
||||
surface.
|
||||
|
||||
**Do not invent restrictions.** Do not reject, prohibit, constrain, cap,
|
||||
harden, or otherwise narrow behavior unless required by the user, an explicit
|
||||
repository contract, or an unavoidable correctness constraint. Preserve
|
||||
operator and caller choice where policy has not been established. If an
|
||||
additional restriction appears advisable, present it as a decision rather than
|
||||
silently implementing it.
|
||||
|
||||
**Do not invent guarantees.** Do not strengthen public contracts, guarantees,
|
||||
or invariants beyond those expressly requested by the user.
|
||||
|
||||
**Use the standard library.** For example: in almost every case, the best
|
||||
approach to copying files is to use the standard library. Writing custom code to
|
||||
implement power-loss protection at the application level, or to guard against
|
||||
bad actors with privileged and concurrent access to the host's filesystem,
|
||||
is wasteful, unnecessary, and not idiomatic. Stick to the simple, standard, and
|
||||
idiomatic approach unless another path is expressly requested by the user.
|
||||
|
||||
**An implementation detail is not repository policy.** Tests and documentation
|
||||
must not elevate incidental behavior into a durable guarantee or restriction
|
||||
unless that behavior was expressly selected as part of the contract.
|
||||
|
||||
## Codebase Discovery
|
||||
|
||||
This project uses codebase-memory-mcp to maintain a knowledge graph of the
|
||||
codebase. Always prefer its graph tools over grep, glob, or file search for code discovery.
|
||||
If the project is not indexed, run `index_repository` first.
|
||||
|
||||
Use this order:
|
||||
|
||||
1. `search_graph` to find functions, classes, routes, and variables by pattern.
|
||||
2. `trace_path` to trace callers and callees.
|
||||
3. `get_code_snippet` to read a specific function or class.
|
||||
4. `query_graph` for complex graph queries.
|
||||
5. `get_architecture` for the high-level project summary.
|
||||
|
||||
Fall back to grep or glob for string literals, error messages, configuration
|
||||
values, non-code files, or when the graph tools return insufficient results.
|
||||
|
||||
Examples:
|
||||
|
||||
- Find a handler: `search_graph(name_pattern=".*OrderHandler.*")`.
|
||||
- Find callers: `trace_path(function_name="OrderHandler", direction="inbound")`.
|
||||
- Read source: `get_code_snippet(qualified_name="pkg/orders.OrderHandler")`.
|
||||
|
||||
## Git Commit Policy
|
||||
|
||||
This policy applies to all coding agents and agent harnesses working in this
|
||||
repository, not only Codex.
|
||||
|
||||
By default, create a commit after completing each coherent unit of work, using a
|
||||
concise, plain-English message describing what changed. This policy is standing
|
||||
authorization to create those local commits; a separate user request is not
|
||||
required. An explicit user instruction to leave work uncommitted takes precedence.
|
||||
|
||||
A unit of work is a focused, reviewable outcome, including its relevant tests and
|
||||
documentation. Do not commit after every intermediate edit or create empty commits.
|
||||
Complete the appropriate validation before committing. If work is incomplete or
|
||||
validation is blocked, report that state rather than presenting it as completed.
|
||||
|
||||
Before committing:
|
||||
|
||||
1. Run `git status --short` and review the relevant diff, including any staged
|
||||
changes. Identify changes that predate the current unit of work.
|
||||
2. Exclude secrets, local environment files, generated junk, and unrelated changes.
|
||||
3. Stage only the files or hunks belonging to the completed unit. Use `git add .`
|
||||
only when every included change belongs to that unit. Do not disturb unrelated
|
||||
staged or unstaged work.
|
||||
4. Review `git diff --staged` and run appropriate checks for the change. Do not
|
||||
repeat checks already passed for unchanged content merely to create a commit.
|
||||
5. Review `git log --oneline -n 10` and use
|
||||
`git commit -m "<concise plain-English commit message>"`.
|
||||
6. Report the commit identifier and any remaining uncommitted work.
|
||||
|
||||
If the unit cannot be safely separated from pre-existing work, leave it uncommitted
|
||||
and explain the overlap rather than silently including that work. If a commit
|
||||
fails, preserve the changes and report the failure; do not bypass hooks.
|
||||
|
||||
Default commit authorization does not authorize pushing, publishing, amending,
|
||||
resetting, reverting, or rewriting history. Those actions require a separate user
|
||||
instruction. Do not use `git commit -a` or `git commit --no-verify` to bypass the
|
||||
staging and validation discipline above.
|
||||
@@ -1,207 +1,105 @@
|
||||
# Architecture Policy
|
||||
|
||||
## Purpose
|
||||
|
||||
This document defines Audita's development architecture and invariants for maintainers and LLM coding agents. It describes how the project is intended to be changed safely, based on behavior implemented in this repository today.
|
||||
|
||||
User-facing behavior belongs in the README and focused runtime docs. Future or proposed work belongs only under `docs/roadmap/`.
|
||||
|
||||
## Project Shape
|
||||
|
||||
Audita is a single-process Go CLI for transcript polishing. The executable entrypoint is `cmd/audita`; command handling lives in `internal/cli`.
|
||||
|
||||
The implemented `audita process` flow is:
|
||||
|
||||
1. load effective config;
|
||||
2. read and validate transcript JSON and glossary YAML;
|
||||
3. normalize transcript segments;
|
||||
4. chunk the working transcript into sections;
|
||||
5. resolve configured module instances;
|
||||
6. run correction modules and validator chains;
|
||||
7. apply approved proposals deterministically;
|
||||
8. write transcript output, reports, and diagnostics artifacts.
|
||||
|
||||
The current built-in modules are `glossary`, `homophones`, `spoken_word`, and `grammar`. The default configured module sequence repeats `glossary`.
|
||||
|
||||
For external behavior and compatibility details, prefer links to existing behavior docs:
|
||||
|
||||
- [CLI reference](../cli.md)
|
||||
- [Configuration](../config.md)
|
||||
- [Operations](../operations.md)
|
||||
- [Troubleshooting](../troubleshooting.md)
|
||||
- [Integration docs](../integrations/subprocess.md)
|
||||
- [Internal docs](../internal/overview.md)
|
||||
|
||||
## Core Design Principles
|
||||
|
||||
- **Hexagonal architecture:** keep domain behavior behind narrow internal contracts. CLI, filesystem, config loading, diagnostics writing, and LLM transport are adapters around the core processing flow.
|
||||
- **Composable modules and validators:** correction stages and validators should remain small, explicit, and independently testable.
|
||||
- **Deterministic orchestration around LLM calls:** LLM responses are nondeterministic inputs. Proposal indexing, validator ordering, proposal application, reports, and output serialization must remain deterministic.
|
||||
- **Bounded and observable concurrency:** use the implemented schedulers and configured concurrency limits for LLM call sites. Preserve utilization diagnostics when changing scheduling or orchestration.
|
||||
- **Conservative correction behavior:** validate proposed corrections before application; apply accepted proposals through deterministic apply-time safety checks.
|
||||
- **Standard-library-first:** prefer the Go standard library. Narrow third-party dependencies are acceptable when they materially improve maintainability, such as `gopkg.in/yaml.v3` for YAML parsing.
|
||||
- **Current-behavior documentation:** non-roadmap docs must describe implemented behavior only.
|
||||
|
||||
## Architectural Boundaries
|
||||
|
||||
`internal/core` owns domain data handling and stable runtime contracts that do not require CLI or provider transport knowledge:
|
||||
|
||||
- config defaults, loading, validation, redaction, and catalogs;
|
||||
- transcript and glossary schemas;
|
||||
- normalization and chunking;
|
||||
- output-schema encoding;
|
||||
- diagnostics artifact naming and run-directory helpers;
|
||||
- public process report shapes.
|
||||
|
||||
`internal/framework` owns orchestration contracts and reusable runtime mechanics:
|
||||
|
||||
- module and validator interfaces;
|
||||
- proposal generation, proposal application, and prompt context;
|
||||
- runner orchestration;
|
||||
- LLM scheduler, OpenAI-compatible adapter, redaction helpers, and diagnostics writers;
|
||||
- structured response schema registry;
|
||||
- process report and correction-ledger assembly.
|
||||
|
||||
`internal/modules/*` owns module-specific correction stages. `internal/validators/*` owns built-in validator implementations, registry, chains, and execution-class metadata. `internal/prompts` owns embedded prompt assets and prompt metadata.
|
||||
|
||||
`internal/cli` owns command parsing, exit codes, stdout/stderr behavior, config command behavior, filesystem input/output wiring, and top-level process orchestration. CLI concerns should not move into modules, validators, or schema logic.
|
||||
|
||||
Tests should stay close to the behavior they protect. Shared test helpers are acceptable when they remove clear duplication without hiding module-specific behavior.
|
||||
|
||||
## Modules and Validators
|
||||
|
||||
Modules implement `contracts.TranscriptModule`. A module must provide:
|
||||
|
||||
- a stable key;
|
||||
- a replacement policy;
|
||||
- a validator chain;
|
||||
- proposal generation from explicit request inputs.
|
||||
|
||||
Module packages should stay separate. Do not collapse module-specific prompts, scope, or validation choices into a broad generic stage abstraction.
|
||||
|
||||
Validators implement the shared validator contract and return one decision per candidate proposal. Deterministic validators and LLM-backed validators are both composable chain elements. Validator identity and execution class metadata are stable enough to affect ordering, diagnostics, reports, and correction-ledger classification.
|
||||
|
||||
Future module or validator changes should preserve:
|
||||
|
||||
- explicit inputs and outputs;
|
||||
- no hidden global state;
|
||||
- explicit config dependencies;
|
||||
- deterministic proposal index handling;
|
||||
- validation before final mutation;
|
||||
- stable reason codes and validator keys where already exposed.
|
||||
|
||||
## LLM Integration and Concurrency
|
||||
|
||||
LLM calls are external effects behind narrow contracts. Production structured completions use `contracts.StructuredLLMClient`; the implemented provider adapter is OpenAI-compatible HTTP code in `internal/framework/llm`.
|
||||
|
||||
Structured response schemas are registered in `internal/framework/responseschema`. Provider-side schema enforcement is not a substitute for local validation: Audita still validates proposal structure, validator decision cardinality, and apply-time safety.
|
||||
|
||||
Concurrency is bounded by configured scheduler limits:
|
||||
|
||||
- total LLM concurrency;
|
||||
- proposal LLM concurrency;
|
||||
- validation LLM concurrency.
|
||||
|
||||
The scheduler is context-aware and releases permits on success, failure, and cancellation. Runner code may collect section-level work concurrently, but transcript mutation is applied later in deterministic proposal-index order.
|
||||
|
||||
Diagnostics for LLM interactions should be useful for debugging without leaking configured secrets. Use the existing redaction helpers and `llm.ConfiguredSecrets`.
|
||||
|
||||
## State, Inputs, and Outputs
|
||||
|
||||
Audita does not implement resume, checkpoint, manifest, or remote storage behavior. Runtime state is in memory plus per-run diagnostics artifacts written under the configured work directory.
|
||||
|
||||
Transcript input accepts the implemented JSON forms documented in the public contract. Parsed source transcripts are normalized into Audita's internal transcript shape before chunking and module execution.
|
||||
|
||||
Proposals and validator decisions are intermediate runtime data. Approved proposals are applied through `internal/framework/proposals`, which clones transcript state, orders by proposal index, and records applied or skipped changes.
|
||||
|
||||
Transcript output is encoded through `internal/core/outputschema`. Reports and correction ledgers are machine-readable artifacts derived from runner outputs; their public shape should not be changed casually.
|
||||
|
||||
## Configuration and CLI Boundaries
|
||||
|
||||
Config behavior is owned by `internal/core/config`; command usage and process wiring are owned by `internal/cli`.
|
||||
|
||||
`audita process` uses implemented precedence: defaults, config file, environment, then CLI flags. `config validate` validates defaults plus a file config and intentionally does not apply environment overrides. `config print-effective` applies defaults, file config, and environment overrides, then prints redacted JSON.
|
||||
|
||||
Do not duplicate full CLI or config reference material here. Use [Configuration](../config.md), [CLI reference](../cli.md), [Operations](../operations.md), and integration docs under [`docs/integrations/`](../integrations/subprocess.md) for current external behavior.
|
||||
|
||||
When adding config fields or CLI flags, update:
|
||||
|
||||
- config defaults, file/env/CLI application, and validation;
|
||||
- CLI flag extraction if applicable;
|
||||
- redaction when secrets are involved;
|
||||
- tests for precedence and source-specific behavior;
|
||||
- user-facing docs if external behavior changes.
|
||||
|
||||
## Errors, Logging, and Diagnostics
|
||||
|
||||
Errors should be phase-specific enough for CLI users and subprocess callers. The CLI writes human-readable errors to stderr and preserves transcript JSON-only stdout behavior on successful stdout output.
|
||||
|
||||
Run diagnostics are best-effort after run-directory creation. Failed runs are retained. Successful run retention follows the implemented work-dir retention policy.
|
||||
|
||||
Diagnostics and reports must not leak configured LLM secrets. Config redaction and LLM payload/error redaction are separate responsibilities and should remain separate.
|
||||
|
||||
Process reports, diagnostics metadata, utilization diagnostics, and correction ledgers are part of the public contract. Prefer additive, compatible changes.
|
||||
|
||||
## Testing Expectations
|
||||
|
||||
Use targeted package tests for touched behavior and `go test ./...` for substantial changes.
|
||||
|
||||
When changing modules, inspect or add:
|
||||
|
||||
- package-local module tests under `internal/modules/*`;
|
||||
- prompt rendering or proposal-generation tests when prompt inputs change;
|
||||
- parity or release fixtures when public output behavior changes.
|
||||
|
||||
When changing validators, inspect or add:
|
||||
|
||||
- validator package tests;
|
||||
- registry and chain tests under `internal/validators`;
|
||||
- framework validator tests for batching, malformed output, diagnostics, and cardinality.
|
||||
|
||||
When changing LLM integration or concurrency, inspect or add:
|
||||
|
||||
- `internal/framework/llm` scheduler/client/redaction tests;
|
||||
- `internal/framework/runner` orchestration and utilization tests;
|
||||
- structured-output malformed classification tests.
|
||||
|
||||
When changing config, CLI, schema, output, reports, or diagnostics, inspect or add:
|
||||
|
||||
- `internal/core/config` tests;
|
||||
- CLI tests under `internal/cli`;
|
||||
- schema and output-schema tests under `internal/core`;
|
||||
- report, diagnostics, parity, and release-fixture tests.
|
||||
|
||||
## Dependency Policy
|
||||
|
||||
Audita should remain dependency-light. Prefer standard-library solutions for CLI parsing, HTTP, JSON, filesystem, synchronization, and tests.
|
||||
|
||||
Third-party dependencies should be narrow, justified, and preferably de facto standard for their purpose. YAML parsing is the current direct dependency exception.
|
||||
|
||||
Do not add broad frameworks for CLI, dependency injection, workflow orchestration, logging, or plugin systems without a concrete implemented need and focused tests.
|
||||
|
||||
## Documentation Expectations
|
||||
|
||||
Follow [Documentation Policy](./documentation.md). Architecture policy must stay concise and aligned with implemented behavior.
|
||||
|
||||
Do not use architecture docs as changelogs. Do not describe planned modules, adapters, modes, persistence, or configuration unless they are implemented. Put future work under `docs/roadmap/`.
|
||||
|
||||
## Architectural Invariants
|
||||
|
||||
- Keep LLM transport behind `StructuredLLMClient` and framework adapter boundaries.
|
||||
- Keep correction modules narrowly scoped and package-separated.
|
||||
- Keep validators modular, composable, and identified by stable keys.
|
||||
- Keep CLI/config/filesystem concerns out of module and validator domain logic.
|
||||
- Preserve deterministic transcript mutation and output handling around nondeterministic LLM calls.
|
||||
- Keep LLM concurrency bounded, configurable, and observable where implemented.
|
||||
- Keep run diagnostics and reports redacted and machine-readable.
|
||||
- Keep public CLI, config, output-schema, diagnostics, report, prompt, module, and validator contracts stable unless a change is explicit and tested.
|
||||
- Prefer small shared helpers over broad rewrites.
|
||||
- Avoid new dependencies unless they are narrow and clearly justified.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- No plugin framework is implemented.
|
||||
- No generic workflow engine is implemented.
|
||||
- No resume, checkpoint, manifest, or remote storage system is implemented.
|
||||
- No multi-process service mode is implemented.
|
||||
- No provider SDK abstraction beyond the current structured LLM client contract and OpenAI-compatible HTTP adapter is implemented.
|
||||
**Base Policy Version:** 1.0
|
||||
|
||||
**Last Updated**: 2026-09-23
|
||||
|
||||
## Purpose and System Shape
|
||||
|
||||
Audita is a single-process Go CLI for transcript polishing. Its architecture
|
||||
separates deterministic transcript processing from external LLM effects through
|
||||
narrow contracts and adapters. Correction modules and validators remain explicit,
|
||||
composable components in an ordered processing pipeline.
|
||||
|
||||
This policy owns architectural requirements. The [internal overview](../internal/overview.md)
|
||||
owns the implemented component inventory and links to runtime mechanics.
|
||||
[CLI](../cli.md), [configuration](../config.md), [operations](../operations.md),
|
||||
and [integration documentation](../integrations/subprocess.md) own external contracts.
|
||||
|
||||
## Ownership and Dependency Direction
|
||||
|
||||
- **Core:** owns domain data handling and reusable configuration, schema,
|
||||
normalization, chunking, output, diagnostics, and reporting contracts. Domain
|
||||
behavior must not require CLI or provider-transport knowledge.
|
||||
- **Framework:** owns orchestration contracts and shared runtime mechanics for
|
||||
modules, validators, proposals, LLM adapters, scheduling, and report assembly.
|
||||
It composes domain operations through explicit inputs and outputs.
|
||||
- **Modules:** own narrowly scoped correction behavior and module-specific prompt
|
||||
choices. Keep module packages separate rather than collapsing their distinct
|
||||
responsibilities into a broad generic stage abstraction.
|
||||
- **Validators:** own validation behavior and composition. Deterministic and
|
||||
LLM-backed validators remain explicit, independently composable elements.
|
||||
- **Prompts:** own embedded prompt assets and metadata used by the runtime.
|
||||
- **CLI and adapters:** own command parsing, streams, filesystem wiring, and
|
||||
transport. Keep these concerns outside module, validator, and schema logic.
|
||||
|
||||
LLM transport belongs behind `StructuredLLMClient` and framework adapter
|
||||
boundaries. Prefer small shared helpers over broad rewrites. Keep module and
|
||||
validator inputs, outputs, and configuration dependencies explicit, without
|
||||
hidden global state.
|
||||
|
||||
## Correction and Concurrency Invariants
|
||||
|
||||
- Treat LLM responses as nondeterministic inputs. Preserve deterministic proposal
|
||||
indexing, validator ordering, transcript mutation, reports, and serialization.
|
||||
- Validate corrections before final mutation, including local proposal checks,
|
||||
decision cardinality, and apply-time safety. Provider-side schema enforcement
|
||||
does not replace these checks.
|
||||
- Keep concurrency bounded, configurable, context-aware, and observable through
|
||||
the existing scheduler and utilization diagnostics. Preserve permit release on
|
||||
success, failure, and cancellation.
|
||||
- Keep module and validator identities, reason codes, and execution-class
|
||||
distinctions stable where exposed as contracts.
|
||||
|
||||
The [pipeline](../internal/pipeline.md), [modules](../internal/modules.md),
|
||||
[validators](../internal/validators.md), and [LLM runtime](../internal/llm-runtime.md)
|
||||
describe how these requirements are implemented.
|
||||
|
||||
## State and Compatibility
|
||||
|
||||
Keep transcript mutation and output handling deterministic around external
|
||||
calls. Reports and correction ledgers are derived from processing results.
|
||||
Preserve the separation between transient processing state and filesystem
|
||||
adapters; physical artifact placement and retention belong to
|
||||
[operations](../operations.md).
|
||||
|
||||
Public CLI, configuration, output-schema, diagnostics, report, prompt, module,
|
||||
and validator contracts should remain stable unless a change is deliberate and
|
||||
tested. Prefer additive, compatible report changes. Current implementation
|
||||
limits are documented in the [internal overview](../internal/overview.md#current-scope);
|
||||
they are not additional prohibitions on future development.
|
||||
|
||||
## Errors and Secret Handling
|
||||
|
||||
Errors should identify the failed processing phase sufficiently for users and
|
||||
subprocess callers. Preserve the stream and exit behavior defined by the
|
||||
[CLI contract](../cli.md) and the artifact handling defined by
|
||||
[operations](../operations.md).
|
||||
|
||||
Diagnostics, reports, and surfaced errors must not leak configured LLM secrets.
|
||||
Configuration redaction and LLM payload/error redaction have separate
|
||||
responsibilities and should remain separate. Implementation details belong in
|
||||
[diagnostics and reporting](../internal/diagnostics-reporting.md).
|
||||
|
||||
## Dependencies
|
||||
|
||||
Keep Audita dependency-light and prefer the Go standard library for CLI parsing,
|
||||
HTTP, JSON, filesystems, synchronization, and tests. Third-party dependencies
|
||||
should be narrow, justified, and preferably established for their purpose.
|
||||
YAML parsing is the existing direct dependency exception.
|
||||
|
||||
Do not introduce broad CLI, dependency-injection, workflow, logging, or plugin
|
||||
frameworks without a concrete implemented need and focused tests.
|
||||
|
||||
## Testing and Documentation
|
||||
|
||||
Follow the [testing policy](testing.md) for risk-based sufficiency and stable test
|
||||
boundaries. Keep tests close to the behavior they protect; shared helpers should
|
||||
remove duplication without obscuring component-specific behavior. Relevant
|
||||
internal docs identify component tests and change recipes; the
|
||||
[contributor guide](../development.md) provides baseline validation commands.
|
||||
|
||||
Follow the [documentation policy](documentation.md) for ownership, current versus
|
||||
future behavior, and the ADR lifecycle. Keep architecture concise and normative;
|
||||
use the roadmap for implementation plans rather than maintaining a changelog or
|
||||
future-feature specification here.
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
# Audita Development Workflow
|
||||
|
||||
## Scope
|
||||
|
||||
This is the canonical contributor workflow for Audita maintainers and coding agents.
|
||||
|
||||
It defines:
|
||||
- repository layout and boundaries;
|
||||
- setup and test commands;
|
||||
- expectations for code changes;
|
||||
- how to add config, CLI, modules, validators, docs, and examples.
|
||||
|
||||
## Setup
|
||||
|
||||
Prerequisites:
|
||||
- Go `1.24` or newer.
|
||||
|
||||
Common commands:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go build ./cmd/audita
|
||||
```
|
||||
|
||||
## Repository Layout
|
||||
|
||||
Top-level areas:
|
||||
- `cmd/audita`: executable entrypoint.
|
||||
- `internal/cli`: command parsing and process/config command orchestration.
|
||||
- `internal/core`: deterministic config/schema/normalization/chunking/output/diagnostics/reporting logic.
|
||||
- `internal/framework`: runner orchestration, contracts, proposal generation/application, validators runtime, LLM runtime, response schemas.
|
||||
- `internal/modules/*`: module-specific correction behavior.
|
||||
- `internal/validators/*`: validator implementations, chains, and metadata.
|
||||
- `internal/prompts`: embedded prompts and prompt metadata.
|
||||
- `docs/`: canonical documentation.
|
||||
- `examples/`: maintained copyable inputs/configs.
|
||||
|
||||
## Change Workflow
|
||||
|
||||
1. Confirm scope and behavior contract before editing.
|
||||
2. Make focused changes in the appropriate ownership area.
|
||||
3. Add or update tests for changed behavior.
|
||||
4. Run targeted package tests for touched areas.
|
||||
5. Run `go test ./...` for substantial changes.
|
||||
6. Update docs/examples when external behavior changes.
|
||||
|
||||
## How To Add or Change Configuration
|
||||
|
||||
1. Add fields/defaults/validation under `internal/core/config`.
|
||||
2. Apply source precedence correctly (defaults, file, env, CLI for `process`).
|
||||
3. Ensure `config validate` remains file-only and `config print-effective` remains redacted.
|
||||
4. Update tests in `internal/core/config` and related CLI tests.
|
||||
5. Update [`docs/config.md`](../config.md) and relevant examples under `examples/`.
|
||||
|
||||
## How To Add or Change CLI Behavior
|
||||
|
||||
1. Implement parsing/wiring in `internal/cli`.
|
||||
2. Keep stdout/stderr and exit behavior compatible unless intentional and documented.
|
||||
3. Update CLI tests under `internal/cli` and integration tests under `cmd/audita`.
|
||||
4. Update [`docs/cli.md`](../cli.md) and related integration docs.
|
||||
|
||||
## How To Add or Change Modules
|
||||
|
||||
1. Add or update one module package under `internal/modules/<module_key>`.
|
||||
2. Keep module-specific prompt ownership in the module + `internal/prompts`.
|
||||
3. Wire module registration/catalog resolution through framework/core module catalog code.
|
||||
4. Verify replacement policy and validator chain selection.
|
||||
5. Add/update module tests and proposal-generation tests.
|
||||
6. Update internal docs when behavior/contracts change.
|
||||
|
||||
## How To Add or Change Validators
|
||||
|
||||
1. Implement validator behavior in `internal/validators` and shared runtime pieces in `internal/framework/validators` only when needed.
|
||||
2. Preserve stable validator keys and decision semantics where already exposed.
|
||||
3. Keep deterministic vs LLM-backed execution-class behavior explicit.
|
||||
4. Add/update validator, chain, batching, and malformed-output tests.
|
||||
5. Update validator documentation when external or developer-facing behavior changes.
|
||||
|
||||
## Documentation and Examples Expectations
|
||||
|
||||
- Keep one canonical home per topic (see [`docs/policy/documentation.md`](documentation.md)).
|
||||
- Do not document future/unimplemented behavior outside `docs/roadmap/`.
|
||||
- Keep command examples and config/examples in sync with current code.
|
||||
- Keep examples secret-free and copyable.
|
||||
|
||||
## Practical Validation Checklist
|
||||
|
||||
Use this checklist for meaningful runtime-impacting changes:
|
||||
|
||||
1. Run core tests:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
```
|
||||
|
||||
2. Verify config commands and examples:
|
||||
|
||||
```sh
|
||||
go run ./cmd/audita config validate --config examples/minimal-config.yml
|
||||
go run ./cmd/audita config validate --config examples/production-config.yml
|
||||
go run ./cmd/audita config print-effective --config examples/minimal-config.yml
|
||||
```
|
||||
|
||||
3. Re-check subprocess/runtime contract when touching CLI/process/report paths:
|
||||
- `--output` success keeps stdout empty;
|
||||
- no `--output` success writes transcript JSON to stdout;
|
||||
- `--report-json` writes file output and is not written to stdout;
|
||||
- failures return nonzero and include diagnostics path when available.
|
||||
|
||||
4. Re-check diagnostics/report/redaction when touching LLM, reporting, or diagnostics code:
|
||||
- report schema metadata fields remain present;
|
||||
- diagnostics artifact paths remain valid;
|
||||
- configured secret values remain redacted in reports/diagnostics/errors.
|
||||
|
||||
5. Re-check output schema behavior when touching serialization/schema code:
|
||||
- default `bare-segments` behavior remains correct unless intentionally changed;
|
||||
- `audita-v1` behavior remains correct unless intentionally changed;
|
||||
- unsupported schemas fail validation/resolve paths clearly.
|
||||
|
||||
## Commit Discipline
|
||||
|
||||
- Keep commits scoped and reviewable.
|
||||
- Avoid mixing unrelated refactors with behavior changes.
|
||||
- Use concise plain-English commit messages.
|
||||
@@ -1,356 +1,164 @@
|
||||
# Go Project Documentation Policy
|
||||
# Documentation Policy
|
||||
|
||||
**Base Policy Version:** 1.1
|
||||
|
||||
**Last Updated**: 2026-09-23
|
||||
|
||||
## Purpose
|
||||
|
||||
Project documentation must help four audiences:
|
||||
|
||||
1. users who need to run the application;
|
||||
2. administrators/operators who need to configure and operate it;
|
||||
3. developers who need to understand and change it safely;
|
||||
4. LLM coding agents that need clear scope, boundaries, and invariants.
|
||||
|
||||
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
||||
This policy assigns each documentation topic to one canonical owner. Its goal is
|
||||
to keep this application's documentation accurate, concise, discoverable, and resistant
|
||||
to drift for users, operators, developers, integrators, and LLM coding agents.
|
||||
|
||||
## Core Rules
|
||||
|
||||
### 1. Keep docs concise
|
||||
|
||||
Each document should cover a defined scope and only the essentials for that scope.
|
||||
|
||||
Avoid:
|
||||
- long background explanations;
|
||||
- repeated reference material;
|
||||
- implementation detail in user-facing docs;
|
||||
- aspirational language outside roadmap docs;
|
||||
- verbose examples where one minimal example is clearer.
|
||||
|
||||
### 2. Document only implemented behavior outside roadmap files
|
||||
|
||||
Unimplemented, planned, aspirational, experimental, or future work may be described only under:
|
||||
|
||||
- `docs/roadmap/`
|
||||
|
||||
No other documentation file, including `README.md`, should describe code, features, modules, stages, commands, config fields, or behaviors that do not currently exist.
|
||||
|
||||
If a feature is partial, non-roadmap docs may describe only the implemented portion and its current boundary.
|
||||
|
||||
### 3. Use canonical homes
|
||||
|
||||
Each type of information should have one canonical location.
|
||||
|
||||
Canonical homes:
|
||||
|
||||
- project purpose and quickstart: `README.md`
|
||||
- development principles: `docs/policy/architecture.md`
|
||||
- configuration reference: `docs/config.md`
|
||||
- CLI reference: `docs/cli.md`
|
||||
- operations and recovery: `docs/operations.md`
|
||||
- troubleshooting: `docs/troubleshooting.md`
|
||||
- implemented internals: `docs/internal/`
|
||||
- future work: `docs/roadmap/`
|
||||
- contributor workflow: `docs/policy/development.md`
|
||||
- copyable examples: `examples/`
|
||||
|
||||
Other files should summarize briefly and link to the canonical source.
|
||||
|
||||
### 4. Keep examples real
|
||||
|
||||
Examples should be valid, maintained, and free of secrets.
|
||||
|
||||
Where practical:
|
||||
- example configs should load successfully;
|
||||
- example commands should match real CLI syntax;
|
||||
- important examples should be covered by tests.
|
||||
|
||||
## Documentation Profiles
|
||||
|
||||
All projects require:
|
||||
|
||||
- `README.md`
|
||||
- `docs/policy/architecture.md`
|
||||
|
||||
Additional docs depend on the project.
|
||||
|
||||
### Small library
|
||||
|
||||
Recommended:
|
||||
- `docs/policy/development.md`, if contributor conventions are non-obvious
|
||||
|
||||
### Simple CLI
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/policy/development.md`
|
||||
|
||||
### Config-driven CLI
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`
|
||||
- `docs/config.md`
|
||||
|
||||
Recommended:
|
||||
- `examples/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
### Stateful or operator-facing application
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- `examples/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
### Modular, staged, service-oriented, or orchestration application
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
- `docs/internal/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- validated examples under `examples/`
|
||||
|
||||
## Required Documents
|
||||
|
||||
### README.md
|
||||
|
||||
**Audience:** users, administrators, operators
|
||||
|
||||
The README is the outward-facing project orientation page.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. concise description;
|
||||
2. elevator pitch;
|
||||
3. shortest useful command or usage example;
|
||||
4. links to targeted docs.
|
||||
|
||||
The README should be short. It is not a manual.
|
||||
|
||||
The “shortest useful command” means the simplest command that performs the project’s core use case. (It does not mean `app --help`.)
|
||||
|
||||
### docs/policy/architecture.md
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
`docs/policy/architecture.md` is required for every project.
|
||||
|
||||
It is an inward-facing development policy document. It should describe how the project is intended to be built and changed.
|
||||
|
||||
It should include:
|
||||
|
||||
- project shape;
|
||||
- core design principles;
|
||||
- package and boundary philosophy;
|
||||
- state/persistence philosophy, if applicable;
|
||||
- external integration philosophy, if applicable;
|
||||
- error-handling and logging principles;
|
||||
- testing expectations;
|
||||
- documentation expectations;
|
||||
- architectural invariants;
|
||||
- explicit non-goals, if useful.
|
||||
|
||||
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
|
||||
|
||||
### docs/policy/development.md
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for projects maintained by humans and LLM coding agents.
|
||||
|
||||
It should include:
|
||||
|
||||
- repository layout;
|
||||
- build/test commands;
|
||||
- coding conventions;
|
||||
- dependency policy;
|
||||
- how to add config fields;
|
||||
- how to add CLI flags;
|
||||
- how to add stages/modules/adapters, if applicable;
|
||||
- how to update examples;
|
||||
- documentation update expectations.
|
||||
|
||||
### docs/config.md
|
||||
|
||||
**Audience:** administrators, operators, advanced users
|
||||
|
||||
Required for applications with configuration files.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. config file locations and discovery precedence;
|
||||
2. minimal working config;
|
||||
3. production-oriented config;
|
||||
4. full configuration reference;
|
||||
5. secrets handling, if applicable;
|
||||
6. links to maintained examples.
|
||||
|
||||
The full configuration reference should be canonical.
|
||||
|
||||
### docs/cli.md
|
||||
|
||||
**Audience:** users, administrators, operators
|
||||
|
||||
Required for CLI applications.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. shortest useful command;
|
||||
2. command overview;
|
||||
3. complete flag reference;
|
||||
4. common workflows;
|
||||
5. diagnostic or recovery commands, if applicable.
|
||||
|
||||
Explain when commands are useful, not just their syntax.
|
||||
|
||||
### docs/operations.md
|
||||
|
||||
**Audience:** administrators, operators
|
||||
|
||||
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
|
||||
|
||||
It should cover:
|
||||
|
||||
- normal workflow;
|
||||
- filesystem layout;
|
||||
- remote storage layout, if applicable;
|
||||
- logs and manifests;
|
||||
- resume/retry behavior;
|
||||
- cleanup behavior;
|
||||
- archive/backup behavior;
|
||||
- safe recovery procedures;
|
||||
- operational caveats.
|
||||
|
||||
### docs/troubleshooting.md
|
||||
|
||||
**Audience:** administrators, operators
|
||||
|
||||
Recommended once recurring failure modes exist.
|
||||
|
||||
Each entry should include:
|
||||
|
||||
- symptom;
|
||||
- likely cause;
|
||||
- diagnostic command or inspection step;
|
||||
- safe fix;
|
||||
- relevant links.
|
||||
|
||||
### docs/internal/
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for modular, staged, service-oriented, or orchestration projects.
|
||||
|
||||
This directory describes implemented internal components. It is not the roadmap.
|
||||
|
||||
Use one file per major component where useful.
|
||||
|
||||
Each component doc should include:
|
||||
|
||||
1. purpose;
|
||||
2. inputs and outputs;
|
||||
3. boundaries;
|
||||
4. config fields used;
|
||||
5. external adapters used;
|
||||
6. state or manifest behavior, if applicable;
|
||||
7. skip/resume behavior, if applicable;
|
||||
8. failure behavior;
|
||||
9. tests to inspect before changing;
|
||||
10. architectural invariants.
|
||||
|
||||
### docs/roadmap/
|
||||
|
||||
**Audience:** maintainers, developers, LLM coding agents
|
||||
|
||||
This is the only place for planned, future, aspirational, experimental, or unimplemented work.
|
||||
|
||||
Roadmap docs should clearly distinguish:
|
||||
|
||||
- proposed work;
|
||||
- accepted plans;
|
||||
- deferred ideas;
|
||||
- rejected ideas;
|
||||
- implementation prompts or task breakdowns, if useful.
|
||||
|
||||
Roadmap docs should not be confused with current behavior.
|
||||
|
||||
### docs/integrations/
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
||||
|
||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
|
||||
|
||||
Use one file per integration where useful.
|
||||
|
||||
## Examples Directory
|
||||
|
||||
Projects with non-trivial configuration or workflows should include `examples/`.
|
||||
|
||||
Useful examples include:
|
||||
|
||||
- minimal working config;
|
||||
- production-oriented config;
|
||||
- full annotated config;
|
||||
- local development config;
|
||||
- remote/object-storage config;
|
||||
- minimal session/input file.
|
||||
|
||||
Examples should be valid, maintained, tested when practical, and linked from relevant docs.
|
||||
|
||||
## Security and Privacy
|
||||
|
||||
Docs and examples must not include:
|
||||
|
||||
- real API keys;
|
||||
- tokens;
|
||||
- passwords;
|
||||
- private keys;
|
||||
- private environment dumps;
|
||||
- sensitive user data;
|
||||
- raw private transcripts;
|
||||
- private infrastructure details unless intentionally public.
|
||||
|
||||
Document secret-handling mechanisms, not actual secret values.
|
||||
|
||||
## Maintenance Rules
|
||||
|
||||
When docs change, verify the affected behavior.
|
||||
|
||||
Where practical:
|
||||
|
||||
- load example config files in tests;
|
||||
- test CLI examples or command parser behavior;
|
||||
- validate documented flags against real flags;
|
||||
- remove stale references;
|
||||
- update links after renames;
|
||||
- keep roadmap content out of non-roadmap docs.
|
||||
|
||||
If documentation and code disagree, fix the documentation and/or open a roadmap item; do not leave aspirational behavior in current-behavior docs.
|
||||
|
||||
Documentation is complete only when it matches the current code.
|
||||
|
||||
## Documentation Change Checklist
|
||||
|
||||
Before merging documentation changes, verify:
|
||||
|
||||
- README is concise and orientation-focused.
|
||||
- `docs/policy/architecture.md` describes development principles.
|
||||
- Future work appears only under `docs/roadmap/`.
|
||||
- User-facing docs avoid unnecessary internals.
|
||||
- Developer-facing docs preserve boundaries and invariants.
|
||||
- Config examples match the schema.
|
||||
- CLI examples match real commands and flags.
|
||||
- Defaults appear in the canonical config reference.
|
||||
- No secrets or private data are included.
|
||||
- Links are accurate.
|
||||
### One Canonical Owner
|
||||
|
||||
Each authoritative fact belongs in one document. A non-owning document may give
|
||||
a short, stable summary for orientation, but it must link to the canonical owner
|
||||
instead of repeating volatile details.
|
||||
|
||||
Volatile details include commands, flags, configuration fields and defaults,
|
||||
module keys, schemas, file names, paths, status codes, retry behavior, and
|
||||
runtime guarantees. If readers could reasonably treat a statement as a
|
||||
contract, maintain it only in the owning document.
|
||||
|
||||
### Current And Future Behavior
|
||||
|
||||
Outside `docs/roadmap/`, documentation describes implemented behavior only.
|
||||
Partial features may be described only to their implemented boundary.
|
||||
|
||||
ADRs are the narrow exception: an ADR may record an accepted architectural
|
||||
decision before implementation, but acceptance must not be presented as proof
|
||||
that the behavior exists. The roadmap owns implementation status and sequencing
|
||||
until the decision is implemented. Current architecture, user, operator,
|
||||
integration, and internal documentation are updated when the behavior lands.
|
||||
|
||||
### Audience And Detail
|
||||
|
||||
Write for the document's stated audience and include only the detail needed for
|
||||
its owned topic. User and operator docs should not expose implementation detail.
|
||||
Developer docs should link to user-facing and external contracts rather than
|
||||
restate them.
|
||||
|
||||
### Examples
|
||||
|
||||
Complete copyable files belong in `examples/`. Documentation may use the
|
||||
smallest illustrative snippet needed to explain its owned topic, but should link
|
||||
to maintained examples instead of embedding a second complete copy.
|
||||
|
||||
Examples must be valid, secret-free, and tested where practical. Commands and
|
||||
configuration used in documentation should match the application.
|
||||
|
||||
### Security And Privacy
|
||||
|
||||
Documentation and examples must not contain real credentials, private keys,
|
||||
private environment dumps, sensitive source material, or private infrastructure
|
||||
details unless intentionally public. Document secret-handling mechanisms, not
|
||||
secret values.
|
||||
|
||||
## Canonical Ownership
|
||||
|
||||
| Topic | Canonical owner | Owned content | Content owned elsewhere |
|
||||
| --- | --- | --- | --- |
|
||||
| Product orientation and minimal end-to-end quickstart | `README.md` | What this application is, why it is useful, one shortest successful invocation, and links onward. | Complete command reference, configuration reference, operational procedures, implementation detail. |
|
||||
| Contributor entry point | `docs/development.md` | Task-oriented reading guide, minimal contributor orientation, baseline validation commands, and links to canonical docs. | Package inventory, architecture rules, subsystem behavior, detailed change recipes. |
|
||||
| Application architecture policy | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, architectural boundaries, invariants, safety properties, and non-goals. | Concrete package inventory, implementation mechanics, contributor procedures, decision history, future work. |
|
||||
| Agent policy | `docs/policy/agents.md` | Implementation discipline for coding agents, including proportionality, idiomatic simplicity, code discovery, Git commit restrictions, and rules against inventing requirements, restrictions, guarantees, or repository policy. | Application contracts, architecture, testing policy, and contributor procedures. |
|
||||
| Documentation policy | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
|
||||
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk-based sufficiency, test boundaries, doubles, coverage guidance, regression-test policy, and criteria for adding, rewriting, or deleting tests. | Subsystem behavior, application contracts, subsystem-specific test inventories, and implementation plans. |
|
||||
| Release policy | `docs/policy/release.md` | Versioning and compatibility rules, release integrity and immutability requirements, release-documentation policy, required gates, publication controls, verification requirements, and exception policy. | Repository-specific commands, product installation guidance, CLI version semantics, historical release summaries, and CI implementation detail. |
|
||||
| Repository-specific release procedure | `docs/release.md` | Maintainer prerequisites, candidate preparation, concrete validation and publication commands, repository and registry guards, deliverable publication, consumer-facing verification, and failure recovery. | General release requirements, product installation guidance, historical release summaries, and current compatibility contracts. |
|
||||
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
|
||||
| Configuration contract | `docs/config.md` | Discovery and precedence, file schema, fields, defaults, environment overrides, validation rules, and user-selectable module or validator keys. | Complete example files, CLI syntax, runtime state lifecycle, module implementation details. |
|
||||
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
|
||||
| Integration contracts for external components | `docs/integrations/`, including `docs/integrations/output-files.md` for serialized outputs | External file formats and protocols, upstream and downstream contracts, logical output bundle paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
||||
| Inventory of implemented internal packages | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
||||
| Behavior and architecture of internal packages | Other files under `docs/internal/` | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
||||
| Contributor change recipes | Relevant files under `docs/internal/`, including `docs/internal/cli-config.md`, `docs/internal/modules.md`, and `docs/internal/validators.md` | Concrete editing and validation steps for each component. | Contributor orientation, global policy, and definitions of external contracts. |
|
||||
| Architectural decision records | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
|
||||
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status; sequencing; and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
||||
| Release-note history | `docs/releases/` | One checked-in historical summary for each source release made under the procedure. The note at the immutable tag is that release's record. | Current commands, behavior, contracts, and compatibility definitions. |
|
||||
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other files intended to be copied or run. | Field-by-field reference, command reference, prose explanation. |
|
||||
|
||||
Documents that do not exist are required only when the corresponding interface
|
||||
or responsibility exists. Do not create placeholder API, consumer, integration,
|
||||
or operations documents for behavior the application does not have. Audita has
|
||||
retired its standalone troubleshooting guide; concrete operating and recovery
|
||||
procedures belong in `docs/operations.md`.
|
||||
|
||||
## Boundary Rules
|
||||
|
||||
### Orientation
|
||||
|
||||
The README owns product orientation. The developer guide routes contributors.
|
||||
Architecture owns normative structure. Internal overview owns the current
|
||||
concrete component map. These documents may link to one another but should not
|
||||
maintain parallel package or behavior descriptions.
|
||||
|
||||
### Commands, Configuration, And Operations
|
||||
|
||||
CLI documentation answers how to invoke the application. Configuration
|
||||
documentation answers what settings mean. Operations answers what happens to
|
||||
runtime state and how to operate or recover the application. When a workflow
|
||||
crosses these topics, choose the document that owns the task and link to the
|
||||
other contracts.
|
||||
|
||||
### Releases
|
||||
|
||||
The release policy defines requirements and invariants that every release must
|
||||
satisfy. The repository-specific release procedure defines the commands and
|
||||
workflow maintainers use to satisfy that policy. Release notes are immutable
|
||||
historical summaries, not current-state contract owners: the checked-in note at
|
||||
an immutable tag records that release, while current canonical documentation
|
||||
must change with the behavior it describes. Do not use a release procedure or
|
||||
release note to weaken policy, redefine current behavior, or defer required
|
||||
documentation updates.
|
||||
|
||||
### Contracts And Implementation
|
||||
|
||||
Integration and API documents define externally observable shapes and
|
||||
semantics. Internal documents explain how this application implements or consumes those
|
||||
contracts. Internal docs may name a field, file, or protocol to identify a
|
||||
dependency, but must link to its canonical contract for the definition.
|
||||
|
||||
### Security Topics
|
||||
|
||||
This policy owns what documentation and examples may contain. Architecture owns
|
||||
application security invariants. Configuration owns credential-supply
|
||||
mechanisms. Operations owns permissions and handling of sensitive runtime
|
||||
artifacts. Internal docs own implementation mechanisms only.
|
||||
|
||||
## Architecture Decision Records
|
||||
|
||||
Use sequentially numbered ADR filenames such as
|
||||
`0001-record-architecture-decisions.md`. Follow the lightweight Nygard format:
|
||||
|
||||
1. title;
|
||||
2. status;
|
||||
3. date;
|
||||
4. context;
|
||||
5. decision;
|
||||
6. alternatives considered;
|
||||
7. consequences.
|
||||
|
||||
Treat the decision content of an accepted ADR as immutable. When a decision
|
||||
changes, create a new ADR and update the earlier ADR's status to superseded.
|
||||
Rejected architectural alternatives belong in the ADR; rejected product ideas
|
||||
belong in the roadmap.
|
||||
|
||||
## Maintenance
|
||||
|
||||
When behavior changes, update its canonical owner in the same change. If
|
||||
ownership moves, remove the old definition and replace it with a link where
|
||||
navigation remains useful.
|
||||
|
||||
Before completing documentation work:
|
||||
|
||||
- verify affected behavior and examples;
|
||||
- check commands, flags, fields, defaults, schemas, and paths against their
|
||||
implementation;
|
||||
- keep unimplemented behavior in the roadmap, subject to the ADR exception;
|
||||
- remove stale references and validate links;
|
||||
- confirm that non-owning documents summarize and link rather than redefine;
|
||||
- confirm that no secrets or sensitive private data were added.
|
||||
|
||||
168
docs/policy/release.md
Normal file
168
docs/policy/release.md
Normal file
@@ -0,0 +1,168 @@
|
||||
# Release Policy
|
||||
|
||||
**Base Policy Version:** 1.0
|
||||
|
||||
**Last Updated**: 2026-09-23
|
||||
|
||||
## Purpose
|
||||
|
||||
This policy defines the minimum requirements for selecting, validating,
|
||||
publishing, verifying, and correcting releases. Its goal is to make every
|
||||
release intentional, reproducible from an exact source revision, documented
|
||||
for consumers, and safe from retrospective replacement.
|
||||
|
||||
The repository-specific commands that implement these requirements belong in
|
||||
`docs/release.md`. Historical release summaries belong in `docs/releases/`.
|
||||
This is an Audita-specific adaptation of the base policy, not a requirement to
|
||||
implement the standard unchanged. Adapt it further when Audita's needs warrant;
|
||||
the canonical procedure must agree with the adopted policy.
|
||||
|
||||
## Release Model
|
||||
|
||||
A release identifies one exact source commit with an immutable version tag and
|
||||
its checked-in release note. The primary release branch is `main`.
|
||||
|
||||
Release success means that the intended version tag has been created and
|
||||
successfully pushed to the authoritative remote with its source commit. The
|
||||
release procedure ends at that point. It must never wait for CI to complete or
|
||||
require successful CI status, artifact publication, or consumer installation as
|
||||
a completion gate.
|
||||
|
||||
CI builds and publishes binaries and checksums asynchronously from the tagged
|
||||
source. Delayed or failed automation does not reverse release success. Artifact
|
||||
availability is a separate status and must not be claimed merely because the tag
|
||||
was pushed. Published artifacts must be traceable to the tagged source.
|
||||
|
||||
Automation must not silently change the source candidate, move a published tag,
|
||||
or replace an existing published artifact.
|
||||
|
||||
## Supported Platforms
|
||||
|
||||
| Platform | Support |
|
||||
| --- | --- |
|
||||
| Linux amd64 | Required production target |
|
||||
| Linux arm64 | Required production target |
|
||||
| macOS arm64 | Best effort for local development; failures do not block a release or Linux artifact publication |
|
||||
|
||||
Windows and macOS amd64 are unsupported. Neither their builds nor tests are
|
||||
release requirements. Native macOS testing and access to a Mac are not release
|
||||
prerequisites.
|
||||
|
||||
## Versioning And Compatibility
|
||||
|
||||
Stable releases use Semantic Versioning in the form `vMAJOR.MINOR.PATCH`, with
|
||||
no leading zeroes. Each version must be unused locally and on the authoritative
|
||||
remote before publication begins.
|
||||
|
||||
Unless an adopted policy explicitly supports and defines prereleases,
|
||||
prerelease versions are not permitted.
|
||||
|
||||
For releases at or after `v1.0.0`:
|
||||
|
||||
- a major release may make intentional incompatible changes;
|
||||
- a minor release may add backward-compatible behavior; and
|
||||
- a patch release may make backward-compatible fixes.
|
||||
|
||||
Before `v1.0.0`, a minor release may change a documented public, operational,
|
||||
configuration, data, integration, or compatibility contract when the release
|
||||
note explains the impact and required consumer action. A patch release must
|
||||
not intentionally break those contracts within its minor line.
|
||||
|
||||
A repository with additional compatibility promises must define them in its
|
||||
adopted policy or canonical contract documentation. A release note may explain
|
||||
a compatibility change, but it must not redefine the current compatibility
|
||||
contract.
|
||||
|
||||
## Release Documentation
|
||||
|
||||
Every release made under this policy must include
|
||||
`docs/releases/<version>.md` in the tagged source tree. The filename must use
|
||||
the complete version tag, and the document must begin with a title containing
|
||||
the product name and that exact version.
|
||||
|
||||
Release notes must contain a short summary and any relevant compatibility or
|
||||
upgrade instructions. Fixed sections and explicit no-action boilerplate are not
|
||||
required. Keep notes proportionate to the change.
|
||||
|
||||
Release notes are immutable historical summaries. They should link to current
|
||||
canonical documentation for exact behavior. Every affected canonical document
|
||||
must be updated in the same candidate; a release note must not substitute for
|
||||
current documentation or defer required documentation changes.
|
||||
|
||||
## Candidate Preparation And Validation
|
||||
|
||||
Use the minimum gates needed for confidence in Audita's release. Provide one
|
||||
local command for the selected automated checks, following the
|
||||
[testing policy](testing.md). Run the deterministic, offline default test suite
|
||||
once for the candidate and verify that both required Linux targets build.
|
||||
Avoid duplicate test runs solely to satisfy separate release phases.
|
||||
|
||||
Additional checks must address a concrete risk not already covered. Do not
|
||||
inherit a mandatory linting, documentation audit, platform matrix, or live-service
|
||||
checklist solely from the base policy. Documentation migration validation is
|
||||
separate work, not automatically a recurring release gate.
|
||||
|
||||
Automate mechanical guards. Do not require manual approvals, sign-offs, laptop
|
||||
testing, or post-publication inspection as release gates. Version selection and
|
||||
a concise release summary remain preparation inputs, not separate approval rounds.
|
||||
|
||||
## Publication Guards
|
||||
|
||||
The procedure must automate the small set of checks needed to publish the
|
||||
intended source under a fresh version:
|
||||
|
||||
- valid, unused version on the authoritative remote;
|
||||
- clean worktree and index on the primary release branch;
|
||||
- the exact candidate that passed the local checks, including its matching note;
|
||||
- publication of that source commit and the exact intended tag.
|
||||
|
||||
Do not require remote CI success. Avoid an additional manual confirmation for
|
||||
each mechanical guard. Publish explicit refs rather than all local tags.
|
||||
A successful push is the completion signal; a failed or ambiguous push must be
|
||||
resolved against remote ref state before declaring success or retrying.
|
||||
|
||||
## Immutability And Correction
|
||||
|
||||
Published version tags and published deliverables are immutable. Do not move,
|
||||
overwrite, recreate, or delete a published tag. Do not replace an artifact,
|
||||
image, package, checksum, signature, provenance statement, or release note
|
||||
under an already published version.
|
||||
|
||||
If validation fails before publication, correct and commit the candidate, then
|
||||
rerun the release command. Validation must apply to the exact candidate. An
|
||||
unpublished local tag may be removed after confirming that it was never published
|
||||
or used by a release channel.
|
||||
|
||||
If a defect is discovered after publication, leave the published release intact. Correct the defect on the primary branch, choose a
|
||||
new patch version or other semantically appropriate new version, write a new
|
||||
matching release note, and perform the complete release procedure again.
|
||||
|
||||
A distribution channel's supported yank, deprecation, or advisory mechanism
|
||||
may warn consumers away from a defective release, but it must not replace that
|
||||
release's contents or reuse its version.
|
||||
|
||||
## Repository Release Procedure
|
||||
|
||||
Maintain [docs/release.md](../release.md) as a short, executable maintainer runbook.
|
||||
It must identify prerequisites, authoritative remote, version and note preparation,
|
||||
one local validation command, automated source/tag guards, exact publication
|
||||
commands, and failure handling. It must name the asynchronous automation and link
|
||||
to this policy for supported platforms and release-success semantics.
|
||||
|
||||
Describe how to diagnose or retry failed artifact delivery separately from release
|
||||
completion. Missing artifacts may be published later from the same immutable tag;
|
||||
existing published artifacts must not be overwritten. A source defect requires a
|
||||
new version. No polling or CI waiting belongs in the release command.
|
||||
|
||||
## Audita Adaptations
|
||||
|
||||
The maintainer approved these deviations from the base policy on 2026-09-23:
|
||||
|
||||
- Audita's needs govern policy adaptation rather than strict template compliance.
|
||||
- Only Linux amd64 and arm64 are required; macOS arm64 is best effort.
|
||||
- A successful tag push completes the release; asynchronous CI cannot block it.
|
||||
- Gates are minimal, automated where practical, and avoid redundant validation
|
||||
and human intervention.
|
||||
|
||||
The [release procedure](../release.md) and its linked scripts implement these
|
||||
requirements. CI delivery remains asynchronous.
|
||||
300
docs/policy/testing.md
Normal file
300
docs/policy/testing.md
Normal file
@@ -0,0 +1,300 @@
|
||||
# Testing Policy
|
||||
|
||||
**Base Policy Version:** 1.0
|
||||
|
||||
**Last Updated**: 2026-09-23
|
||||
|
||||
## Purpose
|
||||
|
||||
Our tests exist to make **incorrect changes expensive and correct changes cheap**.
|
||||
|
||||
We do not optimize for test count, line coverage, exhaustive isolation, or the fewest possible tests. We optimize for sufficient confidence in important behavior while imposing as little unnecessary friction as possible on future development.
|
||||
|
||||
## Every test has a cost
|
||||
|
||||
Testing is not an unqualified good. Every test imposes both an immediate cost and a continuing lifetime cost.
|
||||
|
||||
A test must be:
|
||||
|
||||
- written and reviewed;
|
||||
- understood by future maintainers and coding agents;
|
||||
- executed in local and CI workflows;
|
||||
- diagnosed when it fails;
|
||||
- updated when legitimate behavior changes;
|
||||
- maintained as fixtures, APIs, and dependencies evolve; and
|
||||
- removed or rewritten when it becomes redundant, brittle, misleading, or obsolete.
|
||||
|
||||
Tests also create cognitive and architectural friction. They can constrain refactoring, duplicate policy, slow feedback loops, add noise to failures, and cause harmless implementation changes to require unrelated edits across the suite.
|
||||
|
||||
A test is warranted only when the confidence it provides justifies these costs.
|
||||
|
||||
Apply this cost-benefit analysis at two levels:
|
||||
|
||||
1. **Per test:** What realistic defect does this test detect, how consequential would that defect be, and is that protection worth the test's lifetime cost?
|
||||
2. **Across the suite:** Does this collection provide materially more confidence than a smaller, simpler suite would?
|
||||
|
||||
The preferred test suite is a **lean suite that provides sufficient confidence in the risks that matter, without redundant or low-value tests**. We seek sufficient confidence with the least unnecessary testing friction, not the fewest possible tests.
|
||||
|
||||
Some friction is intentional. Tests should make dangerous changes—such as breaking compatibility, corrupting data, violating security boundaries, or reintroducing subtle bugs—require deliberate review. They should not make ordinary internal changes needlessly expensive.
|
||||
|
||||
The cost of a test is not a reason to omit testing by default. Do not cite maintenance cost abstractly. When omitting a plausible test, be able to state why the protected failure is low-risk, already covered, obvious, reversible, or cheaper to detect elsewhere. For consequential, subtle, or difficult-to-observe behavior, the presumption should favor testing.
|
||||
|
||||
## Default testing style
|
||||
|
||||
Use a **classical/Detroit-style** approach:
|
||||
|
||||
- Test observable behavior, resulting state, contracts, and invariants.
|
||||
- Use real internal collaborators when they are fast and deterministic.
|
||||
- Use fakes, stubs, or mocks primarily at expensive, nondeterministic, destructive, or external boundaries.
|
||||
- Prefer package-level behavioral tests over tests coupled to private helpers or internal call sequences.
|
||||
- Treat exact collaborator interactions as testable behavior only when the interaction itself is a requirement.
|
||||
|
||||
Examples of appropriate seams include clocks, randomness, subprocesses, remote APIs, object storage, email, and paid LLM calls.
|
||||
|
||||
## Test execution requirements
|
||||
|
||||
Tests in the default suite must be deterministic, offline, and independent of real credentials. They must not invoke paid APIs or depend on mutable external services. Tests that require live infrastructure must be explicitly opt-in and clearly separated from the default suite.
|
||||
|
||||
Control clocks, randomness, environment variables, and other process-global or machine-specific state when they affect behavior. Tests should be safe to run repeatedly and alongside other tests without depending on execution order or state left by an earlier test.
|
||||
|
||||
## What deserves tests
|
||||
|
||||
To the extent the following features are present in the application, prioritize tests for:
|
||||
|
||||
1. Public and package-level contracts.
|
||||
2. Domain rules and important invariants.
|
||||
3. Boundary conditions and malformed input.
|
||||
4. Failure handling, cancellation, retries, recovery, and partial success.
|
||||
5. Serialization, schemas, compatibility, and round trips.
|
||||
6. Previously observed or plausible regressions.
|
||||
7. Representative integration and end-to-end workflows.
|
||||
|
||||
A package-level contract is behavior relied upon by another package or major collaborator, not every observable detail of a package implementation.
|
||||
|
||||
For behavior involving **data integrity, destructive operations, compatibility, security, concurrency, idempotency, or recovery**, presume that durable tests are required unless the behavior is already credibly protected at another layer.
|
||||
|
||||
Do not add tests merely because a function, branch, or line exists. Do not add a test when the same meaningful risk is already adequately protected elsewhere.
|
||||
|
||||
## Choose the right test boundary
|
||||
|
||||
Test through the narrowest stable boundary that expresses the behavior clearly.
|
||||
|
||||
This is often the package API, but it may instead be:
|
||||
|
||||
- a smaller pure function when dense domain logic is most clearly isolated there;
|
||||
- a package-level operation when several internal collaborators jointly produce the behavior; or
|
||||
- a larger integration boundary when correctness emerges from interaction with a real dependency.
|
||||
|
||||
Do not force all behavior through oversized end-to-end tests. Do not test every private helper merely because it exists. Choose the boundary that gives durable confidence with the least incidental coupling.
|
||||
|
||||
## Test behavior, not implementation
|
||||
|
||||
A test should protect a decision, contract, or invariant. A test must not memorialize the current implementation.
|
||||
|
||||
Before adding or retaining a test, ask:
|
||||
|
||||
> What realistic defect would this test catch?
|
||||
|
||||
A test is suspect when its main purpose is to detect that someone:
|
||||
|
||||
- changed an internal constant;
|
||||
- renamed or split a private helper;
|
||||
- reordered equivalent internal operations;
|
||||
- changed incidental formatting;
|
||||
- replaced one correct algorithm with another; or
|
||||
- refactored internal object structure without changing behavior.
|
||||
|
||||
Refactoring should normally require no test edits unless the refactored structure is itself part of the contract.
|
||||
|
||||
A test can be factually correct and still have negative value. Accurately describing current behavior is not sufficient; the protected behavior must be important enough to justify the future friction.
|
||||
|
||||
## Expected effects of different changes
|
||||
|
||||
Use the following expectations when evaluating test failures and test maintenance:
|
||||
|
||||
| Change | Expected effect on tests |
|
||||
|---|---|
|
||||
| Internal refactor that preserves behavior | Existing tests should normally remain unchanged and continue to pass. |
|
||||
| Change to an internal default with no contractual significance | Behavioral tests should normally remain unchanged; tests should derive expectations from configuration or relationships rather than duplicate the old value. |
|
||||
| Intentional change to public behavior, policy, schema, or compatibility guarantees | The relevant tests should be reviewed and changed deliberately. |
|
||||
| Accidental violation of a contract or invariant | Tests should fail; fix the production code rather than rewriting the tests to accept the defect. |
|
||||
|
||||
A test failing is not the same as a test needing to be edited. Many tests may correctly fail because of one production defect. The maintenance smell is a correct internal change that requires unrelated expectation updates throughout the suite.
|
||||
|
||||
## Separate mechanism from policy
|
||||
|
||||
Configurable thresholds and defaults must not be duplicated throughout the test suite.
|
||||
|
||||
For example, do not encode an internal concurrency limit indirectly:
|
||||
|
||||
```go
|
||||
// Production policy:
|
||||
const maxConcurrency = 4
|
||||
|
||||
// Brittle test:
|
||||
err := startProcesses(5)
|
||||
require.Error(t, err)
|
||||
```
|
||||
|
||||
Instead, test the mechanism relationally:
|
||||
|
||||
```go
|
||||
const limit = 2
|
||||
runner := NewRunner(limit)
|
||||
|
||||
require.NoError(t, runner.Start(limit))
|
||||
require.ErrorIs(t, runner.Start(limit+1), ErrTooMuchConcurrency)
|
||||
```
|
||||
|
||||
The test should prove:
|
||||
|
||||
- the configured limit is accepted; and
|
||||
- one beyond the configured limit is rejected.
|
||||
|
||||
The production default should be tested exactly only when its literal value is itself a public, operational, safety, protocol, or compatibility requirement.
|
||||
|
||||
Apply the same rule to limits, timeouts, capacities, retry counts, and ranges: test relationships and behavior, not duplicated literals.
|
||||
|
||||
For concurrency limits, test both kinds of behavior when relevant:
|
||||
|
||||
1. **Configuration enforcement:** invalid or excessive requested values are handled correctly.
|
||||
2. **Runtime enforcement:** observed peak concurrency never exceeds the configured limit.
|
||||
|
||||
Use a test-controlled limit and measure the behavior relative to that limit. Do not merely assert today's default value.
|
||||
|
||||
## Avoid semantic duplication across layers
|
||||
|
||||
Each behavior should have a clear test owner.
|
||||
|
||||
- Parser tests own parsing cases.
|
||||
- Validator tests own validation rules.
|
||||
- Domain tests own transformations and invariants.
|
||||
- Adapter tests own external integration behavior.
|
||||
- Orchestrator tests own coordination and failure propagation.
|
||||
- CLI tests own argument and configuration mapping.
|
||||
- End-to-end tests prove that representative assembled workflows work.
|
||||
|
||||
Higher-level tests should not repeat every lower-level case. A single intentional policy change should not require unrelated edits across many test files.
|
||||
|
||||
Tests that are individually reasonable may still be collectively redundant. Evaluate the marginal value of each additional test in light of the protection already provided by the rest of the suite.
|
||||
|
||||
## Use test doubles deliberately
|
||||
|
||||
Choose the least elaborate test double that provides the required control or observation.
|
||||
|
||||
As a default:
|
||||
|
||||
1. Prefer real collaborators when they are fast and deterministic.
|
||||
2. Use small in-memory fakes when realistic stateful behavior is helpful.
|
||||
3. Use stubs when a dependency only needs to provide controlled responses.
|
||||
4. Use mocks when the interaction itself is contractual.
|
||||
|
||||
Mocks are appropriate when the contract includes facts such as:
|
||||
|
||||
- a notification is sent exactly once;
|
||||
- a transaction is committed only after successful writes;
|
||||
- cancellation reaches a subprocess;
|
||||
- an expensive API is called no more than once; or
|
||||
- a security audit event is emitted.
|
||||
|
||||
Do not use mocks merely to isolate every object or reproduce the implementation's call graph.
|
||||
|
||||
## Go-specific guidance
|
||||
|
||||
Use:
|
||||
|
||||
- table-driven tests for meaningful behavioral categories and boundaries;
|
||||
- `t.TempDir()` for real filesystem behavior;
|
||||
- `httptest.Server` for realistic HTTP interactions;
|
||||
- fuzz tests for parsers, normalization, path handling, and broad input spaces;
|
||||
- golden files only when the complete output is intentionally stable;
|
||||
- integration tests where correctness depends on component interaction; and
|
||||
- a small number of representative end-to-end tests.
|
||||
|
||||
Avoid exact error-string assertions unless the wording is itself contractual. Prefer `errors.Is`, `errors.As`, typed errors, or structured error fields.
|
||||
|
||||
At CLI boundaries, prefer exit classifications, structured output, and the smallest stable semantic fragment needed to identify the error. Do not snapshot complete diagnostic wording unless it is contractual.
|
||||
|
||||
Golden-file updates must require an explicit local flag. CI must not update golden files automatically, and reviewers must inspect the semantic diff before accepting an update.
|
||||
|
||||
Keep tests readable and direct. Test helpers and fixture frameworks must earn their own maintenance cost; do not build elaborate test infrastructure for small or isolated needs.
|
||||
|
||||
## Coverage
|
||||
|
||||
Coverage is a diagnostic, not a target.
|
||||
|
||||
Use it to find untested critical branches and unexpectedly weak packages. Do not write low-value tests solely to increase a percentage, and do not infer test quality from coverage alone.
|
||||
|
||||
Pure domain logic will often warrant higher coverage than CLI wiring or external adapters. Uneven coverage is acceptable when it reflects risk.
|
||||
|
||||
Increasing coverage is valuable only when the newly covered behavior protects a meaningful risk at an acceptable cost.
|
||||
|
||||
## Regression tests
|
||||
|
||||
A bug fix should normally include a regression test that fails before the fix and passes afterward.
|
||||
|
||||
Retain the test when the defect could realistically recur and its consequences justify the ongoing cost. Prefer the narrowest durable test of the violated contract or invariant; do not preserve accidental implementation details from the original bug.
|
||||
|
||||
Not every historical bug requires a permanent test. If the underlying design has made recurrence impossible, the test has become redundant, or a stronger invariant test now subsumes it, remove or consolidate it.
|
||||
|
||||
## Deleting or rewriting tests
|
||||
|
||||
Tests are maintained code, not permanent historical artifacts.
|
||||
|
||||
Delete or rewrite a test when its maintenance cost exceeds the confidence it provides.
|
||||
|
||||
Strong candidates include tests that:
|
||||
|
||||
- require updates after harmless internal changes;
|
||||
- directly assert private constants without protecting a real contract;
|
||||
- duplicate the same policy across several layers;
|
||||
- verify mock choreography rather than outcomes;
|
||||
- snapshot large amounts of incidental output;
|
||||
- test trivial private helpers already exercised through stable package behavior;
|
||||
- protect risks already covered more effectively elsewhere;
|
||||
- are flaky, misleading, obsolete, or disproportionately expensive to diagnose; or
|
||||
- no longer correspond to a plausible failure mode.
|
||||
|
||||
Several brittle tests may encode one genuine requirement. Replace them with one durable behavior-level or invariant test rather than preserving all of them.
|
||||
|
||||
Deleting a low-value test can improve the quality of the suite by reducing noise, maintenance burden, and friction around legitimate change.
|
||||
|
||||
## Reviewing a proposed test
|
||||
|
||||
Use the following questions when the value, boundary, or durability of a proposed test is not self-evident. Significant test additions should be reviewable against them, but written answers are not required for every routine test.
|
||||
|
||||
1. What realistic defect would it catch?
|
||||
2. How likely is that defect?
|
||||
3. How consequential would it be?
|
||||
4. Is the behavior already protected elsewhere?
|
||||
5. At which layer should this behavior be owned?
|
||||
6. Does the test assert a durable contract or an incidental implementation detail?
|
||||
7. Could the implementation be refactored without changing the behavior and without editing this test?
|
||||
8. What should cause this test to fail?
|
||||
9. What legitimate changes should not cause this test to fail?
|
||||
10. What ongoing maintenance, execution, and diagnostic cost will the test impose?
|
||||
11. Is there a smaller or more direct test that protects the same risk?
|
||||
|
||||
Do not add the test when its expected lifetime cost exceeds its expected protective value.
|
||||
|
||||
When deciding not to test plausible behavior, record or be able to explain why the risk is low, already protected, obvious, reversible, or cheaper to detect elsewhere.
|
||||
|
||||
## Definition of sufficient
|
||||
|
||||
A test suite is sufficient when:
|
||||
|
||||
- important contracts and invariants are protected;
|
||||
- meaningful boundaries and failure modes are exercised;
|
||||
- realistic and consequential regressions are credibly protected against silent recurrence;
|
||||
- behavior involving data integrity, destructive operations, compatibility, security, concurrency, idempotency, and recovery is credibly protected;
|
||||
- important external boundaries have realistic integration coverage;
|
||||
- representative complete workflows are tested;
|
||||
- failures provide useful signal rather than redundant noise;
|
||||
- legitimate internal changes usually do not require test edits; and
|
||||
- additional tests would mostly repeat existing protection or preserve inconsequential implementation details.
|
||||
|
||||
Sufficiency is a risk judgment, not a coverage percentage or test count. Reassess it as the application, its users, and the consequences of failure evolve.
|
||||
|
||||
The governing rule is:
|
||||
|
||||
> Test heavily where failure is consequential, subtle, or difficult to detect after the fact. Test lightly where failure is obvious, reversible, and inexpensive—and retain no test whose lifetime cost exceeds the confidence it provides.
|
||||
130
docs/release.md
Normal file
130
docs/release.md
Normal file
@@ -0,0 +1,130 @@
|
||||
# Releasing Audita
|
||||
|
||||
The [release policy](policy/release.md) defines versioning, supported platforms,
|
||||
immutability, and success. This runbook implements it with
|
||||
[scripts/release.sh](../scripts/release.sh). Publication ends after a successful
|
||||
source/tag push; binary delivery runs asynchronously.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Use Bash, Git, and Go 1.25 or newer (CI uses Go 1.25), with dependencies available
|
||||
locally or downloadable. Tests themselves are offline. Work from a
|
||||
checkout of `main` with permission to push that branch and tags to `origin`.
|
||||
For this repository, `origin` is `https://gitea.maximumdirect.net/eric/audita.git`.
|
||||
The script uses the checkout's configured `origin`; set it correctly before use.
|
||||
The remote must support atomic pushes. No CI credentials or Mac are needed locally.
|
||||
|
||||
CI uses the existing `GITEA_RELEASE_TOKEN` secret for asynchronous asset delivery.
|
||||
macOS arm64 is a best-effort development environment; no macOS binary is published.
|
||||
|
||||
## Prepare the Candidate
|
||||
|
||||
Choose an unused version following the policy. Write
|
||||
`docs/releases/<version>.md`, beginning with `# Audita <version>`, followed by a
|
||||
short summary and any compatibility or upgrade instructions relevant to the
|
||||
release. There are no mandatory boilerplate sections. Update affected canonical
|
||||
documentation with behavior changes.
|
||||
|
||||
Commit the intended changes, including the note, on `main`. The release script
|
||||
requires a clean index and worktree and does not create commits. There is no
|
||||
separate approval round. Published notes describe their tagged release; existing
|
||||
release history does not need to be reconstructed.
|
||||
|
||||
## Validate and Publish
|
||||
|
||||
From the repository root, substitute the chosen version:
|
||||
|
||||
```sh
|
||||
bash scripts/release.sh --publish v0.2.0
|
||||
```
|
||||
|
||||
This single invocation:
|
||||
|
||||
1. Checks the version, branch, clean candidate, matching note title, and tag absence.
|
||||
2. Runs the default offline test suite once, then builds both required Linux targets
|
||||
in a temporary directory using the same build script as CI.
|
||||
3. Checks that the candidate and worktree are unchanged and that it includes the
|
||||
current `origin/main` history.
|
||||
4. Creates the version tag at that commit and atomically pushes the exact source
|
||||
commit to `main` and the exact tag to `origin`.
|
||||
5. Reports release success and exits without polling CI or checking asset availability.
|
||||
|
||||
To validate a prepared candidate without remote access, creating a tag, or pushing:
|
||||
|
||||
```sh
|
||||
bash scripts/release.sh --check v0.2.0
|
||||
```
|
||||
|
||||
This is an optional diagnostic command, not a required step before `--publish`.
|
||||
Publication validates within its own invocation; it does not use persistent cached
|
||||
results. Both modes require the candidate note and a clean `main` checkout.
|
||||
|
||||
## Artifact Delivery
|
||||
|
||||
[Woodpecker](../.woodpecker/release.yml) runs on tag events. It invokes
|
||||
[scripts/build-release.sh](../scripts/build-release.sh), then publishes:
|
||||
|
||||
- `audita-<version>-linux-amd64`
|
||||
- `audita-<version>-linux-arm64`
|
||||
- `SHA256SUMS`
|
||||
|
||||
Builds disable CGO and use the shared script's release flags. The version is in the
|
||||
asset name and source tag; Audita has no CLI version command. Cross-compilation
|
||||
checks compilation, not native execution on each architecture.
|
||||
|
||||
The CI job status and logs show delivery progress or failure. Assets appear on
|
||||
the matching Gitea release page when delivery succeeds. A successful tag push
|
||||
must not be reported as confirmation that these files are available.
|
||||
|
||||
Consumers can download the binary for their Linux architecture and `SHA256SUMS`
|
||||
from that page, check its listed SHA-256 digest with `sha256sum`, and make it
|
||||
executable. Optional consumer checks are not maintainer release gates. Digests
|
||||
detect file mismatches; they are not an independent signature of authenticity.
|
||||
|
||||
## Failure and Retry
|
||||
|
||||
**Local validation fails:** correct the issue, commit any changes, and rerun the
|
||||
publication command. A failure before tag creation publishes nothing. Temporary
|
||||
local build files are removed on exit.
|
||||
|
||||
**Push fails or the connection is interrupted:** the script retains the local tag
|
||||
and reports failure rather than guessing whether publication succeeded. Inspect
|
||||
remote state with the actual version:
|
||||
|
||||
```sh
|
||||
git rev-parse 'v0.2.0^{commit}'
|
||||
git ls-remote origin refs/heads/main refs/tags/v0.2.0
|
||||
```
|
||||
|
||||
If the remote tag identifies the intended candidate and its source is on remote
|
||||
`main`, the release succeeded; no CI waiting is needed. An atomic push normally
|
||||
updates both refs or neither. If `main` has advanced since publication, fetch it
|
||||
and check candidate ancestry rather than requiring tip equality.
|
||||
|
||||
If the tag is absent remotely and was never published or used by a release
|
||||
channel, remove only that unpublished local tag (`git tag -d v0.2.0`), resolve
|
||||
any rejected branch update, and rerun the command. If the remote tag has a
|
||||
different target, do not overwrite it; choose a fresh version. Never force a
|
||||
branch or tag update to bypass the guards.
|
||||
|
||||
**CI delivery fails:** the release remains successful. Inspect the failed tag
|
||||
job's logs and retry that same tagged job for a transient failure. Publication is
|
||||
configured to skip existing filenames and not overwrite assets. Keep the same
|
||||
source and build environment when retrying; inspect any existing assets/checksum
|
||||
manifest before filling gaps. Do not regenerate and replace a published manifest
|
||||
or binary. If an existing manifest cannot describe the missing rebuilt assets,
|
||||
use a new version instead. A pipeline or source defect also requires a corrected
|
||||
commit and new version. An outage does not require moving the tag.
|
||||
|
||||
**A released source defect is discovered:** fix it on `main` and publish a new
|
||||
appropriate version with a note explaining the correction. Leave the old tag and
|
||||
published files intact.
|
||||
|
||||
## Maintaining the Release Workflow
|
||||
|
||||
The shared build script keeps local and CI build settings aligned. Tests in
|
||||
[scripts/release_test.go](../scripts/release_test.go) exercise publication against
|
||||
temporary local Git remotes, including guard failures and rejected pushes, with
|
||||
Go execution stubbed to avoid recursive builds/tests. They run in `go test ./...`.
|
||||
Changes to CI delivery should preserve explicit asset names, checksums, and
|
||||
non-overwriting publication. CI delivery is separate from the local success path.
|
||||
8
docs/releases/README.md
Normal file
8
docs/releases/README.md
Normal file
@@ -0,0 +1,8 @@
|
||||
# Release Notes
|
||||
|
||||
- [Audita v1.1.0](v1.1.0.md)
|
||||
|
||||
See the [release documentation requirements](../policy/release.md#release-documentation).
|
||||
Use the [release procedure](../release.md) to prepare and publish new releases.
|
||||
|
||||
This index does not reconstruct or replace records for existing tags.
|
||||
22
docs/releases/v1.1.0.md
Normal file
22
docs/releases/v1.1.0.md
Normal file
@@ -0,0 +1,22 @@
|
||||
# Audita v1.1.0
|
||||
|
||||
This release refreshes Audita's documentation and repository policies and adds a
|
||||
small, automated release workflow. Transcript-processing behavior, CLI syntax,
|
||||
configuration, and serialized output formats are unchanged from v1.0.0.
|
||||
|
||||
- Consolidated CLI, configuration, operations, and integration references, including
|
||||
a dedicated reference for output files and reports.
|
||||
- Added contributor navigation and agent, testing, and release policies; retired
|
||||
speculative troubleshooting documentation and the completed migration roadmap.
|
||||
- Added local release validation and publication scripts, with publication complete
|
||||
after the source and version tag are pushed. CI delivers binaries asynchronously.
|
||||
- Isolated CLI tests from host configuration for a reliable local test baseline.
|
||||
|
||||
Release binaries are now provided only for Linux amd64 and Linux arm64. Windows
|
||||
and Intel macOS are no longer supported. macOS arm64 remains a best-effort local
|
||||
build/development environment; macOS binaries are no longer published.
|
||||
|
||||
Linux users need no configuration or data migration. Users of previously published
|
||||
Windows or macOS binaries should account for the distribution changes above before
|
||||
upgrading. See the [release procedure](../release.md) for artifact delivery and the
|
||||
[contributor guide](../development.md) for local build instructions.
|
||||
17
docs/roadmap/runtime-behavior.md
Normal file
17
docs/roadmap/runtime-behavior.md
Normal file
@@ -0,0 +1,17 @@
|
||||
# Runtime Behavior Decisions
|
||||
|
||||
**Last updated:** 2026-09-23
|
||||
|
||||
The documentation migration identified these existing behaviors for separate
|
||||
product consideration. No behavior change is approved or implemented by this list.
|
||||
|
||||
- Decide whether the `never` retention selection should delete successful-run
|
||||
diagnostics; current behavior is documented in
|
||||
[operations](../operations.md#retention-behavior).
|
||||
- Decide whether an explicit zero retry count should disable retries; current
|
||||
adapter behavior is documented in [configuration](../config.md).
|
||||
- Decide whether missing or null input timestamps should be rejected; current
|
||||
decoding is documented in the [input contract](../integrations/transcript-glossary-files.md).
|
||||
|
||||
These decisions are independent of the completed documentation migration and
|
||||
release workflow.
|
||||
@@ -1,153 +0,0 @@
|
||||
# Audita Troubleshooting
|
||||
|
||||
## Scope
|
||||
|
||||
This guide lists recurring implemented failure modes for `audita process` and `audita config`.
|
||||
|
||||
For each entry: symptom, likely cause, inspect, and fix.
|
||||
|
||||
## Config Validation Fails
|
||||
|
||||
Symptom:
|
||||
- `audita config validate --config <path>` exits nonzero.
|
||||
|
||||
Likely causes:
|
||||
- missing `version`;
|
||||
- unsupported config version;
|
||||
- unknown YAML field;
|
||||
- unsupported module key or output schema;
|
||||
- invalid numeric/range/concurrency/retention values.
|
||||
|
||||
Inspect:
|
||||
1. rerun `audita config validate --config <path>` and read stderr.
|
||||
2. if needed, inspect effective config with `audita config print-effective --config <path>`.
|
||||
|
||||
Fix:
|
||||
- set `version: 1`;
|
||||
- remove unknown fields;
|
||||
- use supported module keys and output schemas (`bare-segments`, `audita-v1`);
|
||||
- correct invalid values to satisfy validation constraints.
|
||||
|
||||
## Config File Resolution Errors
|
||||
|
||||
Symptom:
|
||||
- `audita process` fails before processing with config-related errors like `config file not found`.
|
||||
|
||||
Likely causes:
|
||||
- `--config` points to a missing path;
|
||||
- `AUDITA_CONFIG` points to a missing path;
|
||||
- unreadable config path.
|
||||
|
||||
Inspect:
|
||||
1. confirm `--config` or `AUDITA_CONFIG` path exists;
|
||||
2. run `audita config validate --config <path>` directly.
|
||||
|
||||
Fix:
|
||||
- correct the path or unset invalid `AUDITA_CONFIG`;
|
||||
- fix permissions for the config file.
|
||||
|
||||
## Transcript or Glossary Schema Errors
|
||||
|
||||
Symptom:
|
||||
- stderr includes `transcript_schema` or `glossary_schema` and run exits nonzero.
|
||||
|
||||
Likely causes:
|
||||
- transcript is not valid JSON or has invalid segment fields;
|
||||
- glossary is not valid YAML or has missing required glossary entry fields.
|
||||
|
||||
Inspect:
|
||||
1. check stderr for parser/validation details;
|
||||
2. if diagnostics were created, inspect `error.log` and run `report.json` (`error_phase`);
|
||||
3. inspect `source-transcript.json` and `source-transcript-parsed.json` in the run directory.
|
||||
|
||||
Fix:
|
||||
- correct transcript JSON shape/content;
|
||||
- correct glossary YAML shape/content and required entry fields;
|
||||
- rerun validation with known-good tiny examples for comparison:
|
||||
- `examples/tiny-transcript.json`
|
||||
- `examples/tiny-glossary.yaml`
|
||||
|
||||
## LLM Runtime/Backend Failures
|
||||
|
||||
Symptom:
|
||||
- stderr includes `runner_execution` (or backend timeout/error details) and nonzero exit.
|
||||
|
||||
Likely causes:
|
||||
- unreachable/failed LLM endpoint;
|
||||
- timeout/cancellation;
|
||||
- runtime module execution failure.
|
||||
|
||||
Inspect:
|
||||
1. inspect stderr for backend message details;
|
||||
2. inspect run `report.json` (`error_phase`, `module_results`);
|
||||
3. inspect diagnostics payloads and `error.log`.
|
||||
|
||||
Fix:
|
||||
- verify model/base URL/API key settings;
|
||||
- increase timeout if needed;
|
||||
- rerun with `--work-dir-retention always` while debugging.
|
||||
|
||||
## Output File Write Failure
|
||||
|
||||
Symptom:
|
||||
- stderr includes `failed to write output file` and run exits nonzero.
|
||||
|
||||
Likely causes:
|
||||
- output path directory missing;
|
||||
- insufficient filesystem permissions;
|
||||
- invalid output target path.
|
||||
|
||||
Inspect:
|
||||
1. check `--output` target directory exists and is writable;
|
||||
2. inspect run diagnostics `error.log` and report `error_phase`.
|
||||
|
||||
Fix:
|
||||
- write to a valid writable path;
|
||||
- create missing directories;
|
||||
- adjust permissions.
|
||||
|
||||
## Report File Write Failure
|
||||
|
||||
Symptom:
|
||||
- stderr includes `failed to write report JSON file` and run exits nonzero.
|
||||
|
||||
Likely causes:
|
||||
- invalid or unwritable `--report-json` target path.
|
||||
|
||||
Inspect:
|
||||
1. verify parent directory exists and is writable;
|
||||
2. inspect diagnostics `error.log` for `report_write` context.
|
||||
|
||||
Fix:
|
||||
- choose a writable report path;
|
||||
- create missing directories;
|
||||
- rerun.
|
||||
|
||||
## Unsupported Output Schema
|
||||
|
||||
Symptom:
|
||||
- stderr includes `unsupported output schema` and run exits nonzero.
|
||||
|
||||
Likely causes:
|
||||
- unsupported `--output-schema` value;
|
||||
- unsupported `output.schema` in config.
|
||||
|
||||
Inspect:
|
||||
1. check CLI/config schema key;
|
||||
2. run `audita config validate --config <path>` when config is involved.
|
||||
|
||||
Fix:
|
||||
- use `bare-segments` or `audita-v1`.
|
||||
|
||||
## Diagnostics Directory Lookup
|
||||
|
||||
Symptom:
|
||||
- run fails and you need artifacts for debugging.
|
||||
|
||||
Inspect:
|
||||
1. read stderr for `audita process: diagnostics: <run-dir>`;
|
||||
2. open `<run-dir>/report.json` and `<run-dir>/error.log`;
|
||||
3. use diagnostics paths embedded in report metadata for artifact lookup.
|
||||
|
||||
Fix:
|
||||
- rerun with `--work-dir-retention always` to preserve run directories during investigation.
|
||||
@@ -21,6 +21,7 @@ concurrency:
|
||||
proposal_llm: 2
|
||||
validation_llm: 1
|
||||
chunking:
|
||||
# Requires at least this many normalized segments; omit for automatic chunking.
|
||||
target_sections: 8
|
||||
max_section_tokens: 8192
|
||||
min_section_tokens: 2048
|
||||
|
||||
36
internal/cli/main_test.go
Normal file
36
internal/cli/main_test.go
Normal file
@@ -0,0 +1,36 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// Tests supply their own overrides; inherited settings must not affect them.
|
||||
for _, entry := range os.Environ() {
|
||||
key, _, _ := strings.Cut(entry, "=")
|
||||
if strings.HasPrefix(key, "AUDITA_") || key == "OPENROUTER_API_KEY" {
|
||||
if err := os.Unsetenv(key); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dir, err := os.MkdirTemp("", "audita-cli-test-config-")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
configPath := filepath.Join(dir, "config.yml")
|
||||
if err := os.WriteFile(configPath, []byte("version: 1\n"), 0600); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// An explicit environment path prevents discovery of installed system config.
|
||||
// Individual tests can still override it with t.Setenv or --config.
|
||||
if err := os.Setenv("AUDITA_CONFIG", configPath); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
m.Run()
|
||||
}
|
||||
@@ -1173,6 +1173,15 @@ func waitForAtomicAtLeast(t *testing.T, value *int32, want int32) {
|
||||
}
|
||||
|
||||
func TestRunProcessReportJSONSuccessIncludesNormalizationSummary(t *testing.T) {
|
||||
// Exercise the no-config metadata contract without consulting system files.
|
||||
t.Setenv("AUDITA_CONFIG", "")
|
||||
if err := os.Unsetenv("AUDITA_CONFIG"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
searchPaths := config.DefaultConfigSearchPaths
|
||||
config.DefaultConfigSearchPaths = nil
|
||||
t.Cleanup(func() { config.DefaultConfigSearchPaths = searchPaths })
|
||||
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
@@ -1350,7 +1359,7 @@ func TestRunProcessReportJSONAndRunDirReportShareDiagnosticsMetadata(t *testing.
|
||||
if externalReport.Diagnostics == nil || runDirReport.Diagnostics == nil {
|
||||
t.Fatalf("expected diagnostics metadata in both reports")
|
||||
}
|
||||
if externalReport.ReportMetadata != runDirReport.ReportMetadata {
|
||||
if !reflect.DeepEqual(externalReport.ReportMetadata, runDirReport.ReportMetadata) {
|
||||
t.Fatalf("expected same report metadata in --report-json and run-dir report\nexternal=%+v\nrun-dir=%+v",
|
||||
externalReport.ReportMetadata, runDirReport.ReportMetadata)
|
||||
}
|
||||
|
||||
17
scripts/build-release.sh
Executable file
17
scripts/build-release.sh
Executable file
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $# != 2 || ! $1 =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then
|
||||
echo 'Usage: scripts/build-release.sh vMAJOR.MINOR.PATCH OUTPUT_DIRECTORY' >&2
|
||||
exit 2
|
||||
fi
|
||||
version=$1
|
||||
output=$2
|
||||
mkdir -p "$output"
|
||||
output=$(cd "$output" && pwd)
|
||||
cd "$(dirname "$0")/.."
|
||||
for arch in amd64 arm64; do
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=$arch GOWORK=off \
|
||||
go build -mod=readonly -trimpath -ldflags '-s -w' \
|
||||
-o "$output/audita-$version-linux-$arch" ./cmd/audita
|
||||
done
|
||||
48
scripts/release.sh
Executable file
48
scripts/release.sh
Executable file
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
fail() { echo "Release stopped: $*" >&2; exit 1; }
|
||||
if [[ $# != 2 || ( $1 != --check && $1 != --publish ) || ! $2 =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then
|
||||
echo 'Usage: scripts/release.sh --check|--publish vMAJOR.MINOR.PATCH' >&2
|
||||
exit 2
|
||||
fi
|
||||
mode=$1
|
||||
version=$2
|
||||
cd "$(dirname "$0")/.."
|
||||
[[ $(git symbolic-ref --quiet --short HEAD) == main ]] || fail 'switch to main'
|
||||
[[ -z $(git status --porcelain) ]] || fail 'commit or remove outstanding changes first'
|
||||
candidate=$(git rev-parse HEAD)
|
||||
note="docs/releases/$version.md"
|
||||
[[ -s $note ]] || fail "missing release note: $note"
|
||||
IFS= read -r title < "$note" || true
|
||||
[[ $title == "# Audita $version" ]] || fail "note must begin with # Audita $version"
|
||||
git show-ref --verify --quiet "refs/tags/$version" && fail 'version already exists locally'
|
||||
|
||||
# Check mode is local and never contacts a remote or creates a tag.
|
||||
if [[ $mode == --publish ]]; then
|
||||
existing=$(git ls-remote --tags origin "refs/tags/$version")
|
||||
[[ -z $existing ]] || fail 'version already exists on origin'
|
||||
fi
|
||||
build_dir=$(mktemp -d)
|
||||
trap 'rm -rf "$build_dir"' EXIT
|
||||
GOWORK=off go test -mod=readonly ./...
|
||||
bash scripts/build-release.sh "$version" "$build_dir"
|
||||
[[ $(git rev-parse HEAD) == "$candidate" ]] || fail 'HEAD changed during validation'
|
||||
[[ -z $(git status --porcelain) ]] || fail 'worktree changed during validation'
|
||||
[[ $(git symbolic-ref --quiet --short HEAD) == main ]] || fail 'branch changed during validation'
|
||||
if [[ $mode == --check ]]; then
|
||||
echo "Validated $version at $candidate; nothing published."
|
||||
exit 0
|
||||
fi
|
||||
# Fetch the authoritative branch before checking that this is a forward update.
|
||||
git fetch --no-tags origin refs/heads/main
|
||||
git merge-base --is-ancestor FETCH_HEAD "$candidate" || fail 'candidate does not include origin/main'
|
||||
existing=$(git ls-remote --tags origin "refs/tags/$version")
|
||||
[[ -z $existing ]] || fail 'version already exists on origin'
|
||||
git tag "$version" "$candidate"
|
||||
[[ $(git rev-parse "refs/tags/$version") == "$candidate" ]] || fail 'tag target changed'
|
||||
# Atomic publication prevents a rejected tag/branch update from publishing only half.
|
||||
if ! git push --atomic origin "$candidate:refs/heads/main" "refs/tags/$version:refs/tags/$version"; then
|
||||
fail "push failed or its outcome is uncertain; inspect origin/main and refs/tags/$version before retrying (local tag retained)"
|
||||
fi
|
||||
echo "Released $version at $candidate. Binary publication runs asynchronously; CI is not awaited."
|
||||
142
scripts/release_test.go
Normal file
142
scripts/release_test.go
Normal file
@@ -0,0 +1,142 @@
|
||||
package scripts
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Real local Git repositories exercise publication; Go is stubbed at the slow
|
||||
// build/test boundary so these tests remain offline and cannot recurse.
|
||||
func TestReleaseWorkflow(t *testing.T) {
|
||||
for _, scenario := range []string{"check", "publish", "dirty", "tests-fail", "build-fail", "existing-remote-tag", "remote-ahead", "rejected-push", "source-changed", "missing-note", "invalid-version", "existing-local-tag", "wrong-branch", "head-changed"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
repo, remote := filepath.Join(root, "repo"), filepath.Join(root, "remote.git")
|
||||
if err := os.MkdirAll(repo, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL="+os.DevNull, "GIT_AUTHOR_NAME=Release Test", "GIT_AUTHOR_EMAIL=test@example.invalid", "GIT_COMMITTER_NAME=Release Test", "GIT_COMMITTER_EMAIL=test@example.invalid", "GIT_TERMINAL_PROMPT=0")
|
||||
run := func(dir string, args ...string) string {
|
||||
t.Helper()
|
||||
c := exec.Command("git", args...)
|
||||
c.Dir = dir
|
||||
c.Env = env
|
||||
b, e := c.CombinedOutput()
|
||||
if e != nil {
|
||||
t.Fatalf("git %v: %v\n%s", args, e, b)
|
||||
}
|
||||
return strings.TrimSpace(string(b))
|
||||
}
|
||||
write := func(path, data string, mode os.FileMode) {
|
||||
t.Helper()
|
||||
if e := os.MkdirAll(filepath.Dir(path), 0755); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e := os.WriteFile(path, []byte(data), mode); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
run(root, "init", "--bare", "--initial-branch=main", remote)
|
||||
run(repo, "init", "--initial-branch=main")
|
||||
for _, name := range []string{"release.sh", "build-release.sh"} {
|
||||
b, e := os.ReadFile(name)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
write(filepath.Join(repo, "scripts", name), string(b), 0755)
|
||||
}
|
||||
write(filepath.Join(repo, "docs/releases/v1.2.3.md"), "# Audita v1.2.3\n\nA test release.\n", 0644)
|
||||
run(repo, "add", ".")
|
||||
run(repo, "commit", "-m", "Initial fixture")
|
||||
run(repo, "remote", "add", "origin", remote)
|
||||
run(repo, "push", "origin", "main")
|
||||
write(filepath.Join(repo, "change.txt"), "candidate\n", 0644)
|
||||
run(repo, "add", ".")
|
||||
run(repo, "commit", "-m", "Candidate fixture")
|
||||
candidate := run(repo, "rev-parse", "HEAD")
|
||||
log := filepath.Join(root, "go.log")
|
||||
stub := "#!/bin/sh\necho \"$* GOOS=$GOOS GOARCH=$GOARCH CGO_ENABLED=$CGO_ENABLED GOWORK=$GOWORK\" >> \"$GO_LOG\"\n"
|
||||
switch scenario {
|
||||
case "existing-local-tag":
|
||||
run(repo, "tag", "v1.2.3")
|
||||
case "wrong-branch":
|
||||
run(repo, "checkout", "-b", "feature")
|
||||
case "head-changed":
|
||||
stub += "if [ \"$1\" = test ]; then echo changed > change.txt; git add change.txt; git commit -m Changed; fi\n"
|
||||
case "dirty":
|
||||
write(filepath.Join(repo, "untracked"), "dirty", 0644)
|
||||
case "tests-fail":
|
||||
stub += "[ \"$1\" != test ] || exit 1\n"
|
||||
case "build-fail":
|
||||
stub += "[ \"$GOARCH\" != arm64 ] || exit 1\n"
|
||||
case "existing-remote-tag":
|
||||
run(remote, "tag", "v1.2.3", "main")
|
||||
case "remote-ahead":
|
||||
other := filepath.Join(root, "other")
|
||||
run(root, "clone", remote, other)
|
||||
write(filepath.Join(other, "other.txt"), "new remote commit", 0644)
|
||||
run(other, "add", ".")
|
||||
run(other, "commit", "-m", "Remote fixture")
|
||||
run(other, "push", "origin", "main")
|
||||
case "rejected-push":
|
||||
write(filepath.Join(remote, "hooks/pre-receive"), "#!/bin/sh\nexit 1\n", 0755)
|
||||
case "source-changed":
|
||||
stub += "[ \"$1\" != test ] || echo changed > change.txt\n"
|
||||
case "missing-note":
|
||||
run(repo, "rm", "docs/releases/v1.2.3.md")
|
||||
run(repo, "commit", "-m", "Remove fixture note")
|
||||
}
|
||||
stub += "exit 0\n"
|
||||
write(filepath.Join(root, "bin/go"), stub, 0755)
|
||||
mode, version := "--publish", "v1.2.3"
|
||||
if scenario == "check" {
|
||||
mode = "--check"
|
||||
run(repo, "remote", "remove", "origin")
|
||||
}
|
||||
if scenario == "invalid-version" {
|
||||
version = "v01.2.3"
|
||||
}
|
||||
c := exec.Command("bash", "scripts/release.sh", mode, version)
|
||||
c.Dir = repo
|
||||
c.Env = append(env, "PATH="+filepath.Join(root, "bin")+":"+os.Getenv("PATH"), "GO_LOG="+log)
|
||||
b, e := c.CombinedOutput()
|
||||
success := scenario == "check" || scenario == "publish"
|
||||
if (e == nil) != success {
|
||||
t.Fatalf("success=%v: %v\n%s", success, e, b)
|
||||
}
|
||||
tags := run(remote, "tag", "--list")
|
||||
if scenario == "publish" {
|
||||
if run(remote, "rev-parse", "refs/tags/v1.2.3") != candidate || run(remote, "rev-parse", "main") != candidate {
|
||||
t.Fatal("wrong source published")
|
||||
}
|
||||
if !strings.Contains(string(b), "CI is not awaited") {
|
||||
t.Fatal("missing asynchronous completion message")
|
||||
}
|
||||
} else if scenario != "existing-remote-tag" && tags != "" {
|
||||
t.Fatalf("unexpected remote tags: %s", tags)
|
||||
}
|
||||
if scenario == "rejected-push" && run(remote, "rev-parse", "main") == candidate {
|
||||
t.Fatal("rejected push published candidate")
|
||||
}
|
||||
if scenario == "check" && run(repo, "tag", "--list") != "" {
|
||||
t.Fatal("check created a tag")
|
||||
}
|
||||
if success {
|
||||
calls, e := os.ReadFile(log)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
lines := strings.Split(strings.TrimSpace(string(calls)), "\n")
|
||||
if len(lines) != 3 {
|
||||
t.Fatalf("expected one test and two builds: %s", calls)
|
||||
}
|
||||
if !strings.HasPrefix(lines[0], "test -mod=readonly ./...") || !strings.Contains(lines[1], "GOOS=linux GOARCH=amd64 CGO_ENABLED=0 GOWORK=off") || !strings.Contains(lines[2], "GOOS=linux GOARCH=arm64 CGO_ENABLED=0 GOWORK=off") {
|
||||
t.Fatalf("unexpected validation: %s", calls)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user