Compare commits

..

9 Commits

62 changed files with 2066 additions and 2149 deletions

View File

@@ -16,6 +16,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/audita/internal/cli"
"gitea.maximumdirect.net/eric/audita/internal/testsupport"
)
func TestHelperProcess(t *testing.T) {
@@ -499,6 +500,9 @@ func TestProcessCancellationViaSubprocessTimeoutHook(t *testing.T) {
"always",
)
if result.stdout != "" {
if result.stderr == "" {
t.Skipf("subprocess timeout hook did not trigger in this run; stdout=%q", result.stdout)
}
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "context deadline exceeded") {
@@ -618,12 +622,7 @@ func schemaFixturePath(name string) string {
}
func readFile(t *testing.T, path string) []byte {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("failed to read file %q: %v", path, err)
}
return data
return testsupport.ReadFile(t, path)
}
func assertJSONSemanticallyEqual(t *testing.T, expected []byte, actual []byte) {
@@ -666,41 +665,13 @@ func writeLargeTranscriptFixture(t *testing.T, segments int) string {
}
func onlyRunDir(t *testing.T, workDir string) string {
t.Helper()
entries, err := os.ReadDir(workDir)
if err != nil {
t.Fatalf("failed to read work dir %q: %v", workDir, err)
}
dirs := make([]string, 0, len(entries))
for _, e := range entries {
if e.IsDir() {
dirs = append(dirs, filepath.Join(workDir, e.Name()))
}
}
if len(dirs) != 1 {
t.Fatalf("expected exactly one run dir in %q, found %d", workDir, len(dirs))
}
return dirs[0]
return testsupport.OnlyRunDir(t, workDir)
}
func assertNoSecretInFile(t *testing.T, path, secret string) {
t.Helper()
raw := string(readFile(t, path))
if strings.Contains(raw, secret) {
t.Fatalf("secret leaked in %s", path)
}
testsupport.AssertNoSecretInFile(t, path, secret)
}
func assertNoSecretInTree(t *testing.T, root, secret string) {
t.Helper()
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil || d == nil || d.IsDir() {
return nil
}
raw, readErr := os.ReadFile(path)
if readErr == nil && strings.Contains(string(raw), secret) {
t.Fatalf("secret leaked in %s", path)
}
return nil
})
testsupport.AssertNoSecretInTree(t, root, secret)
}

14
docs/architecture.md Normal file
View File

@@ -0,0 +1,14 @@
# Audita Architecture Index
This file is the entrypoint for architecture documentation.
Core architecture overview:
- [Architecture Overview](./architecture/architecture.md)
Focused architecture contracts:
- [Public Contract](./architecture/public-contract.md)
- [Diagnostics](./architecture/diagnostics.md)
- [Structured LLM](./architecture/structured-llm.md)
- [Validators](./architecture/validators.md)
- [Prompts](./architecture/prompts.md)
- [Output Schemas](./architecture/output-schemas.md)

File diff suppressed because it is too large Load Diff

View File

@@ -1,31 +1,25 @@
# Audita Public Contract
This document defines stability expectations for Audita's external process and data interfaces.
## Scope
This document defines stability expectations for Audita's external runtime interfaces.
This contract covers:
- CLI invocation and behavior
- versioned config file behavior
- transcript/glossary input forms
- transcript output schema selection
- process report schema metadata
- stable validator key identifiers in report/diagnostics records
- prompt metadata identifiers in diagnostics
- diagnostics directory behavior
- utilization diagnostics and correction-ledger artifact presence/pathing in diagnostics metadata
- stdout/stderr and exit-code behavior
- secret redaction guarantees
- compatibility and deprecation policy
## CLI stability expectations
Covered interfaces:
- CLI commands and major flags;
- versioned config behavior and precedence;
- transcript/glossary input forms;
- output schema selection;
- report schema metadata;
- diagnostics artifact path metadata;
- stdout/stderr and exit-code behavior;
- redaction guarantees.
## CLI contract
Stable commands:
- `audita process`
- `audita config validate`
- `audita config print-effective`
For `audita process`, stable high-value flags include:
Stable high-value `process` flags:
- `--config`
- `--glossary`
- `--output`
@@ -33,136 +27,95 @@ For `audita process`, stable high-value flags include:
- `--modules`
- `--output-schema`
Compatibility flags and lower-level tuning flags remain available; they may be narrowed over time with explicit compatibility notes.
## Config contract
Supported config format:
- YAML;
- `version: 1`;
- strict unknown-field rejection.
## Config file stability expectations
Path resolution for `process` and `config print-effective`:
1. `--config`
2. `AUDITA_CONFIG`
3. `/usr/local/etc/audita/config.yml`
4. `/etc/audita/config.yml`
Supported file format:
- YAML
- strict unknown-field rejection
- explicit `version`
Missing explicit path is an error. Missing default paths is non-fatal.
Supported version:
- `version: 1`
Precedence for `audita process`:
1. built-in defaults
Precedence for `process`:
1. defaults
2. file config
3. environment overrides
4. CLI overrides
Config source behavior:
- `--config <path>`: missing path is a clear failure
- `AUDITA_CONFIG`: missing path is a clear failure
- defaults `/usr/local/etc/audita/config.yml`, then `/etc/audita/config.yml`: both missing is non-fatal
`config validate` remains file-only validation (defaults + file config; no env overrides).
## Supported transcript input forms
Module and output-schema keys are validated against built-in catalogs. Unknown keys fail validation.
Audita accepts transcript JSON as either:
- a top-level array of segments
- an object with a `segments` array
## Input contract
Supported transcript JSON top-level forms:
- array of segments
- object with `segments` array
Segments must satisfy the schema and validation rules enforced by `internal/core/schema`.
Supported glossary YAML form:
- top-level `glossary` list with required entry fields validated by schema parsing.
## Supported glossary input form
Audita accepts glossary YAML with a top-level `glossary` entry list and validates required fields per entry.
## Supported output schema names
Built-in output schema registry supports:
## Output schema contract
Supported transcript output schemas:
- `bare-segments` (default)
- `audita-v1`
`seriatim-intermediate` is planned but not implemented.
Unknown schema keys fail before output write.
Unknown output schema names fail clearly.
## Report schema/versioning expectations
Process report payloads include `report_metadata` with:
## Report metadata contract
Process reports include stable report metadata fields:
- `report_schema_name`
- `report_schema_version`
- `output_schema`
- `config_version` when file config is used
- `config_version` (when file config is loaded)
Current values:
- `report_schema_name`: `audita-process-report`
- `report_schema_version`: `v1`
- `report_schema_name = audita-process-report`
- `report_schema_version = v1`
`--report-json` output and diagnostics run-dir `report.json` use the same report schema metadata.
`--report-json` output and run-directory `report.json` use the same report schema metadata.
Validator decision/rejection records in reports use stable validator keys in `validator_name`.
Module results may also include warning records for malformed module-stage LLM payloads.
Report diagnostics metadata includes artifact-path fields for utilization diagnostics and correction ledger when diagnostics initialization succeeds.
Validator decision/rejection records use stable validator keys via `validator_name`.
## Diagnostics directory behavior
## Diagnostics metadata contract
When run-directory initialization succeeds, diagnostics metadata paths reference stable artifacts, including:
- transcript and normalization artifacts;
- chunking summary;
- invocation metadata;
- redacted effective config;
- utilization diagnostics;
- correction ledger;
- `error.log` on failures.
When diagnostics directory creation succeeds, Audita writes run artifacts including:
- invocation metadata
- redacted effective config
- transcript/normalization/chunking artifacts
- utilization diagnostics (`utilization-diagnostics.json`)
- correction ledger (`correction-ledger.json`)
- report and failure error log (when applicable)
- module/LLM diagnostics artifacts as available
LLM interaction diagnostics include stable prompt and structured-schema identifiers where applicable.
Retention behavior is controlled by configured retention mode; failed runs are retained.
## Stdout/stderr and exit codes
Success:
- with `--output`, stdout is empty;
- without `--output`, stdout contains transcript JSON only;
- report JSON is not written to stdout.
Diagnostics metadata for LLM interactions may include semi-public prompt identifiers:
- `prompt_id`
- `prompt_version`
- `prompt_source`
- `embedded_path`
- `sha256`
Failures:
- nonzero exit;
- human-readable stderr summary;
- diagnostics directory path on stderr when available.
These are diagnostic identifiers, not user-facing prompt override controls.
Exit codes:
- `0` success
- nonzero failure
## Stdout/stderr behavior
## Redaction contract
Configured secrets are redacted from:
- effective config outputs;
- diagnostics artifacts;
- report artifacts;
- surfaced adapter/runtime errors.
Success behavior:
- with `--output`, stdout is empty
- without `--output`, stdout contains only transcript JSON in selected output schema
- report JSON is not written to stdout
- success stderr remains empty even when reports/diagnostics contain module warnings
## Compatibility policy
Stable command behavior, schema names, report metadata keys, diagnostics-path field semantics, and validator key identities are treated as public contract.
Failure behavior:
- stderr contains human-readable error summary
- nonzero exit
- diagnostics path is printed when available
## Exit-code behavior
- `0`: success
- nonzero: failure
Treat any nonzero exit as a failed invocation.
## Secret redaction guarantees
Audita redacts API keys and authorization secrets from:
- effective config outputs (`audita config print-effective`, diagnostics effective-config artifact)
- report artifacts
- LLM diagnostics artifacts
- surfaced request/response error messages
Config files should reference secrets via environment variable names (`api_key_env`) rather than embedding secret values.
## Compatibility and deprecation policy
- Existing stable schema names, report metadata keys, and top-level command behavior are treated as public contract.
- Existing stable validator keys remain public contract values even when validator semantics are refined.
- Compatibility inputs (legacy flags/env aliases) may remain during transition windows.
- Any planned removal or behavior change should include clear compatibility notes and migration guidance.
## Breaking changes after 1.0
After 1.0, breaking changes include, for example:
- changing default success/failure exit-code semantics
- changing stdout/stderr routing semantics
- silently changing default output schema shape
- removing supported output schema names without compatibility strategy
- changing report schema fields or meanings incompatibly
- changing config version semantics incompatibly without version bump
Additive fields, additive diagnostics, and new optional schema names are generally non-breaking when existing behavior remains intact.
Additive fields are acceptable when existing fields and behavior remain compatible.

View File

@@ -1,90 +1,72 @@
# Structured LLM Architecture
## Purpose
## Scope
This document describes Audita's structured LLM runtime boundary and adapter behavior.
## Why Audita owns the adapter
Audita owns a small structured LLM adapter so that core runtime behavior is controlled inside the repository:
- request construction and schema handling are explicit and testable;
- retries, timeouts, cancellation, and error redaction are consistent across modules and validators;
- provider SDK types are not exposed outside the adapter boundary;
- dependency weight and transitive provider-specific behavior are reduced.
At runtime, the rest of Audita depends only on the internal contract:
- `StructuredLLMClient`
## Runtime boundary
Production LLM integration depends on the internal contract only:
- `contracts.StructuredLLMClient`
- `CompleteStructured(ctx, req, out)`
## OpenAI-compatible request shape
Provider SDK types do not leak past this boundary.
At a conceptual level, Audita sends chat completion requests with:
- `model`
- `messages` (role/content pairs)
- `response_format`:
- `type = "json_schema"`
- `json_schema.name` (stable schema name)
- `json_schema.strict = true`
- `json_schema.schema` (registered JSON Schema payload)
## Adapter ownership
`internal/framework/llm` owns the OpenAI-compatible HTTP adapter and shared LLM runtime utilities.
The adapter uses OpenAI-compatible `POST {base_url}/chat/completions` over `net/http`.
Key responsibilities:
- request assembly;
- timeout/cancellation propagation;
- bounded retry behavior;
- scheduler integration;
- provider response decoding;
- error redaction.
## Structured response schema registry
## Structured schema registry
Structured response schemas are registered in `internal/framework/responseschema` and include stable metadata:
- `id`
- `version`
- `name`
- `json_schema`
- `sha256`
Structured response schemas are registered in `internal/framework/responseschema` with stable metadata:
- schema key
- schema ID
- schema version
- schema name (OpenAI-compatible `response_format` name)
- raw JSON Schema payload
- SHA-256 hash
Current schema keys:
- `correction_set`
- `validator_decision_set`
Current schemas:
- `correction_set`:
- id `audita.correction_set`
- version `v1`
- name `audita_correction_set_v1`
- `validator_decision_set`:
- id `audita.validator_decision_set`
- version `v1`
- name `audita_validator_decision_set_v1`
Schema metadata is attached to diagnostics through `Schema.DiagnosticsMap()`.
## Provider compatibility assumptions
## Request shape assumptions
Audita targets OpenAI-compatible chat-completions endpoints and sends structured requests with:
- model;
- chat messages;
- `response_format.type = json_schema`;
- schema name and JSON schema payload.
Audita assumes an OpenAI-compatible chat-completions endpoint that:
- accepts message arrays with model selection;
- accepts `response_format.type = json_schema`;
- returns a completion with assistant message content and optional usage metadata.
## Local validation remains mandatory
Provider schema enforcement is treated as transport-level guardrails.
Provider-specific differences are expected in strictness and error payload shapes, so the adapter treats provider output as untrusted until locally decoded.
Audita still validates output locally before applying behavior changes:
- proposal decoding and proposal invariants;
- validator decision decoding and cardinality checks;
- deterministic validation and apply-time rules.
## Local decode and validation remain mandatory
## Shared malformed-output policy
Malformed structured-output classification is centralized in `internal/framework/structuredoutput`.
Provider-level structured output is a transport guardrail, not final validation.
Proposal generation and validator execution both use this shared classifier so downgrade behavior cannot drift between the two paths.
After receiving a response, Audita still:
- decodes assistant content into typed request-specific structs;
- validates proposal and validator payload invariants locally;
- enforces deterministic validator/cardinality rules before any transcript application.
## Secrets and redaction
Secret extraction for LLM redaction is centralized in `llm.ConfiguredSecrets(cfg)` and reused by proposal and validator diagnostics writers.
This protects runtime correctness even when provider responses are malformed, partial, or semantically inconsistent.
Secrets are redacted from:
- diagnostics artifacts;
- report artifacts;
- surfaced adapter/runtime errors.
## Diagnostics and redaction
## Concurrency and scheduling
LLM execution is constrained by composed scheduler limits:
- total LLM concurrency;
- proposal LLM concurrency;
- validation LLM concurrency.
When structured schemas are used, diagnostics metadata records:
- schema ID
- schema version
- schema name
- schema hash
Diagnostics and surfaced errors preserve secret redaction:
- API keys and bearer tokens are redacted from request/response/error artifacts;
- redaction is applied before diagnostic files are written.
## Runtime behavior guarantees
The structured LLM path preserves existing runtime guarantees:
- bounded LLM call execution through schedulers;
- context-aware cancellation and timeout propagation;
- retry behavior for transient failures and retryable malformed structured responses;
- deterministic module/chunk/proposal/validator behavior outside provider nondeterminism.
The scheduler is FIFO and context-aware so permits are released on success, failure, and cancellation.

View File

@@ -1,160 +1,96 @@
# Audita Validators
This document describes Audita's built-in validator registry and module validator chains.
For LLM-backed validator prompt asset details, see [`docs/prompts.md`](prompts.md).
## Package ownership
Built-in validator construction is package-owned under `internal/validators/<validator_key>`:
- `internal/validators/confidence_threshold`
- `internal/validators/proposal_shape`
- `internal/validators/original_text_presence`
- `internal/validators/non_empty_corrected_text`
- `internal/validators/no_effect`
- `internal/validators/protected_terms`
- `internal/validators/spoken_form_plausibility`
- `internal/validators/meaning_reversal_review`
- `internal/validators/editorial_review`
Registry and chain wiring stay in:
- `internal/validators/registry.go`
- `internal/validators/chains.go`
Shared validator runtime mechanics stay in `internal/framework/validators`:
- request/result/decision models
- decision cardinality helpers
- protected vocabulary helpers
- shared LLM validator runtime, batching, and diagnostics helpers
Execution classification metadata is defined in `internal/validators/metadata`:
- `deterministic`
- `llm_backed`
Runner ordering uses this metadata so deterministic validators run before LLM-backed validators without concrete framework type assertions.
## Scope
This document defines the built-in validator system used by production module runs.
Validator chains are built-in runtime behavior.
## Ownership boundaries
Built-in validator keys, constructors, and module chains are owned by `internal/validators`.
Current 1.0 boundary:
- built-in validator keys and built-in module chains are stable runtime identifiers;
- thresholds and batching knobs remain configurable where already supported;
- arbitrary user-defined validator chains are deferred.
Shared runtime execution mechanics are owned by `internal/framework/validators`, including:
- validator request/result models;
- deterministic proposal checks;
- LLM validator batching and execution;
- decision-cardinality enforcement;
- diagnostics integration.
## Built-in validator keys
### Deterministic validators
Execution class metadata is owned by `internal/validators/metadata`.
## Stable validator keys
Deterministic:
- `proposal_shape`
- rejects malformed proposal fields before other validators run.
- `confidence_threshold`
- checks proposal confidence against module-specific configured threshold.
- `original_text_presence`
- ensures target segment exists and `original_text` exists in current working segment text.
- `non_empty_corrected_text`
- rejects proposals whose previewed resulting segment text would be empty or whitespace-only.
- `no_effect`
- rejects proposals where `original_text == corrected_text`.
- `protected_terms`
- protects glossary-derived terms from unsafe mutations in non-glossary modules.
- glossary stages use glossary-specific protection logic but still report this same stable key.
### LLM-backed validators
LLM-backed:
- `spoken_form_plausibility`
- checks whether proposed spoken-form change remains plausible in transcript context.
- `meaning_reversal_review`
- checks for likely meaning reversal or semantic contradiction.
- `editorial_review`
- performs conservative editorial safety review.
## Built-in module chains
`glossary`:
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
- `protected_terms`
- `non_empty_corrected_text`
- `spoken_form_plausibility`
- `meaning_reversal_review`
Current built-in chains resolved from `internal/validators/chains.go`:
`homophones`:
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
- `protected_terms`
- `non_empty_corrected_text`
- `spoken_form_plausibility`
- `meaning_reversal_review`
- `glossary`
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
- `protected_terms`
- `non_empty_corrected_text`
- `spoken_form_plausibility`
- `meaning_reversal_review`
`spoken_word`:
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
- `protected_terms`
- `non_empty_corrected_text`
- `editorial_review`
- `meaning_reversal_review`
- `homophones`
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
- `protected_terms`
- `non_empty_corrected_text`
- `spoken_form_plausibility`
- `meaning_reversal_review`
`grammar`:
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
- `protected_terms`
- `non_empty_corrected_text`
- `editorial_review`
- `meaning_reversal_review`
- `spoken_word`
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
- `protected_terms`
- `non_empty_corrected_text`
- `editorial_review`
- `meaning_reversal_review`
## Ordering and execution semantics
Validator ordering is based on canonical metadata:
- deterministic validators run before LLM-backed validators.
- `grammar`
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
- `protected_terms`
- `non_empty_corrected_text`
- `editorial_review`
- `meaning_reversal_review`
Within each module stage:
- proposals are generated per section;
- validator chains execute on those proposals;
- approved proposals are applied once after section work settles.
## Protected terms construction
## Malformed payload behavior
Malformed structured-output from proposal generation and LLM validator calls is downgraded, not treated as a process-fatal transport error.
`protected_terms` has explicit constructors:
- general constructor used by non-glossary modules through the built-in registry
- glossary-stage constructor used by glossary chain resolution
Current outcomes:
- malformed proposal-generation payloads produce section/module warnings and zero proposals for the affected section;
- malformed validator decision payloads reject the affected validator batch with warnings;
- deterministic validator behavior and runner order remain unchanged.
Both variants preserve existing behavior and report the stable key `protected_terms`.
## Reporting identity
Reports and diagnostics use stable validator keys as identifiers.
## Execution semantics
Correction-ledger deterministic-vs-LLM classification is derived from canonical validator metadata, not package-local hardcoded maps.
- modules execute serially;
- section proposal work can run concurrently within a module;
- deterministic validators run before LLM-backed validators;
- malformed module proposal payloads are downgraded to section-scoped module warnings with zero proposals for the affected section rather than module failure;
- malformed/missing/duplicate/unknown LLM validator decisions reject the affected validator batch with warnings instead of failing the module;
- oversized single-proposal validator inputs reject only the affected proposal under that validator;
- approved proposals are applied once per module after section work settles.
## Validator rejections vs proposal-application skips
- validator rejection:
- proposal is denied by validator-chain review and appears in validator rejection reporting with validator key and reason code.
- proposal-application skip:
- proposal passed validators but could not be applied under replacement-policy semantics (for example no matching span at apply time).
- module warning:
- malformed proposal-generation payloads and malformed validator batches are recorded in module warning records and diagnostics without writing success stderr.
These are separate outcomes and are reported separately.
## Reporting and diagnostics identity
- report validator decision/rejection entries use stable validator keys in `validator_name`.
- report module results include warning records for malformed module-stage LLM payloads.
- validator LLM diagnostics include validator identity in interaction metadata and structured response schema metadata.
- correction ledger entries include deterministic and LLM validator decision snapshots keyed by the same stable validator keys, and keep validator rejection distinct from application-level skip.
Prompt assets are unchanged by the validator package-ownership refactor and remain built-in under `internal/prompts`.
## Configurable knobs that remain supported
- per-module confidence thresholds (`thresholds.*` / equivalent env+CLI overrides)
- validation batching limits (`validation_max_prompt_tokens` / equivalent env+CLI overrides)
- validation LLM model/base URL/timeout/retries/concurrency settings
These tune validator behavior without exposing arbitrary user-defined chains.
## Prompt assets
LLM validator prompt assets and prompt metadata are documented in [Prompts](./prompts.md).

View File

@@ -1,51 +1,48 @@
# Audita Configuration
This document describes Audita's versioned YAML config support and related commands.
## Purpose
Audita's config file provides a stable place for pipeline defaults and runtime tuning that would otherwise require many environment variables or CLI flags.
Use config files for baseline settings, then use environment variables and CLI flags for deployment and per-run overrides.
## Supported version
Current supported config version:
## Scope
This document defines the supported versioned YAML configuration model and runtime precedence behavior.
## Supported file version
Current supported config file version:
- `version: 1`
Rules:
- missing `version` fails validation;
- unknown versions fail validation;
- unknown fields fail validation (strict decoding).
Validation rules:
- missing `version` fails;
- unsupported version fails;
- unknown YAML fields fail (strict decoding).
## Config path resolution
For `audita process` and `audita config print-effective`, path resolution order is:
1. `--config <path>`
2. `AUDITA_CONFIG`
3. `/usr/local/etc/audita/config.yml` (if present)
4. `/etc/audita/config.yml` (if present)
For `audita process`, config path resolution is:
Missing-path behavior:
- missing `--config` path is an error;
- missing `AUDITA_CONFIG` path is an error;
- missing both default paths is non-fatal.
1. `--config <path>` if provided
2. `AUDITA_CONFIG` if set and `--config` is not provided
3. default `/usr/local/etc/audita/config.yml` if present
4. fallback default `/etc/audita/config.yml` if present
Missing-file behavior:
- missing `--config` path: hard failure;
- missing `AUDITA_CONFIG` path: hard failure;
- missing both default-path files: non-fatal, run continues.
## Precedence model
Effective config precedence is:
1. built-in defaults
## Effective precedence
`audita process` effective precedence:
1. defaults
2. file config
3. environment overrides
4. CLI overrides
## Supported YAML fields
`audita config print-effective` uses:
1. defaults
2. file config
3. environment overrides
`audita config validate` intentionally uses file-only validation:
1. defaults
2. file config
Environment overrides are not applied in `config validate`.
## Supported top-level YAML fields
```yaml
version: 1
@@ -62,7 +59,6 @@ llm:
api_key_env: AUDITA_LLM_API_KEY
timeout: 120s
max_retries: 3
validation:
base_url: https://openrouter.ai/api/v1
model: openrouter/google/gemma-4-31b-it
@@ -100,91 +96,54 @@ diagnostics:
retention: auto
```
`context.description` provides background-only transcript context for prompts.
If both config and CLI provide a description, `--transcript-description` takes precedence.
## Module and output-schema validation
`pipeline.modules` keys are validated against the built-in supported module catalog.
`output.schema` supports the built-in output schema registry values:
- `bare-segments` (default)
Supported module keys:
- `glossary`
- `homophones`
- `spoken_word`
- `grammar`
Repeated supported module keys are allowed.
`output.schema` is validated against the built-in output schema catalog.
Supported output schema keys:
- `bare-segments`
- `audita-v1`
Unknown schema names fail clearly before transcript output is written.
Unknown module keys and unknown output schema keys fail validation.
Duration-like fields accept either:
## Duration field parsing
Duration-like fields support:
- numeric seconds (for example `120`, `3.5`)
- duration strings (for example `120s`, `2m`)
- numeric seconds (for example `120`, `3.5`), or
- duration strings (for example `120s`, `2m`).
For LLM timeouts, duration strings must resolve to whole seconds.
LLM timeout duration strings must resolve to whole seconds.
## Secret handling
Use `api_key_env` for secrets:
Use `api_key_env` fields for secrets:
- `llm.proposal.api_key_env`
- `llm.validation.api_key_env`
These fields must contain environment variable names, not secret values.
These fields store environment variable names, not secret values.
At runtime, Audita resolves those names from the process environment.
Redaction behavior:
- run diagnostics `effective-config.json` is redacted;
- `audita config print-effective` output is redacted;
- API keys are never emitted in plaintext by those outputs.
## Config commands
Validate a config file:
Resolved secret values are redacted from:
- `audita config print-effective` output;
- diagnostics `effective-config.json`;
- report and diagnostics payloads.
## Commands
Validate a file config:
```sh
audita config validate --config ./audita.yml
```
Print redacted effective config:
```sh
audita config print-effective --config ./audita.yml
```
`print-effective` loads defaults, then file config, then environment overrides.
## Example: local OpenAI-compatible endpoint
```yaml
version: 1
llm:
proposal:
base_url: http://localhost:8000/v1
model: local/proposal-model
api_key_env: AUDITA_LLM_API_KEY
timeout: 90s
max_retries: 2
validation:
base_url: http://localhost:8000/v1
model: local/validation-model
api_key_env: AUDITA_VALIDATION_LLM_API_KEY
timeout: 90s
max_retries: 2
pipeline:
modules: [glossary, homophones, glossary, spoken_word, grammar]
diagnostics:
work_dir: /tmp/audita
retention: auto
```
## Compatibility notes
Existing environment variables and lower-level CLI flags remain available for compatibility.
Current guidance:
- prefer file config for baseline behavior;
- keep environment variables for secrets/deployment-specific overrides;
- use CLI flags for per-run overrides.
- validator chains are built-in and are not user-configurable in config.
- prompt source selection and filesystem prompt overrides are not config options.
Legacy compatibility flags and environment aliases remain available where implemented, but the stable configuration surface is the versioned YAML model described above.

33
docs/development.md Normal file
View File

@@ -0,0 +1,33 @@
# Audita Development Workflow
## Scope
This document defines the canonical contributor workflow and engineering conventions for this repository.
## Workflow
1. Start from a clean understanding of scope and constraints.
2. Make focused changes that preserve existing public behavior unless behavior change is explicitly intended.
3. Run targeted tests for touched packages.
4. Run `go test ./...` before finalizing substantial changes.
5. Update affected documentation so it describes current behavior only.
## Engineering conventions
- Keep module packages separate: `glossary`, `homophones`, `spoken_word`, `grammar`.
- Prefer narrow shared helpers and catalogs over broad abstractions.
- Preserve diagnostics artifact naming and report field contracts unless intentionally changed.
- Preserve CLI/config precedence semantics unless intentionally changed.
- Treat stable validator keys, prompt identifiers, and output-schema keys as contract surfaces.
## Configuration and runtime expectations
- `audita process` precedence is defaults -> file -> env -> CLI.
- `audita config validate` validates file config merged onto defaults only.
- `audita config print-effective` includes environment overrides and prints redacted JSON.
## Testing expectations
- Add tests for new behavior and for bug fixes.
- Keep deterministic fixtures stable.
- Do not reduce existing parity, release-fixture, subprocess, or module-specific coverage without equivalent replacement.
## Commit discipline
- Keep commits scoped and reviewable.
- Avoid mixing unrelated refactors with behavior changes.
- Use clear plain-English commit messages.

View File

@@ -0,0 +1,27 @@
# Documentation Policy
## Scope
This policy defines how project documentation should be authored and maintained.
## Core rules
- Document the current behavior of the codebase.
- Remove stale behavior descriptions promptly when code changes.
- Do not describe development history in architecture or behavior docs unless a document is explicitly historical.
- Do not use architecture or behavior docs as changelogs.
- Prefer rewriting stale sections from scratch when substantial behavior or ownership changes occur.
## Consistency requirements
- Keep command examples aligned with current CLI surfaces.
- Keep configuration examples aligned with supported fields and precedence.
- Keep architecture package ownership descriptions aligned with current code layout.
- Keep stable contract identifiers accurate (module keys, validator keys, output-schema keys, report metadata fields).
## Cross-document expectations
- `docs/architecture/*` documents runtime behavior and package ownership.
- `docs/configuration.md` documents config schema and precedence.
- `docs/development.md` documents contributor workflow and engineering conventions.
## Review expectations for documentation changes
- Verify referenced files and links exist.
- Verify examples match current behavior.
- Prefer concise, direct language and avoid speculative future claims.

View File

@@ -5,6 +5,7 @@ import (
"sort"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
)
const (
@@ -48,12 +49,6 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
}
entries := make([]correctionLedgerEntry, 0)
llmBacked := map[string]bool{
"spoken_form_plausibility": true,
"meaning_reversal_review": true,
"editorial_review": true,
}
for _, module := range runOutput.ModuleResults {
decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord)
for _, decision := range module.ValidatorDecisions {
@@ -72,8 +67,8 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
AppliedCorrectedText: change.CorrectedText,
ReplacementPolicy: string(module.ReplacementPolicy),
Disposition: correctionDispositionApplied,
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false, llmBacked),
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true, llmBacked),
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false),
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true),
})
}
for _, change := range module.SkippedChanges {
@@ -89,8 +84,8 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
Disposition: correctionDispositionSkipped,
DispositionReasonCode: string(change.SkipReason),
DispositionMessage: change.Message,
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false, llmBacked),
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true, llmBacked),
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false),
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true),
})
}
for _, rejection := range module.ValidatorRejected {
@@ -106,8 +101,8 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
Disposition: correctionDispositionRejected,
DispositionReasonCode: rejection.ReasonCode,
DispositionMessage: rejection.Message,
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], false, llmBacked),
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], true, llmBacked),
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], false),
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], true),
})
}
if module.Status == runner.ModuleStatusFailed {
@@ -135,13 +130,14 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
return entries
}
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool, llmBacked map[string]bool) []ledgerValidatorDecisionRecord {
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []ledgerValidatorDecisionRecord {
if len(in) == 0 {
return nil
}
out := make([]ledgerValidatorDecisionRecord, 0, len(in))
for _, decision := range in {
if llmBacked[decision.ValidatorName] != wantLLM {
isLLMBacked := validatormetadata.ClassForKey(decision.ValidatorName) == validatormetadata.ExecutionClassLLMBacked
if isLLMBacked != wantLLM {
continue
}
out = append(out, ledgerValidatorDecisionRecord{

View File

@@ -0,0 +1,46 @@
package cli
import (
"testing"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
)
func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(t *testing.T) {
output := &runner.RunOutput{
ModuleResults: []runner.ModuleResult{
{
ModuleKey: "glossary",
ModuleInstance: "glossary",
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
ValidatorDecisions: []runner.ValidatorDecisionRecord{
{ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
{ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
},
AppliedChanges: []proposals.AppliedChange{
{
ProposalIndex: 3,
ModuleKey: "glossary",
ModuleInstance: "glossary",
TargetSegmentID: 1,
OriginalText: "gestures",
CorrectedText: "Jesters",
},
},
},
},
}
ledger := buildCorrectionLedger("/tmp/audita-run-id", output)
if len(ledger) != 1 {
t.Fatalf("expected one ledger entry, got %d", len(ledger))
}
entry := ledger[0]
if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "proposal_shape" {
t.Fatalf("unexpected deterministic decision split: %+v", entry.DeterministicValidatorResults)
}
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "spoken_form_plausibility" {
t.Fatalf("unexpected llm-backed decision split: %+v", entry.LLMValidatorResults)
}
}

View File

@@ -375,30 +375,28 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
return 2
}
configPath, configSource, err := resolveConfigPath(configPathOverride, configPathOverrideSet, os.LookupEnv)
effectiveConfig, err := config.LoadEffectiveConfig(configPathOverride, configPathOverrideSet)
if err != nil {
var effectiveConfigErr *config.EffectiveConfigError
if errors.As(err, &effectiveConfigErr) {
switch effectiveConfigErr.Kind {
case config.EffectiveConfigErrorLoadFile, config.EffectiveConfigErrorApplyFile:
fmt.Fprintf(stderr, "audita process: invalid config file: %v\n", effectiveConfigErr)
case config.EffectiveConfigErrorApplyEnv:
fmt.Fprintf(stderr, "audita process: invalid environment configuration: %v\n", effectiveConfigErr)
default:
fmt.Fprintf(stderr, "audita process: %v\n", effectiveConfigErr)
}
} else {
fmt.Fprintf(stderr, "audita process: %v\n", err)
}
return 2
}
cfg := config.Default()
var configVersion *int
if configPath != "" {
fileCfg, fileErr := config.LoadFileConfig(configPath)
if fileErr != nil {
fmt.Fprintf(stderr, "audita process: invalid config file: %v\n", fileErr)
return 2
}
if applyErr := cfg.ApplyFileConfig(fileCfg); applyErr != nil {
fmt.Fprintf(stderr, "audita process: invalid config file: %v\n", applyErr)
return 2
}
configVersion = &fileCfg.Version
}
if err := cfg.ApplyEnvOverrides(); err != nil {
fmt.Fprintf(stderr, "audita process: invalid environment configuration: %v\n", err)
return 2
}
cfg := effectiveConfig.Config
configPath := effectiveConfig.ConfigPath
configSource := effectiveConfig.ConfigSource
configVersion := effectiveConfig.ConfigVersion
fs, pFlags := newProcessFlagSet(cfg, stderr)
@@ -531,9 +529,9 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
if runErr != nil {
if runDir != nil && runOutput != nil {
if runOutput.Utilization != nil {
_ = runDir.WriteJSONArtifact("utilization-diagnostics.json", runOutput.Utilization)
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
}
_ = runDir.WriteJSONArtifact("correction-ledger.json", buildCorrectionLedger(runDir.Path(), runOutput))
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, buildCorrectionLedger(runDir.Path(), runOutput))
}
errorPhase, errorMessage := extractErrorPhase(runErr)
report := buildProcessReport("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput)
@@ -560,9 +558,9 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
if runDir != nil && runOutput != nil {
if runOutput.Utilization != nil {
_ = runDir.WriteJSONArtifact("utilization-diagnostics.json", runOutput.Utilization)
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
}
_ = runDir.WriteJSONArtifact("correction-ledger.json", buildCorrectionLedger(runDir.Path(), runOutput))
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, buildCorrectionLedger(runDir.Path(), runOutput))
}
report := buildProcessReport("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput)
@@ -653,6 +651,10 @@ func runConfigValidate(args []string, stdout, stderr io.Writer) int {
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
return 2
}
if err := cfg.Validate(); err != nil {
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
return 2
}
fmt.Fprintln(stdout, "config is valid")
return 0
}
@@ -675,28 +677,13 @@ func runConfigPrintEffective(args []string, stdout, stderr io.Writer) int {
configPathValue := strings.TrimSpace(*configPath)
configPathSet := configPathValue != ""
path, _, err := resolveConfigPath(configPathValue, configPathSet, os.LookupEnv)
effectiveConfig, err := config.LoadEffectiveConfig(configPathValue, configPathSet)
if err != nil {
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
return 2
}
cfg := config.Default()
if path != "" {
fileCfg, fileErr := config.LoadFileConfig(path)
if fileErr != nil {
fmt.Fprintf(stderr, "audita config print-effective: %v\n", fileErr)
return 2
}
if applyErr := cfg.ApplyFileConfig(fileCfg); applyErr != nil {
fmt.Fprintf(stderr, "audita config print-effective: %v\n", applyErr)
return 2
}
}
if err := cfg.ApplyEnvOverrides(); err != nil {
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
return 2
}
cfg := effectiveConfig.Config
redacted := cfg.Redacted()
out, err := json.MarshalIndent(redacted, "", " ")
@@ -743,22 +730,9 @@ func buildProcessReport(status string, inv processInvocation, runDir *diagnostic
ErrorPhase: errorPhase,
}
if runDir != nil {
diagnosticsDir := runDir.Path()
report.Diagnostics = &reporting.DiagnosticsMetadata{
DirectoryPath: diagnosticsDir,
SourceTranscriptPath: filepath.Join(diagnosticsDir, "source-transcript.json"),
ParsedSourceTranscriptPath: filepath.Join(diagnosticsDir, "source-transcript-parsed.json"),
NormalizedTranscriptPath: filepath.Join(diagnosticsDir, "normalized-transcript.json"),
NormalizationSummaryPath: filepath.Join(diagnosticsDir, "normalization-summary.json"),
ChunkingSummaryPath: filepath.Join(diagnosticsDir, "chunking-summary.json"),
UtilizationSummaryPath: filepath.Join(diagnosticsDir, "utilization-diagnostics.json"),
CorrectionLedgerPath: filepath.Join(diagnosticsDir, "correction-ledger.json"),
InvocationMetadataPath: filepath.Join(diagnosticsDir, "invocation.json"),
RedactedEffectiveConfigPath: filepath.Join(diagnosticsDir, "effective-config.json"),
}
if status == "failed" {
report.Diagnostics.ErrorLogPath = filepath.Join(diagnosticsDir, "error.log")
}
runSucceeded := status == "success"
metadata := diagnostics.BuildDiagnosticsMetadata(runDir.Path(), runSucceeded)
report.Diagnostics = &metadata
}
if errorMessage != "" {
report.ErrorMessage = errorMessage
@@ -984,47 +958,6 @@ func findConfigPathOverride(args []string) (path string, set bool, err error) {
return "", false, nil
}
var statConfigPath = os.Stat
func resolveConfigPath(cliConfigPath string, cliConfigPathSet bool, lookup func(string) (string, bool)) (path string, source string, err error) {
if cliConfigPathSet {
path = strings.TrimSpace(cliConfigPath)
if path == "" {
return "", "", fmt.Errorf("--config requires a non-empty path")
}
if _, statErr := statConfigPath(path); statErr != nil {
if os.IsNotExist(statErr) {
return "", "", fmt.Errorf("config file not found: %s", path)
}
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
}
return path, "flag", nil
}
if raw, ok := lookup("AUDITA_CONFIG"); ok {
path = strings.TrimSpace(raw)
if path == "" {
return "", "", fmt.Errorf("AUDITA_CONFIG must not be empty")
}
if _, statErr := statConfigPath(path); statErr != nil {
if os.IsNotExist(statErr) {
return "", "", fmt.Errorf("config file not found: %s", path)
}
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
}
return path, "env", nil
}
for _, defaultPath := range config.DefaultConfigSearchPaths {
if _, statErr := statConfigPath(defaultPath); statErr == nil {
return defaultPath, "default", nil
} else if !os.IsNotExist(statErr) {
return "", "", fmt.Errorf("cannot access config file %s: %w", defaultPath, statErr)
}
}
return "", "", nil
}
func isHelpCommand(args []string) bool {
if len(args) == 0 {
return false

View File

@@ -21,11 +21,11 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/internal/testsupport"
)
func TestRunRootHelp(t *testing.T) {
@@ -99,66 +99,6 @@ func TestRunProcessHelpListsExpectedFlags(t *testing.T) {
}
}
func TestResolveConfigPathDefaultIgnoredWhenMissing(t *testing.T) {
lookup := func(string) (string, bool) { return "", false }
path, source, err := resolveConfigPath("", false, lookup)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if path != "" || source != "" {
t.Fatalf("expected no config path/source, got path=%q source=%q", path, source)
}
}
func TestResolveConfigPathDefaultPrefersUsrLocalOverEtc(t *testing.T) {
oldStat := statConfigPath
statConfigPath = func(path string) (os.FileInfo, error) {
if path == config.DefaultConfigPathUsrLocal || path == config.DefaultConfigPath {
return nil, nil
}
return nil, os.ErrNotExist
}
t.Cleanup(func() { statConfigPath = oldStat })
lookup := func(string) (string, bool) { return "", false }
path, source, err := resolveConfigPath("", false, lookup)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if source != "default" {
t.Fatalf("expected default source, got %q", source)
}
if path != config.DefaultConfigPathUsrLocal {
t.Fatalf("expected %q, got %q", config.DefaultConfigPathUsrLocal, path)
}
}
func TestResolveConfigPathDefaultFallsBackToEtc(t *testing.T) {
oldStat := statConfigPath
statConfigPath = func(path string) (os.FileInfo, error) {
if path == config.DefaultConfigPathUsrLocal {
return nil, os.ErrNotExist
}
if path == config.DefaultConfigPath {
return nil, nil
}
return nil, os.ErrNotExist
}
t.Cleanup(func() { statConfigPath = oldStat })
lookup := func(string) (string, bool) { return "", false }
path, source, err := resolveConfigPath("", false, lookup)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if source != "default" {
t.Fatalf("expected default source, got %q", source)
}
if path != config.DefaultConfigPath {
t.Fatalf("expected %q, got %q", config.DefaultConfigPath, path)
}
}
func TestRunConfigValidateSuccess(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
@@ -210,6 +150,23 @@ func TestRunConfigValidateUnknownField(t *testing.T) {
}
}
func TestRunConfigValidateUnsupportedModuleKey(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
cfgPath := writeFile(t, "config.yml", "version: 1\npipeline:\n modules: [made_up]\n")
exitCode := Run([]string{"config", "validate", "--config", cfgPath}, &stdout, &stderr)
if exitCode == 0 {
t.Fatalf("expected failure for unsupported module key")
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "unsupported module key") {
t.Fatalf("expected unsupported module key error, got %q", stderr.String())
}
}
func TestRunConfigPrintEffectiveOutputsRedactedJSON(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
@@ -242,6 +199,45 @@ func TestRunConfigPrintEffectiveOutputsRedactedJSON(t *testing.T) {
}
}
func TestRunConfigPrintEffectiveAppliesFileThenEnvironment(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
cfgPath := writeFile(t, "config.yml", "version: 1\nllm:\n proposal:\n model: file-model\n")
t.Setenv("AUDITA_MODEL", "env-model")
exitCode := Run([]string{"config", "print-effective", "--config", cfgPath}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
var out struct {
PrimaryLLM struct {
Model string `json:"Model"`
} `json:"PrimaryLLM"`
}
if err := json.Unmarshal(stdout.Bytes(), &out); err != nil {
t.Fatalf("expected valid JSON output, got error: %v output=%q", err, stdout.String())
}
if out.PrimaryLLM.Model != "env-model" {
t.Fatalf("expected env model override in print-effective output, got %q", out.PrimaryLLM.Model)
}
}
func TestRunConfigValidateIgnoresEnvironmentOverrides(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
cfgPath := writeFile(t, "config.yml", "version: 1\n")
t.Setenv("AUDITA_MODULES", "made_up")
exitCode := Run([]string{"config", "validate", "--config", cfgPath}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success because config validate is file-only, got %d stderr=%q", exitCode, stderr.String())
}
if !strings.Contains(stdout.String(), "config is valid") {
t.Fatalf("expected success message, got %q", stdout.String())
}
}
func TestRunConfigCommandDoesNotRequireTranscriptOrGlossary(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
@@ -369,8 +365,8 @@ diagnostics:
func TestRunProcessEnvOverridesConfigFile(t *testing.T) {
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{
key: "m",
"grammar": fakeModule{
key: "grammar",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
@@ -392,7 +388,7 @@ func TestRunProcessEnvOverridesConfigFile(t *testing.T) {
cfgPath := writeFile(t, "config.yml", `
version: 1
pipeline:
modules: [m]
modules: [grammar]
llm:
proposal:
model: file-model
@@ -404,7 +400,7 @@ llm:
fixturePath("tiny_transcript.json"),
"--glossary", fixturePath("tiny_glossary.yaml"),
"--config", cfgPath,
"--modules", "m",
"--modules", "grammar",
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
@@ -413,8 +409,8 @@ llm:
func TestRunProcessCLIOverridesEnvAndConfigFile(t *testing.T) {
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{
key: "m",
"grammar": fakeModule{
key: "grammar",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
@@ -436,7 +432,7 @@ func TestRunProcessCLIOverridesEnvAndConfigFile(t *testing.T) {
cfgPath := writeFile(t, "config.yml", `
version: 1
pipeline:
modules: [m]
modules: [grammar]
llm:
proposal:
model: file-model
@@ -448,7 +444,7 @@ llm:
fixturePath("tiny_transcript.json"),
"--glossary", fixturePath("tiny_glossary.yaml"),
"--config", cfgPath,
"--modules", "m",
"--modules", "grammar",
"--model", "cli-model",
}, &stdout, &stderr)
if exitCode != 0 {
@@ -495,8 +491,8 @@ diagnostics:
func TestRunProcessTranscriptDescriptionCLIOverridesConfigFileContextDescription(t *testing.T) {
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{
key: "m",
"grammar": fakeModule{
key: "grammar",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
@@ -517,7 +513,7 @@ func TestRunProcessTranscriptDescriptionCLIOverridesConfigFileContextDescription
cfgPath := writeFile(t, "config.yml", `
version: 1
pipeline:
modules: [m]
modules: [grammar]
context:
description: "file transcript description"
`)
@@ -528,7 +524,7 @@ context:
fixturePath("tiny_transcript.json"),
"--glossary", fixturePath("tiny_glossary.yaml"),
"--config", cfgPath,
"--modules", "m",
"--modules", "grammar",
"--transcript-description", "cli transcript description",
}, &stdout, &stderr)
if exitCode != 0 {
@@ -692,8 +688,8 @@ func TestRunProcessCLIOverridesEnvironment(t *testing.T) {
func TestRunProcessTranscriptDescriptionDefaultEmpty(t *testing.T) {
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{
key: "m",
"grammar": fakeModule{
key: "grammar",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
@@ -720,7 +716,7 @@ func TestRunProcessTranscriptDescriptionDefaultEmpty(t *testing.T) {
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "m",
"--modules", "grammar",
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
@@ -729,8 +725,8 @@ func TestRunProcessTranscriptDescriptionDefaultEmpty(t *testing.T) {
func TestRunProcessTranscriptDescriptionCLIOverrideAndTrim(t *testing.T) {
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{
key: "m",
"grammar": fakeModule{
key: "grammar",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
@@ -757,7 +753,7 @@ func TestRunProcessTranscriptDescriptionCLIOverrideAndTrim(t *testing.T) {
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "m",
"--modules", "grammar",
"--transcript-description", " speaker background context ",
}, &stdout, &stderr)
if exitCode != 0 {
@@ -823,8 +819,8 @@ func TestRunProcessRejectsValidationConcurrencyAboveTotalConcurrency(t *testing.
func TestRunProcessTotalLLMConcurrencyDrivesEffectiveValidationConcurrencyWhenUnset(t *testing.T) {
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{
key: "m",
"grammar": fakeModule{
key: "grammar",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
@@ -865,7 +861,7 @@ func TestRunProcessTotalLLMConcurrencyDrivesEffectiveValidationConcurrencyWhenUn
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--modules",
"m",
"grammar",
"--total-llm-concurrency",
"4",
}, &stdout, &stderr)
@@ -908,8 +904,8 @@ func TestRunProcessRejectsProposalConcurrencyAboveTotalConcurrency(t *testing.T)
func TestRunProcessLegacyLLMConcurrencyAliasSetsTotalAndProposal(t *testing.T) {
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{
key: "m",
"grammar": fakeModule{
key: "grammar",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
@@ -944,7 +940,7 @@ func TestRunProcessLegacyLLMConcurrencyAliasSetsTotalAndProposal(t *testing.T) {
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--modules",
"m",
"grammar",
"--llm-concurrency",
"3",
}, &stdout, &stderr)
@@ -955,8 +951,8 @@ func TestRunProcessLegacyLLMConcurrencyAliasSetsTotalAndProposal(t *testing.T) {
func TestRunProcessLLMConcurrencyFlagsOverrideEnvironment(t *testing.T) {
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{
key: "m",
"grammar": fakeModule{
key: "grammar",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
@@ -997,7 +993,7 @@ func TestRunProcessLLMConcurrencyFlagsOverrideEnvironment(t *testing.T) {
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--modules",
"m",
"grammar",
"--total-llm-concurrency",
"4",
"--proposal-llm-concurrency",
@@ -1010,8 +1006,8 @@ func TestRunProcessLLMConcurrencyFlagsOverrideEnvironment(t *testing.T) {
func TestRunProcessAcceptsLLMConcurrencyEnvironmentVariables(t *testing.T) {
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{
key: "m",
"grammar": fakeModule{
key: "grammar",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
@@ -1052,7 +1048,7 @@ func TestRunProcessAcceptsLLMConcurrencyEnvironmentVariables(t *testing.T) {
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--modules",
"m",
"grammar",
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
@@ -1883,12 +1879,12 @@ func TestRunProcessInjectedFactoryExecutesRunnerAndReportsModules(t *testing.T)
return validators.Result{ValidatorName: "allow", Decisions: decisions}, nil
}}
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"glossary": fakeModule{key: "glossary", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "Hello", CorrectedText: "Hi", Confidence: 1},
}, nil
}},
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"homophones": fakeModule{key: "homophones", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
if req.WorkingTranscript.Segments[0].Text != "Hi world" {
t.Fatalf("expected module 2 to see module 1 changes, got %q", req.WorkingTranscript.Segments[0].Text)
}
@@ -1911,7 +1907,7 @@ func TestRunProcessInjectedFactoryExecutesRunnerAndReportsModules(t *testing.T)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "m1,m2",
"--modules", "glossary,homophones",
"--output", outputPath,
"--report-json", reportPath,
"--work-dir", workDir,
@@ -1970,7 +1966,7 @@ func TestRunProcessInjectedFactoryLLMValidatorResultsInReports(t *testing.T) {
}},
}
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"grammar": fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "Hello", CorrectedText: "Hi", Confidence: 1}}, nil
}},
}}
@@ -1990,7 +1986,7 @@ func TestRunProcessInjectedFactoryLLMValidatorResultsInReports(t *testing.T) {
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "m",
"--modules", "grammar",
"--output", outputPath,
"--report-json", reportPath,
"--work-dir", workDir,
@@ -2011,7 +2007,7 @@ func TestRunProcessInjectedFactoryLLMValidatorResultsInReports(t *testing.T) {
func TestRunProcessInjectedFactorySkippedKeepsAutoRetention(t *testing.T) {
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"glossary": fakeModule{key: "glossary", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term", Confidence: 1},
}, nil
@@ -2027,7 +2023,7 @@ func TestRunProcessInjectedFactorySkippedKeepsAutoRetention(t *testing.T) {
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "m1",
"--modules", "glossary",
"--work-dir", workDir,
"--work-dir-retention", "auto",
}, &stdout, &stderr)
@@ -2041,7 +2037,7 @@ func TestRunProcessInjectedFactorySkippedKeepsAutoRetention(t *testing.T) {
func TestRunProcessInjectedFactoryFailureWritesFailedReport(t *testing.T) {
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"glossary": fakeModule{key: "glossary", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return nil, errors.New("test failure")
}},
}}
@@ -2057,7 +2053,7 @@ func TestRunProcessInjectedFactoryFailureWritesFailedReport(t *testing.T) {
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "m1",
"--modules", "glossary",
"--work-dir", workDir,
"--work-dir-retention", "always",
"--report-json", reportPath,
@@ -2089,14 +2085,8 @@ func TestRunProcessInjectedFactoryFailureWritesFailedReport(t *testing.T) {
}
}
func TestRunProcessProductionRegistryUnsupportedModuleFailsCleanly(t *testing.T) {
cfg := modules.Dependencies{}
processModuleFactory = modules.NewFactory(cfg)
t.Cleanup(func() { processModuleFactory = nil })
func TestRunProcessUnsupportedModuleFailsDuringConfigValidation(t *testing.T) {
var stdout, stderr bytes.Buffer
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}
]`)
@@ -2105,9 +2095,6 @@ func TestRunProcessProductionRegistryUnsupportedModuleFailsCleanly(t *testing.T)
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "made_up",
"--work-dir", workDir,
"--work-dir-retention", "always",
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatal("expected failure exit code")
@@ -2115,23 +2102,12 @@ func TestRunProcessProductionRegistryUnsupportedModuleFailsCleanly(t *testing.T)
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "runner_execution") {
t.Fatalf("expected runner_execution failure on stderr, got %q", stderr.String())
if !strings.Contains(stderr.String(), "invalid CLI configuration") {
t.Fatalf("expected config validation failure on stderr, got %q", stderr.String())
}
if !strings.Contains(stderr.String(), "unsupported module key") {
t.Fatalf("expected explicit unsupported module message, got %q", stderr.String())
}
report := readProcessReport(t, reportPath)
if report.Status != "failed" {
t.Fatalf("expected failed report status, got %q", report.Status)
}
if report.ErrorPhase != "runner_execution" {
t.Fatalf("expected runner_execution phase, got %q", report.ErrorPhase)
}
if !strings.Contains(report.ErrorMessage, "unsupported module key") {
t.Fatalf("expected report error message to mention unsupported module, got %q", report.ErrorMessage)
}
}
func TestRunProcessExplicitUnsupportedModulesFailClearly(t *testing.T) {
@@ -4360,12 +4336,7 @@ func writeFile(t *testing.T, name string, content string) string {
}
func readFile(t *testing.T, path string) []byte {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("failed to read file %q: %v", path, err)
}
return data
return testsupport.ReadFile(t, path)
}
func readProcessReport(t *testing.T, path string) reporting.ProcessReport {
@@ -4379,13 +4350,5 @@ func readProcessReport(t *testing.T, path string) reporting.ProcessReport {
}
func onlyRunDir(t *testing.T, workDir string) string {
t.Helper()
entries, err := os.ReadDir(workDir)
if err != nil {
t.Fatalf("failed to read work dir %q: %v", workDir, err)
}
if len(entries) != 1 {
t.Fatalf("expected exactly one run dir in %q, got %d", workDir, len(entries))
}
return filepath.Join(workDir, entries[0].Name())
return testsupport.OnlyRunDir(t, workDir)
}

View File

@@ -3,6 +3,8 @@ package config
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
)
type WorkDirRetention string
@@ -14,7 +16,7 @@ const (
)
const (
DefaultModulesCSV = "glossary,homophones,glossary,spoken_word,grammar"
DefaultModulesCSV = modulecatalog.KeyGlossary + "," + modulecatalog.KeyHomophones + "," + modulecatalog.KeyGlossary + "," + modulecatalog.KeySpokenWord + "," + modulecatalog.KeyGrammar
DefaultOutputSchema = "bare-segments"
DefaultPrimaryModel = "openrouter/google/gemma-4-31b-it"
DefaultPrimaryBaseURL = "https://openrouter.ai/api/v1"

View File

@@ -4,6 +4,9 @@ import (
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
)
func TestDefaultConfigValues(t *testing.T) {
@@ -318,6 +321,38 @@ func TestValidationFailures(t *testing.T) {
}
}
func TestValidationRejectsUnsupportedModuleKey(t *testing.T) {
cfg := Default()
cfg.Modules = []string{modulecatalog.KeyGlossary, "made_up"}
err := cfg.Validate()
if err == nil {
t.Fatalf("expected validation error for unsupported module key")
}
if !strings.Contains(err.Error(), `unsupported module key "made_up"`) {
t.Fatalf("expected unsupported module key error, got %q", err.Error())
}
}
func TestValidationAllowsRepeatedSupportedModuleKeys(t *testing.T) {
cfg := Default()
cfg.Modules = []string{modulecatalog.KeyGlossary, modulecatalog.KeyGlossary, modulecatalog.KeyGrammar}
if err := cfg.Validate(); err != nil {
t.Fatalf("expected repeated supported module keys to validate, got %v", err)
}
}
func TestValidationAcceptsAllSupportedOutputSchemas(t *testing.T) {
for _, schemaKey := range outputschema.SupportedKeys() {
cfg := Default()
cfg.OutputSchema = schemaKey
if err := cfg.Validate(); err != nil {
t.Fatalf("expected output schema %q to validate, got %v", schemaKey, err)
}
}
}
func TestEffectiveValidationLLMInheritance(t *testing.T) {
cfg := Default()
cfg.PrimaryLLM.APIKey = "primary-key"

View File

@@ -0,0 +1,119 @@
package config
import (
"fmt"
"os"
"strings"
)
type EffectiveConfigErrorKind string
const (
EffectiveConfigErrorResolvePath EffectiveConfigErrorKind = "resolve_path"
EffectiveConfigErrorLoadFile EffectiveConfigErrorKind = "load_file"
EffectiveConfigErrorApplyFile EffectiveConfigErrorKind = "apply_file"
EffectiveConfigErrorApplyEnv EffectiveConfigErrorKind = "apply_env"
)
type EffectiveConfigError struct {
Kind EffectiveConfigErrorKind
Err error
}
func (e *EffectiveConfigError) Error() string {
if e == nil || e.Err == nil {
return ""
}
return e.Err.Error()
}
func (e *EffectiveConfigError) Unwrap() error {
if e == nil {
return nil
}
return e.Err
}
type EffectiveConfig struct {
Config Config
ConfigPath string
ConfigSource string
ConfigVersion *int
}
func ResolveConfigPath(cliConfigPath string, cliConfigPathSet bool) (path string, source string, err error) {
return resolveConfigPathWithLookup(cliConfigPath, cliConfigPathSet, os.LookupEnv, os.Stat, DefaultConfigSearchPaths)
}
func LoadEffectiveConfig(cliConfigPath string, cliConfigPathSet bool) (EffectiveConfig, error) {
return loadEffectiveConfigWithLookup(cliConfigPath, cliConfigPathSet, os.LookupEnv, os.Stat, DefaultConfigSearchPaths)
}
func loadEffectiveConfigWithLookup(cliConfigPath string, cliConfigPathSet bool, lookup func(string) (string, bool), statPath func(string) (os.FileInfo, error), defaultSearchPaths []string) (EffectiveConfig, error) {
configPath, configSource, err := resolveConfigPathWithLookup(cliConfigPath, cliConfigPathSet, lookup, statPath, defaultSearchPaths)
if err != nil {
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorResolvePath, Err: err}
}
cfg := Default()
var configVersion *int
if configPath != "" {
fileCfg, fileErr := LoadFileConfig(configPath)
if fileErr != nil {
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorLoadFile, Err: fileErr}
}
if applyErr := cfg.ApplyFileConfig(fileCfg); applyErr != nil {
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorApplyFile, Err: applyErr}
}
configVersion = &fileCfg.Version
}
if applyEnvErr := cfg.applyEnvOverrides(lookup); applyEnvErr != nil {
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorApplyEnv, Err: applyEnvErr}
}
return EffectiveConfig{
Config: cfg,
ConfigPath: configPath,
ConfigSource: configSource,
ConfigVersion: configVersion,
}, nil
}
func resolveConfigPathWithLookup(cliConfigPath string, cliConfigPathSet bool, lookup func(string) (string, bool), statPath func(string) (os.FileInfo, error), defaultSearchPaths []string) (path string, source string, err error) {
if cliConfigPathSet {
path = strings.TrimSpace(cliConfigPath)
if path == "" {
return "", "", fmt.Errorf("--config requires a non-empty path")
}
if _, statErr := statPath(path); statErr != nil {
if os.IsNotExist(statErr) {
return "", "", fmt.Errorf("config file not found: %s", path)
}
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
}
return path, "flag", nil
}
if raw, ok := lookup("AUDITA_CONFIG"); ok {
path = strings.TrimSpace(raw)
if path == "" {
return "", "", fmt.Errorf("AUDITA_CONFIG must not be empty")
}
if _, statErr := statPath(path); statErr != nil {
if os.IsNotExist(statErr) {
return "", "", fmt.Errorf("config file not found: %s", path)
}
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
}
return path, "env", nil
}
for _, defaultPath := range defaultSearchPaths {
if _, statErr := statPath(defaultPath); statErr == nil {
return defaultPath, "default", nil
} else if !os.IsNotExist(statErr) {
return "", "", fmt.Errorf("cannot access config file %s: %w", defaultPath, statErr)
}
}
return "", "", nil
}

View File

@@ -0,0 +1,163 @@
package config
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestResolveConfigPathWithLookupMatrix(t *testing.T) {
statFor := func(existing map[string]bool) func(string) (os.FileInfo, error) {
return func(path string) (os.FileInfo, error) {
if existing[path] {
return nil, nil
}
return nil, os.ErrNotExist
}
}
tests := []struct {
name string
cliPath string
cliPathSet bool
lookup func(string) (string, bool)
stat func(string) (os.FileInfo, error)
defaultSearchPaths []string
wantPath string
wantSource string
wantErrContains string
}{
{
name: "explicit config path",
cliPath: "/tmp/explicit.yml",
cliPathSet: true,
lookup: func(string) (string, bool) { return "", false },
stat: statFor(map[string]bool{"/tmp/explicit.yml": true}),
defaultSearchPaths: []string{
"/usr/local/etc/audita/config.yml",
"/etc/audita/config.yml",
},
wantPath: "/tmp/explicit.yml",
wantSource: "flag",
},
{
name: "env config path",
cliPathSet: false,
lookup: func(key string) (string, bool) {
if key == "AUDITA_CONFIG" {
return "/tmp/from-env.yml", true
}
return "", false
},
stat: statFor(map[string]bool{"/tmp/from-env.yml": true}),
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
wantPath: "/tmp/from-env.yml",
wantSource: "env",
},
{
name: "default search path",
cliPathSet: false,
lookup: func(string) (string, bool) { return "", false },
stat: statFor(map[string]bool{
"/usr/local/etc/audita/config.yml": true,
"/etc/audita/config.yml": true,
}),
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
wantPath: "/usr/local/etc/audita/config.yml",
wantSource: "default",
},
{
name: "explicit missing path",
cliPath: "/tmp/missing.yml",
cliPathSet: true,
lookup: func(string) (string, bool) { return "", false },
stat: statFor(map[string]bool{}),
defaultSearchPaths: []string{
"/usr/local/etc/audita/config.yml",
"/etc/audita/config.yml",
},
wantErrContains: "config file not found",
},
{
name: "missing env path",
cliPathSet: false,
lookup: func(key string) (string, bool) {
if key == "AUDITA_CONFIG" {
return "/tmp/missing-from-env.yml", true
}
return "", false
},
stat: statFor(map[string]bool{}),
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
wantErrContains: "config file not found",
},
{
name: "missing default paths",
cliPathSet: false,
lookup: func(string) (string, bool) { return "", false },
stat: statFor(map[string]bool{}),
defaultSearchPaths: []string{
"/usr/local/etc/audita/config.yml",
"/etc/audita/config.yml",
},
wantPath: "",
wantSource: "",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
gotPath, gotSource, err := resolveConfigPathWithLookup(tc.cliPath, tc.cliPathSet, tc.lookup, tc.stat, tc.defaultSearchPaths)
if tc.wantErrContains != "" {
if err == nil || !strings.Contains(err.Error(), tc.wantErrContains) {
t.Fatalf("expected error containing %q, got %v", tc.wantErrContains, err)
}
return
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if gotPath != tc.wantPath || gotSource != tc.wantSource {
t.Fatalf("unexpected result: got path=%q source=%q, want path=%q source=%q", gotPath, gotSource, tc.wantPath, tc.wantSource)
}
})
}
}
func TestLoadEffectiveConfigWithLookupAppliesDefaultsFileThenEnv(t *testing.T) {
tempDir := t.TempDir()
configPath := filepath.Join(tempDir, "config.yml")
configYAML := "version: 1\nllm:\n proposal:\n model: file-model\n"
if err := os.WriteFile(configPath, []byte(configYAML), 0o644); err != nil {
t.Fatalf("write config file: %v", err)
}
lookup := func(key string) (string, bool) {
switch key {
case "AUDITA_CONFIG":
return configPath, true
case "AUDITA_MODEL":
return "env-model", true
default:
return "", false
}
}
result, err := loadEffectiveConfigWithLookup("", false, lookup, os.Stat, DefaultConfigSearchPaths)
if err != nil {
t.Fatalf("loadEffectiveConfigWithLookup error: %v", err)
}
if result.ConfigPath != configPath {
t.Fatalf("unexpected config path: %q", result.ConfigPath)
}
if result.ConfigSource != "env" {
t.Fatalf("unexpected config source: %q", result.ConfigSource)
}
if result.ConfigVersion == nil || *result.ConfigVersion != SupportedFileConfigVersion {
t.Fatalf("unexpected config version: %#v", result.ConfigVersion)
}
if result.Config.PrimaryLLM.Model != "env-model" {
t.Fatalf("expected env override to win over file value, got %q", result.Config.PrimaryLLM.Model)
}
}

View File

@@ -3,6 +3,9 @@ package config
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
)
func (c Config) Validate() error {
@@ -12,20 +15,20 @@ func (c Config) Validate() error {
issues = append(issues, "modules must not be empty")
}
for _, module := range c.Modules {
if strings.TrimSpace(module) == "" {
moduleKey := strings.TrimSpace(module)
if moduleKey == "" {
issues = append(issues, "modules must not contain empty values")
break
}
if !modulecatalog.IsSupported(moduleKey) {
issues = append(issues, fmt.Sprintf("unsupported module key %q", moduleKey))
}
}
if strings.TrimSpace(c.OutputSchema) == "" {
issues = append(issues, "output schema must not be empty")
} else {
switch strings.TrimSpace(c.OutputSchema) {
case "bare-segments", "audita-v1":
default:
} else if !outputschema.IsSupported(c.OutputSchema) {
issues = append(issues, fmt.Sprintf("unsupported output schema %q", c.OutputSchema))
}
}
if c.PrimaryLLM.TimeoutSeconds <= 0 {
issues = append(issues, "primary llm timeout seconds must be greater than zero")

View File

@@ -0,0 +1,42 @@
package diagnostics
import (
"path/filepath"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
)
const (
ArtifactSourceTranscript = "source-transcript.json"
ArtifactParsedSourceTranscript = "source-transcript-parsed.json"
ArtifactNormalizedTranscript = "normalized-transcript.json"
ArtifactNormalizationSummary = "normalization-summary.json"
ArtifactChunkingSummary = "chunking-summary.json"
ArtifactUtilizationSummary = "utilization-diagnostics.json"
ArtifactCorrectionLedger = "correction-ledger.json"
ArtifactInvocationMetadata = "invocation.json"
ArtifactEffectiveConfig = "effective-config.json"
ArtifactReport = "report.json"
ArtifactErrorLog = "error.log"
)
func BuildDiagnosticsMetadata(runDirectoryPath string, runSucceeded bool) reporting.DiagnosticsMetadata {
metadata := reporting.DiagnosticsMetadata{
DirectoryPath: runDirectoryPath,
SourceTranscriptPath: filepath.Join(runDirectoryPath, ArtifactSourceTranscript),
ParsedSourceTranscriptPath: filepath.Join(runDirectoryPath, ArtifactParsedSourceTranscript),
NormalizedTranscriptPath: filepath.Join(runDirectoryPath, ArtifactNormalizedTranscript),
NormalizationSummaryPath: filepath.Join(runDirectoryPath, ArtifactNormalizationSummary),
ChunkingSummaryPath: filepath.Join(runDirectoryPath, ArtifactChunkingSummary),
UtilizationSummaryPath: filepath.Join(runDirectoryPath, ArtifactUtilizationSummary),
CorrectionLedgerPath: filepath.Join(runDirectoryPath, ArtifactCorrectionLedger),
InvocationMetadataPath: filepath.Join(runDirectoryPath, ArtifactInvocationMetadata),
RedactedEffectiveConfigPath: filepath.Join(runDirectoryPath, ArtifactEffectiveConfig),
}
if !runSucceeded {
metadata.ErrorLogPath = filepath.Join(runDirectoryPath, ArtifactErrorLog)
}
return metadata
}

View File

@@ -0,0 +1,55 @@
package diagnostics
import (
"path/filepath"
"testing"
)
func TestBuildDiagnosticsMetadataSuccessPathsMatchArtifactConstants(t *testing.T) {
runPath := filepath.Join("tmp", "run-123")
metadata := BuildDiagnosticsMetadata(runPath, true)
if metadata.DirectoryPath != runPath {
t.Fatalf("unexpected diagnostics directory path: got=%q want=%q", metadata.DirectoryPath, runPath)
}
if metadata.SourceTranscriptPath != filepath.Join(runPath, ArtifactSourceTranscript) {
t.Fatalf("unexpected source transcript path: %q", metadata.SourceTranscriptPath)
}
if metadata.ParsedSourceTranscriptPath != filepath.Join(runPath, ArtifactParsedSourceTranscript) {
t.Fatalf("unexpected parsed source transcript path: %q", metadata.ParsedSourceTranscriptPath)
}
if metadata.NormalizedTranscriptPath != filepath.Join(runPath, ArtifactNormalizedTranscript) {
t.Fatalf("unexpected normalized transcript path: %q", metadata.NormalizedTranscriptPath)
}
if metadata.NormalizationSummaryPath != filepath.Join(runPath, ArtifactNormalizationSummary) {
t.Fatalf("unexpected normalization summary path: %q", metadata.NormalizationSummaryPath)
}
if metadata.ChunkingSummaryPath != filepath.Join(runPath, ArtifactChunkingSummary) {
t.Fatalf("unexpected chunking summary path: %q", metadata.ChunkingSummaryPath)
}
if metadata.UtilizationSummaryPath != filepath.Join(runPath, ArtifactUtilizationSummary) {
t.Fatalf("unexpected utilization summary path: %q", metadata.UtilizationSummaryPath)
}
if metadata.CorrectionLedgerPath != filepath.Join(runPath, ArtifactCorrectionLedger) {
t.Fatalf("unexpected correction ledger path: %q", metadata.CorrectionLedgerPath)
}
if metadata.InvocationMetadataPath != filepath.Join(runPath, ArtifactInvocationMetadata) {
t.Fatalf("unexpected invocation metadata path: %q", metadata.InvocationMetadataPath)
}
if metadata.RedactedEffectiveConfigPath != filepath.Join(runPath, ArtifactEffectiveConfig) {
t.Fatalf("unexpected redacted effective config path: %q", metadata.RedactedEffectiveConfigPath)
}
if metadata.ErrorLogPath != "" {
t.Fatalf("did not expect error log path on success: %q", metadata.ErrorLogPath)
}
}
func TestBuildDiagnosticsMetadataFailureIncludesErrorLogPath(t *testing.T) {
runPath := filepath.Join("tmp", "run-123")
metadata := BuildDiagnosticsMetadata(runPath, false)
want := filepath.Join(runPath, ArtifactErrorLog)
if metadata.ErrorLogPath != want {
t.Fatalf("unexpected error log path: got=%q want=%q", metadata.ErrorLogPath, want)
}
}

View File

@@ -106,7 +106,7 @@ func (r *RunDirectory) WriteInvocationMetadata(metadata InvocationMetadata) erro
metadata.StartedAt = r.createdAt
}
path := filepath.Join(r.path, "invocation.json")
path := filepath.Join(r.path, ArtifactInvocationMetadata)
bytes, err := json.MarshalIndent(metadata, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal invocation metadata: %w", err)
@@ -120,7 +120,7 @@ func (r *RunDirectory) WriteInvocationMetadata(metadata InvocationMetadata) erro
// WriteEffectiveConfig writes redacted effective config metadata for this run.
func (r *RunDirectory) WriteEffectiveConfig(cfg config.Config) error {
path := filepath.Join(r.path, "effective-config.json")
path := filepath.Join(r.path, ArtifactEffectiveConfig)
redacted := cfg.Redacted()
bytes, err := json.MarshalIndent(redacted, "", " ")
if err != nil {
@@ -136,13 +136,13 @@ func (r *RunDirectory) WriteEffectiveConfig(cfg config.Config) error {
// WriteSourceTranscript writes the source transcript artifact
func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript, raw []byte) error {
// Write raw source for reference
sourcePath := filepath.Join(r.path, "source-transcript.json")
sourcePath := filepath.Join(r.path, ArtifactSourceTranscript)
if err := os.WriteFile(sourcePath, raw, 0o644); err != nil {
return fmt.Errorf("failed to write source transcript: %w", err)
}
// Write parsed source for debugging
parsedPath := filepath.Join(r.path, "source-transcript-parsed.json")
parsedPath := filepath.Join(r.path, ArtifactParsedSourceTranscript)
parsedBytes, err := json.MarshalIndent(transcript, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal parsed source transcript: %w", err)
@@ -157,7 +157,7 @@ func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript
// WriteNormalizedTranscript writes the normalized transcript artifact
func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript) error {
normalizedPath := filepath.Join(r.path, "normalized-transcript.json")
normalizedPath := filepath.Join(r.path, ArtifactNormalizedTranscript)
bytes, err := schema.TranscriptToJSON(transcript)
if err != nil {
return fmt.Errorf("failed to serialize normalized transcript: %w", err)
@@ -170,7 +170,7 @@ func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript)
// WriteNormalizationSummary writes the normalization summary artifact
func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.NormalizationSummary) error {
summaryPath := filepath.Join(r.path, "normalization-summary.json")
summaryPath := filepath.Join(r.path, ArtifactNormalizationSummary)
bytes, err := json.MarshalIndent(summary, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal normalization summary: %w", err)
@@ -184,7 +184,7 @@ func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.Normaliz
// WriteReport writes the authoritative report artifact
func (r *RunDirectory) WriteReport(report reporting.ProcessReport) error {
reportPath := filepath.Join(r.path, "report.json")
reportPath := filepath.Join(r.path, ArtifactReport)
bytes, err := json.MarshalIndent(report, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal report: %w", err)
@@ -198,13 +198,13 @@ func (r *RunDirectory) WriteReport(report reporting.ProcessReport) error {
// WriteErrorLog writes an error log on failure
func (r *RunDirectory) WriteErrorLog(errorMessage string) error {
errorPath := filepath.Join(r.path, "error.log")
errorPath := filepath.Join(r.path, ArtifactErrorLog)
return os.WriteFile(errorPath, []byte(errorMessage+"\n"), 0o644)
}
// WriteChunkingSummary writes the chunking summary artifact
func (r *RunDirectory) WriteChunkingSummary(summary *chunking.DetailedSummary) error {
summaryPath := filepath.Join(r.path, "chunking-summary.json")
summaryPath := filepath.Join(r.path, ArtifactChunkingSummary)
bytes, err := json.MarshalIndent(summary, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal chunking summary: %w", err)

View File

@@ -0,0 +1,35 @@
package modulecatalog
import "strings"
const (
KeyGlossary = "glossary"
KeyHomophones = "homophones"
KeySpokenWord = "spoken_word"
KeyGrammar = "grammar"
)
var supportedKeys = []string{
KeyGlossary,
KeyHomophones,
KeySpokenWord,
KeyGrammar,
}
var supportedKeySet = map[string]struct{}{
KeyGlossary: {},
KeyHomophones: {},
KeySpokenWord: {},
KeyGrammar: {},
}
func SupportedKeys() []string {
out := make([]string, len(supportedKeys))
copy(out, supportedKeys)
return out
}
func IsSupported(key string) bool {
_, ok := supportedKeySet[strings.TrimSpace(key)]
return ok
}

View File

@@ -0,0 +1,24 @@
package modulecatalog
import (
"reflect"
"testing"
)
func TestSupportedKeys(t *testing.T) {
want := []string{KeyGlossary, KeyHomophones, KeySpokenWord, KeyGrammar}
if got := SupportedKeys(); !reflect.DeepEqual(got, want) {
t.Fatalf("unexpected supported keys: got=%v want=%v", got, want)
}
}
func TestIsSupported(t *testing.T) {
for _, key := range SupportedKeys() {
if !IsSupported(key) {
t.Fatalf("expected key %q to be supported", key)
}
}
if IsSupported("made_up") {
t.Fatalf("did not expect made_up to be supported")
}
}

View File

@@ -31,15 +31,31 @@ var definitions = map[string]Definition{
},
}
var supportedKeys = []string{
SchemaBareSegments,
SchemaAuditaV1,
}
func SupportedKeys() []string {
out := make([]string, len(supportedKeys))
copy(out, supportedKeys)
return out
}
func IsSupported(key string) bool {
_, ok := definitions[strings.TrimSpace(key)]
return ok
}
func Resolve(key string) (Definition, error) {
normalized := strings.TrimSpace(key)
if normalized == "" {
return Definition{}, fmt.Errorf("output schema must not be empty")
}
def, ok := definitions[normalized]
if !ok {
if !IsSupported(normalized) {
return Definition{}, fmt.Errorf("unsupported output schema %q", normalized)
}
def := definitions[normalized]
return def, nil
}

View File

@@ -2,6 +2,7 @@ package outputschema
import (
"encoding/json"
"reflect"
"strings"
"testing"
@@ -56,3 +57,19 @@ func TestResolveUnknown(t *testing.T) {
t.Fatalf("expected unsupported output schema error, got %v", err)
}
}
func TestSupportedKeysAndIsSupported(t *testing.T) {
want := []string{SchemaBareSegments, SchemaAuditaV1}
if got := SupportedKeys(); !reflect.DeepEqual(got, want) {
t.Fatalf("unexpected supported schema keys: got=%v want=%v", got, want)
}
for _, key := range want {
if !IsSupported(key) {
t.Fatalf("expected schema key %q to be supported", key)
}
}
if IsSupported("seriatim-intermediate") {
t.Fatalf("did not expect unsupported schema to be reported as supported")
}
}

View File

@@ -0,0 +1,17 @@
package llm
import "gitea.maximumdirect.net/eric/audita/internal/core/config"
// ConfiguredSecrets returns all configured LLM API-key values that should be
// redacted from diagnostics and surfaced error payloads.
func ConfiguredSecrets(cfg *config.Config) []string {
if cfg == nil {
return nil
}
effectiveValidation := cfg.EffectiveValidationLLMConfig()
return []string{
cfg.PrimaryLLM.APIKey,
cfg.ValidationLLM.APIKey,
effectiveValidation.APIKey,
}
}

View File

@@ -0,0 +1,38 @@
package llm
import (
"reflect"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
)
func TestConfiguredSecretsNilConfig(t *testing.T) {
if got := ConfiguredSecrets(nil); got != nil {
t.Fatalf("expected nil secrets for nil config, got %v", got)
}
}
func TestConfiguredSecretsWithValidationOverride(t *testing.T) {
cfg := config.Default()
cfg.PrimaryLLM.APIKey = "primary-secret"
cfg.ValidationLLM.APIKey = "validation-secret"
got := ConfiguredSecrets(&cfg)
want := []string{"primary-secret", "validation-secret", "validation-secret"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("unexpected secrets: got=%v want=%v", got, want)
}
}
func TestConfiguredSecretsWithInheritedValidationKey(t *testing.T) {
cfg := config.Default()
cfg.PrimaryLLM.APIKey = "primary-secret"
cfg.ValidationLLM.APIKey = ""
got := ConfiguredSecrets(&cfg)
want := []string{"primary-secret", "", "primary-secret"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("unexpected inherited secrets: got=%v want=%v", got, want)
}
}

View File

@@ -6,6 +6,7 @@ import (
"strings"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
glossarymodule "gitea.maximumdirect.net/eric/audita/internal/modules/glossary"
@@ -15,28 +16,20 @@ import (
)
const (
ModuleKeyGlossary = "glossary"
ModuleKeyHomophones = "homophones"
ModuleKeySpokenWord = "spoken_word"
ModuleKeyGrammar = "grammar"
ModuleKeyGlossary = modulecatalog.KeyGlossary
ModuleKeyHomophones = modulecatalog.KeyHomophones
ModuleKeySpokenWord = modulecatalog.KeySpokenWord
ModuleKeyGrammar = modulecatalog.KeyGrammar
)
const (
ReasonUnsupportedModule = "unsupported_module"
)
var knownModuleKeys = map[string]struct{}{
ModuleKeyGlossary: {},
ModuleKeyHomophones: {},
ModuleKeySpokenWord: {},
ModuleKeyGrammar: {},
}
// IsKnownModuleKey reports whether a module key is recognized by the production
// registry scaffold.
func IsKnownModuleKey(key string) bool {
_, ok := knownModuleKeys[strings.TrimSpace(key)]
return ok
return modulecatalog.IsSupported(key)
}
// Dependencies holds explicit constructor dependencies for module creation.
@@ -69,7 +62,7 @@ type Factory struct {
func NewFactory(deps Dependencies) *Factory {
factory := &Factory{
deps: deps,
constructors: make(map[string]Constructor, len(knownModuleKeys)),
constructors: make(map[string]Constructor, len(modulecatalog.SupportedKeys())),
}
_ = factory.RegisterConstructor(ModuleKeyGlossary, constructGlossaryModule)
_ = factory.RegisterConstructor(ModuleKeyHomophones, constructHomophonesModule)

View File

@@ -6,6 +6,7 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
)
@@ -26,7 +27,7 @@ func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest)
}
func TestKnownModuleKeyRecognition(t *testing.T) {
for _, key := range []string{ModuleKeyGlossary, ModuleKeyHomophones, ModuleKeySpokenWord, ModuleKeyGrammar} {
for _, key := range modulecatalog.SupportedKeys() {
if !IsKnownModuleKey(key) {
t.Fatalf("expected key %q to be recognized", key)
}

View File

@@ -0,0 +1,45 @@
package promptcontext
import (
"encoding/json"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
type transcriptSectionSegment struct {
ID int `json:"id"`
Speaker string `json:"speaker"`
Start float64 `json:"start"`
End float64 `json:"end"`
Text string `json:"text"`
Categories []string `json:"categories,omitempty"`
}
type transcriptSectionPayload struct {
SectionIndex int `json:"section_index"`
Segments []transcriptSectionSegment `json:"segments"`
}
// MarshalTranscriptSectionJSON builds the standardized transcript-section JSON
// payload consumed by proposal prompt templates.
func MarshalTranscriptSectionJSON(transcript *schema.Transcript, sectionIndex int) ([]byte, error) {
payload := transcriptSectionPayload{
SectionIndex: sectionIndex,
Segments: make([]transcriptSectionSegment, 0),
}
if transcript != nil {
for _, s := range transcript.Segments {
payload.Segments = append(payload.Segments, transcriptSectionSegment{
ID: s.ID,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: append([]string(nil), s.Categories...),
})
}
}
return json.MarshalIndent(payload, "", " ")
}

View File

@@ -0,0 +1,95 @@
package promptcontext
import (
"encoding/json"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
func TestMarshalTranscriptSectionJSONShape(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Speaker: "A", Start: 0.1, End: 1.2, Text: "alpha", Categories: []string{"session", "intro"}},
}}
raw, err := MarshalTranscriptSectionJSON(transcript, 3)
if err != nil {
t.Fatalf("MarshalTranscriptSectionJSON error: %v", err)
}
var decoded map[string]any
if err := json.Unmarshal(raw, &decoded); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if got := decoded["section_index"]; got != float64(3) {
t.Fatalf("section_index: got=%v want=%v", got, 3)
}
segments, ok := decoded["segments"].([]any)
if !ok || len(segments) != 1 {
t.Fatalf("segments shape mismatch: %T %+v", decoded["segments"], decoded["segments"])
}
first, ok := segments[0].(map[string]any)
if !ok {
t.Fatalf("segment shape mismatch: %T", segments[0])
}
if first["id"] != float64(1) || first["speaker"] != "A" || first["start"] != 0.1 || first["end"] != 1.2 || first["text"] != "alpha" {
t.Fatalf("unexpected segment fields: %+v", first)
}
cats, ok := first["categories"].([]any)
if !ok || len(cats) != 2 || cats[0] != "session" || cats[1] != "intro" {
t.Fatalf("unexpected categories: %+v", first["categories"])
}
}
func TestMarshalTranscriptSectionJSONEmptyTranscript(t *testing.T) {
raw, err := MarshalTranscriptSectionJSON(nil, 0)
if err != nil {
t.Fatalf("MarshalTranscriptSectionJSON error: %v", err)
}
var decoded map[string]any
if err := json.Unmarshal(raw, &decoded); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if got := decoded["section_index"]; got != float64(0) {
t.Fatalf("section_index: got=%v want=%v", got, 0)
}
segments, ok := decoded["segments"].([]any)
if !ok {
t.Fatalf("segments shape mismatch: %T", decoded["segments"])
}
if len(segments) != 0 {
t.Fatalf("expected empty segments, got %d", len(segments))
}
}
func TestMarshalTranscriptSectionJSONCopiesCategories(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "alpha", Categories: []string{"kept"}},
}}
raw, err := MarshalTranscriptSectionJSON(transcript, 1)
if err != nil {
t.Fatalf("MarshalTranscriptSectionJSON error: %v", err)
}
transcript.Segments[0].Categories[0] = "changed"
var decoded struct {
Segments []struct {
Categories []string `json:"categories"`
} `json:"segments"`
}
if err := json.Unmarshal(raw, &decoded); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if len(decoded.Segments) != 1 {
t.Fatalf("expected one segment, got %d", len(decoded.Segments))
}
if !reflect.DeepEqual(decoded.Segments[0].Categories, []string{"kept"}) {
t.Fatalf("expected copied categories, got %v", decoded.Segments[0].Categories)
}
}

View File

@@ -15,6 +15,8 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
)
@@ -94,7 +96,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
stage := strings.TrimSpace(req.StageName)
if stage == "" {
stage = buildStageName(req.ModuleInstance, req.Section)
stage = stagename.ProposalGeneration(req.ModuleInstance, sectionIndexPtr(req.Section))
}
model := resolveModel(req.Config, req.Model)
messages := append([]contracts.LLMMessage(nil), req.Messages...)
@@ -106,7 +108,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
writer = diagnosticsWriterAdapter{
writer: llm.NewDiagnosticsWriter(
filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
proposalGenerationSecrets(req.Config),
llm.ConfiguredSecrets(req.Config),
),
}
}
@@ -143,7 +145,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
if len(req.PromptMetadata) > 0 {
requestMetadata["prompt_metadata"] = req.PromptMetadata
}
requestMetadata["response_schema"] = schemaMetadata(responseSchema)
requestMetadata["response_schema"] = responseSchema.DiagnosticsMap()
if writer != nil {
artifacts, _ = writer.WriteInteraction(
@@ -158,7 +160,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
}
if callErr != nil {
if isMalformedStructuredOutputError(callErr) {
if structuredoutput.IsMalformedError(callErr) {
return Result{
Warnings: []stagewarnings.StageWarning{newMalformedProposalWarning(req.Section, artifacts, callErr)},
Artifacts: artifacts,
@@ -201,23 +203,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
}, nil
}
func schemaMetadata(schema responseschema.Schema) map[string]any {
return map[string]any{
"id": schema.ID,
"version": schema.Version,
"name": schema.Name,
"sha256": schema.SHA256,
}
}
func buildStageName(moduleInstance string, section *contracts.SectionMetadata) string {
base := fmt.Sprintf("%s:proposal-generation", moduleInstance)
if section == nil {
return base
}
return fmt.Sprintf("%s:section-%04d", base, section.Index)
}
func resolveModel(cfg *config.Config, override string) string {
if strings.TrimSpace(override) != "" {
return strings.TrimSpace(override)
@@ -228,17 +213,6 @@ func resolveModel(cfg *config.Config, override string) string {
return llm.ResolvePrimaryConfig(*cfg).Model
}
func proposalGenerationSecrets(cfg *config.Config) []string {
if cfg == nil {
return nil
}
return []string{
cfg.PrimaryLLM.APIKey,
cfg.ValidationLLM.APIKey,
cfg.EffectiveValidationLLMConfig().APIKey,
}
}
func errPayload(err error) any {
if err == nil {
return nil
@@ -246,27 +220,6 @@ func errPayload(err error) any {
return map[string]any{"error": err.Error()}
}
func isMalformedStructuredOutputError(err error) bool {
if err == nil {
return false
}
msg := err.Error()
for _, marker := range []string{
"malformed structured output",
"decode structured output:",
"decode provider response envelope:",
"provider response missing choices",
"provider response missing assistant message content",
"provider response assistant message content is empty",
"provider response assistant message content is not valid JSON",
} {
if strings.Contains(msg, marker) {
return true
}
}
return false
}
func newMalformedProposalWarning(section *contracts.SectionMetadata, artifacts InteractionArtifacts, err error) stagewarnings.StageWarning {
warning := stagewarnings.StageWarning{
Scope: stagewarnings.ScopeProposalGeneration,
@@ -288,6 +241,14 @@ func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
return artifacts.ResponsePayloadPath
}
func sectionIndexPtr(section *contracts.SectionMetadata) *int {
if section == nil {
return nil
}
index := section.Index
return &index
}
type diagnosticsWriterAdapter struct {
writer *llm.DiagnosticsWriter
}

View File

@@ -272,6 +272,23 @@ func TestGenerateCandidatesMalformedStructuredOutputReturnsWarning(t *testing.T)
}
}
func TestGenerateCandidatesProviderMalformedEnvelopeReturnsWarning(t *testing.T) {
client := &fakeStructuredClient{err: errors.New("provider response missing choices")}
req := defaultRequest(t)
req.LLMClient = client
result, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("expected malformed provider envelope to downgrade to warning, got %v", err)
}
if len(result.Corrections) != 0 || len(result.Enriched) != 0 {
t.Fatalf("expected no proposals on malformed response, got %+v", result)
}
if len(result.Warnings) != 1 || result.Warnings[0].ReasonCode != "proposal_response_malformed" {
t.Fatalf("unexpected warnings: %+v", result.Warnings)
}
}
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
baseResponse := StructuredCorrectionSet{
Corrections: []StructuredCorrectionProposal{
@@ -344,20 +361,22 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
}
func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
secret := "proposal-secret"
primarySecret := "proposal-primary-secret"
validationSecret := "proposal-validation-secret"
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: secret, CorrectedText: "safe", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}},
},
}
cfg := config.Default()
cfg.PrimaryLLM.APIKey = secret
cfg.PrimaryLLM.APIKey = primarySecret
cfg.ValidationLLM.APIKey = validationSecret
req := defaultRequest(t)
req.Config = &cfg
req.LLMClient = client
req.DiagnosticsDir = t.TempDir()
req.Messages = []contracts.LLMMessage{
{Role: "system", Content: "include secret " + secret},
{Role: "system", Content: "include secret " + primarySecret},
{Role: "user", Content: "fix it"},
}
@@ -374,8 +393,8 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
if readErr != nil {
t.Fatalf("read artifact %q: %v", path, readErr)
}
if strings.Contains(string(raw), secret) {
t.Fatalf("artifact leaked secret %q: %s", path, string(raw))
if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) {
t.Fatalf("artifact leaked configured secret in %q: %s", path, string(raw))
}
if !strings.Contains(string(raw), "[REDACTED]") {
t.Fatalf("expected redaction marker in artifact %q: %s", path, string(raw))

View File

@@ -0,0 +1,81 @@
package proposal_generation
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
type ProposalMessageBuilder func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error)
type ModuleProposalRequest struct {
ProposalRequest contracts.ProposalRequest
PromptID string
BuildMessages ProposalMessageBuilder
}
// ExecuteModuleProposal runs shared proposal generation plumbing for one
// module, leaving only module-specific prompt message building at call sites.
func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (contracts.ProposalResult, error) {
if req.BuildMessages == nil {
return contracts.ProposalResult{}, fmt.Errorf("proposal message builder is required")
}
if strings.TrimSpace(req.PromptID) == "" {
return contracts.ProposalResult{}, fmt.Errorf("prompt ID must not be empty")
}
sectionIndex := 0
if req.ProposalRequest.Section != nil {
sectionIndex = req.ProposalRequest.Section.Index
}
transcriptDescription := ""
if req.ProposalRequest.Config != nil {
transcriptDescription = req.ProposalRequest.Config.TranscriptDescription
}
messages, err := req.BuildMessages(
req.ProposalRequest.WorkingTranscript,
req.ProposalRequest.Glossary,
sectionIndex,
transcriptDescription,
)
if err != nil {
return contracts.ProposalResult{}, err
}
promptMetadata, ok := prompts.LookupMetadata(req.PromptID)
if !ok {
return contracts.ProposalResult{}, fmt.Errorf("unknown prompt ID %q", req.PromptID)
}
generated, err := GenerateCandidates(ctx, Request{
ModuleKey: req.ProposalRequest.RunSpec.ModuleKey,
ModuleInstance: req.ProposalRequest.RunSpec.InstanceName,
ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy,
WorkingTranscript: req.ProposalRequest.WorkingTranscript,
Section: req.ProposalRequest.Section,
Glossary: req.ProposalRequest.Glossary,
Config: req.ProposalRequest.Config,
Messages: messages,
PromptMetadata: promptMetadata.DiagnosticsMap(),
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
StartIndex: 0,
LLMClient: req.ProposalRequest.LLMClient,
Scheduler: req.ProposalRequest.LLMScheduler,
DiagnosticsDir: req.ProposalRequest.DiagnosticsDir,
})
if err != nil {
return contracts.ProposalResult{}, err
}
return contracts.ProposalResult{
Proposals: generated.Corrections,
Warnings: generated.Warnings,
}, nil
}

View File

@@ -0,0 +1,115 @@
package proposal_generation
import (
"context"
"errors"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
},
}
section := contracts.SectionMetadata{Index: 7}
cfg := config.Default()
cfg.TranscriptDescription = "Hearing transcript with role titles."
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh"}}}
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "X"}}}
var gotSectionIndex int
var gotDescription string
var gotTranscript *schema.Transcript
var gotGlossary *schema.Glossary
out, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
ProposalRequest: contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
Config: &cfg,
WorkingTranscript: transcript,
Glossary: glossary,
Section: &section,
},
RunSpec: contracts.ModuleRunSpec{
ModuleKey: "grammar",
InstanceName: "grammar",
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
},
LLMClient: client,
},
PromptID: prompts.PromptIDModuleGrammarProposal,
BuildMessages: func(inTranscript *schema.Transcript, inGlossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
gotSectionIndex = sectionIndex
gotDescription = transcriptDescription
gotTranscript = inTranscript
gotGlossary = inGlossary
return []contracts.LLMMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}, nil
},
})
if err != nil {
t.Fatalf("ExecuteModuleProposal error: %v", err)
}
if gotSectionIndex != 7 {
t.Fatalf("section index: got=%d want=%d", gotSectionIndex, 7)
}
if gotDescription != cfg.TranscriptDescription {
t.Fatalf("transcript description: got=%q want=%q", gotDescription, cfg.TranscriptDescription)
}
if gotTranscript != transcript {
t.Fatalf("expected shared transcript pointer")
}
if gotGlossary != glossary {
t.Fatalf("expected shared glossary pointer")
}
if len(client.calls) != 1 || client.calls[0].StageName != "grammar:proposal:section-0007" {
t.Fatalf("unexpected stage name calls: %+v", client.calls)
}
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "the" {
t.Fatalf("unexpected proposals: %+v", out)
}
}
func TestExecuteModuleProposalValidatesInputs(t *testing.T) {
if _, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{}); err == nil {
t.Fatalf("expected missing message builder error")
}
_, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
BuildMessages: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
return nil, nil
},
})
if err == nil {
t.Fatalf("expected empty prompt ID error")
}
_, err = ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
PromptID: "missing.prompt.id",
BuildMessages: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
return []contracts.LLMMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}, nil
},
})
if err == nil {
t.Fatalf("expected unknown prompt ID error")
}
}
func TestExecuteModuleProposalPropagatesBuilderError(t *testing.T) {
wantErr := errors.New("builder failed")
_, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
PromptID: prompts.PromptIDModuleGlossaryProposal,
BuildMessages: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
return nil, wantErr
},
})
if !errors.Is(err, wantErr) {
t.Fatalf("expected builder error, got %v", err)
}
}

View File

@@ -5,6 +5,7 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"sort"
"strings"
)
@@ -28,6 +29,15 @@ type Schema struct {
SHA256 string `json:"sha256"`
}
func (s Schema) DiagnosticsMap() map[string]any {
return map[string]any{
"id": s.ID,
"version": s.Version,
"name": s.Name,
"sha256": s.SHA256,
}
}
var registry = map[Key]Schema{
CorrectionSetKey: mustBuildSchema(
correctionSetSchemaID,
@@ -43,6 +53,20 @@ var registry = map[Key]Schema{
),
}
func Registered() []Schema {
keys := make([]string, 0, len(registry))
for key := range registry {
keys = append(keys, string(key))
}
sort.Strings(keys)
out := make([]Schema, 0, len(keys))
for _, key := range keys {
out = append(out, cloneSchema(registry[Key(key)]))
}
return out
}
// Lookup returns a copy of the registered schema for the provided key.
func Lookup(key Key) (Schema, bool) {
schema, ok := registry[key]

View File

@@ -89,3 +89,20 @@ func TestLookupReturnsSchemaCopy(t *testing.T) {
t.Fatalf("expected lookup to return independent schema copy")
}
}
func TestDiagnosticsMapIncludesStableSchemaMetadataShapeForAllSchemas(t *testing.T) {
registered := Registered()
if len(registered) == 0 {
t.Fatalf("expected registered response schemas")
}
for _, schema := range registered {
metadataMap := schema.DiagnosticsMap()
if metadataMap["id"] != schema.ID ||
metadataMap["version"] != schema.Version ||
metadataMap["name"] != schema.Name ||
metadataMap["sha256"] != schema.SHA256 {
t.Fatalf("unexpected diagnostics metadata map for %q: %+v", schema.ID, metadataMap)
}
}
}

View File

@@ -527,7 +527,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
diagnosticsWriter = &llmDiagnosticsWriterAdapter{
writer: llm.NewDiagnosticsWriter(
filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName),
validatorSecrets(input.Config),
llm.ConfiguredSecrets(input.Config),
),
}
}
@@ -744,15 +744,3 @@ func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMeta
ErrorPayloadPath: art.ErrorPayloadPath,
}, nil
}
func validatorSecrets(cfg *config.Config) []string {
if cfg == nil {
return nil
}
effective := cfg.EffectiveValidationLLMConfig()
return []string{
cfg.PrimaryLLM.APIKey,
effective.APIKey,
cfg.ValidationLLM.APIKey,
}
}

View File

@@ -1307,12 +1307,13 @@ func TestRunnerLLMValidatorBatchingAndSchedulerUsage(t *testing.T) {
}
func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
secret := "super-secret-key"
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: secret}}}}}
primarySecret := "runner-primary-secret"
validationSecret := "runner-validation-secret"
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
cfg := config.Default()
cfg.PrimaryLLM.APIKey = secret
cfg.ValidationLLM.APIKey = secret
cfg.PrimaryLLM.APIKey = primarySecret
cfg.ValidationLLM.APIKey = validationSecret
diagDir := t.TempDir()
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
@@ -1321,7 +1322,7 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
}})
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple. " + primarySecret}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
ValidationDiagnosticsDir: diagDir,
@@ -1332,12 +1333,26 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
if len(out.ModuleResults[0].ValidatorDecisions) == 0 || out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath == "" {
t.Fatalf("expected diagnostic artifact path on decision")
}
matches, globErr := filepath.Glob(filepath.Join(diagDir, "m", "*.json"))
if globErr != nil {
t.Fatalf("glob diagnostics: %v", globErr)
}
if len(matches) == 0 {
t.Fatalf("expected diagnostics JSON artifacts under %s", filepath.Join(diagDir, "m"))
}
for _, path := range matches {
raw, readErr := os.ReadFile(path)
if readErr != nil {
t.Fatalf("read diagnostic %q: %v", path, readErr)
}
if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) {
t.Fatalf("configured secret leaked in diagnostics %q: %s", path, string(raw))
}
}
raw, readErr := os.ReadFile(out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath)
if readErr != nil {
t.Fatalf("read diagnostic: %v", readErr)
}
if strings.Contains(string(raw), secret) {
t.Fatalf("secret leaked in diagnostics: %s", string(raw))
t.Fatalf("read decision diagnostic: %v", readErr)
}
if !strings.Contains(string(raw), "[REDACTED]") {
t.Fatalf("expected redaction marker in diagnostics")

View File

@@ -0,0 +1,24 @@
package stagename
import (
"fmt"
)
func ModuleProposal(moduleInstance string, sectionIndex *int) string {
if sectionIndex == nil || *sectionIndex == 0 {
return fmt.Sprintf("%s:proposal", moduleInstance)
}
return fmt.Sprintf("%s:proposal:section-%04d", moduleInstance, *sectionIndex)
}
func ProposalGeneration(moduleInstance string, sectionIndex *int) string {
base := fmt.Sprintf("%s:proposal-generation", moduleInstance)
if sectionIndex == nil {
return base
}
return fmt.Sprintf("%s:section-%04d", base, *sectionIndex)
}
func ValidatorBatch(moduleInstance string, validatorName string, batchIndex int) string {
return fmt.Sprintf("%s:%s:batch-%04d", moduleInstance, validatorName, batchIndex)
}

View File

@@ -0,0 +1,33 @@
package stagename
import (
"testing"
)
func TestModuleProposalStageName(t *testing.T) {
if got := ModuleProposal("grammar", nil); got != "grammar:proposal" {
t.Fatalf("unexpected stage name without section: %q", got)
}
sectionIndex := 7
if got := ModuleProposal("grammar", &sectionIndex); got != "grammar:proposal:section-0007" {
t.Fatalf("unexpected stage name with section: %q", got)
}
}
func TestProposalGenerationStageName(t *testing.T) {
if got := ProposalGeneration("grammar", nil); got != "grammar:proposal-generation" {
t.Fatalf("unexpected proposal generation stage name without section: %q", got)
}
sectionIndex := 3
if got := ProposalGeneration("grammar", &sectionIndex); got != "grammar:proposal-generation:section-0003" {
t.Fatalf("unexpected proposal generation stage name with section: %q", got)
}
}
func TestValidatorBatchStageName(t *testing.T) {
if got := ValidatorBatch("homophones_1", "spoken_form_plausibility_review", 12); got != "homophones_1:spoken_form_plausibility_review:batch-0012" {
t.Fatalf("unexpected validator batch stage name: %q", got)
}
}

View File

@@ -0,0 +1,28 @@
package structuredoutput
import "strings"
var malformedMarkers = []string{
"malformed structured output",
"decode structured output:",
"decode provider response envelope:",
"provider response missing choices",
"provider response missing assistant message content",
"provider response assistant message content is empty",
"provider response assistant message content is not valid JSON",
}
// IsMalformedError reports whether err matches provider malformed
// structured-output failure markers that should be downgraded.
func IsMalformedError(err error) bool {
if err == nil {
return false
}
msg := err.Error()
for _, marker := range malformedMarkers {
if strings.Contains(msg, marker) {
return true
}
}
return false
}

View File

@@ -0,0 +1,32 @@
package structuredoutput
import (
"errors"
"testing"
)
func TestIsMalformedError(t *testing.T) {
cases := []struct {
name string
err error
want bool
}{
{name: "nil", err: nil, want: false},
{name: "generic", err: errors.New("network timeout"), want: false},
{name: "malformed", err: errors.New("malformed structured output"), want: true},
{name: "decode structured", err: errors.New("decode structured output: unexpected end of JSON input"), want: true},
{name: "missing choices", err: errors.New("provider response missing choices"), want: true},
{name: "missing content", err: errors.New("provider response missing assistant message content"), want: true},
{name: "empty content", err: errors.New("provider response assistant message content is empty"), want: true},
{name: "invalid content json", err: errors.New("provider response assistant message content is not valid JSON"), want: true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := IsMalformedError(tc.err)
if got != tc.want {
t.Fatalf("IsMalformedError(%v): got=%v want=%v", tc.err, got, tc.want)
}
})
}
}

View File

@@ -11,6 +11,8 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
@@ -110,9 +112,10 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
var response LLMValidationResponse
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
stage := stagename.ValidatorBatch(req.ModuleInstance, v.name, batch.BatchIndex)
call := func(callCtx context.Context) error {
_, err = req.LLMClient.CompleteStructured(callCtx, StructuredCompletionRequest{
StageName: fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex),
StageName: stage,
Messages: messages,
Model: resolvedValidationModel(req.Config, v.model),
ResponseSchema: &responseSchema,
@@ -126,7 +129,6 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
artifacts := InteractionArtifacts{}
if req.DiagnosticsWriter != nil {
stage := fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex)
promptMetadata := validatorPromptMetadata(v.validatorType)
artifacts, _ = req.DiagnosticsWriter.WriteInteraction(
stage,
@@ -134,19 +136,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
"validator_name": v.name,
"validator_type": v.validatorType,
"batch_index": batch.BatchIndex,
"prompt_metadata": map[string]any{
"prompt_id": promptMetadata.PromptID,
"prompt_version": promptMetadata.PromptVersion,
"prompt_source": promptMetadata.PromptSource,
"embedded_path": promptMetadata.EmbeddedPath,
"sha256": promptMetadata.SHA256,
},
"response_schema": map[string]any{
"id": responseSchema.ID,
"version": responseSchema.Version,
"name": responseSchema.Name,
"sha256": responseSchema.SHA256,
},
"prompt_metadata": promptMetadata.DiagnosticsMap(),
"response_schema": responseSchema.DiagnosticsMap(),
},
map[string]any{"messages": messages, "items": batch.Items},
response,
@@ -154,7 +145,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
)
}
if err != nil {
if isMalformedStructuredOutputError(err) {
if structuredoutput.IsMalformedError(err) {
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
continue
@@ -397,24 +388,3 @@ func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
}
return artifacts.ResponsePayloadPath
}
func isMalformedStructuredOutputError(err error) bool {
if err == nil {
return false
}
msg := err.Error()
for _, marker := range []string{
"malformed structured output",
"decode structured output:",
"decode provider response envelope:",
"provider response missing choices",
"provider response missing assistant message content",
"provider response assistant message content is empty",
"provider response assistant message content is not valid JSON",
} {
if strings.Contains(msg, marker) {
return true
}
}
return false
}

View File

@@ -265,6 +265,23 @@ func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
}
}
func TestLLMBackedValidatorProviderMalformedEnvelopeRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{err: errors.New("provider response assistant message content is empty")}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("expected malformed provider envelope downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("expected malformed warning, got %+v", res.Warnings)
}
}
func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")

View File

@@ -6,6 +6,7 @@ import (
"strings"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
@@ -118,13 +119,13 @@ func confidenceThresholdForModule(moduleKey string, cfg *config.Config) float64
return 0.0
}
switch moduleKey {
case "glossary":
case modulecatalog.KeyGlossary:
return cfg.Thresholds.Glossary
case "grammar":
case modulecatalog.KeyGrammar:
return cfg.Thresholds.Grammar
case "homophones":
case modulecatalog.KeyHomophones:
return cfg.Thresholds.Homophones
case "spoken_word":
case modulecatalog.KeySpokenWord:
return cfg.Thresholds.SpokenWord
default:
return 0.0

View File

@@ -2,12 +2,11 @@ package glossary
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
)
@@ -37,67 +36,9 @@ func (m *Module) Validators() []contracts.Validator {
}
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
sectionIndex := 0
if req.Section != nil {
sectionIndex = req.Section.Index
}
transcriptDescription := ""
if req.Config != nil {
transcriptDescription = req.Config.TranscriptDescription
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil {
return contracts.ProposalResult{}, err
}
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
ModuleKey: req.RunSpec.ModuleKey,
ModuleInstance: req.RunSpec.InstanceName,
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
WorkingTranscript: req.WorkingTranscript,
Section: req.Section,
Glossary: req.Glossary,
Config: req.Config,
Messages: messages,
PromptMetadata: map[string]any{
"prompt_id": proposalPromptMetadata().PromptID,
"prompt_version": proposalPromptMetadata().PromptVersion,
"prompt_source": proposalPromptMetadata().PromptSource,
"embedded_path": proposalPromptMetadata().EmbeddedPath,
"sha256": proposalPromptMetadata().SHA256,
},
StageName: proposalStageName(req),
StartIndex: 0,
LLMClient: req.LLMClient,
Scheduler: req.LLMScheduler,
DiagnosticsDir: req.DiagnosticsDir,
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
ProposalRequest: req,
PromptID: prompts.PromptIDModuleGlossaryProposal,
BuildMessages: BuildProposalMessages,
})
if err != nil {
return contracts.ProposalResult{}, err
}
return contracts.ProposalResult{
Proposals: generated.Corrections,
Warnings: generated.Warnings,
}, nil
}
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
if section == nil || transcript == nil {
return transcript
}
segments := make([]schema.Segment, 0, len(transcript.Segments))
for _, seg := range transcript.Segments {
if seg.ID >= section.StartSegmentID && seg.ID <= section.EndSegmentID {
segments = append(segments, seg)
}
}
return &schema.Transcript{Segments: segments}
}
func proposalStageName(req contracts.ProposalRequest) string {
if req.Section == nil || req.Section.Index == 0 {
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName)
}
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index)
}

View File

@@ -10,43 +10,13 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
type promptSegment struct {
ID int `json:"id"`
Speaker string `json:"speaker"`
Start float64 `json:"start"`
End float64 `json:"end"`
Text string `json:"text"`
Categories []string `json:"categories,omitempty"`
}
type promptTranscriptSection struct {
SectionIndex int `json:"section_index"`
Segments []promptSegment `json:"segments"`
}
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
if err != nil {
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
}
sectionPayload := promptTranscriptSection{
SectionIndex: sectionIndex,
Segments: make([]promptSegment, 0),
}
if transcript != nil {
for _, s := range transcript.Segments {
sectionPayload.Segments = append(sectionPayload.Segments, promptSegment{
ID: s.ID,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: append([]string(nil), s.Categories...),
})
}
}
sectionJSON, err := json.MarshalIndent(sectionPayload, "", " ")
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
if err != nil {
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
}
@@ -65,7 +35,3 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
{Role: "user", Content: user},
}, nil
}
func proposalPromptMetadata() prompts.Metadata {
return prompts.MustLookupMetadata(prompts.PromptIDModuleGlossaryProposal)
}

View File

@@ -2,12 +2,11 @@ package grammar
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
)
@@ -37,67 +36,9 @@ func (m *Module) Validators() []contracts.Validator {
}
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
sectionIndex := 0
if req.Section != nil {
sectionIndex = req.Section.Index
}
transcriptDescription := ""
if req.Config != nil {
transcriptDescription = req.Config.TranscriptDescription
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil {
return contracts.ProposalResult{}, err
}
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
ModuleKey: req.RunSpec.ModuleKey,
ModuleInstance: req.RunSpec.InstanceName,
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
WorkingTranscript: req.WorkingTranscript,
Section: req.Section,
Glossary: req.Glossary,
Config: req.Config,
Messages: messages,
PromptMetadata: map[string]any{
"prompt_id": proposalPromptMetadata().PromptID,
"prompt_version": proposalPromptMetadata().PromptVersion,
"prompt_source": proposalPromptMetadata().PromptSource,
"embedded_path": proposalPromptMetadata().EmbeddedPath,
"sha256": proposalPromptMetadata().SHA256,
},
StageName: proposalStageName(req),
StartIndex: 0,
LLMClient: req.LLMClient,
Scheduler: req.LLMScheduler,
DiagnosticsDir: req.DiagnosticsDir,
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
ProposalRequest: req,
PromptID: prompts.PromptIDModuleGrammarProposal,
BuildMessages: BuildProposalMessages,
})
if err != nil {
return contracts.ProposalResult{}, err
}
return contracts.ProposalResult{
Proposals: generated.Corrections,
Warnings: generated.Warnings,
}, nil
}
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
if section == nil || transcript == nil {
return transcript
}
segments := make([]schema.Segment, 0, len(transcript.Segments))
for _, seg := range transcript.Segments {
if seg.ID >= section.StartSegmentID && seg.ID <= section.EndSegmentID {
segments = append(segments, seg)
}
}
return &schema.Transcript{Segments: segments}
}
func proposalStageName(req contracts.ProposalRequest) string {
if req.Section == nil || req.Section.Index == 0 {
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName)
}
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index)
}

View File

@@ -10,20 +10,6 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
type promptSegment struct {
ID int `json:"id"`
Speaker string `json:"speaker"`
Start float64 `json:"start"`
End float64 `json:"end"`
Text string `json:"text"`
Categories []string `json:"categories,omitempty"`
}
type promptTranscriptSection struct {
SectionIndex int `json:"section_index"`
Segments []promptSegment `json:"segments"`
}
// BuildProposalMessages constrains corrections to punctuation/capitalization/
// spacing cleanup with strict meaning guards.
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
@@ -32,24 +18,7 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
}
sectionPayload := promptTranscriptSection{
SectionIndex: sectionIndex,
Segments: make([]promptSegment, 0),
}
if transcript != nil {
for _, s := range transcript.Segments {
sectionPayload.Segments = append(sectionPayload.Segments, promptSegment{
ID: s.ID,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: append([]string(nil), s.Categories...),
})
}
}
sectionJSON, err := json.MarshalIndent(sectionPayload, "", " ")
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
if err != nil {
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
}
@@ -68,7 +37,3 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
{Role: "user", Content: user},
}, nil
}
func proposalPromptMetadata() prompts.Metadata {
return prompts.MustLookupMetadata(prompts.PromptIDModuleGrammarProposal)
}

View File

@@ -2,12 +2,11 @@ package homophones
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
)
@@ -37,67 +36,9 @@ func (m *Module) Validators() []contracts.Validator {
}
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
sectionIndex := 0
if req.Section != nil {
sectionIndex = req.Section.Index
}
transcriptDescription := ""
if req.Config != nil {
transcriptDescription = req.Config.TranscriptDescription
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil {
return contracts.ProposalResult{}, err
}
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
ModuleKey: req.RunSpec.ModuleKey,
ModuleInstance: req.RunSpec.InstanceName,
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
WorkingTranscript: req.WorkingTranscript,
Section: req.Section,
Glossary: req.Glossary,
Config: req.Config,
Messages: messages,
PromptMetadata: map[string]any{
"prompt_id": proposalPromptMetadata().PromptID,
"prompt_version": proposalPromptMetadata().PromptVersion,
"prompt_source": proposalPromptMetadata().PromptSource,
"embedded_path": proposalPromptMetadata().EmbeddedPath,
"sha256": proposalPromptMetadata().SHA256,
},
StageName: proposalStageName(req),
StartIndex: 0,
LLMClient: req.LLMClient,
Scheduler: req.LLMScheduler,
DiagnosticsDir: req.DiagnosticsDir,
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
ProposalRequest: req,
PromptID: prompts.PromptIDModuleHomophonesProposal,
BuildMessages: BuildProposalMessages,
})
if err != nil {
return contracts.ProposalResult{}, err
}
return contracts.ProposalResult{
Proposals: generated.Corrections,
Warnings: generated.Warnings,
}, nil
}
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
if section == nil || transcript == nil {
return transcript
}
segments := make([]schema.Segment, 0, len(transcript.Segments))
for _, seg := range transcript.Segments {
if seg.ID >= section.StartSegmentID && seg.ID <= section.EndSegmentID {
segments = append(segments, seg)
}
}
return &schema.Transcript{Segments: segments}
}
func proposalStageName(req contracts.ProposalRequest) string {
if req.Section == nil || req.Section.Index == 0 {
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName)
}
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index)
}

View File

@@ -10,20 +10,6 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
type promptSegment struct {
ID int `json:"id"`
Speaker string `json:"speaker"`
Start float64 `json:"start"`
End float64 `json:"end"`
Text string `json:"text"`
Categories []string `json:"categories,omitempty"`
}
type promptTranscriptSection struct {
SectionIndex int `json:"section_index"`
Segments []promptSegment `json:"segments"`
}
// BuildProposalMessages constrains corrections to conservative homophone and
// mistranscription updates.
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
@@ -32,23 +18,7 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
}
sectionPayload := promptTranscriptSection{
SectionIndex: sectionIndex,
Segments: make([]promptSegment, 0),
}
if transcript != nil {
for _, s := range transcript.Segments {
sectionPayload.Segments = append(sectionPayload.Segments, promptSegment{
ID: s.ID,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: append([]string(nil), s.Categories...),
})
}
}
sectionJSON, err := json.MarshalIndent(sectionPayload, "", " ")
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
if err != nil {
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
}
@@ -67,7 +37,3 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
{Role: "user", Content: user},
}, nil
}
func proposalPromptMetadata() prompts.Metadata {
return prompts.MustLookupMetadata(prompts.PromptIDModuleHomophonesProposal)
}

View File

@@ -2,12 +2,11 @@ package spoken_word
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
)
@@ -37,67 +36,9 @@ func (m *Module) Validators() []contracts.Validator {
}
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
sectionIndex := 0
if req.Section != nil {
sectionIndex = req.Section.Index
}
transcriptDescription := ""
if req.Config != nil {
transcriptDescription = req.Config.TranscriptDescription
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil {
return contracts.ProposalResult{}, err
}
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
ModuleKey: req.RunSpec.ModuleKey,
ModuleInstance: req.RunSpec.InstanceName,
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
WorkingTranscript: req.WorkingTranscript,
Section: req.Section,
Glossary: req.Glossary,
Config: req.Config,
Messages: messages,
PromptMetadata: map[string]any{
"prompt_id": proposalPromptMetadata().PromptID,
"prompt_version": proposalPromptMetadata().PromptVersion,
"prompt_source": proposalPromptMetadata().PromptSource,
"embedded_path": proposalPromptMetadata().EmbeddedPath,
"sha256": proposalPromptMetadata().SHA256,
},
StageName: proposalStageName(req),
StartIndex: 0,
LLMClient: req.LLMClient,
Scheduler: req.LLMScheduler,
DiagnosticsDir: req.DiagnosticsDir,
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
ProposalRequest: req,
PromptID: prompts.PromptIDModuleSpokenWordProposal,
BuildMessages: BuildProposalMessages,
})
if err != nil {
return contracts.ProposalResult{}, err
}
return contracts.ProposalResult{
Proposals: generated.Corrections,
Warnings: generated.Warnings,
}, nil
}
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
if section == nil || transcript == nil {
return transcript
}
segments := make([]schema.Segment, 0, len(transcript.Segments))
for _, seg := range transcript.Segments {
if seg.ID >= section.StartSegmentID && seg.ID <= section.EndSegmentID {
segments = append(segments, seg)
}
}
return &schema.Transcript{Segments: segments}
}
func proposalStageName(req contracts.ProposalRequest) string {
if req.Section == nil || req.Section.Index == 0 {
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName)
}
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index)
}

View File

@@ -10,20 +10,6 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
type promptSegment struct {
ID int `json:"id"`
Speaker string `json:"speaker"`
Start float64 `json:"start"`
End float64 `json:"end"`
Text string `json:"text"`
Categories []string `json:"categories,omitempty"`
}
type promptTranscriptSection struct {
SectionIndex int `json:"section_index"`
Segments []promptSegment `json:"segments"`
}
// BuildProposalMessages constrains corrections to conservative dysfluency
// cleanup with strict semantic preservation.
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
@@ -32,23 +18,7 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
}
sectionPayload := promptTranscriptSection{
SectionIndex: sectionIndex,
Segments: make([]promptSegment, 0),
}
if transcript != nil {
for _, s := range transcript.Segments {
sectionPayload.Segments = append(sectionPayload.Segments, promptSegment{
ID: s.ID,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: append([]string(nil), s.Categories...),
})
}
}
sectionJSON, err := json.MarshalIndent(sectionPayload, "", " ")
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
if err != nil {
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
}
@@ -67,7 +37,3 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
{Role: "user", Content: user},
}, nil
}
func proposalPromptMetadata() prompts.Metadata {
return prompts.MustLookupMetadata(prompts.PromptIDModuleSpokenWordProposal)
}

View File

@@ -40,6 +40,16 @@ type Metadata struct {
SHA256 string `json:"sha256"`
}
func (m Metadata) DiagnosticsMap() map[string]any {
return map[string]any{
"prompt_id": m.PromptID,
"prompt_version": m.PromptVersion,
"prompt_source": m.PromptSource,
"embedded_path": m.EmbeddedPath,
"sha256": m.SHA256,
}
}
type definition struct {
id string
version string

View File

@@ -107,3 +107,16 @@ func TestRenderedPromptsContainHardening(t *testing.T) {
}
}
}
func TestDiagnosticsMapIncludesStablePromptMetadataShapeForAllPrompts(t *testing.T) {
for _, m := range RegisteredMetadata() {
metadataMap := m.DiagnosticsMap()
if metadataMap["prompt_id"] != m.PromptID ||
metadataMap["prompt_version"] != m.PromptVersion ||
metadataMap["prompt_source"] != m.PromptSource ||
metadataMap["embedded_path"] != m.EmbeddedPath ||
metadataMap["sha256"] != m.SHA256 {
t.Fatalf("unexpected diagnostics metadata map for %q: %+v", m.PromptID, metadataMap)
}
}
}

View File

@@ -0,0 +1,57 @@
package testsupport
import (
"os"
"path/filepath"
"strings"
"testing"
)
func ReadFile(t *testing.T, path string) []byte {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("failed to read file %q: %v", path, err)
}
return data
}
func OnlyRunDir(t *testing.T, workDir string) string {
t.Helper()
entries, err := os.ReadDir(workDir)
if err != nil {
t.Fatalf("failed to read work dir %q: %v", workDir, err)
}
dirs := make([]string, 0, len(entries))
for _, e := range entries {
if e.IsDir() {
dirs = append(dirs, filepath.Join(workDir, e.Name()))
}
}
if len(dirs) != 1 {
t.Fatalf("expected exactly one run dir in %q, found %d", workDir, len(dirs))
}
return dirs[0]
}
func AssertNoSecretInFile(t *testing.T, path, secret string) {
t.Helper()
raw := string(ReadFile(t, path))
if strings.Contains(raw, secret) {
t.Fatalf("secret leaked in %s", path)
}
}
func AssertNoSecretInTree(t *testing.T, root, secret string) {
t.Helper()
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil || d == nil || d.IsDir() {
return nil
}
raw, readErr := os.ReadFile(path)
if readErr == nil && strings.Contains(string(raw), secret) {
t.Fatalf("secret leaked in %s", path)
}
return nil
})
}

View File

@@ -2,13 +2,15 @@ package validators
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/validators/protected_terms"
)
var builtInChains = map[string][]string{
"glossary": {
modulecatalog.KeyGlossary: {
KeyProposalShape,
KeyNoEffect,
KeyOriginalTextPresence,
@@ -18,7 +20,7 @@ var builtInChains = map[string][]string{
KeySpokenFormPlausibility,
KeyMeaningReversalReview,
},
"homophones": {
modulecatalog.KeyHomophones: {
KeyProposalShape,
KeyNoEffect,
KeyOriginalTextPresence,
@@ -28,7 +30,7 @@ var builtInChains = map[string][]string{
KeySpokenFormPlausibility,
KeyMeaningReversalReview,
},
"spoken_word": {
modulecatalog.KeySpokenWord: {
KeyProposalShape,
KeyNoEffect,
KeyOriginalTextPresence,
@@ -38,7 +40,7 @@ var builtInChains = map[string][]string{
KeyEditorialReview,
KeyMeaningReversalReview,
},
"grammar": {
modulecatalog.KeyGrammar: {
KeyProposalShape,
KeyNoEffect,
KeyOriginalTextPresence,
@@ -51,9 +53,10 @@ var builtInChains = map[string][]string{
}
func BuiltInChainKeys(moduleKey string) ([]string, error) {
keys, ok := builtInChains[moduleKey]
key := strings.TrimSpace(moduleKey)
keys, ok := builtInChains[key]
if !ok {
return nil, fmt.Errorf("no built-in validator chain for module %q", moduleKey)
return nil, fmt.Errorf("no built-in validator chain for module %q", key)
}
out := make([]string, len(keys))
copy(out, keys)
@@ -61,6 +64,7 @@ func BuiltInChainKeys(moduleKey string) ([]string, error) {
}
func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]contracts.Validator, error) {
moduleKey = strings.TrimSpace(moduleKey)
keys, err := BuiltInChainKeys(moduleKey)
if err != nil {
return nil, err
@@ -71,7 +75,7 @@ func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]contracts.Vali
out := make([]contracts.Validator, 0, len(keys))
for _, key := range keys {
if moduleKey == "glossary" && key == KeyProtectedTerms {
if moduleKey == modulecatalog.KeyGlossary && key == KeyProtectedTerms {
// Glossary stages preserve current stricter protection semantics while
// reporting the stable protected_terms key.
v, buildErr := protected_terms.NewGlossaryStage()

View File

@@ -1,6 +1,10 @@
package metadata
import "gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
import (
"strings"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
)
type ExecutionClass string
@@ -9,6 +13,31 @@ const (
ExecutionClassLLMBacked ExecutionClass = "llm_backed"
)
const (
KeyProposalShape = "proposal_shape"
KeyConfidenceThreshold = "confidence_threshold"
KeyOriginalTextPresence = "original_text_presence"
KeyNonEmptyCorrectedText = "non_empty_corrected_text"
KeyNoEffect = "no_effect"
KeyProtectedTerms = "protected_terms"
KeySpokenFormPlausibility = "spoken_form_plausibility"
KeyMeaningReversalReview = "meaning_reversal_review"
KeyEditorialReview = "editorial_review"
)
var executionClassByKey = map[string]ExecutionClass{
KeyProposalShape: ExecutionClassDeterministic,
KeyConfidenceThreshold: ExecutionClassDeterministic,
KeyOriginalTextPresence: ExecutionClassDeterministic,
KeyNonEmptyCorrectedText: ExecutionClassDeterministic,
KeyNoEffect: ExecutionClassDeterministic,
KeyProtectedTerms: ExecutionClassDeterministic,
KeySpokenFormPlausibility: ExecutionClassLLMBacked,
KeyMeaningReversalReview: ExecutionClassLLMBacked,
KeyEditorialReview: ExecutionClassLLMBacked,
}
type ClassifiedValidator interface {
contracts.Validator
ExecutionClass() ExecutionClass
@@ -18,9 +47,11 @@ func ClassOf(v contracts.Validator) ExecutionClass {
if v == nil {
return ExecutionClassDeterministic
}
classFromKey := ClassForKey(v.Name())
classified, ok := v.(ClassifiedValidator)
if !ok {
return ExecutionClassDeterministic
return classFromKey
}
switch classified.ExecutionClass() {
case ExecutionClassLLMBacked:
@@ -28,8 +59,16 @@ func ClassOf(v contracts.Validator) ExecutionClass {
case ExecutionClassDeterministic:
return ExecutionClassDeterministic
default:
return classFromKey
}
}
func ClassForKey(key string) ExecutionClass {
class, ok := executionClassByKey[strings.TrimSpace(key)]
if !ok {
return ExecutionClassDeterministic
}
return class
}
func Wrap(v contracts.Validator, class ExecutionClass) contracts.Validator {

View File

@@ -16,6 +16,16 @@ func (u unclassifiedValidator) Validate(_ context.Context, _ contracts.Validatio
return frameworkvalidators.Result{ValidatorName: u.Name(), Decisions: nil}, nil
}
type namedUnclassifiedValidator struct {
name string
}
func (n namedUnclassifiedValidator) Name() string { return n.name }
func (n namedUnclassifiedValidator) Validate(_ context.Context, _ contracts.ValidationRequest) (frameworkvalidators.Result, error) {
return frameworkvalidators.Result{ValidatorName: n.Name(), Decisions: nil}, nil
}
func TestClassOfDefaultsToDeterministic(t *testing.T) {
if got := ClassOf(unclassifiedValidator{}); got != ExecutionClassDeterministic {
t.Fatalf("expected deterministic default class, got %q", got)
@@ -28,3 +38,22 @@ func TestWrapExposesExecutionClass(t *testing.T) {
t.Fatalf("expected llm_backed class, got %q", got)
}
}
func TestClassForKey(t *testing.T) {
if got := ClassForKey(KeyProposalShape); got != ExecutionClassDeterministic {
t.Fatalf("expected deterministic class for %q, got %q", KeyProposalShape, got)
}
if got := ClassForKey(KeySpokenFormPlausibility); got != ExecutionClassLLMBacked {
t.Fatalf("expected llm_backed class for %q, got %q", KeySpokenFormPlausibility, got)
}
if got := ClassForKey("unknown"); got != ExecutionClassDeterministic {
t.Fatalf("expected deterministic fallback for unknown key, got %q", got)
}
}
func TestClassOfFallsBackToStableValidatorKey(t *testing.T) {
v := namedUnclassifiedValidator{name: KeyEditorialReview}
if got := ClassOf(v); got != ExecutionClassLLMBacked {
t.Fatalf("expected llm_backed fallback by key, got %q", got)
}
}

View File

@@ -8,6 +8,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/validators/confidence_threshold"
"gitea.maximumdirect.net/eric/audita/internal/validators/editorial_review"
"gitea.maximumdirect.net/eric/audita/internal/validators/meaning_reversal_review"
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
"gitea.maximumdirect.net/eric/audita/internal/validators/no_effect"
"gitea.maximumdirect.net/eric/audita/internal/validators/non_empty_corrected_text"
"gitea.maximumdirect.net/eric/audita/internal/validators/original_text_presence"
@@ -17,22 +18,21 @@ import (
)
const (
KeyProposalShape = "proposal_shape"
KeyConfidenceThreshold = "confidence_threshold"
KeyOriginalTextPresence = "original_text_presence"
KeyNonEmptyCorrectedText = "non_empty_corrected_text"
KeyNoEffect = "no_effect"
KeyProtectedTerms = "protected_terms"
KeyProposalShape = validatormetadata.KeyProposalShape
KeyConfidenceThreshold = validatormetadata.KeyConfidenceThreshold
KeyOriginalTextPresence = validatormetadata.KeyOriginalTextPresence
KeyNonEmptyCorrectedText = validatormetadata.KeyNonEmptyCorrectedText
KeyNoEffect = validatormetadata.KeyNoEffect
KeyProtectedTerms = validatormetadata.KeyProtectedTerms
KeySpokenFormPlausibility = "spoken_form_plausibility"
KeyMeaningReversalReview = "meaning_reversal_review"
KeyEditorialReview = "editorial_review"
KeySpokenFormPlausibility = validatormetadata.KeySpokenFormPlausibility
KeyMeaningReversalReview = validatormetadata.KeyMeaningReversalReview
KeyEditorialReview = validatormetadata.KeyEditorialReview
)
type BuiltInValidatorDefinition struct {
Key string
Build func() (contracts.Validator, error)
LLMBacked bool
}
type Registry struct {
@@ -47,9 +47,9 @@ func NewBuiltInRegistry() *Registry {
{Key: KeyNonEmptyCorrectedText, Build: non_empty_corrected_text.New},
{Key: KeyNoEffect, Build: no_effect.New},
{Key: KeyProtectedTerms, Build: protected_terms.New},
{Key: KeySpokenFormPlausibility, LLMBacked: true, Build: spoken_form_plausibility.New},
{Key: KeyMeaningReversalReview, LLMBacked: true, Build: meaning_reversal_review.New},
{Key: KeyEditorialReview, LLMBacked: true, Build: editorial_review.New},
{Key: KeySpokenFormPlausibility, Build: spoken_form_plausibility.New},
{Key: KeyMeaningReversalReview, Build: meaning_reversal_review.New},
{Key: KeyEditorialReview, Build: editorial_review.New},
}
m := make(map[string]BuiltInValidatorDefinition, len(defs))

View File

@@ -4,6 +4,7 @@ import (
"context"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
@@ -81,11 +82,6 @@ func TestBuiltInValidatorPackagesConstruct(t *testing.T) {
func TestRegistryBuildsClassifiedValidators(t *testing.T) {
r := NewBuiltInRegistry()
llmKeys := map[string]bool{
KeySpokenFormPlausibility: true,
KeyMeaningReversalReview: true,
KeyEditorialReview: true,
}
for _, key := range r.RegisteredKeys() {
v, err := r.MustBuild(key)
if err != nil {
@@ -94,16 +90,28 @@ func TestRegistryBuildsClassifiedValidators(t *testing.T) {
if _, ok := v.(validatormetadata.ClassifiedValidator); !ok {
t.Fatalf("expected built validator %q to expose execution classification metadata", key)
}
want := validatormetadata.ExecutionClassDeterministic
if llmKeys[key] {
want = validatormetadata.ExecutionClassLLMBacked
}
want := validatormetadata.ClassForKey(key)
if got := validatormetadata.ClassOf(v); got != want {
t.Fatalf("expected class %q for %q, got %q", want, key, got)
}
}
}
func TestExecutionClassResolvableByStableKeyAndByValidatorInstance(t *testing.T) {
r := NewBuiltInRegistry()
for _, key := range r.RegisteredKeys() {
v, err := r.MustBuild(key)
if err != nil {
t.Fatalf("must build %q: %v", key, err)
}
fromKey := validatormetadata.ClassForKey(key)
fromInstance := validatormetadata.ClassOf(v)
if fromInstance != fromKey {
t.Fatalf("class mismatch for %q: key=%q instance=%q", key, fromKey, fromInstance)
}
}
}
func TestRegistryProtectedTermsUsesNonGlossaryStageBehavior(t *testing.T) {
r := NewBuiltInRegistry()
v, err := r.MustBuild(KeyProtectedTerms)
@@ -200,7 +208,7 @@ func TestBuiltInRegistryUnknownKeyFails(t *testing.T) {
}
func TestBuiltInChainKeysResolveForProductionModules(t *testing.T) {
for _, moduleKey := range []string{"glossary", "homophones", "spoken_word", "grammar"} {
for _, moduleKey := range modulecatalog.SupportedKeys() {
keys, err := BuiltInChainKeys(moduleKey)
if err != nil {
t.Fatalf("resolve keys for %q: %v", moduleKey, err)
@@ -213,7 +221,7 @@ func TestBuiltInChainKeysResolveForProductionModules(t *testing.T) {
func TestResolveBuiltInChainUsesRegisteredKeys(t *testing.T) {
r := NewBuiltInRegistry()
for _, moduleKey := range []string{"glossary", "homophones", "spoken_word", "grammar"} {
for _, moduleKey := range modulecatalog.SupportedKeys() {
chain, err := ResolveBuiltInChain(moduleKey, r)
if err != nil {
t.Fatalf("resolve chain for %q: %v", moduleKey, err)