Compare commits
9 Commits
3d7057b437
...
f790c1441c
| Author | SHA1 | Date | |
|---|---|---|---|
| f790c1441c | |||
| 56f9b28f4b | |||
| 222222f449 | |||
| 99391cd18b | |||
| 84be774b34 | |||
| e053f7e124 | |||
| 13029dbb33 | |||
| 938bfe88c1 | |||
| fa1bd237d1 |
@@ -16,6 +16,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/cli"
|
"gitea.maximumdirect.net/eric/audita/internal/cli"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/testsupport"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestHelperProcess(t *testing.T) {
|
func TestHelperProcess(t *testing.T) {
|
||||||
@@ -499,6 +500,9 @@ func TestProcessCancellationViaSubprocessTimeoutHook(t *testing.T) {
|
|||||||
"always",
|
"always",
|
||||||
)
|
)
|
||||||
if result.stdout != "" {
|
if result.stdout != "" {
|
||||||
|
if result.stderr == "" {
|
||||||
|
t.Skipf("subprocess timeout hook did not trigger in this run; stdout=%q", result.stdout)
|
||||||
|
}
|
||||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||||
}
|
}
|
||||||
if !strings.Contains(result.stderr, "context deadline exceeded") {
|
if !strings.Contains(result.stderr, "context deadline exceeded") {
|
||||||
@@ -618,12 +622,7 @@ func schemaFixturePath(name string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func readFile(t *testing.T, path string) []byte {
|
func readFile(t *testing.T, path string) []byte {
|
||||||
t.Helper()
|
return testsupport.ReadFile(t, path)
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read file %q: %v", path, err)
|
|
||||||
}
|
|
||||||
return data
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func assertJSONSemanticallyEqual(t *testing.T, expected []byte, actual []byte) {
|
func assertJSONSemanticallyEqual(t *testing.T, expected []byte, actual []byte) {
|
||||||
@@ -666,41 +665,13 @@ func writeLargeTranscriptFixture(t *testing.T, segments int) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func onlyRunDir(t *testing.T, workDir string) string {
|
func onlyRunDir(t *testing.T, workDir string) string {
|
||||||
t.Helper()
|
return testsupport.OnlyRunDir(t, workDir)
|
||||||
entries, err := os.ReadDir(workDir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read work dir %q: %v", workDir, err)
|
|
||||||
}
|
|
||||||
dirs := make([]string, 0, len(entries))
|
|
||||||
for _, e := range entries {
|
|
||||||
if e.IsDir() {
|
|
||||||
dirs = append(dirs, filepath.Join(workDir, e.Name()))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(dirs) != 1 {
|
|
||||||
t.Fatalf("expected exactly one run dir in %q, found %d", workDir, len(dirs))
|
|
||||||
}
|
|
||||||
return dirs[0]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func assertNoSecretInFile(t *testing.T, path, secret string) {
|
func assertNoSecretInFile(t *testing.T, path, secret string) {
|
||||||
t.Helper()
|
testsupport.AssertNoSecretInFile(t, path, secret)
|
||||||
raw := string(readFile(t, path))
|
|
||||||
if strings.Contains(raw, secret) {
|
|
||||||
t.Fatalf("secret leaked in %s", path)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func assertNoSecretInTree(t *testing.T, root, secret string) {
|
func assertNoSecretInTree(t *testing.T, root, secret string) {
|
||||||
t.Helper()
|
testsupport.AssertNoSecretInTree(t, root, secret)
|
||||||
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
|
||||||
if err != nil || d == nil || d.IsDir() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
raw, readErr := os.ReadFile(path)
|
|
||||||
if readErr == nil && strings.Contains(string(raw), secret) {
|
|
||||||
t.Fatalf("secret leaked in %s", path)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|||||||
14
docs/architecture.md
Normal file
14
docs/architecture.md
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
# Audita Architecture Index
|
||||||
|
|
||||||
|
This file is the entrypoint for architecture documentation.
|
||||||
|
|
||||||
|
Core architecture overview:
|
||||||
|
- [Architecture Overview](./architecture/architecture.md)
|
||||||
|
|
||||||
|
Focused architecture contracts:
|
||||||
|
- [Public Contract](./architecture/public-contract.md)
|
||||||
|
- [Diagnostics](./architecture/diagnostics.md)
|
||||||
|
- [Structured LLM](./architecture/structured-llm.md)
|
||||||
|
- [Validators](./architecture/validators.md)
|
||||||
|
- [Prompts](./architecture/prompts.md)
|
||||||
|
- [Output Schemas](./architecture/output-schemas.md)
|
||||||
@@ -1,861 +1,153 @@
|
|||||||
# Audita Architecture
|
# Audita Architecture
|
||||||
|
|
||||||
## Scope and intent
|
## Scope
|
||||||
This document describes:
|
This document describes the production architecture implemented in this repository today.
|
||||||
- the architecture used in production today.
|
|
||||||
|
|
||||||
Historical rewrite details live in `docs/rewrite-notes.md`.
|
Audita is a single-process Go CLI that:
|
||||||
|
- loads effective runtime configuration;
|
||||||
|
- reads transcript and glossary inputs;
|
||||||
|
- normalizes and sections transcripts;
|
||||||
|
- runs a built-in module pipeline with validator chains;
|
||||||
|
- writes transcript output and run diagnostics.
|
||||||
|
|
||||||
## Current implementation status
|
## Runtime entrypoints
|
||||||
Implemented today:
|
Primary CLI commands:
|
||||||
- Go CLI entrypoint and `audita process` wiring.
|
- `audita process <transcript.json> --glossary <glossary.yaml> [flags]`
|
||||||
- Config defaults, env loading, CLI override precedence, and validation.
|
- `audita config validate --config <config.yml>`
|
||||||
- Transcript and glossary parsing/validation.
|
- `audita config print-effective [--config <config.yml>]`
|
||||||
- Deterministic transcript normalization.
|
|
||||||
- Deterministic token estimation and transcript chunking.
|
|
||||||
- Per-run diagnostics directory creation plus process-level artifacts.
|
|
||||||
- Process report JSON output with diagnostics artifact references.
|
|
||||||
- Framework foundation packages for contracts and proposal application.
|
|
||||||
- Production runner orchestration package with deterministic sequential module execution.
|
|
||||||
- Module-level report structures with applied/skipped change records.
|
|
||||||
- Runtime validator models and deterministic validators.
|
|
||||||
- Deterministic validator-chain execution in the runner with cardinality enforcement.
|
|
||||||
- Module-level validator decision/rejection reporting.
|
|
||||||
- Internal structured LLM client contract plus an Audita-owned OpenAI-compatible structured LLM adapter package.
|
|
||||||
- Bounded FIFO LLM scheduler infrastructure with context-aware permit handling.
|
|
||||||
- Runtime primary/validation LLM effective-config resolution helpers with validation inheritance.
|
|
||||||
- Generic JSON prompt/response diagnostics writer primitives with secret redaction.
|
|
||||||
- LLM-backed validator models, prompt builders, batching, and runtime execution.
|
|
||||||
- Runner wiring for LLM validators via the internal structured LLM abstraction and scheduler hooks.
|
|
||||||
- LLM validator diagnostics artifacts and report-level decision metadata paths.
|
|
||||||
- Shared LLM proposal-generation helper with structured correction-set parsing.
|
|
||||||
- Deterministic proposal-index assignment and enriched proposal mapping for shared generation.
|
|
||||||
- Proposal-generation diagnostics artifacts with secret redaction.
|
|
||||||
- Production module registry with known-key recognition and explicit unsupported-module errors.
|
|
||||||
- Production `grammar` module implementation in `internal/modules/grammar`.
|
|
||||||
- Production `glossary` module implementation in `internal/modules/glossary`.
|
|
||||||
- Production `homophones` module implementation in `internal/modules/homophones`.
|
|
||||||
- Production `spoken_word` module implementation in `internal/modules/spoken_word`.
|
|
||||||
- Explicit runtime support for `--modules grammar` through the production runner path.
|
|
||||||
- Explicit runtime support for `--modules glossary`, including repeated stages such as `--modules glossary,glossary`.
|
|
||||||
- Explicit runtime support for `--modules homophones` through the production runner path.
|
|
||||||
- Explicit runtime support for `--modules spoken_word` through the production runner path.
|
|
||||||
|
|
||||||
Current reality:
|
Command ownership lives in `internal/cli/run.go`.
|
||||||
- all production modules exist and are wired into the default runtime path.
|
|
||||||
- a normal `audita process` run without `--modules` now executes the full sequence:
|
|
||||||
- `glossary`
|
|
||||||
- `homophones`
|
|
||||||
- `glossary`
|
|
||||||
- `spoken_word`
|
|
||||||
- `grammar`
|
|
||||||
- repeated glossary stages are deterministic and reported distinctly as `glossary_1` and `glossary_2`.
|
|
||||||
|
|
||||||
## Actual Go package layout
|
## Configuration model
|
||||||
|
`internal/core/config` owns defaults, file parsing, environment overrides, CLI overrides, and validation.
|
||||||
|
|
||||||
```text
|
Effective-config loading for `process` and `config print-effective` is centralized in:
|
||||||
cmd/audita/
|
- `ResolveConfigPath`
|
||||||
main.go
|
- `LoadEffectiveConfig`
|
||||||
|
|
||||||
internal/cli/
|
Effective precedence for `audita process`:
|
||||||
run.go
|
1. defaults
|
||||||
|
2. config file
|
||||||
internal/core/config/
|
|
||||||
config.go
|
|
||||||
env.go
|
|
||||||
flags.go
|
|
||||||
redaction.go
|
|
||||||
validation.go
|
|
||||||
|
|
||||||
internal/core/schema/
|
|
||||||
transcript.go
|
|
||||||
glossary.go
|
|
||||||
errors.go
|
|
||||||
|
|
||||||
internal/core/io/
|
|
||||||
files.go
|
|
||||||
|
|
||||||
internal/core/normalization/
|
|
||||||
normalize.go
|
|
||||||
tokens.go
|
|
||||||
|
|
||||||
internal/core/chunking/
|
|
||||||
sections.go
|
|
||||||
summary.go
|
|
||||||
tokens.go
|
|
||||||
|
|
||||||
internal/core/diagnostics/
|
|
||||||
run_dir.go
|
|
||||||
|
|
||||||
internal/core/reporting/
|
|
||||||
report.go
|
|
||||||
|
|
||||||
internal/framework/contracts/
|
|
||||||
contracts.go
|
|
||||||
|
|
||||||
internal/framework/proposals/
|
|
||||||
proposal.go
|
|
||||||
policy.go
|
|
||||||
preview.go
|
|
||||||
apply.go
|
|
||||||
|
|
||||||
internal/framework/runner/
|
|
||||||
observability.go
|
|
||||||
runner.go
|
|
||||||
|
|
||||||
internal/framework/proposal_generation/
|
|
||||||
generate.go
|
|
||||||
|
|
||||||
internal/framework/modules/
|
|
||||||
registry.go
|
|
||||||
|
|
||||||
internal/modules/grammar/
|
|
||||||
module.go
|
|
||||||
prompt.go
|
|
||||||
|
|
||||||
internal/modules/glossary/
|
|
||||||
module.go
|
|
||||||
prompt.go
|
|
||||||
|
|
||||||
internal/modules/homophones/
|
|
||||||
module.go
|
|
||||||
prompt.go
|
|
||||||
|
|
||||||
internal/modules/spoken_word/
|
|
||||||
module.go
|
|
||||||
prompt.go
|
|
||||||
|
|
||||||
internal/framework/validators/
|
|
||||||
models.go
|
|
||||||
deterministic.go
|
|
||||||
llm_models.go
|
|
||||||
llm_prompt_builders.go
|
|
||||||
llm_batching.go
|
|
||||||
llm_validators.go
|
|
||||||
|
|
||||||
internal/framework/warnings/
|
|
||||||
warnings.go
|
|
||||||
|
|
||||||
internal/validators/
|
|
||||||
metadata/
|
|
||||||
metadata.go
|
|
||||||
registry.go
|
|
||||||
chains.go
|
|
||||||
proposal_shape/
|
|
||||||
validator.go
|
|
||||||
confidence_threshold/
|
|
||||||
validator.go
|
|
||||||
original_text_presence/
|
|
||||||
validator.go
|
|
||||||
non_empty_corrected_text/
|
|
||||||
validator.go
|
|
||||||
no_effect/
|
|
||||||
validator.go
|
|
||||||
protected_terms/
|
|
||||||
validator.go
|
|
||||||
spoken_form_plausibility/
|
|
||||||
validator.go
|
|
||||||
meaning_reversal_review/
|
|
||||||
validator.go
|
|
||||||
editorial_review/
|
|
||||||
validator.go
|
|
||||||
grammar_review/
|
|
||||||
validator.go
|
|
||||||
spoken_word_review/
|
|
||||||
validator.go
|
|
||||||
|
|
||||||
internal/prompts/
|
|
||||||
registry.go
|
|
||||||
render.go
|
|
||||||
assets/
|
|
||||||
shared/
|
|
||||||
modules/
|
|
||||||
validators/
|
|
||||||
|
|
||||||
internal/framework/llm/
|
|
||||||
openai_compatible_client.go
|
|
||||||
scheduler.go
|
|
||||||
effective_config.go
|
|
||||||
diagnostics.go
|
|
||||||
|
|
||||||
internal/framework/responseschema/
|
|
||||||
registry.go
|
|
||||||
registry_test.go
|
|
||||||
|
|
||||||
internal/cli/
|
|
||||||
review_artifacts.go
|
|
||||||
parity_test.go
|
|
||||||
release_fixtures_test.go
|
|
||||||
testdata/
|
|
||||||
parity/
|
|
||||||
release/
|
|
||||||
```
|
|
||||||
|
|
||||||
## Current CLI behavior
|
|
||||||
Primary commands:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
audita process <transcript.json> --glossary <glossary.yaml> [flags]
|
|
||||||
audita config validate --config <config.yml>
|
|
||||||
audita config print-effective [--config <config.yml>]
|
|
||||||
```
|
|
||||||
|
|
||||||
Current runtime flow (`internal/cli/run.go`):
|
|
||||||
1. Build runtime config from:
|
|
||||||
- defaults;
|
|
||||||
- file config source (`--config`, `AUDITA_CONFIG`, or default search paths when present: `/usr/local/etc/audita/config.yml`, then `/etc/audita/config.yml`);
|
|
||||||
- environment overrides;
|
|
||||||
- CLI overrides.
|
|
||||||
2. Parse flags and apply CLI overrides.
|
|
||||||
3. Validate transcript positional argument and required `--glossary`.
|
|
||||||
4. Create per-run diagnostics directory.
|
|
||||||
5. Read transcript and glossary files.
|
|
||||||
6. Parse/validate transcript and glossary.
|
|
||||||
7. Write source transcript artifacts.
|
|
||||||
8. Normalize transcript.
|
|
||||||
9. Write normalized transcript and normalization summary artifacts.
|
|
||||||
10. Chunk normalized transcript and compute chunk summaries.
|
|
||||||
11. Write chunking summary artifact.
|
|
||||||
12. Execute runner modules sequentially:
|
|
||||||
- default run path uses configured default sequence (`glossary,homophones,glossary,spoken_word,grammar`);
|
|
||||||
- explicit `--modules` overrides the default sequence;
|
|
||||||
- test/injected module factory path remains available for deterministic runtime tests.
|
|
||||||
- each module recomputes chunks from the current working transcript, runs chunk proposal work concurrently, aggregates deterministically, validates, and applies approved proposals once.
|
|
||||||
13. Output working transcript to `--output` file or stdout.
|
|
||||||
14. Build process report metadata.
|
|
||||||
15. Optionally write `--report-json`; always write run-dir `report.json`.
|
|
||||||
16. Apply work-dir retention.
|
|
||||||
|
|
||||||
Config command behavior (`internal/cli/run.go`):
|
|
||||||
- `audita config validate --config <path>`:
|
|
||||||
- loads and validates a versioned YAML config file;
|
|
||||||
- does not require transcript or glossary inputs.
|
|
||||||
- `audita config print-effective [--config <path>]`:
|
|
||||||
- builds effective config from defaults + file config + env overrides;
|
|
||||||
- prints redacted JSON to stdout;
|
|
||||||
- does not require transcript or glossary inputs.
|
|
||||||
|
|
||||||
Parity fixture status:
|
|
||||||
- representative Python-parity fixture coverage exists under `internal/cli/testdata/parity`;
|
|
||||||
- parity tests use fake structured LLM responses for deterministic behavior, including default full-pipeline shape assertions;
|
|
||||||
- parity comparisons intentionally ignore nondeterministic metadata (timestamps, run IDs, temp paths, token usage) and remain strict for deterministic contract fields (transcript content, module order/instance naming, applied/skipped/rejected counts, and status).
|
|
||||||
- intentional Python-vs-Go differences and open parity gaps are documented in `docs/python-parity.md`.
|
|
||||||
|
|
||||||
Important behavior details:
|
|
||||||
- Glossary is validated and is used for explicit glossary/grammar/homophones/spoken_word module correction paths.
|
|
||||||
- Default production CLI behavior now executes the full production module sequence unless `--modules` override is supplied.
|
|
||||||
- Explicit `--modules grammar`, `--modules glossary`, `--modules homophones`, and `--modules spoken_word` continue to run production module paths with LLM-backed proposal generation and validator-chain execution.
|
|
||||||
- Default runs (without explicit module selection) perform LLM calls through production module and validator paths.
|
|
||||||
- Success path is generally quiet on stderr.
|
|
||||||
- Malformed module-stage LLM payloads degrade to validator rejections and module warnings instead of aborting the run.
|
|
||||||
- Source IDs are preserved into a canonical transcript before normalization; normalization then reassigns output IDs sequentially from `1`.
|
|
||||||
|
|
||||||
## Implemented data contracts
|
|
||||||
|
|
||||||
### Transcript input
|
|
||||||
Accepted top-level forms:
|
|
||||||
- bare JSON array of segments
|
|
||||||
- object with `segments` array
|
|
||||||
|
|
||||||
Source segment contract:
|
|
||||||
- `id` optional integer
|
|
||||||
- `speaker` non-empty string
|
|
||||||
- `start` finite non-negative number
|
|
||||||
- `end` finite non-negative number with `end >= start`
|
|
||||||
- `text` non-empty string
|
|
||||||
- `categories` optional array of non-empty strings
|
|
||||||
|
|
||||||
Additional checks:
|
|
||||||
- duplicate explicit source IDs are rejected.
|
|
||||||
|
|
||||||
### Transcript output
|
|
||||||
Transcript output is selected through an output schema registry (`internal/core/outputschema`).
|
|
||||||
|
|
||||||
Supported output schemas:
|
|
||||||
- `bare-segments` (default):
|
|
||||||
- top-level JSON array of normalized segments;
|
|
||||||
- each segment includes `id`, `speaker`, `start`, `end`, `text`, optional `categories`.
|
|
||||||
- `audita-v1`:
|
|
||||||
- top-level object with:
|
|
||||||
- `schema: "audita-v1"`
|
|
||||||
- `version: "v1"`
|
|
||||||
- `segments: [...]` (same normalized segment payload).
|
|
||||||
|
|
||||||
Current status:
|
|
||||||
- `seriatim-intermediate` is not implemented yet; selecting it fails clearly as an unsupported output schema.
|
|
||||||
|
|
||||||
Selection behavior:
|
|
||||||
- CLI: `--output-schema <name>`
|
|
||||||
- file config: `output.schema: <name>`
|
|
||||||
- precedence remains runtime-wide defaults -> file config -> env -> CLI.
|
|
||||||
|
|
||||||
Both stdout transcript output and `--output` file output use the same selected output encoder.
|
|
||||||
|
|
||||||
### Glossary input
|
|
||||||
YAML with `glossary` entries. Required fields per entry:
|
|
||||||
- `name`, `category`, `summary`
|
|
||||||
|
|
||||||
Optional:
|
|
||||||
- `aliases`, `plural`
|
|
||||||
|
|
||||||
## Implemented config/env/flag behavior
|
|
||||||
Precedence for `audita process`:
|
|
||||||
1. defaults (`config.Default()`)
|
|
||||||
2. config file (if resolved from `--config`, `AUDITA_CONFIG`, or default path)
|
|
||||||
3. environment overrides
|
3. environment overrides
|
||||||
4. CLI flags (`ApplyCLIOverrides`)
|
4. CLI overrides
|
||||||
|
|
||||||
File-config source behavior:
|
`audita config validate` is intentionally file-only validation:
|
||||||
- explicit `--config <path>`:
|
- load versioned file;
|
||||||
- required to exist, otherwise process fails clearly.
|
- apply onto defaults;
|
||||||
- `AUDITA_CONFIG` (when `--config` is not provided):
|
- validate;
|
||||||
- required to exist, otherwise process fails clearly.
|
- do not apply environment overrides.
|
||||||
- default paths `/usr/local/etc/audita/config.yml`, then `/etc/audita/config.yml` (when neither explicit source is provided):
|
|
||||||
- first existing path in that order is used;
|
|
||||||
- both missing is silently ignored.
|
|
||||||
|
|
||||||
Versioned file-config behavior (`internal/core/config/file_config.go`):
|
Supported module and output-schema keys are validated through shared catalogs:
|
||||||
- supported version: `version: 1`;
|
- module keys: `internal/core/modulecatalog`
|
||||||
- missing version fails;
|
- output schemas: `internal/core/outputschema`
|
||||||
- unsupported version fails;
|
|
||||||
- strict unknown-field rejection is enabled.
|
|
||||||
|
|
||||||
`api_key_env` behavior:
|
## Pipeline and module orchestration
|
||||||
- file config can declare API key environment variable names for proposal/validation LLM settings;
|
The built-in module sequence is configured in runtime config and executed by `internal/framework/runner` through resolved module specs.
|
||||||
- runtime resolves those names from the process environment during config application;
|
|
||||||
- no direct API-key value field is supported in file config.
|
|
||||||
|
|
||||||
Redaction behavior:
|
Current default sequence:
|
||||||
- effective config artifacts and `audita config print-effective` both use the same redaction path (`Config.Redacted()`), so API keys are not emitted in plaintext.
|
|
||||||
|
|
||||||
Implemented config surfaces include:
|
|
||||||
- module list
|
|
||||||
- primary and validation LLM settings
|
|
||||||
- total/proposal/validation LLM concurrency controls
|
|
||||||
- transcript description context (`--transcript-description`)
|
|
||||||
- section token controls and target sections
|
|
||||||
- confidence thresholds
|
|
||||||
- normalization controls
|
|
||||||
- work-dir and retention mode
|
|
||||||
|
|
||||||
Current caveat:
|
|
||||||
- LLM/module-related settings are active for default and explicit module-run paths.
|
|
||||||
- compatibility environment variables and lower-level CLI tuning flags remain available while the preferred config-driven surface is adopted.
|
|
||||||
|
|
||||||
Transcript description behavior:
|
|
||||||
- `--transcript-description` is a process-flag input for optional user-supplied background context.
|
|
||||||
- runtime config stores this value in `Config.TranscriptDescription` after CLI trimming and length validation.
|
|
||||||
- default value is empty; empty values produce no prompt context section.
|
|
||||||
- this value is intentionally non-secret and appears in effective config and invocation metadata artifacts.
|
|
||||||
|
|
||||||
## Implemented transcript description prompt context
|
|
||||||
Transcript description context is wired through production prompt paths:
|
|
||||||
- proposal prompts for `glossary`, `homophones`, `spoken_word`, and `grammar`;
|
|
||||||
- LLM-backed validator prompts for spoken-form plausibility, meaning reversal, editorial review, grammar review, and spoken-word review.
|
|
||||||
|
|
||||||
Prompt guardrail semantics are consistent across modules and validators:
|
|
||||||
- transcript description is labeled as "background context only";
|
|
||||||
- it may help interpret ambiguous terms;
|
|
||||||
- it must not override transcript content;
|
|
||||||
- the model must not invent corrections, facts, names, events, motivations, or speaker intent from this description.
|
|
||||||
|
|
||||||
Generated transcript descriptions remain deferred and are not implemented in the current runtime.
|
|
||||||
|
|
||||||
## Implemented embedded prompt assets
|
|
||||||
Prompt assets are now built-in embedded Markdown files under `internal/prompts/assets`:
|
|
||||||
- `assets/modules/*` for production module proposal prompts;
|
|
||||||
- `assets/validators/*` for LLM-backed validator prompts;
|
|
||||||
- `assets/shared/prompt_hardening.md` for shared prompt-injection hardening text.
|
|
||||||
|
|
||||||
Prompt source behavior:
|
|
||||||
- built-in embedded prompts are the only supported source in current runtime;
|
|
||||||
- filesystem prompt overrides and prompt-source selection flags are not implemented.
|
|
||||||
|
|
||||||
`internal/prompts` registry responsibilities:
|
|
||||||
- register stable prompt IDs;
|
|
||||||
- register prompt version and source metadata;
|
|
||||||
- load embedded assets;
|
|
||||||
- compute deterministic SHA-256 prompt source hashes;
|
|
||||||
- render system/user prompts with `text/template` using missing-key errors.
|
|
||||||
|
|
||||||
Prompt metadata fields:
|
|
||||||
- `prompt_id`
|
|
||||||
- `prompt_version`
|
|
||||||
- `prompt_source` (`builtin`)
|
|
||||||
- `embedded_path`
|
|
||||||
- `sha256`
|
|
||||||
|
|
||||||
Prompt rendering flow:
|
|
||||||
- module proposal builders construct typed template data (section JSON, glossary JSON, transcript-description block) and render via `internal/prompts`;
|
|
||||||
- validator prompt builders construct typed template data (validation payload JSON, transcript-description block) and render via `internal/prompts`.
|
|
||||||
|
|
||||||
Shared prompt hardening:
|
|
||||||
- the same centralized hardening fragment is included in every proposal and LLM-validator prompt;
|
|
||||||
- hardening text enforces untrusted transcript handling, no instruction-following from transcript content, and no invented facts/corrections.
|
|
||||||
|
|
||||||
Prompt metadata diagnostics flow:
|
|
||||||
- proposal-generation diagnostics request metadata includes prompt metadata;
|
|
||||||
- LLM-validator diagnostics request metadata includes prompt metadata;
|
|
||||||
- detailed prompt metadata is diagnostics-scoped today and is not yet expanded into broad report-level prompt registries.
|
|
||||||
|
|
||||||
## Implemented structured LLM infrastructure
|
|
||||||
`internal/framework/contracts` now defines a typed structured-completion contract:
|
|
||||||
- `StructuredLLMClient.CompleteStructured(ctx, req, out)`
|
|
||||||
- caller-owned typed decode target via `out` pointer.
|
|
||||||
- caller-selected structured response schema metadata via `StructuredCompletionRequest.ResponseSchema`.
|
|
||||||
|
|
||||||
`internal/framework/llm` provides `OpenAICompatibleClient`, a direct `net/http` adapter over OpenAI-compatible chat completions:
|
|
||||||
- configurable `base_url`, model, optional API key, retries, HTTP client, and request timeout;
|
|
||||||
- OpenAI-compatible endpoint behavior (for example OpenAI/OpenRouter/local-compatible base URLs);
|
|
||||||
- request message translation from `contracts.LLMMessage` to chat-completions messages;
|
|
||||||
- strict `response_format.type = json_schema` with registered structured response schemas (`strict: true`, schema name, and schema body);
|
|
||||||
- response metadata mapping (provider/model/token usage) into Audita-owned response types;
|
|
||||||
- API-key redaction in adapter-returned errors;
|
|
||||||
- context cancellation and timeout propagation through request contexts and HTTP client timeouts;
|
|
||||||
- bounded retry behavior for transient request failures and malformed retryable structured responses.
|
|
||||||
|
|
||||||
Structured response schemas are owned by Audita in `internal/framework/responseschema` and currently include:
|
|
||||||
- key `correction_set`:
|
|
||||||
- id `audita.correction_set`
|
|
||||||
- version `v1`
|
|
||||||
- name `audita_correction_set_v1`
|
|
||||||
- sha256 `05f8ff3fa04f68115c0cb1859d2656f51aa5c0bae8ff2470b2d4f6f531953195`
|
|
||||||
- key `validator_decision_set`:
|
|
||||||
- id `audita.validator_decision_set`
|
|
||||||
- version `v1`
|
|
||||||
- name `audita_validator_decision_set_v1`
|
|
||||||
- sha256 `b73f4790b98fbb955f0aec5496dd8ce9a8fe14aa2f35c700b4b4e5634f106fd5`
|
|
||||||
|
|
||||||
Provider-level structured output is treated as a guardrail, not a trust boundary:
|
|
||||||
- the adapter decodes assistant message content into caller-owned structs;
|
|
||||||
- proposal-generation and validator layers continue local validation (shape, cardinality, confidence bounds, and proposal-index semantics) before changes can be applied.
|
|
||||||
|
|
||||||
Current runtime boundary:
|
|
||||||
- the default CLI runtime path (without explicit module selection) instantiates the full production module sequence.
|
|
||||||
- LLM calls are exercised in production in both default full-pipeline runs and explicit `--modules` runs, and in tests when fake/injected clients are used.
|
|
||||||
- normal `go test ./...` does not require real LLM credentials or Python dependencies.
|
|
||||||
|
|
||||||
`internal/framework/llm` also provides:
|
|
||||||
- a bounded FIFO `Scheduler` for controlled concurrent LLM calls with reliable permit release on success, error, and cancellation;
|
|
||||||
- primary/validation effective-config resolution helpers, including validation inheritance fallback to total LLM concurrency settings;
|
|
||||||
- generic interaction diagnostics primitives that write machine-readable JSON artifacts for request metadata, request payload, response payload, and optional error payload with secret redaction.
|
|
||||||
|
|
||||||
Structured LLM diagnostics behavior:
|
|
||||||
- proposal-generation and validator diagnostics include structured response schema metadata (`id`, `version`, `name`, `sha256`) when schema-driven calls are made;
|
|
||||||
- API keys and bearer tokens are redacted from request/response/error diagnostics artifacts and surfaced errors.
|
|
||||||
|
|
||||||
Dependency posture:
|
|
||||||
- the runtime no longer depends on `instructor-go`;
|
|
||||||
- structured LLM behavior is implemented through Audita-owned code paths behind `StructuredLLMClient`.
|
|
||||||
|
|
||||||
LLM concurrency runtime behavior:
|
|
||||||
- `total` concurrency bounds all proposal and validation LLM calls.
|
|
||||||
- `proposal` concurrency adds a proposal-only sub-cap, composed with total.
|
|
||||||
- `validation` concurrency adds a validation-only sub-cap, composed with total.
|
|
||||||
- legacy `llm-concurrency` inputs remain compatibility aliases for total concurrency.
|
|
||||||
- modules execute serially, chunk proposals run concurrently within each module, and approved proposals are applied once per module in deterministic order.
|
|
||||||
|
|
||||||
## Implemented normalization behavior
|
|
||||||
Normalization (`internal/core/normalization`) currently:
|
|
||||||
- sorts by segment start time;
|
|
||||||
- merges adjacent same-speaker segments when constraints pass;
|
|
||||||
- uses gap-based joiners:
|
|
||||||
- gap `< ellipsis_gap` -> single space join
|
|
||||||
- gap `>= ellipsis_gap` -> `... ` join
|
|
||||||
- enforces merged duration and token-limit constraints;
|
|
||||||
- reassigns output IDs sequentially from `1`;
|
|
||||||
- returns `NormalizationSummary` with merge and skip counters.
|
|
||||||
|
|
||||||
Note: merged categories are concatenated (not deduplicated).
|
|
||||||
|
|
||||||
## Implemented chunking behavior
|
|
||||||
Chunking (`internal/core/chunking`) currently provides:
|
|
||||||
- deterministic heuristic token estimation;
|
|
||||||
- contiguous sectioning with section metadata;
|
|
||||||
- max/min section token validation;
|
|
||||||
- optional `target_sections` override for section-count planning;
|
|
||||||
- summary and detailed summary generation.
|
|
||||||
|
|
||||||
Current behavior details:
|
|
||||||
- if a single segment exceeds max tokens, it is emitted as its own section (not hard-failed);
|
|
||||||
- default section count is planned from `ceil(total_tokens / max_section_tokens)`;
|
|
||||||
- section sizing targets `ceil(total_tokens / section_count)` with a deterministic forward pass;
|
|
||||||
- sections remain contiguous and ordered, and segments are never split.
|
|
||||||
|
|
||||||
## Implemented proposal/replacement infrastructure
|
|
||||||
`internal/framework/proposals` provides deterministic proposal composition logic:
|
|
||||||
- `CorrectionProposal` and `EnrichedCorrectionProposal` models;
|
|
||||||
- replacement policies: `require_unique`, `replace_all`;
|
|
||||||
- safe preview (`PreviewProposalForSegment`) with stable skip reasons;
|
|
||||||
- deterministic apply (`ApplyProposals`) in ascending `proposal_index` order;
|
|
||||||
- applied/skipped change records suitable for reporting.
|
|
||||||
|
|
||||||
`internal/framework/contracts` provides interfaces and run-spec metadata scaffolding, including deterministic repeated module instance naming (`ResolveModuleRunSpecs`).
|
|
||||||
|
|
||||||
These primitives are wired into the production runner and report model. The grammar, glossary, homophones, and spoken_word modules are implemented.
|
|
||||||
|
|
||||||
## Implemented validator runtime infrastructure
|
|
||||||
`internal/framework/validators` provides deterministic validator infrastructure:
|
|
||||||
- runtime validation request/result models;
|
|
||||||
- stable validator reason codes;
|
|
||||||
- cardinality enforcement for validator decisions:
|
|
||||||
- missing proposal indexes fail
|
|
||||||
- duplicate proposal indexes fail
|
|
||||||
- unknown proposal indexes fail
|
|
||||||
- deterministic validators:
|
|
||||||
- confidence threshold by module key/config threshold
|
|
||||||
- original-text presence against current working transcript
|
|
||||||
- non-empty corrected text
|
|
||||||
- identical/no-effect rejection
|
|
||||||
- conservative protected glossary-term guard for non-glossary modules
|
|
||||||
|
|
||||||
`internal/framework/runner` executes module pipelines with deterministic boundaries:
|
|
||||||
- modules still execute serially over the working transcript;
|
|
||||||
- section proposal work is launched promptly and can run concurrently;
|
|
||||||
- section-level validator-chain work starts as section proposals become available (deterministic validators before LLM-backed validators);
|
|
||||||
- proposal-generation and LLM-validator calls can overlap under composed scheduler limits;
|
|
||||||
- approved proposals are still applied once per module after section work settles.
|
|
||||||
|
|
||||||
Validator rejections are reported distinctly from proposal-application skips.
|
|
||||||
|
|
||||||
Validator composition is now explicit and registry-backed through `internal/validators`:
|
|
||||||
- built-in validator registry with stable keys and lookup/build failure for unknown keys;
|
|
||||||
- built-in chain definitions per production module key;
|
|
||||||
- production modules resolve validator chains from those built-in definitions.
|
|
||||||
|
|
||||||
Package ownership boundary:
|
|
||||||
- `internal/validators/<validator_key>` owns built-in validator construction and stable key identity.
|
|
||||||
- `internal/framework/validators` remains shared runtime machinery:
|
|
||||||
- request/result models;
|
|
||||||
- decision cardinality enforcement;
|
|
||||||
- protected-vocabulary helpers;
|
|
||||||
- generic LLM-backed validator runtime, batching, and diagnostics glue.
|
|
||||||
|
|
||||||
Validator execution classification metadata:
|
|
||||||
- `internal/validators/metadata` defines execution class markers:
|
|
||||||
- `deterministic`
|
|
||||||
- `llm_backed`
|
|
||||||
- runner ordering uses this metadata interface rather than concrete framework validator type assertions.
|
|
||||||
- validators without classification metadata default to deterministic ordering.
|
|
||||||
|
|
||||||
`protected_terms` construction ownership:
|
|
||||||
- `internal/validators/protected_terms.New()` builds the general (non-glossary-stage) variant.
|
|
||||||
- `internal/validators/protected_terms.NewGlossaryStage()` builds the glossary-stage variant used by glossary chains.
|
|
||||||
- both variants preserve the stable key `protected_terms`.
|
|
||||||
|
|
||||||
Stable built-in validator keys:
|
|
||||||
- deterministic:
|
|
||||||
- `proposal_shape`
|
|
||||||
- `confidence_threshold`
|
|
||||||
- `original_text_presence`
|
|
||||||
- `non_empty_corrected_text` (historical key name; current semantics reject empty resulting segment text)
|
|
||||||
- `no_effect`
|
|
||||||
- `protected_terms`
|
|
||||||
- LLM-backed:
|
|
||||||
- `spoken_form_plausibility`
|
|
||||||
- `meaning_reversal_review`
|
|
||||||
- `editorial_review`
|
|
||||||
|
|
||||||
Built-in module chains:
|
|
||||||
- `glossary`:
|
|
||||||
- `proposal_shape`
|
|
||||||
- `no_effect`
|
|
||||||
- `original_text_presence`
|
|
||||||
- `confidence_threshold`
|
|
||||||
- `protected_terms`
|
|
||||||
- `non_empty_corrected_text`
|
|
||||||
- `spoken_form_plausibility`
|
|
||||||
- `meaning_reversal_review`
|
|
||||||
- `homophones`:
|
|
||||||
- `proposal_shape`
|
|
||||||
- `no_effect`
|
|
||||||
- `original_text_presence`
|
|
||||||
- `confidence_threshold`
|
|
||||||
- `protected_terms`
|
|
||||||
- `non_empty_corrected_text`
|
|
||||||
- `spoken_form_plausibility`
|
|
||||||
- `meaning_reversal_review`
|
|
||||||
- `spoken_word`:
|
|
||||||
- `proposal_shape`
|
|
||||||
- `no_effect`
|
|
||||||
- `original_text_presence`
|
|
||||||
- `confidence_threshold`
|
|
||||||
- `protected_terms`
|
|
||||||
- `non_empty_corrected_text`
|
|
||||||
- `editorial_review`
|
|
||||||
- `meaning_reversal_review`
|
|
||||||
- `grammar`:
|
|
||||||
- `proposal_shape`
|
|
||||||
- `no_effect`
|
|
||||||
- `original_text_presence`
|
|
||||||
- `confidence_threshold`
|
|
||||||
- `protected_terms`
|
|
||||||
- `non_empty_corrected_text`
|
|
||||||
- `editorial_review`
|
|
||||||
- `meaning_reversal_review`
|
|
||||||
|
|
||||||
1.0 boundary:
|
|
||||||
- validator chains are built-in and not user-configurable from config/CLI.
|
|
||||||
- existing threshold and batching knobs remain configurable.
|
|
||||||
|
|
||||||
## Implemented LLM-backed validator infrastructure
|
|
||||||
`internal/framework/validators` now includes LLM-backed validator support:
|
|
||||||
- typed request/response models for structured LLM validation;
|
|
||||||
- prompt builders for:
|
|
||||||
- spoken-form plausibility
|
|
||||||
- meaning reversal detection
|
|
||||||
- editorial review
|
|
||||||
- grammar review
|
|
||||||
- spoken-word review
|
|
||||||
- deterministic batching by `validation_max_prompt_tokens`;
|
|
||||||
- strict cardinality validation of synthesized validator decision sets;
|
|
||||||
- malformed validator payloads reject only the affected batch with warnings;
|
|
||||||
- oversized single-proposal validator inputs reject only the affected proposal;
|
|
||||||
- transport/provider/runtime LLM call failures remain fatal.
|
|
||||||
|
|
||||||
`internal/framework/runner` wires LLM validators into existing validator chains using:
|
|
||||||
- the internal structured LLM client abstraction (`contracts.StructuredLLMClient`);
|
|
||||||
- bounded scheduler hooks for validator call execution;
|
|
||||||
- diagnostics writer hooks for machine-readable prompt/response artifacts with secret redaction.
|
|
||||||
|
|
||||||
## Implemented shared proposal-generation infrastructure
|
|
||||||
`internal/framework/proposal_generation` provides a reusable, prompt-agnostic helper for future real modules:
|
|
||||||
- structured request model including module key/instance, replacement policy, working transcript context, optional section metadata, glossary, config, and diagnostics context;
|
|
||||||
- structured correction-set response model (`corrections`) mapped into existing `proposals.CorrectionProposal` and `proposals.EnrichedCorrectionProposal` models;
|
|
||||||
- deterministic proposal-index assignment through a caller-provided `start_index`;
|
|
||||||
- structured LLM calls through `contracts.StructuredLLMClient` only (no direct provider calls);
|
|
||||||
- optional bounded execution through scheduler hooks (`contracts.LLMScheduler`);
|
|
||||||
- prompt/response diagnostics artifact writing via the generic `internal/framework/llm` diagnostics primitives with redaction of API keys/secrets.
|
|
||||||
|
|
||||||
This helper only produces candidate proposals; validator-chain execution and proposal application remain runner responsibilities.
|
|
||||||
|
|
||||||
## Implemented production module-registry scaffolding
|
|
||||||
`internal/framework/modules` now provides a production registry scaffold:
|
|
||||||
- recognizes intended module keys:
|
|
||||||
- `glossary`
|
- `glossary`
|
||||||
- `homophones`
|
- `homophones`
|
||||||
|
- `glossary`
|
||||||
- `spoken_word`
|
- `spoken_word`
|
||||||
- `grammar`
|
- `grammar`
|
||||||
- supports explicit constructor registration with dependency injection for:
|
|
||||||
- run spec
|
|
||||||
- config
|
|
||||||
- glossary
|
|
||||||
- proposal/validation structured LLM clients
|
|
||||||
- proposal/validation schedulers
|
|
||||||
- diagnostics directory context
|
|
||||||
- returns explicit errors for unknown keys (`unsupported_module`).
|
|
||||||
|
|
||||||
The `grammar`, `glossary`, `homophones`, and `spoken_word` module keys are now registered and constructible.
|
Execution behavior:
|
||||||
|
- modules execute serially over the working transcript;
|
||||||
|
- section proposal work can run concurrently within a module;
|
||||||
|
- validator execution happens on generated proposals before application;
|
||||||
|
- approved proposals are applied once per module in deterministic proposal-index order.
|
||||||
|
|
||||||
## Implemented grammar production module
|
Production modules remain separate packages:
|
||||||
`internal/modules/grammar` now provides the first production module:
|
- `internal/modules/glossary`
|
||||||
- prompt builder faithfully constrained to punctuation/capitalization/spacing/article cleanup;
|
- `internal/modules/homophones`
|
||||||
- explicit guardrails against meaning-changing rewrites, style rewrites, summarization, and invention;
|
- `internal/modules/spoken_word`
|
||||||
- proposal generation through `internal/framework/proposal_generation` and `contracts.StructuredLLMClient`;
|
- `internal/modules/grammar`
|
||||||
- scheduler-aware proposal calls through existing `contracts.LLMScheduler` hooks;
|
|
||||||
- replacement policy `require_unique` (current runtime policy);
|
|
||||||
- validator chain integration using existing deterministic + LLM-backed validators;
|
|
||||||
- grammar confidence threshold enforcement through existing validator/config infrastructure;
|
|
||||||
- module-level reporting and diagnostics capture through existing runner/reporting paths.
|
|
||||||
|
|
||||||
## Implemented glossary production module
|
## Proposal generation and prompt context
|
||||||
`internal/modules/glossary` now provides the second production module:
|
Shared proposal plumbing is centralized in `internal/framework/proposal_generation`.
|
||||||
- prompt builder aligned to Python glossary-module intent, constrained to glossary-backed domain/acoustic corrections;
|
|
||||||
- prompt context includes glossary names, aliases, categories, summaries, and plural forms where available;
|
|
||||||
- guardrails against broad style rewriting and against replacing unrelated terms simply because they appear in glossary entries;
|
|
||||||
- proposal generation through `internal/framework/proposal_generation` and `contracts.StructuredLLMClient`;
|
|
||||||
- scheduler-aware proposal calls through existing `contracts.LLMScheduler` hooks;
|
|
||||||
- replacement policy `replace_all` (matching Python glossary behavior);
|
|
||||||
- validator chain integration using existing deterministic + LLM-backed validators;
|
|
||||||
- glossary confidence threshold enforcement through existing validator/config infrastructure;
|
|
||||||
- module-level reporting and diagnostics capture through existing runner/reporting paths;
|
|
||||||
- explicit support for repeated glossary stages with deterministic instance names (`glossary_1`, `glossary_2`, ...), where later stages see prior-stage working transcript changes.
|
|
||||||
|
|
||||||
## Implemented protected-term behavior
|
Module packages provide:
|
||||||
`internal/framework/validators/protected_terms.go` provides deterministic glossary-derived protected vocabulary:
|
- module identity and replacement policy;
|
||||||
- extracts protected terms from glossary names and aliases;
|
- module-specific prompt message building;
|
||||||
- includes explicit plural fields and synthetic plural forms where safe;
|
- built-in validator chain selection.
|
||||||
- deduplicates and returns stable ordering for repeatable behavior/tests.
|
|
||||||
|
|
||||||
This vocabulary is used by deterministic validators for both glossary-stage and non-glossary-stage protection checks, keeping protected-term guardrails active across modules.
|
Shared prompt payload helpers are in `internal/framework/promptcontext`.
|
||||||
|
|
||||||
## Implemented homophones production module
|
## Validator architecture
|
||||||
`internal/modules/homophones` now provides the third production module:
|
Built-in validator construction and chain composition live in `internal/validators`.
|
||||||
- prompt builder aligned to Python homophones-module intent, constrained to conservative homophone/near-homophone/mistranscription corrections;
|
|
||||||
- prompt context includes protected glossary names/aliases/plurals to avoid damaging known terms;
|
|
||||||
- explicit guardrails against punctuation cleanup, grammar cleanup, style rewriting, summarization, and content invention;
|
|
||||||
- proposal generation through `internal/framework/proposal_generation` and `contracts.StructuredLLMClient`;
|
|
||||||
- scheduler-aware proposal calls through existing `contracts.LLMScheduler` hooks;
|
|
||||||
- replacement policy `require_unique` (matching Python homophones behavior);
|
|
||||||
- validator chain integration using existing deterministic + LLM-backed validators;
|
|
||||||
- homophones confidence threshold enforcement through existing validator/config infrastructure;
|
|
||||||
- protected-term guardrails for non-glossary modules remain active and are exercised through the homophones path;
|
|
||||||
- module-level reporting and diagnostics capture through existing runner/reporting paths.
|
|
||||||
|
|
||||||
## Implemented spoken_word production module
|
Shared validator runtime mechanics live in `internal/framework/validators`.
|
||||||
`internal/modules/spoken_word` now provides the fourth production module:
|
|
||||||
- prompt builder aligned to Python spoken_word-module intent, constrained to conservative dysfluency cleanup;
|
|
||||||
- strong prompt guardrails preserving meaning/intent/voice/named entities/domain terms and substantive content;
|
|
||||||
- explicit guardrails against summarization, style rewriting, grammar-only cleanup, punctuation-only cleanup, invention, and meaning-changing rewrites;
|
|
||||||
- proposal generation through `internal/framework/proposal_generation` and `contracts.StructuredLLMClient`;
|
|
||||||
- scheduler-aware proposal calls through existing `contracts.LLMScheduler` hooks;
|
|
||||||
- replacement policy `require_unique` (matching Python spoken_word behavior);
|
|
||||||
- validator chain integration using existing deterministic + LLM-backed validators, including strong semantic guardrails (`spoken_word_review`, `meaning_reversal_review`);
|
|
||||||
- spoken_word confidence threshold enforcement through existing validator/config infrastructure;
|
|
||||||
- protected-term guardrails for non-glossary modules remain active and are exercised through the spoken_word path;
|
|
||||||
- module-level reporting and diagnostics capture through existing runner/reporting paths.
|
|
||||||
|
|
||||||
## Reports and diagnostics (implemented)
|
Execution class metadata (deterministic vs LLM-backed) is centralized in `internal/validators/metadata` and used for ordering and reporting classification.
|
||||||
Current per-run artifacts include:
|
|
||||||
- `source-transcript.json`
|
|
||||||
- `source-transcript-parsed.json`
|
|
||||||
- `normalized-transcript.json`
|
|
||||||
- `normalization-summary.json`
|
|
||||||
- `chunking-summary.json`
|
|
||||||
- `utilization-diagnostics.json`
|
|
||||||
- `correction-ledger.json`
|
|
||||||
- `invocation.json`
|
|
||||||
- `effective-config.json` (redacted credentials)
|
|
||||||
- `report.json`
|
|
||||||
- `error.log` on failure
|
|
||||||
|
|
||||||
`--report-json` writes a separate report file when requested.
|
## Structured LLM boundary
|
||||||
|
All production LLM calls go through the internal contract:
|
||||||
|
- `contracts.StructuredLLMClient`
|
||||||
|
- `CompleteStructured(ctx, req, out)`
|
||||||
|
|
||||||
Current process reports include diagnostics metadata references for:
|
The OpenAI-compatible HTTP adapter is implemented in `internal/framework/llm`.
|
||||||
- diagnostics directory path;
|
|
||||||
- source transcript artifact path;
|
|
||||||
- parsed source transcript artifact path;
|
|
||||||
- normalized transcript artifact path;
|
|
||||||
- normalization summary artifact path;
|
|
||||||
- chunking summary artifact path;
|
|
||||||
- utilization diagnostics artifact path;
|
|
||||||
- correction ledger artifact path;
|
|
||||||
- invocation metadata artifact path;
|
|
||||||
- redacted effective-config artifact path;
|
|
||||||
- error-log artifact path on failure.
|
|
||||||
|
|
||||||
Current process reports also include:
|
Structured response schemas are registered in `internal/framework/responseschema` and attached to requests via `response_format` metadata.
|
||||||
- module-level results (when runner modules execute), including applied/skipped proposal changes;
|
|
||||||
- run-level module summary totals and failed module instance metadata.
|
|
||||||
- module-level warning records for malformed proposal-generation payloads and malformed validator batches.
|
|
||||||
- module-level validator decisions and validator rejections.
|
|
||||||
- optional decision-level diagnostic artifact paths for validator LLM interactions when available.
|
|
||||||
- stable validator keys in `validator_name` fields for validator decisions/rejections.
|
|
||||||
- explicit report metadata:
|
|
||||||
- report schema name;
|
|
||||||
- report schema version;
|
|
||||||
- selected output schema;
|
|
||||||
- config file version when config file input is used.
|
|
||||||
- review/observability artifacts:
|
|
||||||
- run-level and module-level utilization/timing summaries;
|
|
||||||
- flattened correction ledger entries for applied/rejected/skipped/failed correction dispositions.
|
|
||||||
|
|
||||||
Utilization diagnostics collection:
|
Malformed structured-output detection is centralized in `internal/framework/structuredoutput` and reused by proposal generation and validator execution so downgrade behavior stays consistent.
|
||||||
- collection is performed in the runner path via lightweight instrumentation around LLM scheduler and structured-client execution (`internal/framework/runner`);
|
|
||||||
- instrumentation is observational only and does not change scheduler acquisition/release semantics or module execution order;
|
|
||||||
- serialized artifact: `utilization-diagnostics.json`.
|
|
||||||
|
|
||||||
Utilization diagnostics high-level shape:
|
## Stage naming and diagnostics metadata
|
||||||
- `effective_concurrency`:
|
Diagnostics stage naming is centralized in `internal/framework/stagename`:
|
||||||
- `total_llm`, `proposal_llm`, `validation_llm`;
|
- module proposal stage names;
|
||||||
- `run_timing`:
|
- proposal-generation stage names;
|
||||||
- `run_wall_time_ms`;
|
- validator batch stage names.
|
||||||
- `scheduler_queue_wait_ms`;
|
|
||||||
- `llm_execution_time_ms`;
|
|
||||||
- `deterministic_validation_time_ms`;
|
|
||||||
- `max_in_flight_llm_calls`;
|
|
||||||
- `average_in_flight_llm_calls`;
|
|
||||||
- `llm_calls`:
|
|
||||||
- `total_proposal_calls`;
|
|
||||||
- `total_validation_calls`;
|
|
||||||
- `modules`:
|
|
||||||
- per-module key/instance timing summaries including module wall time and per-module call counts;
|
|
||||||
- `validators`:
|
|
||||||
- per-validator summaries keyed by stable validator key with elapsed time and LLM-backed marker.
|
|
||||||
|
|
||||||
Correction ledger construction:
|
Prompt metadata and response-schema metadata each expose canonical diagnostics maps via:
|
||||||
- ledger entries are built from runner module results in the CLI report/diagnostics path (`internal/cli/review_artifacts.go`);
|
- `prompts.Metadata.DiagnosticsMap()`
|
||||||
- serialized artifact: `correction-ledger.json`;
|
- `responseschema.Schema.DiagnosticsMap()`
|
||||||
- one flattened record per applied/validator-rejected/application-skipped outcome where data is available, plus module-failed records for failed module instances.
|
|
||||||
|
|
||||||
Correction ledger high-level shape:
|
## Diagnostics and reporting
|
||||||
- run/module/proposal identity:
|
Run-directory artifacts are owned by `internal/core/diagnostics`.
|
||||||
- `run_id`, `module_key`, `module_instance`, `proposal_index`, `segment_id`;
|
|
||||||
- correction payload:
|
|
||||||
- `original_text`, `proposed_corrected_text`, `applied_corrected_text` (when applied), `replacement_policy`;
|
|
||||||
- disposition:
|
|
||||||
- `disposition` in `{applied,rejected,skipped,failed}`;
|
|
||||||
- `disposition_reason_code`, `disposition_message`;
|
|
||||||
- validator decision snapshots:
|
|
||||||
- `deterministic_validator_decisions[]`;
|
|
||||||
- `llm_validator_decisions[]`;
|
|
||||||
- each decision uses stable validator keys and reason codes.
|
|
||||||
|
|
||||||
Identity and metadata boundaries:
|
Stable artifact names are centralized constants (for example transcript artifacts, `invocation.json`, `effective-config.json`, `utilization-diagnostics.json`, `correction-ledger.json`, `report.json`, `error.log`).
|
||||||
- stable module keys/instance names and stable validator keys are included directly in ledger records;
|
|
||||||
- prompt metadata and structured response schema metadata remain in LLM interaction diagnostics payloads and are not duplicated into every ledger row;
|
|
||||||
- reports reference artifact paths for utilization and ledger files through diagnostics metadata.
|
|
||||||
|
|
||||||
Redaction and retention:
|
Report diagnostics path metadata is constructed through `BuildDiagnosticsMetadata`, which keeps run-directory artifact references consistent between success and failure reports.
|
||||||
- secret redaction guarantees continue to apply to diagnostics/report artifacts;
|
|
||||||
- utilization and ledger artifacts are emitted within the existing run-directory retention model (`auto|always|never`) and are retained/removed with the run directory.
|
|
||||||
|
|
||||||
Current report schema metadata values:
|
## Secret redaction
|
||||||
- `report_metadata.report_schema_name = "audita-process-report"`
|
Redaction responsibilities are split by concern:
|
||||||
- `report_metadata.report_schema_version = "v1"`
|
- structural config redaction: `config.Config.Redacted()`
|
||||||
|
- byte/string payload redaction for diagnostics and surfaced errors: framework redaction utilities.
|
||||||
|
|
||||||
Retention modes implemented in `ApplyRetention`:
|
Configured LLM secret extraction is centralized in `llm.ConfiguredSecrets(cfg)` and reused across proposal and validator diagnostics paths.
|
||||||
- `always`: keep all run directories.
|
|
||||||
- `never`: keep successful run directories.
|
|
||||||
- `auto`: keep failed runs and successful runs with skipped corrections.
|
|
||||||
- failed runs are always retained.
|
|
||||||
|
|
||||||
Current runtime note:
|
## Output contracts
|
||||||
- default non-explicit runs usually have no module-level skipped corrections, so `auto` commonly removes clean successful run directories.
|
Transcript output schema selection is owned by `internal/core/outputschema`.
|
||||||
- explicit grammar/glossary/homophones/spoken_word runs can produce validator rejections and application skips, which are reflected in reports and retention input.
|
|
||||||
|
|
||||||
## Current tests and quality posture
|
Supported schemas:
|
||||||
Implemented tests currently cover:
|
- `bare-segments`
|
||||||
- CLI argument handling and behavior (`internal/cli/run_test.go`)
|
- `audita-v1`
|
||||||
- subprocess stdout/stderr and exit-code behavior (`cmd/audita/main_integration_test.go`)
|
|
||||||
- config/env/override validation (`internal/core/config/*_test.go`)
|
|
||||||
- transcript and glossary schema validation (`internal/core/schema/*_test.go`)
|
|
||||||
- deterministic normalization (`internal/core/normalization/*_test.go`)
|
|
||||||
- deterministic chunking and summaries (`internal/core/chunking/*_test.go`)
|
|
||||||
- proposal preview/apply semantics (`internal/framework/proposals/*_test.go`)
|
|
||||||
- contracts/foundation composition tests (`internal/framework/contracts/*_test.go`)
|
|
||||||
- runner sequencing and failure behavior with deterministic fake modules (`internal/framework/runner/*_test.go`)
|
|
||||||
- CLI runner integration through injected fake module factories (`internal/cli/run_test.go`)
|
|
||||||
- validator models, cardinality enforcement, and deterministic validators (`internal/framework/validators/*_test.go`)
|
|
||||||
- LLM-backed validator batching, prompt builders, structured-response safety, scheduler hooks, and diagnostics redaction (`internal/framework/validators/*_test.go`, `internal/framework/runner/*_test.go`)
|
|
||||||
- shared proposal-generation request/response parsing, deterministic indexing, scheduler hooks, and diagnostics redaction (`internal/framework/proposal_generation/*_test.go`, `internal/framework/runner/*_test.go`)
|
|
||||||
- production module-registry known-key recognition and unsupported/internal-registry error behavior (`internal/framework/modules/*_test.go`, `internal/cli/run_test.go`)
|
|
||||||
- production grammar module prompt constraints, proposal mapping, validator-chain behavior, confidence-threshold enforcement, diagnostics redaction, and explicit CLI/runtime integration (`internal/modules/grammar/*_test.go`, `internal/cli/run_test.go`, `internal/framework/runner/*_test.go`)
|
|
||||||
- production glossary module prompt constraints, proposal mapping, validator-chain behavior, confidence-threshold enforcement, diagnostics redaction, repeated-stage behavior, and explicit CLI/runtime integration (`internal/modules/glossary/*_test.go`, `internal/cli/run_test.go`, `internal/framework/runner/*_test.go`)
|
|
||||||
- production homophones module prompt constraints, proposal mapping, validator-chain behavior, confidence-threshold enforcement, diagnostics redaction, protected-term behavior, and explicit CLI/runtime integration (`internal/modules/homophones/*_test.go`, `internal/cli/run_test.go`, `internal/framework/runner/*_test.go`)
|
|
||||||
- production spoken_word module prompt constraints, proposal mapping, validator-chain behavior, semantic guardrail behavior, confidence-threshold enforcement, diagnostics redaction, protected-term behavior, and explicit CLI/runtime integration (`internal/modules/spoken_word/*_test.go`, `internal/cli/run_test.go`, `internal/framework/runner/*_test.go`)
|
|
||||||
- glossary-derived protected-term extraction and stable behavior (`internal/framework/validators/protected_terms_test.go`)
|
|
||||||
- default full-pipeline runtime shape and ordering (`internal/cli/run_test.go`, `cmd/audita/main_integration_test.go`, `internal/cli/parity_test.go`)
|
|
||||||
- subprocess operational hardening behavior including large-input, failure-mode, timeout/cancellation, backend-failure, and partial-progress paths (`cmd/audita/main_integration_test.go`)
|
|
||||||
- report/diagnostics redaction and artifact-shape behavior across success and failure paths (`internal/cli/run_test.go`, `cmd/audita/main_integration_test.go`)
|
|
||||||
- curated release-fixture and idempotence-oriented readiness checks using fake structured LLM responses (`internal/cli/release_fixtures_test.go`, `internal/cli/testdata/release`)
|
|
||||||
|
|
||||||
## Operational hardening status
|
Unknown schema keys fail validation and runtime resolution.
|
||||||
The runtime now includes hardened subprocess behavior for parent-process callers:
|
|
||||||
- deterministic success/failure exit codes;
|
|
||||||
- strict stdout/stderr separation suitable for machine orchestration;
|
|
||||||
- failure stderr summaries that include diagnostics location when available;
|
|
||||||
- retained failure diagnostics (`report.json`, `error.log`, and artifacts written before failure);
|
|
||||||
- deterministic timeout/cancellation behavior in tests;
|
|
||||||
- redaction coverage for API keys/secrets across reports, diagnostics artifacts, and surfaced errors.
|
|
||||||
- stable output routing behavior:
|
|
||||||
- with `--output`, stdout remains empty on success;
|
|
||||||
- without `--output`, stdout contains only transcript JSON in the selected output schema;
|
|
||||||
- `--report-json` writes report data to file only (never stdout).
|
|
||||||
|
|
||||||
Operational caller guidance is documented in [`docs/subprocess-operations.md`](docs/subprocess-operations.md).
|
## Key package map
|
||||||
|
Core packages:
|
||||||
|
- `internal/core/config`
|
||||||
|
- `internal/core/schema`
|
||||||
|
- `internal/core/normalization`
|
||||||
|
- `internal/core/chunking`
|
||||||
|
- `internal/core/diagnostics`
|
||||||
|
- `internal/core/reporting`
|
||||||
|
- `internal/core/modulecatalog`
|
||||||
|
- `internal/core/outputschema`
|
||||||
|
|
||||||
## Final status
|
Framework packages:
|
||||||
- Audita's default full module-sequence runtime is implemented and tested.
|
- `internal/framework/contracts`
|
||||||
- Parity fixtures and operational hardening coverage are in place.
|
- `internal/framework/proposals`
|
||||||
- Historical migration context is documented in [`docs/migration-from-python.md`](docs/migration-from-python.md).
|
- `internal/framework/proposal_generation`
|
||||||
|
- `internal/framework/promptcontext`
|
||||||
|
- `internal/framework/runner`
|
||||||
|
- `internal/framework/validators`
|
||||||
|
- `internal/framework/llm`
|
||||||
|
- `internal/framework/responseschema`
|
||||||
|
- `internal/framework/stagename`
|
||||||
|
- `internal/framework/structuredoutput`
|
||||||
|
|
||||||
|
Domain packages:
|
||||||
|
- `internal/modules/*`
|
||||||
|
- `internal/validators/*`
|
||||||
|
- `internal/prompts`
|
||||||
|
|||||||
@@ -1,31 +1,25 @@
|
|||||||
# Audita Public Contract
|
# Audita Public Contract
|
||||||
|
|
||||||
This document defines stability expectations for Audita's external process and data interfaces.
|
|
||||||
|
|
||||||
## Scope
|
## Scope
|
||||||
|
This document defines stability expectations for Audita's external runtime interfaces.
|
||||||
|
|
||||||
This contract covers:
|
Covered interfaces:
|
||||||
- CLI invocation and behavior
|
- CLI commands and major flags;
|
||||||
- versioned config file behavior
|
- versioned config behavior and precedence;
|
||||||
- transcript/glossary input forms
|
- transcript/glossary input forms;
|
||||||
- transcript output schema selection
|
- output schema selection;
|
||||||
- process report schema metadata
|
- report schema metadata;
|
||||||
- stable validator key identifiers in report/diagnostics records
|
- diagnostics artifact path metadata;
|
||||||
- prompt metadata identifiers in diagnostics
|
- stdout/stderr and exit-code behavior;
|
||||||
- diagnostics directory behavior
|
- redaction guarantees.
|
||||||
- utilization diagnostics and correction-ledger artifact presence/pathing in diagnostics metadata
|
|
||||||
- stdout/stderr and exit-code behavior
|
|
||||||
- secret redaction guarantees
|
|
||||||
- compatibility and deprecation policy
|
|
||||||
|
|
||||||
## CLI stability expectations
|
|
||||||
|
|
||||||
|
## CLI contract
|
||||||
Stable commands:
|
Stable commands:
|
||||||
- `audita process`
|
- `audita process`
|
||||||
- `audita config validate`
|
- `audita config validate`
|
||||||
- `audita config print-effective`
|
- `audita config print-effective`
|
||||||
|
|
||||||
For `audita process`, stable high-value flags include:
|
Stable high-value `process` flags:
|
||||||
- `--config`
|
- `--config`
|
||||||
- `--glossary`
|
- `--glossary`
|
||||||
- `--output`
|
- `--output`
|
||||||
@@ -33,136 +27,95 @@ For `audita process`, stable high-value flags include:
|
|||||||
- `--modules`
|
- `--modules`
|
||||||
- `--output-schema`
|
- `--output-schema`
|
||||||
|
|
||||||
Compatibility flags and lower-level tuning flags remain available; they may be narrowed over time with explicit compatibility notes.
|
## Config contract
|
||||||
|
Supported config format:
|
||||||
|
- YAML;
|
||||||
|
- `version: 1`;
|
||||||
|
- strict unknown-field rejection.
|
||||||
|
|
||||||
## Config file stability expectations
|
Path resolution for `process` and `config print-effective`:
|
||||||
|
1. `--config`
|
||||||
|
2. `AUDITA_CONFIG`
|
||||||
|
3. `/usr/local/etc/audita/config.yml`
|
||||||
|
4. `/etc/audita/config.yml`
|
||||||
|
|
||||||
Supported file format:
|
Missing explicit path is an error. Missing default paths is non-fatal.
|
||||||
- YAML
|
|
||||||
- strict unknown-field rejection
|
|
||||||
- explicit `version`
|
|
||||||
|
|
||||||
Supported version:
|
Precedence for `process`:
|
||||||
- `version: 1`
|
1. defaults
|
||||||
|
|
||||||
Precedence for `audita process`:
|
|
||||||
1. built-in defaults
|
|
||||||
2. file config
|
2. file config
|
||||||
3. environment overrides
|
3. environment overrides
|
||||||
4. CLI overrides
|
4. CLI overrides
|
||||||
|
|
||||||
Config source behavior:
|
`config validate` remains file-only validation (defaults + file config; no env overrides).
|
||||||
- `--config <path>`: missing path is a clear failure
|
|
||||||
- `AUDITA_CONFIG`: missing path is a clear failure
|
|
||||||
- defaults `/usr/local/etc/audita/config.yml`, then `/etc/audita/config.yml`: both missing is non-fatal
|
|
||||||
|
|
||||||
## Supported transcript input forms
|
Module and output-schema keys are validated against built-in catalogs. Unknown keys fail validation.
|
||||||
|
|
||||||
Audita accepts transcript JSON as either:
|
## Input contract
|
||||||
- a top-level array of segments
|
Supported transcript JSON top-level forms:
|
||||||
- an object with a `segments` array
|
- array of segments
|
||||||
|
- object with `segments` array
|
||||||
|
|
||||||
Segments must satisfy the schema and validation rules enforced by `internal/core/schema`.
|
Supported glossary YAML form:
|
||||||
|
- top-level `glossary` list with required entry fields validated by schema parsing.
|
||||||
|
|
||||||
## Supported glossary input form
|
## Output schema contract
|
||||||
|
Supported transcript output schemas:
|
||||||
Audita accepts glossary YAML with a top-level `glossary` entry list and validates required fields per entry.
|
|
||||||
|
|
||||||
## Supported output schema names
|
|
||||||
|
|
||||||
Built-in output schema registry supports:
|
|
||||||
- `bare-segments` (default)
|
- `bare-segments` (default)
|
||||||
- `audita-v1`
|
- `audita-v1`
|
||||||
|
|
||||||
`seriatim-intermediate` is planned but not implemented.
|
Unknown schema keys fail before output write.
|
||||||
|
|
||||||
Unknown output schema names fail clearly.
|
## Report metadata contract
|
||||||
|
Process reports include stable report metadata fields:
|
||||||
## Report schema/versioning expectations
|
|
||||||
|
|
||||||
Process report payloads include `report_metadata` with:
|
|
||||||
- `report_schema_name`
|
- `report_schema_name`
|
||||||
- `report_schema_version`
|
- `report_schema_version`
|
||||||
- `output_schema`
|
- `output_schema`
|
||||||
- `config_version` when file config is used
|
- `config_version` (when file config is loaded)
|
||||||
|
|
||||||
Current values:
|
Current values:
|
||||||
- `report_schema_name`: `audita-process-report`
|
- `report_schema_name = audita-process-report`
|
||||||
- `report_schema_version`: `v1`
|
- `report_schema_version = v1`
|
||||||
|
|
||||||
`--report-json` output and diagnostics run-dir `report.json` use the same report schema metadata.
|
`--report-json` output and run-directory `report.json` use the same report schema metadata.
|
||||||
|
|
||||||
Validator decision/rejection records in reports use stable validator keys in `validator_name`.
|
Validator decision/rejection records use stable validator keys via `validator_name`.
|
||||||
Module results may also include warning records for malformed module-stage LLM payloads.
|
|
||||||
Report diagnostics metadata includes artifact-path fields for utilization diagnostics and correction ledger when diagnostics initialization succeeds.
|
|
||||||
|
|
||||||
## Diagnostics directory behavior
|
## Diagnostics metadata contract
|
||||||
|
When run-directory initialization succeeds, diagnostics metadata paths reference stable artifacts, including:
|
||||||
|
- transcript and normalization artifacts;
|
||||||
|
- chunking summary;
|
||||||
|
- invocation metadata;
|
||||||
|
- redacted effective config;
|
||||||
|
- utilization diagnostics;
|
||||||
|
- correction ledger;
|
||||||
|
- `error.log` on failures.
|
||||||
|
|
||||||
When diagnostics directory creation succeeds, Audita writes run artifacts including:
|
LLM interaction diagnostics include stable prompt and structured-schema identifiers where applicable.
|
||||||
- invocation metadata
|
|
||||||
- redacted effective config
|
|
||||||
- transcript/normalization/chunking artifacts
|
|
||||||
- utilization diagnostics (`utilization-diagnostics.json`)
|
|
||||||
- correction ledger (`correction-ledger.json`)
|
|
||||||
- report and failure error log (when applicable)
|
|
||||||
- module/LLM diagnostics artifacts as available
|
|
||||||
|
|
||||||
Retention behavior is controlled by configured retention mode; failed runs are retained.
|
## Stdout/stderr and exit codes
|
||||||
|
Success:
|
||||||
|
- with `--output`, stdout is empty;
|
||||||
|
- without `--output`, stdout contains transcript JSON only;
|
||||||
|
- report JSON is not written to stdout.
|
||||||
|
|
||||||
Diagnostics metadata for LLM interactions may include semi-public prompt identifiers:
|
Failures:
|
||||||
- `prompt_id`
|
- nonzero exit;
|
||||||
- `prompt_version`
|
- human-readable stderr summary;
|
||||||
- `prompt_source`
|
- diagnostics directory path on stderr when available.
|
||||||
- `embedded_path`
|
|
||||||
- `sha256`
|
|
||||||
|
|
||||||
These are diagnostic identifiers, not user-facing prompt override controls.
|
Exit codes:
|
||||||
|
- `0` success
|
||||||
|
- nonzero failure
|
||||||
|
|
||||||
## Stdout/stderr behavior
|
## Redaction contract
|
||||||
|
Configured secrets are redacted from:
|
||||||
|
- effective config outputs;
|
||||||
|
- diagnostics artifacts;
|
||||||
|
- report artifacts;
|
||||||
|
- surfaced adapter/runtime errors.
|
||||||
|
|
||||||
Success behavior:
|
## Compatibility policy
|
||||||
- with `--output`, stdout is empty
|
Stable command behavior, schema names, report metadata keys, diagnostics-path field semantics, and validator key identities are treated as public contract.
|
||||||
- without `--output`, stdout contains only transcript JSON in selected output schema
|
|
||||||
- report JSON is not written to stdout
|
|
||||||
- success stderr remains empty even when reports/diagnostics contain module warnings
|
|
||||||
|
|
||||||
Failure behavior:
|
Additive fields are acceptable when existing fields and behavior remain compatible.
|
||||||
- stderr contains human-readable error summary
|
|
||||||
- nonzero exit
|
|
||||||
- diagnostics path is printed when available
|
|
||||||
|
|
||||||
## Exit-code behavior
|
|
||||||
|
|
||||||
- `0`: success
|
|
||||||
- nonzero: failure
|
|
||||||
|
|
||||||
Treat any nonzero exit as a failed invocation.
|
|
||||||
|
|
||||||
## Secret redaction guarantees
|
|
||||||
|
|
||||||
Audita redacts API keys and authorization secrets from:
|
|
||||||
- effective config outputs (`audita config print-effective`, diagnostics effective-config artifact)
|
|
||||||
- report artifacts
|
|
||||||
- LLM diagnostics artifacts
|
|
||||||
- surfaced request/response error messages
|
|
||||||
|
|
||||||
Config files should reference secrets via environment variable names (`api_key_env`) rather than embedding secret values.
|
|
||||||
|
|
||||||
## Compatibility and deprecation policy
|
|
||||||
|
|
||||||
- Existing stable schema names, report metadata keys, and top-level command behavior are treated as public contract.
|
|
||||||
- Existing stable validator keys remain public contract values even when validator semantics are refined.
|
|
||||||
- Compatibility inputs (legacy flags/env aliases) may remain during transition windows.
|
|
||||||
- Any planned removal or behavior change should include clear compatibility notes and migration guidance.
|
|
||||||
|
|
||||||
## Breaking changes after 1.0
|
|
||||||
|
|
||||||
After 1.0, breaking changes include, for example:
|
|
||||||
- changing default success/failure exit-code semantics
|
|
||||||
- changing stdout/stderr routing semantics
|
|
||||||
- silently changing default output schema shape
|
|
||||||
- removing supported output schema names without compatibility strategy
|
|
||||||
- changing report schema fields or meanings incompatibly
|
|
||||||
- changing config version semantics incompatibly without version bump
|
|
||||||
|
|
||||||
Additive fields, additive diagnostics, and new optional schema names are generally non-breaking when existing behavior remains intact.
|
|
||||||
|
|||||||
@@ -1,90 +1,72 @@
|
|||||||
# Structured LLM Architecture
|
# Structured LLM Architecture
|
||||||
|
|
||||||
## Purpose
|
## Scope
|
||||||
|
|
||||||
This document describes Audita's structured LLM runtime boundary and adapter behavior.
|
This document describes Audita's structured LLM runtime boundary and adapter behavior.
|
||||||
|
|
||||||
## Why Audita owns the adapter
|
## Runtime boundary
|
||||||
|
Production LLM integration depends on the internal contract only:
|
||||||
Audita owns a small structured LLM adapter so that core runtime behavior is controlled inside the repository:
|
- `contracts.StructuredLLMClient`
|
||||||
- request construction and schema handling are explicit and testable;
|
|
||||||
- retries, timeouts, cancellation, and error redaction are consistent across modules and validators;
|
|
||||||
- provider SDK types are not exposed outside the adapter boundary;
|
|
||||||
- dependency weight and transitive provider-specific behavior are reduced.
|
|
||||||
|
|
||||||
At runtime, the rest of Audita depends only on the internal contract:
|
|
||||||
- `StructuredLLMClient`
|
|
||||||
- `CompleteStructured(ctx, req, out)`
|
- `CompleteStructured(ctx, req, out)`
|
||||||
|
|
||||||
## OpenAI-compatible request shape
|
Provider SDK types do not leak past this boundary.
|
||||||
|
|
||||||
At a conceptual level, Audita sends chat completion requests with:
|
## Adapter ownership
|
||||||
- `model`
|
`internal/framework/llm` owns the OpenAI-compatible HTTP adapter and shared LLM runtime utilities.
|
||||||
- `messages` (role/content pairs)
|
|
||||||
- `response_format`:
|
|
||||||
- `type = "json_schema"`
|
|
||||||
- `json_schema.name` (stable schema name)
|
|
||||||
- `json_schema.strict = true`
|
|
||||||
- `json_schema.schema` (registered JSON Schema payload)
|
|
||||||
|
|
||||||
The adapter uses OpenAI-compatible `POST {base_url}/chat/completions` over `net/http`.
|
Key responsibilities:
|
||||||
|
- request assembly;
|
||||||
|
- timeout/cancellation propagation;
|
||||||
|
- bounded retry behavior;
|
||||||
|
- scheduler integration;
|
||||||
|
- provider response decoding;
|
||||||
|
- error redaction.
|
||||||
|
|
||||||
## Structured response schema registry
|
## Structured schema registry
|
||||||
|
Structured response schemas are registered in `internal/framework/responseschema` and include stable metadata:
|
||||||
|
- `id`
|
||||||
|
- `version`
|
||||||
|
- `name`
|
||||||
|
- `json_schema`
|
||||||
|
- `sha256`
|
||||||
|
|
||||||
Structured response schemas are registered in `internal/framework/responseschema` with stable metadata:
|
Current schema keys:
|
||||||
- schema key
|
- `correction_set`
|
||||||
- schema ID
|
- `validator_decision_set`
|
||||||
- schema version
|
|
||||||
- schema name (OpenAI-compatible `response_format` name)
|
|
||||||
- raw JSON Schema payload
|
|
||||||
- SHA-256 hash
|
|
||||||
|
|
||||||
Current schemas:
|
Schema metadata is attached to diagnostics through `Schema.DiagnosticsMap()`.
|
||||||
- `correction_set`:
|
|
||||||
- id `audita.correction_set`
|
|
||||||
- version `v1`
|
|
||||||
- name `audita_correction_set_v1`
|
|
||||||
- `validator_decision_set`:
|
|
||||||
- id `audita.validator_decision_set`
|
|
||||||
- version `v1`
|
|
||||||
- name `audita_validator_decision_set_v1`
|
|
||||||
|
|
||||||
## Provider compatibility assumptions
|
## Request shape assumptions
|
||||||
|
Audita targets OpenAI-compatible chat-completions endpoints and sends structured requests with:
|
||||||
|
- model;
|
||||||
|
- chat messages;
|
||||||
|
- `response_format.type = json_schema`;
|
||||||
|
- schema name and JSON schema payload.
|
||||||
|
|
||||||
Audita assumes an OpenAI-compatible chat-completions endpoint that:
|
## Local validation remains mandatory
|
||||||
- accepts message arrays with model selection;
|
Provider schema enforcement is treated as transport-level guardrails.
|
||||||
- accepts `response_format.type = json_schema`;
|
|
||||||
- returns a completion with assistant message content and optional usage metadata.
|
|
||||||
|
|
||||||
Provider-specific differences are expected in strictness and error payload shapes, so the adapter treats provider output as untrusted until locally decoded.
|
Audita still validates output locally before applying behavior changes:
|
||||||
|
- proposal decoding and proposal invariants;
|
||||||
|
- validator decision decoding and cardinality checks;
|
||||||
|
- deterministic validation and apply-time rules.
|
||||||
|
|
||||||
## Local decode and validation remain mandatory
|
## Shared malformed-output policy
|
||||||
|
Malformed structured-output classification is centralized in `internal/framework/structuredoutput`.
|
||||||
|
|
||||||
Provider-level structured output is a transport guardrail, not final validation.
|
Proposal generation and validator execution both use this shared classifier so downgrade behavior cannot drift between the two paths.
|
||||||
|
|
||||||
After receiving a response, Audita still:
|
## Secrets and redaction
|
||||||
- decodes assistant content into typed request-specific structs;
|
Secret extraction for LLM redaction is centralized in `llm.ConfiguredSecrets(cfg)` and reused by proposal and validator diagnostics writers.
|
||||||
- validates proposal and validator payload invariants locally;
|
|
||||||
- enforces deterministic validator/cardinality rules before any transcript application.
|
|
||||||
|
|
||||||
This protects runtime correctness even when provider responses are malformed, partial, or semantically inconsistent.
|
Secrets are redacted from:
|
||||||
|
- diagnostics artifacts;
|
||||||
|
- report artifacts;
|
||||||
|
- surfaced adapter/runtime errors.
|
||||||
|
|
||||||
## Diagnostics and redaction
|
## Concurrency and scheduling
|
||||||
|
LLM execution is constrained by composed scheduler limits:
|
||||||
|
- total LLM concurrency;
|
||||||
|
- proposal LLM concurrency;
|
||||||
|
- validation LLM concurrency.
|
||||||
|
|
||||||
When structured schemas are used, diagnostics metadata records:
|
The scheduler is FIFO and context-aware so permits are released on success, failure, and cancellation.
|
||||||
- schema ID
|
|
||||||
- schema version
|
|
||||||
- schema name
|
|
||||||
- schema hash
|
|
||||||
|
|
||||||
Diagnostics and surfaced errors preserve secret redaction:
|
|
||||||
- API keys and bearer tokens are redacted from request/response/error artifacts;
|
|
||||||
- redaction is applied before diagnostic files are written.
|
|
||||||
|
|
||||||
## Runtime behavior guarantees
|
|
||||||
|
|
||||||
The structured LLM path preserves existing runtime guarantees:
|
|
||||||
- bounded LLM call execution through schedulers;
|
|
||||||
- context-aware cancellation and timeout propagation;
|
|
||||||
- retry behavior for transient failures and retryable malformed structured responses;
|
|
||||||
- deterministic module/chunk/proposal/validator behavior outside provider nondeterminism.
|
|
||||||
|
|||||||
@@ -1,79 +1,36 @@
|
|||||||
# Audita Validators
|
# Audita Validators
|
||||||
|
|
||||||
This document describes Audita's built-in validator registry and module validator chains.
|
|
||||||
|
|
||||||
For LLM-backed validator prompt asset details, see [`docs/prompts.md`](prompts.md).
|
|
||||||
|
|
||||||
## Package ownership
|
|
||||||
|
|
||||||
Built-in validator construction is package-owned under `internal/validators/<validator_key>`:
|
|
||||||
- `internal/validators/confidence_threshold`
|
|
||||||
- `internal/validators/proposal_shape`
|
|
||||||
- `internal/validators/original_text_presence`
|
|
||||||
- `internal/validators/non_empty_corrected_text`
|
|
||||||
- `internal/validators/no_effect`
|
|
||||||
- `internal/validators/protected_terms`
|
|
||||||
- `internal/validators/spoken_form_plausibility`
|
|
||||||
- `internal/validators/meaning_reversal_review`
|
|
||||||
- `internal/validators/editorial_review`
|
|
||||||
|
|
||||||
Registry and chain wiring stay in:
|
|
||||||
- `internal/validators/registry.go`
|
|
||||||
- `internal/validators/chains.go`
|
|
||||||
|
|
||||||
Shared validator runtime mechanics stay in `internal/framework/validators`:
|
|
||||||
- request/result/decision models
|
|
||||||
- decision cardinality helpers
|
|
||||||
- protected vocabulary helpers
|
|
||||||
- shared LLM validator runtime, batching, and diagnostics helpers
|
|
||||||
|
|
||||||
Execution classification metadata is defined in `internal/validators/metadata`:
|
|
||||||
- `deterministic`
|
|
||||||
- `llm_backed`
|
|
||||||
|
|
||||||
Runner ordering uses this metadata so deterministic validators run before LLM-backed validators without concrete framework type assertions.
|
|
||||||
|
|
||||||
## Scope
|
## Scope
|
||||||
|
This document defines the built-in validator system used by production module runs.
|
||||||
|
|
||||||
Validator chains are built-in runtime behavior.
|
## Ownership boundaries
|
||||||
|
Built-in validator keys, constructors, and module chains are owned by `internal/validators`.
|
||||||
|
|
||||||
Current 1.0 boundary:
|
Shared runtime execution mechanics are owned by `internal/framework/validators`, including:
|
||||||
- built-in validator keys and built-in module chains are stable runtime identifiers;
|
- validator request/result models;
|
||||||
- thresholds and batching knobs remain configurable where already supported;
|
- deterministic proposal checks;
|
||||||
- arbitrary user-defined validator chains are deferred.
|
- LLM validator batching and execution;
|
||||||
|
- decision-cardinality enforcement;
|
||||||
|
- diagnostics integration.
|
||||||
|
|
||||||
## Built-in validator keys
|
Execution class metadata is owned by `internal/validators/metadata`.
|
||||||
|
|
||||||
### Deterministic validators
|
|
||||||
|
|
||||||
|
## Stable validator keys
|
||||||
|
Deterministic:
|
||||||
- `proposal_shape`
|
- `proposal_shape`
|
||||||
- rejects malformed proposal fields before other validators run.
|
|
||||||
- `confidence_threshold`
|
- `confidence_threshold`
|
||||||
- checks proposal confidence against module-specific configured threshold.
|
|
||||||
- `original_text_presence`
|
- `original_text_presence`
|
||||||
- ensures target segment exists and `original_text` exists in current working segment text.
|
|
||||||
- `non_empty_corrected_text`
|
- `non_empty_corrected_text`
|
||||||
- rejects proposals whose previewed resulting segment text would be empty or whitespace-only.
|
|
||||||
- `no_effect`
|
- `no_effect`
|
||||||
- rejects proposals where `original_text == corrected_text`.
|
|
||||||
- `protected_terms`
|
- `protected_terms`
|
||||||
- protects glossary-derived terms from unsafe mutations in non-glossary modules.
|
|
||||||
- glossary stages use glossary-specific protection logic but still report this same stable key.
|
|
||||||
|
|
||||||
### LLM-backed validators
|
|
||||||
|
|
||||||
|
LLM-backed:
|
||||||
- `spoken_form_plausibility`
|
- `spoken_form_plausibility`
|
||||||
- checks whether proposed spoken-form change remains plausible in transcript context.
|
|
||||||
- `meaning_reversal_review`
|
- `meaning_reversal_review`
|
||||||
- checks for likely meaning reversal or semantic contradiction.
|
|
||||||
- `editorial_review`
|
- `editorial_review`
|
||||||
- performs conservative editorial safety review.
|
|
||||||
|
|
||||||
## Built-in module chains
|
## Built-in module chains
|
||||||
|
`glossary`:
|
||||||
Current built-in chains resolved from `internal/validators/chains.go`:
|
|
||||||
|
|
||||||
- `glossary`
|
|
||||||
- `proposal_shape`
|
- `proposal_shape`
|
||||||
- `no_effect`
|
- `no_effect`
|
||||||
- `original_text_presence`
|
- `original_text_presence`
|
||||||
@@ -83,7 +40,7 @@ Current built-in chains resolved from `internal/validators/chains.go`:
|
|||||||
- `spoken_form_plausibility`
|
- `spoken_form_plausibility`
|
||||||
- `meaning_reversal_review`
|
- `meaning_reversal_review`
|
||||||
|
|
||||||
- `homophones`
|
`homophones`:
|
||||||
- `proposal_shape`
|
- `proposal_shape`
|
||||||
- `no_effect`
|
- `no_effect`
|
||||||
- `original_text_presence`
|
- `original_text_presence`
|
||||||
@@ -93,7 +50,7 @@ Current built-in chains resolved from `internal/validators/chains.go`:
|
|||||||
- `spoken_form_plausibility`
|
- `spoken_form_plausibility`
|
||||||
- `meaning_reversal_review`
|
- `meaning_reversal_review`
|
||||||
|
|
||||||
- `spoken_word`
|
`spoken_word`:
|
||||||
- `proposal_shape`
|
- `proposal_shape`
|
||||||
- `no_effect`
|
- `no_effect`
|
||||||
- `original_text_presence`
|
- `original_text_presence`
|
||||||
@@ -103,7 +60,7 @@ Current built-in chains resolved from `internal/validators/chains.go`:
|
|||||||
- `editorial_review`
|
- `editorial_review`
|
||||||
- `meaning_reversal_review`
|
- `meaning_reversal_review`
|
||||||
|
|
||||||
- `grammar`
|
`grammar`:
|
||||||
- `proposal_shape`
|
- `proposal_shape`
|
||||||
- `no_effect`
|
- `no_effect`
|
||||||
- `original_text_presence`
|
- `original_text_presence`
|
||||||
@@ -113,48 +70,27 @@ Current built-in chains resolved from `internal/validators/chains.go`:
|
|||||||
- `editorial_review`
|
- `editorial_review`
|
||||||
- `meaning_reversal_review`
|
- `meaning_reversal_review`
|
||||||
|
|
||||||
## Protected terms construction
|
## Ordering and execution semantics
|
||||||
|
Validator ordering is based on canonical metadata:
|
||||||
|
- deterministic validators run before LLM-backed validators.
|
||||||
|
|
||||||
`protected_terms` has explicit constructors:
|
Within each module stage:
|
||||||
- general constructor used by non-glossary modules through the built-in registry
|
- proposals are generated per section;
|
||||||
- glossary-stage constructor used by glossary chain resolution
|
- validator chains execute on those proposals;
|
||||||
|
- approved proposals are applied once after section work settles.
|
||||||
|
|
||||||
Both variants preserve existing behavior and report the stable key `protected_terms`.
|
## Malformed payload behavior
|
||||||
|
Malformed structured-output from proposal generation and LLM validator calls is downgraded, not treated as a process-fatal transport error.
|
||||||
|
|
||||||
## Execution semantics
|
Current outcomes:
|
||||||
|
- malformed proposal-generation payloads produce section/module warnings and zero proposals for the affected section;
|
||||||
|
- malformed validator decision payloads reject the affected validator batch with warnings;
|
||||||
|
- deterministic validator behavior and runner order remain unchanged.
|
||||||
|
|
||||||
- modules execute serially;
|
## Reporting identity
|
||||||
- section proposal work can run concurrently within a module;
|
Reports and diagnostics use stable validator keys as identifiers.
|
||||||
- deterministic validators run before LLM-backed validators;
|
|
||||||
- malformed module proposal payloads are downgraded to section-scoped module warnings with zero proposals for the affected section rather than module failure;
|
|
||||||
- malformed/missing/duplicate/unknown LLM validator decisions reject the affected validator batch with warnings instead of failing the module;
|
|
||||||
- oversized single-proposal validator inputs reject only the affected proposal under that validator;
|
|
||||||
- approved proposals are applied once per module after section work settles.
|
|
||||||
|
|
||||||
## Validator rejections vs proposal-application skips
|
Correction-ledger deterministic-vs-LLM classification is derived from canonical validator metadata, not package-local hardcoded maps.
|
||||||
|
|
||||||
- validator rejection:
|
## Prompt assets
|
||||||
- proposal is denied by validator-chain review and appears in validator rejection reporting with validator key and reason code.
|
LLM validator prompt assets and prompt metadata are documented in [Prompts](./prompts.md).
|
||||||
- proposal-application skip:
|
|
||||||
- proposal passed validators but could not be applied under replacement-policy semantics (for example no matching span at apply time).
|
|
||||||
- module warning:
|
|
||||||
- malformed proposal-generation payloads and malformed validator batches are recorded in module warning records and diagnostics without writing success stderr.
|
|
||||||
|
|
||||||
These are separate outcomes and are reported separately.
|
|
||||||
|
|
||||||
## Reporting and diagnostics identity
|
|
||||||
|
|
||||||
- report validator decision/rejection entries use stable validator keys in `validator_name`.
|
|
||||||
- report module results include warning records for malformed module-stage LLM payloads.
|
|
||||||
- validator LLM diagnostics include validator identity in interaction metadata and structured response schema metadata.
|
|
||||||
- correction ledger entries include deterministic and LLM validator decision snapshots keyed by the same stable validator keys, and keep validator rejection distinct from application-level skip.
|
|
||||||
|
|
||||||
Prompt assets are unchanged by the validator package-ownership refactor and remain built-in under `internal/prompts`.
|
|
||||||
|
|
||||||
## Configurable knobs that remain supported
|
|
||||||
|
|
||||||
- per-module confidence thresholds (`thresholds.*` / equivalent env+CLI overrides)
|
|
||||||
- validation batching limits (`validation_max_prompt_tokens` / equivalent env+CLI overrides)
|
|
||||||
- validation LLM model/base URL/timeout/retries/concurrency settings
|
|
||||||
|
|
||||||
These tune validator behavior without exposing arbitrary user-defined chains.
|
|
||||||
|
|||||||
@@ -1,51 +1,48 @@
|
|||||||
# Audita Configuration
|
# Audita Configuration
|
||||||
|
|
||||||
This document describes Audita's versioned YAML config support and related commands.
|
## Scope
|
||||||
|
This document defines the supported versioned YAML configuration model and runtime precedence behavior.
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Audita's config file provides a stable place for pipeline defaults and runtime tuning that would otherwise require many environment variables or CLI flags.
|
|
||||||
|
|
||||||
Use config files for baseline settings, then use environment variables and CLI flags for deployment and per-run overrides.
|
|
||||||
|
|
||||||
## Supported version
|
|
||||||
|
|
||||||
Current supported config version:
|
|
||||||
|
|
||||||
|
## Supported file version
|
||||||
|
Current supported config file version:
|
||||||
- `version: 1`
|
- `version: 1`
|
||||||
|
|
||||||
Rules:
|
Validation rules:
|
||||||
|
- missing `version` fails;
|
||||||
- missing `version` fails validation;
|
- unsupported version fails;
|
||||||
- unknown versions fail validation;
|
- unknown YAML fields fail (strict decoding).
|
||||||
- unknown fields fail validation (strict decoding).
|
|
||||||
|
|
||||||
## Config path resolution
|
## Config path resolution
|
||||||
|
For `audita process` and `audita config print-effective`, path resolution order is:
|
||||||
|
1. `--config <path>`
|
||||||
|
2. `AUDITA_CONFIG`
|
||||||
|
3. `/usr/local/etc/audita/config.yml` (if present)
|
||||||
|
4. `/etc/audita/config.yml` (if present)
|
||||||
|
|
||||||
For `audita process`, config path resolution is:
|
Missing-path behavior:
|
||||||
|
- missing `--config` path is an error;
|
||||||
|
- missing `AUDITA_CONFIG` path is an error;
|
||||||
|
- missing both default paths is non-fatal.
|
||||||
|
|
||||||
1. `--config <path>` if provided
|
## Effective precedence
|
||||||
2. `AUDITA_CONFIG` if set and `--config` is not provided
|
`audita process` effective precedence:
|
||||||
3. default `/usr/local/etc/audita/config.yml` if present
|
1. defaults
|
||||||
4. fallback default `/etc/audita/config.yml` if present
|
|
||||||
|
|
||||||
Missing-file behavior:
|
|
||||||
|
|
||||||
- missing `--config` path: hard failure;
|
|
||||||
- missing `AUDITA_CONFIG` path: hard failure;
|
|
||||||
- missing both default-path files: non-fatal, run continues.
|
|
||||||
|
|
||||||
## Precedence model
|
|
||||||
|
|
||||||
Effective config precedence is:
|
|
||||||
|
|
||||||
1. built-in defaults
|
|
||||||
2. file config
|
2. file config
|
||||||
3. environment overrides
|
3. environment overrides
|
||||||
4. CLI overrides
|
4. CLI overrides
|
||||||
|
|
||||||
## Supported YAML fields
|
`audita config print-effective` uses:
|
||||||
|
1. defaults
|
||||||
|
2. file config
|
||||||
|
3. environment overrides
|
||||||
|
|
||||||
|
`audita config validate` intentionally uses file-only validation:
|
||||||
|
1. defaults
|
||||||
|
2. file config
|
||||||
|
|
||||||
|
Environment overrides are not applied in `config validate`.
|
||||||
|
|
||||||
|
## Supported top-level YAML fields
|
||||||
```yaml
|
```yaml
|
||||||
version: 1
|
version: 1
|
||||||
|
|
||||||
@@ -62,7 +59,6 @@ llm:
|
|||||||
api_key_env: AUDITA_LLM_API_KEY
|
api_key_env: AUDITA_LLM_API_KEY
|
||||||
timeout: 120s
|
timeout: 120s
|
||||||
max_retries: 3
|
max_retries: 3
|
||||||
|
|
||||||
validation:
|
validation:
|
||||||
base_url: https://openrouter.ai/api/v1
|
base_url: https://openrouter.ai/api/v1
|
||||||
model: openrouter/google/gemma-4-31b-it
|
model: openrouter/google/gemma-4-31b-it
|
||||||
@@ -100,91 +96,54 @@ diagnostics:
|
|||||||
retention: auto
|
retention: auto
|
||||||
```
|
```
|
||||||
|
|
||||||
`context.description` provides background-only transcript context for prompts.
|
## Module and output-schema validation
|
||||||
If both config and CLI provide a description, `--transcript-description` takes precedence.
|
`pipeline.modules` keys are validated against the built-in supported module catalog.
|
||||||
|
|
||||||
`output.schema` supports the built-in output schema registry values:
|
Supported module keys:
|
||||||
- `bare-segments` (default)
|
- `glossary`
|
||||||
|
- `homophones`
|
||||||
|
- `spoken_word`
|
||||||
|
- `grammar`
|
||||||
|
|
||||||
|
Repeated supported module keys are allowed.
|
||||||
|
|
||||||
|
`output.schema` is validated against the built-in output schema catalog.
|
||||||
|
|
||||||
|
Supported output schema keys:
|
||||||
|
- `bare-segments`
|
||||||
- `audita-v1`
|
- `audita-v1`
|
||||||
|
|
||||||
Unknown schema names fail clearly before transcript output is written.
|
Unknown module keys and unknown output schema keys fail validation.
|
||||||
|
|
||||||
Duration-like fields accept either:
|
## Duration field parsing
|
||||||
|
Duration-like fields support:
|
||||||
|
- numeric seconds (for example `120`, `3.5`)
|
||||||
|
- duration strings (for example `120s`, `2m`)
|
||||||
|
|
||||||
- numeric seconds (for example `120`, `3.5`), or
|
LLM timeout duration strings must resolve to whole seconds.
|
||||||
- duration strings (for example `120s`, `2m`).
|
|
||||||
|
|
||||||
For LLM timeouts, duration strings must resolve to whole seconds.
|
|
||||||
|
|
||||||
## Secret handling
|
## Secret handling
|
||||||
|
Use `api_key_env` fields for secrets:
|
||||||
Use `api_key_env` for secrets:
|
|
||||||
|
|
||||||
- `llm.proposal.api_key_env`
|
- `llm.proposal.api_key_env`
|
||||||
- `llm.validation.api_key_env`
|
- `llm.validation.api_key_env`
|
||||||
|
|
||||||
These fields must contain environment variable names, not secret values.
|
These fields store environment variable names, not secret values.
|
||||||
|
|
||||||
At runtime, Audita resolves those names from the process environment.
|
Resolved secret values are redacted from:
|
||||||
|
- `audita config print-effective` output;
|
||||||
Redaction behavior:
|
- diagnostics `effective-config.json`;
|
||||||
|
- report and diagnostics payloads.
|
||||||
- run diagnostics `effective-config.json` is redacted;
|
|
||||||
- `audita config print-effective` output is redacted;
|
|
||||||
- API keys are never emitted in plaintext by those outputs.
|
|
||||||
|
|
||||||
## Config commands
|
|
||||||
|
|
||||||
Validate a config file:
|
|
||||||
|
|
||||||
|
## Commands
|
||||||
|
Validate a file config:
|
||||||
```sh
|
```sh
|
||||||
audita config validate --config ./audita.yml
|
audita config validate --config ./audita.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
Print redacted effective config:
|
Print redacted effective config:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
audita config print-effective --config ./audita.yml
|
audita config print-effective --config ./audita.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
`print-effective` loads defaults, then file config, then environment overrides.
|
|
||||||
|
|
||||||
## Example: local OpenAI-compatible endpoint
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
version: 1
|
|
||||||
|
|
||||||
llm:
|
|
||||||
proposal:
|
|
||||||
base_url: http://localhost:8000/v1
|
|
||||||
model: local/proposal-model
|
|
||||||
api_key_env: AUDITA_LLM_API_KEY
|
|
||||||
timeout: 90s
|
|
||||||
max_retries: 2
|
|
||||||
|
|
||||||
validation:
|
|
||||||
base_url: http://localhost:8000/v1
|
|
||||||
model: local/validation-model
|
|
||||||
api_key_env: AUDITA_VALIDATION_LLM_API_KEY
|
|
||||||
timeout: 90s
|
|
||||||
max_retries: 2
|
|
||||||
|
|
||||||
pipeline:
|
|
||||||
modules: [glossary, homophones, glossary, spoken_word, grammar]
|
|
||||||
|
|
||||||
diagnostics:
|
|
||||||
work_dir: /tmp/audita
|
|
||||||
retention: auto
|
|
||||||
```
|
|
||||||
|
|
||||||
## Compatibility notes
|
## Compatibility notes
|
||||||
|
Legacy compatibility flags and environment aliases remain available where implemented, but the stable configuration surface is the versioned YAML model described above.
|
||||||
Existing environment variables and lower-level CLI flags remain available for compatibility.
|
|
||||||
|
|
||||||
Current guidance:
|
|
||||||
|
|
||||||
- prefer file config for baseline behavior;
|
|
||||||
- keep environment variables for secrets/deployment-specific overrides;
|
|
||||||
- use CLI flags for per-run overrides.
|
|
||||||
- validator chains are built-in and are not user-configurable in config.
|
|
||||||
- prompt source selection and filesystem prompt overrides are not config options.
|
|
||||||
|
|||||||
33
docs/development.md
Normal file
33
docs/development.md
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
# Audita Development Workflow
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
This document defines the canonical contributor workflow and engineering conventions for this repository.
|
||||||
|
|
||||||
|
## Workflow
|
||||||
|
1. Start from a clean understanding of scope and constraints.
|
||||||
|
2. Make focused changes that preserve existing public behavior unless behavior change is explicitly intended.
|
||||||
|
3. Run targeted tests for touched packages.
|
||||||
|
4. Run `go test ./...` before finalizing substantial changes.
|
||||||
|
5. Update affected documentation so it describes current behavior only.
|
||||||
|
|
||||||
|
## Engineering conventions
|
||||||
|
- Keep module packages separate: `glossary`, `homophones`, `spoken_word`, `grammar`.
|
||||||
|
- Prefer narrow shared helpers and catalogs over broad abstractions.
|
||||||
|
- Preserve diagnostics artifact naming and report field contracts unless intentionally changed.
|
||||||
|
- Preserve CLI/config precedence semantics unless intentionally changed.
|
||||||
|
- Treat stable validator keys, prompt identifiers, and output-schema keys as contract surfaces.
|
||||||
|
|
||||||
|
## Configuration and runtime expectations
|
||||||
|
- `audita process` precedence is defaults -> file -> env -> CLI.
|
||||||
|
- `audita config validate` validates file config merged onto defaults only.
|
||||||
|
- `audita config print-effective` includes environment overrides and prints redacted JSON.
|
||||||
|
|
||||||
|
## Testing expectations
|
||||||
|
- Add tests for new behavior and for bug fixes.
|
||||||
|
- Keep deterministic fixtures stable.
|
||||||
|
- Do not reduce existing parity, release-fixture, subprocess, or module-specific coverage without equivalent replacement.
|
||||||
|
|
||||||
|
## Commit discipline
|
||||||
|
- Keep commits scoped and reviewable.
|
||||||
|
- Avoid mixing unrelated refactors with behavior changes.
|
||||||
|
- Use clear plain-English commit messages.
|
||||||
27
docs/documentation/policy.md
Normal file
27
docs/documentation/policy.md
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
# Documentation Policy
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
This policy defines how project documentation should be authored and maintained.
|
||||||
|
|
||||||
|
## Core rules
|
||||||
|
- Document the current behavior of the codebase.
|
||||||
|
- Remove stale behavior descriptions promptly when code changes.
|
||||||
|
- Do not describe development history in architecture or behavior docs unless a document is explicitly historical.
|
||||||
|
- Do not use architecture or behavior docs as changelogs.
|
||||||
|
- Prefer rewriting stale sections from scratch when substantial behavior or ownership changes occur.
|
||||||
|
|
||||||
|
## Consistency requirements
|
||||||
|
- Keep command examples aligned with current CLI surfaces.
|
||||||
|
- Keep configuration examples aligned with supported fields and precedence.
|
||||||
|
- Keep architecture package ownership descriptions aligned with current code layout.
|
||||||
|
- Keep stable contract identifiers accurate (module keys, validator keys, output-schema keys, report metadata fields).
|
||||||
|
|
||||||
|
## Cross-document expectations
|
||||||
|
- `docs/architecture/*` documents runtime behavior and package ownership.
|
||||||
|
- `docs/configuration.md` documents config schema and precedence.
|
||||||
|
- `docs/development.md` documents contributor workflow and engineering conventions.
|
||||||
|
|
||||||
|
## Review expectations for documentation changes
|
||||||
|
- Verify referenced files and links exist.
|
||||||
|
- Verify examples match current behavior.
|
||||||
|
- Prefer concise, direct language and avoid speculative future claims.
|
||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||||
|
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -48,12 +49,6 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
}
|
}
|
||||||
|
|
||||||
entries := make([]correctionLedgerEntry, 0)
|
entries := make([]correctionLedgerEntry, 0)
|
||||||
llmBacked := map[string]bool{
|
|
||||||
"spoken_form_plausibility": true,
|
|
||||||
"meaning_reversal_review": true,
|
|
||||||
"editorial_review": true,
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, module := range runOutput.ModuleResults {
|
for _, module := range runOutput.ModuleResults {
|
||||||
decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord)
|
decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord)
|
||||||
for _, decision := range module.ValidatorDecisions {
|
for _, decision := range module.ValidatorDecisions {
|
||||||
@@ -72,8 +67,8 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
AppliedCorrectedText: change.CorrectedText,
|
AppliedCorrectedText: change.CorrectedText,
|
||||||
ReplacementPolicy: string(module.ReplacementPolicy),
|
ReplacementPolicy: string(module.ReplacementPolicy),
|
||||||
Disposition: correctionDispositionApplied,
|
Disposition: correctionDispositionApplied,
|
||||||
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false, llmBacked),
|
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false),
|
||||||
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true, llmBacked),
|
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
for _, change := range module.SkippedChanges {
|
for _, change := range module.SkippedChanges {
|
||||||
@@ -89,8 +84,8 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
Disposition: correctionDispositionSkipped,
|
Disposition: correctionDispositionSkipped,
|
||||||
DispositionReasonCode: string(change.SkipReason),
|
DispositionReasonCode: string(change.SkipReason),
|
||||||
DispositionMessage: change.Message,
|
DispositionMessage: change.Message,
|
||||||
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false, llmBacked),
|
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false),
|
||||||
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true, llmBacked),
|
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
for _, rejection := range module.ValidatorRejected {
|
for _, rejection := range module.ValidatorRejected {
|
||||||
@@ -106,8 +101,8 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
Disposition: correctionDispositionRejected,
|
Disposition: correctionDispositionRejected,
|
||||||
DispositionReasonCode: rejection.ReasonCode,
|
DispositionReasonCode: rejection.ReasonCode,
|
||||||
DispositionMessage: rejection.Message,
|
DispositionMessage: rejection.Message,
|
||||||
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], false, llmBacked),
|
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], false),
|
||||||
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], true, llmBacked),
|
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], true),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if module.Status == runner.ModuleStatusFailed {
|
if module.Status == runner.ModuleStatusFailed {
|
||||||
@@ -135,13 +130,14 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
|||||||
return entries
|
return entries
|
||||||
}
|
}
|
||||||
|
|
||||||
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool, llmBacked map[string]bool) []ledgerValidatorDecisionRecord {
|
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []ledgerValidatorDecisionRecord {
|
||||||
if len(in) == 0 {
|
if len(in) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
out := make([]ledgerValidatorDecisionRecord, 0, len(in))
|
out := make([]ledgerValidatorDecisionRecord, 0, len(in))
|
||||||
for _, decision := range in {
|
for _, decision := range in {
|
||||||
if llmBacked[decision.ValidatorName] != wantLLM {
|
isLLMBacked := validatormetadata.ClassForKey(decision.ValidatorName) == validatormetadata.ExecutionClassLLMBacked
|
||||||
|
if isLLMBacked != wantLLM {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
out = append(out, ledgerValidatorDecisionRecord{
|
out = append(out, ledgerValidatorDecisionRecord{
|
||||||
|
|||||||
46
internal/cli/review_artifacts_test.go
Normal file
46
internal/cli/review_artifacts_test.go
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(t *testing.T) {
|
||||||
|
output := &runner.RunOutput{
|
||||||
|
ModuleResults: []runner.ModuleResult{
|
||||||
|
{
|
||||||
|
ModuleKey: "glossary",
|
||||||
|
ModuleInstance: "glossary",
|
||||||
|
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
|
||||||
|
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
||||||
|
{ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
||||||
|
{ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
||||||
|
},
|
||||||
|
AppliedChanges: []proposals.AppliedChange{
|
||||||
|
{
|
||||||
|
ProposalIndex: 3,
|
||||||
|
ModuleKey: "glossary",
|
||||||
|
ModuleInstance: "glossary",
|
||||||
|
TargetSegmentID: 1,
|
||||||
|
OriginalText: "gestures",
|
||||||
|
CorrectedText: "Jesters",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
ledger := buildCorrectionLedger("/tmp/audita-run-id", output)
|
||||||
|
if len(ledger) != 1 {
|
||||||
|
t.Fatalf("expected one ledger entry, got %d", len(ledger))
|
||||||
|
}
|
||||||
|
entry := ledger[0]
|
||||||
|
if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "proposal_shape" {
|
||||||
|
t.Fatalf("unexpected deterministic decision split: %+v", entry.DeterministicValidatorResults)
|
||||||
|
}
|
||||||
|
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "spoken_form_plausibility" {
|
||||||
|
t.Fatalf("unexpected llm-backed decision split: %+v", entry.LLMValidatorResults)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -375,30 +375,28 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
|||||||
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
|
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
configPath, configSource, err := resolveConfigPath(configPathOverride, configPathOverrideSet, os.LookupEnv)
|
effectiveConfig, err := config.LoadEffectiveConfig(configPathOverride, configPathOverrideSet)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
var effectiveConfigErr *config.EffectiveConfigError
|
||||||
|
if errors.As(err, &effectiveConfigErr) {
|
||||||
|
switch effectiveConfigErr.Kind {
|
||||||
|
case config.EffectiveConfigErrorLoadFile, config.EffectiveConfigErrorApplyFile:
|
||||||
|
fmt.Fprintf(stderr, "audita process: invalid config file: %v\n", effectiveConfigErr)
|
||||||
|
case config.EffectiveConfigErrorApplyEnv:
|
||||||
|
fmt.Fprintf(stderr, "audita process: invalid environment configuration: %v\n", effectiveConfigErr)
|
||||||
|
default:
|
||||||
|
fmt.Fprintf(stderr, "audita process: %v\n", effectiveConfigErr)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
fmt.Fprintf(stderr, "audita process: %v\n", err)
|
fmt.Fprintf(stderr, "audita process: %v\n", err)
|
||||||
|
}
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg := config.Default()
|
cfg := effectiveConfig.Config
|
||||||
var configVersion *int
|
configPath := effectiveConfig.ConfigPath
|
||||||
if configPath != "" {
|
configSource := effectiveConfig.ConfigSource
|
||||||
fileCfg, fileErr := config.LoadFileConfig(configPath)
|
configVersion := effectiveConfig.ConfigVersion
|
||||||
if fileErr != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: invalid config file: %v\n", fileErr)
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
if applyErr := cfg.ApplyFileConfig(fileCfg); applyErr != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: invalid config file: %v\n", applyErr)
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
configVersion = &fileCfg.Version
|
|
||||||
}
|
|
||||||
if err := cfg.ApplyEnvOverrides(); err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: invalid environment configuration: %v\n", err)
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
|
|
||||||
fs, pFlags := newProcessFlagSet(cfg, stderr)
|
fs, pFlags := newProcessFlagSet(cfg, stderr)
|
||||||
|
|
||||||
@@ -531,9 +529,9 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
|||||||
if runErr != nil {
|
if runErr != nil {
|
||||||
if runDir != nil && runOutput != nil {
|
if runDir != nil && runOutput != nil {
|
||||||
if runOutput.Utilization != nil {
|
if runOutput.Utilization != nil {
|
||||||
_ = runDir.WriteJSONArtifact("utilization-diagnostics.json", runOutput.Utilization)
|
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
||||||
}
|
}
|
||||||
_ = runDir.WriteJSONArtifact("correction-ledger.json", buildCorrectionLedger(runDir.Path(), runOutput))
|
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, buildCorrectionLedger(runDir.Path(), runOutput))
|
||||||
}
|
}
|
||||||
errorPhase, errorMessage := extractErrorPhase(runErr)
|
errorPhase, errorMessage := extractErrorPhase(runErr)
|
||||||
report := buildProcessReport("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput)
|
report := buildProcessReport("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput)
|
||||||
@@ -560,9 +558,9 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
|||||||
|
|
||||||
if runDir != nil && runOutput != nil {
|
if runDir != nil && runOutput != nil {
|
||||||
if runOutput.Utilization != nil {
|
if runOutput.Utilization != nil {
|
||||||
_ = runDir.WriteJSONArtifact("utilization-diagnostics.json", runOutput.Utilization)
|
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
||||||
}
|
}
|
||||||
_ = runDir.WriteJSONArtifact("correction-ledger.json", buildCorrectionLedger(runDir.Path(), runOutput))
|
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, buildCorrectionLedger(runDir.Path(), runOutput))
|
||||||
}
|
}
|
||||||
|
|
||||||
report := buildProcessReport("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput)
|
report := buildProcessReport("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput)
|
||||||
@@ -653,6 +651,10 @@ func runConfigValidate(args []string, stdout, stderr io.Writer) int {
|
|||||||
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
|
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
|
||||||
|
return 2
|
||||||
|
}
|
||||||
fmt.Fprintln(stdout, "config is valid")
|
fmt.Fprintln(stdout, "config is valid")
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
@@ -675,28 +677,13 @@ func runConfigPrintEffective(args []string, stdout, stderr io.Writer) int {
|
|||||||
|
|
||||||
configPathValue := strings.TrimSpace(*configPath)
|
configPathValue := strings.TrimSpace(*configPath)
|
||||||
configPathSet := configPathValue != ""
|
configPathSet := configPathValue != ""
|
||||||
path, _, err := resolveConfigPath(configPathValue, configPathSet, os.LookupEnv)
|
effectiveConfig, err := config.LoadEffectiveConfig(configPathValue, configPathSet)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
|
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg := config.Default()
|
cfg := effectiveConfig.Config
|
||||||
if path != "" {
|
|
||||||
fileCfg, fileErr := config.LoadFileConfig(path)
|
|
||||||
if fileErr != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita config print-effective: %v\n", fileErr)
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
if applyErr := cfg.ApplyFileConfig(fileCfg); applyErr != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita config print-effective: %v\n", applyErr)
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := cfg.ApplyEnvOverrides(); err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
|
|
||||||
redacted := cfg.Redacted()
|
redacted := cfg.Redacted()
|
||||||
out, err := json.MarshalIndent(redacted, "", " ")
|
out, err := json.MarshalIndent(redacted, "", " ")
|
||||||
@@ -743,22 +730,9 @@ func buildProcessReport(status string, inv processInvocation, runDir *diagnostic
|
|||||||
ErrorPhase: errorPhase,
|
ErrorPhase: errorPhase,
|
||||||
}
|
}
|
||||||
if runDir != nil {
|
if runDir != nil {
|
||||||
diagnosticsDir := runDir.Path()
|
runSucceeded := status == "success"
|
||||||
report.Diagnostics = &reporting.DiagnosticsMetadata{
|
metadata := diagnostics.BuildDiagnosticsMetadata(runDir.Path(), runSucceeded)
|
||||||
DirectoryPath: diagnosticsDir,
|
report.Diagnostics = &metadata
|
||||||
SourceTranscriptPath: filepath.Join(diagnosticsDir, "source-transcript.json"),
|
|
||||||
ParsedSourceTranscriptPath: filepath.Join(diagnosticsDir, "source-transcript-parsed.json"),
|
|
||||||
NormalizedTranscriptPath: filepath.Join(diagnosticsDir, "normalized-transcript.json"),
|
|
||||||
NormalizationSummaryPath: filepath.Join(diagnosticsDir, "normalization-summary.json"),
|
|
||||||
ChunkingSummaryPath: filepath.Join(diagnosticsDir, "chunking-summary.json"),
|
|
||||||
UtilizationSummaryPath: filepath.Join(diagnosticsDir, "utilization-diagnostics.json"),
|
|
||||||
CorrectionLedgerPath: filepath.Join(diagnosticsDir, "correction-ledger.json"),
|
|
||||||
InvocationMetadataPath: filepath.Join(diagnosticsDir, "invocation.json"),
|
|
||||||
RedactedEffectiveConfigPath: filepath.Join(diagnosticsDir, "effective-config.json"),
|
|
||||||
}
|
|
||||||
if status == "failed" {
|
|
||||||
report.Diagnostics.ErrorLogPath = filepath.Join(diagnosticsDir, "error.log")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if errorMessage != "" {
|
if errorMessage != "" {
|
||||||
report.ErrorMessage = errorMessage
|
report.ErrorMessage = errorMessage
|
||||||
@@ -984,47 +958,6 @@ func findConfigPathOverride(args []string) (path string, set bool, err error) {
|
|||||||
return "", false, nil
|
return "", false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var statConfigPath = os.Stat
|
|
||||||
|
|
||||||
func resolveConfigPath(cliConfigPath string, cliConfigPathSet bool, lookup func(string) (string, bool)) (path string, source string, err error) {
|
|
||||||
if cliConfigPathSet {
|
|
||||||
path = strings.TrimSpace(cliConfigPath)
|
|
||||||
if path == "" {
|
|
||||||
return "", "", fmt.Errorf("--config requires a non-empty path")
|
|
||||||
}
|
|
||||||
if _, statErr := statConfigPath(path); statErr != nil {
|
|
||||||
if os.IsNotExist(statErr) {
|
|
||||||
return "", "", fmt.Errorf("config file not found: %s", path)
|
|
||||||
}
|
|
||||||
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
|
|
||||||
}
|
|
||||||
return path, "flag", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if raw, ok := lookup("AUDITA_CONFIG"); ok {
|
|
||||||
path = strings.TrimSpace(raw)
|
|
||||||
if path == "" {
|
|
||||||
return "", "", fmt.Errorf("AUDITA_CONFIG must not be empty")
|
|
||||||
}
|
|
||||||
if _, statErr := statConfigPath(path); statErr != nil {
|
|
||||||
if os.IsNotExist(statErr) {
|
|
||||||
return "", "", fmt.Errorf("config file not found: %s", path)
|
|
||||||
}
|
|
||||||
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
|
|
||||||
}
|
|
||||||
return path, "env", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, defaultPath := range config.DefaultConfigSearchPaths {
|
|
||||||
if _, statErr := statConfigPath(defaultPath); statErr == nil {
|
|
||||||
return defaultPath, "default", nil
|
|
||||||
} else if !os.IsNotExist(statErr) {
|
|
||||||
return "", "", fmt.Errorf("cannot access config file %s: %w", defaultPath, statErr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "", "", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func isHelpCommand(args []string) bool {
|
func isHelpCommand(args []string) bool {
|
||||||
if len(args) == 0 {
|
if len(args) == 0 {
|
||||||
return false
|
return false
|
||||||
|
|||||||
@@ -21,11 +21,11 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/testsupport"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRunRootHelp(t *testing.T) {
|
func TestRunRootHelp(t *testing.T) {
|
||||||
@@ -99,66 +99,6 @@ func TestRunProcessHelpListsExpectedFlags(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestResolveConfigPathDefaultIgnoredWhenMissing(t *testing.T) {
|
|
||||||
lookup := func(string) (string, bool) { return "", false }
|
|
||||||
path, source, err := resolveConfigPath("", false, lookup)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if path != "" || source != "" {
|
|
||||||
t.Fatalf("expected no config path/source, got path=%q source=%q", path, source)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveConfigPathDefaultPrefersUsrLocalOverEtc(t *testing.T) {
|
|
||||||
oldStat := statConfigPath
|
|
||||||
statConfigPath = func(path string) (os.FileInfo, error) {
|
|
||||||
if path == config.DefaultConfigPathUsrLocal || path == config.DefaultConfigPath {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return nil, os.ErrNotExist
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { statConfigPath = oldStat })
|
|
||||||
|
|
||||||
lookup := func(string) (string, bool) { return "", false }
|
|
||||||
path, source, err := resolveConfigPath("", false, lookup)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if source != "default" {
|
|
||||||
t.Fatalf("expected default source, got %q", source)
|
|
||||||
}
|
|
||||||
if path != config.DefaultConfigPathUsrLocal {
|
|
||||||
t.Fatalf("expected %q, got %q", config.DefaultConfigPathUsrLocal, path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveConfigPathDefaultFallsBackToEtc(t *testing.T) {
|
|
||||||
oldStat := statConfigPath
|
|
||||||
statConfigPath = func(path string) (os.FileInfo, error) {
|
|
||||||
if path == config.DefaultConfigPathUsrLocal {
|
|
||||||
return nil, os.ErrNotExist
|
|
||||||
}
|
|
||||||
if path == config.DefaultConfigPath {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return nil, os.ErrNotExist
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { statConfigPath = oldStat })
|
|
||||||
|
|
||||||
lookup := func(string) (string, bool) { return "", false }
|
|
||||||
path, source, err := resolveConfigPath("", false, lookup)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if source != "default" {
|
|
||||||
t.Fatalf("expected default source, got %q", source)
|
|
||||||
}
|
|
||||||
if path != config.DefaultConfigPath {
|
|
||||||
t.Fatalf("expected %q, got %q", config.DefaultConfigPath, path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunConfigValidateSuccess(t *testing.T) {
|
func TestRunConfigValidateSuccess(t *testing.T) {
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
@@ -210,6 +150,23 @@ func TestRunConfigValidateUnknownField(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunConfigValidateUnsupportedModuleKey(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
cfgPath := writeFile(t, "config.yml", "version: 1\npipeline:\n modules: [made_up]\n")
|
||||||
|
|
||||||
|
exitCode := Run([]string{"config", "validate", "--config", cfgPath}, &stdout, &stderr)
|
||||||
|
if exitCode == 0 {
|
||||||
|
t.Fatalf("expected failure for unsupported module key")
|
||||||
|
}
|
||||||
|
if stdout.Len() != 0 {
|
||||||
|
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "unsupported module key") {
|
||||||
|
t.Fatalf("expected unsupported module key error, got %q", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunConfigPrintEffectiveOutputsRedactedJSON(t *testing.T) {
|
func TestRunConfigPrintEffectiveOutputsRedactedJSON(t *testing.T) {
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
@@ -242,6 +199,45 @@ func TestRunConfigPrintEffectiveOutputsRedactedJSON(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunConfigPrintEffectiveAppliesFileThenEnvironment(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
cfgPath := writeFile(t, "config.yml", "version: 1\nllm:\n proposal:\n model: file-model\n")
|
||||||
|
t.Setenv("AUDITA_MODEL", "env-model")
|
||||||
|
|
||||||
|
exitCode := Run([]string{"config", "print-effective", "--config", cfgPath}, &stdout, &stderr)
|
||||||
|
if exitCode != 0 {
|
||||||
|
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
var out struct {
|
||||||
|
PrimaryLLM struct {
|
||||||
|
Model string `json:"Model"`
|
||||||
|
} `json:"PrimaryLLM"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(stdout.Bytes(), &out); err != nil {
|
||||||
|
t.Fatalf("expected valid JSON output, got error: %v output=%q", err, stdout.String())
|
||||||
|
}
|
||||||
|
if out.PrimaryLLM.Model != "env-model" {
|
||||||
|
t.Fatalf("expected env model override in print-effective output, got %q", out.PrimaryLLM.Model)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunConfigValidateIgnoresEnvironmentOverrides(t *testing.T) {
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
cfgPath := writeFile(t, "config.yml", "version: 1\n")
|
||||||
|
t.Setenv("AUDITA_MODULES", "made_up")
|
||||||
|
|
||||||
|
exitCode := Run([]string{"config", "validate", "--config", cfgPath}, &stdout, &stderr)
|
||||||
|
if exitCode != 0 {
|
||||||
|
t.Fatalf("expected success because config validate is file-only, got %d stderr=%q", exitCode, stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), "config is valid") {
|
||||||
|
t.Fatalf("expected success message, got %q", stdout.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunConfigCommandDoesNotRequireTranscriptOrGlossary(t *testing.T) {
|
func TestRunConfigCommandDoesNotRequireTranscriptOrGlossary(t *testing.T) {
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
@@ -369,8 +365,8 @@ diagnostics:
|
|||||||
|
|
||||||
func TestRunProcessEnvOverridesConfigFile(t *testing.T) {
|
func TestRunProcessEnvOverridesConfigFile(t *testing.T) {
|
||||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
"m": fakeModule{
|
"grammar": fakeModule{
|
||||||
key: "m",
|
key: "grammar",
|
||||||
policy: proposals.ReplacementPolicyRequireUnique,
|
policy: proposals.ReplacementPolicyRequireUnique,
|
||||||
validators: []contracts.Validator{
|
validators: []contracts.Validator{
|
||||||
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||||
@@ -392,7 +388,7 @@ func TestRunProcessEnvOverridesConfigFile(t *testing.T) {
|
|||||||
cfgPath := writeFile(t, "config.yml", `
|
cfgPath := writeFile(t, "config.yml", `
|
||||||
version: 1
|
version: 1
|
||||||
pipeline:
|
pipeline:
|
||||||
modules: [m]
|
modules: [grammar]
|
||||||
llm:
|
llm:
|
||||||
proposal:
|
proposal:
|
||||||
model: file-model
|
model: file-model
|
||||||
@@ -404,7 +400,7 @@ llm:
|
|||||||
fixturePath("tiny_transcript.json"),
|
fixturePath("tiny_transcript.json"),
|
||||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||||
"--config", cfgPath,
|
"--config", cfgPath,
|
||||||
"--modules", "m",
|
"--modules", "grammar",
|
||||||
}, &stdout, &stderr)
|
}, &stdout, &stderr)
|
||||||
if exitCode != 0 {
|
if exitCode != 0 {
|
||||||
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||||
@@ -413,8 +409,8 @@ llm:
|
|||||||
|
|
||||||
func TestRunProcessCLIOverridesEnvAndConfigFile(t *testing.T) {
|
func TestRunProcessCLIOverridesEnvAndConfigFile(t *testing.T) {
|
||||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
"m": fakeModule{
|
"grammar": fakeModule{
|
||||||
key: "m",
|
key: "grammar",
|
||||||
policy: proposals.ReplacementPolicyRequireUnique,
|
policy: proposals.ReplacementPolicyRequireUnique,
|
||||||
validators: []contracts.Validator{
|
validators: []contracts.Validator{
|
||||||
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||||
@@ -436,7 +432,7 @@ func TestRunProcessCLIOverridesEnvAndConfigFile(t *testing.T) {
|
|||||||
cfgPath := writeFile(t, "config.yml", `
|
cfgPath := writeFile(t, "config.yml", `
|
||||||
version: 1
|
version: 1
|
||||||
pipeline:
|
pipeline:
|
||||||
modules: [m]
|
modules: [grammar]
|
||||||
llm:
|
llm:
|
||||||
proposal:
|
proposal:
|
||||||
model: file-model
|
model: file-model
|
||||||
@@ -448,7 +444,7 @@ llm:
|
|||||||
fixturePath("tiny_transcript.json"),
|
fixturePath("tiny_transcript.json"),
|
||||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||||
"--config", cfgPath,
|
"--config", cfgPath,
|
||||||
"--modules", "m",
|
"--modules", "grammar",
|
||||||
"--model", "cli-model",
|
"--model", "cli-model",
|
||||||
}, &stdout, &stderr)
|
}, &stdout, &stderr)
|
||||||
if exitCode != 0 {
|
if exitCode != 0 {
|
||||||
@@ -495,8 +491,8 @@ diagnostics:
|
|||||||
|
|
||||||
func TestRunProcessTranscriptDescriptionCLIOverridesConfigFileContextDescription(t *testing.T) {
|
func TestRunProcessTranscriptDescriptionCLIOverridesConfigFileContextDescription(t *testing.T) {
|
||||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
"m": fakeModule{
|
"grammar": fakeModule{
|
||||||
key: "m",
|
key: "grammar",
|
||||||
policy: proposals.ReplacementPolicyRequireUnique,
|
policy: proposals.ReplacementPolicyRequireUnique,
|
||||||
validators: []contracts.Validator{
|
validators: []contracts.Validator{
|
||||||
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||||
@@ -517,7 +513,7 @@ func TestRunProcessTranscriptDescriptionCLIOverridesConfigFileContextDescription
|
|||||||
cfgPath := writeFile(t, "config.yml", `
|
cfgPath := writeFile(t, "config.yml", `
|
||||||
version: 1
|
version: 1
|
||||||
pipeline:
|
pipeline:
|
||||||
modules: [m]
|
modules: [grammar]
|
||||||
context:
|
context:
|
||||||
description: "file transcript description"
|
description: "file transcript description"
|
||||||
`)
|
`)
|
||||||
@@ -528,7 +524,7 @@ context:
|
|||||||
fixturePath("tiny_transcript.json"),
|
fixturePath("tiny_transcript.json"),
|
||||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||||
"--config", cfgPath,
|
"--config", cfgPath,
|
||||||
"--modules", "m",
|
"--modules", "grammar",
|
||||||
"--transcript-description", "cli transcript description",
|
"--transcript-description", "cli transcript description",
|
||||||
}, &stdout, &stderr)
|
}, &stdout, &stderr)
|
||||||
if exitCode != 0 {
|
if exitCode != 0 {
|
||||||
@@ -692,8 +688,8 @@ func TestRunProcessCLIOverridesEnvironment(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunProcessTranscriptDescriptionDefaultEmpty(t *testing.T) {
|
func TestRunProcessTranscriptDescriptionDefaultEmpty(t *testing.T) {
|
||||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
"m": fakeModule{
|
"grammar": fakeModule{
|
||||||
key: "m",
|
key: "grammar",
|
||||||
policy: proposals.ReplacementPolicyRequireUnique,
|
policy: proposals.ReplacementPolicyRequireUnique,
|
||||||
validators: []contracts.Validator{
|
validators: []contracts.Validator{
|
||||||
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||||
@@ -720,7 +716,7 @@ func TestRunProcessTranscriptDescriptionDefaultEmpty(t *testing.T) {
|
|||||||
exitCode := Run([]string{
|
exitCode := Run([]string{
|
||||||
"process", transcriptPath,
|
"process", transcriptPath,
|
||||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||||
"--modules", "m",
|
"--modules", "grammar",
|
||||||
}, &stdout, &stderr)
|
}, &stdout, &stderr)
|
||||||
if exitCode != 0 {
|
if exitCode != 0 {
|
||||||
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||||
@@ -729,8 +725,8 @@ func TestRunProcessTranscriptDescriptionDefaultEmpty(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunProcessTranscriptDescriptionCLIOverrideAndTrim(t *testing.T) {
|
func TestRunProcessTranscriptDescriptionCLIOverrideAndTrim(t *testing.T) {
|
||||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
"m": fakeModule{
|
"grammar": fakeModule{
|
||||||
key: "m",
|
key: "grammar",
|
||||||
policy: proposals.ReplacementPolicyRequireUnique,
|
policy: proposals.ReplacementPolicyRequireUnique,
|
||||||
validators: []contracts.Validator{
|
validators: []contracts.Validator{
|
||||||
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||||
@@ -757,7 +753,7 @@ func TestRunProcessTranscriptDescriptionCLIOverrideAndTrim(t *testing.T) {
|
|||||||
exitCode := Run([]string{
|
exitCode := Run([]string{
|
||||||
"process", transcriptPath,
|
"process", transcriptPath,
|
||||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||||
"--modules", "m",
|
"--modules", "grammar",
|
||||||
"--transcript-description", " speaker background context ",
|
"--transcript-description", " speaker background context ",
|
||||||
}, &stdout, &stderr)
|
}, &stdout, &stderr)
|
||||||
if exitCode != 0 {
|
if exitCode != 0 {
|
||||||
@@ -823,8 +819,8 @@ func TestRunProcessRejectsValidationConcurrencyAboveTotalConcurrency(t *testing.
|
|||||||
|
|
||||||
func TestRunProcessTotalLLMConcurrencyDrivesEffectiveValidationConcurrencyWhenUnset(t *testing.T) {
|
func TestRunProcessTotalLLMConcurrencyDrivesEffectiveValidationConcurrencyWhenUnset(t *testing.T) {
|
||||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
"m": fakeModule{
|
"grammar": fakeModule{
|
||||||
key: "m",
|
key: "grammar",
|
||||||
policy: proposals.ReplacementPolicyRequireUnique,
|
policy: proposals.ReplacementPolicyRequireUnique,
|
||||||
validators: []contracts.Validator{
|
validators: []contracts.Validator{
|
||||||
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||||
@@ -865,7 +861,7 @@ func TestRunProcessTotalLLMConcurrencyDrivesEffectiveValidationConcurrencyWhenUn
|
|||||||
"--glossary",
|
"--glossary",
|
||||||
fixturePath("tiny_glossary.yaml"),
|
fixturePath("tiny_glossary.yaml"),
|
||||||
"--modules",
|
"--modules",
|
||||||
"m",
|
"grammar",
|
||||||
"--total-llm-concurrency",
|
"--total-llm-concurrency",
|
||||||
"4",
|
"4",
|
||||||
}, &stdout, &stderr)
|
}, &stdout, &stderr)
|
||||||
@@ -908,8 +904,8 @@ func TestRunProcessRejectsProposalConcurrencyAboveTotalConcurrency(t *testing.T)
|
|||||||
|
|
||||||
func TestRunProcessLegacyLLMConcurrencyAliasSetsTotalAndProposal(t *testing.T) {
|
func TestRunProcessLegacyLLMConcurrencyAliasSetsTotalAndProposal(t *testing.T) {
|
||||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
"m": fakeModule{
|
"grammar": fakeModule{
|
||||||
key: "m",
|
key: "grammar",
|
||||||
policy: proposals.ReplacementPolicyRequireUnique,
|
policy: proposals.ReplacementPolicyRequireUnique,
|
||||||
validators: []contracts.Validator{
|
validators: []contracts.Validator{
|
||||||
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||||
@@ -944,7 +940,7 @@ func TestRunProcessLegacyLLMConcurrencyAliasSetsTotalAndProposal(t *testing.T) {
|
|||||||
"--glossary",
|
"--glossary",
|
||||||
fixturePath("tiny_glossary.yaml"),
|
fixturePath("tiny_glossary.yaml"),
|
||||||
"--modules",
|
"--modules",
|
||||||
"m",
|
"grammar",
|
||||||
"--llm-concurrency",
|
"--llm-concurrency",
|
||||||
"3",
|
"3",
|
||||||
}, &stdout, &stderr)
|
}, &stdout, &stderr)
|
||||||
@@ -955,8 +951,8 @@ func TestRunProcessLegacyLLMConcurrencyAliasSetsTotalAndProposal(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunProcessLLMConcurrencyFlagsOverrideEnvironment(t *testing.T) {
|
func TestRunProcessLLMConcurrencyFlagsOverrideEnvironment(t *testing.T) {
|
||||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
"m": fakeModule{
|
"grammar": fakeModule{
|
||||||
key: "m",
|
key: "grammar",
|
||||||
policy: proposals.ReplacementPolicyRequireUnique,
|
policy: proposals.ReplacementPolicyRequireUnique,
|
||||||
validators: []contracts.Validator{
|
validators: []contracts.Validator{
|
||||||
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||||
@@ -997,7 +993,7 @@ func TestRunProcessLLMConcurrencyFlagsOverrideEnvironment(t *testing.T) {
|
|||||||
"--glossary",
|
"--glossary",
|
||||||
fixturePath("tiny_glossary.yaml"),
|
fixturePath("tiny_glossary.yaml"),
|
||||||
"--modules",
|
"--modules",
|
||||||
"m",
|
"grammar",
|
||||||
"--total-llm-concurrency",
|
"--total-llm-concurrency",
|
||||||
"4",
|
"4",
|
||||||
"--proposal-llm-concurrency",
|
"--proposal-llm-concurrency",
|
||||||
@@ -1010,8 +1006,8 @@ func TestRunProcessLLMConcurrencyFlagsOverrideEnvironment(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunProcessAcceptsLLMConcurrencyEnvironmentVariables(t *testing.T) {
|
func TestRunProcessAcceptsLLMConcurrencyEnvironmentVariables(t *testing.T) {
|
||||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
"m": fakeModule{
|
"grammar": fakeModule{
|
||||||
key: "m",
|
key: "grammar",
|
||||||
policy: proposals.ReplacementPolicyRequireUnique,
|
policy: proposals.ReplacementPolicyRequireUnique,
|
||||||
validators: []contracts.Validator{
|
validators: []contracts.Validator{
|
||||||
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||||
@@ -1052,7 +1048,7 @@ func TestRunProcessAcceptsLLMConcurrencyEnvironmentVariables(t *testing.T) {
|
|||||||
"--glossary",
|
"--glossary",
|
||||||
fixturePath("tiny_glossary.yaml"),
|
fixturePath("tiny_glossary.yaml"),
|
||||||
"--modules",
|
"--modules",
|
||||||
"m",
|
"grammar",
|
||||||
}, &stdout, &stderr)
|
}, &stdout, &stderr)
|
||||||
if exitCode != 0 {
|
if exitCode != 0 {
|
||||||
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||||
@@ -1883,12 +1879,12 @@ func TestRunProcessInjectedFactoryExecutesRunnerAndReportsModules(t *testing.T)
|
|||||||
return validators.Result{ValidatorName: "allow", Decisions: decisions}, nil
|
return validators.Result{ValidatorName: "allow", Decisions: decisions}, nil
|
||||||
}}
|
}}
|
||||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
"glossary": fakeModule{key: "glossary", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||||
return []proposals.CorrectionProposal{
|
return []proposals.CorrectionProposal{
|
||||||
{TargetSegmentID: 1, OriginalText: "Hello", CorrectedText: "Hi", Confidence: 1},
|
{TargetSegmentID: 1, OriginalText: "Hello", CorrectedText: "Hi", Confidence: 1},
|
||||||
}, nil
|
}, nil
|
||||||
}},
|
}},
|
||||||
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
"homophones": fakeModule{key: "homophones", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||||
if req.WorkingTranscript.Segments[0].Text != "Hi world" {
|
if req.WorkingTranscript.Segments[0].Text != "Hi world" {
|
||||||
t.Fatalf("expected module 2 to see module 1 changes, got %q", req.WorkingTranscript.Segments[0].Text)
|
t.Fatalf("expected module 2 to see module 1 changes, got %q", req.WorkingTranscript.Segments[0].Text)
|
||||||
}
|
}
|
||||||
@@ -1911,7 +1907,7 @@ func TestRunProcessInjectedFactoryExecutesRunnerAndReportsModules(t *testing.T)
|
|||||||
exitCode := Run([]string{
|
exitCode := Run([]string{
|
||||||
"process", transcriptPath,
|
"process", transcriptPath,
|
||||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||||
"--modules", "m1,m2",
|
"--modules", "glossary,homophones",
|
||||||
"--output", outputPath,
|
"--output", outputPath,
|
||||||
"--report-json", reportPath,
|
"--report-json", reportPath,
|
||||||
"--work-dir", workDir,
|
"--work-dir", workDir,
|
||||||
@@ -1970,7 +1966,7 @@ func TestRunProcessInjectedFactoryLLMValidatorResultsInReports(t *testing.T) {
|
|||||||
}},
|
}},
|
||||||
}
|
}
|
||||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
"grammar": fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "Hello", CorrectedText: "Hi", Confidence: 1}}, nil
|
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "Hello", CorrectedText: "Hi", Confidence: 1}}, nil
|
||||||
}},
|
}},
|
||||||
}}
|
}}
|
||||||
@@ -1990,7 +1986,7 @@ func TestRunProcessInjectedFactoryLLMValidatorResultsInReports(t *testing.T) {
|
|||||||
exitCode := Run([]string{
|
exitCode := Run([]string{
|
||||||
"process", transcriptPath,
|
"process", transcriptPath,
|
||||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||||
"--modules", "m",
|
"--modules", "grammar",
|
||||||
"--output", outputPath,
|
"--output", outputPath,
|
||||||
"--report-json", reportPath,
|
"--report-json", reportPath,
|
||||||
"--work-dir", workDir,
|
"--work-dir", workDir,
|
||||||
@@ -2011,7 +2007,7 @@ func TestRunProcessInjectedFactoryLLMValidatorResultsInReports(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunProcessInjectedFactorySkippedKeepsAutoRetention(t *testing.T) {
|
func TestRunProcessInjectedFactorySkippedKeepsAutoRetention(t *testing.T) {
|
||||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
"glossary": fakeModule{key: "glossary", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||||
return []proposals.CorrectionProposal{
|
return []proposals.CorrectionProposal{
|
||||||
{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term", Confidence: 1},
|
{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term", Confidence: 1},
|
||||||
}, nil
|
}, nil
|
||||||
@@ -2027,7 +2023,7 @@ func TestRunProcessInjectedFactorySkippedKeepsAutoRetention(t *testing.T) {
|
|||||||
exitCode := Run([]string{
|
exitCode := Run([]string{
|
||||||
"process", transcriptPath,
|
"process", transcriptPath,
|
||||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||||
"--modules", "m1",
|
"--modules", "glossary",
|
||||||
"--work-dir", workDir,
|
"--work-dir", workDir,
|
||||||
"--work-dir-retention", "auto",
|
"--work-dir-retention", "auto",
|
||||||
}, &stdout, &stderr)
|
}, &stdout, &stderr)
|
||||||
@@ -2041,7 +2037,7 @@ func TestRunProcessInjectedFactorySkippedKeepsAutoRetention(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunProcessInjectedFactoryFailureWritesFailedReport(t *testing.T) {
|
func TestRunProcessInjectedFactoryFailureWritesFailedReport(t *testing.T) {
|
||||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
"glossary": fakeModule{key: "glossary", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||||
return nil, errors.New("test failure")
|
return nil, errors.New("test failure")
|
||||||
}},
|
}},
|
||||||
}}
|
}}
|
||||||
@@ -2057,7 +2053,7 @@ func TestRunProcessInjectedFactoryFailureWritesFailedReport(t *testing.T) {
|
|||||||
exitCode := Run([]string{
|
exitCode := Run([]string{
|
||||||
"process", transcriptPath,
|
"process", transcriptPath,
|
||||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||||
"--modules", "m1",
|
"--modules", "glossary",
|
||||||
"--work-dir", workDir,
|
"--work-dir", workDir,
|
||||||
"--work-dir-retention", "always",
|
"--work-dir-retention", "always",
|
||||||
"--report-json", reportPath,
|
"--report-json", reportPath,
|
||||||
@@ -2089,14 +2085,8 @@ func TestRunProcessInjectedFactoryFailureWritesFailedReport(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunProcessProductionRegistryUnsupportedModuleFailsCleanly(t *testing.T) {
|
func TestRunProcessUnsupportedModuleFailsDuringConfigValidation(t *testing.T) {
|
||||||
cfg := modules.Dependencies{}
|
|
||||||
processModuleFactory = modules.NewFactory(cfg)
|
|
||||||
t.Cleanup(func() { processModuleFactory = nil })
|
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
workDir := t.TempDir()
|
|
||||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
|
||||||
transcriptPath := writeFile(t, "transcript.json", `[
|
transcriptPath := writeFile(t, "transcript.json", `[
|
||||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}
|
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}
|
||||||
]`)
|
]`)
|
||||||
@@ -2105,9 +2095,6 @@ func TestRunProcessProductionRegistryUnsupportedModuleFailsCleanly(t *testing.T)
|
|||||||
"process", transcriptPath,
|
"process", transcriptPath,
|
||||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||||
"--modules", "made_up",
|
"--modules", "made_up",
|
||||||
"--work-dir", workDir,
|
|
||||||
"--work-dir-retention", "always",
|
|
||||||
"--report-json", reportPath,
|
|
||||||
}, &stdout, &stderr)
|
}, &stdout, &stderr)
|
||||||
if exitCode == 0 {
|
if exitCode == 0 {
|
||||||
t.Fatal("expected failure exit code")
|
t.Fatal("expected failure exit code")
|
||||||
@@ -2115,23 +2102,12 @@ func TestRunProcessProductionRegistryUnsupportedModuleFailsCleanly(t *testing.T)
|
|||||||
if stdout.Len() != 0 {
|
if stdout.Len() != 0 {
|
||||||
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
|
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
|
||||||
}
|
}
|
||||||
if !strings.Contains(stderr.String(), "runner_execution") {
|
if !strings.Contains(stderr.String(), "invalid CLI configuration") {
|
||||||
t.Fatalf("expected runner_execution failure on stderr, got %q", stderr.String())
|
t.Fatalf("expected config validation failure on stderr, got %q", stderr.String())
|
||||||
}
|
}
|
||||||
if !strings.Contains(stderr.String(), "unsupported module key") {
|
if !strings.Contains(stderr.String(), "unsupported module key") {
|
||||||
t.Fatalf("expected explicit unsupported module message, got %q", stderr.String())
|
t.Fatalf("expected explicit unsupported module message, got %q", stderr.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
report := readProcessReport(t, reportPath)
|
|
||||||
if report.Status != "failed" {
|
|
||||||
t.Fatalf("expected failed report status, got %q", report.Status)
|
|
||||||
}
|
|
||||||
if report.ErrorPhase != "runner_execution" {
|
|
||||||
t.Fatalf("expected runner_execution phase, got %q", report.ErrorPhase)
|
|
||||||
}
|
|
||||||
if !strings.Contains(report.ErrorMessage, "unsupported module key") {
|
|
||||||
t.Fatalf("expected report error message to mention unsupported module, got %q", report.ErrorMessage)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunProcessExplicitUnsupportedModulesFailClearly(t *testing.T) {
|
func TestRunProcessExplicitUnsupportedModulesFailClearly(t *testing.T) {
|
||||||
@@ -4360,12 +4336,7 @@ func writeFile(t *testing.T, name string, content string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func readFile(t *testing.T, path string) []byte {
|
func readFile(t *testing.T, path string) []byte {
|
||||||
t.Helper()
|
return testsupport.ReadFile(t, path)
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read file %q: %v", path, err)
|
|
||||||
}
|
|
||||||
return data
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func readProcessReport(t *testing.T, path string) reporting.ProcessReport {
|
func readProcessReport(t *testing.T, path string) reporting.ProcessReport {
|
||||||
@@ -4379,13 +4350,5 @@ func readProcessReport(t *testing.T, path string) reporting.ProcessReport {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func onlyRunDir(t *testing.T, workDir string) string {
|
func onlyRunDir(t *testing.T, workDir string) string {
|
||||||
t.Helper()
|
return testsupport.OnlyRunDir(t, workDir)
|
||||||
entries, err := os.ReadDir(workDir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read work dir %q: %v", workDir, err)
|
|
||||||
}
|
|
||||||
if len(entries) != 1 {
|
|
||||||
t.Fatalf("expected exactly one run dir in %q, got %d", workDir, len(entries))
|
|
||||||
}
|
|
||||||
return filepath.Join(workDir, entries[0].Name())
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ package config
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||||
)
|
)
|
||||||
|
|
||||||
type WorkDirRetention string
|
type WorkDirRetention string
|
||||||
@@ -14,7 +16,7 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
DefaultModulesCSV = "glossary,homophones,glossary,spoken_word,grammar"
|
DefaultModulesCSV = modulecatalog.KeyGlossary + "," + modulecatalog.KeyHomophones + "," + modulecatalog.KeyGlossary + "," + modulecatalog.KeySpokenWord + "," + modulecatalog.KeyGrammar
|
||||||
DefaultOutputSchema = "bare-segments"
|
DefaultOutputSchema = "bare-segments"
|
||||||
DefaultPrimaryModel = "openrouter/google/gemma-4-31b-it"
|
DefaultPrimaryModel = "openrouter/google/gemma-4-31b-it"
|
||||||
DefaultPrimaryBaseURL = "https://openrouter.ai/api/v1"
|
DefaultPrimaryBaseURL = "https://openrouter.ai/api/v1"
|
||||||
|
|||||||
@@ -4,6 +4,9 @@ import (
|
|||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDefaultConfigValues(t *testing.T) {
|
func TestDefaultConfigValues(t *testing.T) {
|
||||||
@@ -318,6 +321,38 @@ func TestValidationFailures(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidationRejectsUnsupportedModuleKey(t *testing.T) {
|
||||||
|
cfg := Default()
|
||||||
|
cfg.Modules = []string{modulecatalog.KeyGlossary, "made_up"}
|
||||||
|
|
||||||
|
err := cfg.Validate()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("expected validation error for unsupported module key")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), `unsupported module key "made_up"`) {
|
||||||
|
t.Fatalf("expected unsupported module key error, got %q", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidationAllowsRepeatedSupportedModuleKeys(t *testing.T) {
|
||||||
|
cfg := Default()
|
||||||
|
cfg.Modules = []string{modulecatalog.KeyGlossary, modulecatalog.KeyGlossary, modulecatalog.KeyGrammar}
|
||||||
|
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
t.Fatalf("expected repeated supported module keys to validate, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidationAcceptsAllSupportedOutputSchemas(t *testing.T) {
|
||||||
|
for _, schemaKey := range outputschema.SupportedKeys() {
|
||||||
|
cfg := Default()
|
||||||
|
cfg.OutputSchema = schemaKey
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
t.Fatalf("expected output schema %q to validate, got %v", schemaKey, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestEffectiveValidationLLMInheritance(t *testing.T) {
|
func TestEffectiveValidationLLMInheritance(t *testing.T) {
|
||||||
cfg := Default()
|
cfg := Default()
|
||||||
cfg.PrimaryLLM.APIKey = "primary-key"
|
cfg.PrimaryLLM.APIKey = "primary-key"
|
||||||
|
|||||||
119
internal/core/config/effective_config.go
Normal file
119
internal/core/config/effective_config.go
Normal file
@@ -0,0 +1,119 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type EffectiveConfigErrorKind string
|
||||||
|
|
||||||
|
const (
|
||||||
|
EffectiveConfigErrorResolvePath EffectiveConfigErrorKind = "resolve_path"
|
||||||
|
EffectiveConfigErrorLoadFile EffectiveConfigErrorKind = "load_file"
|
||||||
|
EffectiveConfigErrorApplyFile EffectiveConfigErrorKind = "apply_file"
|
||||||
|
EffectiveConfigErrorApplyEnv EffectiveConfigErrorKind = "apply_env"
|
||||||
|
)
|
||||||
|
|
||||||
|
type EffectiveConfigError struct {
|
||||||
|
Kind EffectiveConfigErrorKind
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *EffectiveConfigError) Error() string {
|
||||||
|
if e == nil || e.Err == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return e.Err.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *EffectiveConfigError) Unwrap() error {
|
||||||
|
if e == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return e.Err
|
||||||
|
}
|
||||||
|
|
||||||
|
type EffectiveConfig struct {
|
||||||
|
Config Config
|
||||||
|
ConfigPath string
|
||||||
|
ConfigSource string
|
||||||
|
ConfigVersion *int
|
||||||
|
}
|
||||||
|
|
||||||
|
func ResolveConfigPath(cliConfigPath string, cliConfigPathSet bool) (path string, source string, err error) {
|
||||||
|
return resolveConfigPathWithLookup(cliConfigPath, cliConfigPathSet, os.LookupEnv, os.Stat, DefaultConfigSearchPaths)
|
||||||
|
}
|
||||||
|
|
||||||
|
func LoadEffectiveConfig(cliConfigPath string, cliConfigPathSet bool) (EffectiveConfig, error) {
|
||||||
|
return loadEffectiveConfigWithLookup(cliConfigPath, cliConfigPathSet, os.LookupEnv, os.Stat, DefaultConfigSearchPaths)
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadEffectiveConfigWithLookup(cliConfigPath string, cliConfigPathSet bool, lookup func(string) (string, bool), statPath func(string) (os.FileInfo, error), defaultSearchPaths []string) (EffectiveConfig, error) {
|
||||||
|
configPath, configSource, err := resolveConfigPathWithLookup(cliConfigPath, cliConfigPathSet, lookup, statPath, defaultSearchPaths)
|
||||||
|
if err != nil {
|
||||||
|
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorResolvePath, Err: err}
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := Default()
|
||||||
|
var configVersion *int
|
||||||
|
if configPath != "" {
|
||||||
|
fileCfg, fileErr := LoadFileConfig(configPath)
|
||||||
|
if fileErr != nil {
|
||||||
|
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorLoadFile, Err: fileErr}
|
||||||
|
}
|
||||||
|
if applyErr := cfg.ApplyFileConfig(fileCfg); applyErr != nil {
|
||||||
|
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorApplyFile, Err: applyErr}
|
||||||
|
}
|
||||||
|
configVersion = &fileCfg.Version
|
||||||
|
}
|
||||||
|
if applyEnvErr := cfg.applyEnvOverrides(lookup); applyEnvErr != nil {
|
||||||
|
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorApplyEnv, Err: applyEnvErr}
|
||||||
|
}
|
||||||
|
|
||||||
|
return EffectiveConfig{
|
||||||
|
Config: cfg,
|
||||||
|
ConfigPath: configPath,
|
||||||
|
ConfigSource: configSource,
|
||||||
|
ConfigVersion: configVersion,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveConfigPathWithLookup(cliConfigPath string, cliConfigPathSet bool, lookup func(string) (string, bool), statPath func(string) (os.FileInfo, error), defaultSearchPaths []string) (path string, source string, err error) {
|
||||||
|
if cliConfigPathSet {
|
||||||
|
path = strings.TrimSpace(cliConfigPath)
|
||||||
|
if path == "" {
|
||||||
|
return "", "", fmt.Errorf("--config requires a non-empty path")
|
||||||
|
}
|
||||||
|
if _, statErr := statPath(path); statErr != nil {
|
||||||
|
if os.IsNotExist(statErr) {
|
||||||
|
return "", "", fmt.Errorf("config file not found: %s", path)
|
||||||
|
}
|
||||||
|
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
|
||||||
|
}
|
||||||
|
return path, "flag", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if raw, ok := lookup("AUDITA_CONFIG"); ok {
|
||||||
|
path = strings.TrimSpace(raw)
|
||||||
|
if path == "" {
|
||||||
|
return "", "", fmt.Errorf("AUDITA_CONFIG must not be empty")
|
||||||
|
}
|
||||||
|
if _, statErr := statPath(path); statErr != nil {
|
||||||
|
if os.IsNotExist(statErr) {
|
||||||
|
return "", "", fmt.Errorf("config file not found: %s", path)
|
||||||
|
}
|
||||||
|
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
|
||||||
|
}
|
||||||
|
return path, "env", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, defaultPath := range defaultSearchPaths {
|
||||||
|
if _, statErr := statPath(defaultPath); statErr == nil {
|
||||||
|
return defaultPath, "default", nil
|
||||||
|
} else if !os.IsNotExist(statErr) {
|
||||||
|
return "", "", fmt.Errorf("cannot access config file %s: %w", defaultPath, statErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", "", nil
|
||||||
|
}
|
||||||
163
internal/core/config/effective_config_test.go
Normal file
163
internal/core/config/effective_config_test.go
Normal file
@@ -0,0 +1,163 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResolveConfigPathWithLookupMatrix(t *testing.T) {
|
||||||
|
statFor := func(existing map[string]bool) func(string) (os.FileInfo, error) {
|
||||||
|
return func(path string) (os.FileInfo, error) {
|
||||||
|
if existing[path] {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, os.ErrNotExist
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
cliPath string
|
||||||
|
cliPathSet bool
|
||||||
|
lookup func(string) (string, bool)
|
||||||
|
stat func(string) (os.FileInfo, error)
|
||||||
|
defaultSearchPaths []string
|
||||||
|
wantPath string
|
||||||
|
wantSource string
|
||||||
|
wantErrContains string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "explicit config path",
|
||||||
|
cliPath: "/tmp/explicit.yml",
|
||||||
|
cliPathSet: true,
|
||||||
|
lookup: func(string) (string, bool) { return "", false },
|
||||||
|
stat: statFor(map[string]bool{"/tmp/explicit.yml": true}),
|
||||||
|
defaultSearchPaths: []string{
|
||||||
|
"/usr/local/etc/audita/config.yml",
|
||||||
|
"/etc/audita/config.yml",
|
||||||
|
},
|
||||||
|
wantPath: "/tmp/explicit.yml",
|
||||||
|
wantSource: "flag",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "env config path",
|
||||||
|
cliPathSet: false,
|
||||||
|
lookup: func(key string) (string, bool) {
|
||||||
|
if key == "AUDITA_CONFIG" {
|
||||||
|
return "/tmp/from-env.yml", true
|
||||||
|
}
|
||||||
|
return "", false
|
||||||
|
},
|
||||||
|
stat: statFor(map[string]bool{"/tmp/from-env.yml": true}),
|
||||||
|
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
|
||||||
|
wantPath: "/tmp/from-env.yml",
|
||||||
|
wantSource: "env",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "default search path",
|
||||||
|
cliPathSet: false,
|
||||||
|
lookup: func(string) (string, bool) { return "", false },
|
||||||
|
stat: statFor(map[string]bool{
|
||||||
|
"/usr/local/etc/audita/config.yml": true,
|
||||||
|
"/etc/audita/config.yml": true,
|
||||||
|
}),
|
||||||
|
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
|
||||||
|
wantPath: "/usr/local/etc/audita/config.yml",
|
||||||
|
wantSource: "default",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "explicit missing path",
|
||||||
|
cliPath: "/tmp/missing.yml",
|
||||||
|
cliPathSet: true,
|
||||||
|
lookup: func(string) (string, bool) { return "", false },
|
||||||
|
stat: statFor(map[string]bool{}),
|
||||||
|
defaultSearchPaths: []string{
|
||||||
|
"/usr/local/etc/audita/config.yml",
|
||||||
|
"/etc/audita/config.yml",
|
||||||
|
},
|
||||||
|
wantErrContains: "config file not found",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing env path",
|
||||||
|
cliPathSet: false,
|
||||||
|
lookup: func(key string) (string, bool) {
|
||||||
|
if key == "AUDITA_CONFIG" {
|
||||||
|
return "/tmp/missing-from-env.yml", true
|
||||||
|
}
|
||||||
|
return "", false
|
||||||
|
},
|
||||||
|
stat: statFor(map[string]bool{}),
|
||||||
|
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
|
||||||
|
wantErrContains: "config file not found",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing default paths",
|
||||||
|
cliPathSet: false,
|
||||||
|
lookup: func(string) (string, bool) { return "", false },
|
||||||
|
stat: statFor(map[string]bool{}),
|
||||||
|
defaultSearchPaths: []string{
|
||||||
|
"/usr/local/etc/audita/config.yml",
|
||||||
|
"/etc/audita/config.yml",
|
||||||
|
},
|
||||||
|
wantPath: "",
|
||||||
|
wantSource: "",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
gotPath, gotSource, err := resolveConfigPathWithLookup(tc.cliPath, tc.cliPathSet, tc.lookup, tc.stat, tc.defaultSearchPaths)
|
||||||
|
if tc.wantErrContains != "" {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tc.wantErrContains) {
|
||||||
|
t.Fatalf("expected error containing %q, got %v", tc.wantErrContains, err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if gotPath != tc.wantPath || gotSource != tc.wantSource {
|
||||||
|
t.Fatalf("unexpected result: got path=%q source=%q, want path=%q source=%q", gotPath, gotSource, tc.wantPath, tc.wantSource)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadEffectiveConfigWithLookupAppliesDefaultsFileThenEnv(t *testing.T) {
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
configPath := filepath.Join(tempDir, "config.yml")
|
||||||
|
configYAML := "version: 1\nllm:\n proposal:\n model: file-model\n"
|
||||||
|
if err := os.WriteFile(configPath, []byte(configYAML), 0o644); err != nil {
|
||||||
|
t.Fatalf("write config file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
lookup := func(key string) (string, bool) {
|
||||||
|
switch key {
|
||||||
|
case "AUDITA_CONFIG":
|
||||||
|
return configPath, true
|
||||||
|
case "AUDITA_MODEL":
|
||||||
|
return "env-model", true
|
||||||
|
default:
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := loadEffectiveConfigWithLookup("", false, lookup, os.Stat, DefaultConfigSearchPaths)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("loadEffectiveConfigWithLookup error: %v", err)
|
||||||
|
}
|
||||||
|
if result.ConfigPath != configPath {
|
||||||
|
t.Fatalf("unexpected config path: %q", result.ConfigPath)
|
||||||
|
}
|
||||||
|
if result.ConfigSource != "env" {
|
||||||
|
t.Fatalf("unexpected config source: %q", result.ConfigSource)
|
||||||
|
}
|
||||||
|
if result.ConfigVersion == nil || *result.ConfigVersion != SupportedFileConfigVersion {
|
||||||
|
t.Fatalf("unexpected config version: %#v", result.ConfigVersion)
|
||||||
|
}
|
||||||
|
if result.Config.PrimaryLLM.Model != "env-model" {
|
||||||
|
t.Fatalf("expected env override to win over file value, got %q", result.Config.PrimaryLLM.Model)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,6 +3,9 @@ package config
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (c Config) Validate() error {
|
func (c Config) Validate() error {
|
||||||
@@ -12,20 +15,20 @@ func (c Config) Validate() error {
|
|||||||
issues = append(issues, "modules must not be empty")
|
issues = append(issues, "modules must not be empty")
|
||||||
}
|
}
|
||||||
for _, module := range c.Modules {
|
for _, module := range c.Modules {
|
||||||
if strings.TrimSpace(module) == "" {
|
moduleKey := strings.TrimSpace(module)
|
||||||
|
if moduleKey == "" {
|
||||||
issues = append(issues, "modules must not contain empty values")
|
issues = append(issues, "modules must not contain empty values")
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
if !modulecatalog.IsSupported(moduleKey) {
|
||||||
|
issues = append(issues, fmt.Sprintf("unsupported module key %q", moduleKey))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(c.OutputSchema) == "" {
|
if strings.TrimSpace(c.OutputSchema) == "" {
|
||||||
issues = append(issues, "output schema must not be empty")
|
issues = append(issues, "output schema must not be empty")
|
||||||
} else {
|
} else if !outputschema.IsSupported(c.OutputSchema) {
|
||||||
switch strings.TrimSpace(c.OutputSchema) {
|
|
||||||
case "bare-segments", "audita-v1":
|
|
||||||
default:
|
|
||||||
issues = append(issues, fmt.Sprintf("unsupported output schema %q", c.OutputSchema))
|
issues = append(issues, fmt.Sprintf("unsupported output schema %q", c.OutputSchema))
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if c.PrimaryLLM.TimeoutSeconds <= 0 {
|
if c.PrimaryLLM.TimeoutSeconds <= 0 {
|
||||||
issues = append(issues, "primary llm timeout seconds must be greater than zero")
|
issues = append(issues, "primary llm timeout seconds must be greater than zero")
|
||||||
|
|||||||
42
internal/core/diagnostics/artifacts.go
Normal file
42
internal/core/diagnostics/artifacts.go
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
package diagnostics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
ArtifactSourceTranscript = "source-transcript.json"
|
||||||
|
ArtifactParsedSourceTranscript = "source-transcript-parsed.json"
|
||||||
|
ArtifactNormalizedTranscript = "normalized-transcript.json"
|
||||||
|
ArtifactNormalizationSummary = "normalization-summary.json"
|
||||||
|
ArtifactChunkingSummary = "chunking-summary.json"
|
||||||
|
ArtifactUtilizationSummary = "utilization-diagnostics.json"
|
||||||
|
ArtifactCorrectionLedger = "correction-ledger.json"
|
||||||
|
ArtifactInvocationMetadata = "invocation.json"
|
||||||
|
ArtifactEffectiveConfig = "effective-config.json"
|
||||||
|
ArtifactReport = "report.json"
|
||||||
|
ArtifactErrorLog = "error.log"
|
||||||
|
)
|
||||||
|
|
||||||
|
func BuildDiagnosticsMetadata(runDirectoryPath string, runSucceeded bool) reporting.DiagnosticsMetadata {
|
||||||
|
metadata := reporting.DiagnosticsMetadata{
|
||||||
|
DirectoryPath: runDirectoryPath,
|
||||||
|
SourceTranscriptPath: filepath.Join(runDirectoryPath, ArtifactSourceTranscript),
|
||||||
|
ParsedSourceTranscriptPath: filepath.Join(runDirectoryPath, ArtifactParsedSourceTranscript),
|
||||||
|
NormalizedTranscriptPath: filepath.Join(runDirectoryPath, ArtifactNormalizedTranscript),
|
||||||
|
NormalizationSummaryPath: filepath.Join(runDirectoryPath, ArtifactNormalizationSummary),
|
||||||
|
ChunkingSummaryPath: filepath.Join(runDirectoryPath, ArtifactChunkingSummary),
|
||||||
|
UtilizationSummaryPath: filepath.Join(runDirectoryPath, ArtifactUtilizationSummary),
|
||||||
|
CorrectionLedgerPath: filepath.Join(runDirectoryPath, ArtifactCorrectionLedger),
|
||||||
|
InvocationMetadataPath: filepath.Join(runDirectoryPath, ArtifactInvocationMetadata),
|
||||||
|
RedactedEffectiveConfigPath: filepath.Join(runDirectoryPath, ArtifactEffectiveConfig),
|
||||||
|
}
|
||||||
|
|
||||||
|
if !runSucceeded {
|
||||||
|
metadata.ErrorLogPath = filepath.Join(runDirectoryPath, ArtifactErrorLog)
|
||||||
|
}
|
||||||
|
|
||||||
|
return metadata
|
||||||
|
}
|
||||||
55
internal/core/diagnostics/artifacts_test.go
Normal file
55
internal/core/diagnostics/artifacts_test.go
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
package diagnostics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBuildDiagnosticsMetadataSuccessPathsMatchArtifactConstants(t *testing.T) {
|
||||||
|
runPath := filepath.Join("tmp", "run-123")
|
||||||
|
metadata := BuildDiagnosticsMetadata(runPath, true)
|
||||||
|
|
||||||
|
if metadata.DirectoryPath != runPath {
|
||||||
|
t.Fatalf("unexpected diagnostics directory path: got=%q want=%q", metadata.DirectoryPath, runPath)
|
||||||
|
}
|
||||||
|
if metadata.SourceTranscriptPath != filepath.Join(runPath, ArtifactSourceTranscript) {
|
||||||
|
t.Fatalf("unexpected source transcript path: %q", metadata.SourceTranscriptPath)
|
||||||
|
}
|
||||||
|
if metadata.ParsedSourceTranscriptPath != filepath.Join(runPath, ArtifactParsedSourceTranscript) {
|
||||||
|
t.Fatalf("unexpected parsed source transcript path: %q", metadata.ParsedSourceTranscriptPath)
|
||||||
|
}
|
||||||
|
if metadata.NormalizedTranscriptPath != filepath.Join(runPath, ArtifactNormalizedTranscript) {
|
||||||
|
t.Fatalf("unexpected normalized transcript path: %q", metadata.NormalizedTranscriptPath)
|
||||||
|
}
|
||||||
|
if metadata.NormalizationSummaryPath != filepath.Join(runPath, ArtifactNormalizationSummary) {
|
||||||
|
t.Fatalf("unexpected normalization summary path: %q", metadata.NormalizationSummaryPath)
|
||||||
|
}
|
||||||
|
if metadata.ChunkingSummaryPath != filepath.Join(runPath, ArtifactChunkingSummary) {
|
||||||
|
t.Fatalf("unexpected chunking summary path: %q", metadata.ChunkingSummaryPath)
|
||||||
|
}
|
||||||
|
if metadata.UtilizationSummaryPath != filepath.Join(runPath, ArtifactUtilizationSummary) {
|
||||||
|
t.Fatalf("unexpected utilization summary path: %q", metadata.UtilizationSummaryPath)
|
||||||
|
}
|
||||||
|
if metadata.CorrectionLedgerPath != filepath.Join(runPath, ArtifactCorrectionLedger) {
|
||||||
|
t.Fatalf("unexpected correction ledger path: %q", metadata.CorrectionLedgerPath)
|
||||||
|
}
|
||||||
|
if metadata.InvocationMetadataPath != filepath.Join(runPath, ArtifactInvocationMetadata) {
|
||||||
|
t.Fatalf("unexpected invocation metadata path: %q", metadata.InvocationMetadataPath)
|
||||||
|
}
|
||||||
|
if metadata.RedactedEffectiveConfigPath != filepath.Join(runPath, ArtifactEffectiveConfig) {
|
||||||
|
t.Fatalf("unexpected redacted effective config path: %q", metadata.RedactedEffectiveConfigPath)
|
||||||
|
}
|
||||||
|
if metadata.ErrorLogPath != "" {
|
||||||
|
t.Fatalf("did not expect error log path on success: %q", metadata.ErrorLogPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildDiagnosticsMetadataFailureIncludesErrorLogPath(t *testing.T) {
|
||||||
|
runPath := filepath.Join("tmp", "run-123")
|
||||||
|
metadata := BuildDiagnosticsMetadata(runPath, false)
|
||||||
|
|
||||||
|
want := filepath.Join(runPath, ArtifactErrorLog)
|
||||||
|
if metadata.ErrorLogPath != want {
|
||||||
|
t.Fatalf("unexpected error log path: got=%q want=%q", metadata.ErrorLogPath, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -106,7 +106,7 @@ func (r *RunDirectory) WriteInvocationMetadata(metadata InvocationMetadata) erro
|
|||||||
metadata.StartedAt = r.createdAt
|
metadata.StartedAt = r.createdAt
|
||||||
}
|
}
|
||||||
|
|
||||||
path := filepath.Join(r.path, "invocation.json")
|
path := filepath.Join(r.path, ArtifactInvocationMetadata)
|
||||||
bytes, err := json.MarshalIndent(metadata, "", " ")
|
bytes, err := json.MarshalIndent(metadata, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to marshal invocation metadata: %w", err)
|
return fmt.Errorf("failed to marshal invocation metadata: %w", err)
|
||||||
@@ -120,7 +120,7 @@ func (r *RunDirectory) WriteInvocationMetadata(metadata InvocationMetadata) erro
|
|||||||
|
|
||||||
// WriteEffectiveConfig writes redacted effective config metadata for this run.
|
// WriteEffectiveConfig writes redacted effective config metadata for this run.
|
||||||
func (r *RunDirectory) WriteEffectiveConfig(cfg config.Config) error {
|
func (r *RunDirectory) WriteEffectiveConfig(cfg config.Config) error {
|
||||||
path := filepath.Join(r.path, "effective-config.json")
|
path := filepath.Join(r.path, ArtifactEffectiveConfig)
|
||||||
redacted := cfg.Redacted()
|
redacted := cfg.Redacted()
|
||||||
bytes, err := json.MarshalIndent(redacted, "", " ")
|
bytes, err := json.MarshalIndent(redacted, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -136,13 +136,13 @@ func (r *RunDirectory) WriteEffectiveConfig(cfg config.Config) error {
|
|||||||
// WriteSourceTranscript writes the source transcript artifact
|
// WriteSourceTranscript writes the source transcript artifact
|
||||||
func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript, raw []byte) error {
|
func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript, raw []byte) error {
|
||||||
// Write raw source for reference
|
// Write raw source for reference
|
||||||
sourcePath := filepath.Join(r.path, "source-transcript.json")
|
sourcePath := filepath.Join(r.path, ArtifactSourceTranscript)
|
||||||
if err := os.WriteFile(sourcePath, raw, 0o644); err != nil {
|
if err := os.WriteFile(sourcePath, raw, 0o644); err != nil {
|
||||||
return fmt.Errorf("failed to write source transcript: %w", err)
|
return fmt.Errorf("failed to write source transcript: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write parsed source for debugging
|
// Write parsed source for debugging
|
||||||
parsedPath := filepath.Join(r.path, "source-transcript-parsed.json")
|
parsedPath := filepath.Join(r.path, ArtifactParsedSourceTranscript)
|
||||||
parsedBytes, err := json.MarshalIndent(transcript, "", " ")
|
parsedBytes, err := json.MarshalIndent(transcript, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to marshal parsed source transcript: %w", err)
|
return fmt.Errorf("failed to marshal parsed source transcript: %w", err)
|
||||||
@@ -157,7 +157,7 @@ func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript
|
|||||||
|
|
||||||
// WriteNormalizedTranscript writes the normalized transcript artifact
|
// WriteNormalizedTranscript writes the normalized transcript artifact
|
||||||
func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript) error {
|
func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript) error {
|
||||||
normalizedPath := filepath.Join(r.path, "normalized-transcript.json")
|
normalizedPath := filepath.Join(r.path, ArtifactNormalizedTranscript)
|
||||||
bytes, err := schema.TranscriptToJSON(transcript)
|
bytes, err := schema.TranscriptToJSON(transcript)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to serialize normalized transcript: %w", err)
|
return fmt.Errorf("failed to serialize normalized transcript: %w", err)
|
||||||
@@ -170,7 +170,7 @@ func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript)
|
|||||||
|
|
||||||
// WriteNormalizationSummary writes the normalization summary artifact
|
// WriteNormalizationSummary writes the normalization summary artifact
|
||||||
func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.NormalizationSummary) error {
|
func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.NormalizationSummary) error {
|
||||||
summaryPath := filepath.Join(r.path, "normalization-summary.json")
|
summaryPath := filepath.Join(r.path, ArtifactNormalizationSummary)
|
||||||
bytes, err := json.MarshalIndent(summary, "", " ")
|
bytes, err := json.MarshalIndent(summary, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to marshal normalization summary: %w", err)
|
return fmt.Errorf("failed to marshal normalization summary: %w", err)
|
||||||
@@ -184,7 +184,7 @@ func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.Normaliz
|
|||||||
|
|
||||||
// WriteReport writes the authoritative report artifact
|
// WriteReport writes the authoritative report artifact
|
||||||
func (r *RunDirectory) WriteReport(report reporting.ProcessReport) error {
|
func (r *RunDirectory) WriteReport(report reporting.ProcessReport) error {
|
||||||
reportPath := filepath.Join(r.path, "report.json")
|
reportPath := filepath.Join(r.path, ArtifactReport)
|
||||||
bytes, err := json.MarshalIndent(report, "", " ")
|
bytes, err := json.MarshalIndent(report, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to marshal report: %w", err)
|
return fmt.Errorf("failed to marshal report: %w", err)
|
||||||
@@ -198,13 +198,13 @@ func (r *RunDirectory) WriteReport(report reporting.ProcessReport) error {
|
|||||||
|
|
||||||
// WriteErrorLog writes an error log on failure
|
// WriteErrorLog writes an error log on failure
|
||||||
func (r *RunDirectory) WriteErrorLog(errorMessage string) error {
|
func (r *RunDirectory) WriteErrorLog(errorMessage string) error {
|
||||||
errorPath := filepath.Join(r.path, "error.log")
|
errorPath := filepath.Join(r.path, ArtifactErrorLog)
|
||||||
return os.WriteFile(errorPath, []byte(errorMessage+"\n"), 0o644)
|
return os.WriteFile(errorPath, []byte(errorMessage+"\n"), 0o644)
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteChunkingSummary writes the chunking summary artifact
|
// WriteChunkingSummary writes the chunking summary artifact
|
||||||
func (r *RunDirectory) WriteChunkingSummary(summary *chunking.DetailedSummary) error {
|
func (r *RunDirectory) WriteChunkingSummary(summary *chunking.DetailedSummary) error {
|
||||||
summaryPath := filepath.Join(r.path, "chunking-summary.json")
|
summaryPath := filepath.Join(r.path, ArtifactChunkingSummary)
|
||||||
bytes, err := json.MarshalIndent(summary, "", " ")
|
bytes, err := json.MarshalIndent(summary, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to marshal chunking summary: %w", err)
|
return fmt.Errorf("failed to marshal chunking summary: %w", err)
|
||||||
|
|||||||
35
internal/core/modulecatalog/catalog.go
Normal file
35
internal/core/modulecatalog/catalog.go
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
package modulecatalog
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
|
const (
|
||||||
|
KeyGlossary = "glossary"
|
||||||
|
KeyHomophones = "homophones"
|
||||||
|
KeySpokenWord = "spoken_word"
|
||||||
|
KeyGrammar = "grammar"
|
||||||
|
)
|
||||||
|
|
||||||
|
var supportedKeys = []string{
|
||||||
|
KeyGlossary,
|
||||||
|
KeyHomophones,
|
||||||
|
KeySpokenWord,
|
||||||
|
KeyGrammar,
|
||||||
|
}
|
||||||
|
|
||||||
|
var supportedKeySet = map[string]struct{}{
|
||||||
|
KeyGlossary: {},
|
||||||
|
KeyHomophones: {},
|
||||||
|
KeySpokenWord: {},
|
||||||
|
KeyGrammar: {},
|
||||||
|
}
|
||||||
|
|
||||||
|
func SupportedKeys() []string {
|
||||||
|
out := make([]string, len(supportedKeys))
|
||||||
|
copy(out, supportedKeys)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func IsSupported(key string) bool {
|
||||||
|
_, ok := supportedKeySet[strings.TrimSpace(key)]
|
||||||
|
return ok
|
||||||
|
}
|
||||||
24
internal/core/modulecatalog/catalog_test.go
Normal file
24
internal/core/modulecatalog/catalog_test.go
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
package modulecatalog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSupportedKeys(t *testing.T) {
|
||||||
|
want := []string{KeyGlossary, KeyHomophones, KeySpokenWord, KeyGrammar}
|
||||||
|
if got := SupportedKeys(); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("unexpected supported keys: got=%v want=%v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIsSupported(t *testing.T) {
|
||||||
|
for _, key := range SupportedKeys() {
|
||||||
|
if !IsSupported(key) {
|
||||||
|
t.Fatalf("expected key %q to be supported", key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if IsSupported("made_up") {
|
||||||
|
t.Fatalf("did not expect made_up to be supported")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -31,15 +31,31 @@ var definitions = map[string]Definition{
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var supportedKeys = []string{
|
||||||
|
SchemaBareSegments,
|
||||||
|
SchemaAuditaV1,
|
||||||
|
}
|
||||||
|
|
||||||
|
func SupportedKeys() []string {
|
||||||
|
out := make([]string, len(supportedKeys))
|
||||||
|
copy(out, supportedKeys)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func IsSupported(key string) bool {
|
||||||
|
_, ok := definitions[strings.TrimSpace(key)]
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
|
||||||
func Resolve(key string) (Definition, error) {
|
func Resolve(key string) (Definition, error) {
|
||||||
normalized := strings.TrimSpace(key)
|
normalized := strings.TrimSpace(key)
|
||||||
if normalized == "" {
|
if normalized == "" {
|
||||||
return Definition{}, fmt.Errorf("output schema must not be empty")
|
return Definition{}, fmt.Errorf("output schema must not be empty")
|
||||||
}
|
}
|
||||||
def, ok := definitions[normalized]
|
if !IsSupported(normalized) {
|
||||||
if !ok {
|
|
||||||
return Definition{}, fmt.Errorf("unsupported output schema %q", normalized)
|
return Definition{}, fmt.Errorf("unsupported output schema %q", normalized)
|
||||||
}
|
}
|
||||||
|
def := definitions[normalized]
|
||||||
return def, nil
|
return def, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package outputschema
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -56,3 +57,19 @@ func TestResolveUnknown(t *testing.T) {
|
|||||||
t.Fatalf("expected unsupported output schema error, got %v", err)
|
t.Fatalf("expected unsupported output schema error, got %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSupportedKeysAndIsSupported(t *testing.T) {
|
||||||
|
want := []string{SchemaBareSegments, SchemaAuditaV1}
|
||||||
|
if got := SupportedKeys(); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("unexpected supported schema keys: got=%v want=%v", got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, key := range want {
|
||||||
|
if !IsSupported(key) {
|
||||||
|
t.Fatalf("expected schema key %q to be supported", key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if IsSupported("seriatim-intermediate") {
|
||||||
|
t.Fatalf("did not expect unsupported schema to be reported as supported")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
17
internal/framework/llm/secrets.go
Normal file
17
internal/framework/llm/secrets.go
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import "gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||||
|
|
||||||
|
// ConfiguredSecrets returns all configured LLM API-key values that should be
|
||||||
|
// redacted from diagnostics and surfaced error payloads.
|
||||||
|
func ConfiguredSecrets(cfg *config.Config) []string {
|
||||||
|
if cfg == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
effectiveValidation := cfg.EffectiveValidationLLMConfig()
|
||||||
|
return []string{
|
||||||
|
cfg.PrimaryLLM.APIKey,
|
||||||
|
cfg.ValidationLLM.APIKey,
|
||||||
|
effectiveValidation.APIKey,
|
||||||
|
}
|
||||||
|
}
|
||||||
38
internal/framework/llm/secrets_test.go
Normal file
38
internal/framework/llm/secrets_test.go
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestConfiguredSecretsNilConfig(t *testing.T) {
|
||||||
|
if got := ConfiguredSecrets(nil); got != nil {
|
||||||
|
t.Fatalf("expected nil secrets for nil config, got %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfiguredSecretsWithValidationOverride(t *testing.T) {
|
||||||
|
cfg := config.Default()
|
||||||
|
cfg.PrimaryLLM.APIKey = "primary-secret"
|
||||||
|
cfg.ValidationLLM.APIKey = "validation-secret"
|
||||||
|
|
||||||
|
got := ConfiguredSecrets(&cfg)
|
||||||
|
want := []string{"primary-secret", "validation-secret", "validation-secret"}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("unexpected secrets: got=%v want=%v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfiguredSecretsWithInheritedValidationKey(t *testing.T) {
|
||||||
|
cfg := config.Default()
|
||||||
|
cfg.PrimaryLLM.APIKey = "primary-secret"
|
||||||
|
cfg.ValidationLLM.APIKey = ""
|
||||||
|
|
||||||
|
got := ConfiguredSecrets(&cfg)
|
||||||
|
want := []string{"primary-secret", "", "primary-secret"}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("unexpected inherited secrets: got=%v want=%v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||||
glossarymodule "gitea.maximumdirect.net/eric/audita/internal/modules/glossary"
|
glossarymodule "gitea.maximumdirect.net/eric/audita/internal/modules/glossary"
|
||||||
@@ -15,28 +16,20 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
ModuleKeyGlossary = "glossary"
|
ModuleKeyGlossary = modulecatalog.KeyGlossary
|
||||||
ModuleKeyHomophones = "homophones"
|
ModuleKeyHomophones = modulecatalog.KeyHomophones
|
||||||
ModuleKeySpokenWord = "spoken_word"
|
ModuleKeySpokenWord = modulecatalog.KeySpokenWord
|
||||||
ModuleKeyGrammar = "grammar"
|
ModuleKeyGrammar = modulecatalog.KeyGrammar
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
ReasonUnsupportedModule = "unsupported_module"
|
ReasonUnsupportedModule = "unsupported_module"
|
||||||
)
|
)
|
||||||
|
|
||||||
var knownModuleKeys = map[string]struct{}{
|
|
||||||
ModuleKeyGlossary: {},
|
|
||||||
ModuleKeyHomophones: {},
|
|
||||||
ModuleKeySpokenWord: {},
|
|
||||||
ModuleKeyGrammar: {},
|
|
||||||
}
|
|
||||||
|
|
||||||
// IsKnownModuleKey reports whether a module key is recognized by the production
|
// IsKnownModuleKey reports whether a module key is recognized by the production
|
||||||
// registry scaffold.
|
// registry scaffold.
|
||||||
func IsKnownModuleKey(key string) bool {
|
func IsKnownModuleKey(key string) bool {
|
||||||
_, ok := knownModuleKeys[strings.TrimSpace(key)]
|
return modulecatalog.IsSupported(key)
|
||||||
return ok
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Dependencies holds explicit constructor dependencies for module creation.
|
// Dependencies holds explicit constructor dependencies for module creation.
|
||||||
@@ -69,7 +62,7 @@ type Factory struct {
|
|||||||
func NewFactory(deps Dependencies) *Factory {
|
func NewFactory(deps Dependencies) *Factory {
|
||||||
factory := &Factory{
|
factory := &Factory{
|
||||||
deps: deps,
|
deps: deps,
|
||||||
constructors: make(map[string]Constructor, len(knownModuleKeys)),
|
constructors: make(map[string]Constructor, len(modulecatalog.SupportedKeys())),
|
||||||
}
|
}
|
||||||
_ = factory.RegisterConstructor(ModuleKeyGlossary, constructGlossaryModule)
|
_ = factory.RegisterConstructor(ModuleKeyGlossary, constructGlossaryModule)
|
||||||
_ = factory.RegisterConstructor(ModuleKeyHomophones, constructHomophonesModule)
|
_ = factory.RegisterConstructor(ModuleKeyHomophones, constructHomophonesModule)
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||||
)
|
)
|
||||||
@@ -26,7 +27,7 @@ func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestKnownModuleKeyRecognition(t *testing.T) {
|
func TestKnownModuleKeyRecognition(t *testing.T) {
|
||||||
for _, key := range []string{ModuleKeyGlossary, ModuleKeyHomophones, ModuleKeySpokenWord, ModuleKeyGrammar} {
|
for _, key := range modulecatalog.SupportedKeys() {
|
||||||
if !IsKnownModuleKey(key) {
|
if !IsKnownModuleKey(key) {
|
||||||
t.Fatalf("expected key %q to be recognized", key)
|
t.Fatalf("expected key %q to be recognized", key)
|
||||||
}
|
}
|
||||||
|
|||||||
45
internal/framework/promptcontext/transcript_section.go
Normal file
45
internal/framework/promptcontext/transcript_section.go
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
package promptcontext
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
type transcriptSectionSegment struct {
|
||||||
|
ID int `json:"id"`
|
||||||
|
Speaker string `json:"speaker"`
|
||||||
|
Start float64 `json:"start"`
|
||||||
|
End float64 `json:"end"`
|
||||||
|
Text string `json:"text"`
|
||||||
|
Categories []string `json:"categories,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type transcriptSectionPayload struct {
|
||||||
|
SectionIndex int `json:"section_index"`
|
||||||
|
Segments []transcriptSectionSegment `json:"segments"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalTranscriptSectionJSON builds the standardized transcript-section JSON
|
||||||
|
// payload consumed by proposal prompt templates.
|
||||||
|
func MarshalTranscriptSectionJSON(transcript *schema.Transcript, sectionIndex int) ([]byte, error) {
|
||||||
|
payload := transcriptSectionPayload{
|
||||||
|
SectionIndex: sectionIndex,
|
||||||
|
Segments: make([]transcriptSectionSegment, 0),
|
||||||
|
}
|
||||||
|
|
||||||
|
if transcript != nil {
|
||||||
|
for _, s := range transcript.Segments {
|
||||||
|
payload.Segments = append(payload.Segments, transcriptSectionSegment{
|
||||||
|
ID: s.ID,
|
||||||
|
Speaker: s.Speaker,
|
||||||
|
Start: s.Start,
|
||||||
|
End: s.End,
|
||||||
|
Text: s.Text,
|
||||||
|
Categories: append([]string(nil), s.Categories...),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return json.MarshalIndent(payload, "", " ")
|
||||||
|
}
|
||||||
95
internal/framework/promptcontext/transcript_section_test.go
Normal file
95
internal/framework/promptcontext/transcript_section_test.go
Normal file
@@ -0,0 +1,95 @@
|
|||||||
|
package promptcontext
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMarshalTranscriptSectionJSONShape(t *testing.T) {
|
||||||
|
transcript := &schema.Transcript{Segments: []schema.Segment{
|
||||||
|
{ID: 1, Speaker: "A", Start: 0.1, End: 1.2, Text: "alpha", Categories: []string{"session", "intro"}},
|
||||||
|
}}
|
||||||
|
|
||||||
|
raw, err := MarshalTranscriptSectionJSON(transcript, 3)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MarshalTranscriptSectionJSON error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var decoded map[string]any
|
||||||
|
if err := json.Unmarshal(raw, &decoded); err != nil {
|
||||||
|
t.Fatalf("unmarshal: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := decoded["section_index"]; got != float64(3) {
|
||||||
|
t.Fatalf("section_index: got=%v want=%v", got, 3)
|
||||||
|
}
|
||||||
|
segments, ok := decoded["segments"].([]any)
|
||||||
|
if !ok || len(segments) != 1 {
|
||||||
|
t.Fatalf("segments shape mismatch: %T %+v", decoded["segments"], decoded["segments"])
|
||||||
|
}
|
||||||
|
first, ok := segments[0].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("segment shape mismatch: %T", segments[0])
|
||||||
|
}
|
||||||
|
if first["id"] != float64(1) || first["speaker"] != "A" || first["start"] != 0.1 || first["end"] != 1.2 || first["text"] != "alpha" {
|
||||||
|
t.Fatalf("unexpected segment fields: %+v", first)
|
||||||
|
}
|
||||||
|
cats, ok := first["categories"].([]any)
|
||||||
|
if !ok || len(cats) != 2 || cats[0] != "session" || cats[1] != "intro" {
|
||||||
|
t.Fatalf("unexpected categories: %+v", first["categories"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMarshalTranscriptSectionJSONEmptyTranscript(t *testing.T) {
|
||||||
|
raw, err := MarshalTranscriptSectionJSON(nil, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MarshalTranscriptSectionJSON error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var decoded map[string]any
|
||||||
|
if err := json.Unmarshal(raw, &decoded); err != nil {
|
||||||
|
t.Fatalf("unmarshal: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := decoded["section_index"]; got != float64(0) {
|
||||||
|
t.Fatalf("section_index: got=%v want=%v", got, 0)
|
||||||
|
}
|
||||||
|
segments, ok := decoded["segments"].([]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("segments shape mismatch: %T", decoded["segments"])
|
||||||
|
}
|
||||||
|
if len(segments) != 0 {
|
||||||
|
t.Fatalf("expected empty segments, got %d", len(segments))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMarshalTranscriptSectionJSONCopiesCategories(t *testing.T) {
|
||||||
|
transcript := &schema.Transcript{Segments: []schema.Segment{
|
||||||
|
{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "alpha", Categories: []string{"kept"}},
|
||||||
|
}}
|
||||||
|
|
||||||
|
raw, err := MarshalTranscriptSectionJSON(transcript, 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MarshalTranscriptSectionJSON error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
transcript.Segments[0].Categories[0] = "changed"
|
||||||
|
|
||||||
|
var decoded struct {
|
||||||
|
Segments []struct {
|
||||||
|
Categories []string `json:"categories"`
|
||||||
|
} `json:"segments"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &decoded); err != nil {
|
||||||
|
t.Fatalf("unmarshal: %v", err)
|
||||||
|
}
|
||||||
|
if len(decoded.Segments) != 1 {
|
||||||
|
t.Fatalf("expected one segment, got %d", len(decoded.Segments))
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(decoded.Segments[0].Categories, []string{"kept"}) {
|
||||||
|
t.Fatalf("expected copied categories, got %v", decoded.Segments[0].Categories)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -15,6 +15,8 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
|
||||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -94,7 +96,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
|||||||
|
|
||||||
stage := strings.TrimSpace(req.StageName)
|
stage := strings.TrimSpace(req.StageName)
|
||||||
if stage == "" {
|
if stage == "" {
|
||||||
stage = buildStageName(req.ModuleInstance, req.Section)
|
stage = stagename.ProposalGeneration(req.ModuleInstance, sectionIndexPtr(req.Section))
|
||||||
}
|
}
|
||||||
model := resolveModel(req.Config, req.Model)
|
model := resolveModel(req.Config, req.Model)
|
||||||
messages := append([]contracts.LLMMessage(nil), req.Messages...)
|
messages := append([]contracts.LLMMessage(nil), req.Messages...)
|
||||||
@@ -106,7 +108,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
|||||||
writer = diagnosticsWriterAdapter{
|
writer = diagnosticsWriterAdapter{
|
||||||
writer: llm.NewDiagnosticsWriter(
|
writer: llm.NewDiagnosticsWriter(
|
||||||
filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
|
filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
|
||||||
proposalGenerationSecrets(req.Config),
|
llm.ConfiguredSecrets(req.Config),
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -143,7 +145,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
|||||||
if len(req.PromptMetadata) > 0 {
|
if len(req.PromptMetadata) > 0 {
|
||||||
requestMetadata["prompt_metadata"] = req.PromptMetadata
|
requestMetadata["prompt_metadata"] = req.PromptMetadata
|
||||||
}
|
}
|
||||||
requestMetadata["response_schema"] = schemaMetadata(responseSchema)
|
requestMetadata["response_schema"] = responseSchema.DiagnosticsMap()
|
||||||
|
|
||||||
if writer != nil {
|
if writer != nil {
|
||||||
artifacts, _ = writer.WriteInteraction(
|
artifacts, _ = writer.WriteInteraction(
|
||||||
@@ -158,7 +160,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if callErr != nil {
|
if callErr != nil {
|
||||||
if isMalformedStructuredOutputError(callErr) {
|
if structuredoutput.IsMalformedError(callErr) {
|
||||||
return Result{
|
return Result{
|
||||||
Warnings: []stagewarnings.StageWarning{newMalformedProposalWarning(req.Section, artifacts, callErr)},
|
Warnings: []stagewarnings.StageWarning{newMalformedProposalWarning(req.Section, artifacts, callErr)},
|
||||||
Artifacts: artifacts,
|
Artifacts: artifacts,
|
||||||
@@ -201,23 +203,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func schemaMetadata(schema responseschema.Schema) map[string]any {
|
|
||||||
return map[string]any{
|
|
||||||
"id": schema.ID,
|
|
||||||
"version": schema.Version,
|
|
||||||
"name": schema.Name,
|
|
||||||
"sha256": schema.SHA256,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildStageName(moduleInstance string, section *contracts.SectionMetadata) string {
|
|
||||||
base := fmt.Sprintf("%s:proposal-generation", moduleInstance)
|
|
||||||
if section == nil {
|
|
||||||
return base
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s:section-%04d", base, section.Index)
|
|
||||||
}
|
|
||||||
|
|
||||||
func resolveModel(cfg *config.Config, override string) string {
|
func resolveModel(cfg *config.Config, override string) string {
|
||||||
if strings.TrimSpace(override) != "" {
|
if strings.TrimSpace(override) != "" {
|
||||||
return strings.TrimSpace(override)
|
return strings.TrimSpace(override)
|
||||||
@@ -228,17 +213,6 @@ func resolveModel(cfg *config.Config, override string) string {
|
|||||||
return llm.ResolvePrimaryConfig(*cfg).Model
|
return llm.ResolvePrimaryConfig(*cfg).Model
|
||||||
}
|
}
|
||||||
|
|
||||||
func proposalGenerationSecrets(cfg *config.Config) []string {
|
|
||||||
if cfg == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return []string{
|
|
||||||
cfg.PrimaryLLM.APIKey,
|
|
||||||
cfg.ValidationLLM.APIKey,
|
|
||||||
cfg.EffectiveValidationLLMConfig().APIKey,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func errPayload(err error) any {
|
func errPayload(err error) any {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return nil
|
return nil
|
||||||
@@ -246,27 +220,6 @@ func errPayload(err error) any {
|
|||||||
return map[string]any{"error": err.Error()}
|
return map[string]any{"error": err.Error()}
|
||||||
}
|
}
|
||||||
|
|
||||||
func isMalformedStructuredOutputError(err error) bool {
|
|
||||||
if err == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
msg := err.Error()
|
|
||||||
for _, marker := range []string{
|
|
||||||
"malformed structured output",
|
|
||||||
"decode structured output:",
|
|
||||||
"decode provider response envelope:",
|
|
||||||
"provider response missing choices",
|
|
||||||
"provider response missing assistant message content",
|
|
||||||
"provider response assistant message content is empty",
|
|
||||||
"provider response assistant message content is not valid JSON",
|
|
||||||
} {
|
|
||||||
if strings.Contains(msg, marker) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func newMalformedProposalWarning(section *contracts.SectionMetadata, artifacts InteractionArtifacts, err error) stagewarnings.StageWarning {
|
func newMalformedProposalWarning(section *contracts.SectionMetadata, artifacts InteractionArtifacts, err error) stagewarnings.StageWarning {
|
||||||
warning := stagewarnings.StageWarning{
|
warning := stagewarnings.StageWarning{
|
||||||
Scope: stagewarnings.ScopeProposalGeneration,
|
Scope: stagewarnings.ScopeProposalGeneration,
|
||||||
@@ -288,6 +241,14 @@ func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
|
|||||||
return artifacts.ResponsePayloadPath
|
return artifacts.ResponsePayloadPath
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func sectionIndexPtr(section *contracts.SectionMetadata) *int {
|
||||||
|
if section == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
index := section.Index
|
||||||
|
return &index
|
||||||
|
}
|
||||||
|
|
||||||
type diagnosticsWriterAdapter struct {
|
type diagnosticsWriterAdapter struct {
|
||||||
writer *llm.DiagnosticsWriter
|
writer *llm.DiagnosticsWriter
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -272,6 +272,23 @@ func TestGenerateCandidatesMalformedStructuredOutputReturnsWarning(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGenerateCandidatesProviderMalformedEnvelopeReturnsWarning(t *testing.T) {
|
||||||
|
client := &fakeStructuredClient{err: errors.New("provider response missing choices")}
|
||||||
|
req := defaultRequest(t)
|
||||||
|
req.LLMClient = client
|
||||||
|
|
||||||
|
result, err := GenerateCandidates(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected malformed provider envelope to downgrade to warning, got %v", err)
|
||||||
|
}
|
||||||
|
if len(result.Corrections) != 0 || len(result.Enriched) != 0 {
|
||||||
|
t.Fatalf("expected no proposals on malformed response, got %+v", result)
|
||||||
|
}
|
||||||
|
if len(result.Warnings) != 1 || result.Warnings[0].ReasonCode != "proposal_response_malformed" {
|
||||||
|
t.Fatalf("unexpected warnings: %+v", result.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
|
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
|
||||||
baseResponse := StructuredCorrectionSet{
|
baseResponse := StructuredCorrectionSet{
|
||||||
Corrections: []StructuredCorrectionProposal{
|
Corrections: []StructuredCorrectionProposal{
|
||||||
@@ -344,20 +361,22 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
|
func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||||
secret := "proposal-secret"
|
primarySecret := "proposal-primary-secret"
|
||||||
|
validationSecret := "proposal-validation-secret"
|
||||||
client := &fakeStructuredClient{
|
client := &fakeStructuredClient{
|
||||||
responses: []StructuredCorrectionSet{
|
responses: []StructuredCorrectionSet{
|
||||||
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: secret, CorrectedText: "safe", Confidence: 0.9}}},
|
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
cfg := config.Default()
|
cfg := config.Default()
|
||||||
cfg.PrimaryLLM.APIKey = secret
|
cfg.PrimaryLLM.APIKey = primarySecret
|
||||||
|
cfg.ValidationLLM.APIKey = validationSecret
|
||||||
req := defaultRequest(t)
|
req := defaultRequest(t)
|
||||||
req.Config = &cfg
|
req.Config = &cfg
|
||||||
req.LLMClient = client
|
req.LLMClient = client
|
||||||
req.DiagnosticsDir = t.TempDir()
|
req.DiagnosticsDir = t.TempDir()
|
||||||
req.Messages = []contracts.LLMMessage{
|
req.Messages = []contracts.LLMMessage{
|
||||||
{Role: "system", Content: "include secret " + secret},
|
{Role: "system", Content: "include secret " + primarySecret},
|
||||||
{Role: "user", Content: "fix it"},
|
{Role: "user", Content: "fix it"},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -374,8 +393,8 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
|
|||||||
if readErr != nil {
|
if readErr != nil {
|
||||||
t.Fatalf("read artifact %q: %v", path, readErr)
|
t.Fatalf("read artifact %q: %v", path, readErr)
|
||||||
}
|
}
|
||||||
if strings.Contains(string(raw), secret) {
|
if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) {
|
||||||
t.Fatalf("artifact leaked secret %q: %s", path, string(raw))
|
t.Fatalf("artifact leaked configured secret in %q: %s", path, string(raw))
|
||||||
}
|
}
|
||||||
if !strings.Contains(string(raw), "[REDACTED]") {
|
if !strings.Contains(string(raw), "[REDACTED]") {
|
||||||
t.Fatalf("expected redaction marker in artifact %q: %s", path, string(raw))
|
t.Fatalf("expected redaction marker in artifact %q: %s", path, string(raw))
|
||||||
|
|||||||
81
internal/framework/proposal_generation/module_proposal.go
Normal file
81
internal/framework/proposal_generation/module_proposal.go
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
package proposal_generation
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ProposalMessageBuilder func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error)
|
||||||
|
|
||||||
|
type ModuleProposalRequest struct {
|
||||||
|
ProposalRequest contracts.ProposalRequest
|
||||||
|
PromptID string
|
||||||
|
BuildMessages ProposalMessageBuilder
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExecuteModuleProposal runs shared proposal generation plumbing for one
|
||||||
|
// module, leaving only module-specific prompt message building at call sites.
|
||||||
|
func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (contracts.ProposalResult, error) {
|
||||||
|
if req.BuildMessages == nil {
|
||||||
|
return contracts.ProposalResult{}, fmt.Errorf("proposal message builder is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(req.PromptID) == "" {
|
||||||
|
return contracts.ProposalResult{}, fmt.Errorf("prompt ID must not be empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
sectionIndex := 0
|
||||||
|
if req.ProposalRequest.Section != nil {
|
||||||
|
sectionIndex = req.ProposalRequest.Section.Index
|
||||||
|
}
|
||||||
|
|
||||||
|
transcriptDescription := ""
|
||||||
|
if req.ProposalRequest.Config != nil {
|
||||||
|
transcriptDescription = req.ProposalRequest.Config.TranscriptDescription
|
||||||
|
}
|
||||||
|
|
||||||
|
messages, err := req.BuildMessages(
|
||||||
|
req.ProposalRequest.WorkingTranscript,
|
||||||
|
req.ProposalRequest.Glossary,
|
||||||
|
sectionIndex,
|
||||||
|
transcriptDescription,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ProposalResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
promptMetadata, ok := prompts.LookupMetadata(req.PromptID)
|
||||||
|
if !ok {
|
||||||
|
return contracts.ProposalResult{}, fmt.Errorf("unknown prompt ID %q", req.PromptID)
|
||||||
|
}
|
||||||
|
|
||||||
|
generated, err := GenerateCandidates(ctx, Request{
|
||||||
|
ModuleKey: req.ProposalRequest.RunSpec.ModuleKey,
|
||||||
|
ModuleInstance: req.ProposalRequest.RunSpec.InstanceName,
|
||||||
|
ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy,
|
||||||
|
WorkingTranscript: req.ProposalRequest.WorkingTranscript,
|
||||||
|
Section: req.ProposalRequest.Section,
|
||||||
|
Glossary: req.ProposalRequest.Glossary,
|
||||||
|
Config: req.ProposalRequest.Config,
|
||||||
|
Messages: messages,
|
||||||
|
PromptMetadata: promptMetadata.DiagnosticsMap(),
|
||||||
|
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
|
||||||
|
StartIndex: 0,
|
||||||
|
LLMClient: req.ProposalRequest.LLMClient,
|
||||||
|
Scheduler: req.ProposalRequest.LLMScheduler,
|
||||||
|
DiagnosticsDir: req.ProposalRequest.DiagnosticsDir,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return contracts.ProposalResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return contracts.ProposalResult{
|
||||||
|
Proposals: generated.Corrections,
|
||||||
|
Warnings: generated.Warnings,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
115
internal/framework/proposal_generation/module_proposal_test.go
Normal file
115
internal/framework/proposal_generation/module_proposal_test.go
Normal file
@@ -0,0 +1,115 @@
|
|||||||
|
package proposal_generation
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing.T) {
|
||||||
|
client := &fakeStructuredClient{
|
||||||
|
responses: []StructuredCorrectionSet{
|
||||||
|
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
section := contracts.SectionMetadata{Index: 7}
|
||||||
|
cfg := config.Default()
|
||||||
|
cfg.TranscriptDescription = "Hearing transcript with role titles."
|
||||||
|
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh"}}}
|
||||||
|
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "X"}}}
|
||||||
|
|
||||||
|
var gotSectionIndex int
|
||||||
|
var gotDescription string
|
||||||
|
var gotTranscript *schema.Transcript
|
||||||
|
var gotGlossary *schema.Glossary
|
||||||
|
|
||||||
|
out, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
|
||||||
|
ProposalRequest: contracts.ProposalRequest{
|
||||||
|
ExecutionContext: contracts.ExecutionContext{
|
||||||
|
Config: &cfg,
|
||||||
|
WorkingTranscript: transcript,
|
||||||
|
Glossary: glossary,
|
||||||
|
Section: §ion,
|
||||||
|
},
|
||||||
|
RunSpec: contracts.ModuleRunSpec{
|
||||||
|
ModuleKey: "grammar",
|
||||||
|
InstanceName: "grammar",
|
||||||
|
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||||
|
},
|
||||||
|
LLMClient: client,
|
||||||
|
},
|
||||||
|
PromptID: prompts.PromptIDModuleGrammarProposal,
|
||||||
|
BuildMessages: func(inTranscript *schema.Transcript, inGlossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||||
|
gotSectionIndex = sectionIndex
|
||||||
|
gotDescription = transcriptDescription
|
||||||
|
gotTranscript = inTranscript
|
||||||
|
gotGlossary = inGlossary
|
||||||
|
return []contracts.LLMMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}, nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ExecuteModuleProposal error: %v", err)
|
||||||
|
}
|
||||||
|
if gotSectionIndex != 7 {
|
||||||
|
t.Fatalf("section index: got=%d want=%d", gotSectionIndex, 7)
|
||||||
|
}
|
||||||
|
if gotDescription != cfg.TranscriptDescription {
|
||||||
|
t.Fatalf("transcript description: got=%q want=%q", gotDescription, cfg.TranscriptDescription)
|
||||||
|
}
|
||||||
|
if gotTranscript != transcript {
|
||||||
|
t.Fatalf("expected shared transcript pointer")
|
||||||
|
}
|
||||||
|
if gotGlossary != glossary {
|
||||||
|
t.Fatalf("expected shared glossary pointer")
|
||||||
|
}
|
||||||
|
if len(client.calls) != 1 || client.calls[0].StageName != "grammar:proposal:section-0007" {
|
||||||
|
t.Fatalf("unexpected stage name calls: %+v", client.calls)
|
||||||
|
}
|
||||||
|
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "the" {
|
||||||
|
t.Fatalf("unexpected proposals: %+v", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteModuleProposalValidatesInputs(t *testing.T) {
|
||||||
|
if _, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{}); err == nil {
|
||||||
|
t.Fatalf("expected missing message builder error")
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
|
||||||
|
BuildMessages: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||||
|
return nil, nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("expected empty prompt ID error")
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
|
||||||
|
PromptID: "missing.prompt.id",
|
||||||
|
BuildMessages: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||||
|
return []contracts.LLMMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}, nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("expected unknown prompt ID error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecuteModuleProposalPropagatesBuilderError(t *testing.T) {
|
||||||
|
wantErr := errors.New("builder failed")
|
||||||
|
_, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
|
||||||
|
PromptID: prompts.PromptIDModuleGlossaryProposal,
|
||||||
|
BuildMessages: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||||
|
return nil, wantErr
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if !errors.Is(err, wantErr) {
|
||||||
|
t.Fatalf("expected builder error, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -28,6 +29,15 @@ type Schema struct {
|
|||||||
SHA256 string `json:"sha256"`
|
SHA256 string `json:"sha256"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s Schema) DiagnosticsMap() map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"id": s.ID,
|
||||||
|
"version": s.Version,
|
||||||
|
"name": s.Name,
|
||||||
|
"sha256": s.SHA256,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
var registry = map[Key]Schema{
|
var registry = map[Key]Schema{
|
||||||
CorrectionSetKey: mustBuildSchema(
|
CorrectionSetKey: mustBuildSchema(
|
||||||
correctionSetSchemaID,
|
correctionSetSchemaID,
|
||||||
@@ -43,6 +53,20 @@ var registry = map[Key]Schema{
|
|||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func Registered() []Schema {
|
||||||
|
keys := make([]string, 0, len(registry))
|
||||||
|
for key := range registry {
|
||||||
|
keys = append(keys, string(key))
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
|
||||||
|
out := make([]Schema, 0, len(keys))
|
||||||
|
for _, key := range keys {
|
||||||
|
out = append(out, cloneSchema(registry[Key(key)]))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// Lookup returns a copy of the registered schema for the provided key.
|
// Lookup returns a copy of the registered schema for the provided key.
|
||||||
func Lookup(key Key) (Schema, bool) {
|
func Lookup(key Key) (Schema, bool) {
|
||||||
schema, ok := registry[key]
|
schema, ok := registry[key]
|
||||||
|
|||||||
@@ -89,3 +89,20 @@ func TestLookupReturnsSchemaCopy(t *testing.T) {
|
|||||||
t.Fatalf("expected lookup to return independent schema copy")
|
t.Fatalf("expected lookup to return independent schema copy")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDiagnosticsMapIncludesStableSchemaMetadataShapeForAllSchemas(t *testing.T) {
|
||||||
|
registered := Registered()
|
||||||
|
if len(registered) == 0 {
|
||||||
|
t.Fatalf("expected registered response schemas")
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, schema := range registered {
|
||||||
|
metadataMap := schema.DiagnosticsMap()
|
||||||
|
if metadataMap["id"] != schema.ID ||
|
||||||
|
metadataMap["version"] != schema.Version ||
|
||||||
|
metadataMap["name"] != schema.Name ||
|
||||||
|
metadataMap["sha256"] != schema.SHA256 {
|
||||||
|
t.Fatalf("unexpected diagnostics metadata map for %q: %+v", schema.ID, metadataMap)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -527,7 +527,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
|||||||
diagnosticsWriter = &llmDiagnosticsWriterAdapter{
|
diagnosticsWriter = &llmDiagnosticsWriterAdapter{
|
||||||
writer: llm.NewDiagnosticsWriter(
|
writer: llm.NewDiagnosticsWriter(
|
||||||
filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName),
|
filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName),
|
||||||
validatorSecrets(input.Config),
|
llm.ConfiguredSecrets(input.Config),
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -744,15 +744,3 @@ func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMeta
|
|||||||
ErrorPayloadPath: art.ErrorPayloadPath,
|
ErrorPayloadPath: art.ErrorPayloadPath,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validatorSecrets(cfg *config.Config) []string {
|
|
||||||
if cfg == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
effective := cfg.EffectiveValidationLLMConfig()
|
|
||||||
return []string{
|
|
||||||
cfg.PrimaryLLM.APIKey,
|
|
||||||
effective.APIKey,
|
|
||||||
cfg.ValidationLLM.APIKey,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1307,12 +1307,13 @@ func TestRunnerLLMValidatorBatchingAndSchedulerUsage(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
|
func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||||
secret := "super-secret-key"
|
primarySecret := "runner-primary-secret"
|
||||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: secret}}}}}
|
validationSecret := "runner-validation-secret"
|
||||||
|
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
|
||||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||||
cfg := config.Default()
|
cfg := config.Default()
|
||||||
cfg.PrimaryLLM.APIKey = secret
|
cfg.PrimaryLLM.APIKey = primarySecret
|
||||||
cfg.ValidationLLM.APIKey = secret
|
cfg.ValidationLLM.APIKey = validationSecret
|
||||||
diagDir := t.TempDir()
|
diagDir := t.TempDir()
|
||||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||||
@@ -1321,7 +1322,7 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
|
|||||||
}})
|
}})
|
||||||
out, err := r.Run(context.Background(), RunInput{
|
out, err := r.Run(context.Background(), RunInput{
|
||||||
Config: &cfg,
|
Config: &cfg,
|
||||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
|
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple. " + primarySecret}}},
|
||||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||||
ValidationLLMClient: client,
|
ValidationLLMClient: client,
|
||||||
ValidationDiagnosticsDir: diagDir,
|
ValidationDiagnosticsDir: diagDir,
|
||||||
@@ -1332,12 +1333,26 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
|
|||||||
if len(out.ModuleResults[0].ValidatorDecisions) == 0 || out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath == "" {
|
if len(out.ModuleResults[0].ValidatorDecisions) == 0 || out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath == "" {
|
||||||
t.Fatalf("expected diagnostic artifact path on decision")
|
t.Fatalf("expected diagnostic artifact path on decision")
|
||||||
}
|
}
|
||||||
|
matches, globErr := filepath.Glob(filepath.Join(diagDir, "m", "*.json"))
|
||||||
|
if globErr != nil {
|
||||||
|
t.Fatalf("glob diagnostics: %v", globErr)
|
||||||
|
}
|
||||||
|
if len(matches) == 0 {
|
||||||
|
t.Fatalf("expected diagnostics JSON artifacts under %s", filepath.Join(diagDir, "m"))
|
||||||
|
}
|
||||||
|
for _, path := range matches {
|
||||||
|
raw, readErr := os.ReadFile(path)
|
||||||
|
if readErr != nil {
|
||||||
|
t.Fatalf("read diagnostic %q: %v", path, readErr)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) {
|
||||||
|
t.Fatalf("configured secret leaked in diagnostics %q: %s", path, string(raw))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
raw, readErr := os.ReadFile(out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath)
|
raw, readErr := os.ReadFile(out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath)
|
||||||
if readErr != nil {
|
if readErr != nil {
|
||||||
t.Fatalf("read diagnostic: %v", readErr)
|
t.Fatalf("read decision diagnostic: %v", readErr)
|
||||||
}
|
|
||||||
if strings.Contains(string(raw), secret) {
|
|
||||||
t.Fatalf("secret leaked in diagnostics: %s", string(raw))
|
|
||||||
}
|
}
|
||||||
if !strings.Contains(string(raw), "[REDACTED]") {
|
if !strings.Contains(string(raw), "[REDACTED]") {
|
||||||
t.Fatalf("expected redaction marker in diagnostics")
|
t.Fatalf("expected redaction marker in diagnostics")
|
||||||
|
|||||||
24
internal/framework/stagename/stagename.go
Normal file
24
internal/framework/stagename/stagename.go
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
package stagename
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
)
|
||||||
|
|
||||||
|
func ModuleProposal(moduleInstance string, sectionIndex *int) string {
|
||||||
|
if sectionIndex == nil || *sectionIndex == 0 {
|
||||||
|
return fmt.Sprintf("%s:proposal", moduleInstance)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s:proposal:section-%04d", moduleInstance, *sectionIndex)
|
||||||
|
}
|
||||||
|
|
||||||
|
func ProposalGeneration(moduleInstance string, sectionIndex *int) string {
|
||||||
|
base := fmt.Sprintf("%s:proposal-generation", moduleInstance)
|
||||||
|
if sectionIndex == nil {
|
||||||
|
return base
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s:section-%04d", base, *sectionIndex)
|
||||||
|
}
|
||||||
|
|
||||||
|
func ValidatorBatch(moduleInstance string, validatorName string, batchIndex int) string {
|
||||||
|
return fmt.Sprintf("%s:%s:batch-%04d", moduleInstance, validatorName, batchIndex)
|
||||||
|
}
|
||||||
33
internal/framework/stagename/stagename_test.go
Normal file
33
internal/framework/stagename/stagename_test.go
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
package stagename
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestModuleProposalStageName(t *testing.T) {
|
||||||
|
if got := ModuleProposal("grammar", nil); got != "grammar:proposal" {
|
||||||
|
t.Fatalf("unexpected stage name without section: %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
sectionIndex := 7
|
||||||
|
if got := ModuleProposal("grammar", §ionIndex); got != "grammar:proposal:section-0007" {
|
||||||
|
t.Fatalf("unexpected stage name with section: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProposalGenerationStageName(t *testing.T) {
|
||||||
|
if got := ProposalGeneration("grammar", nil); got != "grammar:proposal-generation" {
|
||||||
|
t.Fatalf("unexpected proposal generation stage name without section: %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
sectionIndex := 3
|
||||||
|
if got := ProposalGeneration("grammar", §ionIndex); got != "grammar:proposal-generation:section-0003" {
|
||||||
|
t.Fatalf("unexpected proposal generation stage name with section: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidatorBatchStageName(t *testing.T) {
|
||||||
|
if got := ValidatorBatch("homophones_1", "spoken_form_plausibility_review", 12); got != "homophones_1:spoken_form_plausibility_review:batch-0012" {
|
||||||
|
t.Fatalf("unexpected validator batch stage name: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
28
internal/framework/structuredoutput/malformed.go
Normal file
28
internal/framework/structuredoutput/malformed.go
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
package structuredoutput
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
|
var malformedMarkers = []string{
|
||||||
|
"malformed structured output",
|
||||||
|
"decode structured output:",
|
||||||
|
"decode provider response envelope:",
|
||||||
|
"provider response missing choices",
|
||||||
|
"provider response missing assistant message content",
|
||||||
|
"provider response assistant message content is empty",
|
||||||
|
"provider response assistant message content is not valid JSON",
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsMalformedError reports whether err matches provider malformed
|
||||||
|
// structured-output failure markers that should be downgraded.
|
||||||
|
func IsMalformedError(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
msg := err.Error()
|
||||||
|
for _, marker := range malformedMarkers {
|
||||||
|
if strings.Contains(msg, marker) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
32
internal/framework/structuredoutput/malformed_test.go
Normal file
32
internal/framework/structuredoutput/malformed_test.go
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
package structuredoutput
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestIsMalformedError(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
err error
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{name: "nil", err: nil, want: false},
|
||||||
|
{name: "generic", err: errors.New("network timeout"), want: false},
|
||||||
|
{name: "malformed", err: errors.New("malformed structured output"), want: true},
|
||||||
|
{name: "decode structured", err: errors.New("decode structured output: unexpected end of JSON input"), want: true},
|
||||||
|
{name: "missing choices", err: errors.New("provider response missing choices"), want: true},
|
||||||
|
{name: "missing content", err: errors.New("provider response missing assistant message content"), want: true},
|
||||||
|
{name: "empty content", err: errors.New("provider response assistant message content is empty"), want: true},
|
||||||
|
{name: "invalid content json", err: errors.New("provider response assistant message content is not valid JSON"), want: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got := IsMalformedError(tc.err)
|
||||||
|
if got != tc.want {
|
||||||
|
t.Fatalf("IsMalformedError(%v): got=%v want=%v", tc.err, got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,6 +11,8 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
|
||||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||||
)
|
)
|
||||||
@@ -110,9 +112,10 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
|||||||
|
|
||||||
var response LLMValidationResponse
|
var response LLMValidationResponse
|
||||||
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
|
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
|
||||||
|
stage := stagename.ValidatorBatch(req.ModuleInstance, v.name, batch.BatchIndex)
|
||||||
call := func(callCtx context.Context) error {
|
call := func(callCtx context.Context) error {
|
||||||
_, err = req.LLMClient.CompleteStructured(callCtx, StructuredCompletionRequest{
|
_, err = req.LLMClient.CompleteStructured(callCtx, StructuredCompletionRequest{
|
||||||
StageName: fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex),
|
StageName: stage,
|
||||||
Messages: messages,
|
Messages: messages,
|
||||||
Model: resolvedValidationModel(req.Config, v.model),
|
Model: resolvedValidationModel(req.Config, v.model),
|
||||||
ResponseSchema: &responseSchema,
|
ResponseSchema: &responseSchema,
|
||||||
@@ -126,7 +129,6 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
|||||||
}
|
}
|
||||||
artifacts := InteractionArtifacts{}
|
artifacts := InteractionArtifacts{}
|
||||||
if req.DiagnosticsWriter != nil {
|
if req.DiagnosticsWriter != nil {
|
||||||
stage := fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex)
|
|
||||||
promptMetadata := validatorPromptMetadata(v.validatorType)
|
promptMetadata := validatorPromptMetadata(v.validatorType)
|
||||||
artifacts, _ = req.DiagnosticsWriter.WriteInteraction(
|
artifacts, _ = req.DiagnosticsWriter.WriteInteraction(
|
||||||
stage,
|
stage,
|
||||||
@@ -134,19 +136,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
|||||||
"validator_name": v.name,
|
"validator_name": v.name,
|
||||||
"validator_type": v.validatorType,
|
"validator_type": v.validatorType,
|
||||||
"batch_index": batch.BatchIndex,
|
"batch_index": batch.BatchIndex,
|
||||||
"prompt_metadata": map[string]any{
|
"prompt_metadata": promptMetadata.DiagnosticsMap(),
|
||||||
"prompt_id": promptMetadata.PromptID,
|
"response_schema": responseSchema.DiagnosticsMap(),
|
||||||
"prompt_version": promptMetadata.PromptVersion,
|
|
||||||
"prompt_source": promptMetadata.PromptSource,
|
|
||||||
"embedded_path": promptMetadata.EmbeddedPath,
|
|
||||||
"sha256": promptMetadata.SHA256,
|
|
||||||
},
|
|
||||||
"response_schema": map[string]any{
|
|
||||||
"id": responseSchema.ID,
|
|
||||||
"version": responseSchema.Version,
|
|
||||||
"name": responseSchema.Name,
|
|
||||||
"sha256": responseSchema.SHA256,
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
map[string]any{"messages": messages, "items": batch.Items},
|
map[string]any{"messages": messages, "items": batch.Items},
|
||||||
response,
|
response,
|
||||||
@@ -154,7 +145,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if isMalformedStructuredOutputError(err) {
|
if structuredoutput.IsMalformedError(err) {
|
||||||
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
|
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
|
||||||
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
|
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
|
||||||
continue
|
continue
|
||||||
@@ -397,24 +388,3 @@ func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
|
|||||||
}
|
}
|
||||||
return artifacts.ResponsePayloadPath
|
return artifacts.ResponsePayloadPath
|
||||||
}
|
}
|
||||||
|
|
||||||
func isMalformedStructuredOutputError(err error) bool {
|
|
||||||
if err == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
msg := err.Error()
|
|
||||||
for _, marker := range []string{
|
|
||||||
"malformed structured output",
|
|
||||||
"decode structured output:",
|
|
||||||
"decode provider response envelope:",
|
|
||||||
"provider response missing choices",
|
|
||||||
"provider response missing assistant message content",
|
|
||||||
"provider response assistant message content is empty",
|
|
||||||
"provider response assistant message content is not valid JSON",
|
|
||||||
} {
|
|
||||||
if strings.Contains(msg, marker) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -265,6 +265,23 @@ func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLLMBackedValidatorProviderMalformedEnvelopeRejectsBatch(t *testing.T) {
|
||||||
|
client := &fakeStructuredLLMClient{err: errors.New("provider response assistant message content is empty")}
|
||||||
|
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||||
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||||
|
req.LLMClient = client
|
||||||
|
res, err := v.Validate(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected malformed provider envelope downgrade, got %v", err)
|
||||||
|
}
|
||||||
|
if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
|
||||||
|
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||||
|
}
|
||||||
|
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorMalformed {
|
||||||
|
t.Fatalf("expected malformed warning, got %+v", res.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) {
|
func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) {
|
||||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}}
|
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}}
|
||||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||||
@@ -118,13 +119,13 @@ func confidenceThresholdForModule(moduleKey string, cfg *config.Config) float64
|
|||||||
return 0.0
|
return 0.0
|
||||||
}
|
}
|
||||||
switch moduleKey {
|
switch moduleKey {
|
||||||
case "glossary":
|
case modulecatalog.KeyGlossary:
|
||||||
return cfg.Thresholds.Glossary
|
return cfg.Thresholds.Glossary
|
||||||
case "grammar":
|
case modulecatalog.KeyGrammar:
|
||||||
return cfg.Thresholds.Grammar
|
return cfg.Thresholds.Grammar
|
||||||
case "homophones":
|
case modulecatalog.KeyHomophones:
|
||||||
return cfg.Thresholds.Homophones
|
return cfg.Thresholds.Homophones
|
||||||
case "spoken_word":
|
case modulecatalog.KeySpokenWord:
|
||||||
return cfg.Thresholds.SpokenWord
|
return cfg.Thresholds.SpokenWord
|
||||||
default:
|
default:
|
||||||
return 0.0
|
return 0.0
|
||||||
|
|||||||
@@ -2,12 +2,11 @@ package glossary
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||||
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
|
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -37,67 +36,9 @@ func (m *Module) Validators() []contracts.Validator {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||||
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
|
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
|
||||||
sectionIndex := 0
|
ProposalRequest: req,
|
||||||
if req.Section != nil {
|
PromptID: prompts.PromptIDModuleGlossaryProposal,
|
||||||
sectionIndex = req.Section.Index
|
BuildMessages: BuildProposalMessages,
|
||||||
}
|
|
||||||
transcriptDescription := ""
|
|
||||||
if req.Config != nil {
|
|
||||||
transcriptDescription = req.Config.TranscriptDescription
|
|
||||||
}
|
|
||||||
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.ProposalResult{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
|
||||||
ModuleKey: req.RunSpec.ModuleKey,
|
|
||||||
ModuleInstance: req.RunSpec.InstanceName,
|
|
||||||
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
|
|
||||||
WorkingTranscript: req.WorkingTranscript,
|
|
||||||
Section: req.Section,
|
|
||||||
Glossary: req.Glossary,
|
|
||||||
Config: req.Config,
|
|
||||||
Messages: messages,
|
|
||||||
PromptMetadata: map[string]any{
|
|
||||||
"prompt_id": proposalPromptMetadata().PromptID,
|
|
||||||
"prompt_version": proposalPromptMetadata().PromptVersion,
|
|
||||||
"prompt_source": proposalPromptMetadata().PromptSource,
|
|
||||||
"embedded_path": proposalPromptMetadata().EmbeddedPath,
|
|
||||||
"sha256": proposalPromptMetadata().SHA256,
|
|
||||||
},
|
|
||||||
StageName: proposalStageName(req),
|
|
||||||
StartIndex: 0,
|
|
||||||
LLMClient: req.LLMClient,
|
|
||||||
Scheduler: req.LLMScheduler,
|
|
||||||
DiagnosticsDir: req.DiagnosticsDir,
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
return contracts.ProposalResult{}, err
|
|
||||||
}
|
|
||||||
return contracts.ProposalResult{
|
|
||||||
Proposals: generated.Corrections,
|
|
||||||
Warnings: generated.Warnings,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
|
|
||||||
if section == nil || transcript == nil {
|
|
||||||
return transcript
|
|
||||||
}
|
|
||||||
segments := make([]schema.Segment, 0, len(transcript.Segments))
|
|
||||||
for _, seg := range transcript.Segments {
|
|
||||||
if seg.ID >= section.StartSegmentID && seg.ID <= section.EndSegmentID {
|
|
||||||
segments = append(segments, seg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &schema.Transcript{Segments: segments}
|
|
||||||
}
|
|
||||||
|
|
||||||
func proposalStageName(req contracts.ProposalRequest) string {
|
|
||||||
if req.Section == nil || req.Section.Index == 0 {
|
|
||||||
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName)
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,43 +10,13 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||||
)
|
)
|
||||||
|
|
||||||
type promptSegment struct {
|
|
||||||
ID int `json:"id"`
|
|
||||||
Speaker string `json:"speaker"`
|
|
||||||
Start float64 `json:"start"`
|
|
||||||
End float64 `json:"end"`
|
|
||||||
Text string `json:"text"`
|
|
||||||
Categories []string `json:"categories,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type promptTranscriptSection struct {
|
|
||||||
SectionIndex int `json:"section_index"`
|
|
||||||
Segments []promptSegment `json:"segments"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||||
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
|
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
sectionPayload := promptTranscriptSection{
|
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
|
||||||
SectionIndex: sectionIndex,
|
|
||||||
Segments: make([]promptSegment, 0),
|
|
||||||
}
|
|
||||||
if transcript != nil {
|
|
||||||
for _, s := range transcript.Segments {
|
|
||||||
sectionPayload.Segments = append(sectionPayload.Segments, promptSegment{
|
|
||||||
ID: s.ID,
|
|
||||||
Speaker: s.Speaker,
|
|
||||||
Start: s.Start,
|
|
||||||
End: s.End,
|
|
||||||
Text: s.Text,
|
|
||||||
Categories: append([]string(nil), s.Categories...),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sectionJSON, err := json.MarshalIndent(sectionPayload, "", " ")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
||||||
}
|
}
|
||||||
@@ -65,7 +35,3 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
|
|||||||
{Role: "user", Content: user},
|
{Role: "user", Content: user},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func proposalPromptMetadata() prompts.Metadata {
|
|
||||||
return prompts.MustLookupMetadata(prompts.PromptIDModuleGlossaryProposal)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -2,12 +2,11 @@ package grammar
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||||
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
|
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -37,67 +36,9 @@ func (m *Module) Validators() []contracts.Validator {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||||
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
|
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
|
||||||
sectionIndex := 0
|
ProposalRequest: req,
|
||||||
if req.Section != nil {
|
PromptID: prompts.PromptIDModuleGrammarProposal,
|
||||||
sectionIndex = req.Section.Index
|
BuildMessages: BuildProposalMessages,
|
||||||
}
|
|
||||||
transcriptDescription := ""
|
|
||||||
if req.Config != nil {
|
|
||||||
transcriptDescription = req.Config.TranscriptDescription
|
|
||||||
}
|
|
||||||
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.ProposalResult{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
|
||||||
ModuleKey: req.RunSpec.ModuleKey,
|
|
||||||
ModuleInstance: req.RunSpec.InstanceName,
|
|
||||||
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
|
|
||||||
WorkingTranscript: req.WorkingTranscript,
|
|
||||||
Section: req.Section,
|
|
||||||
Glossary: req.Glossary,
|
|
||||||
Config: req.Config,
|
|
||||||
Messages: messages,
|
|
||||||
PromptMetadata: map[string]any{
|
|
||||||
"prompt_id": proposalPromptMetadata().PromptID,
|
|
||||||
"prompt_version": proposalPromptMetadata().PromptVersion,
|
|
||||||
"prompt_source": proposalPromptMetadata().PromptSource,
|
|
||||||
"embedded_path": proposalPromptMetadata().EmbeddedPath,
|
|
||||||
"sha256": proposalPromptMetadata().SHA256,
|
|
||||||
},
|
|
||||||
StageName: proposalStageName(req),
|
|
||||||
StartIndex: 0,
|
|
||||||
LLMClient: req.LLMClient,
|
|
||||||
Scheduler: req.LLMScheduler,
|
|
||||||
DiagnosticsDir: req.DiagnosticsDir,
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
return contracts.ProposalResult{}, err
|
|
||||||
}
|
|
||||||
return contracts.ProposalResult{
|
|
||||||
Proposals: generated.Corrections,
|
|
||||||
Warnings: generated.Warnings,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
|
|
||||||
if section == nil || transcript == nil {
|
|
||||||
return transcript
|
|
||||||
}
|
|
||||||
segments := make([]schema.Segment, 0, len(transcript.Segments))
|
|
||||||
for _, seg := range transcript.Segments {
|
|
||||||
if seg.ID >= section.StartSegmentID && seg.ID <= section.EndSegmentID {
|
|
||||||
segments = append(segments, seg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &schema.Transcript{Segments: segments}
|
|
||||||
}
|
|
||||||
|
|
||||||
func proposalStageName(req contracts.ProposalRequest) string {
|
|
||||||
if req.Section == nil || req.Section.Index == 0 {
|
|
||||||
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName)
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,20 +10,6 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||||
)
|
)
|
||||||
|
|
||||||
type promptSegment struct {
|
|
||||||
ID int `json:"id"`
|
|
||||||
Speaker string `json:"speaker"`
|
|
||||||
Start float64 `json:"start"`
|
|
||||||
End float64 `json:"end"`
|
|
||||||
Text string `json:"text"`
|
|
||||||
Categories []string `json:"categories,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type promptTranscriptSection struct {
|
|
||||||
SectionIndex int `json:"section_index"`
|
|
||||||
Segments []promptSegment `json:"segments"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// BuildProposalMessages constrains corrections to punctuation/capitalization/
|
// BuildProposalMessages constrains corrections to punctuation/capitalization/
|
||||||
// spacing cleanup with strict meaning guards.
|
// spacing cleanup with strict meaning guards.
|
||||||
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||||
@@ -32,24 +18,7 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
|
|||||||
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
sectionPayload := promptTranscriptSection{
|
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
|
||||||
SectionIndex: sectionIndex,
|
|
||||||
Segments: make([]promptSegment, 0),
|
|
||||||
}
|
|
||||||
if transcript != nil {
|
|
||||||
for _, s := range transcript.Segments {
|
|
||||||
sectionPayload.Segments = append(sectionPayload.Segments, promptSegment{
|
|
||||||
ID: s.ID,
|
|
||||||
Speaker: s.Speaker,
|
|
||||||
Start: s.Start,
|
|
||||||
End: s.End,
|
|
||||||
Text: s.Text,
|
|
||||||
Categories: append([]string(nil), s.Categories...),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
sectionJSON, err := json.MarshalIndent(sectionPayload, "", " ")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
||||||
}
|
}
|
||||||
@@ -68,7 +37,3 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
|
|||||||
{Role: "user", Content: user},
|
{Role: "user", Content: user},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func proposalPromptMetadata() prompts.Metadata {
|
|
||||||
return prompts.MustLookupMetadata(prompts.PromptIDModuleGrammarProposal)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -2,12 +2,11 @@ package homophones
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||||
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
|
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -37,67 +36,9 @@ func (m *Module) Validators() []contracts.Validator {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||||
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
|
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
|
||||||
sectionIndex := 0
|
ProposalRequest: req,
|
||||||
if req.Section != nil {
|
PromptID: prompts.PromptIDModuleHomophonesProposal,
|
||||||
sectionIndex = req.Section.Index
|
BuildMessages: BuildProposalMessages,
|
||||||
}
|
|
||||||
transcriptDescription := ""
|
|
||||||
if req.Config != nil {
|
|
||||||
transcriptDescription = req.Config.TranscriptDescription
|
|
||||||
}
|
|
||||||
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.ProposalResult{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
|
||||||
ModuleKey: req.RunSpec.ModuleKey,
|
|
||||||
ModuleInstance: req.RunSpec.InstanceName,
|
|
||||||
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
|
|
||||||
WorkingTranscript: req.WorkingTranscript,
|
|
||||||
Section: req.Section,
|
|
||||||
Glossary: req.Glossary,
|
|
||||||
Config: req.Config,
|
|
||||||
Messages: messages,
|
|
||||||
PromptMetadata: map[string]any{
|
|
||||||
"prompt_id": proposalPromptMetadata().PromptID,
|
|
||||||
"prompt_version": proposalPromptMetadata().PromptVersion,
|
|
||||||
"prompt_source": proposalPromptMetadata().PromptSource,
|
|
||||||
"embedded_path": proposalPromptMetadata().EmbeddedPath,
|
|
||||||
"sha256": proposalPromptMetadata().SHA256,
|
|
||||||
},
|
|
||||||
StageName: proposalStageName(req),
|
|
||||||
StartIndex: 0,
|
|
||||||
LLMClient: req.LLMClient,
|
|
||||||
Scheduler: req.LLMScheduler,
|
|
||||||
DiagnosticsDir: req.DiagnosticsDir,
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
return contracts.ProposalResult{}, err
|
|
||||||
}
|
|
||||||
return contracts.ProposalResult{
|
|
||||||
Proposals: generated.Corrections,
|
|
||||||
Warnings: generated.Warnings,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
|
|
||||||
if section == nil || transcript == nil {
|
|
||||||
return transcript
|
|
||||||
}
|
|
||||||
segments := make([]schema.Segment, 0, len(transcript.Segments))
|
|
||||||
for _, seg := range transcript.Segments {
|
|
||||||
if seg.ID >= section.StartSegmentID && seg.ID <= section.EndSegmentID {
|
|
||||||
segments = append(segments, seg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &schema.Transcript{Segments: segments}
|
|
||||||
}
|
|
||||||
|
|
||||||
func proposalStageName(req contracts.ProposalRequest) string {
|
|
||||||
if req.Section == nil || req.Section.Index == 0 {
|
|
||||||
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName)
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,20 +10,6 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||||
)
|
)
|
||||||
|
|
||||||
type promptSegment struct {
|
|
||||||
ID int `json:"id"`
|
|
||||||
Speaker string `json:"speaker"`
|
|
||||||
Start float64 `json:"start"`
|
|
||||||
End float64 `json:"end"`
|
|
||||||
Text string `json:"text"`
|
|
||||||
Categories []string `json:"categories,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type promptTranscriptSection struct {
|
|
||||||
SectionIndex int `json:"section_index"`
|
|
||||||
Segments []promptSegment `json:"segments"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// BuildProposalMessages constrains corrections to conservative homophone and
|
// BuildProposalMessages constrains corrections to conservative homophone and
|
||||||
// mistranscription updates.
|
// mistranscription updates.
|
||||||
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||||
@@ -32,23 +18,7 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
|
|||||||
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
sectionPayload := promptTranscriptSection{
|
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
|
||||||
SectionIndex: sectionIndex,
|
|
||||||
Segments: make([]promptSegment, 0),
|
|
||||||
}
|
|
||||||
if transcript != nil {
|
|
||||||
for _, s := range transcript.Segments {
|
|
||||||
sectionPayload.Segments = append(sectionPayload.Segments, promptSegment{
|
|
||||||
ID: s.ID,
|
|
||||||
Speaker: s.Speaker,
|
|
||||||
Start: s.Start,
|
|
||||||
End: s.End,
|
|
||||||
Text: s.Text,
|
|
||||||
Categories: append([]string(nil), s.Categories...),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sectionJSON, err := json.MarshalIndent(sectionPayload, "", " ")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
||||||
}
|
}
|
||||||
@@ -67,7 +37,3 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
|
|||||||
{Role: "user", Content: user},
|
{Role: "user", Content: user},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func proposalPromptMetadata() prompts.Metadata {
|
|
||||||
return prompts.MustLookupMetadata(prompts.PromptIDModuleHomophonesProposal)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -2,12 +2,11 @@ package spoken_word
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||||
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
|
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -37,67 +36,9 @@ func (m *Module) Validators() []contracts.Validator {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||||
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
|
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
|
||||||
sectionIndex := 0
|
ProposalRequest: req,
|
||||||
if req.Section != nil {
|
PromptID: prompts.PromptIDModuleSpokenWordProposal,
|
||||||
sectionIndex = req.Section.Index
|
BuildMessages: BuildProposalMessages,
|
||||||
}
|
|
||||||
transcriptDescription := ""
|
|
||||||
if req.Config != nil {
|
|
||||||
transcriptDescription = req.Config.TranscriptDescription
|
|
||||||
}
|
|
||||||
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.ProposalResult{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
|
||||||
ModuleKey: req.RunSpec.ModuleKey,
|
|
||||||
ModuleInstance: req.RunSpec.InstanceName,
|
|
||||||
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
|
|
||||||
WorkingTranscript: req.WorkingTranscript,
|
|
||||||
Section: req.Section,
|
|
||||||
Glossary: req.Glossary,
|
|
||||||
Config: req.Config,
|
|
||||||
Messages: messages,
|
|
||||||
PromptMetadata: map[string]any{
|
|
||||||
"prompt_id": proposalPromptMetadata().PromptID,
|
|
||||||
"prompt_version": proposalPromptMetadata().PromptVersion,
|
|
||||||
"prompt_source": proposalPromptMetadata().PromptSource,
|
|
||||||
"embedded_path": proposalPromptMetadata().EmbeddedPath,
|
|
||||||
"sha256": proposalPromptMetadata().SHA256,
|
|
||||||
},
|
|
||||||
StageName: proposalStageName(req),
|
|
||||||
StartIndex: 0,
|
|
||||||
LLMClient: req.LLMClient,
|
|
||||||
Scheduler: req.LLMScheduler,
|
|
||||||
DiagnosticsDir: req.DiagnosticsDir,
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
return contracts.ProposalResult{}, err
|
|
||||||
}
|
|
||||||
return contracts.ProposalResult{
|
|
||||||
Proposals: generated.Corrections,
|
|
||||||
Warnings: generated.Warnings,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
|
|
||||||
if section == nil || transcript == nil {
|
|
||||||
return transcript
|
|
||||||
}
|
|
||||||
segments := make([]schema.Segment, 0, len(transcript.Segments))
|
|
||||||
for _, seg := range transcript.Segments {
|
|
||||||
if seg.ID >= section.StartSegmentID && seg.ID <= section.EndSegmentID {
|
|
||||||
segments = append(segments, seg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &schema.Transcript{Segments: segments}
|
|
||||||
}
|
|
||||||
|
|
||||||
func proposalStageName(req contracts.ProposalRequest) string {
|
|
||||||
if req.Section == nil || req.Section.Index == 0 {
|
|
||||||
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName)
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,20 +10,6 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||||
)
|
)
|
||||||
|
|
||||||
type promptSegment struct {
|
|
||||||
ID int `json:"id"`
|
|
||||||
Speaker string `json:"speaker"`
|
|
||||||
Start float64 `json:"start"`
|
|
||||||
End float64 `json:"end"`
|
|
||||||
Text string `json:"text"`
|
|
||||||
Categories []string `json:"categories,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type promptTranscriptSection struct {
|
|
||||||
SectionIndex int `json:"section_index"`
|
|
||||||
Segments []promptSegment `json:"segments"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// BuildProposalMessages constrains corrections to conservative dysfluency
|
// BuildProposalMessages constrains corrections to conservative dysfluency
|
||||||
// cleanup with strict semantic preservation.
|
// cleanup with strict semantic preservation.
|
||||||
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||||
@@ -32,23 +18,7 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
|
|||||||
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
sectionPayload := promptTranscriptSection{
|
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
|
||||||
SectionIndex: sectionIndex,
|
|
||||||
Segments: make([]promptSegment, 0),
|
|
||||||
}
|
|
||||||
if transcript != nil {
|
|
||||||
for _, s := range transcript.Segments {
|
|
||||||
sectionPayload.Segments = append(sectionPayload.Segments, promptSegment{
|
|
||||||
ID: s.ID,
|
|
||||||
Speaker: s.Speaker,
|
|
||||||
Start: s.Start,
|
|
||||||
End: s.End,
|
|
||||||
Text: s.Text,
|
|
||||||
Categories: append([]string(nil), s.Categories...),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sectionJSON, err := json.MarshalIndent(sectionPayload, "", " ")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
||||||
}
|
}
|
||||||
@@ -67,7 +37,3 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
|
|||||||
{Role: "user", Content: user},
|
{Role: "user", Content: user},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func proposalPromptMetadata() prompts.Metadata {
|
|
||||||
return prompts.MustLookupMetadata(prompts.PromptIDModuleSpokenWordProposal)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -40,6 +40,16 @@ type Metadata struct {
|
|||||||
SHA256 string `json:"sha256"`
|
SHA256 string `json:"sha256"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m Metadata) DiagnosticsMap() map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"prompt_id": m.PromptID,
|
||||||
|
"prompt_version": m.PromptVersion,
|
||||||
|
"prompt_source": m.PromptSource,
|
||||||
|
"embedded_path": m.EmbeddedPath,
|
||||||
|
"sha256": m.SHA256,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type definition struct {
|
type definition struct {
|
||||||
id string
|
id string
|
||||||
version string
|
version string
|
||||||
|
|||||||
@@ -107,3 +107,16 @@ func TestRenderedPromptsContainHardening(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDiagnosticsMapIncludesStablePromptMetadataShapeForAllPrompts(t *testing.T) {
|
||||||
|
for _, m := range RegisteredMetadata() {
|
||||||
|
metadataMap := m.DiagnosticsMap()
|
||||||
|
if metadataMap["prompt_id"] != m.PromptID ||
|
||||||
|
metadataMap["prompt_version"] != m.PromptVersion ||
|
||||||
|
metadataMap["prompt_source"] != m.PromptSource ||
|
||||||
|
metadataMap["embedded_path"] != m.EmbeddedPath ||
|
||||||
|
metadataMap["sha256"] != m.SHA256 {
|
||||||
|
t.Fatalf("unexpected diagnostics metadata map for %q: %+v", m.PromptID, metadataMap)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
57
internal/testsupport/files.go
Normal file
57
internal/testsupport/files.go
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
package testsupport
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func ReadFile(t *testing.T, path string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to read file %q: %v", path, err)
|
||||||
|
}
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
func OnlyRunDir(t *testing.T, workDir string) string {
|
||||||
|
t.Helper()
|
||||||
|
entries, err := os.ReadDir(workDir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to read work dir %q: %v", workDir, err)
|
||||||
|
}
|
||||||
|
dirs := make([]string, 0, len(entries))
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.IsDir() {
|
||||||
|
dirs = append(dirs, filepath.Join(workDir, e.Name()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(dirs) != 1 {
|
||||||
|
t.Fatalf("expected exactly one run dir in %q, found %d", workDir, len(dirs))
|
||||||
|
}
|
||||||
|
return dirs[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
func AssertNoSecretInFile(t *testing.T, path, secret string) {
|
||||||
|
t.Helper()
|
||||||
|
raw := string(ReadFile(t, path))
|
||||||
|
if strings.Contains(raw, secret) {
|
||||||
|
t.Fatalf("secret leaked in %s", path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func AssertNoSecretInTree(t *testing.T, root, secret string) {
|
||||||
|
t.Helper()
|
||||||
|
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||||
|
if err != nil || d == nil || d.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
raw, readErr := os.ReadFile(path)
|
||||||
|
if readErr == nil && strings.Contains(string(raw), secret) {
|
||||||
|
t.Fatalf("secret leaked in %s", path)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -2,13 +2,15 @@ package validators
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/validators/protected_terms"
|
"gitea.maximumdirect.net/eric/audita/internal/validators/protected_terms"
|
||||||
)
|
)
|
||||||
|
|
||||||
var builtInChains = map[string][]string{
|
var builtInChains = map[string][]string{
|
||||||
"glossary": {
|
modulecatalog.KeyGlossary: {
|
||||||
KeyProposalShape,
|
KeyProposalShape,
|
||||||
KeyNoEffect,
|
KeyNoEffect,
|
||||||
KeyOriginalTextPresence,
|
KeyOriginalTextPresence,
|
||||||
@@ -18,7 +20,7 @@ var builtInChains = map[string][]string{
|
|||||||
KeySpokenFormPlausibility,
|
KeySpokenFormPlausibility,
|
||||||
KeyMeaningReversalReview,
|
KeyMeaningReversalReview,
|
||||||
},
|
},
|
||||||
"homophones": {
|
modulecatalog.KeyHomophones: {
|
||||||
KeyProposalShape,
|
KeyProposalShape,
|
||||||
KeyNoEffect,
|
KeyNoEffect,
|
||||||
KeyOriginalTextPresence,
|
KeyOriginalTextPresence,
|
||||||
@@ -28,7 +30,7 @@ var builtInChains = map[string][]string{
|
|||||||
KeySpokenFormPlausibility,
|
KeySpokenFormPlausibility,
|
||||||
KeyMeaningReversalReview,
|
KeyMeaningReversalReview,
|
||||||
},
|
},
|
||||||
"spoken_word": {
|
modulecatalog.KeySpokenWord: {
|
||||||
KeyProposalShape,
|
KeyProposalShape,
|
||||||
KeyNoEffect,
|
KeyNoEffect,
|
||||||
KeyOriginalTextPresence,
|
KeyOriginalTextPresence,
|
||||||
@@ -38,7 +40,7 @@ var builtInChains = map[string][]string{
|
|||||||
KeyEditorialReview,
|
KeyEditorialReview,
|
||||||
KeyMeaningReversalReview,
|
KeyMeaningReversalReview,
|
||||||
},
|
},
|
||||||
"grammar": {
|
modulecatalog.KeyGrammar: {
|
||||||
KeyProposalShape,
|
KeyProposalShape,
|
||||||
KeyNoEffect,
|
KeyNoEffect,
|
||||||
KeyOriginalTextPresence,
|
KeyOriginalTextPresence,
|
||||||
@@ -51,9 +53,10 @@ var builtInChains = map[string][]string{
|
|||||||
}
|
}
|
||||||
|
|
||||||
func BuiltInChainKeys(moduleKey string) ([]string, error) {
|
func BuiltInChainKeys(moduleKey string) ([]string, error) {
|
||||||
keys, ok := builtInChains[moduleKey]
|
key := strings.TrimSpace(moduleKey)
|
||||||
|
keys, ok := builtInChains[key]
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("no built-in validator chain for module %q", moduleKey)
|
return nil, fmt.Errorf("no built-in validator chain for module %q", key)
|
||||||
}
|
}
|
||||||
out := make([]string, len(keys))
|
out := make([]string, len(keys))
|
||||||
copy(out, keys)
|
copy(out, keys)
|
||||||
@@ -61,6 +64,7 @@ func BuiltInChainKeys(moduleKey string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]contracts.Validator, error) {
|
func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]contracts.Validator, error) {
|
||||||
|
moduleKey = strings.TrimSpace(moduleKey)
|
||||||
keys, err := BuiltInChainKeys(moduleKey)
|
keys, err := BuiltInChainKeys(moduleKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -71,7 +75,7 @@ func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]contracts.Vali
|
|||||||
|
|
||||||
out := make([]contracts.Validator, 0, len(keys))
|
out := make([]contracts.Validator, 0, len(keys))
|
||||||
for _, key := range keys {
|
for _, key := range keys {
|
||||||
if moduleKey == "glossary" && key == KeyProtectedTerms {
|
if moduleKey == modulecatalog.KeyGlossary && key == KeyProtectedTerms {
|
||||||
// Glossary stages preserve current stricter protection semantics while
|
// Glossary stages preserve current stricter protection semantics while
|
||||||
// reporting the stable protected_terms key.
|
// reporting the stable protected_terms key.
|
||||||
v, buildErr := protected_terms.NewGlossaryStage()
|
v, buildErr := protected_terms.NewGlossaryStage()
|
||||||
|
|||||||
@@ -1,6 +1,10 @@
|
|||||||
package metadata
|
package metadata
|
||||||
|
|
||||||
import "gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||||
|
)
|
||||||
|
|
||||||
type ExecutionClass string
|
type ExecutionClass string
|
||||||
|
|
||||||
@@ -9,6 +13,31 @@ const (
|
|||||||
ExecutionClassLLMBacked ExecutionClass = "llm_backed"
|
ExecutionClassLLMBacked ExecutionClass = "llm_backed"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
KeyProposalShape = "proposal_shape"
|
||||||
|
KeyConfidenceThreshold = "confidence_threshold"
|
||||||
|
KeyOriginalTextPresence = "original_text_presence"
|
||||||
|
KeyNonEmptyCorrectedText = "non_empty_corrected_text"
|
||||||
|
KeyNoEffect = "no_effect"
|
||||||
|
KeyProtectedTerms = "protected_terms"
|
||||||
|
|
||||||
|
KeySpokenFormPlausibility = "spoken_form_plausibility"
|
||||||
|
KeyMeaningReversalReview = "meaning_reversal_review"
|
||||||
|
KeyEditorialReview = "editorial_review"
|
||||||
|
)
|
||||||
|
|
||||||
|
var executionClassByKey = map[string]ExecutionClass{
|
||||||
|
KeyProposalShape: ExecutionClassDeterministic,
|
||||||
|
KeyConfidenceThreshold: ExecutionClassDeterministic,
|
||||||
|
KeyOriginalTextPresence: ExecutionClassDeterministic,
|
||||||
|
KeyNonEmptyCorrectedText: ExecutionClassDeterministic,
|
||||||
|
KeyNoEffect: ExecutionClassDeterministic,
|
||||||
|
KeyProtectedTerms: ExecutionClassDeterministic,
|
||||||
|
KeySpokenFormPlausibility: ExecutionClassLLMBacked,
|
||||||
|
KeyMeaningReversalReview: ExecutionClassLLMBacked,
|
||||||
|
KeyEditorialReview: ExecutionClassLLMBacked,
|
||||||
|
}
|
||||||
|
|
||||||
type ClassifiedValidator interface {
|
type ClassifiedValidator interface {
|
||||||
contracts.Validator
|
contracts.Validator
|
||||||
ExecutionClass() ExecutionClass
|
ExecutionClass() ExecutionClass
|
||||||
@@ -18,9 +47,11 @@ func ClassOf(v contracts.Validator) ExecutionClass {
|
|||||||
if v == nil {
|
if v == nil {
|
||||||
return ExecutionClassDeterministic
|
return ExecutionClassDeterministic
|
||||||
}
|
}
|
||||||
|
|
||||||
|
classFromKey := ClassForKey(v.Name())
|
||||||
classified, ok := v.(ClassifiedValidator)
|
classified, ok := v.(ClassifiedValidator)
|
||||||
if !ok {
|
if !ok {
|
||||||
return ExecutionClassDeterministic
|
return classFromKey
|
||||||
}
|
}
|
||||||
switch classified.ExecutionClass() {
|
switch classified.ExecutionClass() {
|
||||||
case ExecutionClassLLMBacked:
|
case ExecutionClassLLMBacked:
|
||||||
@@ -28,8 +59,16 @@ func ClassOf(v contracts.Validator) ExecutionClass {
|
|||||||
case ExecutionClassDeterministic:
|
case ExecutionClassDeterministic:
|
||||||
return ExecutionClassDeterministic
|
return ExecutionClassDeterministic
|
||||||
default:
|
default:
|
||||||
|
return classFromKey
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func ClassForKey(key string) ExecutionClass {
|
||||||
|
class, ok := executionClassByKey[strings.TrimSpace(key)]
|
||||||
|
if !ok {
|
||||||
return ExecutionClassDeterministic
|
return ExecutionClassDeterministic
|
||||||
}
|
}
|
||||||
|
return class
|
||||||
}
|
}
|
||||||
|
|
||||||
func Wrap(v contracts.Validator, class ExecutionClass) contracts.Validator {
|
func Wrap(v contracts.Validator, class ExecutionClass) contracts.Validator {
|
||||||
|
|||||||
@@ -16,6 +16,16 @@ func (u unclassifiedValidator) Validate(_ context.Context, _ contracts.Validatio
|
|||||||
return frameworkvalidators.Result{ValidatorName: u.Name(), Decisions: nil}, nil
|
return frameworkvalidators.Result{ValidatorName: u.Name(), Decisions: nil}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type namedUnclassifiedValidator struct {
|
||||||
|
name string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n namedUnclassifiedValidator) Name() string { return n.name }
|
||||||
|
|
||||||
|
func (n namedUnclassifiedValidator) Validate(_ context.Context, _ contracts.ValidationRequest) (frameworkvalidators.Result, error) {
|
||||||
|
return frameworkvalidators.Result{ValidatorName: n.Name(), Decisions: nil}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func TestClassOfDefaultsToDeterministic(t *testing.T) {
|
func TestClassOfDefaultsToDeterministic(t *testing.T) {
|
||||||
if got := ClassOf(unclassifiedValidator{}); got != ExecutionClassDeterministic {
|
if got := ClassOf(unclassifiedValidator{}); got != ExecutionClassDeterministic {
|
||||||
t.Fatalf("expected deterministic default class, got %q", got)
|
t.Fatalf("expected deterministic default class, got %q", got)
|
||||||
@@ -28,3 +38,22 @@ func TestWrapExposesExecutionClass(t *testing.T) {
|
|||||||
t.Fatalf("expected llm_backed class, got %q", got)
|
t.Fatalf("expected llm_backed class, got %q", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestClassForKey(t *testing.T) {
|
||||||
|
if got := ClassForKey(KeyProposalShape); got != ExecutionClassDeterministic {
|
||||||
|
t.Fatalf("expected deterministic class for %q, got %q", KeyProposalShape, got)
|
||||||
|
}
|
||||||
|
if got := ClassForKey(KeySpokenFormPlausibility); got != ExecutionClassLLMBacked {
|
||||||
|
t.Fatalf("expected llm_backed class for %q, got %q", KeySpokenFormPlausibility, got)
|
||||||
|
}
|
||||||
|
if got := ClassForKey("unknown"); got != ExecutionClassDeterministic {
|
||||||
|
t.Fatalf("expected deterministic fallback for unknown key, got %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClassOfFallsBackToStableValidatorKey(t *testing.T) {
|
||||||
|
v := namedUnclassifiedValidator{name: KeyEditorialReview}
|
||||||
|
if got := ClassOf(v); got != ExecutionClassLLMBacked {
|
||||||
|
t.Fatalf("expected llm_backed fallback by key, got %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/audita/internal/validators/confidence_threshold"
|
"gitea.maximumdirect.net/eric/audita/internal/validators/confidence_threshold"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/validators/editorial_review"
|
"gitea.maximumdirect.net/eric/audita/internal/validators/editorial_review"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/validators/meaning_reversal_review"
|
"gitea.maximumdirect.net/eric/audita/internal/validators/meaning_reversal_review"
|
||||||
|
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/validators/no_effect"
|
"gitea.maximumdirect.net/eric/audita/internal/validators/no_effect"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/validators/non_empty_corrected_text"
|
"gitea.maximumdirect.net/eric/audita/internal/validators/non_empty_corrected_text"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/validators/original_text_presence"
|
"gitea.maximumdirect.net/eric/audita/internal/validators/original_text_presence"
|
||||||
@@ -17,22 +18,21 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
KeyProposalShape = "proposal_shape"
|
KeyProposalShape = validatormetadata.KeyProposalShape
|
||||||
KeyConfidenceThreshold = "confidence_threshold"
|
KeyConfidenceThreshold = validatormetadata.KeyConfidenceThreshold
|
||||||
KeyOriginalTextPresence = "original_text_presence"
|
KeyOriginalTextPresence = validatormetadata.KeyOriginalTextPresence
|
||||||
KeyNonEmptyCorrectedText = "non_empty_corrected_text"
|
KeyNonEmptyCorrectedText = validatormetadata.KeyNonEmptyCorrectedText
|
||||||
KeyNoEffect = "no_effect"
|
KeyNoEffect = validatormetadata.KeyNoEffect
|
||||||
KeyProtectedTerms = "protected_terms"
|
KeyProtectedTerms = validatormetadata.KeyProtectedTerms
|
||||||
|
|
||||||
KeySpokenFormPlausibility = "spoken_form_plausibility"
|
KeySpokenFormPlausibility = validatormetadata.KeySpokenFormPlausibility
|
||||||
KeyMeaningReversalReview = "meaning_reversal_review"
|
KeyMeaningReversalReview = validatormetadata.KeyMeaningReversalReview
|
||||||
KeyEditorialReview = "editorial_review"
|
KeyEditorialReview = validatormetadata.KeyEditorialReview
|
||||||
)
|
)
|
||||||
|
|
||||||
type BuiltInValidatorDefinition struct {
|
type BuiltInValidatorDefinition struct {
|
||||||
Key string
|
Key string
|
||||||
Build func() (contracts.Validator, error)
|
Build func() (contracts.Validator, error)
|
||||||
LLMBacked bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type Registry struct {
|
type Registry struct {
|
||||||
@@ -47,9 +47,9 @@ func NewBuiltInRegistry() *Registry {
|
|||||||
{Key: KeyNonEmptyCorrectedText, Build: non_empty_corrected_text.New},
|
{Key: KeyNonEmptyCorrectedText, Build: non_empty_corrected_text.New},
|
||||||
{Key: KeyNoEffect, Build: no_effect.New},
|
{Key: KeyNoEffect, Build: no_effect.New},
|
||||||
{Key: KeyProtectedTerms, Build: protected_terms.New},
|
{Key: KeyProtectedTerms, Build: protected_terms.New},
|
||||||
{Key: KeySpokenFormPlausibility, LLMBacked: true, Build: spoken_form_plausibility.New},
|
{Key: KeySpokenFormPlausibility, Build: spoken_form_plausibility.New},
|
||||||
{Key: KeyMeaningReversalReview, LLMBacked: true, Build: meaning_reversal_review.New},
|
{Key: KeyMeaningReversalReview, Build: meaning_reversal_review.New},
|
||||||
{Key: KeyEditorialReview, LLMBacked: true, Build: editorial_review.New},
|
{Key: KeyEditorialReview, Build: editorial_review.New},
|
||||||
}
|
}
|
||||||
|
|
||||||
m := make(map[string]BuiltInValidatorDefinition, len(defs))
|
m := make(map[string]BuiltInValidatorDefinition, len(defs))
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||||
@@ -81,11 +82,6 @@ func TestBuiltInValidatorPackagesConstruct(t *testing.T) {
|
|||||||
|
|
||||||
func TestRegistryBuildsClassifiedValidators(t *testing.T) {
|
func TestRegistryBuildsClassifiedValidators(t *testing.T) {
|
||||||
r := NewBuiltInRegistry()
|
r := NewBuiltInRegistry()
|
||||||
llmKeys := map[string]bool{
|
|
||||||
KeySpokenFormPlausibility: true,
|
|
||||||
KeyMeaningReversalReview: true,
|
|
||||||
KeyEditorialReview: true,
|
|
||||||
}
|
|
||||||
for _, key := range r.RegisteredKeys() {
|
for _, key := range r.RegisteredKeys() {
|
||||||
v, err := r.MustBuild(key)
|
v, err := r.MustBuild(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -94,16 +90,28 @@ func TestRegistryBuildsClassifiedValidators(t *testing.T) {
|
|||||||
if _, ok := v.(validatormetadata.ClassifiedValidator); !ok {
|
if _, ok := v.(validatormetadata.ClassifiedValidator); !ok {
|
||||||
t.Fatalf("expected built validator %q to expose execution classification metadata", key)
|
t.Fatalf("expected built validator %q to expose execution classification metadata", key)
|
||||||
}
|
}
|
||||||
want := validatormetadata.ExecutionClassDeterministic
|
want := validatormetadata.ClassForKey(key)
|
||||||
if llmKeys[key] {
|
|
||||||
want = validatormetadata.ExecutionClassLLMBacked
|
|
||||||
}
|
|
||||||
if got := validatormetadata.ClassOf(v); got != want {
|
if got := validatormetadata.ClassOf(v); got != want {
|
||||||
t.Fatalf("expected class %q for %q, got %q", want, key, got)
|
t.Fatalf("expected class %q for %q, got %q", want, key, got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExecutionClassResolvableByStableKeyAndByValidatorInstance(t *testing.T) {
|
||||||
|
r := NewBuiltInRegistry()
|
||||||
|
for _, key := range r.RegisteredKeys() {
|
||||||
|
v, err := r.MustBuild(key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("must build %q: %v", key, err)
|
||||||
|
}
|
||||||
|
fromKey := validatormetadata.ClassForKey(key)
|
||||||
|
fromInstance := validatormetadata.ClassOf(v)
|
||||||
|
if fromInstance != fromKey {
|
||||||
|
t.Fatalf("class mismatch for %q: key=%q instance=%q", key, fromKey, fromInstance)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRegistryProtectedTermsUsesNonGlossaryStageBehavior(t *testing.T) {
|
func TestRegistryProtectedTermsUsesNonGlossaryStageBehavior(t *testing.T) {
|
||||||
r := NewBuiltInRegistry()
|
r := NewBuiltInRegistry()
|
||||||
v, err := r.MustBuild(KeyProtectedTerms)
|
v, err := r.MustBuild(KeyProtectedTerms)
|
||||||
@@ -200,7 +208,7 @@ func TestBuiltInRegistryUnknownKeyFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestBuiltInChainKeysResolveForProductionModules(t *testing.T) {
|
func TestBuiltInChainKeysResolveForProductionModules(t *testing.T) {
|
||||||
for _, moduleKey := range []string{"glossary", "homophones", "spoken_word", "grammar"} {
|
for _, moduleKey := range modulecatalog.SupportedKeys() {
|
||||||
keys, err := BuiltInChainKeys(moduleKey)
|
keys, err := BuiltInChainKeys(moduleKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("resolve keys for %q: %v", moduleKey, err)
|
t.Fatalf("resolve keys for %q: %v", moduleKey, err)
|
||||||
@@ -213,7 +221,7 @@ func TestBuiltInChainKeysResolveForProductionModules(t *testing.T) {
|
|||||||
|
|
||||||
func TestResolveBuiltInChainUsesRegisteredKeys(t *testing.T) {
|
func TestResolveBuiltInChainUsesRegisteredKeys(t *testing.T) {
|
||||||
r := NewBuiltInRegistry()
|
r := NewBuiltInRegistry()
|
||||||
for _, moduleKey := range []string{"glossary", "homophones", "spoken_word", "grammar"} {
|
for _, moduleKey := range modulecatalog.SupportedKeys() {
|
||||||
chain, err := ResolveBuiltInChain(moduleKey, r)
|
chain, err := ResolveBuiltInChain(moduleKey, r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("resolve chain for %q: %v", moduleKey, err)
|
t.Fatalf("resolve chain for %q: %v", moduleKey, err)
|
||||||
|
|||||||
Reference in New Issue
Block a user