Compare commits
9 Commits
3d7057b437
...
f790c1441c
| Author | SHA1 | Date | |
|---|---|---|---|
| f790c1441c | |||
| 56f9b28f4b | |||
| 222222f449 | |||
| 99391cd18b | |||
| 84be774b34 | |||
| e053f7e124 | |||
| 13029dbb33 | |||
| 938bfe88c1 | |||
| fa1bd237d1 |
@@ -16,6 +16,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/cli"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/testsupport"
|
||||
)
|
||||
|
||||
func TestHelperProcess(t *testing.T) {
|
||||
@@ -499,6 +500,9 @@ func TestProcessCancellationViaSubprocessTimeoutHook(t *testing.T) {
|
||||
"always",
|
||||
)
|
||||
if result.stdout != "" {
|
||||
if result.stderr == "" {
|
||||
t.Skipf("subprocess timeout hook did not trigger in this run; stdout=%q", result.stdout)
|
||||
}
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "context deadline exceeded") {
|
||||
@@ -618,12 +622,7 @@ func schemaFixturePath(name string) string {
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read file %q: %v", path, err)
|
||||
}
|
||||
return data
|
||||
return testsupport.ReadFile(t, path)
|
||||
}
|
||||
|
||||
func assertJSONSemanticallyEqual(t *testing.T, expected []byte, actual []byte) {
|
||||
@@ -666,41 +665,13 @@ func writeLargeTranscriptFixture(t *testing.T, segments int) string {
|
||||
}
|
||||
|
||||
func onlyRunDir(t *testing.T, workDir string) string {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(workDir)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read work dir %q: %v", workDir, err)
|
||||
}
|
||||
dirs := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
dirs = append(dirs, filepath.Join(workDir, e.Name()))
|
||||
}
|
||||
}
|
||||
if len(dirs) != 1 {
|
||||
t.Fatalf("expected exactly one run dir in %q, found %d", workDir, len(dirs))
|
||||
}
|
||||
return dirs[0]
|
||||
return testsupport.OnlyRunDir(t, workDir)
|
||||
}
|
||||
|
||||
func assertNoSecretInFile(t *testing.T, path, secret string) {
|
||||
t.Helper()
|
||||
raw := string(readFile(t, path))
|
||||
if strings.Contains(raw, secret) {
|
||||
t.Fatalf("secret leaked in %s", path)
|
||||
}
|
||||
testsupport.AssertNoSecretInFile(t, path, secret)
|
||||
}
|
||||
|
||||
func assertNoSecretInTree(t *testing.T, root, secret string) {
|
||||
t.Helper()
|
||||
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil || d == nil || d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
raw, readErr := os.ReadFile(path)
|
||||
if readErr == nil && strings.Contains(string(raw), secret) {
|
||||
t.Fatalf("secret leaked in %s", path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
testsupport.AssertNoSecretInTree(t, root, secret)
|
||||
}
|
||||
|
||||
14
docs/architecture.md
Normal file
14
docs/architecture.md
Normal file
@@ -0,0 +1,14 @@
|
||||
# Audita Architecture Index
|
||||
|
||||
This file is the entrypoint for architecture documentation.
|
||||
|
||||
Core architecture overview:
|
||||
- [Architecture Overview](./architecture/architecture.md)
|
||||
|
||||
Focused architecture contracts:
|
||||
- [Public Contract](./architecture/public-contract.md)
|
||||
- [Diagnostics](./architecture/diagnostics.md)
|
||||
- [Structured LLM](./architecture/structured-llm.md)
|
||||
- [Validators](./architecture/validators.md)
|
||||
- [Prompts](./architecture/prompts.md)
|
||||
- [Output Schemas](./architecture/output-schemas.md)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,31 +1,25 @@
|
||||
# Audita Public Contract
|
||||
|
||||
This document defines stability expectations for Audita's external process and data interfaces.
|
||||
|
||||
## Scope
|
||||
This document defines stability expectations for Audita's external runtime interfaces.
|
||||
|
||||
This contract covers:
|
||||
- CLI invocation and behavior
|
||||
- versioned config file behavior
|
||||
- transcript/glossary input forms
|
||||
- transcript output schema selection
|
||||
- process report schema metadata
|
||||
- stable validator key identifiers in report/diagnostics records
|
||||
- prompt metadata identifiers in diagnostics
|
||||
- diagnostics directory behavior
|
||||
- utilization diagnostics and correction-ledger artifact presence/pathing in diagnostics metadata
|
||||
- stdout/stderr and exit-code behavior
|
||||
- secret redaction guarantees
|
||||
- compatibility and deprecation policy
|
||||
|
||||
## CLI stability expectations
|
||||
Covered interfaces:
|
||||
- CLI commands and major flags;
|
||||
- versioned config behavior and precedence;
|
||||
- transcript/glossary input forms;
|
||||
- output schema selection;
|
||||
- report schema metadata;
|
||||
- diagnostics artifact path metadata;
|
||||
- stdout/stderr and exit-code behavior;
|
||||
- redaction guarantees.
|
||||
|
||||
## CLI contract
|
||||
Stable commands:
|
||||
- `audita process`
|
||||
- `audita config validate`
|
||||
- `audita config print-effective`
|
||||
|
||||
For `audita process`, stable high-value flags include:
|
||||
Stable high-value `process` flags:
|
||||
- `--config`
|
||||
- `--glossary`
|
||||
- `--output`
|
||||
@@ -33,136 +27,95 @@ For `audita process`, stable high-value flags include:
|
||||
- `--modules`
|
||||
- `--output-schema`
|
||||
|
||||
Compatibility flags and lower-level tuning flags remain available; they may be narrowed over time with explicit compatibility notes.
|
||||
## Config contract
|
||||
Supported config format:
|
||||
- YAML;
|
||||
- `version: 1`;
|
||||
- strict unknown-field rejection.
|
||||
|
||||
## Config file stability expectations
|
||||
Path resolution for `process` and `config print-effective`:
|
||||
1. `--config`
|
||||
2. `AUDITA_CONFIG`
|
||||
3. `/usr/local/etc/audita/config.yml`
|
||||
4. `/etc/audita/config.yml`
|
||||
|
||||
Supported file format:
|
||||
- YAML
|
||||
- strict unknown-field rejection
|
||||
- explicit `version`
|
||||
Missing explicit path is an error. Missing default paths is non-fatal.
|
||||
|
||||
Supported version:
|
||||
- `version: 1`
|
||||
|
||||
Precedence for `audita process`:
|
||||
1. built-in defaults
|
||||
Precedence for `process`:
|
||||
1. defaults
|
||||
2. file config
|
||||
3. environment overrides
|
||||
4. CLI overrides
|
||||
|
||||
Config source behavior:
|
||||
- `--config <path>`: missing path is a clear failure
|
||||
- `AUDITA_CONFIG`: missing path is a clear failure
|
||||
- defaults `/usr/local/etc/audita/config.yml`, then `/etc/audita/config.yml`: both missing is non-fatal
|
||||
`config validate` remains file-only validation (defaults + file config; no env overrides).
|
||||
|
||||
## Supported transcript input forms
|
||||
Module and output-schema keys are validated against built-in catalogs. Unknown keys fail validation.
|
||||
|
||||
Audita accepts transcript JSON as either:
|
||||
- a top-level array of segments
|
||||
- an object with a `segments` array
|
||||
## Input contract
|
||||
Supported transcript JSON top-level forms:
|
||||
- array of segments
|
||||
- object with `segments` array
|
||||
|
||||
Segments must satisfy the schema and validation rules enforced by `internal/core/schema`.
|
||||
Supported glossary YAML form:
|
||||
- top-level `glossary` list with required entry fields validated by schema parsing.
|
||||
|
||||
## Supported glossary input form
|
||||
|
||||
Audita accepts glossary YAML with a top-level `glossary` entry list and validates required fields per entry.
|
||||
|
||||
## Supported output schema names
|
||||
|
||||
Built-in output schema registry supports:
|
||||
## Output schema contract
|
||||
Supported transcript output schemas:
|
||||
- `bare-segments` (default)
|
||||
- `audita-v1`
|
||||
|
||||
`seriatim-intermediate` is planned but not implemented.
|
||||
Unknown schema keys fail before output write.
|
||||
|
||||
Unknown output schema names fail clearly.
|
||||
|
||||
## Report schema/versioning expectations
|
||||
|
||||
Process report payloads include `report_metadata` with:
|
||||
## Report metadata contract
|
||||
Process reports include stable report metadata fields:
|
||||
- `report_schema_name`
|
||||
- `report_schema_version`
|
||||
- `output_schema`
|
||||
- `config_version` when file config is used
|
||||
- `config_version` (when file config is loaded)
|
||||
|
||||
Current values:
|
||||
- `report_schema_name`: `audita-process-report`
|
||||
- `report_schema_version`: `v1`
|
||||
- `report_schema_name = audita-process-report`
|
||||
- `report_schema_version = v1`
|
||||
|
||||
`--report-json` output and diagnostics run-dir `report.json` use the same report schema metadata.
|
||||
`--report-json` output and run-directory `report.json` use the same report schema metadata.
|
||||
|
||||
Validator decision/rejection records in reports use stable validator keys in `validator_name`.
|
||||
Module results may also include warning records for malformed module-stage LLM payloads.
|
||||
Report diagnostics metadata includes artifact-path fields for utilization diagnostics and correction ledger when diagnostics initialization succeeds.
|
||||
Validator decision/rejection records use stable validator keys via `validator_name`.
|
||||
|
||||
## Diagnostics directory behavior
|
||||
## Diagnostics metadata contract
|
||||
When run-directory initialization succeeds, diagnostics metadata paths reference stable artifacts, including:
|
||||
- transcript and normalization artifacts;
|
||||
- chunking summary;
|
||||
- invocation metadata;
|
||||
- redacted effective config;
|
||||
- utilization diagnostics;
|
||||
- correction ledger;
|
||||
- `error.log` on failures.
|
||||
|
||||
When diagnostics directory creation succeeds, Audita writes run artifacts including:
|
||||
- invocation metadata
|
||||
- redacted effective config
|
||||
- transcript/normalization/chunking artifacts
|
||||
- utilization diagnostics (`utilization-diagnostics.json`)
|
||||
- correction ledger (`correction-ledger.json`)
|
||||
- report and failure error log (when applicable)
|
||||
- module/LLM diagnostics artifacts as available
|
||||
LLM interaction diagnostics include stable prompt and structured-schema identifiers where applicable.
|
||||
|
||||
Retention behavior is controlled by configured retention mode; failed runs are retained.
|
||||
## Stdout/stderr and exit codes
|
||||
Success:
|
||||
- with `--output`, stdout is empty;
|
||||
- without `--output`, stdout contains transcript JSON only;
|
||||
- report JSON is not written to stdout.
|
||||
|
||||
Diagnostics metadata for LLM interactions may include semi-public prompt identifiers:
|
||||
- `prompt_id`
|
||||
- `prompt_version`
|
||||
- `prompt_source`
|
||||
- `embedded_path`
|
||||
- `sha256`
|
||||
Failures:
|
||||
- nonzero exit;
|
||||
- human-readable stderr summary;
|
||||
- diagnostics directory path on stderr when available.
|
||||
|
||||
These are diagnostic identifiers, not user-facing prompt override controls.
|
||||
Exit codes:
|
||||
- `0` success
|
||||
- nonzero failure
|
||||
|
||||
## Stdout/stderr behavior
|
||||
## Redaction contract
|
||||
Configured secrets are redacted from:
|
||||
- effective config outputs;
|
||||
- diagnostics artifacts;
|
||||
- report artifacts;
|
||||
- surfaced adapter/runtime errors.
|
||||
|
||||
Success behavior:
|
||||
- with `--output`, stdout is empty
|
||||
- without `--output`, stdout contains only transcript JSON in selected output schema
|
||||
- report JSON is not written to stdout
|
||||
- success stderr remains empty even when reports/diagnostics contain module warnings
|
||||
## Compatibility policy
|
||||
Stable command behavior, schema names, report metadata keys, diagnostics-path field semantics, and validator key identities are treated as public contract.
|
||||
|
||||
Failure behavior:
|
||||
- stderr contains human-readable error summary
|
||||
- nonzero exit
|
||||
- diagnostics path is printed when available
|
||||
|
||||
## Exit-code behavior
|
||||
|
||||
- `0`: success
|
||||
- nonzero: failure
|
||||
|
||||
Treat any nonzero exit as a failed invocation.
|
||||
|
||||
## Secret redaction guarantees
|
||||
|
||||
Audita redacts API keys and authorization secrets from:
|
||||
- effective config outputs (`audita config print-effective`, diagnostics effective-config artifact)
|
||||
- report artifacts
|
||||
- LLM diagnostics artifacts
|
||||
- surfaced request/response error messages
|
||||
|
||||
Config files should reference secrets via environment variable names (`api_key_env`) rather than embedding secret values.
|
||||
|
||||
## Compatibility and deprecation policy
|
||||
|
||||
- Existing stable schema names, report metadata keys, and top-level command behavior are treated as public contract.
|
||||
- Existing stable validator keys remain public contract values even when validator semantics are refined.
|
||||
- Compatibility inputs (legacy flags/env aliases) may remain during transition windows.
|
||||
- Any planned removal or behavior change should include clear compatibility notes and migration guidance.
|
||||
|
||||
## Breaking changes after 1.0
|
||||
|
||||
After 1.0, breaking changes include, for example:
|
||||
- changing default success/failure exit-code semantics
|
||||
- changing stdout/stderr routing semantics
|
||||
- silently changing default output schema shape
|
||||
- removing supported output schema names without compatibility strategy
|
||||
- changing report schema fields or meanings incompatibly
|
||||
- changing config version semantics incompatibly without version bump
|
||||
|
||||
Additive fields, additive diagnostics, and new optional schema names are generally non-breaking when existing behavior remains intact.
|
||||
Additive fields are acceptable when existing fields and behavior remain compatible.
|
||||
|
||||
@@ -1,90 +1,72 @@
|
||||
# Structured LLM Architecture
|
||||
|
||||
## Purpose
|
||||
|
||||
## Scope
|
||||
This document describes Audita's structured LLM runtime boundary and adapter behavior.
|
||||
|
||||
## Why Audita owns the adapter
|
||||
|
||||
Audita owns a small structured LLM adapter so that core runtime behavior is controlled inside the repository:
|
||||
- request construction and schema handling are explicit and testable;
|
||||
- retries, timeouts, cancellation, and error redaction are consistent across modules and validators;
|
||||
- provider SDK types are not exposed outside the adapter boundary;
|
||||
- dependency weight and transitive provider-specific behavior are reduced.
|
||||
|
||||
At runtime, the rest of Audita depends only on the internal contract:
|
||||
- `StructuredLLMClient`
|
||||
## Runtime boundary
|
||||
Production LLM integration depends on the internal contract only:
|
||||
- `contracts.StructuredLLMClient`
|
||||
- `CompleteStructured(ctx, req, out)`
|
||||
|
||||
## OpenAI-compatible request shape
|
||||
Provider SDK types do not leak past this boundary.
|
||||
|
||||
At a conceptual level, Audita sends chat completion requests with:
|
||||
- `model`
|
||||
- `messages` (role/content pairs)
|
||||
- `response_format`:
|
||||
- `type = "json_schema"`
|
||||
- `json_schema.name` (stable schema name)
|
||||
- `json_schema.strict = true`
|
||||
- `json_schema.schema` (registered JSON Schema payload)
|
||||
## Adapter ownership
|
||||
`internal/framework/llm` owns the OpenAI-compatible HTTP adapter and shared LLM runtime utilities.
|
||||
|
||||
The adapter uses OpenAI-compatible `POST {base_url}/chat/completions` over `net/http`.
|
||||
Key responsibilities:
|
||||
- request assembly;
|
||||
- timeout/cancellation propagation;
|
||||
- bounded retry behavior;
|
||||
- scheduler integration;
|
||||
- provider response decoding;
|
||||
- error redaction.
|
||||
|
||||
## Structured response schema registry
|
||||
## Structured schema registry
|
||||
Structured response schemas are registered in `internal/framework/responseschema` and include stable metadata:
|
||||
- `id`
|
||||
- `version`
|
||||
- `name`
|
||||
- `json_schema`
|
||||
- `sha256`
|
||||
|
||||
Structured response schemas are registered in `internal/framework/responseschema` with stable metadata:
|
||||
- schema key
|
||||
- schema ID
|
||||
- schema version
|
||||
- schema name (OpenAI-compatible `response_format` name)
|
||||
- raw JSON Schema payload
|
||||
- SHA-256 hash
|
||||
Current schema keys:
|
||||
- `correction_set`
|
||||
- `validator_decision_set`
|
||||
|
||||
Current schemas:
|
||||
- `correction_set`:
|
||||
- id `audita.correction_set`
|
||||
- version `v1`
|
||||
- name `audita_correction_set_v1`
|
||||
- `validator_decision_set`:
|
||||
- id `audita.validator_decision_set`
|
||||
- version `v1`
|
||||
- name `audita_validator_decision_set_v1`
|
||||
Schema metadata is attached to diagnostics through `Schema.DiagnosticsMap()`.
|
||||
|
||||
## Provider compatibility assumptions
|
||||
## Request shape assumptions
|
||||
Audita targets OpenAI-compatible chat-completions endpoints and sends structured requests with:
|
||||
- model;
|
||||
- chat messages;
|
||||
- `response_format.type = json_schema`;
|
||||
- schema name and JSON schema payload.
|
||||
|
||||
Audita assumes an OpenAI-compatible chat-completions endpoint that:
|
||||
- accepts message arrays with model selection;
|
||||
- accepts `response_format.type = json_schema`;
|
||||
- returns a completion with assistant message content and optional usage metadata.
|
||||
## Local validation remains mandatory
|
||||
Provider schema enforcement is treated as transport-level guardrails.
|
||||
|
||||
Provider-specific differences are expected in strictness and error payload shapes, so the adapter treats provider output as untrusted until locally decoded.
|
||||
Audita still validates output locally before applying behavior changes:
|
||||
- proposal decoding and proposal invariants;
|
||||
- validator decision decoding and cardinality checks;
|
||||
- deterministic validation and apply-time rules.
|
||||
|
||||
## Local decode and validation remain mandatory
|
||||
## Shared malformed-output policy
|
||||
Malformed structured-output classification is centralized in `internal/framework/structuredoutput`.
|
||||
|
||||
Provider-level structured output is a transport guardrail, not final validation.
|
||||
Proposal generation and validator execution both use this shared classifier so downgrade behavior cannot drift between the two paths.
|
||||
|
||||
After receiving a response, Audita still:
|
||||
- decodes assistant content into typed request-specific structs;
|
||||
- validates proposal and validator payload invariants locally;
|
||||
- enforces deterministic validator/cardinality rules before any transcript application.
|
||||
## Secrets and redaction
|
||||
Secret extraction for LLM redaction is centralized in `llm.ConfiguredSecrets(cfg)` and reused by proposal and validator diagnostics writers.
|
||||
|
||||
This protects runtime correctness even when provider responses are malformed, partial, or semantically inconsistent.
|
||||
Secrets are redacted from:
|
||||
- diagnostics artifacts;
|
||||
- report artifacts;
|
||||
- surfaced adapter/runtime errors.
|
||||
|
||||
## Diagnostics and redaction
|
||||
## Concurrency and scheduling
|
||||
LLM execution is constrained by composed scheduler limits:
|
||||
- total LLM concurrency;
|
||||
- proposal LLM concurrency;
|
||||
- validation LLM concurrency.
|
||||
|
||||
When structured schemas are used, diagnostics metadata records:
|
||||
- schema ID
|
||||
- schema version
|
||||
- schema name
|
||||
- schema hash
|
||||
|
||||
Diagnostics and surfaced errors preserve secret redaction:
|
||||
- API keys and bearer tokens are redacted from request/response/error artifacts;
|
||||
- redaction is applied before diagnostic files are written.
|
||||
|
||||
## Runtime behavior guarantees
|
||||
|
||||
The structured LLM path preserves existing runtime guarantees:
|
||||
- bounded LLM call execution through schedulers;
|
||||
- context-aware cancellation and timeout propagation;
|
||||
- retry behavior for transient failures and retryable malformed structured responses;
|
||||
- deterministic module/chunk/proposal/validator behavior outside provider nondeterminism.
|
||||
The scheduler is FIFO and context-aware so permits are released on success, failure, and cancellation.
|
||||
|
||||
@@ -1,160 +1,96 @@
|
||||
# Audita Validators
|
||||
|
||||
This document describes Audita's built-in validator registry and module validator chains.
|
||||
|
||||
For LLM-backed validator prompt asset details, see [`docs/prompts.md`](prompts.md).
|
||||
|
||||
## Package ownership
|
||||
|
||||
Built-in validator construction is package-owned under `internal/validators/<validator_key>`:
|
||||
- `internal/validators/confidence_threshold`
|
||||
- `internal/validators/proposal_shape`
|
||||
- `internal/validators/original_text_presence`
|
||||
- `internal/validators/non_empty_corrected_text`
|
||||
- `internal/validators/no_effect`
|
||||
- `internal/validators/protected_terms`
|
||||
- `internal/validators/spoken_form_plausibility`
|
||||
- `internal/validators/meaning_reversal_review`
|
||||
- `internal/validators/editorial_review`
|
||||
|
||||
Registry and chain wiring stay in:
|
||||
- `internal/validators/registry.go`
|
||||
- `internal/validators/chains.go`
|
||||
|
||||
Shared validator runtime mechanics stay in `internal/framework/validators`:
|
||||
- request/result/decision models
|
||||
- decision cardinality helpers
|
||||
- protected vocabulary helpers
|
||||
- shared LLM validator runtime, batching, and diagnostics helpers
|
||||
|
||||
Execution classification metadata is defined in `internal/validators/metadata`:
|
||||
- `deterministic`
|
||||
- `llm_backed`
|
||||
|
||||
Runner ordering uses this metadata so deterministic validators run before LLM-backed validators without concrete framework type assertions.
|
||||
|
||||
## Scope
|
||||
This document defines the built-in validator system used by production module runs.
|
||||
|
||||
Validator chains are built-in runtime behavior.
|
||||
## Ownership boundaries
|
||||
Built-in validator keys, constructors, and module chains are owned by `internal/validators`.
|
||||
|
||||
Current 1.0 boundary:
|
||||
- built-in validator keys and built-in module chains are stable runtime identifiers;
|
||||
- thresholds and batching knobs remain configurable where already supported;
|
||||
- arbitrary user-defined validator chains are deferred.
|
||||
Shared runtime execution mechanics are owned by `internal/framework/validators`, including:
|
||||
- validator request/result models;
|
||||
- deterministic proposal checks;
|
||||
- LLM validator batching and execution;
|
||||
- decision-cardinality enforcement;
|
||||
- diagnostics integration.
|
||||
|
||||
## Built-in validator keys
|
||||
|
||||
### Deterministic validators
|
||||
Execution class metadata is owned by `internal/validators/metadata`.
|
||||
|
||||
## Stable validator keys
|
||||
Deterministic:
|
||||
- `proposal_shape`
|
||||
- rejects malformed proposal fields before other validators run.
|
||||
- `confidence_threshold`
|
||||
- checks proposal confidence against module-specific configured threshold.
|
||||
- `original_text_presence`
|
||||
- ensures target segment exists and `original_text` exists in current working segment text.
|
||||
- `non_empty_corrected_text`
|
||||
- rejects proposals whose previewed resulting segment text would be empty or whitespace-only.
|
||||
- `no_effect`
|
||||
- rejects proposals where `original_text == corrected_text`.
|
||||
- `protected_terms`
|
||||
- protects glossary-derived terms from unsafe mutations in non-glossary modules.
|
||||
- glossary stages use glossary-specific protection logic but still report this same stable key.
|
||||
|
||||
### LLM-backed validators
|
||||
|
||||
LLM-backed:
|
||||
- `spoken_form_plausibility`
|
||||
- checks whether proposed spoken-form change remains plausible in transcript context.
|
||||
- `meaning_reversal_review`
|
||||
- checks for likely meaning reversal or semantic contradiction.
|
||||
- `editorial_review`
|
||||
- performs conservative editorial safety review.
|
||||
|
||||
## Built-in module chains
|
||||
`glossary`:
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
|
||||
Current built-in chains resolved from `internal/validators/chains.go`:
|
||||
`homophones`:
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
|
||||
- `glossary`
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
`spoken_word`:
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `editorial_review`
|
||||
- `meaning_reversal_review`
|
||||
|
||||
- `homophones`
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
`grammar`:
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `editorial_review`
|
||||
- `meaning_reversal_review`
|
||||
|
||||
- `spoken_word`
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `editorial_review`
|
||||
- `meaning_reversal_review`
|
||||
## Ordering and execution semantics
|
||||
Validator ordering is based on canonical metadata:
|
||||
- deterministic validators run before LLM-backed validators.
|
||||
|
||||
- `grammar`
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `editorial_review`
|
||||
- `meaning_reversal_review`
|
||||
Within each module stage:
|
||||
- proposals are generated per section;
|
||||
- validator chains execute on those proposals;
|
||||
- approved proposals are applied once after section work settles.
|
||||
|
||||
## Protected terms construction
|
||||
## Malformed payload behavior
|
||||
Malformed structured-output from proposal generation and LLM validator calls is downgraded, not treated as a process-fatal transport error.
|
||||
|
||||
`protected_terms` has explicit constructors:
|
||||
- general constructor used by non-glossary modules through the built-in registry
|
||||
- glossary-stage constructor used by glossary chain resolution
|
||||
Current outcomes:
|
||||
- malformed proposal-generation payloads produce section/module warnings and zero proposals for the affected section;
|
||||
- malformed validator decision payloads reject the affected validator batch with warnings;
|
||||
- deterministic validator behavior and runner order remain unchanged.
|
||||
|
||||
Both variants preserve existing behavior and report the stable key `protected_terms`.
|
||||
## Reporting identity
|
||||
Reports and diagnostics use stable validator keys as identifiers.
|
||||
|
||||
## Execution semantics
|
||||
Correction-ledger deterministic-vs-LLM classification is derived from canonical validator metadata, not package-local hardcoded maps.
|
||||
|
||||
- modules execute serially;
|
||||
- section proposal work can run concurrently within a module;
|
||||
- deterministic validators run before LLM-backed validators;
|
||||
- malformed module proposal payloads are downgraded to section-scoped module warnings with zero proposals for the affected section rather than module failure;
|
||||
- malformed/missing/duplicate/unknown LLM validator decisions reject the affected validator batch with warnings instead of failing the module;
|
||||
- oversized single-proposal validator inputs reject only the affected proposal under that validator;
|
||||
- approved proposals are applied once per module after section work settles.
|
||||
|
||||
## Validator rejections vs proposal-application skips
|
||||
|
||||
- validator rejection:
|
||||
- proposal is denied by validator-chain review and appears in validator rejection reporting with validator key and reason code.
|
||||
- proposal-application skip:
|
||||
- proposal passed validators but could not be applied under replacement-policy semantics (for example no matching span at apply time).
|
||||
- module warning:
|
||||
- malformed proposal-generation payloads and malformed validator batches are recorded in module warning records and diagnostics without writing success stderr.
|
||||
|
||||
These are separate outcomes and are reported separately.
|
||||
|
||||
## Reporting and diagnostics identity
|
||||
|
||||
- report validator decision/rejection entries use stable validator keys in `validator_name`.
|
||||
- report module results include warning records for malformed module-stage LLM payloads.
|
||||
- validator LLM diagnostics include validator identity in interaction metadata and structured response schema metadata.
|
||||
- correction ledger entries include deterministic and LLM validator decision snapshots keyed by the same stable validator keys, and keep validator rejection distinct from application-level skip.
|
||||
|
||||
Prompt assets are unchanged by the validator package-ownership refactor and remain built-in under `internal/prompts`.
|
||||
|
||||
## Configurable knobs that remain supported
|
||||
|
||||
- per-module confidence thresholds (`thresholds.*` / equivalent env+CLI overrides)
|
||||
- validation batching limits (`validation_max_prompt_tokens` / equivalent env+CLI overrides)
|
||||
- validation LLM model/base URL/timeout/retries/concurrency settings
|
||||
|
||||
These tune validator behavior without exposing arbitrary user-defined chains.
|
||||
## Prompt assets
|
||||
LLM validator prompt assets and prompt metadata are documented in [Prompts](./prompts.md).
|
||||
|
||||
@@ -1,51 +1,48 @@
|
||||
# Audita Configuration
|
||||
|
||||
This document describes Audita's versioned YAML config support and related commands.
|
||||
|
||||
## Purpose
|
||||
|
||||
Audita's config file provides a stable place for pipeline defaults and runtime tuning that would otherwise require many environment variables or CLI flags.
|
||||
|
||||
Use config files for baseline settings, then use environment variables and CLI flags for deployment and per-run overrides.
|
||||
|
||||
## Supported version
|
||||
|
||||
Current supported config version:
|
||||
## Scope
|
||||
This document defines the supported versioned YAML configuration model and runtime precedence behavior.
|
||||
|
||||
## Supported file version
|
||||
Current supported config file version:
|
||||
- `version: 1`
|
||||
|
||||
Rules:
|
||||
|
||||
- missing `version` fails validation;
|
||||
- unknown versions fail validation;
|
||||
- unknown fields fail validation (strict decoding).
|
||||
Validation rules:
|
||||
- missing `version` fails;
|
||||
- unsupported version fails;
|
||||
- unknown YAML fields fail (strict decoding).
|
||||
|
||||
## Config path resolution
|
||||
For `audita process` and `audita config print-effective`, path resolution order is:
|
||||
1. `--config <path>`
|
||||
2. `AUDITA_CONFIG`
|
||||
3. `/usr/local/etc/audita/config.yml` (if present)
|
||||
4. `/etc/audita/config.yml` (if present)
|
||||
|
||||
For `audita process`, config path resolution is:
|
||||
Missing-path behavior:
|
||||
- missing `--config` path is an error;
|
||||
- missing `AUDITA_CONFIG` path is an error;
|
||||
- missing both default paths is non-fatal.
|
||||
|
||||
1. `--config <path>` if provided
|
||||
2. `AUDITA_CONFIG` if set and `--config` is not provided
|
||||
3. default `/usr/local/etc/audita/config.yml` if present
|
||||
4. fallback default `/etc/audita/config.yml` if present
|
||||
|
||||
Missing-file behavior:
|
||||
|
||||
- missing `--config` path: hard failure;
|
||||
- missing `AUDITA_CONFIG` path: hard failure;
|
||||
- missing both default-path files: non-fatal, run continues.
|
||||
|
||||
## Precedence model
|
||||
|
||||
Effective config precedence is:
|
||||
|
||||
1. built-in defaults
|
||||
## Effective precedence
|
||||
`audita process` effective precedence:
|
||||
1. defaults
|
||||
2. file config
|
||||
3. environment overrides
|
||||
4. CLI overrides
|
||||
|
||||
## Supported YAML fields
|
||||
`audita config print-effective` uses:
|
||||
1. defaults
|
||||
2. file config
|
||||
3. environment overrides
|
||||
|
||||
`audita config validate` intentionally uses file-only validation:
|
||||
1. defaults
|
||||
2. file config
|
||||
|
||||
Environment overrides are not applied in `config validate`.
|
||||
|
||||
## Supported top-level YAML fields
|
||||
```yaml
|
||||
version: 1
|
||||
|
||||
@@ -62,7 +59,6 @@ llm:
|
||||
api_key_env: AUDITA_LLM_API_KEY
|
||||
timeout: 120s
|
||||
max_retries: 3
|
||||
|
||||
validation:
|
||||
base_url: https://openrouter.ai/api/v1
|
||||
model: openrouter/google/gemma-4-31b-it
|
||||
@@ -100,91 +96,54 @@ diagnostics:
|
||||
retention: auto
|
||||
```
|
||||
|
||||
`context.description` provides background-only transcript context for prompts.
|
||||
If both config and CLI provide a description, `--transcript-description` takes precedence.
|
||||
## Module and output-schema validation
|
||||
`pipeline.modules` keys are validated against the built-in supported module catalog.
|
||||
|
||||
`output.schema` supports the built-in output schema registry values:
|
||||
- `bare-segments` (default)
|
||||
Supported module keys:
|
||||
- `glossary`
|
||||
- `homophones`
|
||||
- `spoken_word`
|
||||
- `grammar`
|
||||
|
||||
Repeated supported module keys are allowed.
|
||||
|
||||
`output.schema` is validated against the built-in output schema catalog.
|
||||
|
||||
Supported output schema keys:
|
||||
- `bare-segments`
|
||||
- `audita-v1`
|
||||
|
||||
Unknown schema names fail clearly before transcript output is written.
|
||||
Unknown module keys and unknown output schema keys fail validation.
|
||||
|
||||
Duration-like fields accept either:
|
||||
## Duration field parsing
|
||||
Duration-like fields support:
|
||||
- numeric seconds (for example `120`, `3.5`)
|
||||
- duration strings (for example `120s`, `2m`)
|
||||
|
||||
- numeric seconds (for example `120`, `3.5`), or
|
||||
- duration strings (for example `120s`, `2m`).
|
||||
|
||||
For LLM timeouts, duration strings must resolve to whole seconds.
|
||||
LLM timeout duration strings must resolve to whole seconds.
|
||||
|
||||
## Secret handling
|
||||
|
||||
Use `api_key_env` for secrets:
|
||||
|
||||
Use `api_key_env` fields for secrets:
|
||||
- `llm.proposal.api_key_env`
|
||||
- `llm.validation.api_key_env`
|
||||
|
||||
These fields must contain environment variable names, not secret values.
|
||||
These fields store environment variable names, not secret values.
|
||||
|
||||
At runtime, Audita resolves those names from the process environment.
|
||||
|
||||
Redaction behavior:
|
||||
|
||||
- run diagnostics `effective-config.json` is redacted;
|
||||
- `audita config print-effective` output is redacted;
|
||||
- API keys are never emitted in plaintext by those outputs.
|
||||
|
||||
## Config commands
|
||||
|
||||
Validate a config file:
|
||||
Resolved secret values are redacted from:
|
||||
- `audita config print-effective` output;
|
||||
- diagnostics `effective-config.json`;
|
||||
- report and diagnostics payloads.
|
||||
|
||||
## Commands
|
||||
Validate a file config:
|
||||
```sh
|
||||
audita config validate --config ./audita.yml
|
||||
```
|
||||
|
||||
Print redacted effective config:
|
||||
|
||||
```sh
|
||||
audita config print-effective --config ./audita.yml
|
||||
```
|
||||
|
||||
`print-effective` loads defaults, then file config, then environment overrides.
|
||||
|
||||
## Example: local OpenAI-compatible endpoint
|
||||
|
||||
```yaml
|
||||
version: 1
|
||||
|
||||
llm:
|
||||
proposal:
|
||||
base_url: http://localhost:8000/v1
|
||||
model: local/proposal-model
|
||||
api_key_env: AUDITA_LLM_API_KEY
|
||||
timeout: 90s
|
||||
max_retries: 2
|
||||
|
||||
validation:
|
||||
base_url: http://localhost:8000/v1
|
||||
model: local/validation-model
|
||||
api_key_env: AUDITA_VALIDATION_LLM_API_KEY
|
||||
timeout: 90s
|
||||
max_retries: 2
|
||||
|
||||
pipeline:
|
||||
modules: [glossary, homophones, glossary, spoken_word, grammar]
|
||||
|
||||
diagnostics:
|
||||
work_dir: /tmp/audita
|
||||
retention: auto
|
||||
```
|
||||
|
||||
## Compatibility notes
|
||||
|
||||
Existing environment variables and lower-level CLI flags remain available for compatibility.
|
||||
|
||||
Current guidance:
|
||||
|
||||
- prefer file config for baseline behavior;
|
||||
- keep environment variables for secrets/deployment-specific overrides;
|
||||
- use CLI flags for per-run overrides.
|
||||
- validator chains are built-in and are not user-configurable in config.
|
||||
- prompt source selection and filesystem prompt overrides are not config options.
|
||||
Legacy compatibility flags and environment aliases remain available where implemented, but the stable configuration surface is the versioned YAML model described above.
|
||||
|
||||
33
docs/development.md
Normal file
33
docs/development.md
Normal file
@@ -0,0 +1,33 @@
|
||||
# Audita Development Workflow
|
||||
|
||||
## Scope
|
||||
This document defines the canonical contributor workflow and engineering conventions for this repository.
|
||||
|
||||
## Workflow
|
||||
1. Start from a clean understanding of scope and constraints.
|
||||
2. Make focused changes that preserve existing public behavior unless behavior change is explicitly intended.
|
||||
3. Run targeted tests for touched packages.
|
||||
4. Run `go test ./...` before finalizing substantial changes.
|
||||
5. Update affected documentation so it describes current behavior only.
|
||||
|
||||
## Engineering conventions
|
||||
- Keep module packages separate: `glossary`, `homophones`, `spoken_word`, `grammar`.
|
||||
- Prefer narrow shared helpers and catalogs over broad abstractions.
|
||||
- Preserve diagnostics artifact naming and report field contracts unless intentionally changed.
|
||||
- Preserve CLI/config precedence semantics unless intentionally changed.
|
||||
- Treat stable validator keys, prompt identifiers, and output-schema keys as contract surfaces.
|
||||
|
||||
## Configuration and runtime expectations
|
||||
- `audita process` precedence is defaults -> file -> env -> CLI.
|
||||
- `audita config validate` validates file config merged onto defaults only.
|
||||
- `audita config print-effective` includes environment overrides and prints redacted JSON.
|
||||
|
||||
## Testing expectations
|
||||
- Add tests for new behavior and for bug fixes.
|
||||
- Keep deterministic fixtures stable.
|
||||
- Do not reduce existing parity, release-fixture, subprocess, or module-specific coverage without equivalent replacement.
|
||||
|
||||
## Commit discipline
|
||||
- Keep commits scoped and reviewable.
|
||||
- Avoid mixing unrelated refactors with behavior changes.
|
||||
- Use clear plain-English commit messages.
|
||||
27
docs/documentation/policy.md
Normal file
27
docs/documentation/policy.md
Normal file
@@ -0,0 +1,27 @@
|
||||
# Documentation Policy
|
||||
|
||||
## Scope
|
||||
This policy defines how project documentation should be authored and maintained.
|
||||
|
||||
## Core rules
|
||||
- Document the current behavior of the codebase.
|
||||
- Remove stale behavior descriptions promptly when code changes.
|
||||
- Do not describe development history in architecture or behavior docs unless a document is explicitly historical.
|
||||
- Do not use architecture or behavior docs as changelogs.
|
||||
- Prefer rewriting stale sections from scratch when substantial behavior or ownership changes occur.
|
||||
|
||||
## Consistency requirements
|
||||
- Keep command examples aligned with current CLI surfaces.
|
||||
- Keep configuration examples aligned with supported fields and precedence.
|
||||
- Keep architecture package ownership descriptions aligned with current code layout.
|
||||
- Keep stable contract identifiers accurate (module keys, validator keys, output-schema keys, report metadata fields).
|
||||
|
||||
## Cross-document expectations
|
||||
- `docs/architecture/*` documents runtime behavior and package ownership.
|
||||
- `docs/configuration.md` documents config schema and precedence.
|
||||
- `docs/development.md` documents contributor workflow and engineering conventions.
|
||||
|
||||
## Review expectations for documentation changes
|
||||
- Verify referenced files and links exist.
|
||||
- Verify examples match current behavior.
|
||||
- Prefer concise, direct language and avoid speculative future claims.
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"sort"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -48,12 +49,6 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
||||
}
|
||||
|
||||
entries := make([]correctionLedgerEntry, 0)
|
||||
llmBacked := map[string]bool{
|
||||
"spoken_form_plausibility": true,
|
||||
"meaning_reversal_review": true,
|
||||
"editorial_review": true,
|
||||
}
|
||||
|
||||
for _, module := range runOutput.ModuleResults {
|
||||
decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord)
|
||||
for _, decision := range module.ValidatorDecisions {
|
||||
@@ -72,8 +67,8 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
||||
AppliedCorrectedText: change.CorrectedText,
|
||||
ReplacementPolicy: string(module.ReplacementPolicy),
|
||||
Disposition: correctionDispositionApplied,
|
||||
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false, llmBacked),
|
||||
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true, llmBacked),
|
||||
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false),
|
||||
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true),
|
||||
})
|
||||
}
|
||||
for _, change := range module.SkippedChanges {
|
||||
@@ -89,8 +84,8 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
||||
Disposition: correctionDispositionSkipped,
|
||||
DispositionReasonCode: string(change.SkipReason),
|
||||
DispositionMessage: change.Message,
|
||||
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false, llmBacked),
|
||||
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true, llmBacked),
|
||||
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false),
|
||||
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true),
|
||||
})
|
||||
}
|
||||
for _, rejection := range module.ValidatorRejected {
|
||||
@@ -106,8 +101,8 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
||||
Disposition: correctionDispositionRejected,
|
||||
DispositionReasonCode: rejection.ReasonCode,
|
||||
DispositionMessage: rejection.Message,
|
||||
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], false, llmBacked),
|
||||
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], true, llmBacked),
|
||||
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], false),
|
||||
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], true),
|
||||
})
|
||||
}
|
||||
if module.Status == runner.ModuleStatusFailed {
|
||||
@@ -135,13 +130,14 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
|
||||
return entries
|
||||
}
|
||||
|
||||
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool, llmBacked map[string]bool) []ledgerValidatorDecisionRecord {
|
||||
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []ledgerValidatorDecisionRecord {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]ledgerValidatorDecisionRecord, 0, len(in))
|
||||
for _, decision := range in {
|
||||
if llmBacked[decision.ValidatorName] != wantLLM {
|
||||
isLLMBacked := validatormetadata.ClassForKey(decision.ValidatorName) == validatormetadata.ExecutionClassLLMBacked
|
||||
if isLLMBacked != wantLLM {
|
||||
continue
|
||||
}
|
||||
out = append(out, ledgerValidatorDecisionRecord{
|
||||
|
||||
46
internal/cli/review_artifacts_test.go
Normal file
46
internal/cli/review_artifacts_test.go
Normal file
@@ -0,0 +1,46 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
)
|
||||
|
||||
func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(t *testing.T) {
|
||||
output := &runner.RunOutput{
|
||||
ModuleResults: []runner.ModuleResult{
|
||||
{
|
||||
ModuleKey: "glossary",
|
||||
ModuleInstance: "glossary",
|
||||
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
|
||||
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
||||
{ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
||||
{ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
|
||||
},
|
||||
AppliedChanges: []proposals.AppliedChange{
|
||||
{
|
||||
ProposalIndex: 3,
|
||||
ModuleKey: "glossary",
|
||||
ModuleInstance: "glossary",
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "gestures",
|
||||
CorrectedText: "Jesters",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ledger := buildCorrectionLedger("/tmp/audita-run-id", output)
|
||||
if len(ledger) != 1 {
|
||||
t.Fatalf("expected one ledger entry, got %d", len(ledger))
|
||||
}
|
||||
entry := ledger[0]
|
||||
if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "proposal_shape" {
|
||||
t.Fatalf("unexpected deterministic decision split: %+v", entry.DeterministicValidatorResults)
|
||||
}
|
||||
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "spoken_form_plausibility" {
|
||||
t.Fatalf("unexpected llm-backed decision split: %+v", entry.LLMValidatorResults)
|
||||
}
|
||||
}
|
||||
@@ -375,30 +375,28 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
||||
fmt.Fprintf(stderr, "audita process: invalid CLI configuration: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
configPath, configSource, err := resolveConfigPath(configPathOverride, configPathOverrideSet, os.LookupEnv)
|
||||
effectiveConfig, err := config.LoadEffectiveConfig(configPathOverride, configPathOverrideSet)
|
||||
if err != nil {
|
||||
var effectiveConfigErr *config.EffectiveConfigError
|
||||
if errors.As(err, &effectiveConfigErr) {
|
||||
switch effectiveConfigErr.Kind {
|
||||
case config.EffectiveConfigErrorLoadFile, config.EffectiveConfigErrorApplyFile:
|
||||
fmt.Fprintf(stderr, "audita process: invalid config file: %v\n", effectiveConfigErr)
|
||||
case config.EffectiveConfigErrorApplyEnv:
|
||||
fmt.Fprintf(stderr, "audita process: invalid environment configuration: %v\n", effectiveConfigErr)
|
||||
default:
|
||||
fmt.Fprintf(stderr, "audita process: %v\n", effectiveConfigErr)
|
||||
}
|
||||
} else {
|
||||
fmt.Fprintf(stderr, "audita process: %v\n", err)
|
||||
}
|
||||
return 2
|
||||
}
|
||||
|
||||
cfg := config.Default()
|
||||
var configVersion *int
|
||||
if configPath != "" {
|
||||
fileCfg, fileErr := config.LoadFileConfig(configPath)
|
||||
if fileErr != nil {
|
||||
fmt.Fprintf(stderr, "audita process: invalid config file: %v\n", fileErr)
|
||||
return 2
|
||||
}
|
||||
if applyErr := cfg.ApplyFileConfig(fileCfg); applyErr != nil {
|
||||
fmt.Fprintf(stderr, "audita process: invalid config file: %v\n", applyErr)
|
||||
return 2
|
||||
}
|
||||
configVersion = &fileCfg.Version
|
||||
}
|
||||
if err := cfg.ApplyEnvOverrides(); err != nil {
|
||||
fmt.Fprintf(stderr, "audita process: invalid environment configuration: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
cfg := effectiveConfig.Config
|
||||
configPath := effectiveConfig.ConfigPath
|
||||
configSource := effectiveConfig.ConfigSource
|
||||
configVersion := effectiveConfig.ConfigVersion
|
||||
|
||||
fs, pFlags := newProcessFlagSet(cfg, stderr)
|
||||
|
||||
@@ -531,9 +529,9 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
||||
if runErr != nil {
|
||||
if runDir != nil && runOutput != nil {
|
||||
if runOutput.Utilization != nil {
|
||||
_ = runDir.WriteJSONArtifact("utilization-diagnostics.json", runOutput.Utilization)
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
||||
}
|
||||
_ = runDir.WriteJSONArtifact("correction-ledger.json", buildCorrectionLedger(runDir.Path(), runOutput))
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, buildCorrectionLedger(runDir.Path(), runOutput))
|
||||
}
|
||||
errorPhase, errorMessage := extractErrorPhase(runErr)
|
||||
report := buildProcessReport("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput)
|
||||
@@ -560,9 +558,9 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
|
||||
|
||||
if runDir != nil && runOutput != nil {
|
||||
if runOutput.Utilization != nil {
|
||||
_ = runDir.WriteJSONArtifact("utilization-diagnostics.json", runOutput.Utilization)
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, runOutput.Utilization)
|
||||
}
|
||||
_ = runDir.WriteJSONArtifact("correction-ledger.json", buildCorrectionLedger(runDir.Path(), runOutput))
|
||||
_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, buildCorrectionLedger(runDir.Path(), runOutput))
|
||||
}
|
||||
|
||||
report := buildProcessReport("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput)
|
||||
@@ -653,6 +651,10 @@ func runConfigValidate(args []string, stdout, stderr io.Writer) int {
|
||||
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
fmt.Fprintf(stderr, "audita config validate: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
fmt.Fprintln(stdout, "config is valid")
|
||||
return 0
|
||||
}
|
||||
@@ -675,28 +677,13 @@ func runConfigPrintEffective(args []string, stdout, stderr io.Writer) int {
|
||||
|
||||
configPathValue := strings.TrimSpace(*configPath)
|
||||
configPathSet := configPathValue != ""
|
||||
path, _, err := resolveConfigPath(configPathValue, configPathSet, os.LookupEnv)
|
||||
effectiveConfig, err := config.LoadEffectiveConfig(configPathValue, configPathSet)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
|
||||
cfg := config.Default()
|
||||
if path != "" {
|
||||
fileCfg, fileErr := config.LoadFileConfig(path)
|
||||
if fileErr != nil {
|
||||
fmt.Fprintf(stderr, "audita config print-effective: %v\n", fileErr)
|
||||
return 2
|
||||
}
|
||||
if applyErr := cfg.ApplyFileConfig(fileCfg); applyErr != nil {
|
||||
fmt.Fprintf(stderr, "audita config print-effective: %v\n", applyErr)
|
||||
return 2
|
||||
}
|
||||
}
|
||||
if err := cfg.ApplyEnvOverrides(); err != nil {
|
||||
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
cfg := effectiveConfig.Config
|
||||
|
||||
redacted := cfg.Redacted()
|
||||
out, err := json.MarshalIndent(redacted, "", " ")
|
||||
@@ -743,22 +730,9 @@ func buildProcessReport(status string, inv processInvocation, runDir *diagnostic
|
||||
ErrorPhase: errorPhase,
|
||||
}
|
||||
if runDir != nil {
|
||||
diagnosticsDir := runDir.Path()
|
||||
report.Diagnostics = &reporting.DiagnosticsMetadata{
|
||||
DirectoryPath: diagnosticsDir,
|
||||
SourceTranscriptPath: filepath.Join(diagnosticsDir, "source-transcript.json"),
|
||||
ParsedSourceTranscriptPath: filepath.Join(diagnosticsDir, "source-transcript-parsed.json"),
|
||||
NormalizedTranscriptPath: filepath.Join(diagnosticsDir, "normalized-transcript.json"),
|
||||
NormalizationSummaryPath: filepath.Join(diagnosticsDir, "normalization-summary.json"),
|
||||
ChunkingSummaryPath: filepath.Join(diagnosticsDir, "chunking-summary.json"),
|
||||
UtilizationSummaryPath: filepath.Join(diagnosticsDir, "utilization-diagnostics.json"),
|
||||
CorrectionLedgerPath: filepath.Join(diagnosticsDir, "correction-ledger.json"),
|
||||
InvocationMetadataPath: filepath.Join(diagnosticsDir, "invocation.json"),
|
||||
RedactedEffectiveConfigPath: filepath.Join(diagnosticsDir, "effective-config.json"),
|
||||
}
|
||||
if status == "failed" {
|
||||
report.Diagnostics.ErrorLogPath = filepath.Join(diagnosticsDir, "error.log")
|
||||
}
|
||||
runSucceeded := status == "success"
|
||||
metadata := diagnostics.BuildDiagnosticsMetadata(runDir.Path(), runSucceeded)
|
||||
report.Diagnostics = &metadata
|
||||
}
|
||||
if errorMessage != "" {
|
||||
report.ErrorMessage = errorMessage
|
||||
@@ -984,47 +958,6 @@ func findConfigPathOverride(args []string) (path string, set bool, err error) {
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
var statConfigPath = os.Stat
|
||||
|
||||
func resolveConfigPath(cliConfigPath string, cliConfigPathSet bool, lookup func(string) (string, bool)) (path string, source string, err error) {
|
||||
if cliConfigPathSet {
|
||||
path = strings.TrimSpace(cliConfigPath)
|
||||
if path == "" {
|
||||
return "", "", fmt.Errorf("--config requires a non-empty path")
|
||||
}
|
||||
if _, statErr := statConfigPath(path); statErr != nil {
|
||||
if os.IsNotExist(statErr) {
|
||||
return "", "", fmt.Errorf("config file not found: %s", path)
|
||||
}
|
||||
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
|
||||
}
|
||||
return path, "flag", nil
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_CONFIG"); ok {
|
||||
path = strings.TrimSpace(raw)
|
||||
if path == "" {
|
||||
return "", "", fmt.Errorf("AUDITA_CONFIG must not be empty")
|
||||
}
|
||||
if _, statErr := statConfigPath(path); statErr != nil {
|
||||
if os.IsNotExist(statErr) {
|
||||
return "", "", fmt.Errorf("config file not found: %s", path)
|
||||
}
|
||||
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
|
||||
}
|
||||
return path, "env", nil
|
||||
}
|
||||
|
||||
for _, defaultPath := range config.DefaultConfigSearchPaths {
|
||||
if _, statErr := statConfigPath(defaultPath); statErr == nil {
|
||||
return defaultPath, "default", nil
|
||||
} else if !os.IsNotExist(statErr) {
|
||||
return "", "", fmt.Errorf("cannot access config file %s: %w", defaultPath, statErr)
|
||||
}
|
||||
}
|
||||
return "", "", nil
|
||||
}
|
||||
|
||||
func isHelpCommand(args []string) bool {
|
||||
if len(args) == 0 {
|
||||
return false
|
||||
|
||||
@@ -21,11 +21,11 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/testsupport"
|
||||
)
|
||||
|
||||
func TestRunRootHelp(t *testing.T) {
|
||||
@@ -99,66 +99,6 @@ func TestRunProcessHelpListsExpectedFlags(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveConfigPathDefaultIgnoredWhenMissing(t *testing.T) {
|
||||
lookup := func(string) (string, bool) { return "", false }
|
||||
path, source, err := resolveConfigPath("", false, lookup)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if path != "" || source != "" {
|
||||
t.Fatalf("expected no config path/source, got path=%q source=%q", path, source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveConfigPathDefaultPrefersUsrLocalOverEtc(t *testing.T) {
|
||||
oldStat := statConfigPath
|
||||
statConfigPath = func(path string) (os.FileInfo, error) {
|
||||
if path == config.DefaultConfigPathUsrLocal || path == config.DefaultConfigPath {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
t.Cleanup(func() { statConfigPath = oldStat })
|
||||
|
||||
lookup := func(string) (string, bool) { return "", false }
|
||||
path, source, err := resolveConfigPath("", false, lookup)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if source != "default" {
|
||||
t.Fatalf("expected default source, got %q", source)
|
||||
}
|
||||
if path != config.DefaultConfigPathUsrLocal {
|
||||
t.Fatalf("expected %q, got %q", config.DefaultConfigPathUsrLocal, path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveConfigPathDefaultFallsBackToEtc(t *testing.T) {
|
||||
oldStat := statConfigPath
|
||||
statConfigPath = func(path string) (os.FileInfo, error) {
|
||||
if path == config.DefaultConfigPathUsrLocal {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
if path == config.DefaultConfigPath {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
t.Cleanup(func() { statConfigPath = oldStat })
|
||||
|
||||
lookup := func(string) (string, bool) { return "", false }
|
||||
path, source, err := resolveConfigPath("", false, lookup)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if source != "default" {
|
||||
t.Fatalf("expected default source, got %q", source)
|
||||
}
|
||||
if path != config.DefaultConfigPath {
|
||||
t.Fatalf("expected %q, got %q", config.DefaultConfigPath, path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConfigValidateSuccess(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
@@ -210,6 +150,23 @@ func TestRunConfigValidateUnknownField(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConfigValidateUnsupportedModuleKey(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
cfgPath := writeFile(t, "config.yml", "version: 1\npipeline:\n modules: [made_up]\n")
|
||||
|
||||
exitCode := Run([]string{"config", "validate", "--config", cfgPath}, &stdout, &stderr)
|
||||
if exitCode == 0 {
|
||||
t.Fatalf("expected failure for unsupported module key")
|
||||
}
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "unsupported module key") {
|
||||
t.Fatalf("expected unsupported module key error, got %q", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConfigPrintEffectiveOutputsRedactedJSON(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
@@ -242,6 +199,45 @@ func TestRunConfigPrintEffectiveOutputsRedactedJSON(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConfigPrintEffectiveAppliesFileThenEnvironment(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
cfgPath := writeFile(t, "config.yml", "version: 1\nllm:\n proposal:\n model: file-model\n")
|
||||
t.Setenv("AUDITA_MODEL", "env-model")
|
||||
|
||||
exitCode := Run([]string{"config", "print-effective", "--config", cfgPath}, &stdout, &stderr)
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||
}
|
||||
|
||||
var out struct {
|
||||
PrimaryLLM struct {
|
||||
Model string `json:"Model"`
|
||||
} `json:"PrimaryLLM"`
|
||||
}
|
||||
if err := json.Unmarshal(stdout.Bytes(), &out); err != nil {
|
||||
t.Fatalf("expected valid JSON output, got error: %v output=%q", err, stdout.String())
|
||||
}
|
||||
if out.PrimaryLLM.Model != "env-model" {
|
||||
t.Fatalf("expected env model override in print-effective output, got %q", out.PrimaryLLM.Model)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConfigValidateIgnoresEnvironmentOverrides(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
cfgPath := writeFile(t, "config.yml", "version: 1\n")
|
||||
t.Setenv("AUDITA_MODULES", "made_up")
|
||||
|
||||
exitCode := Run([]string{"config", "validate", "--config", cfgPath}, &stdout, &stderr)
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected success because config validate is file-only, got %d stderr=%q", exitCode, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "config is valid") {
|
||||
t.Fatalf("expected success message, got %q", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConfigCommandDoesNotRequireTranscriptOrGlossary(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
@@ -369,8 +365,8 @@ diagnostics:
|
||||
|
||||
func TestRunProcessEnvOverridesConfigFile(t *testing.T) {
|
||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{
|
||||
key: "m",
|
||||
"grammar": fakeModule{
|
||||
key: "grammar",
|
||||
policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{
|
||||
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
@@ -392,7 +388,7 @@ func TestRunProcessEnvOverridesConfigFile(t *testing.T) {
|
||||
cfgPath := writeFile(t, "config.yml", `
|
||||
version: 1
|
||||
pipeline:
|
||||
modules: [m]
|
||||
modules: [grammar]
|
||||
llm:
|
||||
proposal:
|
||||
model: file-model
|
||||
@@ -404,7 +400,7 @@ llm:
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||
"--config", cfgPath,
|
||||
"--modules", "m",
|
||||
"--modules", "grammar",
|
||||
}, &stdout, &stderr)
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||
@@ -413,8 +409,8 @@ llm:
|
||||
|
||||
func TestRunProcessCLIOverridesEnvAndConfigFile(t *testing.T) {
|
||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{
|
||||
key: "m",
|
||||
"grammar": fakeModule{
|
||||
key: "grammar",
|
||||
policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{
|
||||
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
@@ -436,7 +432,7 @@ func TestRunProcessCLIOverridesEnvAndConfigFile(t *testing.T) {
|
||||
cfgPath := writeFile(t, "config.yml", `
|
||||
version: 1
|
||||
pipeline:
|
||||
modules: [m]
|
||||
modules: [grammar]
|
||||
llm:
|
||||
proposal:
|
||||
model: file-model
|
||||
@@ -448,7 +444,7 @@ llm:
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||
"--config", cfgPath,
|
||||
"--modules", "m",
|
||||
"--modules", "grammar",
|
||||
"--model", "cli-model",
|
||||
}, &stdout, &stderr)
|
||||
if exitCode != 0 {
|
||||
@@ -495,8 +491,8 @@ diagnostics:
|
||||
|
||||
func TestRunProcessTranscriptDescriptionCLIOverridesConfigFileContextDescription(t *testing.T) {
|
||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{
|
||||
key: "m",
|
||||
"grammar": fakeModule{
|
||||
key: "grammar",
|
||||
policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{
|
||||
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
@@ -517,7 +513,7 @@ func TestRunProcessTranscriptDescriptionCLIOverridesConfigFileContextDescription
|
||||
cfgPath := writeFile(t, "config.yml", `
|
||||
version: 1
|
||||
pipeline:
|
||||
modules: [m]
|
||||
modules: [grammar]
|
||||
context:
|
||||
description: "file transcript description"
|
||||
`)
|
||||
@@ -528,7 +524,7 @@ context:
|
||||
fixturePath("tiny_transcript.json"),
|
||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||
"--config", cfgPath,
|
||||
"--modules", "m",
|
||||
"--modules", "grammar",
|
||||
"--transcript-description", "cli transcript description",
|
||||
}, &stdout, &stderr)
|
||||
if exitCode != 0 {
|
||||
@@ -692,8 +688,8 @@ func TestRunProcessCLIOverridesEnvironment(t *testing.T) {
|
||||
|
||||
func TestRunProcessTranscriptDescriptionDefaultEmpty(t *testing.T) {
|
||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{
|
||||
key: "m",
|
||||
"grammar": fakeModule{
|
||||
key: "grammar",
|
||||
policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{
|
||||
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
@@ -720,7 +716,7 @@ func TestRunProcessTranscriptDescriptionDefaultEmpty(t *testing.T) {
|
||||
exitCode := Run([]string{
|
||||
"process", transcriptPath,
|
||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||
"--modules", "m",
|
||||
"--modules", "grammar",
|
||||
}, &stdout, &stderr)
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||
@@ -729,8 +725,8 @@ func TestRunProcessTranscriptDescriptionDefaultEmpty(t *testing.T) {
|
||||
|
||||
func TestRunProcessTranscriptDescriptionCLIOverrideAndTrim(t *testing.T) {
|
||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{
|
||||
key: "m",
|
||||
"grammar": fakeModule{
|
||||
key: "grammar",
|
||||
policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{
|
||||
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
@@ -757,7 +753,7 @@ func TestRunProcessTranscriptDescriptionCLIOverrideAndTrim(t *testing.T) {
|
||||
exitCode := Run([]string{
|
||||
"process", transcriptPath,
|
||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||
"--modules", "m",
|
||||
"--modules", "grammar",
|
||||
"--transcript-description", " speaker background context ",
|
||||
}, &stdout, &stderr)
|
||||
if exitCode != 0 {
|
||||
@@ -823,8 +819,8 @@ func TestRunProcessRejectsValidationConcurrencyAboveTotalConcurrency(t *testing.
|
||||
|
||||
func TestRunProcessTotalLLMConcurrencyDrivesEffectiveValidationConcurrencyWhenUnset(t *testing.T) {
|
||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{
|
||||
key: "m",
|
||||
"grammar": fakeModule{
|
||||
key: "grammar",
|
||||
policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{
|
||||
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
@@ -865,7 +861,7 @@ func TestRunProcessTotalLLMConcurrencyDrivesEffectiveValidationConcurrencyWhenUn
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"m",
|
||||
"grammar",
|
||||
"--total-llm-concurrency",
|
||||
"4",
|
||||
}, &stdout, &stderr)
|
||||
@@ -908,8 +904,8 @@ func TestRunProcessRejectsProposalConcurrencyAboveTotalConcurrency(t *testing.T)
|
||||
|
||||
func TestRunProcessLegacyLLMConcurrencyAliasSetsTotalAndProposal(t *testing.T) {
|
||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{
|
||||
key: "m",
|
||||
"grammar": fakeModule{
|
||||
key: "grammar",
|
||||
policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{
|
||||
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
@@ -944,7 +940,7 @@ func TestRunProcessLegacyLLMConcurrencyAliasSetsTotalAndProposal(t *testing.T) {
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"m",
|
||||
"grammar",
|
||||
"--llm-concurrency",
|
||||
"3",
|
||||
}, &stdout, &stderr)
|
||||
@@ -955,8 +951,8 @@ func TestRunProcessLegacyLLMConcurrencyAliasSetsTotalAndProposal(t *testing.T) {
|
||||
|
||||
func TestRunProcessLLMConcurrencyFlagsOverrideEnvironment(t *testing.T) {
|
||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{
|
||||
key: "m",
|
||||
"grammar": fakeModule{
|
||||
key: "grammar",
|
||||
policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{
|
||||
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
@@ -997,7 +993,7 @@ func TestRunProcessLLMConcurrencyFlagsOverrideEnvironment(t *testing.T) {
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"m",
|
||||
"grammar",
|
||||
"--total-llm-concurrency",
|
||||
"4",
|
||||
"--proposal-llm-concurrency",
|
||||
@@ -1010,8 +1006,8 @@ func TestRunProcessLLMConcurrencyFlagsOverrideEnvironment(t *testing.T) {
|
||||
|
||||
func TestRunProcessAcceptsLLMConcurrencyEnvironmentVariables(t *testing.T) {
|
||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{
|
||||
key: "m",
|
||||
"grammar": fakeModule{
|
||||
key: "grammar",
|
||||
policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{
|
||||
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
@@ -1052,7 +1048,7 @@ func TestRunProcessAcceptsLLMConcurrencyEnvironmentVariables(t *testing.T) {
|
||||
"--glossary",
|
||||
fixturePath("tiny_glossary.yaml"),
|
||||
"--modules",
|
||||
"m",
|
||||
"grammar",
|
||||
}, &stdout, &stderr)
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||
@@ -1883,12 +1879,12 @@ func TestRunProcessInjectedFactoryExecutesRunnerAndReportsModules(t *testing.T)
|
||||
return validators.Result{ValidatorName: "allow", Decisions: decisions}, nil
|
||||
}}
|
||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"glossary": fakeModule{key: "glossary", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "Hello", CorrectedText: "Hi", Confidence: 1},
|
||||
}, nil
|
||||
}},
|
||||
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"homophones": fakeModule{key: "homophones", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
if req.WorkingTranscript.Segments[0].Text != "Hi world" {
|
||||
t.Fatalf("expected module 2 to see module 1 changes, got %q", req.WorkingTranscript.Segments[0].Text)
|
||||
}
|
||||
@@ -1911,7 +1907,7 @@ func TestRunProcessInjectedFactoryExecutesRunnerAndReportsModules(t *testing.T)
|
||||
exitCode := Run([]string{
|
||||
"process", transcriptPath,
|
||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||
"--modules", "m1,m2",
|
||||
"--modules", "glossary,homophones",
|
||||
"--output", outputPath,
|
||||
"--report-json", reportPath,
|
||||
"--work-dir", workDir,
|
||||
@@ -1970,7 +1966,7 @@ func TestRunProcessInjectedFactoryLLMValidatorResultsInReports(t *testing.T) {
|
||||
}},
|
||||
}
|
||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"grammar": fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "Hello", CorrectedText: "Hi", Confidence: 1}}, nil
|
||||
}},
|
||||
}}
|
||||
@@ -1990,7 +1986,7 @@ func TestRunProcessInjectedFactoryLLMValidatorResultsInReports(t *testing.T) {
|
||||
exitCode := Run([]string{
|
||||
"process", transcriptPath,
|
||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||
"--modules", "m",
|
||||
"--modules", "grammar",
|
||||
"--output", outputPath,
|
||||
"--report-json", reportPath,
|
||||
"--work-dir", workDir,
|
||||
@@ -2011,7 +2007,7 @@ func TestRunProcessInjectedFactoryLLMValidatorResultsInReports(t *testing.T) {
|
||||
|
||||
func TestRunProcessInjectedFactorySkippedKeepsAutoRetention(t *testing.T) {
|
||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"glossary": fakeModule{key: "glossary", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term", Confidence: 1},
|
||||
}, nil
|
||||
@@ -2027,7 +2023,7 @@ func TestRunProcessInjectedFactorySkippedKeepsAutoRetention(t *testing.T) {
|
||||
exitCode := Run([]string{
|
||||
"process", transcriptPath,
|
||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||
"--modules", "m1",
|
||||
"--modules", "glossary",
|
||||
"--work-dir", workDir,
|
||||
"--work-dir-retention", "auto",
|
||||
}, &stdout, &stderr)
|
||||
@@ -2041,7 +2037,7 @@ func TestRunProcessInjectedFactorySkippedKeepsAutoRetention(t *testing.T) {
|
||||
|
||||
func TestRunProcessInjectedFactoryFailureWritesFailedReport(t *testing.T) {
|
||||
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"glossary": fakeModule{key: "glossary", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return nil, errors.New("test failure")
|
||||
}},
|
||||
}}
|
||||
@@ -2057,7 +2053,7 @@ func TestRunProcessInjectedFactoryFailureWritesFailedReport(t *testing.T) {
|
||||
exitCode := Run([]string{
|
||||
"process", transcriptPath,
|
||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||
"--modules", "m1",
|
||||
"--modules", "glossary",
|
||||
"--work-dir", workDir,
|
||||
"--work-dir-retention", "always",
|
||||
"--report-json", reportPath,
|
||||
@@ -2089,14 +2085,8 @@ func TestRunProcessInjectedFactoryFailureWritesFailedReport(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProcessProductionRegistryUnsupportedModuleFailsCleanly(t *testing.T) {
|
||||
cfg := modules.Dependencies{}
|
||||
processModuleFactory = modules.NewFactory(cfg)
|
||||
t.Cleanup(func() { processModuleFactory = nil })
|
||||
|
||||
func TestRunProcessUnsupportedModuleFailsDuringConfigValidation(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
workDir := t.TempDir()
|
||||
reportPath := filepath.Join(t.TempDir(), "report.json")
|
||||
transcriptPath := writeFile(t, "transcript.json", `[
|
||||
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}
|
||||
]`)
|
||||
@@ -2105,9 +2095,6 @@ func TestRunProcessProductionRegistryUnsupportedModuleFailsCleanly(t *testing.T)
|
||||
"process", transcriptPath,
|
||||
"--glossary", fixturePath("tiny_glossary.yaml"),
|
||||
"--modules", "made_up",
|
||||
"--work-dir", workDir,
|
||||
"--work-dir-retention", "always",
|
||||
"--report-json", reportPath,
|
||||
}, &stdout, &stderr)
|
||||
if exitCode == 0 {
|
||||
t.Fatal("expected failure exit code")
|
||||
@@ -2115,23 +2102,12 @@ func TestRunProcessProductionRegistryUnsupportedModuleFailsCleanly(t *testing.T)
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "runner_execution") {
|
||||
t.Fatalf("expected runner_execution failure on stderr, got %q", stderr.String())
|
||||
if !strings.Contains(stderr.String(), "invalid CLI configuration") {
|
||||
t.Fatalf("expected config validation failure on stderr, got %q", stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "unsupported module key") {
|
||||
t.Fatalf("expected explicit unsupported module message, got %q", stderr.String())
|
||||
}
|
||||
|
||||
report := readProcessReport(t, reportPath)
|
||||
if report.Status != "failed" {
|
||||
t.Fatalf("expected failed report status, got %q", report.Status)
|
||||
}
|
||||
if report.ErrorPhase != "runner_execution" {
|
||||
t.Fatalf("expected runner_execution phase, got %q", report.ErrorPhase)
|
||||
}
|
||||
if !strings.Contains(report.ErrorMessage, "unsupported module key") {
|
||||
t.Fatalf("expected report error message to mention unsupported module, got %q", report.ErrorMessage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProcessExplicitUnsupportedModulesFailClearly(t *testing.T) {
|
||||
@@ -4360,12 +4336,7 @@ func writeFile(t *testing.T, name string, content string) string {
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read file %q: %v", path, err)
|
||||
}
|
||||
return data
|
||||
return testsupport.ReadFile(t, path)
|
||||
}
|
||||
|
||||
func readProcessReport(t *testing.T, path string) reporting.ProcessReport {
|
||||
@@ -4379,13 +4350,5 @@ func readProcessReport(t *testing.T, path string) reporting.ProcessReport {
|
||||
}
|
||||
|
||||
func onlyRunDir(t *testing.T, workDir string) string {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(workDir)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read work dir %q: %v", workDir, err)
|
||||
}
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("expected exactly one run dir in %q, got %d", workDir, len(entries))
|
||||
}
|
||||
return filepath.Join(workDir, entries[0].Name())
|
||||
return testsupport.OnlyRunDir(t, workDir)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,8 @@ package config
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||
)
|
||||
|
||||
type WorkDirRetention string
|
||||
@@ -14,7 +16,7 @@ const (
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultModulesCSV = "glossary,homophones,glossary,spoken_word,grammar"
|
||||
DefaultModulesCSV = modulecatalog.KeyGlossary + "," + modulecatalog.KeyHomophones + "," + modulecatalog.KeyGlossary + "," + modulecatalog.KeySpokenWord + "," + modulecatalog.KeyGrammar
|
||||
DefaultOutputSchema = "bare-segments"
|
||||
DefaultPrimaryModel = "openrouter/google/gemma-4-31b-it"
|
||||
DefaultPrimaryBaseURL = "https://openrouter.ai/api/v1"
|
||||
|
||||
@@ -4,6 +4,9 @@ import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
|
||||
)
|
||||
|
||||
func TestDefaultConfigValues(t *testing.T) {
|
||||
@@ -318,6 +321,38 @@ func TestValidationFailures(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationRejectsUnsupportedModuleKey(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.Modules = []string{modulecatalog.KeyGlossary, "made_up"}
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatalf("expected validation error for unsupported module key")
|
||||
}
|
||||
if !strings.Contains(err.Error(), `unsupported module key "made_up"`) {
|
||||
t.Fatalf("expected unsupported module key error, got %q", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationAllowsRepeatedSupportedModuleKeys(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.Modules = []string{modulecatalog.KeyGlossary, modulecatalog.KeyGlossary, modulecatalog.KeyGrammar}
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("expected repeated supported module keys to validate, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationAcceptsAllSupportedOutputSchemas(t *testing.T) {
|
||||
for _, schemaKey := range outputschema.SupportedKeys() {
|
||||
cfg := Default()
|
||||
cfg.OutputSchema = schemaKey
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("expected output schema %q to validate, got %v", schemaKey, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveValidationLLMInheritance(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.PrimaryLLM.APIKey = "primary-key"
|
||||
|
||||
119
internal/core/config/effective_config.go
Normal file
119
internal/core/config/effective_config.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type EffectiveConfigErrorKind string
|
||||
|
||||
const (
|
||||
EffectiveConfigErrorResolvePath EffectiveConfigErrorKind = "resolve_path"
|
||||
EffectiveConfigErrorLoadFile EffectiveConfigErrorKind = "load_file"
|
||||
EffectiveConfigErrorApplyFile EffectiveConfigErrorKind = "apply_file"
|
||||
EffectiveConfigErrorApplyEnv EffectiveConfigErrorKind = "apply_env"
|
||||
)
|
||||
|
||||
type EffectiveConfigError struct {
|
||||
Kind EffectiveConfigErrorKind
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *EffectiveConfigError) Error() string {
|
||||
if e == nil || e.Err == nil {
|
||||
return ""
|
||||
}
|
||||
return e.Err.Error()
|
||||
}
|
||||
|
||||
func (e *EffectiveConfigError) Unwrap() error {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return e.Err
|
||||
}
|
||||
|
||||
type EffectiveConfig struct {
|
||||
Config Config
|
||||
ConfigPath string
|
||||
ConfigSource string
|
||||
ConfigVersion *int
|
||||
}
|
||||
|
||||
func ResolveConfigPath(cliConfigPath string, cliConfigPathSet bool) (path string, source string, err error) {
|
||||
return resolveConfigPathWithLookup(cliConfigPath, cliConfigPathSet, os.LookupEnv, os.Stat, DefaultConfigSearchPaths)
|
||||
}
|
||||
|
||||
func LoadEffectiveConfig(cliConfigPath string, cliConfigPathSet bool) (EffectiveConfig, error) {
|
||||
return loadEffectiveConfigWithLookup(cliConfigPath, cliConfigPathSet, os.LookupEnv, os.Stat, DefaultConfigSearchPaths)
|
||||
}
|
||||
|
||||
func loadEffectiveConfigWithLookup(cliConfigPath string, cliConfigPathSet bool, lookup func(string) (string, bool), statPath func(string) (os.FileInfo, error), defaultSearchPaths []string) (EffectiveConfig, error) {
|
||||
configPath, configSource, err := resolveConfigPathWithLookup(cliConfigPath, cliConfigPathSet, lookup, statPath, defaultSearchPaths)
|
||||
if err != nil {
|
||||
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorResolvePath, Err: err}
|
||||
}
|
||||
|
||||
cfg := Default()
|
||||
var configVersion *int
|
||||
if configPath != "" {
|
||||
fileCfg, fileErr := LoadFileConfig(configPath)
|
||||
if fileErr != nil {
|
||||
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorLoadFile, Err: fileErr}
|
||||
}
|
||||
if applyErr := cfg.ApplyFileConfig(fileCfg); applyErr != nil {
|
||||
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorApplyFile, Err: applyErr}
|
||||
}
|
||||
configVersion = &fileCfg.Version
|
||||
}
|
||||
if applyEnvErr := cfg.applyEnvOverrides(lookup); applyEnvErr != nil {
|
||||
return EffectiveConfig{}, &EffectiveConfigError{Kind: EffectiveConfigErrorApplyEnv, Err: applyEnvErr}
|
||||
}
|
||||
|
||||
return EffectiveConfig{
|
||||
Config: cfg,
|
||||
ConfigPath: configPath,
|
||||
ConfigSource: configSource,
|
||||
ConfigVersion: configVersion,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func resolveConfigPathWithLookup(cliConfigPath string, cliConfigPathSet bool, lookup func(string) (string, bool), statPath func(string) (os.FileInfo, error), defaultSearchPaths []string) (path string, source string, err error) {
|
||||
if cliConfigPathSet {
|
||||
path = strings.TrimSpace(cliConfigPath)
|
||||
if path == "" {
|
||||
return "", "", fmt.Errorf("--config requires a non-empty path")
|
||||
}
|
||||
if _, statErr := statPath(path); statErr != nil {
|
||||
if os.IsNotExist(statErr) {
|
||||
return "", "", fmt.Errorf("config file not found: %s", path)
|
||||
}
|
||||
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
|
||||
}
|
||||
return path, "flag", nil
|
||||
}
|
||||
|
||||
if raw, ok := lookup("AUDITA_CONFIG"); ok {
|
||||
path = strings.TrimSpace(raw)
|
||||
if path == "" {
|
||||
return "", "", fmt.Errorf("AUDITA_CONFIG must not be empty")
|
||||
}
|
||||
if _, statErr := statPath(path); statErr != nil {
|
||||
if os.IsNotExist(statErr) {
|
||||
return "", "", fmt.Errorf("config file not found: %s", path)
|
||||
}
|
||||
return "", "", fmt.Errorf("cannot access config file %s: %w", path, statErr)
|
||||
}
|
||||
return path, "env", nil
|
||||
}
|
||||
|
||||
for _, defaultPath := range defaultSearchPaths {
|
||||
if _, statErr := statPath(defaultPath); statErr == nil {
|
||||
return defaultPath, "default", nil
|
||||
} else if !os.IsNotExist(statErr) {
|
||||
return "", "", fmt.Errorf("cannot access config file %s: %w", defaultPath, statErr)
|
||||
}
|
||||
}
|
||||
return "", "", nil
|
||||
}
|
||||
163
internal/core/config/effective_config_test.go
Normal file
163
internal/core/config/effective_config_test.go
Normal file
@@ -0,0 +1,163 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestResolveConfigPathWithLookupMatrix(t *testing.T) {
|
||||
statFor := func(existing map[string]bool) func(string) (os.FileInfo, error) {
|
||||
return func(path string) (os.FileInfo, error) {
|
||||
if existing[path] {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
cliPath string
|
||||
cliPathSet bool
|
||||
lookup func(string) (string, bool)
|
||||
stat func(string) (os.FileInfo, error)
|
||||
defaultSearchPaths []string
|
||||
wantPath string
|
||||
wantSource string
|
||||
wantErrContains string
|
||||
}{
|
||||
{
|
||||
name: "explicit config path",
|
||||
cliPath: "/tmp/explicit.yml",
|
||||
cliPathSet: true,
|
||||
lookup: func(string) (string, bool) { return "", false },
|
||||
stat: statFor(map[string]bool{"/tmp/explicit.yml": true}),
|
||||
defaultSearchPaths: []string{
|
||||
"/usr/local/etc/audita/config.yml",
|
||||
"/etc/audita/config.yml",
|
||||
},
|
||||
wantPath: "/tmp/explicit.yml",
|
||||
wantSource: "flag",
|
||||
},
|
||||
{
|
||||
name: "env config path",
|
||||
cliPathSet: false,
|
||||
lookup: func(key string) (string, bool) {
|
||||
if key == "AUDITA_CONFIG" {
|
||||
return "/tmp/from-env.yml", true
|
||||
}
|
||||
return "", false
|
||||
},
|
||||
stat: statFor(map[string]bool{"/tmp/from-env.yml": true}),
|
||||
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
|
||||
wantPath: "/tmp/from-env.yml",
|
||||
wantSource: "env",
|
||||
},
|
||||
{
|
||||
name: "default search path",
|
||||
cliPathSet: false,
|
||||
lookup: func(string) (string, bool) { return "", false },
|
||||
stat: statFor(map[string]bool{
|
||||
"/usr/local/etc/audita/config.yml": true,
|
||||
"/etc/audita/config.yml": true,
|
||||
}),
|
||||
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
|
||||
wantPath: "/usr/local/etc/audita/config.yml",
|
||||
wantSource: "default",
|
||||
},
|
||||
{
|
||||
name: "explicit missing path",
|
||||
cliPath: "/tmp/missing.yml",
|
||||
cliPathSet: true,
|
||||
lookup: func(string) (string, bool) { return "", false },
|
||||
stat: statFor(map[string]bool{}),
|
||||
defaultSearchPaths: []string{
|
||||
"/usr/local/etc/audita/config.yml",
|
||||
"/etc/audita/config.yml",
|
||||
},
|
||||
wantErrContains: "config file not found",
|
||||
},
|
||||
{
|
||||
name: "missing env path",
|
||||
cliPathSet: false,
|
||||
lookup: func(key string) (string, bool) {
|
||||
if key == "AUDITA_CONFIG" {
|
||||
return "/tmp/missing-from-env.yml", true
|
||||
}
|
||||
return "", false
|
||||
},
|
||||
stat: statFor(map[string]bool{}),
|
||||
defaultSearchPaths: []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"},
|
||||
wantErrContains: "config file not found",
|
||||
},
|
||||
{
|
||||
name: "missing default paths",
|
||||
cliPathSet: false,
|
||||
lookup: func(string) (string, bool) { return "", false },
|
||||
stat: statFor(map[string]bool{}),
|
||||
defaultSearchPaths: []string{
|
||||
"/usr/local/etc/audita/config.yml",
|
||||
"/etc/audita/config.yml",
|
||||
},
|
||||
wantPath: "",
|
||||
wantSource: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
gotPath, gotSource, err := resolveConfigPathWithLookup(tc.cliPath, tc.cliPathSet, tc.lookup, tc.stat, tc.defaultSearchPaths)
|
||||
if tc.wantErrContains != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantErrContains) {
|
||||
t.Fatalf("expected error containing %q, got %v", tc.wantErrContains, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if gotPath != tc.wantPath || gotSource != tc.wantSource {
|
||||
t.Fatalf("unexpected result: got path=%q source=%q, want path=%q source=%q", gotPath, gotSource, tc.wantPath, tc.wantSource)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadEffectiveConfigWithLookupAppliesDefaultsFileThenEnv(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
configPath := filepath.Join(tempDir, "config.yml")
|
||||
configYAML := "version: 1\nllm:\n proposal:\n model: file-model\n"
|
||||
if err := os.WriteFile(configPath, []byte(configYAML), 0o644); err != nil {
|
||||
t.Fatalf("write config file: %v", err)
|
||||
}
|
||||
|
||||
lookup := func(key string) (string, bool) {
|
||||
switch key {
|
||||
case "AUDITA_CONFIG":
|
||||
return configPath, true
|
||||
case "AUDITA_MODEL":
|
||||
return "env-model", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
result, err := loadEffectiveConfigWithLookup("", false, lookup, os.Stat, DefaultConfigSearchPaths)
|
||||
if err != nil {
|
||||
t.Fatalf("loadEffectiveConfigWithLookup error: %v", err)
|
||||
}
|
||||
if result.ConfigPath != configPath {
|
||||
t.Fatalf("unexpected config path: %q", result.ConfigPath)
|
||||
}
|
||||
if result.ConfigSource != "env" {
|
||||
t.Fatalf("unexpected config source: %q", result.ConfigSource)
|
||||
}
|
||||
if result.ConfigVersion == nil || *result.ConfigVersion != SupportedFileConfigVersion {
|
||||
t.Fatalf("unexpected config version: %#v", result.ConfigVersion)
|
||||
}
|
||||
if result.Config.PrimaryLLM.Model != "env-model" {
|
||||
t.Fatalf("expected env override to win over file value, got %q", result.Config.PrimaryLLM.Model)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,9 @@ package config
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
|
||||
)
|
||||
|
||||
func (c Config) Validate() error {
|
||||
@@ -12,20 +15,20 @@ func (c Config) Validate() error {
|
||||
issues = append(issues, "modules must not be empty")
|
||||
}
|
||||
for _, module := range c.Modules {
|
||||
if strings.TrimSpace(module) == "" {
|
||||
moduleKey := strings.TrimSpace(module)
|
||||
if moduleKey == "" {
|
||||
issues = append(issues, "modules must not contain empty values")
|
||||
break
|
||||
}
|
||||
if !modulecatalog.IsSupported(moduleKey) {
|
||||
issues = append(issues, fmt.Sprintf("unsupported module key %q", moduleKey))
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(c.OutputSchema) == "" {
|
||||
issues = append(issues, "output schema must not be empty")
|
||||
} else {
|
||||
switch strings.TrimSpace(c.OutputSchema) {
|
||||
case "bare-segments", "audita-v1":
|
||||
default:
|
||||
} else if !outputschema.IsSupported(c.OutputSchema) {
|
||||
issues = append(issues, fmt.Sprintf("unsupported output schema %q", c.OutputSchema))
|
||||
}
|
||||
}
|
||||
|
||||
if c.PrimaryLLM.TimeoutSeconds <= 0 {
|
||||
issues = append(issues, "primary llm timeout seconds must be greater than zero")
|
||||
|
||||
42
internal/core/diagnostics/artifacts.go
Normal file
42
internal/core/diagnostics/artifacts.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
)
|
||||
|
||||
const (
|
||||
ArtifactSourceTranscript = "source-transcript.json"
|
||||
ArtifactParsedSourceTranscript = "source-transcript-parsed.json"
|
||||
ArtifactNormalizedTranscript = "normalized-transcript.json"
|
||||
ArtifactNormalizationSummary = "normalization-summary.json"
|
||||
ArtifactChunkingSummary = "chunking-summary.json"
|
||||
ArtifactUtilizationSummary = "utilization-diagnostics.json"
|
||||
ArtifactCorrectionLedger = "correction-ledger.json"
|
||||
ArtifactInvocationMetadata = "invocation.json"
|
||||
ArtifactEffectiveConfig = "effective-config.json"
|
||||
ArtifactReport = "report.json"
|
||||
ArtifactErrorLog = "error.log"
|
||||
)
|
||||
|
||||
func BuildDiagnosticsMetadata(runDirectoryPath string, runSucceeded bool) reporting.DiagnosticsMetadata {
|
||||
metadata := reporting.DiagnosticsMetadata{
|
||||
DirectoryPath: runDirectoryPath,
|
||||
SourceTranscriptPath: filepath.Join(runDirectoryPath, ArtifactSourceTranscript),
|
||||
ParsedSourceTranscriptPath: filepath.Join(runDirectoryPath, ArtifactParsedSourceTranscript),
|
||||
NormalizedTranscriptPath: filepath.Join(runDirectoryPath, ArtifactNormalizedTranscript),
|
||||
NormalizationSummaryPath: filepath.Join(runDirectoryPath, ArtifactNormalizationSummary),
|
||||
ChunkingSummaryPath: filepath.Join(runDirectoryPath, ArtifactChunkingSummary),
|
||||
UtilizationSummaryPath: filepath.Join(runDirectoryPath, ArtifactUtilizationSummary),
|
||||
CorrectionLedgerPath: filepath.Join(runDirectoryPath, ArtifactCorrectionLedger),
|
||||
InvocationMetadataPath: filepath.Join(runDirectoryPath, ArtifactInvocationMetadata),
|
||||
RedactedEffectiveConfigPath: filepath.Join(runDirectoryPath, ArtifactEffectiveConfig),
|
||||
}
|
||||
|
||||
if !runSucceeded {
|
||||
metadata.ErrorLogPath = filepath.Join(runDirectoryPath, ArtifactErrorLog)
|
||||
}
|
||||
|
||||
return metadata
|
||||
}
|
||||
55
internal/core/diagnostics/artifacts_test.go
Normal file
55
internal/core/diagnostics/artifacts_test.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBuildDiagnosticsMetadataSuccessPathsMatchArtifactConstants(t *testing.T) {
|
||||
runPath := filepath.Join("tmp", "run-123")
|
||||
metadata := BuildDiagnosticsMetadata(runPath, true)
|
||||
|
||||
if metadata.DirectoryPath != runPath {
|
||||
t.Fatalf("unexpected diagnostics directory path: got=%q want=%q", metadata.DirectoryPath, runPath)
|
||||
}
|
||||
if metadata.SourceTranscriptPath != filepath.Join(runPath, ArtifactSourceTranscript) {
|
||||
t.Fatalf("unexpected source transcript path: %q", metadata.SourceTranscriptPath)
|
||||
}
|
||||
if metadata.ParsedSourceTranscriptPath != filepath.Join(runPath, ArtifactParsedSourceTranscript) {
|
||||
t.Fatalf("unexpected parsed source transcript path: %q", metadata.ParsedSourceTranscriptPath)
|
||||
}
|
||||
if metadata.NormalizedTranscriptPath != filepath.Join(runPath, ArtifactNormalizedTranscript) {
|
||||
t.Fatalf("unexpected normalized transcript path: %q", metadata.NormalizedTranscriptPath)
|
||||
}
|
||||
if metadata.NormalizationSummaryPath != filepath.Join(runPath, ArtifactNormalizationSummary) {
|
||||
t.Fatalf("unexpected normalization summary path: %q", metadata.NormalizationSummaryPath)
|
||||
}
|
||||
if metadata.ChunkingSummaryPath != filepath.Join(runPath, ArtifactChunkingSummary) {
|
||||
t.Fatalf("unexpected chunking summary path: %q", metadata.ChunkingSummaryPath)
|
||||
}
|
||||
if metadata.UtilizationSummaryPath != filepath.Join(runPath, ArtifactUtilizationSummary) {
|
||||
t.Fatalf("unexpected utilization summary path: %q", metadata.UtilizationSummaryPath)
|
||||
}
|
||||
if metadata.CorrectionLedgerPath != filepath.Join(runPath, ArtifactCorrectionLedger) {
|
||||
t.Fatalf("unexpected correction ledger path: %q", metadata.CorrectionLedgerPath)
|
||||
}
|
||||
if metadata.InvocationMetadataPath != filepath.Join(runPath, ArtifactInvocationMetadata) {
|
||||
t.Fatalf("unexpected invocation metadata path: %q", metadata.InvocationMetadataPath)
|
||||
}
|
||||
if metadata.RedactedEffectiveConfigPath != filepath.Join(runPath, ArtifactEffectiveConfig) {
|
||||
t.Fatalf("unexpected redacted effective config path: %q", metadata.RedactedEffectiveConfigPath)
|
||||
}
|
||||
if metadata.ErrorLogPath != "" {
|
||||
t.Fatalf("did not expect error log path on success: %q", metadata.ErrorLogPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDiagnosticsMetadataFailureIncludesErrorLogPath(t *testing.T) {
|
||||
runPath := filepath.Join("tmp", "run-123")
|
||||
metadata := BuildDiagnosticsMetadata(runPath, false)
|
||||
|
||||
want := filepath.Join(runPath, ArtifactErrorLog)
|
||||
if metadata.ErrorLogPath != want {
|
||||
t.Fatalf("unexpected error log path: got=%q want=%q", metadata.ErrorLogPath, want)
|
||||
}
|
||||
}
|
||||
@@ -106,7 +106,7 @@ func (r *RunDirectory) WriteInvocationMetadata(metadata InvocationMetadata) erro
|
||||
metadata.StartedAt = r.createdAt
|
||||
}
|
||||
|
||||
path := filepath.Join(r.path, "invocation.json")
|
||||
path := filepath.Join(r.path, ArtifactInvocationMetadata)
|
||||
bytes, err := json.MarshalIndent(metadata, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal invocation metadata: %w", err)
|
||||
@@ -120,7 +120,7 @@ func (r *RunDirectory) WriteInvocationMetadata(metadata InvocationMetadata) erro
|
||||
|
||||
// WriteEffectiveConfig writes redacted effective config metadata for this run.
|
||||
func (r *RunDirectory) WriteEffectiveConfig(cfg config.Config) error {
|
||||
path := filepath.Join(r.path, "effective-config.json")
|
||||
path := filepath.Join(r.path, ArtifactEffectiveConfig)
|
||||
redacted := cfg.Redacted()
|
||||
bytes, err := json.MarshalIndent(redacted, "", " ")
|
||||
if err != nil {
|
||||
@@ -136,13 +136,13 @@ func (r *RunDirectory) WriteEffectiveConfig(cfg config.Config) error {
|
||||
// WriteSourceTranscript writes the source transcript artifact
|
||||
func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript, raw []byte) error {
|
||||
// Write raw source for reference
|
||||
sourcePath := filepath.Join(r.path, "source-transcript.json")
|
||||
sourcePath := filepath.Join(r.path, ArtifactSourceTranscript)
|
||||
if err := os.WriteFile(sourcePath, raw, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write source transcript: %w", err)
|
||||
}
|
||||
|
||||
// Write parsed source for debugging
|
||||
parsedPath := filepath.Join(r.path, "source-transcript-parsed.json")
|
||||
parsedPath := filepath.Join(r.path, ArtifactParsedSourceTranscript)
|
||||
parsedBytes, err := json.MarshalIndent(transcript, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal parsed source transcript: %w", err)
|
||||
@@ -157,7 +157,7 @@ func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript
|
||||
|
||||
// WriteNormalizedTranscript writes the normalized transcript artifact
|
||||
func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript) error {
|
||||
normalizedPath := filepath.Join(r.path, "normalized-transcript.json")
|
||||
normalizedPath := filepath.Join(r.path, ArtifactNormalizedTranscript)
|
||||
bytes, err := schema.TranscriptToJSON(transcript)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to serialize normalized transcript: %w", err)
|
||||
@@ -170,7 +170,7 @@ func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript)
|
||||
|
||||
// WriteNormalizationSummary writes the normalization summary artifact
|
||||
func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.NormalizationSummary) error {
|
||||
summaryPath := filepath.Join(r.path, "normalization-summary.json")
|
||||
summaryPath := filepath.Join(r.path, ArtifactNormalizationSummary)
|
||||
bytes, err := json.MarshalIndent(summary, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal normalization summary: %w", err)
|
||||
@@ -184,7 +184,7 @@ func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.Normaliz
|
||||
|
||||
// WriteReport writes the authoritative report artifact
|
||||
func (r *RunDirectory) WriteReport(report reporting.ProcessReport) error {
|
||||
reportPath := filepath.Join(r.path, "report.json")
|
||||
reportPath := filepath.Join(r.path, ArtifactReport)
|
||||
bytes, err := json.MarshalIndent(report, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal report: %w", err)
|
||||
@@ -198,13 +198,13 @@ func (r *RunDirectory) WriteReport(report reporting.ProcessReport) error {
|
||||
|
||||
// WriteErrorLog writes an error log on failure
|
||||
func (r *RunDirectory) WriteErrorLog(errorMessage string) error {
|
||||
errorPath := filepath.Join(r.path, "error.log")
|
||||
errorPath := filepath.Join(r.path, ArtifactErrorLog)
|
||||
return os.WriteFile(errorPath, []byte(errorMessage+"\n"), 0o644)
|
||||
}
|
||||
|
||||
// WriteChunkingSummary writes the chunking summary artifact
|
||||
func (r *RunDirectory) WriteChunkingSummary(summary *chunking.DetailedSummary) error {
|
||||
summaryPath := filepath.Join(r.path, "chunking-summary.json")
|
||||
summaryPath := filepath.Join(r.path, ArtifactChunkingSummary)
|
||||
bytes, err := json.MarshalIndent(summary, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal chunking summary: %w", err)
|
||||
|
||||
35
internal/core/modulecatalog/catalog.go
Normal file
35
internal/core/modulecatalog/catalog.go
Normal file
@@ -0,0 +1,35 @@
|
||||
package modulecatalog
|
||||
|
||||
import "strings"
|
||||
|
||||
const (
|
||||
KeyGlossary = "glossary"
|
||||
KeyHomophones = "homophones"
|
||||
KeySpokenWord = "spoken_word"
|
||||
KeyGrammar = "grammar"
|
||||
)
|
||||
|
||||
var supportedKeys = []string{
|
||||
KeyGlossary,
|
||||
KeyHomophones,
|
||||
KeySpokenWord,
|
||||
KeyGrammar,
|
||||
}
|
||||
|
||||
var supportedKeySet = map[string]struct{}{
|
||||
KeyGlossary: {},
|
||||
KeyHomophones: {},
|
||||
KeySpokenWord: {},
|
||||
KeyGrammar: {},
|
||||
}
|
||||
|
||||
func SupportedKeys() []string {
|
||||
out := make([]string, len(supportedKeys))
|
||||
copy(out, supportedKeys)
|
||||
return out
|
||||
}
|
||||
|
||||
func IsSupported(key string) bool {
|
||||
_, ok := supportedKeySet[strings.TrimSpace(key)]
|
||||
return ok
|
||||
}
|
||||
24
internal/core/modulecatalog/catalog_test.go
Normal file
24
internal/core/modulecatalog/catalog_test.go
Normal file
@@ -0,0 +1,24 @@
|
||||
package modulecatalog
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSupportedKeys(t *testing.T) {
|
||||
want := []string{KeyGlossary, KeyHomophones, KeySpokenWord, KeyGrammar}
|
||||
if got := SupportedKeys(); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("unexpected supported keys: got=%v want=%v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsSupported(t *testing.T) {
|
||||
for _, key := range SupportedKeys() {
|
||||
if !IsSupported(key) {
|
||||
t.Fatalf("expected key %q to be supported", key)
|
||||
}
|
||||
}
|
||||
if IsSupported("made_up") {
|
||||
t.Fatalf("did not expect made_up to be supported")
|
||||
}
|
||||
}
|
||||
@@ -31,15 +31,31 @@ var definitions = map[string]Definition{
|
||||
},
|
||||
}
|
||||
|
||||
var supportedKeys = []string{
|
||||
SchemaBareSegments,
|
||||
SchemaAuditaV1,
|
||||
}
|
||||
|
||||
func SupportedKeys() []string {
|
||||
out := make([]string, len(supportedKeys))
|
||||
copy(out, supportedKeys)
|
||||
return out
|
||||
}
|
||||
|
||||
func IsSupported(key string) bool {
|
||||
_, ok := definitions[strings.TrimSpace(key)]
|
||||
return ok
|
||||
}
|
||||
|
||||
func Resolve(key string) (Definition, error) {
|
||||
normalized := strings.TrimSpace(key)
|
||||
if normalized == "" {
|
||||
return Definition{}, fmt.Errorf("output schema must not be empty")
|
||||
}
|
||||
def, ok := definitions[normalized]
|
||||
if !ok {
|
||||
if !IsSupported(normalized) {
|
||||
return Definition{}, fmt.Errorf("unsupported output schema %q", normalized)
|
||||
}
|
||||
def := definitions[normalized]
|
||||
return def, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package outputschema
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -56,3 +57,19 @@ func TestResolveUnknown(t *testing.T) {
|
||||
t.Fatalf("expected unsupported output schema error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupportedKeysAndIsSupported(t *testing.T) {
|
||||
want := []string{SchemaBareSegments, SchemaAuditaV1}
|
||||
if got := SupportedKeys(); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("unexpected supported schema keys: got=%v want=%v", got, want)
|
||||
}
|
||||
|
||||
for _, key := range want {
|
||||
if !IsSupported(key) {
|
||||
t.Fatalf("expected schema key %q to be supported", key)
|
||||
}
|
||||
}
|
||||
if IsSupported("seriatim-intermediate") {
|
||||
t.Fatalf("did not expect unsupported schema to be reported as supported")
|
||||
}
|
||||
}
|
||||
|
||||
17
internal/framework/llm/secrets.go
Normal file
17
internal/framework/llm/secrets.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package llm
|
||||
|
||||
import "gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
|
||||
// ConfiguredSecrets returns all configured LLM API-key values that should be
|
||||
// redacted from diagnostics and surfaced error payloads.
|
||||
func ConfiguredSecrets(cfg *config.Config) []string {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
effectiveValidation := cfg.EffectiveValidationLLMConfig()
|
||||
return []string{
|
||||
cfg.PrimaryLLM.APIKey,
|
||||
cfg.ValidationLLM.APIKey,
|
||||
effectiveValidation.APIKey,
|
||||
}
|
||||
}
|
||||
38
internal/framework/llm/secrets_test.go
Normal file
38
internal/framework/llm/secrets_test.go
Normal file
@@ -0,0 +1,38 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
)
|
||||
|
||||
func TestConfiguredSecretsNilConfig(t *testing.T) {
|
||||
if got := ConfiguredSecrets(nil); got != nil {
|
||||
t.Fatalf("expected nil secrets for nil config, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredSecretsWithValidationOverride(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = "primary-secret"
|
||||
cfg.ValidationLLM.APIKey = "validation-secret"
|
||||
|
||||
got := ConfiguredSecrets(&cfg)
|
||||
want := []string{"primary-secret", "validation-secret", "validation-secret"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("unexpected secrets: got=%v want=%v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredSecretsWithInheritedValidationKey(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = "primary-secret"
|
||||
cfg.ValidationLLM.APIKey = ""
|
||||
|
||||
got := ConfiguredSecrets(&cfg)
|
||||
want := []string{"primary-secret", "", "primary-secret"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("unexpected inherited secrets: got=%v want=%v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
glossarymodule "gitea.maximumdirect.net/eric/audita/internal/modules/glossary"
|
||||
@@ -15,28 +16,20 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
ModuleKeyGlossary = "glossary"
|
||||
ModuleKeyHomophones = "homophones"
|
||||
ModuleKeySpokenWord = "spoken_word"
|
||||
ModuleKeyGrammar = "grammar"
|
||||
ModuleKeyGlossary = modulecatalog.KeyGlossary
|
||||
ModuleKeyHomophones = modulecatalog.KeyHomophones
|
||||
ModuleKeySpokenWord = modulecatalog.KeySpokenWord
|
||||
ModuleKeyGrammar = modulecatalog.KeyGrammar
|
||||
)
|
||||
|
||||
const (
|
||||
ReasonUnsupportedModule = "unsupported_module"
|
||||
)
|
||||
|
||||
var knownModuleKeys = map[string]struct{}{
|
||||
ModuleKeyGlossary: {},
|
||||
ModuleKeyHomophones: {},
|
||||
ModuleKeySpokenWord: {},
|
||||
ModuleKeyGrammar: {},
|
||||
}
|
||||
|
||||
// IsKnownModuleKey reports whether a module key is recognized by the production
|
||||
// registry scaffold.
|
||||
func IsKnownModuleKey(key string) bool {
|
||||
_, ok := knownModuleKeys[strings.TrimSpace(key)]
|
||||
return ok
|
||||
return modulecatalog.IsSupported(key)
|
||||
}
|
||||
|
||||
// Dependencies holds explicit constructor dependencies for module creation.
|
||||
@@ -69,7 +62,7 @@ type Factory struct {
|
||||
func NewFactory(deps Dependencies) *Factory {
|
||||
factory := &Factory{
|
||||
deps: deps,
|
||||
constructors: make(map[string]Constructor, len(knownModuleKeys)),
|
||||
constructors: make(map[string]Constructor, len(modulecatalog.SupportedKeys())),
|
||||
}
|
||||
_ = factory.RegisterConstructor(ModuleKeyGlossary, constructGlossaryModule)
|
||||
_ = factory.RegisterConstructor(ModuleKeyHomophones, constructHomophonesModule)
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
)
|
||||
@@ -26,7 +27,7 @@ func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest)
|
||||
}
|
||||
|
||||
func TestKnownModuleKeyRecognition(t *testing.T) {
|
||||
for _, key := range []string{ModuleKeyGlossary, ModuleKeyHomophones, ModuleKeySpokenWord, ModuleKeyGrammar} {
|
||||
for _, key := range modulecatalog.SupportedKeys() {
|
||||
if !IsKnownModuleKey(key) {
|
||||
t.Fatalf("expected key %q to be recognized", key)
|
||||
}
|
||||
|
||||
45
internal/framework/promptcontext/transcript_section.go
Normal file
45
internal/framework/promptcontext/transcript_section.go
Normal file
@@ -0,0 +1,45 @@
|
||||
package promptcontext
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
type transcriptSectionSegment struct {
|
||||
ID int `json:"id"`
|
||||
Speaker string `json:"speaker"`
|
||||
Start float64 `json:"start"`
|
||||
End float64 `json:"end"`
|
||||
Text string `json:"text"`
|
||||
Categories []string `json:"categories,omitempty"`
|
||||
}
|
||||
|
||||
type transcriptSectionPayload struct {
|
||||
SectionIndex int `json:"section_index"`
|
||||
Segments []transcriptSectionSegment `json:"segments"`
|
||||
}
|
||||
|
||||
// MarshalTranscriptSectionJSON builds the standardized transcript-section JSON
|
||||
// payload consumed by proposal prompt templates.
|
||||
func MarshalTranscriptSectionJSON(transcript *schema.Transcript, sectionIndex int) ([]byte, error) {
|
||||
payload := transcriptSectionPayload{
|
||||
SectionIndex: sectionIndex,
|
||||
Segments: make([]transcriptSectionSegment, 0),
|
||||
}
|
||||
|
||||
if transcript != nil {
|
||||
for _, s := range transcript.Segments {
|
||||
payload.Segments = append(payload.Segments, transcriptSectionSegment{
|
||||
ID: s.ID,
|
||||
Speaker: s.Speaker,
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
Text: s.Text,
|
||||
Categories: append([]string(nil), s.Categories...),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return json.MarshalIndent(payload, "", " ")
|
||||
}
|
||||
95
internal/framework/promptcontext/transcript_section_test.go
Normal file
95
internal/framework/promptcontext/transcript_section_test.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package promptcontext
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
func TestMarshalTranscriptSectionJSONShape(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "A", Start: 0.1, End: 1.2, Text: "alpha", Categories: []string{"session", "intro"}},
|
||||
}}
|
||||
|
||||
raw, err := MarshalTranscriptSectionJSON(transcript, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("MarshalTranscriptSectionJSON error: %v", err)
|
||||
}
|
||||
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(raw, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
|
||||
if got := decoded["section_index"]; got != float64(3) {
|
||||
t.Fatalf("section_index: got=%v want=%v", got, 3)
|
||||
}
|
||||
segments, ok := decoded["segments"].([]any)
|
||||
if !ok || len(segments) != 1 {
|
||||
t.Fatalf("segments shape mismatch: %T %+v", decoded["segments"], decoded["segments"])
|
||||
}
|
||||
first, ok := segments[0].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("segment shape mismatch: %T", segments[0])
|
||||
}
|
||||
if first["id"] != float64(1) || first["speaker"] != "A" || first["start"] != 0.1 || first["end"] != 1.2 || first["text"] != "alpha" {
|
||||
t.Fatalf("unexpected segment fields: %+v", first)
|
||||
}
|
||||
cats, ok := first["categories"].([]any)
|
||||
if !ok || len(cats) != 2 || cats[0] != "session" || cats[1] != "intro" {
|
||||
t.Fatalf("unexpected categories: %+v", first["categories"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalTranscriptSectionJSONEmptyTranscript(t *testing.T) {
|
||||
raw, err := MarshalTranscriptSectionJSON(nil, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("MarshalTranscriptSectionJSON error: %v", err)
|
||||
}
|
||||
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(raw, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
|
||||
if got := decoded["section_index"]; got != float64(0) {
|
||||
t.Fatalf("section_index: got=%v want=%v", got, 0)
|
||||
}
|
||||
segments, ok := decoded["segments"].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("segments shape mismatch: %T", decoded["segments"])
|
||||
}
|
||||
if len(segments) != 0 {
|
||||
t.Fatalf("expected empty segments, got %d", len(segments))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalTranscriptSectionJSONCopiesCategories(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "alpha", Categories: []string{"kept"}},
|
||||
}}
|
||||
|
||||
raw, err := MarshalTranscriptSectionJSON(transcript, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("MarshalTranscriptSectionJSON error: %v", err)
|
||||
}
|
||||
|
||||
transcript.Segments[0].Categories[0] = "changed"
|
||||
|
||||
var decoded struct {
|
||||
Segments []struct {
|
||||
Categories []string `json:"categories"`
|
||||
} `json:"segments"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
if len(decoded.Segments) != 1 {
|
||||
t.Fatalf("expected one segment, got %d", len(decoded.Segments))
|
||||
}
|
||||
if !reflect.DeepEqual(decoded.Segments[0].Categories, []string{"kept"}) {
|
||||
t.Fatalf("expected copied categories, got %v", decoded.Segments[0].Categories)
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,8 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
)
|
||||
|
||||
@@ -94,7 +96,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
|
||||
stage := strings.TrimSpace(req.StageName)
|
||||
if stage == "" {
|
||||
stage = buildStageName(req.ModuleInstance, req.Section)
|
||||
stage = stagename.ProposalGeneration(req.ModuleInstance, sectionIndexPtr(req.Section))
|
||||
}
|
||||
model := resolveModel(req.Config, req.Model)
|
||||
messages := append([]contracts.LLMMessage(nil), req.Messages...)
|
||||
@@ -106,7 +108,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
writer = diagnosticsWriterAdapter{
|
||||
writer: llm.NewDiagnosticsWriter(
|
||||
filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
|
||||
proposalGenerationSecrets(req.Config),
|
||||
llm.ConfiguredSecrets(req.Config),
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -143,7 +145,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
if len(req.PromptMetadata) > 0 {
|
||||
requestMetadata["prompt_metadata"] = req.PromptMetadata
|
||||
}
|
||||
requestMetadata["response_schema"] = schemaMetadata(responseSchema)
|
||||
requestMetadata["response_schema"] = responseSchema.DiagnosticsMap()
|
||||
|
||||
if writer != nil {
|
||||
artifacts, _ = writer.WriteInteraction(
|
||||
@@ -158,7 +160,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
}
|
||||
|
||||
if callErr != nil {
|
||||
if isMalformedStructuredOutputError(callErr) {
|
||||
if structuredoutput.IsMalformedError(callErr) {
|
||||
return Result{
|
||||
Warnings: []stagewarnings.StageWarning{newMalformedProposalWarning(req.Section, artifacts, callErr)},
|
||||
Artifacts: artifacts,
|
||||
@@ -201,23 +203,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func schemaMetadata(schema responseschema.Schema) map[string]any {
|
||||
return map[string]any{
|
||||
"id": schema.ID,
|
||||
"version": schema.Version,
|
||||
"name": schema.Name,
|
||||
"sha256": schema.SHA256,
|
||||
}
|
||||
}
|
||||
|
||||
func buildStageName(moduleInstance string, section *contracts.SectionMetadata) string {
|
||||
base := fmt.Sprintf("%s:proposal-generation", moduleInstance)
|
||||
if section == nil {
|
||||
return base
|
||||
}
|
||||
return fmt.Sprintf("%s:section-%04d", base, section.Index)
|
||||
}
|
||||
|
||||
func resolveModel(cfg *config.Config, override string) string {
|
||||
if strings.TrimSpace(override) != "" {
|
||||
return strings.TrimSpace(override)
|
||||
@@ -228,17 +213,6 @@ func resolveModel(cfg *config.Config, override string) string {
|
||||
return llm.ResolvePrimaryConfig(*cfg).Model
|
||||
}
|
||||
|
||||
func proposalGenerationSecrets(cfg *config.Config) []string {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
return []string{
|
||||
cfg.PrimaryLLM.APIKey,
|
||||
cfg.ValidationLLM.APIKey,
|
||||
cfg.EffectiveValidationLLMConfig().APIKey,
|
||||
}
|
||||
}
|
||||
|
||||
func errPayload(err error) any {
|
||||
if err == nil {
|
||||
return nil
|
||||
@@ -246,27 +220,6 @@ func errPayload(err error) any {
|
||||
return map[string]any{"error": err.Error()}
|
||||
}
|
||||
|
||||
func isMalformedStructuredOutputError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := err.Error()
|
||||
for _, marker := range []string{
|
||||
"malformed structured output",
|
||||
"decode structured output:",
|
||||
"decode provider response envelope:",
|
||||
"provider response missing choices",
|
||||
"provider response missing assistant message content",
|
||||
"provider response assistant message content is empty",
|
||||
"provider response assistant message content is not valid JSON",
|
||||
} {
|
||||
if strings.Contains(msg, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func newMalformedProposalWarning(section *contracts.SectionMetadata, artifacts InteractionArtifacts, err error) stagewarnings.StageWarning {
|
||||
warning := stagewarnings.StageWarning{
|
||||
Scope: stagewarnings.ScopeProposalGeneration,
|
||||
@@ -288,6 +241,14 @@ func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
|
||||
return artifacts.ResponsePayloadPath
|
||||
}
|
||||
|
||||
func sectionIndexPtr(section *contracts.SectionMetadata) *int {
|
||||
if section == nil {
|
||||
return nil
|
||||
}
|
||||
index := section.Index
|
||||
return &index
|
||||
}
|
||||
|
||||
type diagnosticsWriterAdapter struct {
|
||||
writer *llm.DiagnosticsWriter
|
||||
}
|
||||
|
||||
@@ -272,6 +272,23 @@ func TestGenerateCandidatesMalformedStructuredOutputReturnsWarning(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesProviderMalformedEnvelopeReturnsWarning(t *testing.T) {
|
||||
client := &fakeStructuredClient{err: errors.New("provider response missing choices")}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
|
||||
result, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected malformed provider envelope to downgrade to warning, got %v", err)
|
||||
}
|
||||
if len(result.Corrections) != 0 || len(result.Enriched) != 0 {
|
||||
t.Fatalf("expected no proposals on malformed response, got %+v", result)
|
||||
}
|
||||
if len(result.Warnings) != 1 || result.Warnings[0].ReasonCode != "proposal_response_malformed" {
|
||||
t.Fatalf("unexpected warnings: %+v", result.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
|
||||
baseResponse := StructuredCorrectionSet{
|
||||
Corrections: []StructuredCorrectionProposal{
|
||||
@@ -344,20 +361,22 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
secret := "proposal-secret"
|
||||
primarySecret := "proposal-primary-secret"
|
||||
validationSecret := "proposal-validation-secret"
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: secret, CorrectedText: "safe", Confidence: 0.9}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}},
|
||||
},
|
||||
}
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = secret
|
||||
cfg.PrimaryLLM.APIKey = primarySecret
|
||||
cfg.ValidationLLM.APIKey = validationSecret
|
||||
req := defaultRequest(t)
|
||||
req.Config = &cfg
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsDir = t.TempDir()
|
||||
req.Messages = []contracts.LLMMessage{
|
||||
{Role: "system", Content: "include secret " + secret},
|
||||
{Role: "system", Content: "include secret " + primarySecret},
|
||||
{Role: "user", Content: "fix it"},
|
||||
}
|
||||
|
||||
@@ -374,8 +393,8 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
if readErr != nil {
|
||||
t.Fatalf("read artifact %q: %v", path, readErr)
|
||||
}
|
||||
if strings.Contains(string(raw), secret) {
|
||||
t.Fatalf("artifact leaked secret %q: %s", path, string(raw))
|
||||
if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) {
|
||||
t.Fatalf("artifact leaked configured secret in %q: %s", path, string(raw))
|
||||
}
|
||||
if !strings.Contains(string(raw), "[REDACTED]") {
|
||||
t.Fatalf("expected redaction marker in artifact %q: %s", path, string(raw))
|
||||
|
||||
81
internal/framework/proposal_generation/module_proposal.go
Normal file
81
internal/framework/proposal_generation/module_proposal.go
Normal file
@@ -0,0 +1,81 @@
|
||||
package proposal_generation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
type ProposalMessageBuilder func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error)
|
||||
|
||||
type ModuleProposalRequest struct {
|
||||
ProposalRequest contracts.ProposalRequest
|
||||
PromptID string
|
||||
BuildMessages ProposalMessageBuilder
|
||||
}
|
||||
|
||||
// ExecuteModuleProposal runs shared proposal generation plumbing for one
|
||||
// module, leaving only module-specific prompt message building at call sites.
|
||||
func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (contracts.ProposalResult, error) {
|
||||
if req.BuildMessages == nil {
|
||||
return contracts.ProposalResult{}, fmt.Errorf("proposal message builder is required")
|
||||
}
|
||||
if strings.TrimSpace(req.PromptID) == "" {
|
||||
return contracts.ProposalResult{}, fmt.Errorf("prompt ID must not be empty")
|
||||
}
|
||||
|
||||
sectionIndex := 0
|
||||
if req.ProposalRequest.Section != nil {
|
||||
sectionIndex = req.ProposalRequest.Section.Index
|
||||
}
|
||||
|
||||
transcriptDescription := ""
|
||||
if req.ProposalRequest.Config != nil {
|
||||
transcriptDescription = req.ProposalRequest.Config.TranscriptDescription
|
||||
}
|
||||
|
||||
messages, err := req.BuildMessages(
|
||||
req.ProposalRequest.WorkingTranscript,
|
||||
req.ProposalRequest.Glossary,
|
||||
sectionIndex,
|
||||
transcriptDescription,
|
||||
)
|
||||
if err != nil {
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
|
||||
promptMetadata, ok := prompts.LookupMetadata(req.PromptID)
|
||||
if !ok {
|
||||
return contracts.ProposalResult{}, fmt.Errorf("unknown prompt ID %q", req.PromptID)
|
||||
}
|
||||
|
||||
generated, err := GenerateCandidates(ctx, Request{
|
||||
ModuleKey: req.ProposalRequest.RunSpec.ModuleKey,
|
||||
ModuleInstance: req.ProposalRequest.RunSpec.InstanceName,
|
||||
ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy,
|
||||
WorkingTranscript: req.ProposalRequest.WorkingTranscript,
|
||||
Section: req.ProposalRequest.Section,
|
||||
Glossary: req.ProposalRequest.Glossary,
|
||||
Config: req.ProposalRequest.Config,
|
||||
Messages: messages,
|
||||
PromptMetadata: promptMetadata.DiagnosticsMap(),
|
||||
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
|
||||
StartIndex: 0,
|
||||
LLMClient: req.ProposalRequest.LLMClient,
|
||||
Scheduler: req.ProposalRequest.LLMScheduler,
|
||||
DiagnosticsDir: req.ProposalRequest.DiagnosticsDir,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
|
||||
return contracts.ProposalResult{
|
||||
Proposals: generated.Corrections,
|
||||
Warnings: generated.Warnings,
|
||||
}, nil
|
||||
}
|
||||
115
internal/framework/proposal_generation/module_proposal_test.go
Normal file
115
internal/framework/proposal_generation/module_proposal_test.go
Normal file
@@ -0,0 +1,115 @@
|
||||
package proposal_generation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
|
||||
},
|
||||
}
|
||||
section := contracts.SectionMetadata{Index: 7}
|
||||
cfg := config.Default()
|
||||
cfg.TranscriptDescription = "Hearing transcript with role titles."
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh"}}}
|
||||
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "X"}}}
|
||||
|
||||
var gotSectionIndex int
|
||||
var gotDescription string
|
||||
var gotTranscript *schema.Transcript
|
||||
var gotGlossary *schema.Glossary
|
||||
|
||||
out, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
|
||||
ProposalRequest: contracts.ProposalRequest{
|
||||
ExecutionContext: contracts.ExecutionContext{
|
||||
Config: &cfg,
|
||||
WorkingTranscript: transcript,
|
||||
Glossary: glossary,
|
||||
Section: §ion,
|
||||
},
|
||||
RunSpec: contracts.ModuleRunSpec{
|
||||
ModuleKey: "grammar",
|
||||
InstanceName: "grammar",
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
},
|
||||
LLMClient: client,
|
||||
},
|
||||
PromptID: prompts.PromptIDModuleGrammarProposal,
|
||||
BuildMessages: func(inTranscript *schema.Transcript, inGlossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
gotSectionIndex = sectionIndex
|
||||
gotDescription = transcriptDescription
|
||||
gotTranscript = inTranscript
|
||||
gotGlossary = inGlossary
|
||||
return []contracts.LLMMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}, nil
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ExecuteModuleProposal error: %v", err)
|
||||
}
|
||||
if gotSectionIndex != 7 {
|
||||
t.Fatalf("section index: got=%d want=%d", gotSectionIndex, 7)
|
||||
}
|
||||
if gotDescription != cfg.TranscriptDescription {
|
||||
t.Fatalf("transcript description: got=%q want=%q", gotDescription, cfg.TranscriptDescription)
|
||||
}
|
||||
if gotTranscript != transcript {
|
||||
t.Fatalf("expected shared transcript pointer")
|
||||
}
|
||||
if gotGlossary != glossary {
|
||||
t.Fatalf("expected shared glossary pointer")
|
||||
}
|
||||
if len(client.calls) != 1 || client.calls[0].StageName != "grammar:proposal:section-0007" {
|
||||
t.Fatalf("unexpected stage name calls: %+v", client.calls)
|
||||
}
|
||||
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "the" {
|
||||
t.Fatalf("unexpected proposals: %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteModuleProposalValidatesInputs(t *testing.T) {
|
||||
if _, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{}); err == nil {
|
||||
t.Fatalf("expected missing message builder error")
|
||||
}
|
||||
|
||||
_, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
|
||||
BuildMessages: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
return nil, nil
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("expected empty prompt ID error")
|
||||
}
|
||||
|
||||
_, err = ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
|
||||
PromptID: "missing.prompt.id",
|
||||
BuildMessages: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
return []contracts.LLMMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}, nil
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("expected unknown prompt ID error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteModuleProposalPropagatesBuilderError(t *testing.T) {
|
||||
wantErr := errors.New("builder failed")
|
||||
_, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
|
||||
PromptID: prompts.PromptIDModuleGlossaryProposal,
|
||||
BuildMessages: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
return nil, wantErr
|
||||
},
|
||||
})
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("expected builder error, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -28,6 +29,15 @@ type Schema struct {
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
|
||||
func (s Schema) DiagnosticsMap() map[string]any {
|
||||
return map[string]any{
|
||||
"id": s.ID,
|
||||
"version": s.Version,
|
||||
"name": s.Name,
|
||||
"sha256": s.SHA256,
|
||||
}
|
||||
}
|
||||
|
||||
var registry = map[Key]Schema{
|
||||
CorrectionSetKey: mustBuildSchema(
|
||||
correctionSetSchemaID,
|
||||
@@ -43,6 +53,20 @@ var registry = map[Key]Schema{
|
||||
),
|
||||
}
|
||||
|
||||
func Registered() []Schema {
|
||||
keys := make([]string, 0, len(registry))
|
||||
for key := range registry {
|
||||
keys = append(keys, string(key))
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
out := make([]Schema, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
out = append(out, cloneSchema(registry[Key(key)]))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Lookup returns a copy of the registered schema for the provided key.
|
||||
func Lookup(key Key) (Schema, bool) {
|
||||
schema, ok := registry[key]
|
||||
|
||||
@@ -89,3 +89,20 @@ func TestLookupReturnsSchemaCopy(t *testing.T) {
|
||||
t.Fatalf("expected lookup to return independent schema copy")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsMapIncludesStableSchemaMetadataShapeForAllSchemas(t *testing.T) {
|
||||
registered := Registered()
|
||||
if len(registered) == 0 {
|
||||
t.Fatalf("expected registered response schemas")
|
||||
}
|
||||
|
||||
for _, schema := range registered {
|
||||
metadataMap := schema.DiagnosticsMap()
|
||||
if metadataMap["id"] != schema.ID ||
|
||||
metadataMap["version"] != schema.Version ||
|
||||
metadataMap["name"] != schema.Name ||
|
||||
metadataMap["sha256"] != schema.SHA256 {
|
||||
t.Fatalf("unexpected diagnostics metadata map for %q: %+v", schema.ID, metadataMap)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -527,7 +527,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
diagnosticsWriter = &llmDiagnosticsWriterAdapter{
|
||||
writer: llm.NewDiagnosticsWriter(
|
||||
filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName),
|
||||
validatorSecrets(input.Config),
|
||||
llm.ConfiguredSecrets(input.Config),
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -744,15 +744,3 @@ func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMeta
|
||||
ErrorPayloadPath: art.ErrorPayloadPath,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func validatorSecrets(cfg *config.Config) []string {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
effective := cfg.EffectiveValidationLLMConfig()
|
||||
return []string{
|
||||
cfg.PrimaryLLM.APIKey,
|
||||
effective.APIKey,
|
||||
cfg.ValidationLLM.APIKey,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1307,12 +1307,13 @@ func TestRunnerLLMValidatorBatchingAndSchedulerUsage(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
secret := "super-secret-key"
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: secret}}}}}
|
||||
primarySecret := "runner-primary-secret"
|
||||
validationSecret := "runner-validation-secret"
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = secret
|
||||
cfg.ValidationLLM.APIKey = secret
|
||||
cfg.PrimaryLLM.APIKey = primarySecret
|
||||
cfg.ValidationLLM.APIKey = validationSecret
|
||||
diagDir := t.TempDir()
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
@@ -1321,7 +1322,7 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
}})
|
||||
out, err := r.Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple. " + primarySecret}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||
ValidationLLMClient: client,
|
||||
ValidationDiagnosticsDir: diagDir,
|
||||
@@ -1332,12 +1333,26 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
if len(out.ModuleResults[0].ValidatorDecisions) == 0 || out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath == "" {
|
||||
t.Fatalf("expected diagnostic artifact path on decision")
|
||||
}
|
||||
matches, globErr := filepath.Glob(filepath.Join(diagDir, "m", "*.json"))
|
||||
if globErr != nil {
|
||||
t.Fatalf("glob diagnostics: %v", globErr)
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
t.Fatalf("expected diagnostics JSON artifacts under %s", filepath.Join(diagDir, "m"))
|
||||
}
|
||||
for _, path := range matches {
|
||||
raw, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
t.Fatalf("read diagnostic %q: %v", path, readErr)
|
||||
}
|
||||
if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) {
|
||||
t.Fatalf("configured secret leaked in diagnostics %q: %s", path, string(raw))
|
||||
}
|
||||
}
|
||||
|
||||
raw, readErr := os.ReadFile(out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath)
|
||||
if readErr != nil {
|
||||
t.Fatalf("read diagnostic: %v", readErr)
|
||||
}
|
||||
if strings.Contains(string(raw), secret) {
|
||||
t.Fatalf("secret leaked in diagnostics: %s", string(raw))
|
||||
t.Fatalf("read decision diagnostic: %v", readErr)
|
||||
}
|
||||
if !strings.Contains(string(raw), "[REDACTED]") {
|
||||
t.Fatalf("expected redaction marker in diagnostics")
|
||||
|
||||
24
internal/framework/stagename/stagename.go
Normal file
24
internal/framework/stagename/stagename.go
Normal file
@@ -0,0 +1,24 @@
|
||||
package stagename
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func ModuleProposal(moduleInstance string, sectionIndex *int) string {
|
||||
if sectionIndex == nil || *sectionIndex == 0 {
|
||||
return fmt.Sprintf("%s:proposal", moduleInstance)
|
||||
}
|
||||
return fmt.Sprintf("%s:proposal:section-%04d", moduleInstance, *sectionIndex)
|
||||
}
|
||||
|
||||
func ProposalGeneration(moduleInstance string, sectionIndex *int) string {
|
||||
base := fmt.Sprintf("%s:proposal-generation", moduleInstance)
|
||||
if sectionIndex == nil {
|
||||
return base
|
||||
}
|
||||
return fmt.Sprintf("%s:section-%04d", base, *sectionIndex)
|
||||
}
|
||||
|
||||
func ValidatorBatch(moduleInstance string, validatorName string, batchIndex int) string {
|
||||
return fmt.Sprintf("%s:%s:batch-%04d", moduleInstance, validatorName, batchIndex)
|
||||
}
|
||||
33
internal/framework/stagename/stagename_test.go
Normal file
33
internal/framework/stagename/stagename_test.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package stagename
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestModuleProposalStageName(t *testing.T) {
|
||||
if got := ModuleProposal("grammar", nil); got != "grammar:proposal" {
|
||||
t.Fatalf("unexpected stage name without section: %q", got)
|
||||
}
|
||||
|
||||
sectionIndex := 7
|
||||
if got := ModuleProposal("grammar", §ionIndex); got != "grammar:proposal:section-0007" {
|
||||
t.Fatalf("unexpected stage name with section: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProposalGenerationStageName(t *testing.T) {
|
||||
if got := ProposalGeneration("grammar", nil); got != "grammar:proposal-generation" {
|
||||
t.Fatalf("unexpected proposal generation stage name without section: %q", got)
|
||||
}
|
||||
|
||||
sectionIndex := 3
|
||||
if got := ProposalGeneration("grammar", §ionIndex); got != "grammar:proposal-generation:section-0003" {
|
||||
t.Fatalf("unexpected proposal generation stage name with section: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorBatchStageName(t *testing.T) {
|
||||
if got := ValidatorBatch("homophones_1", "spoken_form_plausibility_review", 12); got != "homophones_1:spoken_form_plausibility_review:batch-0012" {
|
||||
t.Fatalf("unexpected validator batch stage name: %q", got)
|
||||
}
|
||||
}
|
||||
28
internal/framework/structuredoutput/malformed.go
Normal file
28
internal/framework/structuredoutput/malformed.go
Normal file
@@ -0,0 +1,28 @@
|
||||
package structuredoutput
|
||||
|
||||
import "strings"
|
||||
|
||||
var malformedMarkers = []string{
|
||||
"malformed structured output",
|
||||
"decode structured output:",
|
||||
"decode provider response envelope:",
|
||||
"provider response missing choices",
|
||||
"provider response missing assistant message content",
|
||||
"provider response assistant message content is empty",
|
||||
"provider response assistant message content is not valid JSON",
|
||||
}
|
||||
|
||||
// IsMalformedError reports whether err matches provider malformed
|
||||
// structured-output failure markers that should be downgraded.
|
||||
func IsMalformedError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := err.Error()
|
||||
for _, marker := range malformedMarkers {
|
||||
if strings.Contains(msg, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
32
internal/framework/structuredoutput/malformed_test.go
Normal file
32
internal/framework/structuredoutput/malformed_test.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package structuredoutput
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIsMalformedError(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{name: "nil", err: nil, want: false},
|
||||
{name: "generic", err: errors.New("network timeout"), want: false},
|
||||
{name: "malformed", err: errors.New("malformed structured output"), want: true},
|
||||
{name: "decode structured", err: errors.New("decode structured output: unexpected end of JSON input"), want: true},
|
||||
{name: "missing choices", err: errors.New("provider response missing choices"), want: true},
|
||||
{name: "missing content", err: errors.New("provider response missing assistant message content"), want: true},
|
||||
{name: "empty content", err: errors.New("provider response assistant message content is empty"), want: true},
|
||||
{name: "invalid content json", err: errors.New("provider response assistant message content is not valid JSON"), want: true},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := IsMalformedError(tc.err)
|
||||
if got != tc.want {
|
||||
t.Fatalf("IsMalformedError(%v): got=%v want=%v", tc.err, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,8 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
@@ -110,9 +112,10 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
|
||||
var response LLMValidationResponse
|
||||
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
|
||||
stage := stagename.ValidatorBatch(req.ModuleInstance, v.name, batch.BatchIndex)
|
||||
call := func(callCtx context.Context) error {
|
||||
_, err = req.LLMClient.CompleteStructured(callCtx, StructuredCompletionRequest{
|
||||
StageName: fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex),
|
||||
StageName: stage,
|
||||
Messages: messages,
|
||||
Model: resolvedValidationModel(req.Config, v.model),
|
||||
ResponseSchema: &responseSchema,
|
||||
@@ -126,7 +129,6 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
}
|
||||
artifacts := InteractionArtifacts{}
|
||||
if req.DiagnosticsWriter != nil {
|
||||
stage := fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex)
|
||||
promptMetadata := validatorPromptMetadata(v.validatorType)
|
||||
artifacts, _ = req.DiagnosticsWriter.WriteInteraction(
|
||||
stage,
|
||||
@@ -134,19 +136,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
"validator_name": v.name,
|
||||
"validator_type": v.validatorType,
|
||||
"batch_index": batch.BatchIndex,
|
||||
"prompt_metadata": map[string]any{
|
||||
"prompt_id": promptMetadata.PromptID,
|
||||
"prompt_version": promptMetadata.PromptVersion,
|
||||
"prompt_source": promptMetadata.PromptSource,
|
||||
"embedded_path": promptMetadata.EmbeddedPath,
|
||||
"sha256": promptMetadata.SHA256,
|
||||
},
|
||||
"response_schema": map[string]any{
|
||||
"id": responseSchema.ID,
|
||||
"version": responseSchema.Version,
|
||||
"name": responseSchema.Name,
|
||||
"sha256": responseSchema.SHA256,
|
||||
},
|
||||
"prompt_metadata": promptMetadata.DiagnosticsMap(),
|
||||
"response_schema": responseSchema.DiagnosticsMap(),
|
||||
},
|
||||
map[string]any{"messages": messages, "items": batch.Items},
|
||||
response,
|
||||
@@ -154,7 +145,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
)
|
||||
}
|
||||
if err != nil {
|
||||
if isMalformedStructuredOutputError(err) {
|
||||
if structuredoutput.IsMalformedError(err) {
|
||||
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
|
||||
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
|
||||
continue
|
||||
@@ -397,24 +388,3 @@ func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
|
||||
}
|
||||
return artifacts.ResponsePayloadPath
|
||||
}
|
||||
|
||||
func isMalformedStructuredOutputError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := err.Error()
|
||||
for _, marker := range []string{
|
||||
"malformed structured output",
|
||||
"decode structured output:",
|
||||
"decode provider response envelope:",
|
||||
"provider response missing choices",
|
||||
"provider response missing assistant message content",
|
||||
"provider response assistant message content is empty",
|
||||
"provider response assistant message content is not valid JSON",
|
||||
} {
|
||||
if strings.Contains(msg, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -265,6 +265,23 @@ func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorProviderMalformedEnvelopeRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{err: errors.New("provider response assistant message content is empty")}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
res, err := v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected malformed provider envelope downgrade, got %v", err)
|
||||
}
|
||||
if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
|
||||
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||
}
|
||||
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorMalformed {
|
||||
t.Fatalf("expected malformed warning, got %+v", res.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
@@ -118,13 +119,13 @@ func confidenceThresholdForModule(moduleKey string, cfg *config.Config) float64
|
||||
return 0.0
|
||||
}
|
||||
switch moduleKey {
|
||||
case "glossary":
|
||||
case modulecatalog.KeyGlossary:
|
||||
return cfg.Thresholds.Glossary
|
||||
case "grammar":
|
||||
case modulecatalog.KeyGrammar:
|
||||
return cfg.Thresholds.Grammar
|
||||
case "homophones":
|
||||
case modulecatalog.KeyHomophones:
|
||||
return cfg.Thresholds.Homophones
|
||||
case "spoken_word":
|
||||
case modulecatalog.KeySpokenWord:
|
||||
return cfg.Thresholds.SpokenWord
|
||||
default:
|
||||
return 0.0
|
||||
|
||||
@@ -2,12 +2,11 @@ package glossary
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
|
||||
)
|
||||
|
||||
@@ -37,67 +36,9 @@ func (m *Module) Validators() []contracts.Validator {
|
||||
}
|
||||
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
|
||||
sectionIndex := 0
|
||||
if req.Section != nil {
|
||||
sectionIndex = req.Section.Index
|
||||
}
|
||||
transcriptDescription := ""
|
||||
if req.Config != nil {
|
||||
transcriptDescription = req.Config.TranscriptDescription
|
||||
}
|
||||
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
|
||||
if err != nil {
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
|
||||
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
ModuleKey: req.RunSpec.ModuleKey,
|
||||
ModuleInstance: req.RunSpec.InstanceName,
|
||||
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
|
||||
WorkingTranscript: req.WorkingTranscript,
|
||||
Section: req.Section,
|
||||
Glossary: req.Glossary,
|
||||
Config: req.Config,
|
||||
Messages: messages,
|
||||
PromptMetadata: map[string]any{
|
||||
"prompt_id": proposalPromptMetadata().PromptID,
|
||||
"prompt_version": proposalPromptMetadata().PromptVersion,
|
||||
"prompt_source": proposalPromptMetadata().PromptSource,
|
||||
"embedded_path": proposalPromptMetadata().EmbeddedPath,
|
||||
"sha256": proposalPromptMetadata().SHA256,
|
||||
},
|
||||
StageName: proposalStageName(req),
|
||||
StartIndex: 0,
|
||||
LLMClient: req.LLMClient,
|
||||
Scheduler: req.LLMScheduler,
|
||||
DiagnosticsDir: req.DiagnosticsDir,
|
||||
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
|
||||
ProposalRequest: req,
|
||||
PromptID: prompts.PromptIDModuleGlossaryProposal,
|
||||
BuildMessages: BuildProposalMessages,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
return contracts.ProposalResult{
|
||||
Proposals: generated.Corrections,
|
||||
Warnings: generated.Warnings,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
|
||||
if section == nil || transcript == nil {
|
||||
return transcript
|
||||
}
|
||||
segments := make([]schema.Segment, 0, len(transcript.Segments))
|
||||
for _, seg := range transcript.Segments {
|
||||
if seg.ID >= section.StartSegmentID && seg.ID <= section.EndSegmentID {
|
||||
segments = append(segments, seg)
|
||||
}
|
||||
}
|
||||
return &schema.Transcript{Segments: segments}
|
||||
}
|
||||
|
||||
func proposalStageName(req contracts.ProposalRequest) string {
|
||||
if req.Section == nil || req.Section.Index == 0 {
|
||||
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName)
|
||||
}
|
||||
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index)
|
||||
}
|
||||
|
||||
@@ -10,43 +10,13 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
type promptSegment struct {
|
||||
ID int `json:"id"`
|
||||
Speaker string `json:"speaker"`
|
||||
Start float64 `json:"start"`
|
||||
End float64 `json:"end"`
|
||||
Text string `json:"text"`
|
||||
Categories []string `json:"categories,omitempty"`
|
||||
}
|
||||
|
||||
type promptTranscriptSection struct {
|
||||
SectionIndex int `json:"section_index"`
|
||||
Segments []promptSegment `json:"segments"`
|
||||
}
|
||||
|
||||
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
||||
}
|
||||
|
||||
sectionPayload := promptTranscriptSection{
|
||||
SectionIndex: sectionIndex,
|
||||
Segments: make([]promptSegment, 0),
|
||||
}
|
||||
if transcript != nil {
|
||||
for _, s := range transcript.Segments {
|
||||
sectionPayload.Segments = append(sectionPayload.Segments, promptSegment{
|
||||
ID: s.ID,
|
||||
Speaker: s.Speaker,
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
Text: s.Text,
|
||||
Categories: append([]string(nil), s.Categories...),
|
||||
})
|
||||
}
|
||||
}
|
||||
sectionJSON, err := json.MarshalIndent(sectionPayload, "", " ")
|
||||
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
||||
}
|
||||
@@ -65,7 +35,3 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
|
||||
{Role: "user", Content: user},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func proposalPromptMetadata() prompts.Metadata {
|
||||
return prompts.MustLookupMetadata(prompts.PromptIDModuleGlossaryProposal)
|
||||
}
|
||||
|
||||
@@ -2,12 +2,11 @@ package grammar
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
|
||||
)
|
||||
|
||||
@@ -37,67 +36,9 @@ func (m *Module) Validators() []contracts.Validator {
|
||||
}
|
||||
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
|
||||
sectionIndex := 0
|
||||
if req.Section != nil {
|
||||
sectionIndex = req.Section.Index
|
||||
}
|
||||
transcriptDescription := ""
|
||||
if req.Config != nil {
|
||||
transcriptDescription = req.Config.TranscriptDescription
|
||||
}
|
||||
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
|
||||
if err != nil {
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
|
||||
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
ModuleKey: req.RunSpec.ModuleKey,
|
||||
ModuleInstance: req.RunSpec.InstanceName,
|
||||
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
|
||||
WorkingTranscript: req.WorkingTranscript,
|
||||
Section: req.Section,
|
||||
Glossary: req.Glossary,
|
||||
Config: req.Config,
|
||||
Messages: messages,
|
||||
PromptMetadata: map[string]any{
|
||||
"prompt_id": proposalPromptMetadata().PromptID,
|
||||
"prompt_version": proposalPromptMetadata().PromptVersion,
|
||||
"prompt_source": proposalPromptMetadata().PromptSource,
|
||||
"embedded_path": proposalPromptMetadata().EmbeddedPath,
|
||||
"sha256": proposalPromptMetadata().SHA256,
|
||||
},
|
||||
StageName: proposalStageName(req),
|
||||
StartIndex: 0,
|
||||
LLMClient: req.LLMClient,
|
||||
Scheduler: req.LLMScheduler,
|
||||
DiagnosticsDir: req.DiagnosticsDir,
|
||||
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
|
||||
ProposalRequest: req,
|
||||
PromptID: prompts.PromptIDModuleGrammarProposal,
|
||||
BuildMessages: BuildProposalMessages,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
return contracts.ProposalResult{
|
||||
Proposals: generated.Corrections,
|
||||
Warnings: generated.Warnings,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
|
||||
if section == nil || transcript == nil {
|
||||
return transcript
|
||||
}
|
||||
segments := make([]schema.Segment, 0, len(transcript.Segments))
|
||||
for _, seg := range transcript.Segments {
|
||||
if seg.ID >= section.StartSegmentID && seg.ID <= section.EndSegmentID {
|
||||
segments = append(segments, seg)
|
||||
}
|
||||
}
|
||||
return &schema.Transcript{Segments: segments}
|
||||
}
|
||||
|
||||
func proposalStageName(req contracts.ProposalRequest) string {
|
||||
if req.Section == nil || req.Section.Index == 0 {
|
||||
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName)
|
||||
}
|
||||
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index)
|
||||
}
|
||||
|
||||
@@ -10,20 +10,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
type promptSegment struct {
|
||||
ID int `json:"id"`
|
||||
Speaker string `json:"speaker"`
|
||||
Start float64 `json:"start"`
|
||||
End float64 `json:"end"`
|
||||
Text string `json:"text"`
|
||||
Categories []string `json:"categories,omitempty"`
|
||||
}
|
||||
|
||||
type promptTranscriptSection struct {
|
||||
SectionIndex int `json:"section_index"`
|
||||
Segments []promptSegment `json:"segments"`
|
||||
}
|
||||
|
||||
// BuildProposalMessages constrains corrections to punctuation/capitalization/
|
||||
// spacing cleanup with strict meaning guards.
|
||||
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
@@ -32,24 +18,7 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
|
||||
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
||||
}
|
||||
|
||||
sectionPayload := promptTranscriptSection{
|
||||
SectionIndex: sectionIndex,
|
||||
Segments: make([]promptSegment, 0),
|
||||
}
|
||||
if transcript != nil {
|
||||
for _, s := range transcript.Segments {
|
||||
sectionPayload.Segments = append(sectionPayload.Segments, promptSegment{
|
||||
ID: s.ID,
|
||||
Speaker: s.Speaker,
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
Text: s.Text,
|
||||
Categories: append([]string(nil), s.Categories...),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
sectionJSON, err := json.MarshalIndent(sectionPayload, "", " ")
|
||||
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
||||
}
|
||||
@@ -68,7 +37,3 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
|
||||
{Role: "user", Content: user},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func proposalPromptMetadata() prompts.Metadata {
|
||||
return prompts.MustLookupMetadata(prompts.PromptIDModuleGrammarProposal)
|
||||
}
|
||||
|
||||
@@ -2,12 +2,11 @@ package homophones
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
|
||||
)
|
||||
|
||||
@@ -37,67 +36,9 @@ func (m *Module) Validators() []contracts.Validator {
|
||||
}
|
||||
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
|
||||
sectionIndex := 0
|
||||
if req.Section != nil {
|
||||
sectionIndex = req.Section.Index
|
||||
}
|
||||
transcriptDescription := ""
|
||||
if req.Config != nil {
|
||||
transcriptDescription = req.Config.TranscriptDescription
|
||||
}
|
||||
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
|
||||
if err != nil {
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
|
||||
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
ModuleKey: req.RunSpec.ModuleKey,
|
||||
ModuleInstance: req.RunSpec.InstanceName,
|
||||
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
|
||||
WorkingTranscript: req.WorkingTranscript,
|
||||
Section: req.Section,
|
||||
Glossary: req.Glossary,
|
||||
Config: req.Config,
|
||||
Messages: messages,
|
||||
PromptMetadata: map[string]any{
|
||||
"prompt_id": proposalPromptMetadata().PromptID,
|
||||
"prompt_version": proposalPromptMetadata().PromptVersion,
|
||||
"prompt_source": proposalPromptMetadata().PromptSource,
|
||||
"embedded_path": proposalPromptMetadata().EmbeddedPath,
|
||||
"sha256": proposalPromptMetadata().SHA256,
|
||||
},
|
||||
StageName: proposalStageName(req),
|
||||
StartIndex: 0,
|
||||
LLMClient: req.LLMClient,
|
||||
Scheduler: req.LLMScheduler,
|
||||
DiagnosticsDir: req.DiagnosticsDir,
|
||||
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
|
||||
ProposalRequest: req,
|
||||
PromptID: prompts.PromptIDModuleHomophonesProposal,
|
||||
BuildMessages: BuildProposalMessages,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
return contracts.ProposalResult{
|
||||
Proposals: generated.Corrections,
|
||||
Warnings: generated.Warnings,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
|
||||
if section == nil || transcript == nil {
|
||||
return transcript
|
||||
}
|
||||
segments := make([]schema.Segment, 0, len(transcript.Segments))
|
||||
for _, seg := range transcript.Segments {
|
||||
if seg.ID >= section.StartSegmentID && seg.ID <= section.EndSegmentID {
|
||||
segments = append(segments, seg)
|
||||
}
|
||||
}
|
||||
return &schema.Transcript{Segments: segments}
|
||||
}
|
||||
|
||||
func proposalStageName(req contracts.ProposalRequest) string {
|
||||
if req.Section == nil || req.Section.Index == 0 {
|
||||
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName)
|
||||
}
|
||||
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index)
|
||||
}
|
||||
|
||||
@@ -10,20 +10,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
type promptSegment struct {
|
||||
ID int `json:"id"`
|
||||
Speaker string `json:"speaker"`
|
||||
Start float64 `json:"start"`
|
||||
End float64 `json:"end"`
|
||||
Text string `json:"text"`
|
||||
Categories []string `json:"categories,omitempty"`
|
||||
}
|
||||
|
||||
type promptTranscriptSection struct {
|
||||
SectionIndex int `json:"section_index"`
|
||||
Segments []promptSegment `json:"segments"`
|
||||
}
|
||||
|
||||
// BuildProposalMessages constrains corrections to conservative homophone and
|
||||
// mistranscription updates.
|
||||
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
@@ -32,23 +18,7 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
|
||||
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
||||
}
|
||||
|
||||
sectionPayload := promptTranscriptSection{
|
||||
SectionIndex: sectionIndex,
|
||||
Segments: make([]promptSegment, 0),
|
||||
}
|
||||
if transcript != nil {
|
||||
for _, s := range transcript.Segments {
|
||||
sectionPayload.Segments = append(sectionPayload.Segments, promptSegment{
|
||||
ID: s.ID,
|
||||
Speaker: s.Speaker,
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
Text: s.Text,
|
||||
Categories: append([]string(nil), s.Categories...),
|
||||
})
|
||||
}
|
||||
}
|
||||
sectionJSON, err := json.MarshalIndent(sectionPayload, "", " ")
|
||||
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
||||
}
|
||||
@@ -67,7 +37,3 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
|
||||
{Role: "user", Content: user},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func proposalPromptMetadata() prompts.Metadata {
|
||||
return prompts.MustLookupMetadata(prompts.PromptIDModuleHomophonesProposal)
|
||||
}
|
||||
|
||||
@@ -2,12 +2,11 @@ package spoken_word
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
|
||||
)
|
||||
|
||||
@@ -37,67 +36,9 @@ func (m *Module) Validators() []contracts.Validator {
|
||||
}
|
||||
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
|
||||
sectionIndex := 0
|
||||
if req.Section != nil {
|
||||
sectionIndex = req.Section.Index
|
||||
}
|
||||
transcriptDescription := ""
|
||||
if req.Config != nil {
|
||||
transcriptDescription = req.Config.TranscriptDescription
|
||||
}
|
||||
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
|
||||
if err != nil {
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
|
||||
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
ModuleKey: req.RunSpec.ModuleKey,
|
||||
ModuleInstance: req.RunSpec.InstanceName,
|
||||
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
|
||||
WorkingTranscript: req.WorkingTranscript,
|
||||
Section: req.Section,
|
||||
Glossary: req.Glossary,
|
||||
Config: req.Config,
|
||||
Messages: messages,
|
||||
PromptMetadata: map[string]any{
|
||||
"prompt_id": proposalPromptMetadata().PromptID,
|
||||
"prompt_version": proposalPromptMetadata().PromptVersion,
|
||||
"prompt_source": proposalPromptMetadata().PromptSource,
|
||||
"embedded_path": proposalPromptMetadata().EmbeddedPath,
|
||||
"sha256": proposalPromptMetadata().SHA256,
|
||||
},
|
||||
StageName: proposalStageName(req),
|
||||
StartIndex: 0,
|
||||
LLMClient: req.LLMClient,
|
||||
Scheduler: req.LLMScheduler,
|
||||
DiagnosticsDir: req.DiagnosticsDir,
|
||||
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
|
||||
ProposalRequest: req,
|
||||
PromptID: prompts.PromptIDModuleSpokenWordProposal,
|
||||
BuildMessages: BuildProposalMessages,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
return contracts.ProposalResult{
|
||||
Proposals: generated.Corrections,
|
||||
Warnings: generated.Warnings,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
|
||||
if section == nil || transcript == nil {
|
||||
return transcript
|
||||
}
|
||||
segments := make([]schema.Segment, 0, len(transcript.Segments))
|
||||
for _, seg := range transcript.Segments {
|
||||
if seg.ID >= section.StartSegmentID && seg.ID <= section.EndSegmentID {
|
||||
segments = append(segments, seg)
|
||||
}
|
||||
}
|
||||
return &schema.Transcript{Segments: segments}
|
||||
}
|
||||
|
||||
func proposalStageName(req contracts.ProposalRequest) string {
|
||||
if req.Section == nil || req.Section.Index == 0 {
|
||||
return fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName)
|
||||
}
|
||||
return fmt.Sprintf("%s:proposal:section-%04d", req.RunSpec.InstanceName, req.Section.Index)
|
||||
}
|
||||
|
||||
@@ -10,20 +10,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
type promptSegment struct {
|
||||
ID int `json:"id"`
|
||||
Speaker string `json:"speaker"`
|
||||
Start float64 `json:"start"`
|
||||
End float64 `json:"end"`
|
||||
Text string `json:"text"`
|
||||
Categories []string `json:"categories,omitempty"`
|
||||
}
|
||||
|
||||
type promptTranscriptSection struct {
|
||||
SectionIndex int `json:"section_index"`
|
||||
Segments []promptSegment `json:"segments"`
|
||||
}
|
||||
|
||||
// BuildProposalMessages constrains corrections to conservative dysfluency
|
||||
// cleanup with strict semantic preservation.
|
||||
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
@@ -32,23 +18,7 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
|
||||
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
||||
}
|
||||
|
||||
sectionPayload := promptTranscriptSection{
|
||||
SectionIndex: sectionIndex,
|
||||
Segments: make([]promptSegment, 0),
|
||||
}
|
||||
if transcript != nil {
|
||||
for _, s := range transcript.Segments {
|
||||
sectionPayload.Segments = append(sectionPayload.Segments, promptSegment{
|
||||
ID: s.ID,
|
||||
Speaker: s.Speaker,
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
Text: s.Text,
|
||||
Categories: append([]string(nil), s.Categories...),
|
||||
})
|
||||
}
|
||||
}
|
||||
sectionJSON, err := json.MarshalIndent(sectionPayload, "", " ")
|
||||
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
||||
}
|
||||
@@ -67,7 +37,3 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
|
||||
{Role: "user", Content: user},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func proposalPromptMetadata() prompts.Metadata {
|
||||
return prompts.MustLookupMetadata(prompts.PromptIDModuleSpokenWordProposal)
|
||||
}
|
||||
|
||||
@@ -40,6 +40,16 @@ type Metadata struct {
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
|
||||
func (m Metadata) DiagnosticsMap() map[string]any {
|
||||
return map[string]any{
|
||||
"prompt_id": m.PromptID,
|
||||
"prompt_version": m.PromptVersion,
|
||||
"prompt_source": m.PromptSource,
|
||||
"embedded_path": m.EmbeddedPath,
|
||||
"sha256": m.SHA256,
|
||||
}
|
||||
}
|
||||
|
||||
type definition struct {
|
||||
id string
|
||||
version string
|
||||
|
||||
@@ -107,3 +107,16 @@ func TestRenderedPromptsContainHardening(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticsMapIncludesStablePromptMetadataShapeForAllPrompts(t *testing.T) {
|
||||
for _, m := range RegisteredMetadata() {
|
||||
metadataMap := m.DiagnosticsMap()
|
||||
if metadataMap["prompt_id"] != m.PromptID ||
|
||||
metadataMap["prompt_version"] != m.PromptVersion ||
|
||||
metadataMap["prompt_source"] != m.PromptSource ||
|
||||
metadataMap["embedded_path"] != m.EmbeddedPath ||
|
||||
metadataMap["sha256"] != m.SHA256 {
|
||||
t.Fatalf("unexpected diagnostics metadata map for %q: %+v", m.PromptID, metadataMap)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
57
internal/testsupport/files.go
Normal file
57
internal/testsupport/files.go
Normal file
@@ -0,0 +1,57 @@
|
||||
package testsupport
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func ReadFile(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read file %q: %v", path, err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func OnlyRunDir(t *testing.T, workDir string) string {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(workDir)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read work dir %q: %v", workDir, err)
|
||||
}
|
||||
dirs := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
dirs = append(dirs, filepath.Join(workDir, e.Name()))
|
||||
}
|
||||
}
|
||||
if len(dirs) != 1 {
|
||||
t.Fatalf("expected exactly one run dir in %q, found %d", workDir, len(dirs))
|
||||
}
|
||||
return dirs[0]
|
||||
}
|
||||
|
||||
func AssertNoSecretInFile(t *testing.T, path, secret string) {
|
||||
t.Helper()
|
||||
raw := string(ReadFile(t, path))
|
||||
if strings.Contains(raw, secret) {
|
||||
t.Fatalf("secret leaked in %s", path)
|
||||
}
|
||||
}
|
||||
|
||||
func AssertNoSecretInTree(t *testing.T, root, secret string) {
|
||||
t.Helper()
|
||||
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil || d == nil || d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
raw, readErr := os.ReadFile(path)
|
||||
if readErr == nil && strings.Contains(string(raw), secret) {
|
||||
t.Fatalf("secret leaked in %s", path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -2,13 +2,15 @@ package validators
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/protected_terms"
|
||||
)
|
||||
|
||||
var builtInChains = map[string][]string{
|
||||
"glossary": {
|
||||
modulecatalog.KeyGlossary: {
|
||||
KeyProposalShape,
|
||||
KeyNoEffect,
|
||||
KeyOriginalTextPresence,
|
||||
@@ -18,7 +20,7 @@ var builtInChains = map[string][]string{
|
||||
KeySpokenFormPlausibility,
|
||||
KeyMeaningReversalReview,
|
||||
},
|
||||
"homophones": {
|
||||
modulecatalog.KeyHomophones: {
|
||||
KeyProposalShape,
|
||||
KeyNoEffect,
|
||||
KeyOriginalTextPresence,
|
||||
@@ -28,7 +30,7 @@ var builtInChains = map[string][]string{
|
||||
KeySpokenFormPlausibility,
|
||||
KeyMeaningReversalReview,
|
||||
},
|
||||
"spoken_word": {
|
||||
modulecatalog.KeySpokenWord: {
|
||||
KeyProposalShape,
|
||||
KeyNoEffect,
|
||||
KeyOriginalTextPresence,
|
||||
@@ -38,7 +40,7 @@ var builtInChains = map[string][]string{
|
||||
KeyEditorialReview,
|
||||
KeyMeaningReversalReview,
|
||||
},
|
||||
"grammar": {
|
||||
modulecatalog.KeyGrammar: {
|
||||
KeyProposalShape,
|
||||
KeyNoEffect,
|
||||
KeyOriginalTextPresence,
|
||||
@@ -51,9 +53,10 @@ var builtInChains = map[string][]string{
|
||||
}
|
||||
|
||||
func BuiltInChainKeys(moduleKey string) ([]string, error) {
|
||||
keys, ok := builtInChains[moduleKey]
|
||||
key := strings.TrimSpace(moduleKey)
|
||||
keys, ok := builtInChains[key]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no built-in validator chain for module %q", moduleKey)
|
||||
return nil, fmt.Errorf("no built-in validator chain for module %q", key)
|
||||
}
|
||||
out := make([]string, len(keys))
|
||||
copy(out, keys)
|
||||
@@ -61,6 +64,7 @@ func BuiltInChainKeys(moduleKey string) ([]string, error) {
|
||||
}
|
||||
|
||||
func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]contracts.Validator, error) {
|
||||
moduleKey = strings.TrimSpace(moduleKey)
|
||||
keys, err := BuiltInChainKeys(moduleKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -71,7 +75,7 @@ func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]contracts.Vali
|
||||
|
||||
out := make([]contracts.Validator, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
if moduleKey == "glossary" && key == KeyProtectedTerms {
|
||||
if moduleKey == modulecatalog.KeyGlossary && key == KeyProtectedTerms {
|
||||
// Glossary stages preserve current stricter protection semantics while
|
||||
// reporting the stable protected_terms key.
|
||||
v, buildErr := protected_terms.NewGlossaryStage()
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
package metadata
|
||||
|
||||
import "gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type ExecutionClass string
|
||||
|
||||
@@ -9,6 +13,31 @@ const (
|
||||
ExecutionClassLLMBacked ExecutionClass = "llm_backed"
|
||||
)
|
||||
|
||||
const (
|
||||
KeyProposalShape = "proposal_shape"
|
||||
KeyConfidenceThreshold = "confidence_threshold"
|
||||
KeyOriginalTextPresence = "original_text_presence"
|
||||
KeyNonEmptyCorrectedText = "non_empty_corrected_text"
|
||||
KeyNoEffect = "no_effect"
|
||||
KeyProtectedTerms = "protected_terms"
|
||||
|
||||
KeySpokenFormPlausibility = "spoken_form_plausibility"
|
||||
KeyMeaningReversalReview = "meaning_reversal_review"
|
||||
KeyEditorialReview = "editorial_review"
|
||||
)
|
||||
|
||||
var executionClassByKey = map[string]ExecutionClass{
|
||||
KeyProposalShape: ExecutionClassDeterministic,
|
||||
KeyConfidenceThreshold: ExecutionClassDeterministic,
|
||||
KeyOriginalTextPresence: ExecutionClassDeterministic,
|
||||
KeyNonEmptyCorrectedText: ExecutionClassDeterministic,
|
||||
KeyNoEffect: ExecutionClassDeterministic,
|
||||
KeyProtectedTerms: ExecutionClassDeterministic,
|
||||
KeySpokenFormPlausibility: ExecutionClassLLMBacked,
|
||||
KeyMeaningReversalReview: ExecutionClassLLMBacked,
|
||||
KeyEditorialReview: ExecutionClassLLMBacked,
|
||||
}
|
||||
|
||||
type ClassifiedValidator interface {
|
||||
contracts.Validator
|
||||
ExecutionClass() ExecutionClass
|
||||
@@ -18,9 +47,11 @@ func ClassOf(v contracts.Validator) ExecutionClass {
|
||||
if v == nil {
|
||||
return ExecutionClassDeterministic
|
||||
}
|
||||
|
||||
classFromKey := ClassForKey(v.Name())
|
||||
classified, ok := v.(ClassifiedValidator)
|
||||
if !ok {
|
||||
return ExecutionClassDeterministic
|
||||
return classFromKey
|
||||
}
|
||||
switch classified.ExecutionClass() {
|
||||
case ExecutionClassLLMBacked:
|
||||
@@ -28,8 +59,16 @@ func ClassOf(v contracts.Validator) ExecutionClass {
|
||||
case ExecutionClassDeterministic:
|
||||
return ExecutionClassDeterministic
|
||||
default:
|
||||
return classFromKey
|
||||
}
|
||||
}
|
||||
|
||||
func ClassForKey(key string) ExecutionClass {
|
||||
class, ok := executionClassByKey[strings.TrimSpace(key)]
|
||||
if !ok {
|
||||
return ExecutionClassDeterministic
|
||||
}
|
||||
return class
|
||||
}
|
||||
|
||||
func Wrap(v contracts.Validator, class ExecutionClass) contracts.Validator {
|
||||
|
||||
@@ -16,6 +16,16 @@ func (u unclassifiedValidator) Validate(_ context.Context, _ contracts.Validatio
|
||||
return frameworkvalidators.Result{ValidatorName: u.Name(), Decisions: nil}, nil
|
||||
}
|
||||
|
||||
type namedUnclassifiedValidator struct {
|
||||
name string
|
||||
}
|
||||
|
||||
func (n namedUnclassifiedValidator) Name() string { return n.name }
|
||||
|
||||
func (n namedUnclassifiedValidator) Validate(_ context.Context, _ contracts.ValidationRequest) (frameworkvalidators.Result, error) {
|
||||
return frameworkvalidators.Result{ValidatorName: n.Name(), Decisions: nil}, nil
|
||||
}
|
||||
|
||||
func TestClassOfDefaultsToDeterministic(t *testing.T) {
|
||||
if got := ClassOf(unclassifiedValidator{}); got != ExecutionClassDeterministic {
|
||||
t.Fatalf("expected deterministic default class, got %q", got)
|
||||
@@ -28,3 +38,22 @@ func TestWrapExposesExecutionClass(t *testing.T) {
|
||||
t.Fatalf("expected llm_backed class, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassForKey(t *testing.T) {
|
||||
if got := ClassForKey(KeyProposalShape); got != ExecutionClassDeterministic {
|
||||
t.Fatalf("expected deterministic class for %q, got %q", KeyProposalShape, got)
|
||||
}
|
||||
if got := ClassForKey(KeySpokenFormPlausibility); got != ExecutionClassLLMBacked {
|
||||
t.Fatalf("expected llm_backed class for %q, got %q", KeySpokenFormPlausibility, got)
|
||||
}
|
||||
if got := ClassForKey("unknown"); got != ExecutionClassDeterministic {
|
||||
t.Fatalf("expected deterministic fallback for unknown key, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassOfFallsBackToStableValidatorKey(t *testing.T) {
|
||||
v := namedUnclassifiedValidator{name: KeyEditorialReview}
|
||||
if got := ClassOf(v); got != ExecutionClassLLMBacked {
|
||||
t.Fatalf("expected llm_backed fallback by key, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/confidence_threshold"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/editorial_review"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/meaning_reversal_review"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/no_effect"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/non_empty_corrected_text"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/original_text_presence"
|
||||
@@ -17,22 +18,21 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
KeyProposalShape = "proposal_shape"
|
||||
KeyConfidenceThreshold = "confidence_threshold"
|
||||
KeyOriginalTextPresence = "original_text_presence"
|
||||
KeyNonEmptyCorrectedText = "non_empty_corrected_text"
|
||||
KeyNoEffect = "no_effect"
|
||||
KeyProtectedTerms = "protected_terms"
|
||||
KeyProposalShape = validatormetadata.KeyProposalShape
|
||||
KeyConfidenceThreshold = validatormetadata.KeyConfidenceThreshold
|
||||
KeyOriginalTextPresence = validatormetadata.KeyOriginalTextPresence
|
||||
KeyNonEmptyCorrectedText = validatormetadata.KeyNonEmptyCorrectedText
|
||||
KeyNoEffect = validatormetadata.KeyNoEffect
|
||||
KeyProtectedTerms = validatormetadata.KeyProtectedTerms
|
||||
|
||||
KeySpokenFormPlausibility = "spoken_form_plausibility"
|
||||
KeyMeaningReversalReview = "meaning_reversal_review"
|
||||
KeyEditorialReview = "editorial_review"
|
||||
KeySpokenFormPlausibility = validatormetadata.KeySpokenFormPlausibility
|
||||
KeyMeaningReversalReview = validatormetadata.KeyMeaningReversalReview
|
||||
KeyEditorialReview = validatormetadata.KeyEditorialReview
|
||||
)
|
||||
|
||||
type BuiltInValidatorDefinition struct {
|
||||
Key string
|
||||
Build func() (contracts.Validator, error)
|
||||
LLMBacked bool
|
||||
}
|
||||
|
||||
type Registry struct {
|
||||
@@ -47,9 +47,9 @@ func NewBuiltInRegistry() *Registry {
|
||||
{Key: KeyNonEmptyCorrectedText, Build: non_empty_corrected_text.New},
|
||||
{Key: KeyNoEffect, Build: no_effect.New},
|
||||
{Key: KeyProtectedTerms, Build: protected_terms.New},
|
||||
{Key: KeySpokenFormPlausibility, LLMBacked: true, Build: spoken_form_plausibility.New},
|
||||
{Key: KeyMeaningReversalReview, LLMBacked: true, Build: meaning_reversal_review.New},
|
||||
{Key: KeyEditorialReview, LLMBacked: true, Build: editorial_review.New},
|
||||
{Key: KeySpokenFormPlausibility, Build: spoken_form_plausibility.New},
|
||||
{Key: KeyMeaningReversalReview, Build: meaning_reversal_review.New},
|
||||
{Key: KeyEditorialReview, Build: editorial_review.New},
|
||||
}
|
||||
|
||||
m := make(map[string]BuiltInValidatorDefinition, len(defs))
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
@@ -81,11 +82,6 @@ func TestBuiltInValidatorPackagesConstruct(t *testing.T) {
|
||||
|
||||
func TestRegistryBuildsClassifiedValidators(t *testing.T) {
|
||||
r := NewBuiltInRegistry()
|
||||
llmKeys := map[string]bool{
|
||||
KeySpokenFormPlausibility: true,
|
||||
KeyMeaningReversalReview: true,
|
||||
KeyEditorialReview: true,
|
||||
}
|
||||
for _, key := range r.RegisteredKeys() {
|
||||
v, err := r.MustBuild(key)
|
||||
if err != nil {
|
||||
@@ -94,16 +90,28 @@ func TestRegistryBuildsClassifiedValidators(t *testing.T) {
|
||||
if _, ok := v.(validatormetadata.ClassifiedValidator); !ok {
|
||||
t.Fatalf("expected built validator %q to expose execution classification metadata", key)
|
||||
}
|
||||
want := validatormetadata.ExecutionClassDeterministic
|
||||
if llmKeys[key] {
|
||||
want = validatormetadata.ExecutionClassLLMBacked
|
||||
}
|
||||
want := validatormetadata.ClassForKey(key)
|
||||
if got := validatormetadata.ClassOf(v); got != want {
|
||||
t.Fatalf("expected class %q for %q, got %q", want, key, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutionClassResolvableByStableKeyAndByValidatorInstance(t *testing.T) {
|
||||
r := NewBuiltInRegistry()
|
||||
for _, key := range r.RegisteredKeys() {
|
||||
v, err := r.MustBuild(key)
|
||||
if err != nil {
|
||||
t.Fatalf("must build %q: %v", key, err)
|
||||
}
|
||||
fromKey := validatormetadata.ClassForKey(key)
|
||||
fromInstance := validatormetadata.ClassOf(v)
|
||||
if fromInstance != fromKey {
|
||||
t.Fatalf("class mismatch for %q: key=%q instance=%q", key, fromKey, fromInstance)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryProtectedTermsUsesNonGlossaryStageBehavior(t *testing.T) {
|
||||
r := NewBuiltInRegistry()
|
||||
v, err := r.MustBuild(KeyProtectedTerms)
|
||||
@@ -200,7 +208,7 @@ func TestBuiltInRegistryUnknownKeyFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestBuiltInChainKeysResolveForProductionModules(t *testing.T) {
|
||||
for _, moduleKey := range []string{"glossary", "homophones", "spoken_word", "grammar"} {
|
||||
for _, moduleKey := range modulecatalog.SupportedKeys() {
|
||||
keys, err := BuiltInChainKeys(moduleKey)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve keys for %q: %v", moduleKey, err)
|
||||
@@ -213,7 +221,7 @@ func TestBuiltInChainKeysResolveForProductionModules(t *testing.T) {
|
||||
|
||||
func TestResolveBuiltInChainUsesRegisteredKeys(t *testing.T) {
|
||||
r := NewBuiltInRegistry()
|
||||
for _, moduleKey := range []string{"glossary", "homophones", "spoken_word", "grammar"} {
|
||||
for _, moduleKey := range modulecatalog.SupportedKeys() {
|
||||
chain, err := ResolveBuiltInChain(moduleKey, r)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve chain for %q: %v", moduleKey, err)
|
||||
|
||||
Reference in New Issue
Block a user