Compare commits
3 Commits
v0.9.1
...
3d7057b437
| Author | SHA1 | Date | |
|---|---|---|---|
| 3d7057b437 | |||
| 32c8c8b446 | |||
| a3655f5540 |
10
README.md
10
README.md
@@ -19,6 +19,7 @@ Pipeline behavior includes:
|
||||
- conservative spoken-word dysfluency cleanup with semantic guardrails
|
||||
- grammar/punctuation/capitalization/formatting cleanup
|
||||
- validator-chain enforcement before application
|
||||
- malformed module-stage LLM payloads degrade to warnings/rejections instead of failing the run
|
||||
- run reports and diagnostics artifacts with secret redaction
|
||||
|
||||
## Build and Install
|
||||
@@ -130,6 +131,7 @@ audita process transcript.json \
|
||||
- Without `--output`, stdout contains transcript JSON only on success.
|
||||
- `--report-json` writes a file and is never printed to stdout.
|
||||
- stderr is human-readable diagnostics/errors.
|
||||
- successful runs remain quiet on stderr even when module warnings are recorded in report/diagnostics artifacts.
|
||||
|
||||
For subprocess orchestration guidance, see [`docs/subprocess-operations.md`](docs/subprocess-operations.md).
|
||||
|
||||
@@ -149,10 +151,10 @@ audita config print-effective --config audita.yml
|
||||
```
|
||||
|
||||
For full config-file schema and examples, see [`docs/configuration.md`](docs/configuration.md).
|
||||
For output-schema details, see [`docs/output-schemas.md`](docs/output-schemas.md).
|
||||
For built-in validator keys and chain definitions, see [`docs/validators.md`](docs/validators.md).
|
||||
For embedded prompt assets and prompt metadata behavior, see [`docs/prompts.md`](docs/prompts.md).
|
||||
For CLI/process compatibility guarantees, see [`docs/public-contract.md`](docs/public-contract.md).
|
||||
For output-schema details, see [`docs/architecture/output-schemas.md`](docs/architecture/output-schemas.md).
|
||||
For built-in validator keys and chain definitions, see [`docs/architecture/validators.md`](docs/architecture/validators.md).
|
||||
For embedded prompt assets and prompt metadata behavior, see [`docs/architecture/prompts.md`](docs/architecture/prompts.md).
|
||||
For CLI/process compatibility guarantees, see [`docs/architecture/public-contract.md`](docs/architecture/public-contract.md).
|
||||
|
||||
### Modules
|
||||
|
||||
|
||||
@@ -382,25 +382,22 @@ func TestProcessFailureMalformedStructuredLLMResponseViaSubprocessHook(t *testin
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected zero exit code, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
if !json.Valid([]byte(result.stdout)) {
|
||||
t.Fatalf("expected transcript JSON on stdout, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "runner_execution") {
|
||||
t.Fatalf("expected runner_execution failure, got %q", result.stderr)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "diagnostics:") {
|
||||
t.Fatalf("expected diagnostics path in stderr, got %q", result.stderr)
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
report := readFile(t, reportPath)
|
||||
if !json.Valid(report) {
|
||||
t.Fatalf("expected valid failure report JSON")
|
||||
t.Fatalf("expected valid success report JSON")
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log, got: %v", err)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
|
||||
t.Fatalf("did not expect error.log, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -133,11 +133,16 @@ internal/framework/validators/
|
||||
llm_batching.go
|
||||
llm_validators.go
|
||||
|
||||
internal/framework/warnings/
|
||||
warnings.go
|
||||
|
||||
internal/validators/
|
||||
metadata/
|
||||
metadata.go
|
||||
registry.go
|
||||
chains.go
|
||||
proposal_shape/
|
||||
validator.go
|
||||
confidence_threshold/
|
||||
validator.go
|
||||
original_text_presence/
|
||||
@@ -242,6 +247,7 @@ Important behavior details:
|
||||
- Explicit `--modules grammar`, `--modules glossary`, `--modules homophones`, and `--modules spoken_word` continue to run production module paths with LLM-backed proposal generation and validator-chain execution.
|
||||
- Default runs (without explicit module selection) perform LLM calls through production module and validator paths.
|
||||
- Success path is generally quiet on stderr.
|
||||
- Malformed module-stage LLM payloads degrade to validator rejections and module warnings instead of aborting the run.
|
||||
- Source IDs are preserved into a canonical transcript before normalization; normalization then reassigns output IDs sequentially from `1`.
|
||||
|
||||
## Implemented data contracts
|
||||
@@ -539,20 +545,20 @@ Validator execution classification metadata:
|
||||
|
||||
Stable built-in validator keys:
|
||||
- deterministic:
|
||||
- `proposal_shape`
|
||||
- `confidence_threshold`
|
||||
- `original_text_presence`
|
||||
- `non_empty_corrected_text`
|
||||
- `non_empty_corrected_text` (historical key name; current semantics reject empty resulting segment text)
|
||||
- `no_effect`
|
||||
- `protected_terms`
|
||||
- LLM-backed:
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
- `editorial_review`
|
||||
- `grammar_review`
|
||||
- `spoken_word_review`
|
||||
|
||||
Built-in module chains:
|
||||
- `glossary`:
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
@@ -561,6 +567,7 @@ Built-in module chains:
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
- `homophones`:
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
@@ -569,20 +576,22 @@ Built-in module chains:
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
- `spoken_word`:
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `spoken_word_review`
|
||||
- `editorial_review`
|
||||
- `meaning_reversal_review`
|
||||
- `grammar`:
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `grammar_review`
|
||||
- `editorial_review`
|
||||
- `meaning_reversal_review`
|
||||
|
||||
1.0 boundary:
|
||||
@@ -595,12 +604,14 @@ Built-in module chains:
|
||||
- prompt builders for:
|
||||
- spoken-form plausibility
|
||||
- meaning reversal detection
|
||||
- editorial review
|
||||
- grammar review
|
||||
- spoken-word review
|
||||
- editorial review
|
||||
- grammar review
|
||||
- spoken-word review
|
||||
- deterministic batching by `validation_max_prompt_tokens`;
|
||||
- strict cardinality validation of structured LLM decisions (missing/duplicate/unknown indexes fail);
|
||||
- safe failure behavior for malformed/invalid structured responses.
|
||||
- strict cardinality validation of synthesized validator decision sets;
|
||||
- malformed validator payloads reject only the affected batch with warnings;
|
||||
- oversized single-proposal validator inputs reject only the affected proposal;
|
||||
- transport/provider/runtime LLM call failures remain fatal.
|
||||
|
||||
`internal/framework/runner` wires LLM validators into existing validator chains using:
|
||||
- the internal structured LLM client abstraction (`contracts.StructuredLLMClient`);
|
||||
@@ -726,6 +737,7 @@ Current process reports include diagnostics metadata references for:
|
||||
Current process reports also include:
|
||||
- module-level results (when runner modules execute), including applied/skipped proposal changes;
|
||||
- run-level module summary totals and failed module instance metadata.
|
||||
- module-level warning records for malformed proposal-generation payloads and malformed validator batches.
|
||||
- module-level validator decisions and validator rejections.
|
||||
- optional decision-level diagnostic artifact paths for validator LLM interactions when available.
|
||||
- stable validator keys in `validator_name` fields for validator decisions/rejections.
|
||||
|
||||
@@ -63,6 +63,7 @@ Ledger records are flattened review entries derived from module results and incl
|
||||
- deterministic and LLM validator decision snapshots using stable validator keys.
|
||||
|
||||
Validator rejection and proposal-application skip are distinct dispositions.
|
||||
Module warnings are reported in module results and diagnostics metadata, but do not create standalone correction-ledger rows.
|
||||
|
||||
## Report references
|
||||
|
||||
@@ -71,6 +72,8 @@ Validator rejection and proposal-application skip are distinct dispositions.
|
||||
- correction ledger artifact;
|
||||
- existing transcript/normalization/chunking/invocation/effective-config artifacts.
|
||||
|
||||
Module report entries also include warning records for malformed proposal-generation payloads and malformed validator batches.
|
||||
|
||||
## Retention behavior
|
||||
|
||||
Run-directory retention follows configured policy:
|
||||
@@ -93,6 +96,9 @@ When debugging:
|
||||
- validator rejections:
|
||||
- inspect `correction-ledger.json` rejected entries and matching validator decisions;
|
||||
- inspect validator response diagnostics payloads.
|
||||
- module warnings:
|
||||
- inspect module `warnings` entries in `report.json` or `--report-json`;
|
||||
- follow any diagnostic artifact path on the warning to the recorded error/response payload.
|
||||
- application skips:
|
||||
- inspect `correction-ledger.json` skipped entries and skip reason codes;
|
||||
- compare with validator decisions to distinguish validation rejection vs apply-time skip.
|
||||
|
||||
@@ -93,6 +93,7 @@ Current values:
|
||||
`--report-json` output and diagnostics run-dir `report.json` use the same report schema metadata.
|
||||
|
||||
Validator decision/rejection records in reports use stable validator keys in `validator_name`.
|
||||
Module results may also include warning records for malformed module-stage LLM payloads.
|
||||
Report diagnostics metadata includes artifact-path fields for utilization diagnostics and correction ledger when diagnostics initialization succeeds.
|
||||
|
||||
## Diagnostics directory behavior
|
||||
@@ -123,6 +124,7 @@ Success behavior:
|
||||
- with `--output`, stdout is empty
|
||||
- without `--output`, stdout contains only transcript JSON in selected output schema
|
||||
- report JSON is not written to stdout
|
||||
- success stderr remains empty even when reports/diagnostics contain module warnings
|
||||
|
||||
Failure behavior:
|
||||
- stderr contains human-readable error summary
|
||||
@@ -149,6 +151,7 @@ Config files should reference secrets via environment variable names (`api_key_e
|
||||
## Compatibility and deprecation policy
|
||||
|
||||
- Existing stable schema names, report metadata keys, and top-level command behavior are treated as public contract.
|
||||
- Existing stable validator keys remain public contract values even when validator semantics are refined.
|
||||
- Compatibility inputs (legacy flags/env aliases) may remain during transition windows.
|
||||
- Any planned removal or behavior change should include clear compatibility notes and migration guidance.
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ For LLM-backed validator prompt asset details, see [`docs/prompts.md`](prompts.m
|
||||
|
||||
Built-in validator construction is package-owned under `internal/validators/<validator_key>`:
|
||||
- `internal/validators/confidence_threshold`
|
||||
- `internal/validators/proposal_shape`
|
||||
- `internal/validators/original_text_presence`
|
||||
- `internal/validators/non_empty_corrected_text`
|
||||
- `internal/validators/no_effect`
|
||||
@@ -45,12 +46,14 @@ Current 1.0 boundary:
|
||||
|
||||
### Deterministic validators
|
||||
|
||||
- `proposal_shape`
|
||||
- rejects malformed proposal fields before other validators run.
|
||||
- `confidence_threshold`
|
||||
- checks proposal confidence against module-specific configured threshold.
|
||||
- `original_text_presence`
|
||||
- ensures target segment exists and `original_text` exists in current working segment text.
|
||||
- `non_empty_corrected_text`
|
||||
- rejects blank/whitespace-only `corrected_text`.
|
||||
- rejects proposals whose previewed resulting segment text would be empty or whitespace-only.
|
||||
- `no_effect`
|
||||
- rejects proposals where `original_text == corrected_text`.
|
||||
- `protected_terms`
|
||||
@@ -71,6 +74,7 @@ Current 1.0 boundary:
|
||||
Current built-in chains resolved from `internal/validators/chains.go`:
|
||||
|
||||
- `glossary`
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
@@ -80,6 +84,7 @@ Current built-in chains resolved from `internal/validators/chains.go`:
|
||||
- `meaning_reversal_review`
|
||||
|
||||
- `homophones`
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
@@ -89,6 +94,7 @@ Current built-in chains resolved from `internal/validators/chains.go`:
|
||||
- `meaning_reversal_review`
|
||||
|
||||
- `spoken_word`
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
@@ -98,6 +104,7 @@ Current built-in chains resolved from `internal/validators/chains.go`:
|
||||
- `meaning_reversal_review`
|
||||
|
||||
- `grammar`
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
@@ -119,7 +126,9 @@ Both variants preserve existing behavior and report the stable key `protected_te
|
||||
- modules execute serially;
|
||||
- section proposal work can run concurrently within a module;
|
||||
- deterministic validators run before LLM-backed validators;
|
||||
- malformed/missing/duplicate/unknown LLM validator decisions fail safely;
|
||||
- malformed module proposal payloads are downgraded to section-scoped module warnings with zero proposals for the affected section rather than module failure;
|
||||
- malformed/missing/duplicate/unknown LLM validator decisions reject the affected validator batch with warnings instead of failing the module;
|
||||
- oversized single-proposal validator inputs reject only the affected proposal under that validator;
|
||||
- approved proposals are applied once per module after section work settles.
|
||||
|
||||
## Validator rejections vs proposal-application skips
|
||||
@@ -128,12 +137,15 @@ Both variants preserve existing behavior and report the stable key `protected_te
|
||||
- proposal is denied by validator-chain review and appears in validator rejection reporting with validator key and reason code.
|
||||
- proposal-application skip:
|
||||
- proposal passed validators but could not be applied under replacement-policy semantics (for example no matching span at apply time).
|
||||
- module warning:
|
||||
- malformed proposal-generation payloads and malformed validator batches are recorded in module warning records and diagnostics without writing success stderr.
|
||||
|
||||
These are separate outcomes and are reported separately.
|
||||
|
||||
## Reporting and diagnostics identity
|
||||
|
||||
- report validator decision/rejection entries use stable validator keys in `validator_name`.
|
||||
- report module results include warning records for malformed module-stage LLM payloads.
|
||||
- validator LLM diagnostics include validator identity in interaction metadata and structured response schema metadata.
|
||||
- correction ledger entries include deterministic and LLM validator decision snapshots keyed by the same stable validator keys, and keep validator rejection distinct from application-level skip.
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ Use this checklist before cutting a pre-1.0 or 1.0 release candidate.
|
||||
- Verify structured response schemas are attached via `response_format.type=json_schema`.
|
||||
- Verify diagnostics metadata includes structured schema `id/version/name/sha256`.
|
||||
- Verify provider output is still locally decoded/validated before use.
|
||||
- Verify malformed module-stage structured payloads degrade to warnings/rejections instead of failing the run.
|
||||
|
||||
## Report and diagnostics schema checks
|
||||
|
||||
@@ -67,6 +68,7 @@ Use this checklist before cutting a pre-1.0 or 1.0 release candidate.
|
||||
- Verify prompt metadata appears in LLM request metadata diagnostics:
|
||||
- `prompt_id`, `prompt_version`, `prompt_source`, `embedded_path`, `sha256`.
|
||||
- Verify stable validator keys appear in report decisions/rejections.
|
||||
- Verify module warning records appear in reports for malformed proposal-generation payloads and malformed validator batches.
|
||||
- Verify built-in validator chains resolve and execute for default and explicit module runs.
|
||||
|
||||
## Utilization diagnostics checks
|
||||
@@ -96,6 +98,8 @@ Use this checklist before cutting a pre-1.0 or 1.0 release candidate.
|
||||
## Failure and cancellation checks
|
||||
|
||||
- Verify controlled failure paths retain diagnostics and produce best-effort failure reports.
|
||||
- Verify malformed proposal-generation payloads keep exit code `0`, keep stderr empty on success, and record warnings in reports/diagnostics.
|
||||
- Verify malformed validator payloads reject only the affected batch and do not fail the module.
|
||||
- Verify timeout/cancellation paths exit nonzero, do not hang, and retain failure diagnostics when initialized.
|
||||
|
||||
## Release fixture/idempotence checks
|
||||
|
||||
791
docs/roadmap/audit.md
Normal file
791
docs/roadmap/audit.md
Normal file
@@ -0,0 +1,791 @@
|
||||
# Pre-1.0 Code Quality and Deduplication Audit
|
||||
|
||||
## 1. Executive summary
|
||||
|
||||
Audita is in good shape for a limited pre-1.0 cleanup pass. The repository is small, package boundaries are mostly explicit, and the core public contract is already documented around `audita process`, config loading, output schemas, diagnostics, reports, embedded prompts, modules, and validators. The highest-value improvements are targeted centralization, not a rewrite.
|
||||
|
||||
Top three refactoring targets before 1.0:
|
||||
|
||||
1. Centralize module proposal plumbing and prompt payload construction across the four production modules.
|
||||
2. Centralize effective config loading plus schema/module catalog validation so `process`, `config print-effective`, and `config validate` cannot drift.
|
||||
3. Centralize diagnostics artifact names, stage names, and validator classification metadata used by reports and the correction ledger.
|
||||
|
||||
No major architectural risk is apparent. The main pre-1.0 risk is public-behavior drift from repeated policy strings, catalog values, artifact paths, and nearly identical command/module scaffolding.
|
||||
|
||||
This report was written to `docs/roadmap/audit.md`. `docs/roadmap/` already exists in the repository, although its previous `publish.md` file is currently deleted in the worktree by an unrelated change.
|
||||
|
||||
## 2. Repository map reviewed
|
||||
|
||||
Reviewed documentation:
|
||||
|
||||
- `README.md`
|
||||
- `docs/configuration.md`
|
||||
- `docs/architecture/architecture.md`
|
||||
- `docs/architecture/public-contract.md`
|
||||
- `docs/architecture/diagnostics.md`
|
||||
- `docs/architecture/output-schemas.md`
|
||||
- `docs/architecture/prompts.md`
|
||||
- `docs/architecture/validators.md`
|
||||
- `docs/architecture/structured-llm.md`
|
||||
- `docs/integration/subprocess-operations.md`
|
||||
- `docs/release-checklist.md`
|
||||
|
||||
Reviewed implementation areas:
|
||||
|
||||
- `cmd/audita`
|
||||
- `internal/cli`
|
||||
- `internal/core/config`
|
||||
- `internal/core/schema`
|
||||
- `internal/core/io`
|
||||
- `internal/core/normalization`
|
||||
- `internal/core/chunking`
|
||||
- `internal/core/diagnostics`
|
||||
- `internal/core/outputschema`
|
||||
- `internal/core/reporting`
|
||||
- `internal/framework/contracts`
|
||||
- `internal/framework/modules`
|
||||
- `internal/framework/proposal_generation`
|
||||
- `internal/framework/proposals`
|
||||
- `internal/framework/runner`
|
||||
- `internal/framework/validators`
|
||||
- `internal/framework/llm`
|
||||
- `internal/framework/responseschema`
|
||||
- `internal/framework/promptcontext`
|
||||
- `internal/framework/warnings`
|
||||
- `internal/modules/glossary`
|
||||
- `internal/modules/homophones`
|
||||
- `internal/modules/spoken_word`
|
||||
- `internal/modules/grammar`
|
||||
- `internal/prompts`
|
||||
- `internal/validators`
|
||||
- package tests and CLI parity/release fixtures under `internal/cli/testdata`
|
||||
|
||||
Major execution paths reviewed:
|
||||
|
||||
- `audita process <transcript.json> --glossary <glossary.yaml>`
|
||||
- `audita config validate --config <path>`
|
||||
- `audita config print-effective [--config <path>]`
|
||||
- default module sequence resolution and repeated glossary instance naming
|
||||
- proposal generation, validator execution, proposal application, report writing, diagnostics writing, and retention
|
||||
|
||||
Important absent or not-applicable areas:
|
||||
|
||||
- No `pkg/` directory exists.
|
||||
- No `examples/` directory exists.
|
||||
- No `docs/internal/` directory exists.
|
||||
- No `internal/app`, `internal/stage`, `internal/storage`, `internal/artifacts`, or `internal/manifest` packages exist. Their closest equivalents are `internal/cli`, `internal/framework/runner`, `internal/core/diagnostics`, and `internal/core/reporting`.
|
||||
|
||||
## 3. High-confidence deduplication opportunities
|
||||
|
||||
### 3.1 Module proposal plumbing is duplicated across all production modules
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/modules/glossary/module.go`
|
||||
- `internal/modules/homophones/module.go`
|
||||
- `internal/modules/spoken_word/module.go`
|
||||
- `internal/modules/grammar/module.go`
|
||||
- `internal/modules/*/prompt.go`
|
||||
- `internal/framework/proposal_generation`
|
||||
- `internal/framework/promptcontext`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Each module has the same `Module` struct shape, `Validators` copy behavior, `Propose` flow, section transcript extraction, transcript description extraction, `proposal_generation.GenerateCandidates` request construction, prompt metadata map construction, and stage-name formatting.
|
||||
- Each module also has a near-identical prompt payload builder with local `promptSegment` and `promptTranscriptSection` types, glossary JSON marshaling, transcript section JSON marshaling, transcript description block rendering, and two-message return shape.
|
||||
- `collectSectionProposals` already passes a section transcript to each module, but each module then filters that transcript again by section metadata.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- A diagnostics or prompt-context bug fix would need to be repeated in four modules.
|
||||
- Prompt metadata fields and stage names are diagnostics-visible and could drift by module.
|
||||
- The double section filtering is currently harmless, but it obscures the runner/module contract.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Add a small shared helper for module proposal execution, likely in `internal/framework/proposal_generation` or a narrow `internal/modules/modulekit` package.
|
||||
- Keep domain-specific prompt IDs and prompt text local to each module.
|
||||
- Move transcript section prompt payload construction into a shared prompt-context helper, for example `promptcontext.MarshalTranscriptSection`.
|
||||
- Provide one helper for prompt metadata maps instead of manually expanding `prompt_id`, `prompt_version`, `prompt_source`, `embedded_path`, and `sha256` in every module.
|
||||
- Preserve current module `Key`, replacement policy, and validator chain ownership.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Keep one golden or table-driven prompt payload test per module for domain-specific wording.
|
||||
- Add shared tests for transcript section JSON shape, empty transcript handling, categories copy behavior, and prompt metadata fields.
|
||||
- Add a parity test that all four module `Propose` methods still write diagnostics under the same module instance directory and produce the same correction mapping.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Low to medium. The behavior is highly duplicated, but prompt and diagnostics behavior is sensitive. Refactor behind existing module tests and CLI parity fixtures.
|
||||
|
||||
### 3.2 Effective config loading is repeated between commands
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/cli/run.go`
|
||||
- `internal/core/config`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- `runProcess` and `runConfigPrintEffective` both resolve config path, start from defaults, optionally load/apply file config, then apply environment overrides.
|
||||
- `runConfigValidate` separately loads a file, applies it to defaults, and validates it.
|
||||
- Path source metadata is computed in `internal/cli`, not `internal/core/config`, even though the precedence contract is documented as config behavior.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Config precedence is part of the public contract. If a future setting is added, three command paths may need coordinated updates.
|
||||
- `config print-effective` is the user-visible diagnostic for effective config. It should use the same loader as `process`, except for intentionally omitted CLI overrides.
|
||||
- The current code is understandable, but the behavior is repeated in a way that makes drift likely as config grows.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Add a narrow effective-config loader in `internal/core/config`, returning `Config`, source path, source type, and version metadata.
|
||||
- Keep command-specific CLI flag parsing in `internal/cli`.
|
||||
- Model the intentional differences explicitly:
|
||||
- `process`: defaults + file + env + CLI overrides
|
||||
- `config print-effective`: defaults + file + env
|
||||
- `config validate`: file schema + default-backed config validation, no env
|
||||
- Move `resolveConfigPath` or an equivalent path resolver into `internal/core/config`.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- One table-driven config loader test covering explicit `--config`, `AUDITA_CONFIG`, default search paths, missing explicit paths, and missing default paths.
|
||||
- CLI tests asserting `process` and `print-effective` share file+env behavior.
|
||||
- A regression test that `config validate` remains file-only and does not read environment overrides.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Low. Behavior is already explicit and well tested; the refactor can be done by moving code without changing precedence.
|
||||
|
||||
### 3.3 Module catalog validation is split across config, contracts, and module factory
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/core/config/validation.go`
|
||||
- `internal/framework/contracts/contracts.go`
|
||||
- `internal/framework/modules/registry.go`
|
||||
- `internal/validators/chains.go`
|
||||
- `internal/framework/validators/models.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Module keys appear in multiple places:
|
||||
- config default CSV: `glossary,homophones,glossary,spoken_word,grammar`
|
||||
- module factory constants and known-key map
|
||||
- built-in validator chains
|
||||
- confidence threshold lookup
|
||||
- individual module `Key()` methods
|
||||
- `Config.Validate` checks only that module names are non-empty. An unsupported configured module can pass `audita config validate` and fail later in `process` runner setup.
|
||||
- `contracts.ResolveModuleRunSpecs` only assigns instance names; it does not validate production module support.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- `audita config validate` is documented as a CI/preflight command. Letting unsupported modules pass weakens that preflight.
|
||||
- Module key drift could affect thresholds, validator chains, reports, and unsupported-module errors.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Introduce a small canonical module catalog or key package that can be imported by config validation, module factory construction, validator chain resolution, and threshold lookup without creating a cycle.
|
||||
- Keep module construction in `internal/framework/modules`; the catalog should expose keys and validation only.
|
||||
- Make `Config.Validate` reject unknown built-in module keys through that catalog.
|
||||
- Keep repeated module instances valid.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- `internal/core/config` test: unknown `pipeline.modules` fails validation.
|
||||
- `internal/cli` test: `audita config validate --config` rejects an unsupported module before runtime.
|
||||
- Existing `internal/framework/modules` unknown-module tests should continue to pass.
|
||||
- Validator chain tests should assert every catalog module has a built-in chain.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Medium. This tightens validation behavior. It is desirable before 1.0, but if unknown modules were intentionally allowed for future extension, document that explicitly instead.
|
||||
|
||||
### 3.4 Output schema support is hardcoded in config validation and registry
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/core/config/validation.go`
|
||||
- `internal/core/outputschema/registry.go`
|
||||
- `docs/architecture/output-schemas.md`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- `Config.Validate` hardcodes `bare-segments` and `audita-v1`.
|
||||
- `outputschema.Resolve` owns the actual output schema registry and returns the runtime error for unsupported schema names.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Adding or deferring a schema requires updating multiple places.
|
||||
- Public behavior could drift: a schema might validate in config but fail at output time, or vice versa.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Make `internal/core/outputschema` expose `IsSupported`, `SupportedKeys`, or a validation function.
|
||||
- Have config validation call that helper or consume shared constants.
|
||||
- Keep actual encoding logic in `outputschema`; config should not know encoder details.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Config validation test for every output schema returned by the registry.
|
||||
- Output schema registry test that unsupported `seriatim-intermediate` still fails clearly until implemented.
|
||||
- CLI test that unsupported `--output-schema` fails before output write.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Low. This is a straightforward catalog centralization.
|
||||
|
||||
### 3.5 Diagnostics artifact names and report metadata paths are repeated
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/core/diagnostics/run_dir.go`
|
||||
- `internal/cli/run.go`
|
||||
- `internal/core/reporting/report.go`
|
||||
- docs under `docs/architecture` and `docs/integration`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Artifact filenames such as `source-transcript.json`, `source-transcript-parsed.json`, `normalized-transcript.json`, `normalization-summary.json`, `chunking-summary.json`, `utilization-diagnostics.json`, `correction-ledger.json`, `invocation.json`, `effective-config.json`, `report.json`, and `error.log` are repeated between run-directory writers and `buildProcessReport`.
|
||||
- `runProcess` writes `utilization-diagnostics.json` and `correction-ledger.json` by raw string on both success and failure paths.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- These names are part of the documented diagnostics contract.
|
||||
- A filename change would need to be made in multiple places, and report metadata could point at files that are no longer written.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Define diagnostics artifact name constants in `internal/core/diagnostics`.
|
||||
- Add a helper that returns `reporting.DiagnosticsMetadata` for a run directory and status.
|
||||
- Add named methods for utilization diagnostics and correction ledger writes, or at least constants used by `WriteJSONArtifact`.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Unit test that `diagnostics.MetadataForRunDirectory` matches files written by `RunDirectory`.
|
||||
- CLI success/failure tests should continue to assert report metadata paths and actual file existence.
|
||||
- Add a test for failure report metadata including `error.log`.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Low. This is mostly string centralization, with high public-contract value.
|
||||
|
||||
### 3.6 Validator execution class is duplicated and partially hardcoded
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/validators/registry.go`
|
||||
- `internal/validators/metadata/metadata.go`
|
||||
- `internal/validators/*/validator.go`
|
||||
- `internal/framework/runner/runner.go`
|
||||
- `internal/cli/review_artifacts.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Validator constructors wrap validators with execution class metadata.
|
||||
- `BuiltInValidatorDefinition` also has an `LLMBacked` field.
|
||||
- Runner uses `metadata.ClassOf` to order deterministic validators before LLM-backed validators.
|
||||
- Correction ledger classification uses a local hardcoded map of LLM-backed validator names.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Adding a new LLM-backed validator could be ordered correctly by runner metadata but appear in the wrong correction-ledger section.
|
||||
- Validator class is domain metadata, not report-building policy. It should have one source of truth.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Make validator classification resolvable by validator instance or stable key from a single metadata source.
|
||||
- Remove the unused or redundant `LLMBacked` field, or make it the canonical source used by constructors, runner ordering, and ledger formatting.
|
||||
- Replace the local ledger map with `metadata.ClassOf` when possible, or a registry lookup by stable key.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Correction ledger test that LLM-backed decisions are classified from validator metadata, not a local string map.
|
||||
- Registry test that every registered LLM-backed validator reports the same class through every public metadata path.
|
||||
- Runner ordering test should remain in place.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Low to medium. The implementation is small, but correction-ledger shape is diagnostics-visible.
|
||||
|
||||
### 3.7 Malformed structured-output classification is duplicated
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/framework/proposal_generation/generate.go`
|
||||
- `internal/framework/validators/llm_validators.go`
|
||||
- `internal/framework/llm/openai_compatible_client.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Proposal generation and LLM validators both classify malformed structured-output errors by scanning error message substrings.
|
||||
- The marker lists are currently the same, but they are maintained independently.
|
||||
- The actual errors originate in the LLM adapter.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Proposal-generation malformed payloads become warnings with zero proposals, while validator malformed payloads reject affected batches with warnings. If classifiers drift, similar adapter failures could be downgraded in one workflow and hard-fail in another.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Prefer a typed error or exported classifier from `internal/framework/llm`.
|
||||
- If typed errors are too invasive, create one shared classifier function in a lower framework package used by both proposal generation and validators.
|
||||
- Preserve the different handling semantics at each call site.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Shared classifier table for all adapter malformed-output errors.
|
||||
- Proposal-generation test and validator test should assert the same representative malformed adapter errors are downgraded.
|
||||
- Adapter tests should assert typed/classified errors wrap useful context and still redact secrets.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Medium. Error typing can accidentally affect retry and wrapping behavior; do this with focused tests.
|
||||
|
||||
## 4. Medium-confidence opportunities
|
||||
|
||||
### 4.1 CLI flag registration and override extraction are large and repetitive
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/cli/run.go`
|
||||
- `internal/core/config/flags.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Each process flag has a field in `processFlags`, a registration entry in `newProcessFlagSet`, a case in `fs.Visit`, and an assignment in `config.ApplyCLIOverrides`.
|
||||
- File config and environment config also set many of the same effective config fields.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Adding a new config option requires multiple edits. Missing one edit could create a flag that displays but does not override, or a config field with no CLI override.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Avoid a generic reflection-heavy flag system before 1.0.
|
||||
- Consider a small metadata table only for simple scalar flags, or a focused helper that maps visited flags to `CLIOverrides`.
|
||||
- Keep nontrivial semantics, such as legacy concurrency alias precedence, explicit in code.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- CLI override parity test for every stable flag that mutates config.
|
||||
- A test that default flag values reflect file+env effective config before CLI overrides.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Medium. A broad flag abstraction would be riskier than the current duplication. Do only a small helper if it clearly reduces missed updates.
|
||||
|
||||
### 4.2 Config source application repeats field-level assignments
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/core/config/file_config.go`
|
||||
- `internal/core/config/env.go`
|
||||
- `internal/core/config/flags.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- The same effective fields are assigned from file config, env vars, and CLI overrides.
|
||||
- Some semantics differ intentionally: file config supports `api_key_env`, env supports `OPENROUTER_API_KEY` fallback, CLI uses direct values.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Field additions are easy to miss in one source.
|
||||
- Error messages and trimming behavior can drift.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Do not force all config sources through one generic mapper.
|
||||
- Add small setter helpers for repeated config subdomains such as LLM target, concurrency, thresholds, normalization, and diagnostics.
|
||||
- Keep source-specific parsing and error labels local.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Cross-source table proving file, env, and CLI all reach the same effective fields where they are meant to.
|
||||
- Tests for intentional differences: API key env resolution, `OPENROUTER_API_KEY` fallback, CLI direct API key, and transcript description trimming.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Medium. Useful, but only after the effective loader and catalog cleanup.
|
||||
|
||||
### 4.3 Prompt metadata and response schema metadata map construction repeats
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/modules/*/module.go`
|
||||
- `internal/framework/proposal_generation/generate.go`
|
||||
- `internal/framework/validators/llm_validators.go`
|
||||
- `internal/prompts`
|
||||
- `internal/framework/responseschema`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Prompt metadata maps are manually expanded in module proposal generation and validator diagnostics.
|
||||
- Response schema metadata maps are built independently in proposal generation and validator diagnostics.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Metadata fields are diagnostics-visible and useful for reproducibility.
|
||||
- Adding a metadata field requires updating multiple call sites.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Add `Metadata.Map()` or a typed diagnostics metadata struct in `internal/prompts`.
|
||||
- Add `responseschema.Metadata()` or a method returning a stable diagnostics shape.
|
||||
- Prefer typed structs over `map[string]any` where possible.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Prompt metadata rendering test should assert all registered prompts expose stable metadata.
|
||||
- Proposal and validator diagnostics tests should assert the shared metadata helper is used.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Low.
|
||||
|
||||
### 4.4 Secret redaction logic is split across config, LLM diagnostics, and adapter errors
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/core/config/redaction.go`
|
||||
- `internal/framework/llm/diagnostics.go`
|
||||
- `internal/framework/llm/client_common.go`
|
||||
- `internal/framework/proposal_generation/generate.go`
|
||||
- `internal/framework/runner/runner.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Config redaction replaces non-empty API keys with `[REDACTED]`.
|
||||
- LLM diagnostics replace configured secret values and `Bearer <secret>`.
|
||||
- Adapter error sanitization separately replaces secrets and bearer values.
|
||||
- Proposal and validator paths separately assemble secret lists.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Secret redaction is a public guarantee.
|
||||
- New secret-bearing config fields could be missed in one path.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Add a small redaction helper package or keep it in `internal/framework/llm` only if it remains LLM-specific.
|
||||
- Centralize `[]string` secret extraction from `config.Config`.
|
||||
- Keep config structural redaction separate from byte/string payload redaction, but share the redaction token and value replacement behavior.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- One test that a proposal-generation error, validator diagnostic artifact, effective config artifact, and surfaced provider error all redact the same configured secrets.
|
||||
- Existing subprocess no-secret-leak test should remain as an end-to-end guard.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Medium. The current coverage appears strong; change carefully.
|
||||
|
||||
### 4.5 Test fakes and fixture helpers are duplicated across packages
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/modules/*/module_test.go`
|
||||
- `internal/framework/proposal_generation/generate_test.go`
|
||||
- `internal/framework/validators/llm_validators_test.go`
|
||||
- `internal/cli/run_test.go`
|
||||
- `cmd/audita/main_integration_test.go`
|
||||
- `internal/cli/release_fixtures_test.go`
|
||||
- `internal/cli/parity_test.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Several packages define fake structured LLM clients, fixture path helpers, read/write helpers, diagnostics glob assertions, and run-directory helpers.
|
||||
- The four module test files have particularly similar fake clients and proposal-diagnostics assertions.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Refactors in LLM or diagnostics behavior require updating many tests.
|
||||
- Some duplicated tests are valuable because they preserve per-module public behavior; the issue is helper duplication, not coverage volume.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Add package-local helper files where duplication is within a package.
|
||||
- For cross-package fakes, prefer a small internal test support package only if it does not create import cycles or hide test intent.
|
||||
- Keep module-specific assertions local.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- This is test infrastructure cleanup. Existing tests should remain semantically equivalent.
|
||||
- Add helper tests only if helpers contain nontrivial behavior, such as fake response sequencing.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Low.
|
||||
|
||||
### 4.6 Stage-name construction is inconsistent enough to centralize, but not enough to redesign
|
||||
|
||||
Affected files/packages:
|
||||
|
||||
- `internal/modules/*/module.go`
|
||||
- `internal/framework/proposal_generation/generate.go`
|
||||
- `internal/framework/validators/llm_validators.go`
|
||||
- `internal/framework/runner/observability.go`
|
||||
|
||||
Duplicated or near-duplicated behavior:
|
||||
|
||||
- Modules pass stage names like `<module_instance>:proposal:section-0001`.
|
||||
- `proposal_generation` has a default builder using `<module_instance>:proposal-generation:section-0001`, but production modules bypass it.
|
||||
- Validators build `<module_instance>:<validator>:batch-0001`.
|
||||
- Utilization extracts module instance by splitting stage names on `:`.
|
||||
|
||||
Why it matters:
|
||||
|
||||
- Stage names affect diagnostics filenames and observability grouping.
|
||||
- Current behavior works, but the naming grammar is implicit.
|
||||
|
||||
Recommended refactor:
|
||||
|
||||
- Add narrow helpers for proposal and validator stage names.
|
||||
- Preserve current production stage names unless there is a deliberate pre-1.0 diagnostics compatibility decision.
|
||||
- Keep filename sanitization in `internal/framework/llm`.
|
||||
|
||||
Suggested tests:
|
||||
|
||||
- Unit tests for stage-name helper output.
|
||||
- Utilization test that module instance extraction still works for proposal and validator stage names.
|
||||
|
||||
Risk level:
|
||||
|
||||
- Medium. Renaming stages can change diagnostics filenames, so avoid unnecessary churn.
|
||||
|
||||
## 5. Boundary and responsibility concerns
|
||||
|
||||
### CLI owns too much report and diagnostics metadata assembly
|
||||
|
||||
`internal/cli/run.go` is doing orchestration, command parsing, config loading, output routing, report assembly, diagnostics metadata path assembly, and correction-ledger construction. This is acceptable for a small CLI, but two pieces are drifting beyond command responsibility:
|
||||
|
||||
- diagnostics artifact path metadata belongs closer to `internal/core/diagnostics`;
|
||||
- report assembly and correction-ledger mapping belong closer to `internal/core/reporting` or a narrow reporting adapter package.
|
||||
|
||||
Recommended home:
|
||||
|
||||
- `internal/core/diagnostics`: artifact constants and diagnostics metadata path construction.
|
||||
- `internal/core/reporting`: pure mapping from runner/config/diagnostics state into report payloads.
|
||||
- `internal/cli`: command parsing, invocation wiring, exit codes, stdout/stderr behavior.
|
||||
|
||||
### Config validation lacks catalog ownership
|
||||
|
||||
`internal/core/config` currently validates only generic module list shape and hardcodes output schema keys. Because modules and output schemas are public contract values, config validation should use a catalog owned by the relevant domain.
|
||||
|
||||
Recommended home:
|
||||
|
||||
- output schema validation: `internal/core/outputschema`;
|
||||
- module key validation: a small catalog package or lower-level constants package importable by config, module factory, validator chains, and threshold lookup.
|
||||
|
||||
### Runner owns adapter shims between contracts and validator framework
|
||||
|
||||
`internal/framework/runner` contains `validationLLMClientAdapter` and `llmDiagnosticsWriterAdapter`. This is not a serious problem today because runner wires proposal and validation workflows. If these adapters grow, move them to `internal/framework/validators` or a small integration package so runner remains focused on orchestration.
|
||||
|
||||
### LLM malformed-output policy is spread across callers
|
||||
|
||||
The LLM adapter emits the errors, while proposal generation and validators classify them by message text. The policy decision is caller-specific, but the classification should live with the LLM/framework error type.
|
||||
|
||||
## 6. Path, key, and naming construction review
|
||||
|
||||
Centralized enough:
|
||||
|
||||
- LLM diagnostics artifact suffixes and stage sanitization are centralized in `internal/framework/llm/diagnostics.go`.
|
||||
- Output file writing is routed through `internal/core/io.WriteFile`.
|
||||
- Run directories are created in `internal/core/diagnostics.NewRunDirectory`.
|
||||
|
||||
Needs cleanup:
|
||||
|
||||
- Core diagnostics artifact names are repeated between `RunDirectory` writer methods and `buildProcessReport`.
|
||||
- `utilization-diagnostics.json` and `correction-ledger.json` are raw strings in both success and failure paths.
|
||||
- Proposal and validator diagnostics subdirectory construction repeats `filepath.Join(diagnosticsDir, moduleInstance)`.
|
||||
- Proposal and validator stage names are manually formatted in multiple packages.
|
||||
- Module keys are repeated across config defaults, module factory, validator chains, confidence threshold lookup, and module implementations.
|
||||
- Output schema names are repeated between config validation and `outputschema`.
|
||||
|
||||
Recommendation:
|
||||
|
||||
- Start with artifact constants and metadata helpers because that is the lowest-risk path/key cleanup.
|
||||
- Then centralize stage-name helpers without changing current production naming.
|
||||
- Defer any broader "path manager" abstraction.
|
||||
|
||||
## 7. Resolution and catalog review
|
||||
|
||||
Modules:
|
||||
|
||||
- Runtime module construction has a production registry in `internal/framework/modules`.
|
||||
- Instance naming for repeated modules is centralized in `contracts.ResolveModuleRunSpecs`.
|
||||
- Unknown module failure exists in the factory, but config validation does not catch unknown modules.
|
||||
- Built-in validator chain resolution separately maps module key to validator keys.
|
||||
|
||||
Output schemas:
|
||||
|
||||
- Encoding is centralized in `internal/core/outputschema`.
|
||||
- Validation is duplicated in config.
|
||||
|
||||
Prompts:
|
||||
|
||||
- Prompt asset lookup and metadata are centralized in `internal/prompts`.
|
||||
- Prompt metadata map construction is repeated at call sites.
|
||||
- Prompt source selection is intentionally built-in only and should remain that way for 1.0.
|
||||
|
||||
Validators:
|
||||
|
||||
- Validator construction is package-owned under `internal/validators`.
|
||||
- Chains are centralized in `internal/validators/chains.go`.
|
||||
- Execution class metadata exists, but reporting/correction-ledger classification does not fully use it.
|
||||
|
||||
Schemas:
|
||||
|
||||
- Transcript and glossary parsing/validation are centralized in `internal/core/schema`.
|
||||
- Structured LLM response schemas are centralized in `internal/framework/responseschema`.
|
||||
- Output schema registry and response schema registry are appropriately separate.
|
||||
|
||||
Recommendation:
|
||||
|
||||
- Introduce only small catalog helpers for module keys, output schema keys, prompt metadata maps, response schema metadata maps, and validator execution class.
|
||||
- Avoid user-configurable modules, validators, prompts, or schemas before 1.0 unless already planned elsewhere.
|
||||
|
||||
## 8. Config and command-loading review
|
||||
|
||||
Consistent behavior:
|
||||
|
||||
- The documented precedence for `process` is implemented: defaults, file config, environment, CLI.
|
||||
- `config print-effective` intentionally omits CLI process flags and uses defaults, file config, and environment.
|
||||
- `config validate` intentionally requires `--config` and does not require transcript/glossary inputs.
|
||||
- Missing explicit config paths are hard failures; missing default paths are non-fatal.
|
||||
- Environment parsing and CLI parsing both preserve legacy total-concurrency alias behavior.
|
||||
|
||||
Likely accidental or high-risk differences:
|
||||
|
||||
- Unsupported module names pass `Config.Validate` and `audita config validate`.
|
||||
- Output schema support is duplicated instead of delegated to the output schema registry.
|
||||
- Config path resolution lives in CLI even though it is part of config behavior.
|
||||
|
||||
Intentional differences:
|
||||
|
||||
- File config resolves `api_key_env`; env and CLI set direct API key values.
|
||||
- `OPENROUTER_API_KEY` is an environment fallback only for the primary LLM.
|
||||
- `transcript-description` has CLI/config support but no `AUDITA_*` environment variable, matching documentation.
|
||||
|
||||
Recommendation:
|
||||
|
||||
- Build a shared effective config context helper and keep source-specific parsing semantics explicit.
|
||||
- Tighten catalog validation before 1.0 if unknown modules are not meant to be accepted.
|
||||
|
||||
## 9. State, manifest, or progress handling review
|
||||
|
||||
Audita does not currently have a manifest/checkpoint/resume model. State is per-run diagnostics and report artifacts.
|
||||
|
||||
Consistent behavior:
|
||||
|
||||
- `process` creates one diagnostics run directory when diagnostics initialization succeeds.
|
||||
- Failures after run-dir creation write `error.log`, best-effort report artifacts, and retain diagnostics.
|
||||
- Success writes optional `--report-json`, run-dir `report.json`, utilization diagnostics, and correction ledger.
|
||||
- Retention is centralized in `diagnostics.ShouldRetainRunDirectory`.
|
||||
- There is no resume/retry/force behavior to preserve.
|
||||
|
||||
Drift risks:
|
||||
|
||||
- Success and failure paths both write utilization and correction-ledger artifacts with duplicated raw filenames.
|
||||
- Report diagnostics metadata is assembled independently from the run-directory writer methods.
|
||||
- Retention mode `never` currently still retains successful run directories in `ShouldRetainRunDirectory`, which may be intentional per tests or a naming/documentation mismatch. Do not change it in a dedup pass without first confirming semantics.
|
||||
|
||||
Recommendation:
|
||||
|
||||
- Centralize artifact names and report metadata path construction.
|
||||
- Keep retention behavior unchanged unless a separate bug review confirms the intended meaning of `never`.
|
||||
|
||||
## 10. Refactors to avoid before 1.0
|
||||
|
||||
- Do not introduce a generic workflow engine. The current sequential runner is clear and explicit.
|
||||
- Do not add a plugin architecture for modules, validators, prompts, or schemas before 1.0.
|
||||
- Do not redesign the CLI or replace `flag` with a larger framework only for deduplication.
|
||||
- Do not collapse all config source parsing into a reflection-based mapper; source semantics differ intentionally.
|
||||
- Do not merge module packages into one generic module type. Keep domain-specific prompt assets, keys, validator chains, and replacement policies visible.
|
||||
- Do not rewrite diagnostics or reporting schemas broadly. Centralize names and mapping helpers first.
|
||||
- Do not change diagnostics stage names casually; they affect artifact filenames and debugging workflows.
|
||||
- Do not consolidate deterministic and LLM validator behavior just because both return decisions. Their failure and batching semantics differ.
|
||||
- Do not generalize transcript/glossary schema parsing into a broad schema framework.
|
||||
- Do not reduce duplicated tests where the duplication protects distinct public command/module behavior.
|
||||
|
||||
## 11. Recommended implementation sequence
|
||||
|
||||
1. Centralize diagnostics artifact constants and diagnostics metadata path construction.
|
||||
2. Centralize output schema validation through `internal/core/outputschema`.
|
||||
3. Introduce a small module key catalog and use it in config validation, module factory, validator chains, and threshold lookup.
|
||||
4. Add an effective config loading context helper for defaults + file + env, then update `process` and `config print-effective`.
|
||||
5. Extract shared module proposal plumbing and prompt transcript-section payload construction.
|
||||
6. Centralize prompt metadata and response schema metadata map construction.
|
||||
7. Centralize validator execution-class lookup and update correction-ledger classification.
|
||||
8. Centralize malformed structured-output classification through a typed/shared LLM error helper.
|
||||
9. Add or consolidate focused test helpers for module LLM fakes, diagnostics assertions, and fixture paths.
|
||||
10. Do a final dead-code and legacy sweep for redundant helper fields such as unused validator definition metadata.
|
||||
|
||||
Each item can be a separate commit with package-level tests and at least one CLI regression where public behavior is involved.
|
||||
|
||||
## 12. Test strategy
|
||||
|
||||
Tests to add before refactoring:
|
||||
|
||||
- `internal/core/config`: unknown module key fails validation, if unsupported modules are not intended to be accepted.
|
||||
- `internal/core/config`: every output schema registry key validates through config.
|
||||
- `internal/core/diagnostics`: report metadata paths match run-directory artifact names.
|
||||
- `internal/validators`: validator class by key/instance is consistent for all registered validators.
|
||||
- `internal/framework/llm`: shared malformed structured-output classifier covers all current adapter malformed errors.
|
||||
|
||||
Tests to add during refactoring:
|
||||
|
||||
- `internal/framework/promptcontext`: transcript section prompt payload preserves IDs, speaker, timestamps, text, and categories.
|
||||
- `internal/framework/proposal_generation`: shared module proposal helper preserves current stage name, diagnostics dir, schema metadata, and malformed-output warning behavior.
|
||||
- `internal/cli`: `process` and `config print-effective` share defaults+file+env behavior.
|
||||
- `internal/cli`: `config validate` remains file-only and does not read env overrides.
|
||||
- `internal/cli`: correction ledger classifies deterministic and LLM validator decisions through canonical metadata.
|
||||
|
||||
Existing tests to run after each cleanup:
|
||||
|
||||
- `go test ./internal/core/config ./internal/core/outputschema`
|
||||
- `go test ./internal/core/diagnostics ./internal/core/reporting`
|
||||
- `go test ./internal/framework/proposal_generation ./internal/framework/validators ./internal/framework/runner`
|
||||
- `go test ./internal/validators/...`
|
||||
- `go test ./internal/modules/...`
|
||||
- `go test ./internal/cli ./cmd/audita`
|
||||
- Run `go test ./...` before merging a multi-package cleanup.
|
||||
|
||||
Validation note:
|
||||
|
||||
- During this report-only pass, no full test suite was run. A lightweight `go list ./...` completed package listing but emitted a sandbox warning while trying to write the Go module stat cache outside the repository.
|
||||
|
||||
## 13. Appendix: findings not worth acting on
|
||||
|
||||
### Separate module packages
|
||||
|
||||
The four production module packages contain visible repetition, but keeping separate packages is useful. The module domains, prompt assets, validator chains, and tests are distinct enough that a single generic module package would hide important behavior.
|
||||
|
||||
Do not refactor now beyond shared proposal/prompt plumbing.
|
||||
|
||||
### Report type duplication between runner and reporting
|
||||
|
||||
`runner.ModuleResult` and `reporting.ModuleReport` look similar. Keeping separate runtime and public report shapes is reasonable because runner owns execution state and reporting owns serialized public schema.
|
||||
|
||||
Only centralize mapping helpers; do not merge the types.
|
||||
|
||||
### Transcript and glossary parsing stay separate
|
||||
|
||||
Transcript JSON and glossary YAML parsing have different formats, validation rules, and error messages. There is no useful shared parser abstraction to extract.
|
||||
|
||||
### Response schema registry and output schema registry stay separate
|
||||
|
||||
Structured LLM response schemas and transcript output schemas are both "schemas", but they serve different users and have different lifecycles. Do not combine their registries.
|
||||
|
||||
### `flag` package usage
|
||||
|
||||
The CLI command surface is small. Replacing `flag` with a larger CLI framework would not pay for itself before 1.0.
|
||||
|
||||
### Local test duplication that protects public behavior
|
||||
|
||||
Some test duplication in CLI, subprocess, parity, and release fixtures is intentional. These tests exercise different public surfaces and should remain explicit even if helpers are shared.
|
||||
|
||||
### Filesystem state as diagnostics state
|
||||
|
||||
Audita has no resume/checkpoint semantics. Treating diagnostics artifacts as filesystem outputs is currently acceptable. A manifest system would be speculative before there is a resume or audit workflow that needs it.
|
||||
329
docs/roadmap/implementation.md
Normal file
329
docs/roadmap/implementation.md
Normal file
@@ -0,0 +1,329 @@
|
||||
# Pre-1.0 Deduplication Implementation Plan
|
||||
|
||||
This plan turns `docs/roadmap/audit.md` into staged, prompt-sized cleanup work for an LLM coding agent. Each stage should be implemented in order and kept small enough to review as an independent commit.
|
||||
|
||||
## Operating rules
|
||||
|
||||
- Read `docs/roadmap/audit.md` before starting any stage.
|
||||
- Preserve public CLI, report, diagnostics, config precedence, prompt metadata, and output-schema behavior unless a stage explicitly calls out an intended behavior change.
|
||||
- Keep the four production module packages separate: `glossary`, `homophones`, `spoken_word`, and `grammar`.
|
||||
- Do not introduce plugin systems, generic workflow engines, broad CLI framework rewrites, reflection-heavy config mappers, or merged module packages.
|
||||
- Prefer narrow helpers, catalogs, constants, and pure mapping functions over broad abstractions.
|
||||
- Run the targeted tests listed in each stage before moving to the next stage.
|
||||
- Run `go test ./...` before declaring the full sequence complete.
|
||||
- Ignore unrelated worktree changes, including the existing deletion of `docs/roadmap/publish.md`, unless the user explicitly asks to handle them.
|
||||
- Do not reduce parity, release-fixture, subprocess, or module-specific behavior coverage while consolidating helpers.
|
||||
|
||||
## Stages
|
||||
|
||||
### Stage 1: Diagnostics artifact constants and metadata paths
|
||||
|
||||
Goal:
|
||||
|
||||
- Centralize diagnostics artifact names and report diagnostics metadata path construction without changing any filenames or report fields.
|
||||
|
||||
Key edits:
|
||||
|
||||
- Define constants in `internal/core/diagnostics` for:
|
||||
- `source-transcript.json`
|
||||
- `source-transcript-parsed.json`
|
||||
- `normalized-transcript.json`
|
||||
- `normalization-summary.json`
|
||||
- `chunking-summary.json`
|
||||
- `utilization-diagnostics.json`
|
||||
- `correction-ledger.json`
|
||||
- `invocation.json`
|
||||
- `effective-config.json`
|
||||
- `report.json`
|
||||
- `error.log`
|
||||
- Add a diagnostics helper that builds `reporting.DiagnosticsMetadata` from a run directory path and failure/success status.
|
||||
- Update `RunDirectory` methods to use the constants.
|
||||
- Update CLI report assembly and utilization/correction-ledger writes to use the constants/helper instead of raw strings.
|
||||
|
||||
Behavior changes:
|
||||
|
||||
- None. All artifact names, report JSON keys, and path values must remain byte-for-byte compatible except for normal timestamp/order differences in existing outputs.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add or update `internal/core/diagnostics` tests proving metadata helper paths match the artifact constants.
|
||||
- Run `go test ./internal/core/diagnostics ./internal/core/reporting ./internal/cli`.
|
||||
- Run any existing CLI report/diagnostics tests touched by this stage.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- No raw core diagnostics artifact filename strings remain in CLI report metadata assembly.
|
||||
- Existing success and failure reports still point to files that are actually written.
|
||||
- Retention behavior is unchanged.
|
||||
|
||||
### Stage 2: Output schema validation and module catalog
|
||||
|
||||
Goal:
|
||||
|
||||
- Move public key validation to small canonical catalogs so config validation, runtime resolution, and factory behavior cannot drift.
|
||||
|
||||
Key edits:
|
||||
|
||||
- Add `SupportedKeys`, `IsSupported`, or an equivalent validation helper to `internal/core/outputschema`.
|
||||
- Update `config.Validate` to use `internal/core/outputschema` for output schema validation.
|
||||
- Add a small canonical module key catalog that is importable by:
|
||||
- `internal/core/config`
|
||||
- `internal/framework/modules`
|
||||
- `internal/validators`
|
||||
- `internal/framework/validators`
|
||||
- Use the module catalog for default module key constants, known-key checks, validator chain keys, and confidence-threshold lookup.
|
||||
- Keep module construction in `internal/framework/modules`; the catalog must not construct modules.
|
||||
|
||||
Behavior changes:
|
||||
|
||||
- Intended behavior change: unsupported configured module keys should fail during config validation, including `audita config validate`.
|
||||
- Repeated supported module keys remain valid.
|
||||
- Output schema behavior remains unchanged for `bare-segments`, `audita-v1`, and unsupported names.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add `internal/core/config` tests for unsupported module keys and repeated supported module keys.
|
||||
- Add config validation tests that every supported output schema validates.
|
||||
- Add or update output schema registry tests for supported and unsupported schemas.
|
||||
- Update module registry and validator chain tests to use the shared catalog where appropriate.
|
||||
- Run `go test ./internal/core/config ./internal/core/outputschema ./internal/framework/modules ./internal/framework/validators ./internal/validators/... ./internal/cli`.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- Unknown modules fail before runner setup in config validation paths.
|
||||
- No duplicated hardcoded output schema support list remains in config validation.
|
||||
- No import cycle is introduced.
|
||||
|
||||
### Stage 3: Effective config loading context
|
||||
|
||||
Goal:
|
||||
|
||||
- Centralize config path resolution and defaults+file+env loading while keeping command-specific CLI overrides explicit.
|
||||
|
||||
Key edits:
|
||||
|
||||
- Move config path resolution from `internal/cli` into `internal/core/config` or add an equivalent exported helper there.
|
||||
- Add an effective config loader that returns:
|
||||
- effective `config.Config`
|
||||
- config path
|
||||
- config source (`flag`, `env`, `default`, or empty)
|
||||
- config version pointer when a file was loaded
|
||||
- Use the shared loader in `audita process` before applying CLI overrides.
|
||||
- Use the shared loader in `audita config print-effective`.
|
||||
- Keep `audita config validate` as file-only: load file, apply to defaults, validate, and do not apply environment overrides.
|
||||
|
||||
Behavior changes:
|
||||
|
||||
- None. Preserve existing precedence:
|
||||
- `process`: defaults, file config, environment, CLI flags
|
||||
- `config print-effective`: defaults, file config, environment
|
||||
- `config validate`: file config applied to defaults only
|
||||
- Preserve explicit config path failure behavior and missing default path non-fatal behavior.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add table-driven config loader tests for:
|
||||
- explicit `--config`
|
||||
- `AUDITA_CONFIG`
|
||||
- default search paths
|
||||
- missing explicit path
|
||||
- missing env path
|
||||
- missing default paths
|
||||
- Add or update CLI tests proving `process` and `config print-effective` share file+env behavior.
|
||||
- Add or update CLI tests proving `config validate` ignores environment overrides.
|
||||
- Run `go test ./internal/core/config ./internal/cli ./cmd/audita`.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- Config precedence is unchanged.
|
||||
- Config source/path/version metadata in invocation and reports is unchanged.
|
||||
- Config command stdout/stderr and exit-code behavior is unchanged except for the intended unknown-module validation from Stage 2.
|
||||
|
||||
### Stage 4: Prompt/schema metadata and stage-name helpers
|
||||
|
||||
Goal:
|
||||
|
||||
- Centralize diagnostics-visible metadata and stage-name construction without changing production diagnostics names.
|
||||
|
||||
Key edits:
|
||||
|
||||
- Add a helper or method in `internal/prompts` that returns the stable prompt metadata diagnostics shape currently expanded by call sites.
|
||||
- Add a helper or method in `internal/framework/responseschema` that returns the stable response schema metadata diagnostics shape currently expanded by call sites.
|
||||
- Add shared proposal and validator stage-name helpers in the lowest package that avoids import cycles.
|
||||
- Use the helpers in proposal generation, LLM validators, and production modules.
|
||||
|
||||
Behavior changes:
|
||||
|
||||
- None. Preserve current production stage names:
|
||||
- module proposal stages keep their existing `proposal` naming form;
|
||||
- validator batch stages keep their existing validator/batch naming form.
|
||||
- Preserve all prompt metadata and response schema metadata field names and values.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add prompt metadata helper tests covering every registered prompt.
|
||||
- Add response schema metadata helper tests covering every registered response schema.
|
||||
- Add stage-name helper tests for no-section, section, and validator batch cases.
|
||||
- Run `go test ./internal/prompts ./internal/framework/responseschema ./internal/framework/proposal_generation ./internal/framework/validators ./internal/modules/...`.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- No manual prompt metadata map expansion remains in production module proposal plumbing.
|
||||
- No duplicated response schema metadata map construction remains in proposal generation and LLM validators.
|
||||
- Existing diagnostics fixture/path assertions still pass.
|
||||
|
||||
### Stage 5: Shared module proposal and prompt payload plumbing
|
||||
|
||||
Goal:
|
||||
|
||||
- Remove duplicated proposal execution and transcript-section prompt payload construction while preserving module-specific domain behavior.
|
||||
|
||||
Key edits:
|
||||
|
||||
- Add a narrow shared proposal execution helper, preferably in `internal/framework/proposal_generation` unless import cycles require a small module helper package.
|
||||
- The helper should own:
|
||||
- transcript description extraction from config;
|
||||
- `GenerateCandidates` request construction;
|
||||
- prompt metadata attachment;
|
||||
- stage-name selection;
|
||||
- conversion from generated corrections/warnings to `contracts.ProposalResult`.
|
||||
- Add shared transcript-section prompt payload construction in `internal/framework/promptcontext`.
|
||||
- Update each production module to provide only:
|
||||
- module key;
|
||||
- replacement policy;
|
||||
- validator chain;
|
||||
- prompt ID;
|
||||
- domain-specific `BuildProposalMessages` call or message builder.
|
||||
- Remove each module's redundant section transcript filtering if the runner already passes section-limited transcripts.
|
||||
|
||||
Behavior changes:
|
||||
|
||||
- None. Preserve module keys, replacement policies, validator chains, prompt IDs, diagnostics directories, proposal indexes, warning behavior, and correction mapping.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add promptcontext tests for transcript section payload shape, empty transcript handling, section index, and category copying.
|
||||
- Keep one module-specific prompt test per production module for domain wording and constraints.
|
||||
- Add or update module proposal tests proving diagnostics are still written under the same module instance directory.
|
||||
- Run `go test ./internal/framework/promptcontext ./internal/framework/proposal_generation ./internal/modules/... ./internal/cli`.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- Four production modules share proposal execution plumbing.
|
||||
- Module packages remain separate and readable.
|
||||
- CLI parity and release fixture behavior is unchanged.
|
||||
|
||||
### Stage 6: Validator classification and malformed LLM output policy
|
||||
|
||||
Goal:
|
||||
|
||||
- Use one source of truth for validator execution class and one shared classifier for malformed structured-output errors.
|
||||
|
||||
Key edits:
|
||||
|
||||
- Make validator execution class resolvable by stable validator key and by validator instance.
|
||||
- Replace the correction-ledger hardcoded LLM-backed validator map with the canonical metadata source.
|
||||
- Remove redundant validator metadata fields only after all call sites use the canonical source.
|
||||
- Add a shared malformed structured-output classifier in `internal/framework/llm` or another low-level framework package.
|
||||
- Update proposal generation and LLM validators to use the shared classifier while preserving their different handling outcomes.
|
||||
|
||||
Behavior changes:
|
||||
|
||||
- None. Proposal-generation malformed payloads still downgrade to warnings with zero proposals for affected sections.
|
||||
- Validator malformed payloads still reject affected batches with warnings.
|
||||
- Correction-ledger deterministic vs LLM validator sections should be unchanged for current validators.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add validator metadata tests proving every registered validator has the expected execution class by key and instance.
|
||||
- Add correction-ledger tests proving deterministic and LLM-backed decisions are classified through canonical metadata.
|
||||
- Add shared malformed-output classifier tests covering current adapter malformed-output messages.
|
||||
- Update proposal-generation and validator tests to assert representative malformed adapter errors are still downgraded.
|
||||
- Run `go test ./internal/validators/... ./internal/framework/validators ./internal/framework/proposal_generation ./internal/framework/llm ./internal/cli`.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- No local hardcoded LLM-backed validator map remains in correction-ledger construction.
|
||||
- Proposal-generation and validator malformed-output classifier lists cannot drift.
|
||||
- Existing runner validator ordering is unchanged.
|
||||
|
||||
### Stage 7: Redaction and adapter workflow cleanup
|
||||
|
||||
Goal:
|
||||
|
||||
- Reduce duplicated secret extraction/redaction setup while preserving all no-secret-leak guarantees.
|
||||
|
||||
Key edits:
|
||||
|
||||
- Add a shared helper that extracts all configured LLM secret values from `config.Config`.
|
||||
- Use the helper in proposal-generation diagnostics and validator diagnostics setup.
|
||||
- Keep config structural redaction (`Config.Redacted`) separate from byte/string payload redaction.
|
||||
- Keep adapter error redaction behavior compatible with current surfaced errors.
|
||||
- Move runner adapter shims only if Stage 6 or this stage makes them materially larger; otherwise leave them in runner.
|
||||
|
||||
Behavior changes:
|
||||
|
||||
- None. Redaction token and no-secret-leak behavior remain unchanged.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add or update tests proving proposal diagnostics, validator diagnostics, effective config artifacts, and surfaced adapter errors redact the same configured secrets.
|
||||
- Keep existing subprocess no-secret-leak tests.
|
||||
- Run `go test ./internal/core/config ./internal/framework/llm ./internal/framework/proposal_generation ./internal/framework/validators ./internal/cli ./cmd/audita`.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- Secret-list assembly is no longer duplicated between proposal and validator paths.
|
||||
- No plaintext configured API key appears in diagnostics, reports, stdout, or stderr in existing redaction tests.
|
||||
- No unrelated adapter behavior changes.
|
||||
|
||||
### Stage 8: Test helper cleanup and dead-code sweep
|
||||
|
||||
Goal:
|
||||
|
||||
- Consolidate test-only duplication and remove dead/redundant code left by prior stages.
|
||||
|
||||
Key edits:
|
||||
|
||||
- Consolidate package-local fake LLM clients, fixture readers, diagnostics glob helpers, and run-directory helpers where duplication is clear.
|
||||
- Use cross-package test support only if it does not obscure test intent or introduce awkward imports.
|
||||
- Remove redundant metadata fields, constants, or helper functions made obsolete by earlier stages.
|
||||
- Keep module-specific prompt and behavior assertions local to each module package.
|
||||
|
||||
Behavior changes:
|
||||
|
||||
- None.
|
||||
|
||||
Tests:
|
||||
|
||||
- Run all package tests touched by helper cleanup.
|
||||
- Run `go test ./internal/modules/... ./internal/framework/... ./internal/cli ./cmd/audita`.
|
||||
- Run `go test ./...` before completing the full sequence.
|
||||
|
||||
Acceptance criteria:
|
||||
|
||||
- Test helpers are simpler without reducing coverage.
|
||||
- No parity or release fixture assertions are removed unless replaced by equivalent or stronger assertions.
|
||||
- No production behavior changes.
|
||||
|
||||
## Final verification
|
||||
|
||||
Before declaring the staged cleanup complete:
|
||||
|
||||
- Run:
|
||||
- `go test ./internal/core/config ./internal/core/outputschema`
|
||||
- `go test ./internal/core/diagnostics ./internal/core/reporting`
|
||||
- `go test ./internal/framework/proposal_generation ./internal/framework/validators ./internal/framework/runner`
|
||||
- `go test ./internal/validators/...`
|
||||
- `go test ./internal/modules/...`
|
||||
- `go test ./internal/cli ./cmd/audita`
|
||||
- `go test ./...`
|
||||
- Inspect `git diff` for accidental public CLI, config, report, diagnostics, prompt metadata, stage-name, or output-schema changes.
|
||||
- Update docs only when behavior intentionally changes, especially the intended Stage 2 unknown-module validation change.
|
||||
- Keep commits stage-sized and mention behavior-preservation tests in each commit message or PR description.
|
||||
|
||||
## Assumptions
|
||||
|
||||
- Unknown configured module keys should become config-validation failures before 1.0.
|
||||
- Diagnostics filenames and stage names are public enough to preserve unless a stage explicitly says otherwise.
|
||||
- Each stage should be implemented and reviewed separately.
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
)
|
||||
|
||||
type noOpStructuredLLMClient struct{}
|
||||
@@ -803,6 +804,7 @@ func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummar
|
||||
ReplacementPolicy: string(r.ReplacementPolicy),
|
||||
Status: r.Status,
|
||||
ProposalCount: r.ProposalCount,
|
||||
Warnings: append([]stagewarnings.StageWarning(nil), r.Warnings...),
|
||||
ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions),
|
||||
ValidatorRejected: mapValidatorRejected(r.ValidatorRejected),
|
||||
AppliedChanges: r.AppliedChanges,
|
||||
|
||||
@@ -1793,11 +1793,12 @@ type fakeModule struct {
|
||||
func (m fakeModule) Key() string { return m.key }
|
||||
func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
|
||||
func (m fakeModule) Validators() []contracts.Validator { return m.validators }
|
||||
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
if m.proposeF == nil {
|
||||
return nil, nil
|
||||
return contracts.ProposalResult{}, nil
|
||||
}
|
||||
return m.proposeF(req)
|
||||
proposalsOut, err := m.proposeF(req)
|
||||
return contracts.ProposalResult{Proposals: proposalsOut}, err
|
||||
}
|
||||
|
||||
type fakeValidator struct {
|
||||
@@ -2384,7 +2385,7 @@ func TestRunProcessExplicitGrammarRejectedAndApplicationSkipAreDistinct(t *testi
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProcessExplicitGrammarMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
|
||||
func TestRunProcessExplicitGrammarMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
|
||||
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
|
||||
t.Cleanup(func() { processProposalLLMClient = nil })
|
||||
|
||||
@@ -2403,22 +2404,32 @@ func TestRunProcessExplicitGrammarMalformedLLMOutputFailsWithErrorLog(t *testing
|
||||
"--work-dir-retention", "always",
|
||||
"--report-json", reportPath,
|
||||
}, &stdout, &stderr)
|
||||
if exitCode == 0 {
|
||||
t.Fatal("expected failure")
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||
}
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("expected empty stderr on success, got %q", stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "runner_execution") {
|
||||
t.Fatalf("expected runner_execution error, got %q", stderr.String())
|
||||
parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("expected transcript stdout on success: %v", err)
|
||||
}
|
||||
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
|
||||
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log on failed grammar run: %v", err)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
|
||||
t.Fatalf("did not expect error.log on successful grammar run: %v", err)
|
||||
}
|
||||
report := readProcessReport(t, reportPath)
|
||||
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
|
||||
t.Fatalf("expected failed runner_execution report, got %+v", report)
|
||||
if report.Status != "success" || report.ErrorPhase != "" {
|
||||
t.Fatalf("expected successful report, got %+v", report)
|
||||
}
|
||||
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
|
||||
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
|
||||
}
|
||||
if report.ModuleResults[0].Warnings[0].ReasonCode != "proposal_response_malformed" {
|
||||
t.Fatalf("unexpected warning: %+v", report.ModuleResults[0].Warnings[0])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3041,7 +3052,7 @@ func TestRunProcessExplicitGlossaryRepeatedStagesUseDeterministicInstanceNamesAn
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProcessExplicitGlossaryMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
|
||||
func TestRunProcessExplicitGlossaryMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
|
||||
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
|
||||
t.Cleanup(func() { processProposalLLMClient = nil })
|
||||
|
||||
@@ -3060,22 +3071,29 @@ func TestRunProcessExplicitGlossaryMalformedLLMOutputFailsWithErrorLog(t *testin
|
||||
"--work-dir-retention", "always",
|
||||
"--report-json", reportPath,
|
||||
}, &stdout, &stderr)
|
||||
if exitCode == 0 {
|
||||
t.Fatal("expected failure")
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||
}
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("expected empty stderr on success, got %q", stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "runner_execution") {
|
||||
t.Fatalf("expected runner_execution error, got %q", stderr.String())
|
||||
parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("expected transcript stdout on success: %v", err)
|
||||
}
|
||||
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
|
||||
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log on failed glossary run: %v", err)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
|
||||
t.Fatalf("did not expect error.log on successful glossary run: %v", err)
|
||||
}
|
||||
report := readProcessReport(t, reportPath)
|
||||
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
|
||||
t.Fatalf("expected failed runner_execution report, got %+v", report)
|
||||
if report.Status != "success" || report.ErrorPhase != "" {
|
||||
t.Fatalf("expected successful report, got %+v", report)
|
||||
}
|
||||
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
|
||||
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3295,7 +3313,7 @@ func TestRunProcessExplicitHomophonesProtectedGlossaryTermRejected(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProcessExplicitHomophonesMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
|
||||
func TestRunProcessExplicitHomophonesMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
|
||||
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
|
||||
t.Cleanup(func() { processProposalLLMClient = nil })
|
||||
|
||||
@@ -3314,22 +3332,29 @@ func TestRunProcessExplicitHomophonesMalformedLLMOutputFailsWithErrorLog(t *test
|
||||
"--work-dir-retention", "always",
|
||||
"--report-json", reportPath,
|
||||
}, &stdout, &stderr)
|
||||
if exitCode == 0 {
|
||||
t.Fatal("expected failure")
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||
}
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("expected empty stderr on success, got %q", stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "runner_execution") {
|
||||
t.Fatalf("expected runner_execution error, got %q", stderr.String())
|
||||
parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("expected transcript stdout on success: %v", err)
|
||||
}
|
||||
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
|
||||
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log on failed homophones run: %v", err)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
|
||||
t.Fatalf("did not expect error.log on successful homophones run: %v", err)
|
||||
}
|
||||
report := readProcessReport(t, reportPath)
|
||||
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
|
||||
t.Fatalf("expected failed runner_execution report, got %+v", report)
|
||||
if report.Status != "success" || report.ErrorPhase != "" {
|
||||
t.Fatalf("expected successful report, got %+v", report)
|
||||
}
|
||||
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
|
||||
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3674,7 +3699,7 @@ func TestRunProcessExplicitSpokenWordProtectedGlossaryTermRejected(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProcessExplicitSpokenWordMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
|
||||
func TestRunProcessExplicitSpokenWordMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
|
||||
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
|
||||
t.Cleanup(func() { processProposalLLMClient = nil })
|
||||
|
||||
@@ -3693,22 +3718,29 @@ func TestRunProcessExplicitSpokenWordMalformedLLMOutputFailsWithErrorLog(t *test
|
||||
"--work-dir-retention", "always",
|
||||
"--report-json", reportPath,
|
||||
}, &stdout, &stderr)
|
||||
if exitCode == 0 {
|
||||
t.Fatal("expected failure")
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||
}
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("expected empty stderr on success, got %q", stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "runner_execution") {
|
||||
t.Fatalf("expected runner_execution error, got %q", stderr.String())
|
||||
parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("expected transcript stdout on success: %v", err)
|
||||
}
|
||||
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
|
||||
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log on failed spoken_word run: %v", err)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
|
||||
t.Fatalf("did not expect error.log on successful spoken_word run: %v", err)
|
||||
}
|
||||
report := readProcessReport(t, reportPath)
|
||||
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
|
||||
t.Fatalf("expected failed runner_execution report, got %+v", report)
|
||||
if report.Status != "success" || report.ErrorPhase != "" {
|
||||
t.Fatalf("expected successful report, got %+v", report)
|
||||
}
|
||||
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
|
||||
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -78,25 +78,22 @@ func (c *subprocessTestLLMClient) CompleteStructured(ctx context.Context, req co
|
||||
}
|
||||
}
|
||||
case "mid_pipeline_fail":
|
||||
switch target := out.(type) {
|
||||
case *proposal_generation.StructuredCorrectionSet:
|
||||
if _, ok := out.(*proposal_generation.StructuredCorrectionSet); ok {
|
||||
c.mu.Lock()
|
||||
c.proposals++
|
||||
proposalCall := c.proposals
|
||||
c.mu.Unlock()
|
||||
|
||||
if proposalCall >= 3 {
|
||||
*target = proposal_generation.StructuredCorrectionSet{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y", Confidence: 0.99},
|
||||
},
|
||||
}
|
||||
} else {
|
||||
*target = proposal_generation.StructuredCorrectionSet{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99},
|
||||
},
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("synthetic mid-pipeline failure")
|
||||
}
|
||||
}
|
||||
switch target := out.(type) {
|
||||
case *proposal_generation.StructuredCorrectionSet:
|
||||
*target = proposal_generation.StructuredCorrectionSet{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99},
|
||||
},
|
||||
}
|
||||
case *validators.LLMValidationResponse:
|
||||
*target = validators.LLMValidationResponse{Validations: nil}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
)
|
||||
|
||||
type ProcessReport struct {
|
||||
@@ -46,18 +47,19 @@ type ReportMetadata struct {
|
||||
}
|
||||
|
||||
type ModuleReport struct {
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ReplacementPolicy string `json:"replacement_policy,omitempty"`
|
||||
Status string `json:"status"`
|
||||
ProposalCount int `json:"proposal_count"`
|
||||
ValidatorDecisions []ValidatorDecisionReport `json:"validator_decisions,omitempty"`
|
||||
ValidatorRejected []ValidatorRejectedReport `json:"validator_rejected,omitempty"`
|
||||
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
|
||||
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ReplacementPolicy string `json:"replacement_policy,omitempty"`
|
||||
Status string `json:"status"`
|
||||
ProposalCount int `json:"proposal_count"`
|
||||
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
|
||||
ValidatorDecisions []ValidatorDecisionReport `json:"validator_decisions,omitempty"`
|
||||
ValidatorRejected []ValidatorRejectedReport `json:"validator_rejected,omitempty"`
|
||||
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
|
||||
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
}
|
||||
|
||||
type ValidatorDecisionReport struct {
|
||||
|
||||
@@ -47,7 +47,7 @@ func (h testChunkProposalHarness) collectEnrichedProposals(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, proposal := range base {
|
||||
for _, proposal := range base.Proposals {
|
||||
sectionIndex := section.Index
|
||||
out = append(out, proposals.EnrichedCorrectionProposal{
|
||||
CorrectionProposal: proposal,
|
||||
@@ -85,11 +85,11 @@ func (m deterministicFakeModule) ReplacementPolicy() proposals.ReplacementPolicy
|
||||
|
||||
func (m deterministicFakeModule) Validators() []Validator { return nil }
|
||||
|
||||
func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) {
|
||||
_ = ctx
|
||||
|
||||
if req.WorkingTranscript == nil || req.Section == nil {
|
||||
return []proposals.CorrectionProposal{}, nil
|
||||
return ProposalResult{}, nil
|
||||
}
|
||||
|
||||
out := make([]proposals.CorrectionProposal, 0)
|
||||
@@ -110,7 +110,7 @@ func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalReques
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
return ProposalResult{Proposals: out}, nil
|
||||
}
|
||||
|
||||
func TestChunkProposalMetadataAssociation(t *testing.T) {
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
)
|
||||
|
||||
// StructuredLLMClient provides provider-agnostic structured completion.
|
||||
@@ -28,7 +29,7 @@ type TranscriptModule interface {
|
||||
Key() string
|
||||
ReplacementPolicy() proposals.ReplacementPolicy
|
||||
Validators() []Validator
|
||||
Propose(ctx context.Context, req ProposalRequest) ([]proposals.CorrectionProposal, error)
|
||||
Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error)
|
||||
}
|
||||
|
||||
// Validator evaluates candidate proposals and returns one decision per proposal index.
|
||||
@@ -103,6 +104,11 @@ type ProposalRequest struct {
|
||||
LLMScheduler LLMScheduler `json:"-"`
|
||||
}
|
||||
|
||||
type ProposalResult struct {
|
||||
Proposals []proposals.CorrectionProposal `json:"proposals,omitempty"`
|
||||
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
// ValidationRequest is the input to validator execution.
|
||||
type ValidationRequest = validators.Request
|
||||
|
||||
|
||||
@@ -50,11 +50,13 @@ func (f *fakeModule) Validators() []Validator {
|
||||
return []Validator{&fakeValidator{}}
|
||||
}
|
||||
|
||||
func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) {
|
||||
_ = ctx
|
||||
_ = req
|
||||
return []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "b", Confidence: 0.9},
|
||||
return ProposalResult{
|
||||
Proposals: []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "b", Confidence: 0.9},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -68,12 +70,12 @@ func TestInterfaceContractsCompileWithFakes(t *testing.T) {
|
||||
t.Fatalf("unexpected module key: %q", got)
|
||||
}
|
||||
|
||||
proposalsOut, err := module.Propose(context.Background(), ProposalRequest{})
|
||||
proposalResult, err := module.Propose(context.Background(), ProposalRequest{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected propose error: %v", err)
|
||||
}
|
||||
if len(proposalsOut) != 1 {
|
||||
t.Fatalf("expected one proposal, got %d", len(proposalsOut))
|
||||
if len(proposalResult.Proposals) != 1 {
|
||||
t.Fatalf("expected one proposal, got %d", len(proposalResult.Proposals))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -19,10 +19,10 @@ func (m noopModule) ReplacementPolicy() proposals.ReplacementPolicy {
|
||||
return proposals.ReplacementPolicyRequireUnique
|
||||
}
|
||||
func (m noopModule) Validators() []contracts.Validator { return nil }
|
||||
func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
_ = ctx
|
||||
_ = req
|
||||
return nil, nil
|
||||
return contracts.ProposalResult{}, nil
|
||||
}
|
||||
|
||||
func TestKnownModuleKeyRecognition(t *testing.T) {
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
)
|
||||
|
||||
// InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts.
|
||||
@@ -68,6 +69,7 @@ type Request struct {
|
||||
type Result struct {
|
||||
Corrections []proposals.CorrectionProposal `json:"corrections"`
|
||||
Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"`
|
||||
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
|
||||
Artifacts InteractionArtifacts `json:"artifacts,omitempty"`
|
||||
}
|
||||
|
||||
@@ -156,6 +158,12 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
}
|
||||
|
||||
if callErr != nil {
|
||||
if isMalformedStructuredOutputError(callErr) {
|
||||
return Result{
|
||||
Warnings: []stagewarnings.StageWarning{newMalformedProposalWarning(req.Section, artifacts, callErr)},
|
||||
Artifacts: artifacts,
|
||||
}, nil
|
||||
}
|
||||
return Result{}, fmt.Errorf("proposal generation completion failed: %w", callErr)
|
||||
}
|
||||
|
||||
@@ -168,9 +176,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
CorrectedText: raw.CorrectedText,
|
||||
Confidence: raw.Confidence,
|
||||
}
|
||||
if err := candidate.Validate(); err != nil {
|
||||
return Result{}, fmt.Errorf("invalid structured correction at index %d: %w", i, err)
|
||||
}
|
||||
|
||||
corrections = append(corrections, candidate)
|
||||
enrichedCandidate := proposals.EnrichedCorrectionProposal{
|
||||
@@ -191,6 +196,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
return Result{
|
||||
Corrections: corrections,
|
||||
Enriched: enriched,
|
||||
Warnings: nil,
|
||||
Artifacts: artifacts,
|
||||
}, nil
|
||||
}
|
||||
@@ -240,6 +246,48 @@ func errPayload(err error) any {
|
||||
return map[string]any{"error": err.Error()}
|
||||
}
|
||||
|
||||
func isMalformedStructuredOutputError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := err.Error()
|
||||
for _, marker := range []string{
|
||||
"malformed structured output",
|
||||
"decode structured output:",
|
||||
"decode provider response envelope:",
|
||||
"provider response missing choices",
|
||||
"provider response missing assistant message content",
|
||||
"provider response assistant message content is empty",
|
||||
"provider response assistant message content is not valid JSON",
|
||||
} {
|
||||
if strings.Contains(msg, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func newMalformedProposalWarning(section *contracts.SectionMetadata, artifacts InteractionArtifacts, err error) stagewarnings.StageWarning {
|
||||
warning := stagewarnings.StageWarning{
|
||||
Scope: stagewarnings.ScopeProposalGeneration,
|
||||
ReasonCode: "proposal_response_malformed",
|
||||
Message: strings.TrimSpace(err.Error()),
|
||||
DiagnosticArtifactPath: diagnosticArtifactPath(artifacts),
|
||||
}
|
||||
if section != nil {
|
||||
sectionIndex := section.Index
|
||||
warning.SectionIndex = §ionIndex
|
||||
}
|
||||
return warning
|
||||
}
|
||||
|
||||
func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
|
||||
if artifacts.ErrorPayloadPath != "" {
|
||||
return artifacts.ErrorPayloadPath
|
||||
}
|
||||
return artifacts.ResponsePayloadPath
|
||||
}
|
||||
|
||||
type diagnosticsWriterAdapter struct {
|
||||
writer *llm.DiagnosticsWriter
|
||||
}
|
||||
|
||||
@@ -224,12 +224,12 @@ func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) {
|
||||
func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{
|
||||
Corrections: []StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "x", CorrectedText: "", Confidence: 0.9},
|
||||
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -237,9 +237,38 @@ func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) {
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
|
||||
_, err := GenerateCandidates(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "invalid structured correction") {
|
||||
t.Fatalf("expected structured response validation failure, got %v", err)
|
||||
result, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected invalid correction to survive generation, got %v", err)
|
||||
}
|
||||
if len(result.Corrections) != 1 {
|
||||
t.Fatalf("expected one correction, got %+v", result)
|
||||
}
|
||||
if result.Corrections[0].TargetSegmentID != 0 || result.Corrections[0].Confidence != 1.2 {
|
||||
t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0])
|
||||
}
|
||||
if len(result.Warnings) != 0 {
|
||||
t.Fatalf("did not expect warnings for individually invalid corrections, got %+v", result.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesMalformedStructuredOutputReturnsWarning(t *testing.T) {
|
||||
client := &fakeStructuredClient{err: errors.New("malformed structured output")}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
|
||||
result, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected malformed structured output to downgrade to warning, got %v", err)
|
||||
}
|
||||
if len(result.Corrections) != 0 || len(result.Enriched) != 0 {
|
||||
t.Fatalf("expected no proposals on malformed response, got %+v", result)
|
||||
}
|
||||
if len(result.Warnings) != 1 {
|
||||
t.Fatalf("expected one warning, got %+v", result.Warnings)
|
||||
}
|
||||
if result.Warnings[0].ReasonCode != "proposal_response_malformed" {
|
||||
t.Fatalf("unexpected warning: %+v", result.Warnings[0])
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -147,6 +147,8 @@ func skipReasonMessage(reason ProposalSkipReason) string {
|
||||
return "original_text matched multiple spans under require_unique policy"
|
||||
case SkipReasonNoEffect:
|
||||
return "original_text and corrected_text are identical"
|
||||
case SkipReasonEmptyResultingText:
|
||||
return "proposal would leave the segment empty"
|
||||
case SkipReasonInvalidProposal:
|
||||
return "proposal failed structural or policy validation"
|
||||
default:
|
||||
|
||||
@@ -15,6 +15,7 @@ const (
|
||||
SkipReasonMissingOriginalText ProposalSkipReason = "missing_original_text"
|
||||
SkipReasonAmbiguousOriginal ProposalSkipReason = "ambiguous_original_text"
|
||||
SkipReasonNoEffect ProposalSkipReason = "no_effect"
|
||||
SkipReasonEmptyResultingText ProposalSkipReason = "empty_resulting_segment"
|
||||
SkipReasonInvalidProposal ProposalSkipReason = "invalid_proposal"
|
||||
)
|
||||
|
||||
@@ -62,6 +63,9 @@ func PreviewProposalForSegment(segment *schema.Segment, proposal CorrectionPropo
|
||||
}
|
||||
|
||||
corrected := strings.Replace(segment.Text, proposal.OriginalText, proposal.CorrectedText, 1)
|
||||
if strings.TrimSpace(corrected) == "" {
|
||||
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText}
|
||||
}
|
||||
return SegmentPreviewResult{
|
||||
Applicable: true,
|
||||
CorrectedSegmentText: corrected,
|
||||
@@ -70,6 +74,9 @@ func PreviewProposalForSegment(segment *schema.Segment, proposal CorrectionPropo
|
||||
|
||||
case ReplacementPolicyReplaceAll:
|
||||
corrected := strings.ReplaceAll(segment.Text, proposal.OriginalText, proposal.CorrectedText)
|
||||
if strings.TrimSpace(corrected) == "" {
|
||||
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText}
|
||||
}
|
||||
return SegmentPreviewResult{
|
||||
Applicable: true,
|
||||
CorrectedSegmentText: corrected,
|
||||
|
||||
@@ -121,6 +121,42 @@ func TestPreviewProposalForSegmentNoEffectReplacement(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewProposalForSegmentAllowsEmptyCorrectedTextWhenSegmentRemainsNonEmpty(t *testing.T) {
|
||||
segment := &schema.Segment{ID: 8, Text: "uh hello"}
|
||||
proposal := CorrectionProposal{
|
||||
TargetSegmentID: 8,
|
||||
OriginalText: "uh ",
|
||||
CorrectedText: "",
|
||||
Confidence: 0.9,
|
||||
}
|
||||
|
||||
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
|
||||
if !result.Applicable {
|
||||
t.Fatalf("expected applicable preview, got skip reason %q", result.SkipReason)
|
||||
}
|
||||
if result.CorrectedSegmentText != "hello" {
|
||||
t.Fatalf("unexpected corrected text: %q", result.CorrectedSegmentText)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewProposalForSegmentRejectsEmptyResultingSegment(t *testing.T) {
|
||||
segment := &schema.Segment{ID: 8, Text: "uh"}
|
||||
proposal := CorrectionProposal{
|
||||
TargetSegmentID: 8,
|
||||
OriginalText: "uh",
|
||||
CorrectedText: "",
|
||||
Confidence: 0.9,
|
||||
}
|
||||
|
||||
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
|
||||
if result.Applicable {
|
||||
t.Fatal("expected non-applicable preview")
|
||||
}
|
||||
if result.SkipReason != SkipReasonEmptyResultingText {
|
||||
t.Fatalf("expected skip reason %q, got %q", SkipReasonEmptyResultingText, result.SkipReason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewProposalForSegmentPreservesInputSegment(t *testing.T) {
|
||||
segment := &schema.Segment{ID: 9, Speaker: "A", Start: 1.0, End: 2.0, Text: "rank rank"}
|
||||
original := *segment
|
||||
|
||||
@@ -38,9 +38,6 @@ func (p CorrectionProposal) Validate() error {
|
||||
if strings.TrimSpace(p.OriginalText) == "" {
|
||||
return fmt.Errorf("proposal original_text must not be empty")
|
||||
}
|
||||
if strings.TrimSpace(p.CorrectedText) == "" {
|
||||
return fmt.Errorf("proposal corrected_text must not be empty")
|
||||
}
|
||||
if p.Confidence < 0.0 || p.Confidence > 1.0 {
|
||||
return fmt.Errorf("proposal confidence must be between 0.0 and 1.0")
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ func TestCorrectionProposalValidate_InvalidEmptyOriginalText(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorrectionProposalValidate_InvalidEmptyCorrectedText(t *testing.T) {
|
||||
func TestCorrectionProposalValidate_AllowsEmptyCorrectedText(t *testing.T) {
|
||||
proposal := CorrectionProposal{
|
||||
TargetSegmentID: 42,
|
||||
OriginalText: "gestures",
|
||||
@@ -43,12 +43,8 @@ func TestCorrectionProposalValidate_InvalidEmptyCorrectedText(t *testing.T) {
|
||||
Confidence: 0.95,
|
||||
}
|
||||
|
||||
err := proposal.Validate()
|
||||
if err == nil {
|
||||
t.Fatal("expected validation error, got nil")
|
||||
}
|
||||
if err.Error() != "proposal corrected_text must not be empty" {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
if err := proposal.Validate(); err != nil {
|
||||
t.Fatalf("expected empty corrected_text to be allowed, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
|
||||
)
|
||||
|
||||
@@ -37,18 +38,19 @@ type ValidationScheduler = contracts.LLMScheduler
|
||||
|
||||
// ModuleResult captures deterministic per-module execution output.
|
||||
type ModuleResult struct {
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
Status string `json:"status"`
|
||||
ProposalCount int `json:"proposal_count"`
|
||||
ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"`
|
||||
ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"`
|
||||
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
|
||||
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
Status string `json:"status"`
|
||||
ProposalCount int `json:"proposal_count"`
|
||||
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
|
||||
ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"`
|
||||
ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"`
|
||||
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
|
||||
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
}
|
||||
|
||||
type ValidatorDecisionRecord struct {
|
||||
@@ -176,6 +178,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
||||
ReplacementPolicy: policy,
|
||||
Status: ModuleStatusFailed,
|
||||
ProposalCount: pipelineResult.ProposalCount,
|
||||
Warnings: pipelineResult.Warnings,
|
||||
ValidatorDecisions: pipelineResult.ValidatorDecisions,
|
||||
ValidatorRejected: pipelineResult.ValidatorRejected,
|
||||
ErrorMessage: pipelineErr.Error(),
|
||||
@@ -199,6 +202,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
||||
ReplacementPolicy: policy,
|
||||
Status: ModuleStatusSuccess,
|
||||
ProposalCount: pipelineResult.ProposalCount,
|
||||
Warnings: pipelineResult.Warnings,
|
||||
ValidatorDecisions: pipelineResult.ValidatorDecisions,
|
||||
ValidatorRejected: pipelineResult.ValidatorRejected,
|
||||
AppliedChanges: applyResult.Applied,
|
||||
@@ -232,12 +236,14 @@ type collectSectionProposalsInput struct {
|
||||
type sectionProposals struct {
|
||||
meta contracts.SectionMetadata
|
||||
corrected []proposals.CorrectionProposal
|
||||
warnings []stagewarnings.StageWarning
|
||||
}
|
||||
|
||||
type sectionProposalResult struct {
|
||||
sectionPos int
|
||||
section chunking.Section
|
||||
corrected []proposals.CorrectionProposal
|
||||
warnings []stagewarnings.StageWarning
|
||||
err error
|
||||
}
|
||||
|
||||
@@ -247,12 +253,14 @@ type sectionValidationResult struct {
|
||||
approved []proposals.EnrichedCorrectionProposal
|
||||
decisions []ValidatorDecisionRecord
|
||||
rejected []ValidatorRejectedChange
|
||||
warnings []stagewarnings.StageWarning
|
||||
err error
|
||||
}
|
||||
|
||||
type modulePipelineResult struct {
|
||||
ProposalCount int
|
||||
Approved []proposals.EnrichedCorrectionProposal
|
||||
Warnings []stagewarnings.StageWarning
|
||||
ValidatorDecisions []ValidatorDecisionRecord
|
||||
ValidatorRejected []ValidatorRejectedChange
|
||||
}
|
||||
@@ -271,7 +279,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
meta := contracts.SectionMetadataFromSection(section)
|
||||
corrected, err := input.Module.Propose(withModuleInstanceContext(runCtx, input.Spec.InstanceName), contracts.ProposalRequest{
|
||||
proposalResult, err := input.Module.Propose(withModuleInstanceContext(runCtx, input.Spec.InstanceName), contracts.ProposalRequest{
|
||||
ExecutionContext: contracts.ExecutionContext{
|
||||
Config: input.Config,
|
||||
WorkingTranscript: transcriptFromSection(section),
|
||||
@@ -291,7 +299,8 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
|
||||
case results <- sectionProposalResult{
|
||||
sectionPos: sectionPos,
|
||||
section: section,
|
||||
corrected: corrected,
|
||||
corrected: proposalResult.Proposals,
|
||||
warnings: proposalResult.Warnings,
|
||||
err: err,
|
||||
}:
|
||||
case <-runCtx.Done():
|
||||
@@ -308,6 +317,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
|
||||
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
|
||||
out := modulePipelineResult{
|
||||
Approved: make([]proposals.EnrichedCorrectionProposal, 0),
|
||||
Warnings: make([]stagewarnings.StageWarning, 0),
|
||||
ValidatorDecisions: make([]ValidatorDecisionRecord, 0),
|
||||
ValidatorRejected: make([]ValidatorRejectedChange, 0),
|
||||
}
|
||||
@@ -352,6 +362,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
if firstErr != nil {
|
||||
continue
|
||||
}
|
||||
out.Warnings = append(out.Warnings, result.warnings...)
|
||||
pending[result.sectionPos] = result
|
||||
|
||||
for {
|
||||
@@ -401,6 +412,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
approved: validated.approved,
|
||||
decisions: validated.decisions,
|
||||
rejected: validated.rejected,
|
||||
warnings: validated.warnings,
|
||||
err: err,
|
||||
}
|
||||
}(nextSectionToProcess, sectionEnriched, sectionMeta)
|
||||
@@ -424,6 +436,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
break
|
||||
}
|
||||
out.Approved = append(out.Approved, res.approved...)
|
||||
out.Warnings = append(out.Warnings, res.warnings...)
|
||||
out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...)
|
||||
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...)
|
||||
}
|
||||
@@ -440,6 +453,38 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
}
|
||||
return validatorOrder[out.ValidatorRejected[i].ValidatorName] < validatorOrder[out.ValidatorRejected[j].ValidatorName]
|
||||
})
|
||||
sort.SliceStable(out.Warnings, func(i, j int) bool {
|
||||
leftSection, rightSection := -1, -1
|
||||
if out.Warnings[i].SectionIndex != nil {
|
||||
leftSection = *out.Warnings[i].SectionIndex
|
||||
}
|
||||
if out.Warnings[j].SectionIndex != nil {
|
||||
rightSection = *out.Warnings[j].SectionIndex
|
||||
}
|
||||
if leftSection != rightSection {
|
||||
return leftSection < rightSection
|
||||
}
|
||||
leftBatch, rightBatch := -1, -1
|
||||
if out.Warnings[i].BatchIndex != nil {
|
||||
leftBatch = *out.Warnings[i].BatchIndex
|
||||
}
|
||||
if out.Warnings[j].BatchIndex != nil {
|
||||
rightBatch = *out.Warnings[j].BatchIndex
|
||||
}
|
||||
if leftBatch != rightBatch {
|
||||
return leftBatch < rightBatch
|
||||
}
|
||||
if out.Warnings[i].ValidatorName != out.Warnings[j].ValidatorName {
|
||||
return validatorOrder[out.Warnings[i].ValidatorName] < validatorOrder[out.Warnings[j].ValidatorName]
|
||||
}
|
||||
if out.Warnings[i].Scope != out.Warnings[j].Scope {
|
||||
return out.Warnings[i].Scope < out.Warnings[j].Scope
|
||||
}
|
||||
if out.Warnings[i].ReasonCode != out.Warnings[j].ReasonCode {
|
||||
return out.Warnings[i].ReasonCode < out.Warnings[j].ReasonCode
|
||||
}
|
||||
return out.Warnings[i].Message < out.Warnings[j].Message
|
||||
})
|
||||
|
||||
if firstErr != nil {
|
||||
return out, firstErr
|
||||
@@ -467,11 +512,13 @@ type validateSectionCandidatesResult struct {
|
||||
approved []proposals.EnrichedCorrectionProposal
|
||||
decisions []ValidatorDecisionRecord
|
||||
rejected []ValidatorRejectedChange
|
||||
warnings []stagewarnings.StageWarning
|
||||
}
|
||||
|
||||
func validateSectionCandidates(ctx context.Context, input validateSectionCandidatesInput) (validateSectionCandidatesResult, error) {
|
||||
decisions := make([]ValidatorDecisionRecord, 0)
|
||||
rejected := make([]ValidatorRejectedChange, 0)
|
||||
warnings := make([]stagewarnings.StageWarning, 0)
|
||||
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
|
||||
|
||||
for _, validator := range input.Validators {
|
||||
@@ -512,6 +559,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
approved: eligible,
|
||||
decisions: decisions,
|
||||
rejected: rejected,
|
||||
warnings: warnings,
|
||||
}, fmt.Errorf("validator %q failed: %w", validator.Name(), err)
|
||||
}
|
||||
if err := validators.EnforceDecisionCardinality(eligible, vResult.Decisions); err != nil {
|
||||
@@ -519,8 +567,10 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
approved: eligible,
|
||||
decisions: decisions,
|
||||
rejected: rejected,
|
||||
warnings: warnings,
|
||||
}, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), err)
|
||||
}
|
||||
warnings = append(warnings, vResult.Warnings...)
|
||||
|
||||
nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(eligible))
|
||||
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(eligible))
|
||||
@@ -562,6 +612,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
approved: eligible,
|
||||
decisions: decisions,
|
||||
rejected: rejected,
|
||||
warnings: warnings,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -47,11 +47,12 @@ type fakeModule struct {
|
||||
func (m fakeModule) Key() string { return m.key }
|
||||
func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
|
||||
func (m fakeModule) Validators() []contracts.Validator { return m.validators }
|
||||
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
if m.proposeF == nil {
|
||||
return nil, nil
|
||||
return contracts.ProposalResult{}, nil
|
||||
}
|
||||
return m.proposeF(req)
|
||||
proposalsOut, err := m.proposeF(req)
|
||||
return contracts.ProposalResult{Proposals: proposalsOut}, err
|
||||
}
|
||||
|
||||
type fakeValidator struct {
|
||||
@@ -1191,7 +1192,7 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing.T) {
|
||||
func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "bad index"}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
@@ -1209,15 +1210,21 @@ func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing.
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}},
|
||||
ValidationLLMClient: client,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected llm validator failure")
|
||||
if err != nil {
|
||||
t.Fatalf("expected malformed validator response to downgrade, got %v", err)
|
||||
}
|
||||
if out.FinalTranscript.Segments[0].Text != "the cat" {
|
||||
t.Fatalf("expected partial progress retained")
|
||||
}
|
||||
if len(out.ModuleResults) != 2 || len(out.ModuleResults[1].ValidatorRejected) != 1 {
|
||||
t.Fatalf("expected second module rejection, got %+v", out.ModuleResults)
|
||||
}
|
||||
if len(out.ModuleResults[1].Warnings) != 1 || out.ModuleResults[1].Warnings[0].ReasonCode != validators.ReasonValidatorMalformed {
|
||||
t.Fatalf("expected malformed warning, got %+v", out.ModuleResults[1].Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) {
|
||||
func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
@@ -1232,12 +1239,12 @@ func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) {
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||
ValidationLLMClient: client,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected missing decision failure")
|
||||
if err != nil {
|
||||
t.Fatalf("expected missing decision downgrade, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) {
|
||||
func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
|
||||
@@ -1255,8 +1262,8 @@ func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) {
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||
ValidationLLMClient: client,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate decision failure")
|
||||
if err != nil {
|
||||
t.Fatalf("expected duplicate decision downgrade, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1355,7 +1362,7 @@ type proposalGenerationModule struct {
|
||||
func (m proposalGenerationModule) Key() string { return m.key }
|
||||
func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
|
||||
func (m proposalGenerationModule) Validators() []contracts.Validator { return nil }
|
||||
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
ModuleKey: req.RunSpec.ModuleKey,
|
||||
ModuleInstance: req.RunSpec.InstanceName,
|
||||
@@ -1372,9 +1379,9 @@ func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.Pro
|
||||
DiagnosticsDir: req.DiagnosticsDir,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
return result.Corrections, nil
|
||||
return contracts.ProposalResult{Proposals: result.Corrections, Warnings: result.Warnings}, nil
|
||||
}
|
||||
|
||||
type fakeProposalStructuredClient struct {
|
||||
|
||||
@@ -4,8 +4,35 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
)
|
||||
|
||||
type ProposalShapeValidator struct{}
|
||||
|
||||
func (v ProposalShapeValidator) Name() string { return "proposal_shape" }
|
||||
|
||||
func (v ProposalShapeValidator) Validate(_ context.Context, req Request) (Result, error) {
|
||||
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
||||
for _, c := range req.CandidateProposal {
|
||||
switch {
|
||||
case c.TargetSegmentID <= 0:
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidTargetSegment, "proposal target segment id must be positive"))
|
||||
case strings.TrimSpace(c.OriginalText) == "":
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyOriginalText, "proposal original_text must not be empty"))
|
||||
case c.Confidence < 0.0 || c.Confidence > 1.0:
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidConfidence, "proposal confidence must be between 0.0 and 1.0"))
|
||||
default:
|
||||
decisions = append(decisions, approval(c.ProposalIndex))
|
||||
}
|
||||
}
|
||||
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
|
||||
}
|
||||
|
||||
type ConfidenceThresholdValidator struct{}
|
||||
|
||||
func (v ConfidenceThresholdValidator) Name() string { return "confidence_threshold" }
|
||||
@@ -62,10 +89,23 @@ type NonEmptyCorrectionValidator struct{}
|
||||
func (v NonEmptyCorrectionValidator) Name() string { return "non_empty_corrected_text" }
|
||||
|
||||
func (v NonEmptyCorrectionValidator) Validate(_ context.Context, req Request) (Result, error) {
|
||||
segmentsByID := make(map[int]schema.Segment)
|
||||
if req.WorkingTranscript != nil {
|
||||
for _, seg := range req.WorkingTranscript.Segments {
|
||||
segmentsByID[seg.ID] = seg
|
||||
}
|
||||
}
|
||||
|
||||
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
||||
for _, c := range req.CandidateProposal {
|
||||
if strings.TrimSpace(c.CorrectedText) == "" {
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyCorrectedText, "corrected_text must not be empty"))
|
||||
segment, ok := segmentsByID[c.TargetSegmentID]
|
||||
if !ok {
|
||||
decisions = append(decisions, approval(c.ProposalIndex))
|
||||
continue
|
||||
}
|
||||
preview := proposals.PreviewProposalForSegment(&segment, c.CorrectionProposal, req.ReplacementPolicy)
|
||||
if preview.SkipReason == proposals.SkipReasonEmptyResultingText {
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyResultingText, "proposal would leave the segment empty"))
|
||||
continue
|
||||
}
|
||||
decisions = append(decisions, approval(c.ProposalIndex))
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
@@ -78,7 +79,20 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
maxTokens = req.Config.ValidationMaxPromptTokens
|
||||
}
|
||||
|
||||
batches, err := ChunkLLMValidationItems(validationReq.Items, maxTokens, v.estimator)
|
||||
warnings := append([]stagewarnings.StageWarning(nil), oversizedValidationWarnings(v.name, maxTokens, validationReq.Items, v.estimator)...)
|
||||
oversized := oversizedValidationDecisions(validationReq.Items, maxTokens, v.estimator)
|
||||
itemsForBatching := filterItemsByDecision(validationReq.Items, oversized)
|
||||
if len(itemsForBatching) == 0 {
|
||||
all := append([]Decision(nil), immediate...)
|
||||
all = append(all, oversized...)
|
||||
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
|
||||
return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
|
||||
}
|
||||
|
||||
batches, err := ChunkLLMValidationItems(itemsForBatching, maxTokens, v.estimator)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
@@ -140,12 +154,19 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
)
|
||||
}
|
||||
if err != nil {
|
||||
if isMalformedStructuredOutputError(err) {
|
||||
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
|
||||
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
|
||||
continue
|
||||
}
|
||||
return Result{}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
|
||||
}
|
||||
|
||||
batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("LLM validator %q response invalid: %w", v.name, err)
|
||||
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
|
||||
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
|
||||
continue
|
||||
}
|
||||
for i := range batchDecisions {
|
||||
batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
|
||||
@@ -154,12 +175,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
}
|
||||
|
||||
all := append([]Decision(nil), immediate...)
|
||||
all = append(all, oversized...)
|
||||
all = append(all, llmDecisions...)
|
||||
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
|
||||
return Result{ValidatorName: v.name, Decisions: all}, nil
|
||||
return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
|
||||
}
|
||||
|
||||
func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata {
|
||||
@@ -301,3 +323,98 @@ func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidation
|
||||
}
|
||||
return decisions, nil
|
||||
}
|
||||
|
||||
func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) []Decision {
|
||||
out := make([]Decision, 0)
|
||||
for _, item := range items {
|
||||
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
|
||||
if err != nil || singleTokens <= maxPromptTokens {
|
||||
continue
|
||||
}
|
||||
out = append(out, rejection(item.CorrectionIndex, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens"))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func oversizedValidationWarnings(validatorName string, maxPromptTokens int, items []LLMValidationItem, estimator chunking.TokenEstimator) []stagewarnings.StageWarning {
|
||||
out := make([]stagewarnings.StageWarning, 0)
|
||||
for _, item := range items {
|
||||
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
|
||||
if err != nil || singleTokens <= maxPromptTokens {
|
||||
continue
|
||||
}
|
||||
out = append(out, stagewarnings.StageWarning{
|
||||
Scope: stagewarnings.ScopeValidator,
|
||||
ValidatorName: validatorName,
|
||||
ReasonCode: ReasonValidatorInputTooLarge,
|
||||
Message: fmt.Sprintf("validation input exceeds max prompt tokens for proposal %d", item.CorrectionIndex),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func filterItemsByDecision(items []LLMValidationItem, decisions []Decision) []LLMValidationItem {
|
||||
if len(decisions) == 0 {
|
||||
return append([]LLMValidationItem(nil), items...)
|
||||
}
|
||||
rejected := make(map[int]struct{}, len(decisions))
|
||||
for _, decision := range decisions {
|
||||
rejected[decision.ProposalIndex] = struct{}{}
|
||||
}
|
||||
out := make([]LLMValidationItem, 0, len(items))
|
||||
for _, item := range items {
|
||||
if _, ok := rejected[item.CorrectionIndex]; ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func rejectBatch(items []LLMValidationItem, reasonCode string, message string) []Decision {
|
||||
out := make([]Decision, 0, len(items))
|
||||
for _, item := range items {
|
||||
out = append(out, rejection(item.CorrectionIndex, reasonCode, message))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func newValidatorWarning(validatorName string, batchIndex int, reasonCode string, message string, artifacts InteractionArtifacts) stagewarnings.StageWarning {
|
||||
idx := batchIndex
|
||||
return stagewarnings.StageWarning{
|
||||
Scope: stagewarnings.ScopeValidator,
|
||||
ValidatorName: validatorName,
|
||||
BatchIndex: &idx,
|
||||
ReasonCode: reasonCode,
|
||||
Message: strings.TrimSpace(message),
|
||||
DiagnosticArtifactPath: diagnosticArtifactPath(artifacts),
|
||||
}
|
||||
}
|
||||
|
||||
func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
|
||||
if artifacts.ErrorPayloadPath != "" {
|
||||
return artifacts.ErrorPayloadPath
|
||||
}
|
||||
return artifacts.ResponsePayloadPath
|
||||
}
|
||||
|
||||
func isMalformedStructuredOutputError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := err.Error()
|
||||
for _, marker := range []string{
|
||||
"malformed structured output",
|
||||
"decode structured output:",
|
||||
"decode provider response envelope:",
|
||||
"provider response missing choices",
|
||||
"provider response missing assistant message content",
|
||||
"provider response assistant message content is empty",
|
||||
"provider response assistant message content is not valid JSON",
|
||||
} {
|
||||
if strings.Contains(msg, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -248,29 +248,38 @@ func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorMalformedOutputFails(t *testing.T) {
|
||||
func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
_, err := v.Validate(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "completion failed") {
|
||||
t.Fatalf("expected malformed output error, got %v", err)
|
||||
res, err := v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected malformed output downgrade, got %v", err)
|
||||
}
|
||||
if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
|
||||
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||
}
|
||||
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorMalformed {
|
||||
t.Fatalf("expected malformed warning, got %+v", res.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorMissingDecisionFails(t *testing.T) {
|
||||
func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
_, err := v.Validate(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "response invalid") {
|
||||
t.Fatalf("expected missing decision error, got %v", err)
|
||||
res, err := v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected missing decision downgrade, got %v", err)
|
||||
}
|
||||
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
|
||||
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) {
|
||||
func TestLLMBackedValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
|
||||
@@ -278,20 +287,74 @@ func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) {
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
_, err := v.Validate(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "duplicate") {
|
||||
t.Fatalf("expected duplicate decision error, got %v", err)
|
||||
res, err := v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected duplicate decision downgrade, got %v", err)
|
||||
}
|
||||
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
|
||||
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorUnknownProposalIndexFails(t *testing.T) {
|
||||
func TestLLMBackedValidatorUnknownProposalIndexRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
_, err := v.Validate(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "unknown") {
|
||||
t.Fatalf("expected unknown index error, got %v", err)
|
||||
res, err := v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected unknown index downgrade, got %v", err)
|
||||
}
|
||||
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
|
||||
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
}}}}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||
huge := strings.Repeat("gestures ", 200)
|
||||
req := Request{
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{
|
||||
{ID: 1, Text: huge},
|
||||
{ID: 2, Text: "There were gestures at the temple.", Categories: []string{"narration"}},
|
||||
}},
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: huge, CorrectedText: "Jesters", Confidence: 0.9},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
||||
},
|
||||
{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 2, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.9},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
||||
},
|
||||
},
|
||||
ModuleKey: "homophones",
|
||||
ModuleInstance: "homophones",
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
}
|
||||
req.LLMClient = client
|
||||
cfg := config.Default()
|
||||
cfg.ValidationMaxPromptTokens = 200
|
||||
req.Config = &cfg
|
||||
|
||||
res, err := v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected oversize downgrade, got %v", err)
|
||||
}
|
||||
if len(res.Decisions) != 2 {
|
||||
t.Fatalf("expected two decisions, got %+v", res.Decisions)
|
||||
}
|
||||
if res.Decisions[0].ReasonCode != ReasonValidatorInputTooLarge || res.Decisions[0].Approved {
|
||||
t.Fatalf("expected first decision oversize rejection, got %+v", res.Decisions[0])
|
||||
}
|
||||
if !res.Decisions[1].Approved {
|
||||
t.Fatalf("expected second decision approved, got %+v", res.Decisions[1])
|
||||
}
|
||||
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorInputTooLarge {
|
||||
t.Fatalf("expected one oversize warning, got %+v", res.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -8,16 +8,22 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
)
|
||||
|
||||
const (
|
||||
ReasonApproved = "approved"
|
||||
ReasonLowConfidence = "low_confidence"
|
||||
ReasonMissingOriginalText = "missing_original_text"
|
||||
ReasonMissingTargetSegment = "missing_target_segment"
|
||||
ReasonEmptyCorrectedText = "empty_corrected_text"
|
||||
ReasonNoEffect = "no_effect"
|
||||
ReasonProtectedGlossaryTerm = "protected_glossary_term"
|
||||
ReasonApproved = "approved"
|
||||
ReasonLowConfidence = "low_confidence"
|
||||
ReasonMissingOriginalText = "missing_original_text"
|
||||
ReasonMissingTargetSegment = "missing_target_segment"
|
||||
ReasonEmptyResultingText = "empty_resulting_segment"
|
||||
ReasonNoEffect = "no_effect"
|
||||
ReasonProtectedGlossaryTerm = "protected_glossary_term"
|
||||
ReasonInvalidTargetSegment = "invalid_target_segment_id"
|
||||
ReasonEmptyOriginalText = "empty_original_text"
|
||||
ReasonInvalidConfidence = "invalid_confidence"
|
||||
ReasonValidatorMalformed = "validator_response_malformed"
|
||||
ReasonValidatorInputTooLarge = "validator_input_too_large"
|
||||
)
|
||||
|
||||
// Request is the runtime input shared by deterministic validators.
|
||||
@@ -45,8 +51,9 @@ type Decision struct {
|
||||
|
||||
// Result is one validator output containing exactly one decision per proposal index.
|
||||
type Result struct {
|
||||
ValidatorName string `json:"validator_name"`
|
||||
Decisions []Decision `json:"decisions"`
|
||||
ValidatorName string `json:"validator_name"`
|
||||
Decisions []Decision `json:"decisions"`
|
||||
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
// ValidationScheduler provides bounded execution for validator LLM calls.
|
||||
|
||||
@@ -68,6 +68,31 @@ func TestConfidenceThresholdValidator(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProposalShapeValidator(t *testing.T) {
|
||||
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "teh", "the", 0.9),
|
||||
mkCandidate(1, 0, "teh", "the", 0.9),
|
||||
mkCandidate(2, 1, " ", "the", 0.9),
|
||||
mkCandidate(3, 1, "teh", "the", 1.5),
|
||||
}}
|
||||
res, err := (ProposalShapeValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Validate error: %v", err)
|
||||
}
|
||||
if !res.Decisions[0].Approved {
|
||||
t.Fatalf("expected proposal 0 approved")
|
||||
}
|
||||
if res.Decisions[1].ReasonCode != ReasonInvalidTargetSegment {
|
||||
t.Fatalf("expected invalid target segment rejection, got %+v", res.Decisions[1])
|
||||
}
|
||||
if res.Decisions[2].ReasonCode != ReasonEmptyOriginalText {
|
||||
t.Fatalf("expected empty original rejection, got %+v", res.Decisions[2])
|
||||
}
|
||||
if res.Decisions[3].ReasonCode != ReasonInvalidConfidence {
|
||||
t.Fatalf("expected invalid confidence rejection, got %+v", res.Decisions[3])
|
||||
}
|
||||
}
|
||||
|
||||
func TestOriginalTextPresenceValidator(t *testing.T) {
|
||||
req := Request{WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "hello", "hi", 0.9),
|
||||
@@ -90,10 +115,13 @@ func TestOriginalTextPresenceValidator(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestNonEmptyCorrectionValidator(t *testing.T) {
|
||||
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "hello", "hi", 0.9),
|
||||
mkCandidate(1, 1, "hello", " ", 0.9),
|
||||
}}
|
||||
req := Request{
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}},
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "hello", "hi", 0.9),
|
||||
mkCandidate(1, 1, "hello world", " ", 0.9),
|
||||
}}
|
||||
res, err := (NonEmptyCorrectionValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Validate error: %v", err)
|
||||
@@ -101,8 +129,8 @@ func TestNonEmptyCorrectionValidator(t *testing.T) {
|
||||
if !res.Decisions[0].Approved {
|
||||
t.Fatalf("expected proposal 0 approved")
|
||||
}
|
||||
if res.Decisions[1].ReasonCode != ReasonEmptyCorrectedText {
|
||||
t.Fatalf("expected empty_corrected_text, got %+v", res.Decisions[1])
|
||||
if res.Decisions[1].ReasonCode != ReasonEmptyResultingText {
|
||||
t.Fatalf("expected empty_resulting_segment, got %+v", res.Decisions[1])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,9 +179,14 @@ func TestStableReasonCodes(t *testing.T) {
|
||||
ReasonLowConfidence,
|
||||
ReasonMissingOriginalText,
|
||||
ReasonMissingTargetSegment,
|
||||
ReasonEmptyCorrectedText,
|
||||
ReasonEmptyResultingText,
|
||||
ReasonNoEffect,
|
||||
ReasonProtectedGlossaryTerm,
|
||||
ReasonInvalidTargetSegment,
|
||||
ReasonEmptyOriginalText,
|
||||
ReasonInvalidConfidence,
|
||||
ReasonValidatorMalformed,
|
||||
ReasonValidatorInputTooLarge,
|
||||
}
|
||||
for _, code := range codes {
|
||||
if strings.TrimSpace(code) == "" {
|
||||
|
||||
18
internal/framework/warnings/warnings.go
Normal file
18
internal/framework/warnings/warnings.go
Normal file
@@ -0,0 +1,18 @@
|
||||
package warnings
|
||||
|
||||
type Scope string
|
||||
|
||||
const (
|
||||
ScopeProposalGeneration Scope = "proposal_generation"
|
||||
ScopeValidator Scope = "validator"
|
||||
)
|
||||
|
||||
type StageWarning struct {
|
||||
Scope Scope `json:"scope"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
SectionIndex *int `json:"section_index,omitempty"`
|
||||
ValidatorName string `json:"validator_name,omitempty"`
|
||||
BatchIndex *int `json:"batch_index,omitempty"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
}
|
||||
@@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator {
|
||||
return append([]contracts.Validator(nil), m.validators...)
|
||||
}
|
||||
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
|
||||
sectionIndex := 0
|
||||
if req.Section != nil {
|
||||
@@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
||||
}
|
||||
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
|
||||
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
@@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
||||
DiagnosticsDir: req.DiagnosticsDir,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
return generated.Corrections, nil
|
||||
return contracts.ProposalResult{
|
||||
Proposals: generated.Corrections,
|
||||
Warnings: generated.Warnings,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
|
||||
|
||||
@@ -129,6 +129,7 @@ func TestGlossaryModuleValidatorChain(t *testing.T) {
|
||||
got = append(got, v.Name())
|
||||
}
|
||||
want := []string{
|
||||
"proposal_shape",
|
||||
"no_effect",
|
||||
"original_text_presence",
|
||||
"confidence_threshold",
|
||||
@@ -185,7 +186,7 @@ func TestGlossaryModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T)
|
||||
if len(client.calls) != 1 || client.calls[0].StageName != "glossary:proposal" {
|
||||
t.Fatalf("expected one glossary:proposal call, got %+v", client.calls)
|
||||
}
|
||||
if len(out) != 1 || out[0].CorrectedText != "Jesters" {
|
||||
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Jesters" {
|
||||
t.Fatalf("unexpected proposals: %+v", out)
|
||||
}
|
||||
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "glossary", "*proposal*response-payload.json"))
|
||||
|
||||
@@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator {
|
||||
return append([]contracts.Validator(nil), m.validators...)
|
||||
}
|
||||
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
|
||||
sectionIndex := 0
|
||||
if req.Section != nil {
|
||||
@@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
||||
}
|
||||
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
|
||||
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
@@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
||||
DiagnosticsDir: req.DiagnosticsDir,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
return generated.Corrections, nil
|
||||
return contracts.ProposalResult{
|
||||
Proposals: generated.Corrections,
|
||||
Warnings: generated.Warnings,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
|
||||
|
||||
@@ -131,6 +131,7 @@ func TestGrammarModuleValidatorChain(t *testing.T) {
|
||||
got = append(got, v.Name())
|
||||
}
|
||||
want := []string{
|
||||
"proposal_shape",
|
||||
"no_effect",
|
||||
"original_text_presence",
|
||||
"confidence_threshold",
|
||||
@@ -187,7 +188,7 @@ func TestGrammarModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) {
|
||||
if len(client.calls) != 1 || client.calls[0].StageName != "grammar:proposal" {
|
||||
t.Fatalf("expected one grammar:proposal call, got %+v", client.calls)
|
||||
}
|
||||
if len(out) != 1 || out[0].CorrectedText != "Hello, world" {
|
||||
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Hello, world" {
|
||||
t.Fatalf("unexpected proposals: %+v", out)
|
||||
}
|
||||
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "grammar", "*proposal*response-payload.json"))
|
||||
|
||||
@@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator {
|
||||
return append([]contracts.Validator(nil), m.validators...)
|
||||
}
|
||||
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
|
||||
sectionIndex := 0
|
||||
if req.Section != nil {
|
||||
@@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
||||
}
|
||||
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
|
||||
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
@@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
||||
DiagnosticsDir: req.DiagnosticsDir,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
return generated.Corrections, nil
|
||||
return contracts.ProposalResult{
|
||||
Proposals: generated.Corrections,
|
||||
Warnings: generated.Warnings,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
|
||||
|
||||
@@ -145,6 +145,7 @@ func TestHomophonesModuleValidatorChain(t *testing.T) {
|
||||
got = append(got, v.Name())
|
||||
}
|
||||
want := []string{
|
||||
"proposal_shape",
|
||||
"no_effect",
|
||||
"original_text_presence",
|
||||
"confidence_threshold",
|
||||
@@ -201,7 +202,7 @@ func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
|
||||
if len(client.calls) != 1 || client.calls[0].StageName != "homophones:proposal" {
|
||||
t.Fatalf("expected one homophones:proposal call, got %+v", client.calls)
|
||||
}
|
||||
if len(out) != 1 || out[0].CorrectedText != "Jesters" {
|
||||
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Jesters" {
|
||||
t.Fatalf("unexpected proposals: %+v", out)
|
||||
}
|
||||
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "homophones", "*proposal*response-payload.json"))
|
||||
|
||||
@@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator {
|
||||
return append([]contracts.Validator(nil), m.validators...)
|
||||
}
|
||||
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
|
||||
sectionIndex := 0
|
||||
if req.Section != nil {
|
||||
@@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
||||
}
|
||||
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
|
||||
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
@@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
||||
DiagnosticsDir: req.DiagnosticsDir,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
return generated.Corrections, nil
|
||||
return contracts.ProposalResult{
|
||||
Proposals: generated.Corrections,
|
||||
Warnings: generated.Warnings,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
|
||||
|
||||
@@ -132,6 +132,7 @@ func TestSpokenWordModuleValidatorChain(t *testing.T) {
|
||||
got = append(got, v.Name())
|
||||
}
|
||||
want := []string{
|
||||
"proposal_shape",
|
||||
"no_effect",
|
||||
"original_text_presence",
|
||||
"confidence_threshold",
|
||||
@@ -188,7 +189,7 @@ func TestSpokenWordModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
|
||||
if len(client.calls) != 1 || client.calls[0].StageName != "spoken_word:proposal" {
|
||||
t.Fatalf("expected one spoken_word:proposal call, got %+v", client.calls)
|
||||
}
|
||||
if len(out) != 1 || out[0].CorrectedText != "I think" {
|
||||
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "I think" {
|
||||
t.Fatalf("unexpected proposals: %+v", out)
|
||||
}
|
||||
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "spoken_word", "*proposal*response-payload.json"))
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
var builtInChains = map[string][]string{
|
||||
"glossary": {
|
||||
KeyProposalShape,
|
||||
KeyNoEffect,
|
||||
KeyOriginalTextPresence,
|
||||
KeyConfidenceThreshold,
|
||||
@@ -18,6 +19,7 @@ var builtInChains = map[string][]string{
|
||||
KeyMeaningReversalReview,
|
||||
},
|
||||
"homophones": {
|
||||
KeyProposalShape,
|
||||
KeyNoEffect,
|
||||
KeyOriginalTextPresence,
|
||||
KeyConfidenceThreshold,
|
||||
@@ -27,6 +29,7 @@ var builtInChains = map[string][]string{
|
||||
KeyMeaningReversalReview,
|
||||
},
|
||||
"spoken_word": {
|
||||
KeyProposalShape,
|
||||
KeyNoEffect,
|
||||
KeyOriginalTextPresence,
|
||||
KeyConfidenceThreshold,
|
||||
@@ -36,6 +39,7 @@ var builtInChains = map[string][]string{
|
||||
KeyMeaningReversalReview,
|
||||
},
|
||||
"grammar": {
|
||||
KeyProposalShape,
|
||||
KeyNoEffect,
|
||||
KeyOriginalTextPresence,
|
||||
KeyConfidenceThreshold,
|
||||
|
||||
13
internal/validators/proposal_shape/validator.go
Normal file
13
internal/validators/proposal_shape/validator.go
Normal file
@@ -0,0 +1,13 @@
|
||||
package proposal_shape
|
||||
|
||||
import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
|
||||
)
|
||||
|
||||
const Key = "proposal_shape"
|
||||
|
||||
func New() (contracts.Validator, error) {
|
||||
return validatormetadata.Wrap(frameworkvalidators.ProposalShapeValidator{}, validatormetadata.ExecutionClassDeterministic), nil
|
||||
}
|
||||
@@ -11,11 +11,13 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/no_effect"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/non_empty_corrected_text"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/original_text_presence"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/proposal_shape"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/protected_terms"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/spoken_form_plausibility"
|
||||
)
|
||||
|
||||
const (
|
||||
KeyProposalShape = "proposal_shape"
|
||||
KeyConfidenceThreshold = "confidence_threshold"
|
||||
KeyOriginalTextPresence = "original_text_presence"
|
||||
KeyNonEmptyCorrectedText = "non_empty_corrected_text"
|
||||
@@ -39,6 +41,7 @@ type Registry struct {
|
||||
|
||||
func NewBuiltInRegistry() *Registry {
|
||||
defs := []BuiltInValidatorDefinition{
|
||||
{Key: KeyProposalShape, Build: proposal_shape.New},
|
||||
{Key: KeyConfidenceThreshold, Build: confidence_threshold.New},
|
||||
{Key: KeyOriginalTextPresence, Build: original_text_presence.New},
|
||||
{Key: KeyNonEmptyCorrectedText, Build: non_empty_corrected_text.New},
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/no_effect"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/non_empty_corrected_text"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/original_text_presence"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/proposal_shape"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/protected_terms"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/spoken_form_plausibility"
|
||||
)
|
||||
@@ -21,6 +22,7 @@ import (
|
||||
func TestBuiltInRegistryRegistersAllKeys(t *testing.T) {
|
||||
r := NewBuiltInRegistry()
|
||||
for _, key := range []string{
|
||||
KeyProposalShape,
|
||||
KeyConfidenceThreshold,
|
||||
KeyOriginalTextPresence,
|
||||
KeyNonEmptyCorrectedText,
|
||||
@@ -52,6 +54,7 @@ func TestBuiltInValidatorPackagesConstruct(t *testing.T) {
|
||||
wantClass validatormetadata.ExecutionClass
|
||||
}
|
||||
cases := []validatorCtor{
|
||||
{name: "proposal_shape", key: KeyProposalShape, build: proposal_shape.New, wantClass: validatormetadata.ExecutionClassDeterministic},
|
||||
{name: "confidence_threshold", key: KeyConfidenceThreshold, build: confidence_threshold.New, wantClass: validatormetadata.ExecutionClassDeterministic},
|
||||
{name: "original_text_presence", key: KeyOriginalTextPresence, build: original_text_presence.New, wantClass: validatormetadata.ExecutionClassDeterministic},
|
||||
{name: "non_empty_corrected_text", key: KeyNonEmptyCorrectedText, build: non_empty_corrected_text.New, wantClass: validatormetadata.ExecutionClassDeterministic},
|
||||
|
||||
Reference in New Issue
Block a user