diff --git a/README.md b/README.md index c5c4cfd..546f3bc 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,7 @@ audita config print-effective --config audita.yml For full config-file schema and examples, see [`docs/configuration.md`](docs/configuration.md). For output-schema details, see [`docs/output-schemas.md`](docs/output-schemas.md). +For built-in validator keys and chain definitions, see [`docs/validators.md`](docs/validators.md). For CLI/process compatibility guarantees, see [`docs/public-contract.md`](docs/public-contract.md). ### Modules diff --git a/docs/architecture.md b/docs/architecture.md index f66a0ef..93fea9e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -132,6 +132,10 @@ internal/framework/validators/ llm_batching.go llm_validators.go +internal/validators/ + registry.go + chains.go + internal/framework/llm/ openai_compatible_client.go scheduler.go @@ -428,6 +432,63 @@ These primitives are wired into the production runner and report model. The gram Validator rejections are reported distinctly from proposal-application skips. +Validator composition is now explicit and registry-backed through `internal/validators`: +- built-in validator registry with stable keys and lookup/build failure for unknown keys; +- built-in chain definitions per production module key; +- production modules resolve validator chains from those built-in definitions. + +Stable built-in validator keys: +- deterministic: + - `confidence_threshold` + - `original_text_presence` + - `non_empty_corrected_text` + - `no_effect` + - `protected_terms` +- LLM-backed: + - `spoken_form_plausibility` + - `meaning_reversal_review` + - `editorial_review` + - `grammar_review` + - `spoken_word_review` + +Built-in module chains: +- `glossary`: + - `no_effect` + - `original_text_presence` + - `confidence_threshold` + - `protected_terms` + - `non_empty_corrected_text` + - `spoken_form_plausibility` + - `meaning_reversal_review` +- `homophones`: + - `no_effect` + - `original_text_presence` + - `confidence_threshold` + - `protected_terms` + - `non_empty_corrected_text` + - `spoken_form_plausibility` + - `meaning_reversal_review` +- `spoken_word`: + - `no_effect` + - `original_text_presence` + - `confidence_threshold` + - `protected_terms` + - `non_empty_corrected_text` + - `spoken_word_review` + - `meaning_reversal_review` +- `grammar`: + - `no_effect` + - `original_text_presence` + - `confidence_threshold` + - `protected_terms` + - `non_empty_corrected_text` + - `grammar_review` + - `meaning_reversal_review` + +1.0 boundary: +- validator chains are built-in and not user-configurable from config/CLI. +- existing threshold and batching knobs remain configurable. + ## Implemented LLM-backed validator infrastructure `internal/framework/validators` now includes LLM-backed validator support: - typed request/response models for structured LLM validation; @@ -563,6 +624,7 @@ Current process reports also include: - run-level module summary totals and failed module instance metadata. - module-level validator decisions and validator rejections. - optional decision-level diagnostic artifact paths for validator LLM interactions when available. +- stable validator keys in `validator_name` fields for validator decisions/rejections. - explicit report metadata: - report schema name; - report schema version; diff --git a/docs/configuration.md b/docs/configuration.md index a53cc5f..c02c48a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -182,3 +182,4 @@ Current guidance: - prefer file config for baseline behavior; - keep environment variables for secrets/deployment-specific overrides; - use CLI flags for per-run overrides. +- validator chains are built-in and are not user-configurable in config. diff --git a/docs/public-contract.md b/docs/public-contract.md index 7f2c380..aaf6df2 100644 --- a/docs/public-contract.md +++ b/docs/public-contract.md @@ -10,6 +10,7 @@ This contract covers: - transcript/glossary input forms - transcript output schema selection - process report schema metadata +- stable validator key identifiers in report/diagnostics records - diagnostics directory behavior - stdout/stderr and exit-code behavior - secret redaction guarantees @@ -89,6 +90,8 @@ Current values: `--report-json` output and diagnostics run-dir `report.json` use the same report schema metadata. +Validator decision/rejection records in reports use stable validator keys in `validator_name`. + ## Diagnostics directory behavior When diagnostics directory creation succeeds, Audita writes run artifacts including: diff --git a/docs/roadmap.md b/docs/roadmap.md index 376c40a..a7d9e00 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -511,6 +511,20 @@ Add explicit report schema metadata, for example: Make validators as easy to reason about as modules. +## Implementation status (2026-05-13) + +This workstream is now implemented for built-in validator composition: +- first-class built-in validator registry exists in `internal/validators`; +- stable validator keys are defined and used for built-in chains and runtime validator names; +- explicit built-in module validator-chain definitions are implemented and resolved through registry-backed chain wiring; +- production module constructors use built-in chain resolution instead of ad hoc manual validator construction; +- runner execution preserves deterministic ordering (deterministic validators before LLM-backed validators); +- report validator decision/rejection records now carry stable validator keys. + +Current boundary: +- validator chains are built-in and not user-configurable. +- prompt-asset registries, prompt metadata, scheduler utilization diagnostics, correction ledgers, and generated summaries remain planned. + Validators are now central runtime components. They are reused across modules, have deterministic and LLM-backed implementations, produce diagnostics, and affect final correction acceptance. They should therefore have stable identities, registry metadata, and composable chain definitions. ## Package structure diff --git a/docs/validators.md b/docs/validators.md new file mode 100644 index 0000000..81714f5 --- /dev/null +++ b/docs/validators.md @@ -0,0 +1,111 @@ +# Audita Validators + +This document describes Audita's built-in validator registry and module validator chains. + +## Scope + +Validator chains are built-in runtime behavior. + +Current 1.0 boundary: +- built-in validator keys and built-in module chains are stable runtime identifiers; +- thresholds and batching knobs remain configurable where already supported; +- arbitrary user-defined validator chains are deferred. + +## Built-in validator keys + +### Deterministic validators + +- `confidence_threshold` + - checks proposal confidence against module-specific configured threshold. +- `original_text_presence` + - ensures target segment exists and `original_text` exists in current working segment text. +- `non_empty_corrected_text` + - rejects blank/whitespace-only `corrected_text`. +- `no_effect` + - rejects proposals where `original_text == corrected_text`. +- `protected_terms` + - protects glossary-derived terms from unsafe mutations in non-glossary modules. + - glossary stages use glossary-specific protection logic but still report this same stable key. + +### LLM-backed validators + +- `spoken_form_plausibility` + - checks whether proposed spoken-form change remains plausible in transcript context. +- `meaning_reversal_review` + - checks for likely meaning reversal or semantic contradiction. +- `editorial_review` + - performs conservative editorial safety review. +- `grammar_review` + - checks grammar-stage proposals for grammar-focused safety constraints. +- `spoken_word_review` + - checks spoken-word-stage proposals for dysfluency-cleanup safety constraints. + +## Built-in module chains + +Current built-in chains resolved from `internal/validators/chains.go`: + +- `glossary` + - `no_effect` + - `original_text_presence` + - `confidence_threshold` + - `protected_terms` + - `non_empty_corrected_text` + - `spoken_form_plausibility` + - `meaning_reversal_review` + +- `homophones` + - `no_effect` + - `original_text_presence` + - `confidence_threshold` + - `protected_terms` + - `non_empty_corrected_text` + - `spoken_form_plausibility` + - `meaning_reversal_review` + +- `spoken_word` + - `no_effect` + - `original_text_presence` + - `confidence_threshold` + - `protected_terms` + - `non_empty_corrected_text` + - `spoken_word_review` + - `meaning_reversal_review` + +- `grammar` + - `no_effect` + - `original_text_presence` + - `confidence_threshold` + - `protected_terms` + - `non_empty_corrected_text` + - `grammar_review` + - `meaning_reversal_review` + +## Execution semantics + +- modules execute serially; +- section proposal work can run concurrently within a module; +- deterministic validators run before LLM-backed validators; +- malformed/missing/duplicate/unknown LLM validator decisions fail safely; +- approved proposals are applied once per module after section work settles. + +## Validator rejections vs proposal-application skips + +- validator rejection: + - proposal is denied by validator-chain review and appears in validator rejection reporting with validator key and reason code. +- proposal-application skip: + - proposal passed validators but could not be applied under replacement-policy semantics (for example no matching span at apply time). + +These are separate outcomes and are reported separately. + +## Reporting and diagnostics identity + +- report validator decision/rejection entries use stable validator keys in `validator_name`. +- validator LLM diagnostics include validator identity in interaction metadata and structured response schema metadata. + +## Configurable knobs that remain supported + +- per-module confidence thresholds (`thresholds.*` / equivalent env+CLI overrides) +- validation batching limits (`validation_max_prompt_tokens` / equivalent env+CLI overrides) +- validation LLM model/base URL/timeout/retries/concurrency settings + +These tune validator behavior without exposing arbitrary user-defined chains. diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index f0cac13..1e8e848 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -3048,6 +3048,9 @@ func TestRunProcessExplicitHomophonesProtectedGlossaryTermRejected(t *testing.T) if module.ValidatorRejected[0].ReasonCode != validators.ReasonProtectedGlossaryTerm { t.Fatalf("expected protected glossary term rejection, got %+v", module.ValidatorRejected[0]) } + if module.ValidatorRejected[0].ValidatorName != "protected_terms" { + t.Fatalf("expected stable validator key protected_terms, got %+v", module.ValidatorRejected[0]) + } } func TestRunProcessExplicitHomophonesMalformedLLMOutputFailsWithErrorLog(t *testing.T) { diff --git a/internal/cli/testdata/parity/llm-validator-rejection.case.json b/internal/cli/testdata/parity/llm-validator-rejection.case.json index 27a72e2..faaf3ba 100644 --- a/internal/cli/testdata/parity/llm-validator-rejection.case.json +++ b/internal/cli/testdata/parity/llm-validator-rejection.case.json @@ -14,6 +14,6 @@ "total_skipped_changes": 1, "validator_rejected_reason_codes": ["llm_rejected"], "expected_proposal_calls": ["grammar:proposal"], - "expected_validation_calls": ["grammar:section-0000:grammar_only_guard:batch-0000"] + "expected_validation_calls": ["grammar:section-0000:grammar_review:batch-0000"] } } diff --git a/internal/framework/validators/deterministic.go b/internal/framework/validators/deterministic.go index dbc99d9..76da533 100644 --- a/internal/framework/validators/deterministic.go +++ b/internal/framework/validators/deterministic.go @@ -59,7 +59,7 @@ func (v OriginalTextPresenceValidator) Validate(_ context.Context, req Request) type NonEmptyCorrectionValidator struct{} -func (v NonEmptyCorrectionValidator) Name() string { return "non_empty_correction" } +func (v NonEmptyCorrectionValidator) Name() string { return "non_empty_corrected_text" } func (v NonEmptyCorrectionValidator) Validate(_ context.Context, req Request) (Result, error) { decisions := make([]Decision, 0, len(req.CandidateProposal)) @@ -97,7 +97,7 @@ func (v NoEffectValidator) Validate(_ context.Context, req Request) (Result, err type ProtectedGlossaryTermValidator struct{} -func (v ProtectedGlossaryTermValidator) Name() string { return "protected_glossary_terms" } +func (v ProtectedGlossaryTermValidator) Name() string { return "protected_terms" } func (v ProtectedGlossaryTermValidator) Validate(_ context.Context, req Request) (Result, error) { if req.ModuleKey == "glossary" { @@ -127,7 +127,7 @@ func (v ProtectedGlossaryTermValidator) Validate(_ context.Context, req Request) type GlossaryStageProtectedGlossaryTermValidator struct{} func (v GlossaryStageProtectedGlossaryTermValidator) Name() string { - return "glossary_stage_protected_glossary_terms" + return "protected_terms" } func (v GlossaryStageProtectedGlossaryTermValidator) Validate(_ context.Context, req Request) (Result, error) { diff --git a/internal/modules/glossary/module.go b/internal/modules/glossary/module.go index 1430dbb..408a9f8 100644 --- a/internal/modules/glossary/module.go +++ b/internal/modules/glossary/module.go @@ -8,7 +8,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" - "gitea.maximumdirect.net/eric/audita/internal/framework/validators" + builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators" ) type Module struct { @@ -16,25 +16,12 @@ type Module struct { } func New() (*Module, error) { - spokenForm, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "") + chain, err := builtinvalidators.ResolveBuiltInChain("glossary", builtinvalidators.NewBuiltInRegistry()) if err != nil { return nil, err } - meaningReversal, err := validators.NewLLMBackedValidator("meaning_reversal_review", validators.LLMValidatorTypeMeaningReversal, "") - if err != nil { - return nil, err - } - return &Module{ - validators: []contracts.Validator{ - validators.NoEffectValidator{}, - validators.OriginalTextPresenceValidator{}, - validators.ConfidenceThresholdValidator{}, - validators.GlossaryStageProtectedGlossaryTermValidator{}, - validators.NonEmptyCorrectionValidator{}, - spokenForm, - meaningReversal, - }, + validators: chain, }, nil } diff --git a/internal/modules/glossary/module_test.go b/internal/modules/glossary/module_test.go index 77dda1f..d832a3f 100644 --- a/internal/modules/glossary/module_test.go +++ b/internal/modules/glossary/module_test.go @@ -132,9 +132,9 @@ func TestGlossaryModuleValidatorChain(t *testing.T) { "no_effect", "original_text_presence", "confidence_threshold", - "glossary_stage_protected_glossary_terms", - "non_empty_correction", - "spoken_form_plausibility_review", + "protected_terms", + "non_empty_corrected_text", + "spoken_form_plausibility", "meaning_reversal_review", } if strings.Join(got, ",") != strings.Join(want, ",") { diff --git a/internal/modules/grammar/module.go b/internal/modules/grammar/module.go index 7c0770f..6c2df41 100644 --- a/internal/modules/grammar/module.go +++ b/internal/modules/grammar/module.go @@ -8,7 +8,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" - "gitea.maximumdirect.net/eric/audita/internal/framework/validators" + builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators" ) type Module struct { @@ -16,25 +16,12 @@ type Module struct { } func New() (*Module, error) { - grammarOnlyGuard, err := validators.NewLLMBackedValidator("grammar_only_guard", validators.LLMValidatorTypeGrammarReview, "") + chain, err := builtinvalidators.ResolveBuiltInChain("grammar", builtinvalidators.NewBuiltInRegistry()) if err != nil { return nil, err } - meaningReversal, err := validators.NewLLMBackedValidator("meaning_reversal_review", validators.LLMValidatorTypeMeaningReversal, "") - if err != nil { - return nil, err - } - return &Module{ - validators: []contracts.Validator{ - validators.NoEffectValidator{}, - validators.OriginalTextPresenceValidator{}, - validators.ConfidenceThresholdValidator{}, - validators.ProtectedGlossaryTermValidator{}, - validators.NonEmptyCorrectionValidator{}, - grammarOnlyGuard, - meaningReversal, - }, + validators: chain, }, nil } diff --git a/internal/modules/grammar/module_test.go b/internal/modules/grammar/module_test.go index 90b3b2c..da69cd2 100644 --- a/internal/modules/grammar/module_test.go +++ b/internal/modules/grammar/module_test.go @@ -134,9 +134,9 @@ func TestGrammarModuleValidatorChain(t *testing.T) { "no_effect", "original_text_presence", "confidence_threshold", - "protected_glossary_terms", - "non_empty_correction", - "grammar_only_guard", + "protected_terms", + "non_empty_corrected_text", + "grammar_review", "meaning_reversal_review", } if strings.Join(got, ",") != strings.Join(want, ",") { diff --git a/internal/modules/homophones/module.go b/internal/modules/homophones/module.go index 489084b..3c4c79c 100644 --- a/internal/modules/homophones/module.go +++ b/internal/modules/homophones/module.go @@ -8,7 +8,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" - "gitea.maximumdirect.net/eric/audita/internal/framework/validators" + builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators" ) type Module struct { @@ -16,25 +16,12 @@ type Module struct { } func New() (*Module, error) { - spokenForm, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "") + chain, err := builtinvalidators.ResolveBuiltInChain("homophones", builtinvalidators.NewBuiltInRegistry()) if err != nil { return nil, err } - meaningReversal, err := validators.NewLLMBackedValidator("meaning_reversal_review", validators.LLMValidatorTypeMeaningReversal, "") - if err != nil { - return nil, err - } - return &Module{ - validators: []contracts.Validator{ - validators.NoEffectValidator{}, - validators.OriginalTextPresenceValidator{}, - validators.ConfidenceThresholdValidator{}, - validators.ProtectedGlossaryTermValidator{}, - validators.NonEmptyCorrectionValidator{}, - spokenForm, - meaningReversal, - }, + validators: chain, }, nil } diff --git a/internal/modules/homophones/module_test.go b/internal/modules/homophones/module_test.go index 4ce29df..3e795c9 100644 --- a/internal/modules/homophones/module_test.go +++ b/internal/modules/homophones/module_test.go @@ -148,9 +148,9 @@ func TestHomophonesModuleValidatorChain(t *testing.T) { "no_effect", "original_text_presence", "confidence_threshold", - "protected_glossary_terms", - "non_empty_correction", - "spoken_form_plausibility_review", + "protected_terms", + "non_empty_corrected_text", + "spoken_form_plausibility", "meaning_reversal_review", } if strings.Join(got, ",") != strings.Join(want, ",") { @@ -276,13 +276,13 @@ func TestHomophonesProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi } var protectedValidator contracts.Validator for _, v := range module.Validators() { - if v.Name() == "protected_glossary_terms" { + if v.Name() == "protected_terms" { protectedValidator = v break } } if protectedValidator == nil { - t.Fatal("expected protected_glossary_terms validator") + t.Fatal("expected protected_terms validator") } working := &schema.Transcript{Segments: []schema.Segment{ diff --git a/internal/modules/spoken_word/module.go b/internal/modules/spoken_word/module.go index c4c5008..afb8d19 100644 --- a/internal/modules/spoken_word/module.go +++ b/internal/modules/spoken_word/module.go @@ -8,7 +8,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" - "gitea.maximumdirect.net/eric/audita/internal/framework/validators" + builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators" ) type Module struct { @@ -16,25 +16,12 @@ type Module struct { } func New() (*Module, error) { - spokenWordReview, err := validators.NewLLMBackedValidator("spoken_word_review", validators.LLMValidatorTypeSpokenWordReview, "") + chain, err := builtinvalidators.ResolveBuiltInChain("spoken_word", builtinvalidators.NewBuiltInRegistry()) if err != nil { return nil, err } - meaningReversal, err := validators.NewLLMBackedValidator("meaning_reversal_review", validators.LLMValidatorTypeMeaningReversal, "") - if err != nil { - return nil, err - } - return &Module{ - validators: []contracts.Validator{ - validators.NoEffectValidator{}, - validators.OriginalTextPresenceValidator{}, - validators.ConfidenceThresholdValidator{}, - validators.ProtectedGlossaryTermValidator{}, - validators.NonEmptyCorrectionValidator{}, - spokenWordReview, - meaningReversal, - }, + validators: chain, }, nil } diff --git a/internal/modules/spoken_word/module_test.go b/internal/modules/spoken_word/module_test.go index b88212f..2fdb19b 100644 --- a/internal/modules/spoken_word/module_test.go +++ b/internal/modules/spoken_word/module_test.go @@ -136,8 +136,8 @@ func TestSpokenWordModuleValidatorChain(t *testing.T) { "no_effect", "original_text_presence", "confidence_threshold", - "protected_glossary_terms", - "non_empty_correction", + "protected_terms", + "non_empty_corrected_text", "spoken_word_review", "meaning_reversal_review", } @@ -264,13 +264,13 @@ func TestSpokenWordProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi } var protectedValidator contracts.Validator for _, v := range module.Validators() { - if v.Name() == "protected_glossary_terms" { + if v.Name() == "protected_terms" { protectedValidator = v break } } if protectedValidator == nil { - t.Fatal("expected protected_glossary_terms validator") + t.Fatal("expected protected_terms validator") } working := &schema.Transcript{Segments: []schema.Segment{ diff --git a/internal/validators/chains.go b/internal/validators/chains.go new file mode 100644 index 0000000..22d96c2 --- /dev/null +++ b/internal/validators/chains.go @@ -0,0 +1,84 @@ +package validators + +import ( + "fmt" + + "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" + frameworkvalidators "gitea.maximumdirect.net/eric/audita/internal/framework/validators" +) + +var builtInChains = map[string][]string{ + "glossary": { + KeyNoEffect, + KeyOriginalTextPresence, + KeyConfidenceThreshold, + KeyProtectedTerms, + KeyNonEmptyCorrectedText, + KeySpokenFormPlausibility, + KeyMeaningReversalReview, + }, + "homophones": { + KeyNoEffect, + KeyOriginalTextPresence, + KeyConfidenceThreshold, + KeyProtectedTerms, + KeyNonEmptyCorrectedText, + KeySpokenFormPlausibility, + KeyMeaningReversalReview, + }, + "spoken_word": { + KeyNoEffect, + KeyOriginalTextPresence, + KeyConfidenceThreshold, + KeyProtectedTerms, + KeyNonEmptyCorrectedText, + KeySpokenWordReview, + KeyMeaningReversalReview, + }, + "grammar": { + KeyNoEffect, + KeyOriginalTextPresence, + KeyConfidenceThreshold, + KeyProtectedTerms, + KeyNonEmptyCorrectedText, + KeyGrammarReview, + KeyMeaningReversalReview, + }, +} + +func BuiltInChainKeys(moduleKey string) ([]string, error) { + keys, ok := builtInChains[moduleKey] + if !ok { + return nil, fmt.Errorf("no built-in validator chain for module %q", moduleKey) + } + out := make([]string, len(keys)) + copy(out, keys) + return out, nil +} + +func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]contracts.Validator, error) { + keys, err := BuiltInChainKeys(moduleKey) + if err != nil { + return nil, err + } + if registry == nil { + registry = NewBuiltInRegistry() + } + + out := make([]contracts.Validator, 0, len(keys)) + for _, key := range keys { + if moduleKey == "glossary" && key == KeyProtectedTerms { + // Glossary stages preserve current stricter protection semantics while + // reporting the stable protected_terms key. + v := frameworkvalidators.GlossaryStageProtectedGlossaryTermValidator{} + out = append(out, v) + continue + } + v, buildErr := registry.MustBuild(key) + if buildErr != nil { + return nil, buildErr + } + out = append(out, v) + } + return out, nil +} diff --git a/internal/validators/module_chains_test.go b/internal/validators/module_chains_test.go new file mode 100644 index 0000000..3092dd1 --- /dev/null +++ b/internal/validators/module_chains_test.go @@ -0,0 +1,66 @@ +package validators_test + +import ( + "testing" + + "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/internal/modules/glossary" + "gitea.maximumdirect.net/eric/audita/internal/modules/grammar" + "gitea.maximumdirect.net/eric/audita/internal/modules/homophones" + "gitea.maximumdirect.net/eric/audita/internal/modules/spoken_word" + builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators" +) + +func TestProductionModulesUseRegisteredValidatorKeys(t *testing.T) { + registry := builtinvalidators.NewBuiltInRegistry() + keys := map[string]struct{}{} + for _, key := range registry.RegisteredKeys() { + keys[key] = struct{}{} + } + + cases := []struct { + name string + build func() ([]contracts.Validator, error) + }{ + {name: "glossary", build: func() ([]contracts.Validator, error) { + m, err := glossary.New() + if err != nil { + return nil, err + } + return m.Validators(), nil + }}, + {name: "homophones", build: func() ([]contracts.Validator, error) { + m, err := homophones.New() + if err != nil { + return nil, err + } + return m.Validators(), nil + }}, + {name: "spoken_word", build: func() ([]contracts.Validator, error) { + m, err := spoken_word.New() + if err != nil { + return nil, err + } + return m.Validators(), nil + }}, + {name: "grammar", build: func() ([]contracts.Validator, error) { + m, err := grammar.New() + if err != nil { + return nil, err + } + return m.Validators(), nil + }}, + } + + for _, tc := range cases { + validators, err := tc.build() + if err != nil { + t.Fatalf("build module %s: %v", tc.name, err) + } + for _, v := range validators { + if _, ok := keys[v.Name()]; !ok { + t.Fatalf("module %s uses unregistered validator key %q", tc.name, v.Name()) + } + } + } +} diff --git a/internal/validators/registry.go b/internal/validators/registry.go new file mode 100644 index 0000000..f39a9f9 --- /dev/null +++ b/internal/validators/registry.go @@ -0,0 +1,104 @@ +package validators + +import ( + "fmt" + "strings" + + "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" + frameworkvalidators "gitea.maximumdirect.net/eric/audita/internal/framework/validators" +) + +const ( + KeyConfidenceThreshold = "confidence_threshold" + KeyOriginalTextPresence = "original_text_presence" + KeyNonEmptyCorrectedText = "non_empty_corrected_text" + KeyNoEffect = "no_effect" + KeyProtectedTerms = "protected_terms" + + KeySpokenFormPlausibility = "spoken_form_plausibility" + KeyMeaningReversalReview = "meaning_reversal_review" + KeyEditorialReview = "editorial_review" + KeyGrammarReview = "grammar_review" + KeySpokenWordReview = "spoken_word_review" +) + +type BuiltInValidatorDefinition struct { + Key string + Build func() (contracts.Validator, error) + LLMBacked bool +} + +type Registry struct { + definitions map[string]BuiltInValidatorDefinition +} + +func NewBuiltInRegistry() *Registry { + defs := []BuiltInValidatorDefinition{ + {Key: KeyConfidenceThreshold, Build: func() (contracts.Validator, error) { return frameworkvalidators.ConfidenceThresholdValidator{}, nil }}, + {Key: KeyOriginalTextPresence, Build: func() (contracts.Validator, error) { return frameworkvalidators.OriginalTextPresenceValidator{}, nil }}, + {Key: KeyNonEmptyCorrectedText, Build: func() (contracts.Validator, error) { return frameworkvalidators.NonEmptyCorrectionValidator{}, nil }}, + {Key: KeyNoEffect, Build: func() (contracts.Validator, error) { return frameworkvalidators.NoEffectValidator{}, nil }}, + {Key: KeyProtectedTerms, Build: func() (contracts.Validator, error) { return frameworkvalidators.ProtectedGlossaryTermValidator{}, nil }}, + {Key: KeySpokenFormPlausibility, LLMBacked: true, Build: func() (contracts.Validator, error) { + return frameworkvalidators.NewLLMBackedValidator(KeySpokenFormPlausibility, frameworkvalidators.LLMValidatorTypeSpokenFormPlausibility, "") + }}, + {Key: KeyMeaningReversalReview, LLMBacked: true, Build: func() (contracts.Validator, error) { + return frameworkvalidators.NewLLMBackedValidator(KeyMeaningReversalReview, frameworkvalidators.LLMValidatorTypeMeaningReversal, "") + }}, + {Key: KeyEditorialReview, LLMBacked: true, Build: func() (contracts.Validator, error) { + return frameworkvalidators.NewLLMBackedValidator(KeyEditorialReview, frameworkvalidators.LLMValidatorTypeEditorialReview, "") + }}, + {Key: KeyGrammarReview, LLMBacked: true, Build: func() (contracts.Validator, error) { + return frameworkvalidators.NewLLMBackedValidator(KeyGrammarReview, frameworkvalidators.LLMValidatorTypeGrammarReview, "") + }}, + {Key: KeySpokenWordReview, LLMBacked: true, Build: func() (contracts.Validator, error) { + return frameworkvalidators.NewLLMBackedValidator(KeySpokenWordReview, frameworkvalidators.LLMValidatorTypeSpokenWordReview, "") + }}, + } + + m := make(map[string]BuiltInValidatorDefinition, len(defs)) + for _, def := range defs { + m[def.Key] = def + } + return &Registry{definitions: m} +} + +func (r *Registry) Lookup(key string) (BuiltInValidatorDefinition, bool) { + if r == nil { + return BuiltInValidatorDefinition{}, false + } + def, ok := r.definitions[strings.TrimSpace(key)] + return def, ok +} + +func (r *Registry) MustBuild(key string) (contracts.Validator, error) { + if r == nil { + return nil, fmt.Errorf("validator registry is nil") + } + def, ok := r.Lookup(key) + if !ok { + return nil, fmt.Errorf("unknown validator key %q", key) + } + v, err := def.Build() + if err != nil { + return nil, fmt.Errorf("build validator %q: %w", key, err) + } + if v == nil { + return nil, fmt.Errorf("build validator %q: returned nil validator", key) + } + if v.Name() != def.Key { + return nil, fmt.Errorf("validator key/name mismatch for %q: got %q", key, v.Name()) + } + return v, nil +} + +func (r *Registry) RegisteredKeys() []string { + if r == nil { + return nil + } + keys := make([]string, 0, len(r.definitions)) + for k := range r.definitions { + keys = append(keys, k) + } + return keys +} diff --git a/internal/validators/registry_test.go b/internal/validators/registry_test.go new file mode 100644 index 0000000..6021248 --- /dev/null +++ b/internal/validators/registry_test.go @@ -0,0 +1,79 @@ +package validators + +import ( + "testing" +) + +func TestBuiltInRegistryRegistersAllKeys(t *testing.T) { + r := NewBuiltInRegistry() + for _, key := range []string{ + KeyConfidenceThreshold, + KeyOriginalTextPresence, + KeyNonEmptyCorrectedText, + KeyNoEffect, + KeyProtectedTerms, + KeySpokenFormPlausibility, + KeyMeaningReversalReview, + KeyEditorialReview, + KeyGrammarReview, + KeySpokenWordReview, + } { + def, ok := r.Lookup(key) + if !ok { + t.Fatalf("expected validator key %q to be registered", key) + } + v, err := def.Build() + if err != nil { + t.Fatalf("build validator %q: %v", key, err) + } + if v.Name() != key { + t.Fatalf("expected validator name %q, got %q", key, v.Name()) + } + } +} + +func TestBuiltInRegistryUnknownKeyFails(t *testing.T) { + r := NewBuiltInRegistry() + if _, err := r.MustBuild("missing"); err == nil { + t.Fatalf("expected unknown validator key error") + } +} + +func TestBuiltInChainKeysResolveForProductionModules(t *testing.T) { + for _, moduleKey := range []string{"glossary", "homophones", "spoken_word", "grammar"} { + keys, err := BuiltInChainKeys(moduleKey) + if err != nil { + t.Fatalf("resolve keys for %q: %v", moduleKey, err) + } + if len(keys) == 0 { + t.Fatalf("expected non-empty chain for %q", moduleKey) + } + } +} + +func TestResolveBuiltInChainUsesRegisteredKeys(t *testing.T) { + r := NewBuiltInRegistry() + for _, moduleKey := range []string{"glossary", "homophones", "spoken_word", "grammar"} { + chain, err := ResolveBuiltInChain(moduleKey, r) + if err != nil { + t.Fatalf("resolve chain for %q: %v", moduleKey, err) + } + if len(chain) == 0 { + t.Fatalf("expected non-empty chain for %q", moduleKey) + } + for _, v := range chain { + if v == nil { + t.Fatalf("nil validator in %q chain", moduleKey) + } + } + } +} + +func TestBuiltInChainUnknownModuleFails(t *testing.T) { + if _, err := BuiltInChainKeys("unknown"); err == nil { + t.Fatalf("expected unknown module chain failure") + } + if _, err := ResolveBuiltInChain("unknown", NewBuiltInRegistry()); err == nil { + t.Fatalf("expected unknown module chain failure") + } +}