eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Refactor validators into built-in registry chains

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-13 18:36:46 +00:00
parent 1bc5936681
commit d6126bf52b
21 changed files with 559 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
internal/cli/run_test.go
View File

@@ -3048,6 +3048,9 @@ func TestRunProcessExplicitHomophonesProtectedGlossaryTermRejected(t *testing.T)
if module.ValidatorRejected[0].ReasonCode != validators.ReasonProtectedGlossaryTerm {
t.Fatalf("expected protected glossary term rejection, got %+v", module.ValidatorRejected[0])
}
if module.ValidatorRejected[0].ValidatorName != "protected_terms" {
t.Fatalf("expected stable validator key protected_terms, got %+v", module.ValidatorRejected[0])
}
}
func TestRunProcessExplicitHomophonesMalformedLLMOutputFailsWithErrorLog(t *testing.T) {

2
internal/cli/testdata/parity/llm-validator-rejection.case.json vendored
View File

@@ -14,6 +14,6 @@
"total_skipped_changes": 1,
"validator_rejected_reason_codes": ["llm_rejected"],
"expected_proposal_calls": ["grammar:proposal"],
"expected_validation_calls": ["grammar:section-0000:grammar_only_guard:batch-0000"]
"expected_validation_calls": ["grammar:section-0000:grammar_review:batch-0000"]
}
}

6
internal/framework/validators/deterministic.go
View File

@@ -59,7 +59,7 @@ func (v OriginalTextPresenceValidator) Validate(_ context.Context, req Request)
type NonEmptyCorrectionValidator struct{}
func (v NonEmptyCorrectionValidator) Name() string { return "non_empty_correction" }
func (v NonEmptyCorrectionValidator) Name() string { return "non_empty_corrected_text" }
func (v NonEmptyCorrectionValidator) Validate(_ context.Context, req Request) (Result, error) {
decisions := make([]Decision, 0, len(req.CandidateProposal))
@@ -97,7 +97,7 @@ func (v NoEffectValidator) Validate(_ context.Context, req Request) (Result, err
type ProtectedGlossaryTermValidator struct{}
func (v ProtectedGlossaryTermValidator) Name() string { return "protected_glossary_terms" }
func (v ProtectedGlossaryTermValidator) Name() string { return "protected_terms" }
func (v ProtectedGlossaryTermValidator) Validate(_ context.Context, req Request) (Result, error) {
if req.ModuleKey == "glossary" {
@@ -127,7 +127,7 @@ func (v ProtectedGlossaryTermValidator) Validate(_ context.Context, req Request)
type GlossaryStageProtectedGlossaryTermValidator struct{}
func (v GlossaryStageProtectedGlossaryTermValidator) Name() string {
return "glossary_stage_protected_glossary_terms"
return "protected_terms"
}
func (v GlossaryStageProtectedGlossaryTermValidator) Validate(_ context.Context, req Request) (Result, error) {

19
internal/modules/glossary/module.go
View File

@@ -8,7 +8,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
)
type Module struct {
@@ -16,25 +16,12 @@ type Module struct {
}
func New() (*Module, error) {
spokenForm, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
chain, err := builtinvalidators.ResolveBuiltInChain("glossary", builtinvalidators.NewBuiltInRegistry())
if err != nil {
return nil, err
}
meaningReversal, err := validators.NewLLMBackedValidator("meaning_reversal_review", validators.LLMValidatorTypeMeaningReversal, "")
if err != nil {
return nil, err
}
return &Module{
validators: []contracts.Validator{
validators.NoEffectValidator{},
validators.OriginalTextPresenceValidator{},
validators.ConfidenceThresholdValidator{},
validators.GlossaryStageProtectedGlossaryTermValidator{},
validators.NonEmptyCorrectionValidator{},
spokenForm,
meaningReversal,
},
validators: chain,
}, nil
}

6
internal/modules/glossary/module_test.go
View File

@@ -132,9 +132,9 @@ func TestGlossaryModuleValidatorChain(t *testing.T) {
"no_effect",
"original_text_presence",
"confidence_threshold",
"glossary_stage_protected_glossary_terms",
"non_empty_correction",
"spoken_form_plausibility_review",
"protected_terms",
"non_empty_corrected_text",
"spoken_form_plausibility",
"meaning_reversal_review",
}
if strings.Join(got, ",") != strings.Join(want, ",") {

19
internal/modules/grammar/module.go
View File

@@ -8,7 +8,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
)
type Module struct {
@@ -16,25 +16,12 @@ type Module struct {
}
func New() (*Module, error) {
grammarOnlyGuard, err := validators.NewLLMBackedValidator("grammar_only_guard", validators.LLMValidatorTypeGrammarReview, "")
chain, err := builtinvalidators.ResolveBuiltInChain("grammar", builtinvalidators.NewBuiltInRegistry())
if err != nil {
return nil, err
}
meaningReversal, err := validators.NewLLMBackedValidator("meaning_reversal_review", validators.LLMValidatorTypeMeaningReversal, "")
if err != nil {
return nil, err
}
return &Module{
validators: []contracts.Validator{
validators.NoEffectValidator{},
validators.OriginalTextPresenceValidator{},
validators.ConfidenceThresholdValidator{},
validators.ProtectedGlossaryTermValidator{},
validators.NonEmptyCorrectionValidator{},
grammarOnlyGuard,
meaningReversal,
},
validators: chain,
}, nil
}

6
internal/modules/grammar/module_test.go
View File

@@ -134,9 +134,9 @@ func TestGrammarModuleValidatorChain(t *testing.T) {
"no_effect",
"original_text_presence",
"confidence_threshold",
"protected_glossary_terms",
"non_empty_correction",
"grammar_only_guard",
"protected_terms",
"non_empty_corrected_text",
"grammar_review",
"meaning_reversal_review",
}
if strings.Join(got, ",") != strings.Join(want, ",") {

19
internal/modules/homophones/module.go
View File

@@ -8,7 +8,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
)
type Module struct {
@@ -16,25 +16,12 @@ type Module struct {
}
func New() (*Module, error) {
spokenForm, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
chain, err := builtinvalidators.ResolveBuiltInChain("homophones", builtinvalidators.NewBuiltInRegistry())
if err != nil {
return nil, err
}
meaningReversal, err := validators.NewLLMBackedValidator("meaning_reversal_review", validators.LLMValidatorTypeMeaningReversal, "")
if err != nil {
return nil, err
}
return &Module{
validators: []contracts.Validator{
validators.NoEffectValidator{},
validators.OriginalTextPresenceValidator{},
validators.ConfidenceThresholdValidator{},
validators.ProtectedGlossaryTermValidator{},
validators.NonEmptyCorrectionValidator{},
spokenForm,
meaningReversal,
},
validators: chain,
}, nil
}

10
internal/modules/homophones/module_test.go
View File

@@ -148,9 +148,9 @@ func TestHomophonesModuleValidatorChain(t *testing.T) {
"no_effect",
"original_text_presence",
"confidence_threshold",
"protected_glossary_terms",
"non_empty_correction",
"spoken_form_plausibility_review",
"protected_terms",
"non_empty_corrected_text",
"spoken_form_plausibility",
"meaning_reversal_review",
}
if strings.Join(got, ",") != strings.Join(want, ",") {
@@ -276,13 +276,13 @@ func TestHomophonesProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi
}
var protectedValidator contracts.Validator
for _, v := range module.Validators() {
if v.Name() == "protected_glossary_terms" {
if v.Name() == "protected_terms" {
protectedValidator = v
break
}
}
if protectedValidator == nil {
t.Fatal("expected protected_glossary_terms validator")
t.Fatal("expected protected_terms validator")
}
working := &schema.Transcript{Segments: []schema.Segment{

19
internal/modules/spoken_word/module.go
View File

@@ -8,7 +8,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
)
type Module struct {
@@ -16,25 +16,12 @@ type Module struct {
}
func New() (*Module, error) {
spokenWordReview, err := validators.NewLLMBackedValidator("spoken_word_review", validators.LLMValidatorTypeSpokenWordReview, "")
chain, err := builtinvalidators.ResolveBuiltInChain("spoken_word", builtinvalidators.NewBuiltInRegistry())
if err != nil {
return nil, err
}
meaningReversal, err := validators.NewLLMBackedValidator("meaning_reversal_review", validators.LLMValidatorTypeMeaningReversal, "")
if err != nil {
return nil, err
}
return &Module{
validators: []contracts.Validator{
validators.NoEffectValidator{},
validators.OriginalTextPresenceValidator{},
validators.ConfidenceThresholdValidator{},
validators.ProtectedGlossaryTermValidator{},
validators.NonEmptyCorrectionValidator{},
spokenWordReview,
meaningReversal,
},
validators: chain,
}, nil
}

8
internal/modules/spoken_word/module_test.go
View File

@@ -136,8 +136,8 @@ func TestSpokenWordModuleValidatorChain(t *testing.T) {
"no_effect",
"original_text_presence",
"confidence_threshold",
"protected_glossary_terms",
"non_empty_correction",
"protected_terms",
"non_empty_corrected_text",
"spoken_word_review",
"meaning_reversal_review",
}
@@ -264,13 +264,13 @@ func TestSpokenWordProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi
}
var protectedValidator contracts.Validator
for _, v := range module.Validators() {
if v.Name() == "protected_glossary_terms" {
if v.Name() == "protected_terms" {
protectedValidator = v
break
}
}
if protectedValidator == nil {
t.Fatal("expected protected_glossary_terms validator")
t.Fatal("expected protected_terms validator")
}
working := &schema.Transcript{Segments: []schema.Segment{

84
internal/validators/chains.go Normal file
View File

@@ -0,0 +1,84 @@
package validators
import (
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
var builtInChains = map[string][]string{
"glossary": {
KeyNoEffect,
KeyOriginalTextPresence,
KeyConfidenceThreshold,
KeyProtectedTerms,
KeyNonEmptyCorrectedText,
KeySpokenFormPlausibility,
KeyMeaningReversalReview,
},
"homophones": {
KeyNoEffect,
KeyOriginalTextPresence,
KeyConfidenceThreshold,
KeyProtectedTerms,
KeyNonEmptyCorrectedText,
KeySpokenFormPlausibility,
KeyMeaningReversalReview,
},
"spoken_word": {
KeyNoEffect,
KeyOriginalTextPresence,
KeyConfidenceThreshold,
KeyProtectedTerms,
KeyNonEmptyCorrectedText,
KeySpokenWordReview,
KeyMeaningReversalReview,
},
"grammar": {
KeyNoEffect,
KeyOriginalTextPresence,
KeyConfidenceThreshold,
KeyProtectedTerms,
KeyNonEmptyCorrectedText,
KeyGrammarReview,
KeyMeaningReversalReview,
},
}
func BuiltInChainKeys(moduleKey string) ([]string, error) {
keys, ok := builtInChains[moduleKey]
if !ok {
return nil, fmt.Errorf("no built-in validator chain for module %q", moduleKey)
}
out := make([]string, len(keys))
copy(out, keys)
return out, nil
}
func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]contracts.Validator, error) {
keys, err := BuiltInChainKeys(moduleKey)
if err != nil {
return nil, err
}
if registry == nil {
registry = NewBuiltInRegistry()
}
out := make([]contracts.Validator, 0, len(keys))
for _, key := range keys {
if moduleKey == "glossary" && key == KeyProtectedTerms {
// Glossary stages preserve current stricter protection semantics while
// reporting the stable protected_terms key.
v := frameworkvalidators.GlossaryStageProtectedGlossaryTermValidator{}
out = append(out, v)
continue
}
v, buildErr := registry.MustBuild(key)
if buildErr != nil {
return nil, buildErr
}
out = append(out, v)
}
return out, nil
}

66
internal/validators/module_chains_test.go Normal file
View File

@@ -0,0 +1,66 @@
package validators_test
import (
"testing"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/modules/glossary"
"gitea.maximumdirect.net/eric/audita/internal/modules/grammar"
"gitea.maximumdirect.net/eric/audita/internal/modules/homophones"
"gitea.maximumdirect.net/eric/audita/internal/modules/spoken_word"
builtinvalidators "gitea.maximumdirect.net/eric/audita/internal/validators"
)
func TestProductionModulesUseRegisteredValidatorKeys(t *testing.T) {
registry := builtinvalidators.NewBuiltInRegistry()
keys := map[string]struct{}{}
for _, key := range registry.RegisteredKeys() {
keys[key] = struct{}{}
}
cases := []struct {
name string
build func() ([]contracts.Validator, error)
}{
{name: "glossary", build: func() ([]contracts.Validator, error) {
m, err := glossary.New()
if err != nil {
return nil, err
}
return m.Validators(), nil
}},
{name: "homophones", build: func() ([]contracts.Validator, error) {
m, err := homophones.New()
if err != nil {
return nil, err
}
return m.Validators(), nil
}},
{name: "spoken_word", build: func() ([]contracts.Validator, error) {
m, err := spoken_word.New()
if err != nil {
return nil, err
}
return m.Validators(), nil
}},
{name: "grammar", build: func() ([]contracts.Validator, error) {
m, err := grammar.New()
if err != nil {
return nil, err
}
return m.Validators(), nil
}},
}
for _, tc := range cases {
validators, err := tc.build()
if err != nil {
t.Fatalf("build module %s: %v", tc.name, err)
}
for _, v := range validators {
if _, ok := keys[v.Name()]; !ok {
t.Fatalf("module %s uses unregistered validator key %q", tc.name, v.Name())
}
}
}
}

104
internal/validators/registry.go Normal file
View File

@@ -0,0 +1,104 @@
package validators
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
const (
KeyConfidenceThreshold = "confidence_threshold"
KeyOriginalTextPresence = "original_text_presence"
KeyNonEmptyCorrectedText = "non_empty_corrected_text"
KeyNoEffect = "no_effect"
KeyProtectedTerms = "protected_terms"
KeySpokenFormPlausibility = "spoken_form_plausibility"
KeyMeaningReversalReview = "meaning_reversal_review"
KeyEditorialReview = "editorial_review"
KeyGrammarReview = "grammar_review"
KeySpokenWordReview = "spoken_word_review"
)
type BuiltInValidatorDefinition struct {
Key string
Build func() (contracts.Validator, error)
LLMBacked bool
}
type Registry struct {
definitions map[string]BuiltInValidatorDefinition
}
func NewBuiltInRegistry() *Registry {
defs := []BuiltInValidatorDefinition{
{Key: KeyConfidenceThreshold, Build: func() (contracts.Validator, error) { return frameworkvalidators.ConfidenceThresholdValidator{}, nil }},
{Key: KeyOriginalTextPresence, Build: func() (contracts.Validator, error) { return frameworkvalidators.OriginalTextPresenceValidator{}, nil }},
{Key: KeyNonEmptyCorrectedText, Build: func() (contracts.Validator, error) { return frameworkvalidators.NonEmptyCorrectionValidator{}, nil }},
{Key: KeyNoEffect, Build: func() (contracts.Validator, error) { return frameworkvalidators.NoEffectValidator{}, nil }},
{Key: KeyProtectedTerms, Build: func() (contracts.Validator, error) { return frameworkvalidators.ProtectedGlossaryTermValidator{}, nil }},
{Key: KeySpokenFormPlausibility, LLMBacked: true, Build: func() (contracts.Validator, error) {
return frameworkvalidators.NewLLMBackedValidator(KeySpokenFormPlausibility, frameworkvalidators.LLMValidatorTypeSpokenFormPlausibility, "")
}},
{Key: KeyMeaningReversalReview, LLMBacked: true, Build: func() (contracts.Validator, error) {
return frameworkvalidators.NewLLMBackedValidator(KeyMeaningReversalReview, frameworkvalidators.LLMValidatorTypeMeaningReversal, "")
}},
{Key: KeyEditorialReview, LLMBacked: true, Build: func() (contracts.Validator, error) {
return frameworkvalidators.NewLLMBackedValidator(KeyEditorialReview, frameworkvalidators.LLMValidatorTypeEditorialReview, "")
}},
{Key: KeyGrammarReview, LLMBacked: true, Build: func() (contracts.Validator, error) {
return frameworkvalidators.NewLLMBackedValidator(KeyGrammarReview, frameworkvalidators.LLMValidatorTypeGrammarReview, "")
}},
{Key: KeySpokenWordReview, LLMBacked: true, Build: func() (contracts.Validator, error) {
return frameworkvalidators.NewLLMBackedValidator(KeySpokenWordReview, frameworkvalidators.LLMValidatorTypeSpokenWordReview, "")
}},
}
m := make(map[string]BuiltInValidatorDefinition, len(defs))
for _, def := range defs {
m[def.Key] = def
}
return &Registry{definitions: m}
}
func (r *Registry) Lookup(key string) (BuiltInValidatorDefinition, bool) {
if r == nil {
return BuiltInValidatorDefinition{}, false
}
def, ok := r.definitions[strings.TrimSpace(key)]
return def, ok
}
func (r *Registry) MustBuild(key string) (contracts.Validator, error) {
if r == nil {
return nil, fmt.Errorf("validator registry is nil")
}
def, ok := r.Lookup(key)
if !ok {
return nil, fmt.Errorf("unknown validator key %q", key)
}
v, err := def.Build()
if err != nil {
return nil, fmt.Errorf("build validator %q: %w", key, err)
}
if v == nil {
return nil, fmt.Errorf("build validator %q: returned nil validator", key)
}
if v.Name() != def.Key {
return nil, fmt.Errorf("validator key/name mismatch for %q: got %q", key, v.Name())
}
return v, nil
}
func (r *Registry) RegisteredKeys() []string {
if r == nil {
return nil
}
keys := make([]string, 0, len(r.definitions))
for k := range r.definitions {
keys = append(keys, k)
}
return keys
}

79
internal/validators/registry_test.go Normal file
View File

@@ -0,0 +1,79 @@
package validators
import (
"testing"
)
func TestBuiltInRegistryRegistersAllKeys(t *testing.T) {
r := NewBuiltInRegistry()
for _, key := range []string{
KeyConfidenceThreshold,
KeyOriginalTextPresence,
KeyNonEmptyCorrectedText,
KeyNoEffect,
KeyProtectedTerms,
KeySpokenFormPlausibility,
KeyMeaningReversalReview,
KeyEditorialReview,
KeyGrammarReview,
KeySpokenWordReview,
} {
def, ok := r.Lookup(key)
if !ok {
t.Fatalf("expected validator key %q to be registered", key)
}
v, err := def.Build()
if err != nil {
t.Fatalf("build validator %q: %v", key, err)
}
if v.Name() != key {
t.Fatalf("expected validator name %q, got %q", key, v.Name())
}
}
}
func TestBuiltInRegistryUnknownKeyFails(t *testing.T) {
r := NewBuiltInRegistry()
if _, err := r.MustBuild("missing"); err == nil {
t.Fatalf("expected unknown validator key error")
}
}
func TestBuiltInChainKeysResolveForProductionModules(t *testing.T) {
for _, moduleKey := range []string{"glossary", "homophones", "spoken_word", "grammar"} {
keys, err := BuiltInChainKeys(moduleKey)
if err != nil {
t.Fatalf("resolve keys for %q: %v", moduleKey, err)
}
if len(keys) == 0 {
t.Fatalf("expected non-empty chain for %q", moduleKey)
}
}
}
func TestResolveBuiltInChainUsesRegisteredKeys(t *testing.T) {
r := NewBuiltInRegistry()
for _, moduleKey := range []string{"glossary", "homophones", "spoken_word", "grammar"} {
chain, err := ResolveBuiltInChain(moduleKey, r)
if err != nil {
t.Fatalf("resolve chain for %q: %v", moduleKey, err)
}
if len(chain) == 0 {
t.Fatalf("expected non-empty chain for %q", moduleKey)
}
for _, v := range chain {
if v == nil {
t.Fatalf("nil validator in %q chain", moduleKey)
}
}
}
}
func TestBuiltInChainUnknownModuleFails(t *testing.T) {
if _, err := BuiltInChainKeys("unknown"); err == nil {
t.Fatalf("expected unknown module chain failure")
}
if _, err := ResolveBuiltInChain("unknown", NewBuiltInRegistry()); err == nil {
t.Fatalf("expected unknown module chain failure")
}
}