eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Refactor validators into package-owned components

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-14 00:28:28 +00:00
parent 3b160cf05b
commit 52ffe42e73
26 changed files with 775 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
internal/framework/runner/observability.go
View File

@@ -8,7 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
)
type UtilizationDiagnostics struct {
@@ -337,11 +337,7 @@ func withModuleInstanceContext(ctx context.Context, moduleInstance string) conte
}
func isLLMBackedValidator(v contracts.Validator) bool {
if v == nil {
return false
}
_, ok := v.(*frameworkvalidators.LLMBackedValidator)
return ok
return validatormetadata.ClassOf(v) == validatormetadata.ExecutionClassLLMBacked
}
func sortValidatorSummaries(in []ValidatorTimingSummary) {

3
internal/framework/runner/runner.go
View File

@@ -15,6 +15,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
)
const (
@@ -568,7 +569,7 @@ func reorderValidatorsForPipeline(in []contracts.Validator) ([]contracts.Validat
deterministic := make([]contracts.Validator, 0, len(in))
llmBacked := make([]contracts.Validator, 0, len(in))
for _, validator := range in {
if _, ok := validator.(*validators.LLMBackedValidator); ok {
if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked {
llmBacked = append(llmBacked, validator)
continue
}

53
internal/framework/runner/runner_test.go
View File

@@ -22,6 +22,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
)
type fakeFactory struct {
@@ -64,6 +65,58 @@ func (v fakeValidator) Validate(ctx context.Context, req contracts.ValidationReq
return v.validateF(req)
}
type classifiedFakeValidator struct {
fakeValidator
class validatormetadata.ExecutionClass
}
func (v classifiedFakeValidator) ExecutionClass() validatormetadata.ExecutionClass {
return v.class
}
func TestReorderValidatorsDeterministicBeforeLLMBacked(t *testing.T) {
llm := classifiedFakeValidator{
fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
return validators.Result{ValidatorName: "llm"}, nil
}},
class: validatormetadata.ExecutionClassLLMBacked,
}
deterministic := classifiedFakeValidator{
fakeValidator: fakeValidator{name: "deterministic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
return validators.Result{ValidatorName: "deterministic"}, nil
}},
class: validatormetadata.ExecutionClassDeterministic,
}
ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, deterministic})
if len(ordered) != 2 {
t.Fatalf("expected 2 validators, got %d", len(ordered))
}
if ordered[0].Name() != "deterministic" || ordered[1].Name() != "llm" {
t.Fatalf("unexpected validator order: %s, %s", ordered[0].Name(), ordered[1].Name())
}
}
func TestReorderValidatorsDefaultsUnclassifiedToDeterministic(t *testing.T) {
unclassified := fakeValidator{name: "plain", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
return validators.Result{ValidatorName: "plain"}, nil
}}
llm := classifiedFakeValidator{
fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
return validators.Result{ValidatorName: "llm"}, nil
}},
class: validatormetadata.ExecutionClassLLMBacked,
}
ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, unclassified})
if len(ordered) != 2 {
t.Fatalf("expected 2 validators, got %d", len(ordered))
}
if ordered[0].Name() != "plain" || ordered[1].Name() != "llm" {
t.Fatalf("unexpected validator order with unclassified validator: %s, %s", ordered[0].Name(), ordered[1].Name())
}
}
func TestRunnerOneModuleAppliesProposal(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{

46
internal/framework/validators/deterministic.go
View File

@@ -95,23 +95,16 @@ func (v NoEffectValidator) Validate(_ context.Context, req Request) (Result, err
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
}
type ProtectedGlossaryTermValidator struct{}
func (v ProtectedGlossaryTermValidator) Name() string { return "protected_terms" }
func (v ProtectedGlossaryTermValidator) Validate(_ context.Context, req Request) (Result, error) {
if req.ModuleKey == "glossary" {
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
decisions = append(decisions, approval(c.ProposalIndex))
}
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
}
func ValidateProtectedTerms(req Request, glossaryStage bool) (Result, error) {
vocab := NewProtectedVocabulary(req.Glossary)
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
reason := vocab.violationReason(c.OriginalText, c.CorrectedText)
var reason string
if glossaryStage {
reason = vocab.glossaryStageViolationReason(c.OriginalText, c.CorrectedText)
} else {
reason = vocab.violationReason(c.OriginalText, c.CorrectedText)
}
if reason != "" {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonProtectedGlossaryTerm, reason))
continue
@@ -121,28 +114,5 @@ func (v ProtectedGlossaryTermValidator) Validate(_ context.Context, req Request)
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
return Result{}, err
}
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
}
type GlossaryStageProtectedGlossaryTermValidator struct{}
func (v GlossaryStageProtectedGlossaryTermValidator) Name() string {
return "protected_terms"
}
func (v GlossaryStageProtectedGlossaryTermValidator) Validate(_ context.Context, req Request) (Result, error) {
vocab := NewProtectedVocabulary(req.Glossary)
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
reason := vocab.glossaryStageViolationReason(c.OriginalText, c.CorrectedText)
if reason != "" {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonProtectedGlossaryTerm, reason))
continue
}
decisions = append(decisions, approval(c.ProposalIndex))
}
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
return Result{}, err
}
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
return Result{ValidatorName: "protected_terms", Decisions: decisions}, nil
}

9
internal/framework/validators/protected_terms_test.go
View File

@@ -1,7 +1,6 @@
package validators
import (
"context"
"reflect"
"testing"
@@ -31,7 +30,7 @@ func TestExtractProtectedTermsEmptyGlossary(t *testing.T) {
}
}
func TestProtectedGlossaryTermValidatorAppliesToNonGlossaryModule(t *testing.T) {
func TestValidateProtectedTermsAppliesToNonGlossaryModule(t *testing.T) {
req := Request{
ModuleKey: "grammar",
Glossary: &schema.Glossary{
@@ -49,7 +48,7 @@ func TestProtectedGlossaryTermValidatorAppliesToNonGlossaryModule(t *testing.T)
},
},
}
res, err := (ProtectedGlossaryTermValidator{}).Validate(context.Background(), req)
res, err := ValidateProtectedTerms(req, false)
if err != nil {
t.Fatalf("validator error: %v", err)
}
@@ -58,7 +57,7 @@ func TestProtectedGlossaryTermValidatorAppliesToNonGlossaryModule(t *testing.T)
}
}
func TestGlossaryStageProtectedGlossaryTermValidatorAllowsProtectedTermSwap(t *testing.T) {
func TestValidateProtectedTermsGlossaryStageAllowsProtectedTermSwap(t *testing.T) {
req := Request{
ModuleKey: "glossary",
Glossary: &schema.Glossary{
@@ -76,7 +75,7 @@ func TestGlossaryStageProtectedGlossaryTermValidatorAllowsProtectedTermSwap(t *t
},
},
}
res, err := (GlossaryStageProtectedGlossaryTermValidator{}).Validate(context.Background(), req)
res, err := ValidateProtectedTerms(req, true)
if err != nil {
t.Fatalf("validator error: %v", err)
}

22
internal/framework/validators/validators_test.go
View File

@@ -123,7 +123,7 @@ func TestNoEffectValidator(t *testing.T) {
}
}
func TestProtectedGlossaryTermValidator(t *testing.T) {
func TestValidateProtectedTerms(t *testing.T) {
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "OpenAI", Aliases: []string{"Open AI"}, Plural: "OpenAIs", Category: "brand", Summary: "brand"}}}
req := Request{
Glossary: glossary,
@@ -133,7 +133,7 @@ func TestProtectedGlossaryTermValidator(t *testing.T) {
mkCandidate(1, 1, "teh", "the", 0.9),
},
}
res, err := (ProtectedGlossaryTermValidator{}).Validate(context.Background(), req)
res, err := ValidateProtectedTerms(req, false)
if err != nil {
t.Fatalf("Validate error: %v", err)
}
@@ -145,24 +145,6 @@ func TestProtectedGlossaryTermValidator(t *testing.T) {
}
}
func TestProtectedGlossaryTermValidatorAllowsGlossaryModule(t *testing.T) {
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "OpenAI", Category: "brand", Summary: "brand"}}}
req := Request{
Glossary: glossary,
ModuleKey: "glossary",
CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "OpenAI", "Open A I", 0.9),
},
}
res, err := (ProtectedGlossaryTermValidator{}).Validate(context.Background(), req)
if err != nil {
t.Fatalf("Validate error: %v", err)
}
if !res.Decisions[0].Approved {
t.Fatalf("expected glossary module approval, got %+v", res.Decisions[0])
}
}
func TestStableReasonCodes(t *testing.T) {
codes := []string{
ReasonApproved,