eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Refactor validators into package-owned components

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-14 00:28:28 +00:00
parent 3b160cf05b
commit 52ffe42e73
26 changed files with 775 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
internal/framework/runner/observability.go
View File

@@ -8,7 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
)
type UtilizationDiagnostics struct {
@@ -337,11 +337,7 @@ func withModuleInstanceContext(ctx context.Context, moduleInstance string) conte
}
func isLLMBackedValidator(v contracts.Validator) bool {
if v == nil {
return false
}
_, ok := v.(*frameworkvalidators.LLMBackedValidator)
return ok
return validatormetadata.ClassOf(v) == validatormetadata.ExecutionClassLLMBacked
}
func sortValidatorSummaries(in []ValidatorTimingSummary) {

3
internal/framework/runner/runner.go
View File

@@ -15,6 +15,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
)
const (
@@ -568,7 +569,7 @@ func reorderValidatorsForPipeline(in []contracts.Validator) ([]contracts.Validat
deterministic := make([]contracts.Validator, 0, len(in))
llmBacked := make([]contracts.Validator, 0, len(in))
for _, validator := range in {
if _, ok := validator.(*validators.LLMBackedValidator); ok {
if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked {
llmBacked = append(llmBacked, validator)
continue
}

53
internal/framework/runner/runner_test.go
View File

@@ -22,6 +22,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
)
type fakeFactory struct {
@@ -64,6 +65,58 @@ func (v fakeValidator) Validate(ctx context.Context, req contracts.ValidationReq
return v.validateF(req)
}
type classifiedFakeValidator struct {
fakeValidator
class validatormetadata.ExecutionClass
}
func (v classifiedFakeValidator) ExecutionClass() validatormetadata.ExecutionClass {
return v.class
}
func TestReorderValidatorsDeterministicBeforeLLMBacked(t *testing.T) {
llm := classifiedFakeValidator{
fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
return validators.Result{ValidatorName: "llm"}, nil
}},
class: validatormetadata.ExecutionClassLLMBacked,
}
deterministic := classifiedFakeValidator{
fakeValidator: fakeValidator{name: "deterministic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
return validators.Result{ValidatorName: "deterministic"}, nil
}},
class: validatormetadata.ExecutionClassDeterministic,
}
ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, deterministic})
if len(ordered) != 2 {
t.Fatalf("expected 2 validators, got %d", len(ordered))
}
if ordered[0].Name() != "deterministic" || ordered[1].Name() != "llm" {
t.Fatalf("unexpected validator order: %s, %s", ordered[0].Name(), ordered[1].Name())
}
}
func TestReorderValidatorsDefaultsUnclassifiedToDeterministic(t *testing.T) {
unclassified := fakeValidator{name: "plain", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
return validators.Result{ValidatorName: "plain"}, nil
}}
llm := classifiedFakeValidator{
fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
return validators.Result{ValidatorName: "llm"}, nil
}},
class: validatormetadata.ExecutionClassLLMBacked,
}
ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, unclassified})
if len(ordered) != 2 {
t.Fatalf("expected 2 validators, got %d", len(ordered))
}
if ordered[0].Name() != "plain" || ordered[1].Name() != "llm" {
t.Fatalf("unexpected validator order with unclassified validator: %s, %s", ordered[0].Name(), ordered[1].Name())
}
}
func TestRunnerOneModuleAppliesProposal(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{