eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace validator contracts with raw output validation

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-07 20:55:49 +00:00
parent 249e49c928
commit b7ad66f0e0
13 changed files with 161 additions and 672 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
internal/framework/contracts/composition_test.go
View File

@@ -255,20 +255,15 @@ func (validator compositionValidator) Name() string {
return "generic-validator"
}
func (validator compositionValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates))
for _, candidate := range req.Candidates {
decisions = append(decisions, contracts.ValidationDecision{
CandidateIndex: candidate.Index,
Approved: true,
ReasonCode: "accepted",
Message: "candidate accepted",
})
}
func (validator compositionValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (validator compositionValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{
ValidatorName: validator.Name(),
Decisions: decisions,
Approved: true,
ReasonCode: "accepted",
Message: "output accepted",
}, nil
}

73
internal/framework/contracts/contracts.go
View File

@@ -203,20 +203,37 @@ type RawPayload struct {
Warnings []Warning `json:"warnings,omitempty"`
}
type RawValidationRequest struct {
Stage string `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key"`
Source *source.SourceDocument `json:"-"`
SourceID string `json:"source_id,omitempty"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
Schema ResponseSchema `json:"schema,omitempty"`
Payload RawPayload `json:"payload"`
Metadata map[string]any `json:"metadata,omitempty"`
type ExecutionClass string
const (
ExecutionClassDeterministic ExecutionClass = "deterministic"
ExecutionClassLLMBacked ExecutionClass = "llm_backed"
)
type ValidationRequest struct {
Stage string `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key"`
Source *source.SourceDocument `json:"-"`
SourceID string `json:"source_id,omitempty"`
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
SessionID string `json:"session_id,omitempty"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
Schema ResponseSchema `json:"schema,omitempty"`
Payload RawPayload `json:"payload"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
Chunk *SourceChunk `json:"chunk,omitempty"`
Chunks []SourceChunk `json:"chunks,omitempty"`
ExtractOutputs []ExtractOutput `json:"extract_outputs,omitempty"`
MergeOutput MergeOutput `json:"merge_output,omitempty"`
}
type RawValidationResult struct {
type ValidationResult struct {
Approved bool `json:"approved"`
ReasonCode string `json:"reason_code,omitempty"`
Message string `json:"message,omitempty"`
@@ -224,9 +241,10 @@ type RawValidationResult struct {
Warnings []Warning `json:"warnings,omitempty"`
}
type RawValidator interface {
type Validator interface {
Name() string
ValidateRaw(ctx context.Context, req RawValidationRequest) (RawValidationResult, error)
ExecutionClass() ExecutionClass
Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error)
}
type ResponseSchema struct {
@@ -308,33 +326,6 @@ type Normalizer interface {
Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error)
}
type ValidationRequest struct {
Source *source.SourceDocument `json:"-"`
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type ValidationDecision struct {
CandidateIndex int `json:"candidate_index"`
Approved bool `json:"approved"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
}
type ValidationResult struct {
ValidatorName string `json:"validator_name"`
Decisions []ValidationDecision `json:"decisions"`
Warnings []Warning `json:"warnings,omitempty"`
}
type Validator interface {
Name() string
Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error)
}
type Warning struct {
Scope string `json:"scope,omitempty"`
ReasonCode string `json:"reason_code"`

19
internal/framework/contracts/contracts_test.go
View File

@@ -577,20 +577,15 @@ func (validator fakeValidator) Name() string {
return validator.name
}
func (validator fakeValidator) Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error) {
decisions := make([]ValidationDecision, 0, len(req.Candidates))
for _, candidate := range req.Candidates {
decisions = append(decisions, ValidationDecision{
CandidateIndex: candidate.Index,
Approved: true,
ReasonCode: "accepted",
Message: "candidate accepted",
})
}
func (validator fakeValidator) ExecutionClass() ExecutionClass {
return ExecutionClassDeterministic
}
func (validator fakeValidator) Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error) {
return ValidationResult{
ValidatorName: validator.name,
Decisions: decisions,
Approved: true,
ReasonCode: "accepted",
Message: "output accepted",
}, nil
}

8
internal/framework/pipeline/chunker_registry_test.go
View File

@@ -386,6 +386,10 @@ func (validator registryValidator) Name() string {
return validator.name
}
func (validator registryValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{}, nil
func (validator registryValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (validator registryValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}

21
internal/framework/pipeline/raw_validation_registry.go
View File

@@ -13,16 +13,16 @@ type rawValidationKey struct {
}
type RawValidationRegistry struct {
chains map[rawValidationKey][]contracts.RawValidator
chains map[rawValidationKey][]contracts.Validator
}
func NewRawValidationRegistry() *RawValidationRegistry {
return &RawValidationRegistry{
chains: make(map[rawValidationKey][]contracts.RawValidator),
chains: make(map[rawValidationKey][]contracts.Validator),
}
}
func (r *RawValidationRegistry) Register(stage ModuleStage, module string, validators ...contracts.RawValidator) error {
func (r *RawValidationRegistry) Register(stage ModuleStage, module string, validators ...contracts.Validator) error {
if r == nil {
return fmt.Errorf("raw validation registry must not be nil")
}
@@ -39,7 +39,7 @@ func (r *RawValidationRegistry) Register(stage ModuleStage, module string, valid
return fmt.Errorf("raw validation chain for %q %q must not be empty", stage, normalizedModule)
}
chain := make([]contracts.RawValidator, 0, len(validators))
chain := make([]contracts.Validator, 0, len(validators))
for i, validator := range validators {
if validator == nil {
return fmt.Errorf("raw validator %d for %q %q must not be nil", i, stage, normalizedModule)
@@ -47,21 +47,26 @@ func (r *RawValidationRegistry) Register(stage ModuleStage, module string, valid
if strings.TrimSpace(validator.Name()) == "" {
return fmt.Errorf("raw validator %d for %q %q must not have an empty name", i, stage, normalizedModule)
}
switch validator.ExecutionClass() {
case contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked:
default:
return fmt.Errorf("raw validator %q for %q %q has unsupported execution class %q", validator.Name(), stage, normalizedModule, validator.ExecutionClass())
}
chain = append(chain, validator)
}
if r.chains == nil {
r.chains = make(map[rawValidationKey][]contracts.RawValidator)
r.chains = make(map[rawValidationKey][]contracts.Validator)
}
key := rawValidationKey{stage: stage, module: normalizedModule}
if _, exists := r.chains[key]; exists {
return fmt.Errorf("raw validation chain for %q %q is already registered", stage, normalizedModule)
}
r.chains[key] = append([]contracts.RawValidator(nil), chain...)
r.chains[key] = append([]contracts.Validator(nil), chain...)
return nil
}
func (r *RawValidationRegistry) Validators(stage ModuleStage, module string) []contracts.RawValidator {
func (r *RawValidationRegistry) Validators(stage ModuleStage, module string) []contracts.Validator {
if r == nil {
return nil
}
@@ -69,5 +74,5 @@ func (r *RawValidationRegistry) Validators(stage ModuleStage, module string) []c
if len(chain) == 0 {
return nil
}
return append([]contracts.RawValidator(nil), chain...)
return append([]contracts.Validator(nil), chain...)
}

4
internal/framework/pipeline/runner.go
View File

@@ -461,7 +461,7 @@ func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([
return nil, nil, nil
}
request := contracts.RawValidationRequest{
request := contracts.ValidationRequest{
Stage: string(target.stage),
LaneID: target.laneID,
ModuleKey: target.moduleKey,
@@ -476,7 +476,7 @@ func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([
var warnings []contracts.Warning
for _, validator := range validators {
result, err := validator.ValidateRaw(ctx, request)
result, err := validator.Validate(ctx, request)
if err != nil {
return nil, nil, fmt.Errorf("validate raw %s output with validator %q: %w", target.stage, validator.Name(), err)
}

56
internal/framework/pipeline/runner_test.go
View File

@@ -1796,14 +1796,15 @@ func (normalizer *runnerNormalizer) Normalize(ctx context.Context, req contracts
}
type runnerValidator struct {
name string
resultName string
decisions func([]artifacts.ArtifactCandidate) []contracts.ValidationDecision
warnings []contracts.Warning
err error
order *[]string
calls int
requests []contracts.ValidationRequest
name string
approved []bool
reason string
message string
warnings []contracts.Warning
err error
order *[]string
calls int
requests []contracts.ValidationRequest
}
type runnerRawValidator struct {
@@ -1814,18 +1815,22 @@ type runnerRawValidator struct {
warnings []contracts.Warning
err error
calls int
requests []contracts.RawValidationRequest
requests []contracts.ValidationRequest
}
func (validator *runnerRawValidator) Name() string {
return validator.name
}
func (validator *runnerRawValidator) ValidateRaw(ctx context.Context, req contracts.RawValidationRequest) (contracts.RawValidationResult, error) {
func (validator *runnerRawValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (validator *runnerRawValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
validator.calls++
validator.requests = append(validator.requests, req)
if validator.err != nil {
return contracts.RawValidationResult{}, validator.err
return contracts.ValidationResult{}, validator.err
}
approved := true
if len(validator.approved) > 0 {
@@ -1835,7 +1840,7 @@ func (validator *runnerRawValidator) ValidateRaw(ctx context.Context, req contra
}
approved = validator.approved[index]
}
return contracts.RawValidationResult{
return contracts.ValidationResult{
Approved: approved,
ReasonCode: validator.reason,
Message: validator.message,
@@ -1847,24 +1852,29 @@ func (validator *runnerValidator) Name() string {
return validator.name
}
func (validator *runnerValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (validator *runnerValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
validator.calls++
validator.requests = append(validator.requests, req)
if validator.order != nil {
*validator.order = append(*validator.order, validator.name)
}
resultName := validator.resultName
if resultName == "" {
resultName = validator.name
}
var decisions []contracts.ValidationDecision
if validator.decisions != nil {
decisions = validator.decisions(req.Candidates)
approved := true
if len(validator.approved) > 0 {
index := validator.calls - 1
if index >= len(validator.approved) {
index = len(validator.approved) - 1
}
approved = validator.approved[index]
}
return contracts.ValidationResult{
ValidatorName: resultName,
Decisions: decisions,
Warnings: validator.warnings,
Approved: approved,
ReasonCode: validator.reason,
Message: validator.message,
Warnings: validator.warnings,
}, validator.err
}
@@ -2027,7 +2037,7 @@ func assertRunError(t *testing.T, err error, want string) {
}
}
func rawValidationRegistry(t *testing.T, stage ModuleStage, module string, validators ...contracts.RawValidator) *RawValidationRegistry {
func rawValidationRegistry(t *testing.T, stage ModuleStage, module string, validators ...contracts.Validator) *RawValidationRegistry {
t.Helper()
registry := NewRawValidationRegistry()

5
internal/framework/pipeline/validator_registry.go
View File

@@ -77,6 +77,11 @@ func (r *ValidatorRegistry) Build(key string) (contracts.Validator, error) {
if validator.Name() != normalizedKey {
return nil, fmt.Errorf("validator %q returned name %q", normalizedKey, validator.Name())
}
switch validator.ExecutionClass() {
case contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked:
default:
return nil, fmt.Errorf("validator %q returned unsupported execution class %q", normalizedKey, validator.ExecutionClass())
}
return validator, nil
}

31
internal/framework/pipeline/validator_registry_test.go
View File

@@ -1,6 +1,7 @@
package pipeline
import (
"context"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -56,3 +57,33 @@ func TestValidatorRegistryBehavior(t *testing.T) {
},
})
}
func TestValidatorRegistryRejectsUnsupportedExecutionClass(t *testing.T) {
registry := NewValidatorRegistry()
if err := registry.Register("invalid-validator", func() (contracts.Validator, error) {
return invalidExecutionClassValidator{name: "invalid-validator"}, nil
}); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
_, err := registry.Build("invalid-validator")
if err == nil {
t.Fatal("Build() error = nil, want unsupported execution class error")
}
}
type invalidExecutionClassValidator struct {
name string
}
func (validator invalidExecutionClassValidator) Name() string {
return validator.name
}
func (validator invalidExecutionClassValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClass("unsupported")
}
func (validator invalidExecutionClassValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}

57
internal/framework/validate/validate.go
View File

@@ -1,10 +1,8 @@
package validate
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
@@ -12,53 +10,18 @@ const (
ReasonApproved = "approved"
)
func Approved(candidateIndex int) contracts.ValidationDecision {
return contracts.ValidationDecision{
CandidateIndex: candidateIndex,
Approved: true,
ReasonCode: ReasonApproved,
Message: ReasonApproved,
func Approved() contracts.ValidationResult {
return contracts.ValidationResult{
Approved: true,
ReasonCode: ReasonApproved,
Message: ReasonApproved,
}
}
func Rejected(candidateIndex int, reasonCode string, message string) contracts.ValidationDecision {
return contracts.ValidationDecision{
CandidateIndex: candidateIndex,
Approved: false,
ReasonCode: strings.TrimSpace(reasonCode),
Message: strings.TrimSpace(message),
func Rejected(reasonCode string, message string) contracts.ValidationResult {
return contracts.ValidationResult{
Approved: false,
ReasonCode: strings.TrimSpace(reasonCode),
Message: strings.TrimSpace(message),
}
}
func EnforceDecisionCardinality(candidates []artifacts.ArtifactCandidate, decisions []contracts.ValidationDecision) error {
if len(candidates) != len(decisions) {
return fmt.Errorf("validator returned %d decisions for %d candidates", len(decisions), len(candidates))
}
expected := make(map[int]struct{}, len(candidates))
for _, candidate := range candidates {
if _, ok := expected[candidate.Index]; ok {
return fmt.Errorf("candidate index %d is duplicated", candidate.Index)
}
expected[candidate.Index] = struct{}{}
}
seen := make(map[int]struct{}, len(decisions))
for _, decision := range decisions {
if _, ok := expected[decision.CandidateIndex]; !ok {
return fmt.Errorf("validator returned decision for unknown candidate index %d", decision.CandidateIndex)
}
if _, ok := seen[decision.CandidateIndex]; ok {
return fmt.Errorf("validator returned duplicate decision for candidate index %d", decision.CandidateIndex)
}
seen[decision.CandidateIndex] = struct{}{}
}
for candidateIndex := range expected {
if _, ok := seen[candidateIndex]; !ok {
return fmt.Errorf("validator did not return decision for candidate index %d", candidateIndex)
}
}
return nil
}

95
internal/framework/validate/validate_test.go
View File

@@ -1,105 +1,40 @@
package validate
import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestApproved(t *testing.T) {
decision := Approved(7)
result := Approved()
if decision.CandidateIndex != 7 {
t.Fatalf("CandidateIndex = %d, want 7", decision.CandidateIndex)
}
if !decision.Approved {
if !result.Approved {
t.Fatal("Approved = false, want true")
}
if decision.ReasonCode != ReasonApproved {
t.Fatalf("ReasonCode = %q, want %q", decision.ReasonCode, ReasonApproved)
if result.ReasonCode != ReasonApproved {
t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonApproved)
}
if decision.Message != "approved" {
t.Fatalf("Message = %q, want approved", decision.Message)
if result.Message != "approved" {
t.Fatalf("Message = %q, want approved", result.Message)
}
}
func TestRejectedTrimsReasonAndMessage(t *testing.T) {
decision := Rejected(3, " invalid ", "\tmessage\n")
result := Rejected(" invalid ", "\tmessage\n")
if decision.CandidateIndex != 3 {
t.Fatalf("CandidateIndex = %d, want 3", decision.CandidateIndex)
}
if decision.Approved {
if result.Approved {
t.Fatal("Approved = true, want false")
}
if decision.ReasonCode != "invalid" {
t.Fatalf("ReasonCode = %q, want invalid", decision.ReasonCode)
if result.ReasonCode != "invalid" {
t.Fatalf("ReasonCode = %q, want invalid", result.ReasonCode)
}
if decision.Message != "message" {
t.Fatalf("Message = %q, want message", decision.Message)
if result.Message != "message" {
t.Fatalf("Message = %q, want message", result.Message)
}
}
func TestEnforceDecisionCardinalityAllowsNonZeroCandidateIndices(t *testing.T) {
candidates := []artifacts.ArtifactCandidate{{Index: 4}, {Index: 8}}
decisions := []contracts.ValidationDecision{Approved(8), Approved(4)}
if err := EnforceDecisionCardinality(candidates, decisions); err != nil {
t.Fatalf("EnforceDecisionCardinality() error = %v, want nil", err)
}
}
func TestEnforceDecisionCardinalityAllowsEmptyInputs(t *testing.T) {
if err := EnforceDecisionCardinality(nil, nil); err != nil {
t.Fatalf("EnforceDecisionCardinality() error = %v, want nil", err)
}
}
func TestEnforceDecisionCardinalityRejectsUnknownDecisionIndex(t *testing.T) {
err := EnforceDecisionCardinality(
[]artifacts.ArtifactCandidate{{Index: 1}},
[]contracts.ValidationDecision{Approved(2)},
)
assertCardinalityError(t, err, "unknown candidate index 2")
}
func TestEnforceDecisionCardinalityRejectsDuplicateDecisionIndex(t *testing.T) {
err := EnforceDecisionCardinality(
[]artifacts.ArtifactCandidate{{Index: 1}, {Index: 2}},
[]contracts.ValidationDecision{Approved(1), Approved(1)},
)
assertCardinalityError(t, err, "duplicate decision")
}
func TestEnforceDecisionCardinalityRejectsMissingDecisionIndex(t *testing.T) {
err := EnforceDecisionCardinality(
[]artifacts.ArtifactCandidate{{Index: 1}, {Index: 2}},
[]contracts.ValidationDecision{Approved(1)},
)
assertCardinalityError(t, err, "1 decisions for 2 candidates")
}
func TestEnforceDecisionCardinalityRejectsDuplicateCandidateIndex(t *testing.T) {
err := EnforceDecisionCardinality(
[]artifacts.ArtifactCandidate{{Index: 1}, {Index: 1}},
[]contracts.ValidationDecision{Approved(1), Approved(1)},
)
assertCardinalityError(t, err, "candidate index 1 is duplicated")
}
func assertCardinalityError(t *testing.T, err error, want string) {
t.Helper()
if err == nil {
t.Fatal("EnforceDecisionCardinality() error = nil, want error")
}
if !strings.Contains(err.Error(), want) {
t.Fatalf("EnforceDecisionCardinality() error = %q, want substring %q", err.Error(), want)
}
func TestHelpersReturnValidationResults(t *testing.T) {
var _ contracts.ValidationResult = Approved()
var _ contracts.ValidationResult = Rejected("reason", "message")
}