eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace validator contracts with raw output validation

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-07 20:55:49 +00:00
parent 249e49c928
commit b7ad66f0e0
13 changed files with 161 additions and 672 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
internal/framework/contracts/composition_test.go
View File

@@ -255,20 +255,15 @@ func (validator compositionValidator) Name() string {
return "generic-validator"
}
func (validator compositionValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates))
for _, candidate := range req.Candidates {
decisions = append(decisions, contracts.ValidationDecision{
CandidateIndex: candidate.Index,
Approved: true,
ReasonCode: "accepted",
Message: "candidate accepted",
})
}
func (validator compositionValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (validator compositionValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{
ValidatorName: validator.Name(),
Decisions: decisions,
Approved: true,
ReasonCode: "accepted",
Message: "output accepted",
}, nil
}

73
internal/framework/contracts/contracts.go
View File

@@ -203,20 +203,37 @@ type RawPayload struct {
Warnings []Warning `json:"warnings,omitempty"`
}
type RawValidationRequest struct {
Stage string `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key"`
Source *source.SourceDocument `json:"-"`
SourceID string `json:"source_id,omitempty"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
Schema ResponseSchema `json:"schema,omitempty"`
Payload RawPayload `json:"payload"`
Metadata map[string]any `json:"metadata,omitempty"`
type ExecutionClass string
const (
ExecutionClassDeterministic ExecutionClass = "deterministic"
ExecutionClassLLMBacked ExecutionClass = "llm_backed"
)
type ValidationRequest struct {
Stage string `json:"stage"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key"`
Source *source.SourceDocument `json:"-"`
SourceID string `json:"source_id,omitempty"`
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
SessionID string `json:"session_id,omitempty"`
References ReferenceSet `json:"references,omitempty"`
LLMClient StructuredLLMClient `json:"-"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
Schema ResponseSchema `json:"schema,omitempty"`
Payload RawPayload `json:"payload"`
ChunkID string `json:"chunk_id,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
Chunk *SourceChunk `json:"chunk,omitempty"`
Chunks []SourceChunk `json:"chunks,omitempty"`
ExtractOutputs []ExtractOutput `json:"extract_outputs,omitempty"`
MergeOutput MergeOutput `json:"merge_output,omitempty"`
}
type RawValidationResult struct {
type ValidationResult struct {
Approved bool `json:"approved"`
ReasonCode string `json:"reason_code,omitempty"`
Message string `json:"message,omitempty"`
@@ -224,9 +241,10 @@ type RawValidationResult struct {
Warnings []Warning `json:"warnings,omitempty"`
}
type RawValidator interface {
type Validator interface {
Name() string
ValidateRaw(ctx context.Context, req RawValidationRequest) (RawValidationResult, error)
ExecutionClass() ExecutionClass
Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error)
}
type ResponseSchema struct {
@@ -308,33 +326,6 @@ type Normalizer interface {
Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error)
}
type ValidationRequest struct {
Source *source.SourceDocument `json:"-"`
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
LLMProfile string `json:"llm_profile,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type ValidationDecision struct {
CandidateIndex int `json:"candidate_index"`
Approved bool `json:"approved"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
}
type ValidationResult struct {
ValidatorName string `json:"validator_name"`
Decisions []ValidationDecision `json:"decisions"`
Warnings []Warning `json:"warnings,omitempty"`
}
type Validator interface {
Name() string
Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error)
}
type Warning struct {
Scope string `json:"scope,omitempty"`
ReasonCode string `json:"reason_code"`

19
internal/framework/contracts/contracts_test.go
View File

@@ -577,20 +577,15 @@ func (validator fakeValidator) Name() string {
return validator.name
}
func (validator fakeValidator) Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error) {
decisions := make([]ValidationDecision, 0, len(req.Candidates))
for _, candidate := range req.Candidates {
decisions = append(decisions, ValidationDecision{
CandidateIndex: candidate.Index,
Approved: true,
ReasonCode: "accepted",
Message: "candidate accepted",
})
}
func (validator fakeValidator) ExecutionClass() ExecutionClass {
return ExecutionClassDeterministic
}
func (validator fakeValidator) Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error) {
return ValidationResult{
ValidatorName: validator.name,
Decisions: decisions,
Approved: true,
ReasonCode: "accepted",
Message: "output accepted",
}, nil
}

8
internal/framework/pipeline/chunker_registry_test.go
View File

@@ -386,6 +386,10 @@ func (validator registryValidator) Name() string {
return validator.name
}
func (validator registryValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{}, nil
func (validator registryValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (validator registryValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}

21
internal/framework/pipeline/raw_validation_registry.go
View File

@@ -13,16 +13,16 @@ type rawValidationKey struct {
}
type RawValidationRegistry struct {
chains map[rawValidationKey][]contracts.RawValidator
chains map[rawValidationKey][]contracts.Validator
}
func NewRawValidationRegistry() *RawValidationRegistry {
return &RawValidationRegistry{
chains: make(map[rawValidationKey][]contracts.RawValidator),
chains: make(map[rawValidationKey][]contracts.Validator),
}
}
func (r *RawValidationRegistry) Register(stage ModuleStage, module string, validators ...contracts.RawValidator) error {
func (r *RawValidationRegistry) Register(stage ModuleStage, module string, validators ...contracts.Validator) error {
if r == nil {
return fmt.Errorf("raw validation registry must not be nil")
}
@@ -39,7 +39,7 @@ func (r *RawValidationRegistry) Register(stage ModuleStage, module string, valid
return fmt.Errorf("raw validation chain for %q %q must not be empty", stage, normalizedModule)
}
chain := make([]contracts.RawValidator, 0, len(validators))
chain := make([]contracts.Validator, 0, len(validators))
for i, validator := range validators {
if validator == nil {
return fmt.Errorf("raw validator %d for %q %q must not be nil", i, stage, normalizedModule)
@@ -47,21 +47,26 @@ func (r *RawValidationRegistry) Register(stage ModuleStage, module string, valid
if strings.TrimSpace(validator.Name()) == "" {
return fmt.Errorf("raw validator %d for %q %q must not have an empty name", i, stage, normalizedModule)
}
switch validator.ExecutionClass() {
case contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked:
default:
return fmt.Errorf("raw validator %q for %q %q has unsupported execution class %q", validator.Name(), stage, normalizedModule, validator.ExecutionClass())
}
chain = append(chain, validator)
}
if r.chains == nil {
r.chains = make(map[rawValidationKey][]contracts.RawValidator)
r.chains = make(map[rawValidationKey][]contracts.Validator)
}
key := rawValidationKey{stage: stage, module: normalizedModule}
if _, exists := r.chains[key]; exists {
return fmt.Errorf("raw validation chain for %q %q is already registered", stage, normalizedModule)
}
r.chains[key] = append([]contracts.RawValidator(nil), chain...)
r.chains[key] = append([]contracts.Validator(nil), chain...)
return nil
}
func (r *RawValidationRegistry) Validators(stage ModuleStage, module string) []contracts.RawValidator {
func (r *RawValidationRegistry) Validators(stage ModuleStage, module string) []contracts.Validator {
if r == nil {
return nil
}
@@ -69,5 +74,5 @@ func (r *RawValidationRegistry) Validators(stage ModuleStage, module string) []c
if len(chain) == 0 {
return nil
}
return append([]contracts.RawValidator(nil), chain...)
return append([]contracts.Validator(nil), chain...)
}

4
internal/framework/pipeline/runner.go
View File

@@ -461,7 +461,7 @@ func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([
return nil, nil, nil
}
request := contracts.RawValidationRequest{
request := contracts.ValidationRequest{
Stage: string(target.stage),
LaneID: target.laneID,
ModuleKey: target.moduleKey,
@@ -476,7 +476,7 @@ func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([
var warnings []contracts.Warning
for _, validator := range validators {
result, err := validator.ValidateRaw(ctx, request)
result, err := validator.Validate(ctx, request)
if err != nil {
return nil, nil, fmt.Errorf("validate raw %s output with validator %q: %w", target.stage, validator.Name(), err)
}

56
internal/framework/pipeline/runner_test.go
View File

@@ -1796,14 +1796,15 @@ func (normalizer *runnerNormalizer) Normalize(ctx context.Context, req contracts
}
type runnerValidator struct {
name string
resultName string
decisions func([]artifacts.ArtifactCandidate) []contracts.ValidationDecision
warnings []contracts.Warning
err error
order *[]string
calls int
requests []contracts.ValidationRequest
name string
approved []bool
reason string
message string
warnings []contracts.Warning
err error
order *[]string
calls int
requests []contracts.ValidationRequest
}
type runnerRawValidator struct {
@@ -1814,18 +1815,22 @@ type runnerRawValidator struct {
warnings []contracts.Warning
err error
calls int
requests []contracts.RawValidationRequest
requests []contracts.ValidationRequest
}
func (validator *runnerRawValidator) Name() string {
return validator.name
}
func (validator *runnerRawValidator) ValidateRaw(ctx context.Context, req contracts.RawValidationRequest) (contracts.RawValidationResult, error) {
func (validator *runnerRawValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (validator *runnerRawValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
validator.calls++
validator.requests = append(validator.requests, req)
if validator.err != nil {
return contracts.RawValidationResult{}, validator.err
return contracts.ValidationResult{}, validator.err
}
approved := true
if len(validator.approved) > 0 {
@@ -1835,7 +1840,7 @@ func (validator *runnerRawValidator) ValidateRaw(ctx context.Context, req contra
}
approved = validator.approved[index]
}
return contracts.RawValidationResult{
return contracts.ValidationResult{
Approved: approved,
ReasonCode: validator.reason,
Message: validator.message,
@@ -1847,24 +1852,29 @@ func (validator *runnerValidator) Name() string {
return validator.name
}
func (validator *runnerValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (validator *runnerValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
validator.calls++
validator.requests = append(validator.requests, req)
if validator.order != nil {
*validator.order = append(*validator.order, validator.name)
}
resultName := validator.resultName
if resultName == "" {
resultName = validator.name
}
var decisions []contracts.ValidationDecision
if validator.decisions != nil {
decisions = validator.decisions(req.Candidates)
approved := true
if len(validator.approved) > 0 {
index := validator.calls - 1
if index >= len(validator.approved) {
index = len(validator.approved) - 1
}
approved = validator.approved[index]
}
return contracts.ValidationResult{
ValidatorName: resultName,
Decisions: decisions,
Warnings: validator.warnings,
Approved: approved,
ReasonCode: validator.reason,
Message: validator.message,
Warnings: validator.warnings,
}, validator.err
}
@@ -2027,7 +2037,7 @@ func assertRunError(t *testing.T, err error, want string) {
}
}
func rawValidationRegistry(t *testing.T, stage ModuleStage, module string, validators ...contracts.RawValidator) *RawValidationRegistry {
func rawValidationRegistry(t *testing.T, stage ModuleStage, module string, validators ...contracts.Validator) *RawValidationRegistry {
t.Helper()
registry := NewRawValidationRegistry()

5
internal/framework/pipeline/validator_registry.go
View File

@@ -77,6 +77,11 @@ func (r *ValidatorRegistry) Build(key string) (contracts.Validator, error) {
if validator.Name() != normalizedKey {
return nil, fmt.Errorf("validator %q returned name %q", normalizedKey, validator.Name())
}
switch validator.ExecutionClass() {
case contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked:
default:
return nil, fmt.Errorf("validator %q returned unsupported execution class %q", normalizedKey, validator.ExecutionClass())
}
return validator, nil
}

31
internal/framework/pipeline/validator_registry_test.go
View File

@@ -1,6 +1,7 @@
package pipeline
import (
"context"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -56,3 +57,33 @@ func TestValidatorRegistryBehavior(t *testing.T) {
},
})
}
func TestValidatorRegistryRejectsUnsupportedExecutionClass(t *testing.T) {
registry := NewValidatorRegistry()
if err := registry.Register("invalid-validator", func() (contracts.Validator, error) {
return invalidExecutionClassValidator{name: "invalid-validator"}, nil
}); err != nil {
t.Fatalf("Register() error = %v, want nil", err)
}
_, err := registry.Build("invalid-validator")
if err == nil {
t.Fatal("Build() error = nil, want unsupported execution class error")
}
}
type invalidExecutionClassValidator struct {
name string
}
func (validator invalidExecutionClassValidator) Name() string {
return validator.name
}
func (validator invalidExecutionClassValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClass("unsupported")
}
func (validator invalidExecutionClassValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: true}, nil
}

57
internal/framework/validate/validate.go
View File

@@ -1,10 +1,8 @@
package validate
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
@@ -12,53 +10,18 @@ const (
ReasonApproved = "approved"
)
func Approved(candidateIndex int) contracts.ValidationDecision {
return contracts.ValidationDecision{
CandidateIndex: candidateIndex,
Approved: true,
ReasonCode: ReasonApproved,
Message: ReasonApproved,
func Approved() contracts.ValidationResult {
return contracts.ValidationResult{
Approved: true,
ReasonCode: ReasonApproved,
Message: ReasonApproved,
}
}
func Rejected(candidateIndex int, reasonCode string, message string) contracts.ValidationDecision {
return contracts.ValidationDecision{
CandidateIndex: candidateIndex,
Approved: false,
ReasonCode: strings.TrimSpace(reasonCode),
Message: strings.TrimSpace(message),
func Rejected(reasonCode string, message string) contracts.ValidationResult {
return contracts.ValidationResult{
Approved: false,
ReasonCode: strings.TrimSpace(reasonCode),
Message: strings.TrimSpace(message),
}
}
func EnforceDecisionCardinality(candidates []artifacts.ArtifactCandidate, decisions []contracts.ValidationDecision) error {
if len(candidates) != len(decisions) {
return fmt.Errorf("validator returned %d decisions for %d candidates", len(decisions), len(candidates))
}
expected := make(map[int]struct{}, len(candidates))
for _, candidate := range candidates {
if _, ok := expected[candidate.Index]; ok {
return fmt.Errorf("candidate index %d is duplicated", candidate.Index)
}
expected[candidate.Index] = struct{}{}
}
seen := make(map[int]struct{}, len(decisions))
for _, decision := range decisions {
if _, ok := expected[decision.CandidateIndex]; !ok {
return fmt.Errorf("validator returned decision for unknown candidate index %d", decision.CandidateIndex)
}
if _, ok := seen[decision.CandidateIndex]; ok {
return fmt.Errorf("validator returned duplicate decision for candidate index %d", decision.CandidateIndex)
}
seen[decision.CandidateIndex] = struct{}{}
}
for candidateIndex := range expected {
if _, ok := seen[candidateIndex]; !ok {
return fmt.Errorf("validator did not return decision for candidate index %d", candidateIndex)
}
}
return nil
}

95
internal/framework/validate/validate_test.go
View File

@@ -1,105 +1,40 @@
package validate
import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestApproved(t *testing.T) {
decision := Approved(7)
result := Approved()
if decision.CandidateIndex != 7 {
t.Fatalf("CandidateIndex = %d, want 7", decision.CandidateIndex)
}
if !decision.Approved {
if !result.Approved {
t.Fatal("Approved = false, want true")
}
if decision.ReasonCode != ReasonApproved {
t.Fatalf("ReasonCode = %q, want %q", decision.ReasonCode, ReasonApproved)
if result.ReasonCode != ReasonApproved {
t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonApproved)
}
if decision.Message != "approved" {
t.Fatalf("Message = %q, want approved", decision.Message)
if result.Message != "approved" {
t.Fatalf("Message = %q, want approved", result.Message)
}
}
func TestRejectedTrimsReasonAndMessage(t *testing.T) {
decision := Rejected(3, " invalid ", "\tmessage\n")
result := Rejected(" invalid ", "\tmessage\n")
if decision.CandidateIndex != 3 {
t.Fatalf("CandidateIndex = %d, want 3", decision.CandidateIndex)
}
if decision.Approved {
if result.Approved {
t.Fatal("Approved = true, want false")
}
if decision.ReasonCode != "invalid" {
t.Fatalf("ReasonCode = %q, want invalid", decision.ReasonCode)
if result.ReasonCode != "invalid" {
t.Fatalf("ReasonCode = %q, want invalid", result.ReasonCode)
}
if decision.Message != "message" {
t.Fatalf("Message = %q, want message", decision.Message)
if result.Message != "message" {
t.Fatalf("Message = %q, want message", result.Message)
}
}
func TestEnforceDecisionCardinalityAllowsNonZeroCandidateIndices(t *testing.T) {
candidates := []artifacts.ArtifactCandidate{{Index: 4}, {Index: 8}}
decisions := []contracts.ValidationDecision{Approved(8), Approved(4)}
if err := EnforceDecisionCardinality(candidates, decisions); err != nil {
t.Fatalf("EnforceDecisionCardinality() error = %v, want nil", err)
}
}
func TestEnforceDecisionCardinalityAllowsEmptyInputs(t *testing.T) {
if err := EnforceDecisionCardinality(nil, nil); err != nil {
t.Fatalf("EnforceDecisionCardinality() error = %v, want nil", err)
}
}
func TestEnforceDecisionCardinalityRejectsUnknownDecisionIndex(t *testing.T) {
err := EnforceDecisionCardinality(
[]artifacts.ArtifactCandidate{{Index: 1}},
[]contracts.ValidationDecision{Approved(2)},
)
assertCardinalityError(t, err, "unknown candidate index 2")
}
func TestEnforceDecisionCardinalityRejectsDuplicateDecisionIndex(t *testing.T) {
err := EnforceDecisionCardinality(
[]artifacts.ArtifactCandidate{{Index: 1}, {Index: 2}},
[]contracts.ValidationDecision{Approved(1), Approved(1)},
)
assertCardinalityError(t, err, "duplicate decision")
}
func TestEnforceDecisionCardinalityRejectsMissingDecisionIndex(t *testing.T) {
err := EnforceDecisionCardinality(
[]artifacts.ArtifactCandidate{{Index: 1}, {Index: 2}},
[]contracts.ValidationDecision{Approved(1)},
)
assertCardinalityError(t, err, "1 decisions for 2 candidates")
}
func TestEnforceDecisionCardinalityRejectsDuplicateCandidateIndex(t *testing.T) {
err := EnforceDecisionCardinality(
[]artifacts.ArtifactCandidate{{Index: 1}, {Index: 1}},
[]contracts.ValidationDecision{Approved(1), Approved(1)},
)
assertCardinalityError(t, err, "candidate index 1 is duplicated")
}
func assertCardinalityError(t *testing.T, err error, want string) {
t.Helper()
if err == nil {
t.Fatal("EnforceDecisionCardinality() error = nil, want error")
}
if !strings.Contains(err.Error(), want) {
t.Fatalf("EnforceDecisionCardinality() error = %q, want substring %q", err.Error(), want)
}
func TestHelpersReturnValidationResults(t *testing.T) {
var _ contracts.ValidationResult = Approved()
var _ contracts.ValidationResult = Rejected("reason", "message")
}

168
internal/modules/extract/dnd/spells/validator.go
View File

@@ -1,168 +0,0 @@
package spells
import (
"context"
"encoding/json"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/validate"
)
const (
shapeValidatorName = "dnd/spells/shape"
sourceRefValidatorName = "dnd/spells/source_refs"
reasonInvalidPayload = "invalid_payload"
reasonMissingRequiredField = "missing_required_field"
reasonMissingSourceRef = "missing_source_ref"
reasonInvalidSourceRef = "invalid_source_ref"
reasonSpellNotNearSource = "spell_not_near_source"
)
var _ contracts.Validator = ShapeValidator{}
var _ contracts.Validator = SourceRefValidator{}
type ShapeValidator struct{}
func (validator ShapeValidator) Name() string {
return shapeValidatorName
}
func (validator ShapeValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates))
for _, candidate := range req.Candidates {
decisions = append(decisions, validateShape(candidate))
}
return contracts.ValidationResult{
ValidatorName: validator.Name(),
Decisions: decisions,
}, nil
}
type SourceRefValidator struct{}
func (validator SourceRefValidator) Name() string {
return sourceRefValidatorName
}
func (validator SourceRefValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
if req.Source == nil {
return contracts.ValidationResult{}, fmt.Errorf("dnd spells source refs validator: source must not be nil")
}
decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates))
var warnings []contracts.Warning
for _, candidate := range req.Candidates {
decisions = append(decisions, validateSourceRefs(req.Source, candidate))
warnings = append(warnings, sourceRelatednessWarnings(req.Source, candidate)...)
}
return contracts.ValidationResult{
ValidatorName: validator.Name(),
Decisions: decisions,
Warnings: warnings,
}, nil
}
func validateShape(candidate artifacts.ArtifactCandidate) contracts.ValidationDecision {
var payload SpellCast
if err := json.Unmarshal(candidate.Payload, &payload); err != nil {
return validate.Rejected(candidate.Index, reasonInvalidPayload, fmt.Sprintf("invalid spell cast payload: %v", err))
}
for _, field := range requiredSpellCastFields(payload) {
if strings.TrimSpace(field.value) == "" {
return validate.Rejected(candidate.Index, reasonMissingRequiredField, fmt.Sprintf("missing required field %q", field.name))
}
}
return validate.Approved(candidate.Index)
}
func validateSourceRefs(doc *source.SourceDocument, candidate artifacts.ArtifactCandidate) contracts.ValidationDecision {
if len(candidate.SourceRefs) == 0 {
return validate.Rejected(candidate.Index, reasonMissingSourceRef, "spell cast candidate must include at least one source ref")
}
for _, ref := range candidate.SourceRefs {
if err := source.ValidateRef(doc, ref); err != nil {
return validate.Rejected(candidate.Index, reasonInvalidSourceRef, err.Error())
}
}
return validate.Approved(candidate.Index)
}
func sourceRelatednessWarnings(doc *source.SourceDocument, candidate artifacts.ArtifactCandidate) []contracts.Warning {
if doc == nil || len(candidate.SourceRefs) == 0 {
return nil
}
var payload SpellCast
if err := json.Unmarshal(candidate.Payload, &payload); err != nil {
return nil
}
spell := strings.TrimSpace(payload.Spell)
if spell == "" {
return nil
}
needle := strings.ToLower(spell)
for _, ref := range candidate.SourceRefs {
text, ok := sourceRefText(doc, ref)
if !ok {
continue
}
if strings.Contains(strings.ToLower(text), needle) {
return nil
}
}
return []contracts.Warning{
{
Scope: fmt.Sprintf("candidate.%d", candidate.Index),
ReasonCode: reasonSpellNotNearSource,
Message: fmt.Sprintf("spell %q was not found in the cited source text", spell),
},
}
}
func sourceRefText(doc *source.SourceDocument, ref source.SourceRef) (string, bool) {
if err := source.ValidateRef(doc, ref); err != nil {
return "", false
}
start := -1
end := -1
for i, unit := range doc.Units {
if unit.ID == ref.StartUnitID {
start = i
}
if unit.ID == ref.EndUnitID {
end = i
}
}
if start < 0 || end < start {
return "", false
}
var b strings.Builder
for i := start; i <= end; i++ {
if b.Len() > 0 {
b.WriteString("\n")
}
b.WriteString(doc.Units[i].Text)
}
return b.String(), true
}
func requiredSpellCastFields(payload SpellCast) []struct {
name string
value string
} {
return []struct {
name string
value string
}{
{name: "caster", value: payload.Caster},
{name: "spell", value: payload.Spell},
{name: "effect", value: payload.Effect},
{name: "narrative_description", value: payload.NarrativeDescription},
}
}

277
internal/modules/extract/dnd/spells/validator_test.go
View File

@@ -1,277 +0,0 @@
package spells
import (
"context"
"encoding/json"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/validate"
)
func TestValidatorsApproveValidCandidate(t *testing.T) {
candidate := validSpellCandidate(7)
shapeResult, err := ShapeValidator{}.Validate(context.Background(), contracts.ValidationRequest{
Candidates: []artifacts.ArtifactCandidate{candidate},
})
if err != nil {
t.Fatalf("ShapeValidator.Validate() error = %v, want nil", err)
}
assertSingleDecision(t, shapeResult, shapeValidatorName, 7, true, validate.ReasonApproved)
sourceRefResult, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{
Source: promptSourceDocument(),
Candidates: []artifacts.ArtifactCandidate{candidate},
})
if err != nil {
t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err)
}
assertSingleDecision(t, sourceRefResult, sourceRefValidatorName, 7, true, validate.ReasonApproved)
if len(sourceRefResult.Warnings) != 0 {
t.Fatalf("warnings = %#v, want none", sourceRefResult.Warnings)
}
}
func TestShapeValidatorRejectsMalformedPayload(t *testing.T) {
candidate := validSpellCandidate(3)
candidate.Payload = json.RawMessage(`{"caster":`)
result, err := ShapeValidator{}.Validate(context.Background(), contracts.ValidationRequest{
Candidates: []artifacts.ArtifactCandidate{candidate},
})
if err != nil {
t.Fatalf("ShapeValidator.Validate() error = %v, want nil", err)
}
assertSingleDecision(t, result, shapeValidatorName, 3, false, reasonInvalidPayload)
}
func TestShapeValidatorRejectsBlankRequiredFields(t *testing.T) {
tests := []struct {
name string
mutate func(*SpellCast)
}{
{name: "caster", mutate: func(payload *SpellCast) { payload.Caster = " \t" }},
{name: "spell", mutate: func(payload *SpellCast) { payload.Spell = "" }},
{name: "effect", mutate: func(payload *SpellCast) { payload.Effect = "\n" }},
{name: "narrative description", mutate: func(payload *SpellCast) { payload.NarrativeDescription = " " }},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
payload := validSpellPayload()
tt.mutate(&payload)
candidate := validSpellCandidate(5)
candidate.Payload = mustSpellPayload(t, payload)
result, err := ShapeValidator{}.Validate(context.Background(), contracts.ValidationRequest{
Candidates: []artifacts.ArtifactCandidate{candidate},
})
if err != nil {
t.Fatalf("ShapeValidator.Validate() error = %v, want nil", err)
}
assertSingleDecision(t, result, shapeValidatorName, 5, false, reasonMissingRequiredField)
})
}
}
func TestShapeValidatorDoesNotRequireSourceDocument(t *testing.T) {
result, err := ShapeValidator{}.Validate(context.Background(), contracts.ValidationRequest{
Candidates: []artifacts.ArtifactCandidate{validSpellCandidate(11)},
})
if err != nil {
t.Fatalf("ShapeValidator.Validate() error = %v, want nil", err)
}
assertSingleDecision(t, result, shapeValidatorName, 11, true, validate.ReasonApproved)
}
func TestSourceRefValidatorRejectsMissingRefs(t *testing.T) {
candidate := validSpellCandidate(13)
candidate.SourceRefs = nil
result, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{
Source: promptSourceDocument(),
Candidates: []artifacts.ArtifactCandidate{candidate},
})
if err != nil {
t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err)
}
assertSingleDecision(t, result, sourceRefValidatorName, 13, false, reasonMissingSourceRef)
}
func TestSourceRefValidatorRejectsInvalidRefs(t *testing.T) {
tests := []struct {
name string
ref source.SourceRef
want string
}{
{
name: "unknown source id",
ref: source.SourceRef{SourceID: "session-beta", StartUnitID: 1, EndUnitID: 2},
want: "does not match",
},
{
name: "unknown start unit",
ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 999, EndUnitID: 2},
want: "start_unit_id",
},
{
name: "unknown end unit",
ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 999},
want: "end_unit_id",
},
{
name: "reversed unit range",
ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 1},
want: "appears after",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
candidate := validSpellCandidate(17)
candidate.SourceRefs = []source.SourceRef{tt.ref}
result, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{
Source: promptSourceDocument(),
Candidates: []artifacts.ArtifactCandidate{candidate},
})
if err != nil {
t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err)
}
assertSingleDecision(t, result, sourceRefValidatorName, 17, false, reasonInvalidSourceRef)
if !strings.Contains(result.Decisions[0].Message, tt.want) {
t.Fatalf("Message = %q, want substring %q", result.Decisions[0].Message, tt.want)
}
})
}
}
func TestSourceRefValidatorWarnsWhenSpellNameIsNotInCitedSource(t *testing.T) {
candidate := validSpellCandidate(31)
payload := validSpellPayload()
payload.Spell = "Shield"
candidate.Payload = mustSpellPayload(t, payload)
result, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{
Source: promptSourceDocument(),
Candidates: []artifacts.ArtifactCandidate{candidate},
})
if err != nil {
t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err)
}
assertSingleDecision(t, result, sourceRefValidatorName, 31, true, validate.ReasonApproved)
if len(result.Warnings) != 1 {
t.Fatalf("warnings = %#v, want one relatedness warning", result.Warnings)
}
warning := result.Warnings[0]
if warning.ReasonCode != reasonSpellNotNearSource || !strings.Contains(warning.Message, "Shield") {
t.Fatalf("warning = %#v, want spell relatedness warning", warning)
}
}
func TestSourceRefValidatorRequiresSourceDocument(t *testing.T) {
_, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{
Candidates: []artifacts.ArtifactCandidate{validSpellCandidate(19)},
})
if err == nil {
t.Fatal("SourceRefValidator.Validate() error = nil, want source error")
}
if !strings.Contains(err.Error(), "dnd spells") || !strings.Contains(err.Error(), "source") {
t.Fatalf("SourceRefValidator.Validate() error = %q, want source context", err.Error())
}
}
func TestValidatorsPreserveCandidateIndexes(t *testing.T) {
candidates := []artifacts.ArtifactCandidate{
validSpellCandidate(23),
validSpellCandidate(29),
}
shapeResult, err := ShapeValidator{}.Validate(context.Background(), contracts.ValidationRequest{
Candidates: candidates,
})
if err != nil {
t.Fatalf("ShapeValidator.Validate() error = %v, want nil", err)
}
assertDecisionIndexes(t, shapeResult.Decisions, []int{23, 29})
sourceRefResult, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{
Source: promptSourceDocument(),
Candidates: candidates,
})
if err != nil {
t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err)
}
assertDecisionIndexes(t, sourceRefResult.Decisions, []int{23, 29})
}
func validSpellCandidate(index int) artifacts.ArtifactCandidate {
return artifacts.ArtifactCandidate{
Index: index,
Payload: spellPayload(validSpellPayload()),
SourceRefs: []source.SourceRef{
{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 2},
},
}
}
func validSpellPayload() SpellCast {
return SpellCast{
Caster: "Aria",
Spell: "Cure Wounds",
Effect: "Heals an injured ally.",
NarrativeDescription: "Aria restores the fighter after the fight.",
}
}
func mustSpellPayload(t *testing.T, payload SpellCast) json.RawMessage {
t.Helper()
return spellPayload(payload)
}
func spellPayload(payload SpellCast) json.RawMessage {
encoded, err := json.Marshal(payload)
if err != nil {
panic(err)
}
return encoded
}
func assertSingleDecision(t *testing.T, result contracts.ValidationResult, wantName string, wantIndex int, wantApproved bool, wantReason string) {
t.Helper()
if result.ValidatorName != wantName {
t.Fatalf("ValidatorName = %q, want %q", result.ValidatorName, wantName)
}
if len(result.Decisions) != 1 {
t.Fatalf("len(Decisions) = %d, want 1", len(result.Decisions))
}
decision := result.Decisions[0]
if decision.CandidateIndex != wantIndex {
t.Fatalf("CandidateIndex = %d, want %d", decision.CandidateIndex, wantIndex)
}
if decision.Approved != wantApproved {
t.Fatalf("Approved = %t, want %t", decision.Approved, wantApproved)
}
if decision.ReasonCode != wantReason {
t.Fatalf("ReasonCode = %q, want %q", decision.ReasonCode, wantReason)
}
}
func assertDecisionIndexes(t *testing.T, decisions []contracts.ValidationDecision, want []int) {
t.Helper()
if len(decisions) != len(want) {
t.Fatalf("len(Decisions) = %d, want %d", len(decisions), len(want))
}
for i := range want {
if decisions[i].CandidateIndex != want[i] {
t.Fatalf("Decisions[%d].CandidateIndex = %d, want %d", i, decisions[i].CandidateIndex, want[i])
}
}
}