diff --git a/internal/framework/contracts/composition_test.go b/internal/framework/contracts/composition_test.go index 2752a53..542ba23 100644 --- a/internal/framework/contracts/composition_test.go +++ b/internal/framework/contracts/composition_test.go @@ -255,20 +255,15 @@ func (validator compositionValidator) Name() string { return "generic-validator" } -func (validator compositionValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) { - decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates)) - for _, candidate := range req.Candidates { - decisions = append(decisions, contracts.ValidationDecision{ - CandidateIndex: candidate.Index, - Approved: true, - ReasonCode: "accepted", - Message: "candidate accepted", - }) - } +func (validator compositionValidator) ExecutionClass() contracts.ExecutionClass { + return contracts.ExecutionClassDeterministic +} +func (validator compositionValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) { return contracts.ValidationResult{ - ValidatorName: validator.Name(), - Decisions: decisions, + Approved: true, + ReasonCode: "accepted", + Message: "output accepted", }, nil } diff --git a/internal/framework/contracts/contracts.go b/internal/framework/contracts/contracts.go index be52de1..fc4eaf5 100644 --- a/internal/framework/contracts/contracts.go +++ b/internal/framework/contracts/contracts.go @@ -203,20 +203,37 @@ type RawPayload struct { Warnings []Warning `json:"warnings,omitempty"` } -type RawValidationRequest struct { - Stage string `json:"stage"` - LaneID string `json:"lane_id,omitempty"` - ModuleKey string `json:"module_key"` - Source *source.SourceDocument `json:"-"` - SourceID string `json:"source_id,omitempty"` - ChunkID string `json:"chunk_id,omitempty"` - ChunkIndex int `json:"chunk_index,omitempty"` - Schema ResponseSchema `json:"schema,omitempty"` - Payload RawPayload `json:"payload"` - Metadata map[string]any `json:"metadata,omitempty"` +type ExecutionClass string + +const ( + ExecutionClassDeterministic ExecutionClass = "deterministic" + ExecutionClassLLMBacked ExecutionClass = "llm_backed" +) + +type ValidationRequest struct { + Stage string `json:"stage"` + LaneID string `json:"lane_id,omitempty"` + ModuleKey string `json:"module_key"` + Source *source.SourceDocument `json:"-"` + SourceID string `json:"source_id,omitempty"` + SourceInput LLMInputMaterial `json:"source_input,omitempty"` + SessionID string `json:"session_id,omitempty"` + References ReferenceSet `json:"references,omitempty"` + LLMClient StructuredLLMClient `json:"-"` + LLMProfile string `json:"llm_profile,omitempty"` + Options map[string]any `json:"options,omitempty"` + Metadata map[string]any `json:"metadata,omitempty"` + Schema ResponseSchema `json:"schema,omitempty"` + Payload RawPayload `json:"payload"` + ChunkID string `json:"chunk_id,omitempty"` + ChunkIndex int `json:"chunk_index,omitempty"` + Chunk *SourceChunk `json:"chunk,omitempty"` + Chunks []SourceChunk `json:"chunks,omitempty"` + ExtractOutputs []ExtractOutput `json:"extract_outputs,omitempty"` + MergeOutput MergeOutput `json:"merge_output,omitempty"` } -type RawValidationResult struct { +type ValidationResult struct { Approved bool `json:"approved"` ReasonCode string `json:"reason_code,omitempty"` Message string `json:"message,omitempty"` @@ -224,9 +241,10 @@ type RawValidationResult struct { Warnings []Warning `json:"warnings,omitempty"` } -type RawValidator interface { +type Validator interface { Name() string - ValidateRaw(ctx context.Context, req RawValidationRequest) (RawValidationResult, error) + ExecutionClass() ExecutionClass + Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error) } type ResponseSchema struct { @@ -308,33 +326,6 @@ type Normalizer interface { Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error) } -type ValidationRequest struct { - Source *source.SourceDocument `json:"-"` - Candidates []artifacts.ArtifactCandidate `json:"candidates"` - LLMProfile string `json:"llm_profile,omitempty"` - Options map[string]any `json:"options,omitempty"` - Metadata map[string]any `json:"metadata,omitempty"` -} - -type ValidationDecision struct { - CandidateIndex int `json:"candidate_index"` - Approved bool `json:"approved"` - ReasonCode string `json:"reason_code"` - Message string `json:"message"` - DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"` -} - -type ValidationResult struct { - ValidatorName string `json:"validator_name"` - Decisions []ValidationDecision `json:"decisions"` - Warnings []Warning `json:"warnings,omitempty"` -} - -type Validator interface { - Name() string - Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error) -} - type Warning struct { Scope string `json:"scope,omitempty"` ReasonCode string `json:"reason_code"` diff --git a/internal/framework/contracts/contracts_test.go b/internal/framework/contracts/contracts_test.go index 2900887..27479c2 100644 --- a/internal/framework/contracts/contracts_test.go +++ b/internal/framework/contracts/contracts_test.go @@ -577,20 +577,15 @@ func (validator fakeValidator) Name() string { return validator.name } -func (validator fakeValidator) Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error) { - decisions := make([]ValidationDecision, 0, len(req.Candidates)) - for _, candidate := range req.Candidates { - decisions = append(decisions, ValidationDecision{ - CandidateIndex: candidate.Index, - Approved: true, - ReasonCode: "accepted", - Message: "candidate accepted", - }) - } +func (validator fakeValidator) ExecutionClass() ExecutionClass { + return ExecutionClassDeterministic +} +func (validator fakeValidator) Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error) { return ValidationResult{ - ValidatorName: validator.name, - Decisions: decisions, + Approved: true, + ReasonCode: "accepted", + Message: "output accepted", }, nil } diff --git a/internal/framework/pipeline/chunker_registry_test.go b/internal/framework/pipeline/chunker_registry_test.go index 42c77a8..46ba57e 100644 --- a/internal/framework/pipeline/chunker_registry_test.go +++ b/internal/framework/pipeline/chunker_registry_test.go @@ -386,6 +386,10 @@ func (validator registryValidator) Name() string { return validator.name } -func (validator registryValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) { - return contracts.ValidationResult{}, nil +func (validator registryValidator) ExecutionClass() contracts.ExecutionClass { + return contracts.ExecutionClassDeterministic +} + +func (validator registryValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) { + return contracts.ValidationResult{Approved: true}, nil } diff --git a/internal/framework/pipeline/raw_validation_registry.go b/internal/framework/pipeline/raw_validation_registry.go index a412f35..efb0c6f 100644 --- a/internal/framework/pipeline/raw_validation_registry.go +++ b/internal/framework/pipeline/raw_validation_registry.go @@ -13,16 +13,16 @@ type rawValidationKey struct { } type RawValidationRegistry struct { - chains map[rawValidationKey][]contracts.RawValidator + chains map[rawValidationKey][]contracts.Validator } func NewRawValidationRegistry() *RawValidationRegistry { return &RawValidationRegistry{ - chains: make(map[rawValidationKey][]contracts.RawValidator), + chains: make(map[rawValidationKey][]contracts.Validator), } } -func (r *RawValidationRegistry) Register(stage ModuleStage, module string, validators ...contracts.RawValidator) error { +func (r *RawValidationRegistry) Register(stage ModuleStage, module string, validators ...contracts.Validator) error { if r == nil { return fmt.Errorf("raw validation registry must not be nil") } @@ -39,7 +39,7 @@ func (r *RawValidationRegistry) Register(stage ModuleStage, module string, valid return fmt.Errorf("raw validation chain for %q %q must not be empty", stage, normalizedModule) } - chain := make([]contracts.RawValidator, 0, len(validators)) + chain := make([]contracts.Validator, 0, len(validators)) for i, validator := range validators { if validator == nil { return fmt.Errorf("raw validator %d for %q %q must not be nil", i, stage, normalizedModule) @@ -47,21 +47,26 @@ func (r *RawValidationRegistry) Register(stage ModuleStage, module string, valid if strings.TrimSpace(validator.Name()) == "" { return fmt.Errorf("raw validator %d for %q %q must not have an empty name", i, stage, normalizedModule) } + switch validator.ExecutionClass() { + case contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked: + default: + return fmt.Errorf("raw validator %q for %q %q has unsupported execution class %q", validator.Name(), stage, normalizedModule, validator.ExecutionClass()) + } chain = append(chain, validator) } if r.chains == nil { - r.chains = make(map[rawValidationKey][]contracts.RawValidator) + r.chains = make(map[rawValidationKey][]contracts.Validator) } key := rawValidationKey{stage: stage, module: normalizedModule} if _, exists := r.chains[key]; exists { return fmt.Errorf("raw validation chain for %q %q is already registered", stage, normalizedModule) } - r.chains[key] = append([]contracts.RawValidator(nil), chain...) + r.chains[key] = append([]contracts.Validator(nil), chain...) return nil } -func (r *RawValidationRegistry) Validators(stage ModuleStage, module string) []contracts.RawValidator { +func (r *RawValidationRegistry) Validators(stage ModuleStage, module string) []contracts.Validator { if r == nil { return nil } @@ -69,5 +74,5 @@ func (r *RawValidationRegistry) Validators(stage ModuleStage, module string) []c if len(chain) == 0 { return nil } - return append([]contracts.RawValidator(nil), chain...) + return append([]contracts.Validator(nil), chain...) } diff --git a/internal/framework/pipeline/runner.go b/internal/framework/pipeline/runner.go index 740b4ca..ce2d1d7 100644 --- a/internal/framework/pipeline/runner.go +++ b/internal/framework/pipeline/runner.go @@ -461,7 +461,7 @@ func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([ return nil, nil, nil } - request := contracts.RawValidationRequest{ + request := contracts.ValidationRequest{ Stage: string(target.stage), LaneID: target.laneID, ModuleKey: target.moduleKey, @@ -476,7 +476,7 @@ func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([ var warnings []contracts.Warning for _, validator := range validators { - result, err := validator.ValidateRaw(ctx, request) + result, err := validator.Validate(ctx, request) if err != nil { return nil, nil, fmt.Errorf("validate raw %s output with validator %q: %w", target.stage, validator.Name(), err) } diff --git a/internal/framework/pipeline/runner_test.go b/internal/framework/pipeline/runner_test.go index 57475e0..6f6d580 100644 --- a/internal/framework/pipeline/runner_test.go +++ b/internal/framework/pipeline/runner_test.go @@ -1796,14 +1796,15 @@ func (normalizer *runnerNormalizer) Normalize(ctx context.Context, req contracts } type runnerValidator struct { - name string - resultName string - decisions func([]artifacts.ArtifactCandidate) []contracts.ValidationDecision - warnings []contracts.Warning - err error - order *[]string - calls int - requests []contracts.ValidationRequest + name string + approved []bool + reason string + message string + warnings []contracts.Warning + err error + order *[]string + calls int + requests []contracts.ValidationRequest } type runnerRawValidator struct { @@ -1814,18 +1815,22 @@ type runnerRawValidator struct { warnings []contracts.Warning err error calls int - requests []contracts.RawValidationRequest + requests []contracts.ValidationRequest } func (validator *runnerRawValidator) Name() string { return validator.name } -func (validator *runnerRawValidator) ValidateRaw(ctx context.Context, req contracts.RawValidationRequest) (contracts.RawValidationResult, error) { +func (validator *runnerRawValidator) ExecutionClass() contracts.ExecutionClass { + return contracts.ExecutionClassDeterministic +} + +func (validator *runnerRawValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) { validator.calls++ validator.requests = append(validator.requests, req) if validator.err != nil { - return contracts.RawValidationResult{}, validator.err + return contracts.ValidationResult{}, validator.err } approved := true if len(validator.approved) > 0 { @@ -1835,7 +1840,7 @@ func (validator *runnerRawValidator) ValidateRaw(ctx context.Context, req contra } approved = validator.approved[index] } - return contracts.RawValidationResult{ + return contracts.ValidationResult{ Approved: approved, ReasonCode: validator.reason, Message: validator.message, @@ -1847,24 +1852,29 @@ func (validator *runnerValidator) Name() string { return validator.name } +func (validator *runnerValidator) ExecutionClass() contracts.ExecutionClass { + return contracts.ExecutionClassDeterministic +} + func (validator *runnerValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) { validator.calls++ validator.requests = append(validator.requests, req) if validator.order != nil { *validator.order = append(*validator.order, validator.name) } - resultName := validator.resultName - if resultName == "" { - resultName = validator.name - } - var decisions []contracts.ValidationDecision - if validator.decisions != nil { - decisions = validator.decisions(req.Candidates) + approved := true + if len(validator.approved) > 0 { + index := validator.calls - 1 + if index >= len(validator.approved) { + index = len(validator.approved) - 1 + } + approved = validator.approved[index] } return contracts.ValidationResult{ - ValidatorName: resultName, - Decisions: decisions, - Warnings: validator.warnings, + Approved: approved, + ReasonCode: validator.reason, + Message: validator.message, + Warnings: validator.warnings, }, validator.err } @@ -2027,7 +2037,7 @@ func assertRunError(t *testing.T, err error, want string) { } } -func rawValidationRegistry(t *testing.T, stage ModuleStage, module string, validators ...contracts.RawValidator) *RawValidationRegistry { +func rawValidationRegistry(t *testing.T, stage ModuleStage, module string, validators ...contracts.Validator) *RawValidationRegistry { t.Helper() registry := NewRawValidationRegistry() diff --git a/internal/framework/pipeline/validator_registry.go b/internal/framework/pipeline/validator_registry.go index 865a417..17e6b15 100644 --- a/internal/framework/pipeline/validator_registry.go +++ b/internal/framework/pipeline/validator_registry.go @@ -77,6 +77,11 @@ func (r *ValidatorRegistry) Build(key string) (contracts.Validator, error) { if validator.Name() != normalizedKey { return nil, fmt.Errorf("validator %q returned name %q", normalizedKey, validator.Name()) } + switch validator.ExecutionClass() { + case contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked: + default: + return nil, fmt.Errorf("validator %q returned unsupported execution class %q", normalizedKey, validator.ExecutionClass()) + } return validator, nil } diff --git a/internal/framework/pipeline/validator_registry_test.go b/internal/framework/pipeline/validator_registry_test.go index 286d59d..a20d497 100644 --- a/internal/framework/pipeline/validator_registry_test.go +++ b/internal/framework/pipeline/validator_registry_test.go @@ -1,6 +1,7 @@ package pipeline import ( + "context" "testing" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" @@ -56,3 +57,33 @@ func TestValidatorRegistryBehavior(t *testing.T) { }, }) } + +func TestValidatorRegistryRejectsUnsupportedExecutionClass(t *testing.T) { + registry := NewValidatorRegistry() + if err := registry.Register("invalid-validator", func() (contracts.Validator, error) { + return invalidExecutionClassValidator{name: "invalid-validator"}, nil + }); err != nil { + t.Fatalf("Register() error = %v, want nil", err) + } + + _, err := registry.Build("invalid-validator") + if err == nil { + t.Fatal("Build() error = nil, want unsupported execution class error") + } +} + +type invalidExecutionClassValidator struct { + name string +} + +func (validator invalidExecutionClassValidator) Name() string { + return validator.name +} + +func (validator invalidExecutionClassValidator) ExecutionClass() contracts.ExecutionClass { + return contracts.ExecutionClass("unsupported") +} + +func (validator invalidExecutionClassValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) { + return contracts.ValidationResult{Approved: true}, nil +} diff --git a/internal/framework/validate/validate.go b/internal/framework/validate/validate.go index 21ad62c..7bc6c24 100644 --- a/internal/framework/validate/validate.go +++ b/internal/framework/validate/validate.go @@ -1,10 +1,8 @@ package validate import ( - "fmt" "strings" - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) @@ -12,53 +10,18 @@ const ( ReasonApproved = "approved" ) -func Approved(candidateIndex int) contracts.ValidationDecision { - return contracts.ValidationDecision{ - CandidateIndex: candidateIndex, - Approved: true, - ReasonCode: ReasonApproved, - Message: ReasonApproved, +func Approved() contracts.ValidationResult { + return contracts.ValidationResult{ + Approved: true, + ReasonCode: ReasonApproved, + Message: ReasonApproved, } } -func Rejected(candidateIndex int, reasonCode string, message string) contracts.ValidationDecision { - return contracts.ValidationDecision{ - CandidateIndex: candidateIndex, - Approved: false, - ReasonCode: strings.TrimSpace(reasonCode), - Message: strings.TrimSpace(message), +func Rejected(reasonCode string, message string) contracts.ValidationResult { + return contracts.ValidationResult{ + Approved: false, + ReasonCode: strings.TrimSpace(reasonCode), + Message: strings.TrimSpace(message), } } - -func EnforceDecisionCardinality(candidates []artifacts.ArtifactCandidate, decisions []contracts.ValidationDecision) error { - if len(candidates) != len(decisions) { - return fmt.Errorf("validator returned %d decisions for %d candidates", len(decisions), len(candidates)) - } - - expected := make(map[int]struct{}, len(candidates)) - for _, candidate := range candidates { - if _, ok := expected[candidate.Index]; ok { - return fmt.Errorf("candidate index %d is duplicated", candidate.Index) - } - expected[candidate.Index] = struct{}{} - } - - seen := make(map[int]struct{}, len(decisions)) - for _, decision := range decisions { - if _, ok := expected[decision.CandidateIndex]; !ok { - return fmt.Errorf("validator returned decision for unknown candidate index %d", decision.CandidateIndex) - } - if _, ok := seen[decision.CandidateIndex]; ok { - return fmt.Errorf("validator returned duplicate decision for candidate index %d", decision.CandidateIndex) - } - seen[decision.CandidateIndex] = struct{}{} - } - - for candidateIndex := range expected { - if _, ok := seen[candidateIndex]; !ok { - return fmt.Errorf("validator did not return decision for candidate index %d", candidateIndex) - } - } - - return nil -} diff --git a/internal/framework/validate/validate_test.go b/internal/framework/validate/validate_test.go index 8d51662..c39df16 100644 --- a/internal/framework/validate/validate_test.go +++ b/internal/framework/validate/validate_test.go @@ -1,105 +1,40 @@ package validate import ( - "strings" "testing" - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) func TestApproved(t *testing.T) { - decision := Approved(7) + result := Approved() - if decision.CandidateIndex != 7 { - t.Fatalf("CandidateIndex = %d, want 7", decision.CandidateIndex) - } - if !decision.Approved { + if !result.Approved { t.Fatal("Approved = false, want true") } - if decision.ReasonCode != ReasonApproved { - t.Fatalf("ReasonCode = %q, want %q", decision.ReasonCode, ReasonApproved) + if result.ReasonCode != ReasonApproved { + t.Fatalf("ReasonCode = %q, want %q", result.ReasonCode, ReasonApproved) } - if decision.Message != "approved" { - t.Fatalf("Message = %q, want approved", decision.Message) + if result.Message != "approved" { + t.Fatalf("Message = %q, want approved", result.Message) } } func TestRejectedTrimsReasonAndMessage(t *testing.T) { - decision := Rejected(3, " invalid ", "\tmessage\n") + result := Rejected(" invalid ", "\tmessage\n") - if decision.CandidateIndex != 3 { - t.Fatalf("CandidateIndex = %d, want 3", decision.CandidateIndex) - } - if decision.Approved { + if result.Approved { t.Fatal("Approved = true, want false") } - if decision.ReasonCode != "invalid" { - t.Fatalf("ReasonCode = %q, want invalid", decision.ReasonCode) + if result.ReasonCode != "invalid" { + t.Fatalf("ReasonCode = %q, want invalid", result.ReasonCode) } - if decision.Message != "message" { - t.Fatalf("Message = %q, want message", decision.Message) + if result.Message != "message" { + t.Fatalf("Message = %q, want message", result.Message) } } -func TestEnforceDecisionCardinalityAllowsNonZeroCandidateIndices(t *testing.T) { - candidates := []artifacts.ArtifactCandidate{{Index: 4}, {Index: 8}} - decisions := []contracts.ValidationDecision{Approved(8), Approved(4)} - - if err := EnforceDecisionCardinality(candidates, decisions); err != nil { - t.Fatalf("EnforceDecisionCardinality() error = %v, want nil", err) - } -} - -func TestEnforceDecisionCardinalityAllowsEmptyInputs(t *testing.T) { - if err := EnforceDecisionCardinality(nil, nil); err != nil { - t.Fatalf("EnforceDecisionCardinality() error = %v, want nil", err) - } -} - -func TestEnforceDecisionCardinalityRejectsUnknownDecisionIndex(t *testing.T) { - err := EnforceDecisionCardinality( - []artifacts.ArtifactCandidate{{Index: 1}}, - []contracts.ValidationDecision{Approved(2)}, - ) - - assertCardinalityError(t, err, "unknown candidate index 2") -} - -func TestEnforceDecisionCardinalityRejectsDuplicateDecisionIndex(t *testing.T) { - err := EnforceDecisionCardinality( - []artifacts.ArtifactCandidate{{Index: 1}, {Index: 2}}, - []contracts.ValidationDecision{Approved(1), Approved(1)}, - ) - - assertCardinalityError(t, err, "duplicate decision") -} - -func TestEnforceDecisionCardinalityRejectsMissingDecisionIndex(t *testing.T) { - err := EnforceDecisionCardinality( - []artifacts.ArtifactCandidate{{Index: 1}, {Index: 2}}, - []contracts.ValidationDecision{Approved(1)}, - ) - - assertCardinalityError(t, err, "1 decisions for 2 candidates") -} - -func TestEnforceDecisionCardinalityRejectsDuplicateCandidateIndex(t *testing.T) { - err := EnforceDecisionCardinality( - []artifacts.ArtifactCandidate{{Index: 1}, {Index: 1}}, - []contracts.ValidationDecision{Approved(1), Approved(1)}, - ) - - assertCardinalityError(t, err, "candidate index 1 is duplicated") -} - -func assertCardinalityError(t *testing.T, err error, want string) { - t.Helper() - - if err == nil { - t.Fatal("EnforceDecisionCardinality() error = nil, want error") - } - if !strings.Contains(err.Error(), want) { - t.Fatalf("EnforceDecisionCardinality() error = %q, want substring %q", err.Error(), want) - } +func TestHelpersReturnValidationResults(t *testing.T) { + var _ contracts.ValidationResult = Approved() + var _ contracts.ValidationResult = Rejected("reason", "message") } diff --git a/internal/modules/extract/dnd/spells/validator.go b/internal/modules/extract/dnd/spells/validator.go deleted file mode 100644 index 1b33ef6..0000000 --- a/internal/modules/extract/dnd/spells/validator.go +++ /dev/null @@ -1,168 +0,0 @@ -package spells - -import ( - "context" - "encoding/json" - "fmt" - "strings" - - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" - "gitea.maximumdirect.net/eric/notarius/internal/core/source" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" - "gitea.maximumdirect.net/eric/notarius/internal/framework/validate" -) - -const ( - shapeValidatorName = "dnd/spells/shape" - sourceRefValidatorName = "dnd/spells/source_refs" - - reasonInvalidPayload = "invalid_payload" - reasonMissingRequiredField = "missing_required_field" - reasonMissingSourceRef = "missing_source_ref" - reasonInvalidSourceRef = "invalid_source_ref" - reasonSpellNotNearSource = "spell_not_near_source" -) - -var _ contracts.Validator = ShapeValidator{} -var _ contracts.Validator = SourceRefValidator{} - -type ShapeValidator struct{} - -func (validator ShapeValidator) Name() string { - return shapeValidatorName -} - -func (validator ShapeValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) { - decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates)) - for _, candidate := range req.Candidates { - decisions = append(decisions, validateShape(candidate)) - } - return contracts.ValidationResult{ - ValidatorName: validator.Name(), - Decisions: decisions, - }, nil -} - -type SourceRefValidator struct{} - -func (validator SourceRefValidator) Name() string { - return sourceRefValidatorName -} - -func (validator SourceRefValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) { - if req.Source == nil { - return contracts.ValidationResult{}, fmt.Errorf("dnd spells source refs validator: source must not be nil") - } - - decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates)) - var warnings []contracts.Warning - for _, candidate := range req.Candidates { - decisions = append(decisions, validateSourceRefs(req.Source, candidate)) - warnings = append(warnings, sourceRelatednessWarnings(req.Source, candidate)...) - } - return contracts.ValidationResult{ - ValidatorName: validator.Name(), - Decisions: decisions, - Warnings: warnings, - }, nil -} - -func validateShape(candidate artifacts.ArtifactCandidate) contracts.ValidationDecision { - var payload SpellCast - if err := json.Unmarshal(candidate.Payload, &payload); err != nil { - return validate.Rejected(candidate.Index, reasonInvalidPayload, fmt.Sprintf("invalid spell cast payload: %v", err)) - } - for _, field := range requiredSpellCastFields(payload) { - if strings.TrimSpace(field.value) == "" { - return validate.Rejected(candidate.Index, reasonMissingRequiredField, fmt.Sprintf("missing required field %q", field.name)) - } - } - return validate.Approved(candidate.Index) -} - -func validateSourceRefs(doc *source.SourceDocument, candidate artifacts.ArtifactCandidate) contracts.ValidationDecision { - if len(candidate.SourceRefs) == 0 { - return validate.Rejected(candidate.Index, reasonMissingSourceRef, "spell cast candidate must include at least one source ref") - } - for _, ref := range candidate.SourceRefs { - if err := source.ValidateRef(doc, ref); err != nil { - return validate.Rejected(candidate.Index, reasonInvalidSourceRef, err.Error()) - } - } - return validate.Approved(candidate.Index) -} - -func sourceRelatednessWarnings(doc *source.SourceDocument, candidate artifacts.ArtifactCandidate) []contracts.Warning { - if doc == nil || len(candidate.SourceRefs) == 0 { - return nil - } - var payload SpellCast - if err := json.Unmarshal(candidate.Payload, &payload); err != nil { - return nil - } - spell := strings.TrimSpace(payload.Spell) - if spell == "" { - return nil - } - - needle := strings.ToLower(spell) - for _, ref := range candidate.SourceRefs { - text, ok := sourceRefText(doc, ref) - if !ok { - continue - } - if strings.Contains(strings.ToLower(text), needle) { - return nil - } - } - - return []contracts.Warning{ - { - Scope: fmt.Sprintf("candidate.%d", candidate.Index), - ReasonCode: reasonSpellNotNearSource, - Message: fmt.Sprintf("spell %q was not found in the cited source text", spell), - }, - } -} - -func sourceRefText(doc *source.SourceDocument, ref source.SourceRef) (string, bool) { - if err := source.ValidateRef(doc, ref); err != nil { - return "", false - } - start := -1 - end := -1 - for i, unit := range doc.Units { - if unit.ID == ref.StartUnitID { - start = i - } - if unit.ID == ref.EndUnitID { - end = i - } - } - if start < 0 || end < start { - return "", false - } - var b strings.Builder - for i := start; i <= end; i++ { - if b.Len() > 0 { - b.WriteString("\n") - } - b.WriteString(doc.Units[i].Text) - } - return b.String(), true -} - -func requiredSpellCastFields(payload SpellCast) []struct { - name string - value string -} { - return []struct { - name string - value string - }{ - {name: "caster", value: payload.Caster}, - {name: "spell", value: payload.Spell}, - {name: "effect", value: payload.Effect}, - {name: "narrative_description", value: payload.NarrativeDescription}, - } -} diff --git a/internal/modules/extract/dnd/spells/validator_test.go b/internal/modules/extract/dnd/spells/validator_test.go deleted file mode 100644 index 3ce0e95..0000000 --- a/internal/modules/extract/dnd/spells/validator_test.go +++ /dev/null @@ -1,277 +0,0 @@ -package spells - -import ( - "context" - "encoding/json" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" - "gitea.maximumdirect.net/eric/notarius/internal/core/source" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" - "gitea.maximumdirect.net/eric/notarius/internal/framework/validate" -) - -func TestValidatorsApproveValidCandidate(t *testing.T) { - candidate := validSpellCandidate(7) - - shapeResult, err := ShapeValidator{}.Validate(context.Background(), contracts.ValidationRequest{ - Candidates: []artifacts.ArtifactCandidate{candidate}, - }) - if err != nil { - t.Fatalf("ShapeValidator.Validate() error = %v, want nil", err) - } - assertSingleDecision(t, shapeResult, shapeValidatorName, 7, true, validate.ReasonApproved) - - sourceRefResult, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{ - Source: promptSourceDocument(), - Candidates: []artifacts.ArtifactCandidate{candidate}, - }) - if err != nil { - t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err) - } - assertSingleDecision(t, sourceRefResult, sourceRefValidatorName, 7, true, validate.ReasonApproved) - if len(sourceRefResult.Warnings) != 0 { - t.Fatalf("warnings = %#v, want none", sourceRefResult.Warnings) - } -} - -func TestShapeValidatorRejectsMalformedPayload(t *testing.T) { - candidate := validSpellCandidate(3) - candidate.Payload = json.RawMessage(`{"caster":`) - - result, err := ShapeValidator{}.Validate(context.Background(), contracts.ValidationRequest{ - Candidates: []artifacts.ArtifactCandidate{candidate}, - }) - if err != nil { - t.Fatalf("ShapeValidator.Validate() error = %v, want nil", err) - } - assertSingleDecision(t, result, shapeValidatorName, 3, false, reasonInvalidPayload) -} - -func TestShapeValidatorRejectsBlankRequiredFields(t *testing.T) { - tests := []struct { - name string - mutate func(*SpellCast) - }{ - {name: "caster", mutate: func(payload *SpellCast) { payload.Caster = " \t" }}, - {name: "spell", mutate: func(payload *SpellCast) { payload.Spell = "" }}, - {name: "effect", mutate: func(payload *SpellCast) { payload.Effect = "\n" }}, - {name: "narrative description", mutate: func(payload *SpellCast) { payload.NarrativeDescription = " " }}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - payload := validSpellPayload() - tt.mutate(&payload) - candidate := validSpellCandidate(5) - candidate.Payload = mustSpellPayload(t, payload) - - result, err := ShapeValidator{}.Validate(context.Background(), contracts.ValidationRequest{ - Candidates: []artifacts.ArtifactCandidate{candidate}, - }) - if err != nil { - t.Fatalf("ShapeValidator.Validate() error = %v, want nil", err) - } - assertSingleDecision(t, result, shapeValidatorName, 5, false, reasonMissingRequiredField) - }) - } -} - -func TestShapeValidatorDoesNotRequireSourceDocument(t *testing.T) { - result, err := ShapeValidator{}.Validate(context.Background(), contracts.ValidationRequest{ - Candidates: []artifacts.ArtifactCandidate{validSpellCandidate(11)}, - }) - if err != nil { - t.Fatalf("ShapeValidator.Validate() error = %v, want nil", err) - } - assertSingleDecision(t, result, shapeValidatorName, 11, true, validate.ReasonApproved) -} - -func TestSourceRefValidatorRejectsMissingRefs(t *testing.T) { - candidate := validSpellCandidate(13) - candidate.SourceRefs = nil - - result, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{ - Source: promptSourceDocument(), - Candidates: []artifacts.ArtifactCandidate{candidate}, - }) - if err != nil { - t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err) - } - assertSingleDecision(t, result, sourceRefValidatorName, 13, false, reasonMissingSourceRef) -} - -func TestSourceRefValidatorRejectsInvalidRefs(t *testing.T) { - tests := []struct { - name string - ref source.SourceRef - want string - }{ - { - name: "unknown source id", - ref: source.SourceRef{SourceID: "session-beta", StartUnitID: 1, EndUnitID: 2}, - want: "does not match", - }, - { - name: "unknown start unit", - ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 999, EndUnitID: 2}, - want: "start_unit_id", - }, - { - name: "unknown end unit", - ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 999}, - want: "end_unit_id", - }, - { - name: "reversed unit range", - ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 1}, - want: "appears after", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - candidate := validSpellCandidate(17) - candidate.SourceRefs = []source.SourceRef{tt.ref} - - result, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{ - Source: promptSourceDocument(), - Candidates: []artifacts.ArtifactCandidate{candidate}, - }) - if err != nil { - t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err) - } - assertSingleDecision(t, result, sourceRefValidatorName, 17, false, reasonInvalidSourceRef) - if !strings.Contains(result.Decisions[0].Message, tt.want) { - t.Fatalf("Message = %q, want substring %q", result.Decisions[0].Message, tt.want) - } - }) - } -} - -func TestSourceRefValidatorWarnsWhenSpellNameIsNotInCitedSource(t *testing.T) { - candidate := validSpellCandidate(31) - payload := validSpellPayload() - payload.Spell = "Shield" - candidate.Payload = mustSpellPayload(t, payload) - - result, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{ - Source: promptSourceDocument(), - Candidates: []artifacts.ArtifactCandidate{candidate}, - }) - if err != nil { - t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err) - } - assertSingleDecision(t, result, sourceRefValidatorName, 31, true, validate.ReasonApproved) - if len(result.Warnings) != 1 { - t.Fatalf("warnings = %#v, want one relatedness warning", result.Warnings) - } - warning := result.Warnings[0] - if warning.ReasonCode != reasonSpellNotNearSource || !strings.Contains(warning.Message, "Shield") { - t.Fatalf("warning = %#v, want spell relatedness warning", warning) - } -} - -func TestSourceRefValidatorRequiresSourceDocument(t *testing.T) { - _, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{ - Candidates: []artifacts.ArtifactCandidate{validSpellCandidate(19)}, - }) - if err == nil { - t.Fatal("SourceRefValidator.Validate() error = nil, want source error") - } - if !strings.Contains(err.Error(), "dnd spells") || !strings.Contains(err.Error(), "source") { - t.Fatalf("SourceRefValidator.Validate() error = %q, want source context", err.Error()) - } -} - -func TestValidatorsPreserveCandidateIndexes(t *testing.T) { - candidates := []artifacts.ArtifactCandidate{ - validSpellCandidate(23), - validSpellCandidate(29), - } - - shapeResult, err := ShapeValidator{}.Validate(context.Background(), contracts.ValidationRequest{ - Candidates: candidates, - }) - if err != nil { - t.Fatalf("ShapeValidator.Validate() error = %v, want nil", err) - } - assertDecisionIndexes(t, shapeResult.Decisions, []int{23, 29}) - - sourceRefResult, err := SourceRefValidator{}.Validate(context.Background(), contracts.ValidationRequest{ - Source: promptSourceDocument(), - Candidates: candidates, - }) - if err != nil { - t.Fatalf("SourceRefValidator.Validate() error = %v, want nil", err) - } - assertDecisionIndexes(t, sourceRefResult.Decisions, []int{23, 29}) -} - -func validSpellCandidate(index int) artifacts.ArtifactCandidate { - return artifacts.ArtifactCandidate{ - Index: index, - Payload: spellPayload(validSpellPayload()), - SourceRefs: []source.SourceRef{ - {SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 2}, - }, - } -} - -func validSpellPayload() SpellCast { - return SpellCast{ - Caster: "Aria", - Spell: "Cure Wounds", - Effect: "Heals an injured ally.", - NarrativeDescription: "Aria restores the fighter after the fight.", - } -} - -func mustSpellPayload(t *testing.T, payload SpellCast) json.RawMessage { - t.Helper() - - return spellPayload(payload) -} - -func spellPayload(payload SpellCast) json.RawMessage { - encoded, err := json.Marshal(payload) - if err != nil { - panic(err) - } - return encoded -} - -func assertSingleDecision(t *testing.T, result contracts.ValidationResult, wantName string, wantIndex int, wantApproved bool, wantReason string) { - t.Helper() - - if result.ValidatorName != wantName { - t.Fatalf("ValidatorName = %q, want %q", result.ValidatorName, wantName) - } - if len(result.Decisions) != 1 { - t.Fatalf("len(Decisions) = %d, want 1", len(result.Decisions)) - } - decision := result.Decisions[0] - if decision.CandidateIndex != wantIndex { - t.Fatalf("CandidateIndex = %d, want %d", decision.CandidateIndex, wantIndex) - } - if decision.Approved != wantApproved { - t.Fatalf("Approved = %t, want %t", decision.Approved, wantApproved) - } - if decision.ReasonCode != wantReason { - t.Fatalf("ReasonCode = %q, want %q", decision.ReasonCode, wantReason) - } -} - -func assertDecisionIndexes(t *testing.T, decisions []contracts.ValidationDecision, want []int) { - t.Helper() - - if len(decisions) != len(want) { - t.Fatalf("len(Decisions) = %d, want %d", len(decisions), len(want)) - } - for i := range want { - if decisions[i].CandidateIndex != want[i] { - t.Fatalf("Decisions[%d].CandidateIndex = %d, want %d", i, decisions[i].CandidateIndex, want[i]) - } - } -}