Add D&D scene description validation

internal/modules/dnd/validate/scenedescriptions/source_relatedness/validator.go

@@ -0,0 +1,139 @@
+// Package sourcerelatedness warns when scene prose has no lexical grounding.
+package sourcerelatedness
+
+import (
+	"context"
+	"fmt"
+	"unicode/utf8"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
+	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
+	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
+	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape"
+)
+
+const (
+	Key               = "extract/dnd/scene-descriptions/source_relatedness"
+	WarningReasonCode = "scene_description_not_near_source"
+	OmittedReasonCode = "scene_description_relatedness_warnings_omitted"
+	policy            = "dnd.scene_descriptions.validator.source_relatedness.v1"
+)
+
+var stopwords = map[string]struct{}{
+	"a": {}, "an": {}, "and": {}, "are": {}, "as": {}, "at": {}, "be": {}, "but": {}, "by": {}, "for": {}, "from": {},
+	"had": {}, "has": {}, "have": {}, "he": {}, "her": {}, "him": {}, "his": {}, "in": {}, "into": {}, "is": {}, "it": {}, "its": {},
+	"of": {}, "on": {}, "or": {}, "she": {}, "that": {}, "the": {}, "their": {}, "them": {}, "they": {}, "this": {}, "to": {},
+	"was": {}, "were": {}, "with": {},
+}
+
+type Options struct{}
+type Validator struct{}
+
+var _ contracts.TypedValidator[dnd.SceneDescriptionList] = (*Validator)(nil)
+var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil)
+
+func New(Options) *Validator      { return &Validator{} }
+func (v *Validator) Name() string { return Key }
+func (v *Validator) ExecutionClass() contracts.ExecutionClass {
+	return contracts.ExecutionClassDeterministic
+}
+func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
+	return []pipeline.CheckpointFingerprint{{Name: "policy", Value: policy}}
+}
+
+func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.ValidationResult, error) {
+	if shape.Validate(req.Value) != nil || !sourceRefsValid(req.Source, req.Value) {
+		return contracts.ValidationResult{Approved: true}, nil
+	}
+	warnings := make([]contracts.Warning, 0)
+	for index, scene := range req.Value.Scenes {
+		citedText, err := shared.CitedText(req.Source, []source.SourceRef{scene.SourceRef})
+		if err != nil {
+			return contracts.ValidationResult{Approved: true}, nil
+		}
+		citedTokens := tokenSet(citedText)
+		if !hasGroundedToken(citedTokens, scene.Title) {
+			warnings = append(warnings, warning(index, "title"))
+		}
+		if !hasGroundedToken(citedTokens, scene.Summary) {
+			warnings = append(warnings, warning(index, "summary"))
+		}
+	}
+	return contracts.ValidationResult{
+		Approved: true,
+		Warnings: diagnostics.LimitWarnings(warnings, "scenes", OmittedReasonCode),
+	}, nil
+}
+
+func sourceRefsValid(doc *source.SourceDocument, value dnd.SceneDescriptionList) bool {
+	for _, scene := range value.Scenes {
+		if source.ValidateRef(doc, scene.SourceRef) != nil {
+			return false
+		}
+	}
+	return true
+}
+
+func tokenSet(value string) map[string]struct{} {
+	tokens := make(map[string]struct{})
+	for _, token := range shared.NormalizedTokens(value) {
+		if significant(token) {
+			tokens[token] = struct{}{}
+		}
+	}
+	return tokens
+}
+
+func hasGroundedToken(cited map[string]struct{}, value string) bool {
+	for _, token := range shared.NormalizedTokens(value) {
+		if !significant(token) {
+			continue
+		}
+		if _, ok := cited[token]; ok {
+			return true
+		}
+	}
+	return false
+}
+
+func significant(token string) bool {
+	if utf8.RuneCountInString(token) < 3 {
+		return false
+	}
+	_, ignored := stopwords[token]
+	return !ignored
+}
+
+func warning(index int, field string) contracts.Warning {
+	return contracts.Warning{
+		Scope:      fmt.Sprintf("scenes[%d].%s", index, field),
+		ReasonCode: WarningReasonCode,
+		Message:    "scene description " + field + " has no significant token in cited source text",
+	}
+}
+
+func Spec() pipeline.ValidatorSpec {
+	return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
+}
+
+func Register(registry *pipeline.ValidatorRegistry) error {
+	return pipeline.RegisterTypedValidatorBuilder(registry, dnd.SceneDescriptionListKind, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.TypedValidator[dnd.SceneDescriptionList], error) {
+		options, err := DecodeOptions(request.Options)
+		if err != nil {
+			return nil, err
+		}
+		return New(options), nil
+	})
+}
+
+func DecodeOptions(options map[string]any) (Options, error) {
+	if err := pipeline.RejectUnknownOptions(options); err != nil {
+		return Options{}, err
+	}
+	return Options{}, nil
+}
+
+func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }