// Package sourcerelatedness warns when scene prose has no lexical grounding. package sourcerelatedness import ( "context" "fmt" "unicode/utf8" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape" ) const ( Key = "extract/dnd/scene-descriptions/source_relatedness" WarningReasonCode = "scene_description_not_near_source" OmittedReasonCode = "scene_description_relatedness_warnings_omitted" policy = "dnd.scene_descriptions.validator.source_relatedness.v1" ) var stopwords = map[string]struct{}{ "a": {}, "an": {}, "and": {}, "are": {}, "as": {}, "at": {}, "be": {}, "but": {}, "by": {}, "for": {}, "from": {}, "had": {}, "has": {}, "have": {}, "he": {}, "her": {}, "him": {}, "his": {}, "in": {}, "into": {}, "is": {}, "it": {}, "its": {}, "of": {}, "on": {}, "or": {}, "she": {}, "that": {}, "the": {}, "their": {}, "them": {}, "they": {}, "this": {}, "to": {}, "was": {}, "were": {}, "with": {}, } type Options struct{} type Validator struct{} var _ contracts.TypedValidator[dnd.SceneDescriptionList] = (*Validator)(nil) var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil) func New(Options) *Validator { return &Validator{} } func (v *Validator) Name() string { return Key } func (v *Validator) ExecutionClass() contracts.ExecutionClass { return contracts.ExecutionClassDeterministic } func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint { return []pipeline.CheckpointFingerprint{{Name: "policy", Value: policy}} } func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.ValidationResult, error) { if shape.Validate(req.Value) != nil || !sourceRefsValid(req.Source, req.Value) { return contracts.ValidationResult{Approved: true}, nil } warnings := make([]contracts.Warning, 0) for index, scene := range req.Value.Scenes { citedText, err := shared.CitedText(req.Source, []source.SourceRef{scene.SourceRef}) if err != nil { return contracts.ValidationResult{Approved: true}, nil } citedTokens := tokenSet(citedText) if !hasGroundedToken(citedTokens, scene.Title) { warnings = append(warnings, warning(index, "title")) } if !hasGroundedToken(citedTokens, scene.Summary) { warnings = append(warnings, warning(index, "summary")) } } return contracts.ValidationResult{ Approved: true, Warnings: diagnostics.LimitWarnings(warnings, "scenes", OmittedReasonCode), }, nil } func sourceRefsValid(doc *source.SourceDocument, value dnd.SceneDescriptionList) bool { for _, scene := range value.Scenes { if source.ValidateRef(doc, scene.SourceRef) != nil { return false } } return true } func tokenSet(value string) map[string]struct{} { tokens := make(map[string]struct{}) for _, token := range shared.NormalizedTokens(value) { if significant(token) { tokens[token] = struct{}{} } } return tokens } func hasGroundedToken(cited map[string]struct{}, value string) bool { for _, token := range shared.NormalizedTokens(value) { if !significant(token) { continue } if _, ok := cited[token]; ok { return true } } return false } func significant(token string) bool { if utf8.RuneCountInString(token) < 3 { return false } _, ignored := stopwords[token] return !ignored } func warning(index int, field string) contracts.Warning { return contracts.Warning{ Scope: fmt.Sprintf("scenes[%d].%s", index, field), ReasonCode: WarningReasonCode, Message: "scene description " + field + " has no significant token in cited source text", } } func Spec() pipeline.ValidatorSpec { return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic} } func Register(registry *pipeline.ValidatorRegistry) error { return pipeline.RegisterTypedValidatorBuilder(registry, dnd.SceneDescriptionListKind, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.TypedValidator[dnd.SceneDescriptionList], error) { options, err := DecodeOptions(request.Options) if err != nil { return nil, err } return New(options), nil }) } func DecodeOptions(options map[string]any) (Options, error) { if err := pipeline.RejectUnknownOptions(options); err != nil { return Options{}, err } return Options{}, nil } func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }