Add D&D scene description validation
This commit is contained in:
@@ -0,0 +1,139 @@
|
||||
// Package sourcerelatedness warns when scene prose has no lexical grounding.
|
||||
package sourcerelatedness
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape"
|
||||
)
|
||||
|
||||
const (
|
||||
Key = "extract/dnd/scene-descriptions/source_relatedness"
|
||||
WarningReasonCode = "scene_description_not_near_source"
|
||||
OmittedReasonCode = "scene_description_relatedness_warnings_omitted"
|
||||
policy = "dnd.scene_descriptions.validator.source_relatedness.v1"
|
||||
)
|
||||
|
||||
var stopwords = map[string]struct{}{
|
||||
"a": {}, "an": {}, "and": {}, "are": {}, "as": {}, "at": {}, "be": {}, "but": {}, "by": {}, "for": {}, "from": {},
|
||||
"had": {}, "has": {}, "have": {}, "he": {}, "her": {}, "him": {}, "his": {}, "in": {}, "into": {}, "is": {}, "it": {}, "its": {},
|
||||
"of": {}, "on": {}, "or": {}, "she": {}, "that": {}, "the": {}, "their": {}, "them": {}, "they": {}, "this": {}, "to": {},
|
||||
"was": {}, "were": {}, "with": {},
|
||||
}
|
||||
|
||||
type Options struct{}
|
||||
type Validator struct{}
|
||||
|
||||
var _ contracts.TypedValidator[dnd.SceneDescriptionList] = (*Validator)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil)
|
||||
|
||||
func New(Options) *Validator { return &Validator{} }
|
||||
func (v *Validator) Name() string { return Key }
|
||||
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
return []pipeline.CheckpointFingerprint{{Name: "policy", Value: policy}}
|
||||
}
|
||||
|
||||
func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.ValidationResult, error) {
|
||||
if shape.Validate(req.Value) != nil || !sourceRefsValid(req.Source, req.Value) {
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
warnings := make([]contracts.Warning, 0)
|
||||
for index, scene := range req.Value.Scenes {
|
||||
citedText, err := shared.CitedText(req.Source, []source.SourceRef{scene.SourceRef})
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
citedTokens := tokenSet(citedText)
|
||||
if !hasGroundedToken(citedTokens, scene.Title) {
|
||||
warnings = append(warnings, warning(index, "title"))
|
||||
}
|
||||
if !hasGroundedToken(citedTokens, scene.Summary) {
|
||||
warnings = append(warnings, warning(index, "summary"))
|
||||
}
|
||||
}
|
||||
return contracts.ValidationResult{
|
||||
Approved: true,
|
||||
Warnings: diagnostics.LimitWarnings(warnings, "scenes", OmittedReasonCode),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func sourceRefsValid(doc *source.SourceDocument, value dnd.SceneDescriptionList) bool {
|
||||
for _, scene := range value.Scenes {
|
||||
if source.ValidateRef(doc, scene.SourceRef) != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func tokenSet(value string) map[string]struct{} {
|
||||
tokens := make(map[string]struct{})
|
||||
for _, token := range shared.NormalizedTokens(value) {
|
||||
if significant(token) {
|
||||
tokens[token] = struct{}{}
|
||||
}
|
||||
}
|
||||
return tokens
|
||||
}
|
||||
|
||||
func hasGroundedToken(cited map[string]struct{}, value string) bool {
|
||||
for _, token := range shared.NormalizedTokens(value) {
|
||||
if !significant(token) {
|
||||
continue
|
||||
}
|
||||
if _, ok := cited[token]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func significant(token string) bool {
|
||||
if utf8.RuneCountInString(token) < 3 {
|
||||
return false
|
||||
}
|
||||
_, ignored := stopwords[token]
|
||||
return !ignored
|
||||
}
|
||||
|
||||
func warning(index int, field string) contracts.Warning {
|
||||
return contracts.Warning{
|
||||
Scope: fmt.Sprintf("scenes[%d].%s", index, field),
|
||||
ReasonCode: WarningReasonCode,
|
||||
Message: "scene description " + field + " has no significant token in cited source text",
|
||||
}
|
||||
}
|
||||
|
||||
func Spec() pipeline.ValidatorSpec {
|
||||
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||
return pipeline.RegisterTypedValidatorBuilder(registry, dnd.SceneDescriptionListKind, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.TypedValidator[dnd.SceneDescriptionList], error) {
|
||||
options, err := DecodeOptions(request.Options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(options), nil
|
||||
})
|
||||
}
|
||||
|
||||
func DecodeOptions(options map[string]any) (Options, error) {
|
||||
if err := pipeline.RejectUnknownOptions(options); err != nil {
|
||||
return Options{}, err
|
||||
}
|
||||
return Options{}, nil
|
||||
}
|
||||
|
||||
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
|
||||
@@ -0,0 +1,78 @@
|
||||
package sourcerelatedness
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func TestValidatorWarnsIndependentlyForUngroundedTitleAndSummary(t *testing.T) {
|
||||
doc := document()
|
||||
value := dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{
|
||||
scene("one", "Watchtower arrival", "The group enters the ruined watchtower."),
|
||||
scene("two", "Moonlit council", "Whispers linger."),
|
||||
scene("three", "The and to", "An or the"),
|
||||
}}
|
||||
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.SceneDescriptionList]{Source: doc, Value: value})
|
||||
if err != nil || !result.Approved {
|
||||
t.Fatalf("Validate() = %#v, %v", result, err)
|
||||
}
|
||||
wantScopes := []string{"scenes[1].title", "scenes[1].summary", "scenes[2].title", "scenes[2].summary"}
|
||||
if len(result.Warnings) != len(wantScopes) {
|
||||
t.Fatalf("warnings = %#v, want %d", result.Warnings, len(wantScopes))
|
||||
}
|
||||
for index, scope := range wantScopes {
|
||||
item := result.Warnings[index]
|
||||
if item.Scope != scope || item.ReasonCode != WarningReasonCode {
|
||||
t.Fatalf("warning[%d] = %#v, want scope %q", index, item, scope)
|
||||
}
|
||||
if len(item.Message) > 4096 || strings.Contains(item.Message, doc.Units[0].Text) {
|
||||
t.Fatalf("warning[%d] is not safely bounded: %#v", index, item)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorUsesTranscriptOnlyAndDefersInvalidInputs(t *testing.T) {
|
||||
doc := document()
|
||||
value := dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{scene("one", "Greencloak", "Greencloak arrives.")}}
|
||||
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.SceneDescriptionList]{
|
||||
Source: doc, Value: value,
|
||||
References: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||
"glossary": {Items: []contracts.ReferenceItem{{Content: []byte("Greencloak: title")}}},
|
||||
}},
|
||||
})
|
||||
if err != nil || len(result.Warnings) != 2 {
|
||||
t.Fatalf("Validate() = %#v, %v; want transcript-only warnings", result, err)
|
||||
}
|
||||
malformed, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.SceneDescriptionList]{Source: doc, Value: dnd.SceneDescriptionList{}})
|
||||
if err != nil || !malformed.Approved || len(malformed.Warnings) != 0 {
|
||||
t.Fatalf("malformed Validate() = %#v, %v; want deferral", malformed, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorRegistrationAndPolicyFingerprint(t *testing.T) {
|
||||
registry := pipeline.NewValidatorRegistry()
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := registry.Spec(Key); !ok {
|
||||
t.Fatalf("registry missing %q", Key)
|
||||
}
|
||||
if got, want := New(Options{}).CheckpointFingerprints(), []pipeline.CheckpointFingerprint{{Name: "policy", Value: policy}}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("fingerprints = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func document() *source.SourceDocument {
|
||||
return &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Text: "The group enters the ruined watchtower."}}}
|
||||
}
|
||||
|
||||
func scene(id, title, summary string) dnd.SceneDescription {
|
||||
return dnd.SceneDescription{ID: id, SourceRef: source.SourceRef{SourceID: "session", StartUnitID: 1, EndUnitID: 1}, Kind: dnd.SceneKindNarrative, Title: title, Summary: summary}
|
||||
}
|
||||
Reference in New Issue
Block a user