eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve D&D validation reliability

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-29 01:24:45 +00:00
parent 4da9360d74
commit 917d150279
55 changed files with 1300 additions and 565 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
internal/modules/dnd/validate/spells/catalog/validator.go
View File

@@ -5,19 +5,19 @@ import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape"
)
const (
Key = "extract/dnd/spells/catalog"
ReasonCode = "unknown_spell"
maxIssues = 20
maxDisplayedNameRunes = 128
maxMessageBytes = 4096
Key = "extract/dnd/spells/catalog"
ReasonCode = "unknown_spell"
policy = "dnd.spells.validator.catalog.v2"
)
type Options struct{}
@@ -54,7 +54,10 @@ func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
if v == nil {
return nil
}
return []pipeline.CheckpointFingerprint{{Name: "effective_catalog", Value: v.catalog.Digest()}}
return []pipeline.CheckpointFingerprint{
{Name: "policy", Value: policy},
{Name: "effective_catalog", Value: v.catalog.Digest()},
}
}
func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) {
@@ -69,7 +72,7 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
continue
}
if _, ok := v.catalog.Lookup(name); !ok {
unknown = append(unknown, unknownSpell{index: index, name: name})
unknown = append(unknown, unknownSpell{index: index, name: name, caster: spell.Caster, sourceRefs: spell.SourceRefs})
}
}
if len(unknown) == 0 {
@@ -79,42 +82,29 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
}
type unknownSpell struct {
index int
name string
index int
name string
caster string
sourceRefs []source.SourceRef
}
func rejection(unknown []unknownSpell) contracts.ValidationResult {
limit := len(unknown)
if limit > maxIssues {
limit = maxIssues
issues := make([]string, 0, len(unknown))
var corrections diagnostics.Corrections
for _, item := range unknown {
issues = append(issues, fmt.Sprintf("spell_casts[%d].spell %s", item.index, diagnostics.Quote(item.name)))
corrections.Add(
"recognized-spell",
"Use a recognized D&D spell name supported by the transcript, or omit the record when the evidence does not establish a spell cast.",
fmt.Sprintf("Affected spell %s by caster %s %s.", diagnostics.Quote(item.name), diagnostics.Quote(strings.TrimSpace(item.caster)), diagnostics.SourceRange(item.sourceRefs)),
)
}
issues := make([]string, 0, limit)
for _, item := range unknown[:limit] {
issue := fmt.Sprintf("spell_casts[%d].spell %q", item.index, truncateDisplayedName(item.name))
candidate := rejectionMessage(append(issues, issue), len(unknown)-len(issues)-1)
if len(candidate) > maxMessageBytes {
break
}
issues = append(issues, issue)
return contracts.ValidationResult{
Approved: false,
ReasonCode: ReasonCode,
Message: diagnostics.Aggregate("unknown spell names", issues),
CorrectionGuidance: corrections.Guidance("Correct every unrecognized spell and return the complete replacement spell-cast list"),
}
message := rejectionMessage(issues, len(unknown)-len(issues))
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message, CorrectionGuidance: "Use recognized D&D spell names supported by the supplied transcript evidence, and omit any candidate that is not a spell cast."}
}
func truncateDisplayedName(name string) string {
runes := []rune(name)
if len(runes) <= maxDisplayedNameRunes {
return name
}
return string(runes[:maxDisplayedNameRunes-1]) + "…"
}
func rejectionMessage(issues []string, omitted int) string {
message := "unknown spell names: " + strings.Join(issues, ", ")
if omitted > 0 {
message += fmt.Sprintf("; %d additional issue(s) omitted", omitted)
}
return message
}
func Spec() pipeline.ValidatorSpec {

62
internal/modules/dnd/validate/spells/catalog/validator_test.go
View File

@@ -2,11 +2,9 @@ package catalog
import (
"context"
"fmt"
"reflect"
"strings"
"testing"
"unicode/utf8"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -36,7 +34,7 @@ func TestValidatorCheckpointFingerprintUsesEffectiveCatalogDigest(t *testing.T)
t.Fatal(err)
}
fingerprints := validator.CheckpointFingerprints()
if len(fingerprints) != 1 || fingerprints[0].Name != "effective_catalog" || fingerprints[0].Value != validator.catalog.Digest() {
if len(fingerprints) != 2 || fingerprints[0].Name != "policy" || fingerprints[0].Value != policy || fingerprints[1].Name != "effective_catalog" || fingerprints[1].Value != validator.catalog.Digest() {
t.Fatalf("checkpoint fingerprints = %#v, want effective catalog digest %q", fingerprints, validator.catalog.Digest())
}
}
@@ -60,59 +58,13 @@ func TestValidatorRejectsMultipleUnknownCastsInStableOrder(t *testing.T) {
if want := `spell_casts[0].spell "Unknown First", spell_casts[2].spell "Unknown Second"`; !strings.Contains(result.Message, want) {
t.Fatalf("message = %q, want %q", result.Message, want)
}
}
func TestValidatorBoundsUnknownCastMessage(t *testing.T) {
value := dnd.SpellList{SpellCasts: make([]dnd.SpellCast, 22)}
for index := range value.SpellCasts {
value.SpellCasts[index] = validCast(fmt.Sprintf("Unknown Spell %02d", index))
for _, want := range []string{"Unknown First", "Unknown Second", "source unit 1", "complete replacement"} {
if !strings.Contains(result.CorrectionGuidance, want) {
t.Fatalf("CorrectionGuidance = %q, want %q", result.CorrectionGuidance, want)
}
}
validator, err := New(Options{})
if err != nil {
t.Fatalf("New() error = %v, want nil", err)
}
result, err := validator.Validate(context.Background(), validationRequest(value))
if err != nil {
t.Fatalf("Validate() error = %v, want nil", err)
}
if result.Approved || result.ReasonCode != ReasonCode {
t.Fatalf("Validate() = %#v, want unknown-spell rejection", result)
}
if got := strings.Count(result.Message, "spell_casts["); got != maxIssues {
t.Fatalf("message includes %d issues, want %d: %q", got, maxIssues, result.Message)
}
if !strings.Contains(result.Message, "2 additional issue(s) omitted") || strings.Contains(result.Message, "Unknown Spell 21") {
t.Fatalf("message = %q, want bounded diagnostics", result.Message)
}
}
func TestValidatorBoundsUnknownSpellNamesAndTotalMessage(t *testing.T) {
longName := strings.Repeat("火", maxDisplayedNameRunes+100) + "\n\t"
value := dnd.SpellList{SpellCasts: make([]dnd.SpellCast, maxIssues)}
for index := range value.SpellCasts {
value.SpellCasts[index] = validCast(fmt.Sprintf("%s-%d", longName, index))
}
validator, err := New(Options{})
if err != nil {
t.Fatal(err)
}
result, err := validator.Validate(context.Background(), validationRequest(value))
if err != nil {
t.Fatal(err)
}
if result.Approved || len([]byte(result.Message)) > maxMessageBytes || !strings.Contains(result.Message, "…") {
t.Fatalf("message length/content = %d/%q, want bounded message with truncation", len([]byte(result.Message)), result.Message)
}
if !strings.Contains(result.Message, "additional issue(s) omitted") {
t.Fatalf("message = %q, want byte-budget omitted count", result.Message)
}
displayed := strings.Count(result.Message, "spell_casts[")
wantOmitted := fmt.Sprintf("%d additional issue(s) omitted", len(value.SpellCasts)-displayed)
if !strings.Contains(result.Message, wantOmitted) {
t.Fatalf("message = %q, want omitted count %q", result.Message, wantOmitted)
}
if !utf8.ValidString(result.Message) {
t.Fatal("bounded message is not valid UTF-8")
if strings.Contains(result.CorrectionGuidance, "spell_casts[") || strings.Contains(result.CorrectionGuidance, ReasonCode) {
t.Fatalf("CorrectionGuidance exposed internal diagnostics: %q", result.CorrectionGuidance)
}
}