eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Classify source relatedness as data-quality advisories

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-27 15:49:53 +00:00
parent 1f1967c8d2
commit ccba2ce3f9
25 changed files with 232 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
internal/modules/dnd/validate/npcregistry/source_relatedness/validator.go
View File

@@ -13,9 +13,9 @@ import (
)
const (
Key = "extract/dnd/npc-registry/source_relatedness"
WarningReasonCode = "npc_not_near_source"
policy = "dnd.npc_registry.validator.source_relatedness.v2"
Key = "extract/dnd/npc-registry/source_relatedness"
ReasonCode = "npc_not_near_source"
policy = "dnd.npc_registry.validator.source_relatedness.v2"
)
type Options struct{}
@@ -56,11 +56,11 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
}
warnings = append(warnings, contracts.Warning{
Scope: fmt.Sprintf("npcs[%d]", npcIndex),
ReasonCode: WarningReasonCode,
ReasonCode: ReasonCode,
Message: fmt.Sprintf("NPC %s was not found in cited source text", diagnostics.Quote(npc.Name)),
})
}
return contracts.ValidationResult{Approved: true, Warnings: warnings}, nil
return diagnostics.DataQualityResult(warnings)
}
func npcAppearsInCitedText(citedText string, npc dnd.NPC) bool {

29
internal/modules/dnd/validate/npcregistry/source_relatedness/validator_test.go
View File

@@ -26,7 +26,7 @@ func TestValidatorMatchesCanonicalNamesWithUnicodeVariants(t *testing.T) {
{ID: 2, Kind: "message", Text: "The greencloak watches."},
}}
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.NPCRegistry]{Source: doc, Value: value})
if err != nil || !result.Approved || len(result.Warnings) != 0 {
if err != nil || !result.Approved || len(result.Diagnostics) != 0 {
t.Fatalf("Validate() = %#v, %v; want canonical-name relatedness approval", result, err)
}
}
@@ -36,12 +36,13 @@ func TestValidatorWarnsAtMostOncePerNPCForUnrelatedCitations(t *testing.T) {
{ID: "one", Name: "Missing\nName", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}, {SourceID: "session", StartUnitID: 1, EndUnitID: 1}}},
}}
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.NPCRegistry]{Source: relatednessDocument(), Value: value})
if err != nil || !result.Approved || len(result.Warnings) != 1 {
t.Fatalf("Validate() = %#v, %v; want one warning", result, err)
if err != nil || !result.Approved || len(result.Warnings) != 0 || len(result.Diagnostics) != 1 {
t.Fatalf("Validate() = %#v, %v; want one advisory", result, err)
}
warning := result.Warnings[0]
if warning.Scope != "npcs[0]" || warning.ReasonCode != WarningReasonCode || !strings.Contains(warning.Message, `Missing\nName`) || strings.Contains(warning.Message, "Missing\nName") || !utf8.ValidString(warning.Message) {
t.Fatalf("warning = %#v, want safely quoted bounded warning", warning)
diagnostic := result.Diagnostics[0]
sample := diagnostic.Samples[0]
if sample.Scope != "npcs[0]" || diagnostic.ReasonCode != ReasonCode || !strings.Contains(sample.Message, `Missing\nName`) || strings.Contains(sample.Message, "Missing\nName") || !utf8.ValidString(sample.Message) {
t.Fatalf("diagnostic = %#v, want safely quoted advisory", diagnostic)
}
}
@@ -51,21 +52,21 @@ func TestValidatorDoesNotMatchShortNameSubstring(t *testing.T) {
}}}
doc := &source.SourceDocument{ID: "session", Kind: "transcript", Format: "application/json", Digest: "sha256:session", Units: []source.SourceUnit{{ID: 1, Kind: "message", Text: "A cart rolls past."}}}
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.NPCRegistry]{Source: doc, Value: value})
if err != nil || !result.Approved || len(result.Warnings) != 1 {
t.Fatalf("Validate() = %#v, %v; want short-name boundary warning", result, err)
if err != nil || !result.Approved || len(result.Diagnostics) != 1 {
t.Fatalf("Validate() = %#v, %v; want short-name boundary advisory", result, err)
}
}
func TestValidatorDefersMalformedShapeAndInvalidRangesDoNotPanic(t *testing.T) {
invalidShape := dnd.NPCRegistry{NPCs: []dnd.NPC{{Name: "Mira Thorn"}}}
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.NPCRegistry]{Source: relatednessDocument(), Value: invalidShape})
if err != nil || !result.Approved || len(result.Warnings) != 0 {
t.Fatalf("shape deferral = %#v, %v; want approval without warning", result, err)
if err != nil || !result.Approved || len(result.Diagnostics) != 0 {
t.Fatalf("shape deferral = %#v, %v; want approval without finding", result, err)
}
invalidRange := dnd.NPCRegistry{NPCs: []dnd.NPC{{ID: "one", Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 99, EndUnitID: 99}}}}}
result, err = New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.NPCRegistry]{Source: relatednessDocument(), Value: invalidRange})
if err != nil || !result.Approved || len(result.Warnings) != 0 {
t.Fatalf("invalid-range relatedness = %#v, %v; want approval without warning", result, err)
if err != nil || !result.Approved || len(result.Diagnostics) != 0 {
t.Fatalf("invalid-range relatedness = %#v, %v; want approval without finding", result, err)
}
}
@@ -73,8 +74,8 @@ func TestValidatorUsesOnlyTranscriptEvidenceAndRegistersPolicy(t *testing.T) {
value := dnd.NPCRegistry{NPCs: []dnd.NPC{{ID: "one", Name: "Opaque NPC", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}}}}
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"glossary": {Items: []contracts.ReferenceItem{{Content: []byte("Opaque NPC")}}}}}
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.NPCRegistry]{Source: relatednessDocument(), References: references, Value: value})
if err != nil || len(result.Warnings) != 1 {
t.Fatalf("reference-only relatedness = %#v, %v; want warning", result, err)
if err != nil || len(result.Diagnostics) != 1 {
t.Fatalf("reference-only relatedness = %#v, %v; want advisory", result, err)
}
if got := New(Options{}).CheckpointFingerprints(); len(got) != 1 || got[0].Name != "policy" || got[0].Value != "dnd.npc_registry.validator.source_relatedness.v2" {
t.Fatalf("CheckpointFingerprints() = %#v, want local policy", got)