eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Classify source relatedness as data-quality advisories

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-27 15:49:53 +00:00
parent 1f1967c8d2
commit ccba2ce3f9
25 changed files with 232 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
internal/modules/dnd/validate/combatturns/source_relatedness/validator.go
View File

@@ -13,10 +13,9 @@ import (
)
const (
Key = "extract/dnd/combat-turns/source_relatedness"
WarningReasonCode = "combat_turn_not_near_source"
OmittedReasonCode = "combat_turn_relatedness_warnings_omitted"
policy = "dnd.combat_turns.validator.source_relatedness.v2"
Key = "extract/dnd/combat-turns/source_relatedness"
ReasonCode = "combat_turn_not_near_source"
policy = "dnd.combat_turns.validator.source_relatedness.v2"
)
type Options struct{}
@@ -59,16 +58,13 @@ func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationReq
}
warnings = append(warnings, contracts.Warning{
Scope: fmt.Sprintf("combat_turns[%d]", turnIndex),
ReasonCode: WarningReasonCode,
ReasonCode: ReasonCode,
Message: diagnostics.Aggregate("combat turn not near source", []string{
fmt.Sprintf("actor %s was not found in cited source text", diagnostics.Quote(turn.Actor)),
}),
})
}
return contracts.ValidationResult{
Approved: true,
Warnings: diagnostics.LimitWarnings(warnings, "combat_turns", OmittedReasonCode),
}, nil
return diagnostics.DataQualityResult(warnings)
}
func actorAppearsInCitedText(citedText string, actor string) bool {
return shared.ContainsTokenSequence(citedText, actor)

45
internal/modules/dnd/validate/combatturns/source_relatedness/validator_test.go
View File

@@ -10,7 +10,6 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
)
func TestValidatorUsesDocumentOrderAndUnicodeComparisonForActor(t *testing.T) {
@@ -26,8 +25,8 @@ func TestValidatorUsesDocumentOrderAndUnicodeComparisonForActor(t *testing.T) {
{ID: 2, Kind: "message", Text: "O’rin attacks the goblin."},
}}
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.CombatTurnList]{Source: doc, Value: value})
if err != nil || !result.Approved || len(result.Warnings) != 0 {
t.Fatalf("Validate() = %#v, %v; want Unicode-related approval without target warning", result, err)
if err != nil || !result.Approved || len(result.Diagnostics) != 0 {
t.Fatalf("Validate() = %#v, %v; want Unicode-related approval without a finding", result, err)
}
}
@@ -37,17 +36,18 @@ func TestValidatorWarnsOncePerTurnForUnrelatedActor(t *testing.T) {
SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}, {SourceID: "session", StartUnitID: 1, EndUnitID: 1}},
}}}
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.CombatTurnList]{Source: relatednessDocument(), Value: value})
if err != nil || !result.Approved || len(result.Warnings) != 1 {
t.Fatalf("Validate() = %#v, %v; want one warning for the turn", result, err)
if err != nil || !result.Approved || len(result.Warnings) != 0 || len(result.Diagnostics) != 1 {
t.Fatalf("Validate() = %#v, %v; want one advisory for the turn", result, err)
}
warning := result.Warnings[0]
if warning.Scope != "combat_turns[0]" || warning.ReasonCode != WarningReasonCode || !strings.Contains(warning.Message, `Missing\nName`) || strings.Contains(warning.Message, "Missing\nName") || !utf8.ValidString(warning.Message) {
t.Fatalf("warning = %#v, want one safely quoted bounded warning", warning)
diagnostic := result.Diagnostics[0]
sample := diagnostic.Samples[0]
if diagnostic.Disposition != contracts.DiagnosticDispositionAdvisory || diagnostic.Category != contracts.DiagnosticCategoryDataQuality || diagnostic.ReasonCode != ReasonCode || sample.Scope != "combat_turns[0]" || !strings.Contains(sample.Message, `Missing\nName`) || strings.Contains(sample.Message, "Missing\nName") || !utf8.ValidString(sample.Message) {
t.Fatalf("diagnostic = %#v, want one safely quoted advisory", diagnostic)
}
}
func TestValidatorLimitsUnrelatedActorWarnings(t *testing.T) {
turns := make([]dnd.CombatTurn, diagnostics.MaxWarnings+1)
turns := make([]dnd.CombatTurn, contracts.MaxDiagnosticSamples+2)
for index := range turns {
turns[index] = dnd.CombatTurn{
Actor: "Missing Actor",
@@ -62,15 +62,8 @@ func TestValidatorLimitsUnrelatedActorWarnings(t *testing.T) {
if err != nil || !result.Approved {
t.Fatalf("Validate() = %#v, %v", result, err)
}
if len(result.Warnings) != diagnostics.MaxWarnings {
t.Fatalf("warning count = %d, want %d", len(result.Warnings), diagnostics.MaxWarnings)
}
if first := result.Warnings[0]; first.Scope != "combat_turns[0]" || first.ReasonCode != WarningReasonCode {
t.Fatalf("first warning = %#v, want first turn warning", first)
}
summary := result.Warnings[len(result.Warnings)-1]
if summary.Scope != "combat_turns" || summary.ReasonCode != OmittedReasonCode || summary.Message != "2 additional warning(s) omitted" {
t.Fatalf("warning summary = %#v", summary)
if len(result.Warnings) != 0 || len(result.Diagnostics) != 1 || result.Diagnostics[0].OccurrenceCount != len(turns) || len(result.Diagnostics[0].Samples) != contracts.MaxDiagnosticSamples || result.Diagnostics[0].OmittedSampleCount != len(turns)-contracts.MaxDiagnosticSamples {
t.Fatalf("diagnostics = %#v", result.Diagnostics)
}
}
@@ -81,22 +74,22 @@ func TestValidatorDoesNotMatchShortActorSubstring(t *testing.T) {
}}}
doc := &source.SourceDocument{ID: "session", Kind: "transcript", Format: "application/json", Digest: "sha256:session", Units: []source.SourceUnit{{ID: 1, Kind: "message", Text: "The cart attacks."}}}
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.CombatTurnList]{Source: doc, Value: value})
if err != nil || !result.Approved || len(result.Warnings) != 1 || !strings.Contains(result.Warnings[0].Message, "actor") {
t.Fatalf("Validate() = %#v, %v; want short-actor boundary warning", result, err)
if err != nil || !result.Approved || len(result.Diagnostics) != 1 || !strings.Contains(result.Diagnostics[0].Samples[0].Message, "actor") {
t.Fatalf("Validate() = %#v, %v; want short-actor boundary advisory", result, err)
}
}
func TestValidatorDefersMalformedShapeAndInvalidRanges(t *testing.T) {
invalidShape := dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{{Actor: "Aria"}}}
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.CombatTurnList]{Source: relatednessDocument(), Value: invalidShape})
if err != nil || !result.Approved || len(result.Warnings) != 0 {
t.Fatalf("shape deferral = %#v, %v; want approval without warning", result, err)
if err != nil || !result.Approved || len(result.Diagnostics) != 0 {
t.Fatalf("shape deferral = %#v, %v; want approval without finding", result, err)
}
invalidRange := validCombatTurnList()
invalidRange.CombatTurns[0].SourceRefs[0].StartUnitID = 99
result, err = New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.CombatTurnList]{Source: relatednessDocument(), Value: invalidRange})
if err != nil || !result.Approved || len(result.Warnings) != 0 {
t.Fatalf("invalid-range deferral = %#v, %v; want approval without warning", result, err)
if err != nil || !result.Approved || len(result.Diagnostics) != 0 {
t.Fatalf("invalid-range deferral = %#v, %v; want approval without finding", result, err)
}
}
@@ -105,8 +98,8 @@ func TestValidatorIgnoresReferenceMaterialAndRegistersPolicy(t *testing.T) {
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"glossary": {Items: []contracts.ReferenceItem{{Content: []byte("Aria attacks")}}}}}
doc := &source.SourceDocument{ID: "session", Kind: "transcript", Format: "application/json", Digest: "sha256:session", Units: []source.SourceUnit{{ID: 1, Kind: "message", Text: "The party waits."}}}
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.CombatTurnList]{Source: doc, References: references, Value: value})
if err != nil || !result.Approved || len(result.Warnings) != 1 {
t.Fatalf("reference-only relatedness = %#v, %v; want warning from transcript-only evidence", result, err)
if err != nil || !result.Approved || len(result.Diagnostics) != 1 {
t.Fatalf("reference-only relatedness = %#v, %v; want advisory from transcript-only evidence", result, err)
}
if got := New(Options{}).CheckpointFingerprints(); len(got) != 1 || got[0].Name != "policy" || got[0].Value != "dnd.combat_turns.validator.source_relatedness.v2" {
t.Fatalf("CheckpointFingerprints() = %#v, want relatedness policy", got)