eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve D&D validation reliability

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-29 01:24:45 +00:00
parent 4da9360d74
commit 917d150279
55 changed files with 1300 additions and 565 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
internal/modules/dnd/extract/combatturns/extractor_test.go
View File

@@ -66,7 +66,7 @@ func TestExtractMapsAndOrdersCombatTurnsBySourcePosition(t *testing.T) {
func TestExtractPreservesInvalidCandidatesForValidators(t *testing.T) {
client := &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{
{
Actor: " ", TurnKind: "unsupported",
Actor: " ", TurnKind: "turn",
SourceRefs: []combatSourceRefResponse{{StartUnitID: 99, EndUnitID: 0}},
},
}}}
@@ -75,7 +75,7 @@ func TestExtractPreservesInvalidCandidatesForValidators(t *testing.T) {
t.Fatalf("Extract() error = %v, want nil for candidate values", err)
}
turn := result.Value.CombatTurns[0]
if turn.Actor != " " || turn.TurnKind != "unsupported" {
if turn.Actor != " " || turn.TurnKind != "turn" {
t.Fatalf("invalid turn fields = %#v, want preserved candidate values", turn)
}
if turn.SourceRefs[0] != (source.SourceRef{SourceID: "session-alpha", StartUnitID: 99}) {

26
internal/modules/dnd/extract/combatturns/schema_test.go
View File

@@ -7,6 +7,8 @@ import (
"testing"
"github.com/santhosh-tekuri/jsonschema/v6"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
)
func TestLoadResponseSchemaUsesPrivateCombatShape(t *testing.T) {
@@ -37,7 +39,7 @@ func TestLoadResponseSchemaUsesPrivateCombatShape(t *testing.T) {
}
}
func TestResponseSchemaLeavesSemanticConstraintsToDeterministicValidators(t *testing.T) {
func TestResponseSchemaLeavesNonCategoricalSemanticsToDeterministicValidators(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatal(err)
@@ -45,7 +47,6 @@ func TestResponseSchemaLeavesSemanticConstraintsToDeterministicValidators(t *tes
semanticCandidate := validCombatResponse()
turn := semanticCandidate["combat_turns"].([]any)[0].(map[string]any)
turn["actor"] = ""
turn["turn_kind"] = "unsupported"
ref := turn["source_refs"].([]any)[0].(map[string]any)
ref["start_unit_id"] = 0
ref["end_unit_id"] = -1
@@ -54,7 +55,7 @@ func TestResponseSchemaLeavesSemanticConstraintsToDeterministicValidators(t *tes
t.Fatal(err)
}
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
t.Fatalf("private schema rejected validator-owned semantics: %v", err)
t.Fatalf("private schema rejected non-categorical validator-owned semantics: %v", err)
}
turn["source_refs"] = []any{}
content, err = json.Marshal(semanticCandidate)
@@ -79,6 +80,7 @@ func TestResponseSchemaRetainsStructuralBoundary(t *testing.T) {
{name: "wrong actor type", mutate: func(turn map[string]any) { turn["actor"] = 1 }},
{name: "unknown field", mutate: func(turn map[string]any) { turn["unexpected"] = true }},
{name: "missing source refs", mutate: func(turn map[string]any) { delete(turn, "source_refs") }},
{name: "unsupported turn kind", mutate: func(turn map[string]any) { turn["turn_kind"] = "unsupported" }},
} {
t.Run(test.name, func(t *testing.T) {
candidate := validCombatResponse()
@@ -94,6 +96,24 @@ func TestResponseSchemaRetainsStructuralBoundary(t *testing.T) {
}
}
func TestResponseSchemaAcceptsEverySupportedTurnKind(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatal(err)
}
for _, kind := range []string{string(dnd.CombatTurnKindTurn), string(dnd.CombatTurnKindReaction), string(dnd.CombatTurnKindLegendaryAction), string(dnd.CombatTurnKindLairAction), string(dnd.CombatTurnKindOther)} {
candidate := validCombatResponse()
candidate["combat_turns"].([]any)[0].(map[string]any)["turn_kind"] = kind
content, err := json.Marshal(candidate)
if err != nil {
t.Fatal(err)
}
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
t.Fatalf("supported turn kind %q was rejected: %v", kind, err)
}
}
}
func TestResponseSchemaJSONIsMutationSafe(t *testing.T) {
first, err := loadResponseSchema()
if err != nil {

4
internal/modules/dnd/extract/enemyevents/extractor_test.go
View File

@@ -59,14 +59,14 @@ func TestExtractMapsEnemyEventsInSourceOrder(t *testing.T) {
func TestExtractPreservesSemanticCandidatesAndResponseOwnership(t *testing.T) {
client := &fakeEnemyEventsLLMClient{response: extractionResponse{Events: []enemyEventResponse{{
Name: " ", Kind: "unsupported", SourceRefs: []enemySourceRefResponse{{StartUnitID: 99, EndUnitID: 0}},
Name: " ", Kind: "engaged", SourceRefs: []enemySourceRefResponse{{StartUnitID: 99, EndUnitID: 0}},
}}}}
result, err := newEnemyExtractor(t, client).Extract(context.Background(), enemyExtractionRequest(t))
if err != nil {
t.Fatal(err)
}
event := result.Value.Events[0]
if event.Name != " " || event.Kind != "unsupported" || event.SourceRefs[0] != (source.SourceRef{SourceID: "combat-session", StartUnitID: 99}) {
if event.Name != " " || event.Kind != "engaged" || event.SourceRefs[0] != (source.SourceRef{SourceID: "combat-session", StartUnitID: 99}) {
t.Fatalf("semantic candidate = %#v", event)
}
result.Value.Events[0].SourceRefs[0].StartUnitID = 7

24
internal/modules/dnd/extract/enemyevents/schema_test.go
View File

@@ -7,6 +7,8 @@ import (
"testing"
"github.com/santhosh-tekuri/jsonschema/v6"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
)
func TestResponseSchemaDefinesPrivateStructuralBoundary(t *testing.T) {
@@ -28,13 +30,12 @@ func TestResponseSchemaDefinesPrivateStructuralBoundary(t *testing.T) {
semantic := validEnemyResponse()
event := semantic["events"].([]any)[0].(map[string]any)
event["name"] = ""
event["kind"] = "unsupported"
ref := event["source_refs"].([]any)[0].(map[string]any)
ref["start_unit_id"] = 0
ref["end_unit_id"] = -1
content, err = json.Marshal(semantic)
if err != nil || validateEnemySchema(content, schema.JSONSchema) != nil {
t.Fatalf("validator-owned semantics were rejected: %v", err)
t.Fatalf("non-categorical validator-owned semantics were rejected: %v", err)
}
}
@@ -46,6 +47,7 @@ func TestResponseSchemaRejectsInvalidStructure(t *testing.T) {
for _, mutate := range []func(map[string]any){
func(event map[string]any) { delete(event, "name") },
func(event map[string]any) { event["kind"] = 1 },
func(event map[string]any) { event["kind"] = "unsupported" },
func(event map[string]any) { event["unexpected"] = true },
func(event map[string]any) { event["source_refs"].([]any)[0].(map[string]any)["source_id"] = "session" },
} {
@@ -70,6 +72,24 @@ func TestResponseSchemaRejectsInvalidStructure(t *testing.T) {
}
}
func TestResponseSchemaAcceptsEverySupportedKind(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatal(err)
}
for _, kind := range []string{string(dnd.EnemyEventKindEngaged), string(dnd.EnemyEventKindKilled), string(dnd.EnemyEventKindFled), string(dnd.EnemyEventKindCaptured), string(dnd.EnemyEventKindIncapacitated)} {
candidate := validEnemyResponse()
candidate["events"].([]any)[0].(map[string]any)["kind"] = kind
content, err := json.Marshal(candidate)
if err != nil {
t.Fatal(err)
}
if err := validateEnemySchema(content, schema.JSONSchema); err != nil {
t.Fatalf("supported enemy-event kind %q was rejected: %v", kind, err)
}
}
}
func validEnemyResponse() map[string]any {
return map[string]any{"events": []any{map[string]any{
"name": "Ashfang", "kind": "engaged", "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1}},

6
internal/modules/dnd/extract/npcoccurrences/extractor_test.go
View File

@@ -24,7 +24,7 @@ func TestExtractMapsEveryKindAndOrdersBySourcePosition(t *testing.T) {
{Name: "Speaker", Kind: "dialogue", SourceRefs: append(occurrenceRefs(2, 2), occurrenceRefs(2, 2)...)},
{Name: "Present", Kind: "noncombat_presence", SourceRefs: occurrenceRefs(7, 2)},
{Name: "Mentioned", Kind: "mentioned", SourceRefs: occurrenceRefs(10, 10)},
{Name: "Invalid", Kind: "unsupported", SourceRefs: occurrenceRefs(0, 0)},
{Name: "Invalid", Kind: "other", SourceRefs: occurrenceRefs(0, 0)},
}}}
references := requiredRegistryReferences(t, "Mentioned", "Speaker", "Present", "Ally", "Opponent", "Other", "Invalid")
req := extractionRequest()
@@ -44,7 +44,7 @@ func TestExtractMapsEveryKindAndOrdersBySourcePosition(t *testing.T) {
dnd.NPCOccurrenceKindCombatAlly,
dnd.NPCOccurrenceKindCombatOpponent,
dnd.NPCOccurrenceKindOther,
"unsupported",
dnd.NPCOccurrenceKindOther,
}) {
t.Fatalf("occurrence kinds = %#v", got)
}
@@ -57,7 +57,7 @@ func TestExtractMapsEveryKindAndOrdersBySourcePosition(t *testing.T) {
if id := result.Value.Occurrences[0].NPCID; id != identity.DeriveID("Mentioned") {
t.Fatalf("durable NPC ID = %q, want registry identity", id)
}
if invalid := result.Value.Occurrences[6]; invalid.Name != "Invalid" || invalid.Kind != "unsupported" || !reflect.DeepEqual(invalid.SourceRefs, []source.SourceRef{{SourceID: "session-alpha"}}) {
if invalid := result.Value.Occurrences[6]; invalid.Name != "Invalid" || invalid.Kind != dnd.NPCOccurrenceKindOther || !reflect.DeepEqual(invalid.SourceRefs, []source.SourceRef{{SourceID: "session-alpha"}}) {
t.Fatalf("invalid candidate = %#v, want preserved values with current source identity", invalid)
}
if len(client.requests) != 1 {

24
internal/modules/dnd/extract/npcoccurrences/schema_test.go
View File

@@ -7,6 +7,8 @@ import (
"testing"
"github.com/santhosh-tekuri/jsonschema/v6"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
)
func TestResponseSchemaOwnsOnlyPrivateStructuralContract(t *testing.T) {
@@ -29,7 +31,6 @@ func TestResponseSchemaOwnsOnlyPrivateStructuralContract(t *testing.T) {
semanticCandidate := validOccurrenceResponse()
occurrence := semanticCandidate["occurrences"].([]any)[0].(map[string]any)
occurrence["name"] = ""
occurrence["kind"] = "unsupported"
ref := occurrence["source_refs"].([]any)[0].(map[string]any)
ref["start_unit_id"] = 0
ref["end_unit_id"] = -1
@@ -38,12 +39,13 @@ func TestResponseSchemaOwnsOnlyPrivateStructuralContract(t *testing.T) {
t.Fatal(err)
}
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
t.Fatalf("schema rejected validator-owned semantics: %v", err)
t.Fatalf("schema rejected non-categorical validator-owned semantics: %v", err)
}
for _, mutate := range []func(map[string]any){
func(record map[string]any) { delete(record, "name") },
func(record map[string]any) { record["kind"] = 1 },
func(record map[string]any) { record["kind"] = "unsupported" },
func(record map[string]any) { record["npc_id"] = "npc:sha256:opaque" },
func(record map[string]any) { record["unexpected"] = true },
func(record map[string]any) {
@@ -79,6 +81,24 @@ func TestResponseSchemaIsDefensiveAndContentSafe(t *testing.T) {
}
}
func TestResponseSchemaAcceptsEverySupportedKind(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatal(err)
}
for _, kind := range []string{string(dnd.NPCOccurrenceKindMentioned), string(dnd.NPCOccurrenceKindNoncombatPresence), string(dnd.NPCOccurrenceKindDialogue), string(dnd.NPCOccurrenceKindCombatAlly), string(dnd.NPCOccurrenceKindCombatOpponent), string(dnd.NPCOccurrenceKindOther)} {
candidate := validOccurrenceResponse()
candidate["occurrences"].([]any)[0].(map[string]any)["kind"] = kind
content, err := json.Marshal(candidate)
if err != nil {
t.Fatal(err)
}
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
t.Fatalf("supported NPC-occurrence kind %q was rejected: %v", kind, err)
}
}
}
func validOccurrenceResponse() map[string]any {
return map[string]any{"occurrences": []any{map[string]any{
"name": "Mira Thorn", "kind": "dialogue",

6
internal/modules/dnd/extract/scenedescriptions/extractor_test.go
View File

@@ -75,18 +75,18 @@ func TestExtractReturnsSemanticallyInvalidResponseForDeterministicValidation(t *
if err != nil {
t.Fatalf("loadResponseSchema() error = %v", err)
}
if err := validateJSONSchema(t, map[string]any{"kind": "unrecognized", "title": " ", "summary": ""}, schema.JSONSchema); err != nil {
if err := validateJSONSchema(t, map[string]any{"kind": "narrative", "title": " ", "summary": ""}, schema.JSONSchema); err != nil {
t.Fatalf("semantic candidate rejected by private schema: %v", err)
}
client := &fakeSceneDescriptionsLLMClient{response: extractionResponse{
Kind: dnd.SceneKind("unrecognized"), Title: " ", Summary: "",
Kind: dnd.SceneKindNarrative, Title: " ", Summary: "",
}}
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
scene := result.Value.Scenes[0]
if scene.Kind != dnd.SceneKind("unrecognized") || scene.Title != "" || scene.Summary != "" {
if scene.Kind != dnd.SceneKindNarrative || scene.Title != "" || scene.Summary != "" {
t.Fatalf("scene = %#v, want semantic candidates returned for deterministic validation", scene)
}
}

17
internal/modules/dnd/extract/scenedescriptions/schema_test.go
View File

@@ -7,6 +7,8 @@ import (
"testing"
"github.com/santhosh-tekuri/jsonschema/v6"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
)
func TestLoadResponseSchemaUsesStrictPrivateSceneDescriptionContract(t *testing.T) {
@@ -30,7 +32,7 @@ func TestLoadResponseSchemaUsesStrictPrivateSceneDescriptionContract(t *testing.
{name: "unknown framework field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "id": "assigned-later"}},
{name: "unknown application field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "source_ref": map[string]any{}}},
{name: "collection is not allowed", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "scenes": []any{}}},
{name: "unsupported kind", response: map[string]any{"kind": "interlude", "title": "Ambush", "summary": "Bandits strike."}, valid: true},
{name: "unsupported kind", response: map[string]any{"kind": "interlude", "title": "Ambush", "summary": "Bandits strike."}},
{name: "empty title", response: map[string]any{"kind": "combat", "title": "", "summary": "Bandits strike."}, valid: true},
{name: "empty summary", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": ""}, valid: true},
{name: "wrong kind type", response: map[string]any{"kind": 7, "title": "Ambush", "summary": "Bandits strike."}},
@@ -64,6 +66,19 @@ func TestResponseSchemaIsMutationSafeAndDiagnosticsRedactContent(t *testing.T) {
}
}
func TestResponseSchemaAcceptsEverySupportedKind(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatal(err)
}
for _, kind := range []string{string(dnd.SceneKindCombat), string(dnd.SceneKindNarrative), string(dnd.SceneKindRecap), string(dnd.SceneKindMeta)} {
response := map[string]any{"kind": kind, "title": "Title", "summary": "Summary"}
if err := validateJSONSchema(t, response, schema.JSONSchema); err != nil {
t.Fatalf("supported scene kind %q was rejected: %v", kind, err)
}
}
}
func validateJSONSchema(t *testing.T, instance map[string]any, schemaContent []byte) error {
t.Helper()
content, err := json.Marshal(instance)