From 9cb74628006d000874ae93f1149df7584a2e694b Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 28 Aug 2026 00:22:56 +0000 Subject: [PATCH] Test combat semantics correction flow --- docs/roadmap/implementation.md | 2 + .../integration/dnd_npc_grounded_test.go | 92 +++++++++++++++++-- 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 452001f1..203b87c8 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -308,6 +308,8 @@ This stage is small enough for one implementation prompt. ## Stage 5: Prove Feedback-Aware Producer Correction +✅ Complete + ### Goal Prove the assembled D&D path can reject a well-formed wrong scene kind, guide diff --git a/internal/modules/integration/dnd_npc_grounded_test.go b/internal/modules/integration/dnd_npc_grounded_test.go index 345fb71c..1ca9b9ca 100644 --- a/internal/modules/integration/dnd_npc_grounded_test.go +++ b/internal/modules/integration/dnd_npc_grounded_test.go @@ -28,6 +28,7 @@ import ( sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry" + combatsemantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics" ) func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T) { @@ -258,6 +259,70 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T assertCurrentEvidence(t, combatValue.CombatTurns[0].SourceRefs) } +func TestSceneCombatSemanticsCorrectionRetriesAndRevalidates(t *testing.T) { + registries := productionNPCRegistries(t) + configValue := loadGroundedPipelineConfig(t) + profile := configValue.Pipelines["dnd-npc-grounded"] + sceneLane := profile.Steps[0].Artifacts["scene-descriptions"] + sceneLane.Extract.Retries = 1 + sceneLane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{ + pipeline.Binding("generic/valid_json"), + pipeline.Binding("extract/dnd/scene-descriptions/shape"), + pipeline.Binding("extract/dnd/scene-descriptions/source_refs"), + pipeline.Binding("generic/valid_json_schema"), + pipeline.Binding("extract/dnd/scene-descriptions/source_relatedness"), + pipeline.Binding(combatsemantics.Key), + }} + profile.Steps[0].Artifacts["scene-descriptions"] = sceneLane + configValue.Pipelines["dnd-npc-grounded"] = profile + + client := &groundedDNDLLMClient{sceneKinds: []dnd.SceneKind{dnd.SceneKindNarrative, dnd.SceneKindCombat}, combatSemanticsVerdicts: []string{"combat_should_be_added", "approved"}} + output := runGroundedPipeline(t, configValue, registries, client, nil) + if len(output.Rejected) != 0 { + t.Fatalf("rejected = %#v, want corrected acceptance", output.Rejected) + } + for _, group := range output.Diagnostics.Groups { + if group.Origin.ValidatorKey == combatsemantics.Key { + t.Fatalf("combat-semantics validator left a diagnostic after correction: %#v", group) + } + } + var scenes dnd.SceneDescriptionList + for _, serialized := range output.NormalizeOutputs { + if serialized.LaneID == "scene-descriptions" { + var err error + scenes, err = scenecodec.New().Decode(serialized.Artifact.Content) + if err != nil { + t.Fatal(err) + } + } + } + if len(scenes.Scenes) != 1 || scenes.Scenes[0].Kind != dnd.SceneKindCombat { + t.Fatalf("accepted scenes = %#v, want corrected combat scene", scenes) + } + requests := client.requestsSnapshot() + var sceneRequests, validatorRequests []contracts.StructuredCompletionRequest + for _, request := range requests { + switch request.PromptID { + case sceneextract.PromptID: + sceneRequests = append(sceneRequests, request) + case combatsemantics.PromptID: + validatorRequests = append(validatorRequests, request) + } + } + if len(sceneRequests) != 2 || len(validatorRequests) != 2 || sceneRequests[0].Correction != nil || sceneRequests[1].Correction == nil { + t.Fatalf("scene/validator requests = %d/%d; want correction and revalidation", len(sceneRequests), len(validatorRequests)) + } + correction := sceneRequests[1].Correction + if !strings.Contains(string(correction.AssistantResponse), `"kind":"narrative"`) || !strings.Contains(correction.UserGuidance, "Return kind: combat") || !strings.Contains(correction.UserGuidance, "active encounter") { + t.Fatalf("correction = %#v, want latest candidate and useful guidance", correction) + } + for _, forbidden := range []string{combatsemantics.Key, combatsemantics.ReasonCodeActiveCombatNotClassified, "session:chunk"} { + if strings.Contains(correction.UserGuidance, forbidden) { + t.Fatalf("correction guidance leaked internal detail %q: %q", forbidden, correction.UserGuidance) + } + } +} + func TestProductionDNDOutputPublishesSelectedEvidenceContext(t *testing.T) { registries := productionNPCRegistries(t) configValue := loadGroundedPipelineConfig(t) @@ -508,12 +573,16 @@ func (loader *generatedReferenceCheckpointLoader) extractDependencies(laneID str } type groundedDNDLLMClient struct { - mu sync.Mutex - requests []contracts.StructuredCompletionRequest - sceneKind dnd.SceneKind - sceneTitle string - firstUnitID int - thirdUnitID int + mu sync.Mutex + requests []contracts.StructuredCompletionRequest + sceneKind dnd.SceneKind + sceneTitle string + firstUnitID int + thirdUnitID int + sceneKinds []dnd.SceneKind + combatSemanticsVerdicts []string + sceneCalls int + combatSemanticsCalls int } func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { @@ -547,6 +616,10 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ payload = map[string]any{"duplicate_groups": []any{}} case sceneextract.PromptID: kind := client.sceneKind + if client.sceneCalls < len(client.sceneKinds) { + kind = client.sceneKinds[client.sceneCalls] + } + client.sceneCalls++ if kind == "" { kind = dnd.SceneKindCombat } @@ -555,6 +628,13 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ title = "A combat encounter" } payload = map[string]any{"kind": kind, "title": title, "summary": "The party faces an active encounter."} + case combatsemantics.PromptID: + verdict := "approved" + if client.combatSemanticsCalls < len(client.combatSemanticsVerdicts) { + verdict = client.combatSemanticsVerdicts[client.combatSemanticsCalls] + } + client.combatSemanticsCalls++ + payload = map[string]any{"verdict": verdict, "explanation": "The transcript shows an active encounter with hostile action."} case spells.PromptID: payload = map[string]any{"spell_casts": []any{map[string]any{ "caster": "Mira Thorn", "spell": "Cure Wounds",