Test combat semantics correction flow

This commit is contained in:
2026-08-28 00:22:56 +00:00
parent 87385b7e14
commit 9cb7462800
2 changed files with 88 additions and 6 deletions

View File

@@ -308,6 +308,8 @@ This stage is small enough for one implementation prompt.
## Stage 5: Prove Feedback-Aware Producer Correction ## Stage 5: Prove Feedback-Aware Producer Correction
✅ Complete
### Goal ### Goal
Prove the assembled D&D path can reject a well-formed wrong scene kind, guide Prove the assembled D&D path can reject a well-formed wrong scene kind, guide

View File

@@ -28,6 +28,7 @@ import (
sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions" sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry" npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
combatsemantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
) )
func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T) { func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T) {
@@ -258,6 +259,70 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
assertCurrentEvidence(t, combatValue.CombatTurns[0].SourceRefs) assertCurrentEvidence(t, combatValue.CombatTurns[0].SourceRefs)
} }
func TestSceneCombatSemanticsCorrectionRetriesAndRevalidates(t *testing.T) {
registries := productionNPCRegistries(t)
configValue := loadGroundedPipelineConfig(t)
profile := configValue.Pipelines["dnd-npc-grounded"]
sceneLane := profile.Steps[0].Artifacts["scene-descriptions"]
sceneLane.Extract.Retries = 1
sceneLane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{
pipeline.Binding("generic/valid_json"),
pipeline.Binding("extract/dnd/scene-descriptions/shape"),
pipeline.Binding("extract/dnd/scene-descriptions/source_refs"),
pipeline.Binding("generic/valid_json_schema"),
pipeline.Binding("extract/dnd/scene-descriptions/source_relatedness"),
pipeline.Binding(combatsemantics.Key),
}}
profile.Steps[0].Artifacts["scene-descriptions"] = sceneLane
configValue.Pipelines["dnd-npc-grounded"] = profile
client := &groundedDNDLLMClient{sceneKinds: []dnd.SceneKind{dnd.SceneKindNarrative, dnd.SceneKindCombat}, combatSemanticsVerdicts: []string{"combat_should_be_added", "approved"}}
output := runGroundedPipeline(t, configValue, registries, client, nil)
if len(output.Rejected) != 0 {
t.Fatalf("rejected = %#v, want corrected acceptance", output.Rejected)
}
for _, group := range output.Diagnostics.Groups {
if group.Origin.ValidatorKey == combatsemantics.Key {
t.Fatalf("combat-semantics validator left a diagnostic after correction: %#v", group)
}
}
var scenes dnd.SceneDescriptionList
for _, serialized := range output.NormalizeOutputs {
if serialized.LaneID == "scene-descriptions" {
var err error
scenes, err = scenecodec.New().Decode(serialized.Artifact.Content)
if err != nil {
t.Fatal(err)
}
}
}
if len(scenes.Scenes) != 1 || scenes.Scenes[0].Kind != dnd.SceneKindCombat {
t.Fatalf("accepted scenes = %#v, want corrected combat scene", scenes)
}
requests := client.requestsSnapshot()
var sceneRequests, validatorRequests []contracts.StructuredCompletionRequest
for _, request := range requests {
switch request.PromptID {
case sceneextract.PromptID:
sceneRequests = append(sceneRequests, request)
case combatsemantics.PromptID:
validatorRequests = append(validatorRequests, request)
}
}
if len(sceneRequests) != 2 || len(validatorRequests) != 2 || sceneRequests[0].Correction != nil || sceneRequests[1].Correction == nil {
t.Fatalf("scene/validator requests = %d/%d; want correction and revalidation", len(sceneRequests), len(validatorRequests))
}
correction := sceneRequests[1].Correction
if !strings.Contains(string(correction.AssistantResponse), `"kind":"narrative"`) || !strings.Contains(correction.UserGuidance, "Return kind: combat") || !strings.Contains(correction.UserGuidance, "active encounter") {
t.Fatalf("correction = %#v, want latest candidate and useful guidance", correction)
}
for _, forbidden := range []string{combatsemantics.Key, combatsemantics.ReasonCodeActiveCombatNotClassified, "session:chunk"} {
if strings.Contains(correction.UserGuidance, forbidden) {
t.Fatalf("correction guidance leaked internal detail %q: %q", forbidden, correction.UserGuidance)
}
}
}
func TestProductionDNDOutputPublishesSelectedEvidenceContext(t *testing.T) { func TestProductionDNDOutputPublishesSelectedEvidenceContext(t *testing.T) {
registries := productionNPCRegistries(t) registries := productionNPCRegistries(t)
configValue := loadGroundedPipelineConfig(t) configValue := loadGroundedPipelineConfig(t)
@@ -514,6 +579,10 @@ type groundedDNDLLMClient struct {
sceneTitle string sceneTitle string
firstUnitID int firstUnitID int
thirdUnitID int thirdUnitID int
sceneKinds []dnd.SceneKind
combatSemanticsVerdicts []string
sceneCalls int
combatSemanticsCalls int
} }
func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
@@ -547,6 +616,10 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ
payload = map[string]any{"duplicate_groups": []any{}} payload = map[string]any{"duplicate_groups": []any{}}
case sceneextract.PromptID: case sceneextract.PromptID:
kind := client.sceneKind kind := client.sceneKind
if client.sceneCalls < len(client.sceneKinds) {
kind = client.sceneKinds[client.sceneCalls]
}
client.sceneCalls++
if kind == "" { if kind == "" {
kind = dnd.SceneKindCombat kind = dnd.SceneKindCombat
} }
@@ -555,6 +628,13 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ
title = "A combat encounter" title = "A combat encounter"
} }
payload = map[string]any{"kind": kind, "title": title, "summary": "The party faces an active encounter."} payload = map[string]any{"kind": kind, "title": title, "summary": "The party faces an active encounter."}
case combatsemantics.PromptID:
verdict := "approved"
if client.combatSemanticsCalls < len(client.combatSemanticsVerdicts) {
verdict = client.combatSemanticsVerdicts[client.combatSemanticsCalls]
}
client.combatSemanticsCalls++
payload = map[string]any{"verdict": verdict, "explanation": "The transcript shows an active encounter with hostile action."}
case spells.PromptID: case spells.PromptID:
payload = map[string]any{"spell_casts": []any{map[string]any{ payload = map[string]any{"spell_casts": []any{map[string]any{
"caster": "Mira Thorn", "spell": "Cure Wounds", "caster": "Mira Thorn", "spell": "Cure Wounds",