Test combat semantics correction flow
This commit is contained in:
@@ -308,6 +308,8 @@ This stage is small enough for one implementation prompt.
|
||||
|
||||
## Stage 5: Prove Feedback-Aware Producer Correction
|
||||
|
||||
✅ Complete
|
||||
|
||||
### Goal
|
||||
|
||||
Prove the assembled D&D path can reject a well-formed wrong scene kind, guide
|
||||
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
||||
combatsemantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
|
||||
)
|
||||
|
||||
func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T) {
|
||||
@@ -258,6 +259,70 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
|
||||
assertCurrentEvidence(t, combatValue.CombatTurns[0].SourceRefs)
|
||||
}
|
||||
|
||||
func TestSceneCombatSemanticsCorrectionRetriesAndRevalidates(t *testing.T) {
|
||||
registries := productionNPCRegistries(t)
|
||||
configValue := loadGroundedPipelineConfig(t)
|
||||
profile := configValue.Pipelines["dnd-npc-grounded"]
|
||||
sceneLane := profile.Steps[0].Artifacts["scene-descriptions"]
|
||||
sceneLane.Extract.Retries = 1
|
||||
sceneLane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{
|
||||
pipeline.Binding("generic/valid_json"),
|
||||
pipeline.Binding("extract/dnd/scene-descriptions/shape"),
|
||||
pipeline.Binding("extract/dnd/scene-descriptions/source_refs"),
|
||||
pipeline.Binding("generic/valid_json_schema"),
|
||||
pipeline.Binding("extract/dnd/scene-descriptions/source_relatedness"),
|
||||
pipeline.Binding(combatsemantics.Key),
|
||||
}}
|
||||
profile.Steps[0].Artifacts["scene-descriptions"] = sceneLane
|
||||
configValue.Pipelines["dnd-npc-grounded"] = profile
|
||||
|
||||
client := &groundedDNDLLMClient{sceneKinds: []dnd.SceneKind{dnd.SceneKindNarrative, dnd.SceneKindCombat}, combatSemanticsVerdicts: []string{"combat_should_be_added", "approved"}}
|
||||
output := runGroundedPipeline(t, configValue, registries, client, nil)
|
||||
if len(output.Rejected) != 0 {
|
||||
t.Fatalf("rejected = %#v, want corrected acceptance", output.Rejected)
|
||||
}
|
||||
for _, group := range output.Diagnostics.Groups {
|
||||
if group.Origin.ValidatorKey == combatsemantics.Key {
|
||||
t.Fatalf("combat-semantics validator left a diagnostic after correction: %#v", group)
|
||||
}
|
||||
}
|
||||
var scenes dnd.SceneDescriptionList
|
||||
for _, serialized := range output.NormalizeOutputs {
|
||||
if serialized.LaneID == "scene-descriptions" {
|
||||
var err error
|
||||
scenes, err = scenecodec.New().Decode(serialized.Artifact.Content)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(scenes.Scenes) != 1 || scenes.Scenes[0].Kind != dnd.SceneKindCombat {
|
||||
t.Fatalf("accepted scenes = %#v, want corrected combat scene", scenes)
|
||||
}
|
||||
requests := client.requestsSnapshot()
|
||||
var sceneRequests, validatorRequests []contracts.StructuredCompletionRequest
|
||||
for _, request := range requests {
|
||||
switch request.PromptID {
|
||||
case sceneextract.PromptID:
|
||||
sceneRequests = append(sceneRequests, request)
|
||||
case combatsemantics.PromptID:
|
||||
validatorRequests = append(validatorRequests, request)
|
||||
}
|
||||
}
|
||||
if len(sceneRequests) != 2 || len(validatorRequests) != 2 || sceneRequests[0].Correction != nil || sceneRequests[1].Correction == nil {
|
||||
t.Fatalf("scene/validator requests = %d/%d; want correction and revalidation", len(sceneRequests), len(validatorRequests))
|
||||
}
|
||||
correction := sceneRequests[1].Correction
|
||||
if !strings.Contains(string(correction.AssistantResponse), `"kind":"narrative"`) || !strings.Contains(correction.UserGuidance, "Return kind: combat") || !strings.Contains(correction.UserGuidance, "active encounter") {
|
||||
t.Fatalf("correction = %#v, want latest candidate and useful guidance", correction)
|
||||
}
|
||||
for _, forbidden := range []string{combatsemantics.Key, combatsemantics.ReasonCodeActiveCombatNotClassified, "session:chunk"} {
|
||||
if strings.Contains(correction.UserGuidance, forbidden) {
|
||||
t.Fatalf("correction guidance leaked internal detail %q: %q", forbidden, correction.UserGuidance)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionDNDOutputPublishesSelectedEvidenceContext(t *testing.T) {
|
||||
registries := productionNPCRegistries(t)
|
||||
configValue := loadGroundedPipelineConfig(t)
|
||||
@@ -508,12 +573,16 @@ func (loader *generatedReferenceCheckpointLoader) extractDependencies(laneID str
|
||||
}
|
||||
|
||||
type groundedDNDLLMClient struct {
|
||||
mu sync.Mutex
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
sceneKind dnd.SceneKind
|
||||
sceneTitle string
|
||||
firstUnitID int
|
||||
thirdUnitID int
|
||||
mu sync.Mutex
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
sceneKind dnd.SceneKind
|
||||
sceneTitle string
|
||||
firstUnitID int
|
||||
thirdUnitID int
|
||||
sceneKinds []dnd.SceneKind
|
||||
combatSemanticsVerdicts []string
|
||||
sceneCalls int
|
||||
combatSemanticsCalls int
|
||||
}
|
||||
|
||||
func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
@@ -547,6 +616,10 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ
|
||||
payload = map[string]any{"duplicate_groups": []any{}}
|
||||
case sceneextract.PromptID:
|
||||
kind := client.sceneKind
|
||||
if client.sceneCalls < len(client.sceneKinds) {
|
||||
kind = client.sceneKinds[client.sceneCalls]
|
||||
}
|
||||
client.sceneCalls++
|
||||
if kind == "" {
|
||||
kind = dnd.SceneKindCombat
|
||||
}
|
||||
@@ -555,6 +628,13 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ
|
||||
title = "A combat encounter"
|
||||
}
|
||||
payload = map[string]any{"kind": kind, "title": title, "summary": "The party faces an active encounter."}
|
||||
case combatsemantics.PromptID:
|
||||
verdict := "approved"
|
||||
if client.combatSemanticsCalls < len(client.combatSemanticsVerdicts) {
|
||||
verdict = client.combatSemanticsVerdicts[client.combatSemanticsCalls]
|
||||
}
|
||||
client.combatSemanticsCalls++
|
||||
payload = map[string]any{"verdict": verdict, "explanation": "The transcript shows an active encounter with hostile action."}
|
||||
case spells.PromptID:
|
||||
payload = map[string]any{"spell_casts": []any{map[string]any{
|
||||
"caster": "Mira Thorn", "spell": "Cure Wounds",
|
||||
|
||||
Reference in New Issue
Block a user