Test combat semantics correction flow
This commit is contained in:
@@ -308,6 +308,8 @@ This stage is small enough for one implementation prompt.
|
|||||||
|
|
||||||
## Stage 5: Prove Feedback-Aware Producer Correction
|
## Stage 5: Prove Feedback-Aware Producer Correction
|
||||||
|
|
||||||
|
✅ Complete
|
||||||
|
|
||||||
### Goal
|
### Goal
|
||||||
|
|
||||||
Prove the assembled D&D path can reject a well-formed wrong scene kind, guide
|
Prove the assembled D&D path can reject a well-formed wrong scene kind, guide
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ import (
|
|||||||
sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
|
sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||||
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
||||||
|
combatsemantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T) {
|
func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T) {
|
||||||
@@ -258,6 +259,70 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
|
|||||||
assertCurrentEvidence(t, combatValue.CombatTurns[0].SourceRefs)
|
assertCurrentEvidence(t, combatValue.CombatTurns[0].SourceRefs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSceneCombatSemanticsCorrectionRetriesAndRevalidates(t *testing.T) {
|
||||||
|
registries := productionNPCRegistries(t)
|
||||||
|
configValue := loadGroundedPipelineConfig(t)
|
||||||
|
profile := configValue.Pipelines["dnd-npc-grounded"]
|
||||||
|
sceneLane := profile.Steps[0].Artifacts["scene-descriptions"]
|
||||||
|
sceneLane.Extract.Retries = 1
|
||||||
|
sceneLane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{
|
||||||
|
pipeline.Binding("generic/valid_json"),
|
||||||
|
pipeline.Binding("extract/dnd/scene-descriptions/shape"),
|
||||||
|
pipeline.Binding("extract/dnd/scene-descriptions/source_refs"),
|
||||||
|
pipeline.Binding("generic/valid_json_schema"),
|
||||||
|
pipeline.Binding("extract/dnd/scene-descriptions/source_relatedness"),
|
||||||
|
pipeline.Binding(combatsemantics.Key),
|
||||||
|
}}
|
||||||
|
profile.Steps[0].Artifacts["scene-descriptions"] = sceneLane
|
||||||
|
configValue.Pipelines["dnd-npc-grounded"] = profile
|
||||||
|
|
||||||
|
client := &groundedDNDLLMClient{sceneKinds: []dnd.SceneKind{dnd.SceneKindNarrative, dnd.SceneKindCombat}, combatSemanticsVerdicts: []string{"combat_should_be_added", "approved"}}
|
||||||
|
output := runGroundedPipeline(t, configValue, registries, client, nil)
|
||||||
|
if len(output.Rejected) != 0 {
|
||||||
|
t.Fatalf("rejected = %#v, want corrected acceptance", output.Rejected)
|
||||||
|
}
|
||||||
|
for _, group := range output.Diagnostics.Groups {
|
||||||
|
if group.Origin.ValidatorKey == combatsemantics.Key {
|
||||||
|
t.Fatalf("combat-semantics validator left a diagnostic after correction: %#v", group)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var scenes dnd.SceneDescriptionList
|
||||||
|
for _, serialized := range output.NormalizeOutputs {
|
||||||
|
if serialized.LaneID == "scene-descriptions" {
|
||||||
|
var err error
|
||||||
|
scenes, err = scenecodec.New().Decode(serialized.Artifact.Content)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(scenes.Scenes) != 1 || scenes.Scenes[0].Kind != dnd.SceneKindCombat {
|
||||||
|
t.Fatalf("accepted scenes = %#v, want corrected combat scene", scenes)
|
||||||
|
}
|
||||||
|
requests := client.requestsSnapshot()
|
||||||
|
var sceneRequests, validatorRequests []contracts.StructuredCompletionRequest
|
||||||
|
for _, request := range requests {
|
||||||
|
switch request.PromptID {
|
||||||
|
case sceneextract.PromptID:
|
||||||
|
sceneRequests = append(sceneRequests, request)
|
||||||
|
case combatsemantics.PromptID:
|
||||||
|
validatorRequests = append(validatorRequests, request)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(sceneRequests) != 2 || len(validatorRequests) != 2 || sceneRequests[0].Correction != nil || sceneRequests[1].Correction == nil {
|
||||||
|
t.Fatalf("scene/validator requests = %d/%d; want correction and revalidation", len(sceneRequests), len(validatorRequests))
|
||||||
|
}
|
||||||
|
correction := sceneRequests[1].Correction
|
||||||
|
if !strings.Contains(string(correction.AssistantResponse), `"kind":"narrative"`) || !strings.Contains(correction.UserGuidance, "Return kind: combat") || !strings.Contains(correction.UserGuidance, "active encounter") {
|
||||||
|
t.Fatalf("correction = %#v, want latest candidate and useful guidance", correction)
|
||||||
|
}
|
||||||
|
for _, forbidden := range []string{combatsemantics.Key, combatsemantics.ReasonCodeActiveCombatNotClassified, "session:chunk"} {
|
||||||
|
if strings.Contains(correction.UserGuidance, forbidden) {
|
||||||
|
t.Fatalf("correction guidance leaked internal detail %q: %q", forbidden, correction.UserGuidance)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestProductionDNDOutputPublishesSelectedEvidenceContext(t *testing.T) {
|
func TestProductionDNDOutputPublishesSelectedEvidenceContext(t *testing.T) {
|
||||||
registries := productionNPCRegistries(t)
|
registries := productionNPCRegistries(t)
|
||||||
configValue := loadGroundedPipelineConfig(t)
|
configValue := loadGroundedPipelineConfig(t)
|
||||||
@@ -514,6 +579,10 @@ type groundedDNDLLMClient struct {
|
|||||||
sceneTitle string
|
sceneTitle string
|
||||||
firstUnitID int
|
firstUnitID int
|
||||||
thirdUnitID int
|
thirdUnitID int
|
||||||
|
sceneKinds []dnd.SceneKind
|
||||||
|
combatSemanticsVerdicts []string
|
||||||
|
sceneCalls int
|
||||||
|
combatSemanticsCalls int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||||
@@ -547,6 +616,10 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ
|
|||||||
payload = map[string]any{"duplicate_groups": []any{}}
|
payload = map[string]any{"duplicate_groups": []any{}}
|
||||||
case sceneextract.PromptID:
|
case sceneextract.PromptID:
|
||||||
kind := client.sceneKind
|
kind := client.sceneKind
|
||||||
|
if client.sceneCalls < len(client.sceneKinds) {
|
||||||
|
kind = client.sceneKinds[client.sceneCalls]
|
||||||
|
}
|
||||||
|
client.sceneCalls++
|
||||||
if kind == "" {
|
if kind == "" {
|
||||||
kind = dnd.SceneKindCombat
|
kind = dnd.SceneKindCombat
|
||||||
}
|
}
|
||||||
@@ -555,6 +628,13 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ
|
|||||||
title = "A combat encounter"
|
title = "A combat encounter"
|
||||||
}
|
}
|
||||||
payload = map[string]any{"kind": kind, "title": title, "summary": "The party faces an active encounter."}
|
payload = map[string]any{"kind": kind, "title": title, "summary": "The party faces an active encounter."}
|
||||||
|
case combatsemantics.PromptID:
|
||||||
|
verdict := "approved"
|
||||||
|
if client.combatSemanticsCalls < len(client.combatSemanticsVerdicts) {
|
||||||
|
verdict = client.combatSemanticsVerdicts[client.combatSemanticsCalls]
|
||||||
|
}
|
||||||
|
client.combatSemanticsCalls++
|
||||||
|
payload = map[string]any{"verdict": verdict, "explanation": "The transcript shows an active encounter with hostile action."}
|
||||||
case spells.PromptID:
|
case spells.PromptID:
|
||||||
payload = map[string]any{"spell_casts": []any{map[string]any{
|
payload = map[string]any{"spell_casts": []any{map[string]any{
|
||||||
"caster": "Mira Thorn", "spell": "Cure Wounds",
|
"caster": "Mira Thorn", "spell": "Cure Wounds",
|
||||||
|
|||||||
Reference in New Issue
Block a user