eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Test scene-aware combat handoff behavior

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-25 19:29:15 +00:00
parent 989f2c220b
commit b1c0faa748
2 changed files with 271 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
internal/modules/integration/dnd_combat_runner_test.go
View File

@@ -181,6 +181,56 @@ func TestProductionCombatPipelineAttributesExhaustedInvalidEnumsToShapeValidatio
}
}
func TestCombatPipelinePublishesUnavailableSceneWarningForMismatchedReference(t *testing.T) {
registries := productionNPCRegistries(t)
configValue := combatOnlyConfig()
profile := configValue.Pipelines["dnd-combat"]
profile.Chunk.Options["max_units"] = 100
profile.References["scene_descriptions"] = pipeline.ExternalReference(combatSceneDescriptionPath(t, []dnd.SceneDescription{{
ID: "chunk-000001",
SourceRef: source.SourceRef{SourceID: "npc-session", StartUnitID: 2, EndUnitID: 5},
Kind: dnd.SceneKindCombat,
Title: "Mismatched combat encounter",
Summary: "The classification range does not match the accepted chunk.",
}}))
configValue.Pipelines["dnd-combat"] = profile
catalog := moduleCatalog(registries)
effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-combat", Catalog: catalog})
if err != nil {
t.Fatalf("Resolve() error = %v", err)
}
materialized, warnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{})
if err != nil || len(warnings) != 0 {
t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings)
}
client := &fakeCombatLLMClient{}
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client})
if err != nil {
t.Fatalf("Prepare() error = %v", err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readNPCFixture(t),
ExtractWorkers: 1,
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(client.requests) != 0 || len(output.Rejected) != 0 || !hasCombatWarning(output.Warnings, "scene_classification_unavailable") {
t.Fatalf("requests/rejected/warnings = %#v / %#v / %#v, want accepted unavailable-scene skip without a combat request", client.requests, output.Rejected, output.Warnings)
}
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("normalized outputs = %#v, want accepted empty combat lane", output.NormalizeOutputs)
}
value, err := combatcodec.New().Decode(output.NormalizeOutputs[0].Artifact.Content)
if err != nil {
t.Fatalf("Decode(combat output) error = %v", err)
}
if value.CombatTurns == nil || len(value.CombatTurns) != 0 {
t.Fatalf("mismatched scene combat output = %#v, want non-nil empty turns", value)
}
}
func TestCombatNormalizerRejectsCampaignReferenceBinding(t *testing.T) {
registries := productionNPCRegistries(t)
catalog := moduleCatalog(registries)

228
internal/modules/integration/dnd_npc_grounded_test.go
View File

@@ -5,6 +5,8 @@ import (
"crypto/sha256"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"testing"
@@ -28,6 +30,7 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
registries := productionNPCRegistries(t)
catalog := moduleCatalog(registries)
configValue := loadGroundedPipelineConfig(t)
campaignInputs := configureGroundedCampaignReferences(t, &configValue)
effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-grounded", Catalog: catalog})
if err != nil {
t.Fatalf("Resolve() error = %v", err)
@@ -63,10 +66,12 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
} {
assertCombatFingerprint(t, prepared.CheckpointFingerprints(), name)
}
checkpoint := newGeneratedReferenceCheckpointLoader()
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readNPCFixture(t),
ExtractWorkers: 1,
Checkpoint: checkpoint,
})
if err != nil {
t.Fatalf("Run() error = %v", err)
@@ -94,16 +99,26 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
}
}
var npcPayload []byte
var npcPayload, scenePayload []byte
seenSteps := map[string]string{}
for _, serialized := range output.NormalizeOutputs {
if serialized.LaneID == "npcs" {
seenSteps[serialized.LaneID] = serialized.StepID
switch serialized.LaneID {
case "npcs":
npcPayload = append([]byte(nil), serialized.Artifact.Content...)
break
case "scene-descriptions":
scenePayload = append([]byte(nil), serialized.Artifact.Content...)
}
}
if seenSteps["npcs"] != "identify-npcs" || seenSteps["scene-descriptions"] != "identify-npcs" || seenSteps["spells"] != "grounded-events" || seenSteps["combat"] != "grounded-events" {
t.Fatalf("normalized output steps = %#v, want ordered producer and consumer steps", seenSteps)
}
if len(npcPayload) == 0 {
t.Fatal("NPC producer did not publish a canonical payload")
}
if len(scenePayload) == 0 {
t.Fatal("scene producer did not publish a canonical payload")
}
npcValue, err := npccodec.New().Decode(npcPayload)
if err != nil {
t.Fatalf("decode NPC producer payload: %v", err)
@@ -113,6 +128,15 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
t.Fatalf("encode canonical NPC producer payload: %v", err)
}
canonicalDigest := digestBytes(npcPayload)
sceneValue, err := scenecodec.New().Decode(scenePayload)
if err != nil {
t.Fatalf("decode scene producer payload: %v", err)
}
scenePayload, err = scenecodec.New().Encode(sceneValue)
if err != nil {
t.Fatalf("encode canonical scene producer payload: %v", err)
}
sceneDigest := digestBytes(scenePayload)
projection := []byte(`{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}`)
projectionDigest := digestBytes(projection)
seenConsumers := map[string]bool{}
@@ -124,11 +148,29 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
if input.MediaType != npccodec.MediaType || input.Digest != projectionDigest || string(input.Content) != string(projection) || input.OriginURI != "" {
t.Fatalf("%s NPC prompt input = %#v, want names-only generated registry", request.PromptID, input)
}
if request.PromptID == combatextract.PromptID {
transcript, ok := request.Inputs["transcript"]
if !ok || !strings.Contains(string(transcript.Content), "Mira Thorn asks the party") {
t.Fatalf("combat transcript input = %#v, want configured transcript content", transcript)
}
for name, want := range campaignInputs {
input, ok := request.Inputs[name]
if !ok || string(input.Content) != want {
t.Fatalf("combat %s input = %#v, want configured campaign reference", name, input)
}
}
if _, ok := request.Inputs["scene_descriptions"]; ok {
t.Fatalf("combat prompt inputs = %#v, must not contain scene descriptions", request.Inputs)
}
}
seenConsumers[request.PromptID] = true
}
if !seenConsumers[spells.PromptID] || !seenConsumers[combatextract.PromptID] {
t.Fatalf("consumer prompt IDs = %#v, want spell and combat requests", seenConsumers)
}
if combatCompletionCount(client.requestsSnapshot()) != 1 {
t.Fatalf("combat completion requests = %#v, want exactly one for the exact combat scene", client.requestsSnapshot())
}
provenanceCount := 0
for _, reference := range output.Manifest.References {
@@ -147,21 +189,34 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
for _, reference := range output.Manifest.References {
if reference.SlotName == "scene_descriptions" {
sceneProvenanceCount++
if reference.Stage != string(pipeline.StageExtract) || reference.StepID != "grounded-events" || reference.LaneID != "combat" || reference.OriginType != "generated" || reference.Digest != sceneDigest || reference.ArtifactKind != string(dnd.SceneDescriptionListKind) || reference.SchemaID != scenecodec.SchemaID || reference.SchemaName != scenecodec.SchemaName || reference.SchemaVersion != scenecodec.SchemaVersion || reference.MediaType != scenecodec.MediaType || reference.ProducerPipeline != "dnd-npc-grounded" || reference.ProducerStep != "identify-npcs" || reference.ProducerLane != "scene-descriptions" || reference.ProducerModule != "dnd/scene-descriptions" {
t.Fatalf("scene generated provenance = %#v, want canonical combat handoff identity", reference)
}
}
}
if sceneProvenanceCount != 1 {
t.Fatalf("scene generated provenance count = %d, want combat extractor handoff", sceneProvenanceCount)
}
assertGeneratedSceneDependency(t, checkpoint.extractDependencies("combat"))
manifestContent, err := json.Marshal(output.Manifest)
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"Mira Thorn", "Hooded Guard", "For every NPC record", "Extract Dungeons & Dragons"} {
for _, forbidden := range []string{"Mira Thorn", "Hooded Guard", "For every NPC record", "Extract Dungeons & Dragons", "A combat encounter", "The party faces an active encounter.", "Mira Thorn asks the party", `"scenes":`} {
if strings.Contains(string(manifestContent), forbidden) {
t.Fatalf("manifest exposes prompt or reference payload content %q", forbidden)
}
}
for _, lane := range output.Manifest.ArtifactLanes {
componentContent, err := json.Marshal(lane.Metadata)
if err != nil {
t.Fatalf("marshal %s component metadata: %v", lane.ID, err)
}
for _, forbidden := range []string{"A combat encounter", "The party faces an active encounter.", "Mira Thorn asks the party", `"scenes":`} {
if strings.Contains(string(componentContent), forbidden) {
t.Fatalf("%s component metadata exposes scene or transcript content %q", lane.ID, forbidden)
}
}
for _, component := range []string{"extractor", "normalizer"} {
metadata, ok := lane.Metadata[component].(map[string]any)
if ok && (metadata["npc_registry_digest"] != nil || metadata["scene_eligibility_digest"] != nil) {
@@ -193,6 +248,38 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
assertCurrentEvidence(t, combatValue.CombatTurns[0].SourceRefs)
}
func TestGroundedPipelineSkipsCombatForExactNarrativeScene(t *testing.T) {
registries := productionNPCRegistries(t)
configValue := loadGroundedPipelineConfig(t)
profile := configValue.Pipelines["dnd-npc-grounded"]
combat := profile.Steps[1].Artifacts["combat"]
combat.Extract.Retries = 2
profile.Steps[1].Artifacts["combat"] = combat
configValue.Pipelines["dnd-npc-grounded"] = profile
client := &groundedDNDLLMClient{sceneKind: dnd.SceneKindNarrative}
output := runGroundedPipeline(t, configValue, registries, client, nil)
if combatCompletionCount(client.requestsSnapshot()) != 0 {
t.Fatalf("combat completion requests = %#v, want none for an exact narrative scene", client.requestsSnapshot())
}
if len(output.Rejected) != 0 || hasCombatWarning(output.Warnings, "scene_classification_unavailable") {
t.Fatalf("narrative output rejected/warnings = %#v / %#v, want accepted skip without unavailable warning", output.Rejected, output.Warnings)
}
combatValue := normalizedCombatOutput(t, output)
if combatValue.CombatTurns == nil || len(combatValue.CombatTurns) != 0 {
t.Fatalf("narrative combat output = %#v, want an accepted empty combat list", combatValue)
}
}
func TestGeneratedSceneReferenceChangesCombatCheckpointDependency(t *testing.T) {
registries := productionNPCRegistries(t)
first := generatedSceneDependency(t, registries, "A combat encounter")
second := generatedSceneDependency(t, registries, "A revised combat encounter")
if first == second {
t.Fatalf("generated combat dependencies = %q and %q, want canonical scene artifact change to invalidate reuse", first, second)
}
}
func assertFingerprintValue(t *testing.T, fingerprints []pipeline.CheckpointFingerprint, name, want string) {
t.Helper()
for _, fingerprint := range fingerprints {
@@ -225,9 +312,128 @@ func loadGroundedPipelineConfig(t *testing.T) config.Config {
return cfg
}
func configureGroundedCampaignReferences(t *testing.T, cfg *config.Config) map[string]string {
t.Helper()
values := map[string]string{
"party": "Mira Thorn: ally\n",
"glossary": "Greencloak: Mira Thorn's title\n",
}
profile := cfg.Pipelines["dnd-npc-grounded"]
profile.References = make(map[string]pipeline.ReferenceSource, len(values))
for name, content := range values {
path := filepath.Join(t.TempDir(), name+".txt")
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
t.Fatalf("write %s reference: %v", name, err)
}
profile.References[name] = pipeline.ExternalReference(path)
}
cfg.Pipelines["dnd-npc-grounded"] = profile
return values
}
func runGroundedPipeline(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient, checkpoint pipeline.CheckpointLoader) pipeline.RunOutput {
t.Helper()
catalog := moduleCatalog(registries)
effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-grounded", Catalog: catalog})
if err != nil {
t.Fatalf("Resolve() error = %v", err)
}
materialized, warnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{})
if err != nil || len(warnings) != 0 {
t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings)
}
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client})
if err != nil {
t.Fatalf("Prepare() error = %v", err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readNPCFixture(t),
ExtractWorkers: 1,
Checkpoint: checkpoint,
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
return output
}
func normalizedCombatOutput(t *testing.T, output pipeline.RunOutput) dnd.CombatTurnList {
t.Helper()
for _, serialized := range output.NormalizeOutputs {
if serialized.LaneID != "combat" {
continue
}
value, err := combatcodec.New().Decode(serialized.Artifact.Content)
if err != nil {
t.Fatalf("decode combat output: %v", err)
}
return value
}
t.Fatalf("normalized outputs = %#v, want combat lane", output.NormalizeOutputs)
return dnd.CombatTurnList{}
}
func combatCompletionCount(requests []contracts.StructuredCompletionRequest) int {
count := 0
for _, request := range requests {
if request.PromptID == combatextract.PromptID {
count++
}
}
return count
}
func generatedSceneDependency(t *testing.T, registries pipeline.Registries, sceneTitle string) string {
t.Helper()
client := &groundedDNDLLMClient{sceneTitle: sceneTitle}
checkpoint := newGeneratedReferenceCheckpointLoader()
runGroundedPipeline(t, loadGroundedPipelineConfig(t), registries, client, checkpoint)
return assertGeneratedSceneDependency(t, checkpoint.extractDependencies("combat"))
}
func assertGeneratedSceneDependency(t *testing.T, dependencies []pipeline.CheckpointFingerprint) string {
t.Helper()
for _, dependency := range dependencies {
if dependency.Name == "generated-reference:scene_descriptions:0" && strings.HasPrefix(dependency.Value, "sha256:") {
return dependency.Value
}
}
t.Fatalf("combat extract checkpoint dependencies = %#v, want generated scene reference fingerprint", dependencies)
return ""
}
type generatedReferenceCheckpointLoader struct {
pipeline.CheckpointLoader
extract map[string][][]pipeline.CheckpointFingerprint
}
func newGeneratedReferenceCheckpointLoader() *generatedReferenceCheckpointLoader {
return &generatedReferenceCheckpointLoader{
CheckpointLoader: pipeline.NoopCheckpointLoader(),
extract: make(map[string][][]pipeline.CheckpointFingerprint),
}
}
func (loader *generatedReferenceCheckpointLoader) Enabled() bool { return true }
func (loader *generatedReferenceCheckpointLoader) Extract(laneID, _ string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) {
loader.extract[laneID] = append(loader.extract[laneID], append([]pipeline.CheckpointFingerprint(nil), dependencies...))
return pipeline.ExtractCheckpoint{}, pipeline.NewCheckpointDecision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonMissing)
}
func (loader *generatedReferenceCheckpointLoader) extractDependencies(laneID string) []pipeline.CheckpointFingerprint {
if len(loader.extract[laneID]) != 1 {
return nil
}
return loader.extract[laneID][0]
}
type groundedDNDLLMClient struct {
mu sync.Mutex
requests []contracts.StructuredCompletionRequest
mu sync.Mutex
requests []contracts.StructuredCompletionRequest
sceneKind dnd.SceneKind
sceneTitle string
}
func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
@@ -250,7 +456,15 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ
},
}}
case sceneextract.PromptID:
payload = map[string]any{"kind": "combat", "title": "A combat encounter", "summary": "The party faces an active encounter."}
kind := client.sceneKind
if kind == "" {
kind = dnd.SceneKindCombat
}
title := client.sceneTitle
if title == "" {
title = "A combat encounter"
}
payload = map[string]any{"kind": kind, "title": title, "summary": "The party faces an active encounter."}
case spells.PromptID:
payload = map[string]any{"spell_casts": []any{map[string]any{
"caster": "Mira Thorn", "spell": "Cure Wounds",