diff --git a/internal/modules/integration/dnd_combat_runner_test.go b/internal/modules/integration/dnd_combat_runner_test.go index 75ffb99..bfbbfc4 100644 --- a/internal/modules/integration/dnd_combat_runner_test.go +++ b/internal/modules/integration/dnd_combat_runner_test.go @@ -181,6 +181,56 @@ func TestProductionCombatPipelineAttributesExhaustedInvalidEnumsToShapeValidatio } } +func TestCombatPipelinePublishesUnavailableSceneWarningForMismatchedReference(t *testing.T) { + registries := productionNPCRegistries(t) + configValue := combatOnlyConfig() + profile := configValue.Pipelines["dnd-combat"] + profile.Chunk.Options["max_units"] = 100 + profile.References["scene_descriptions"] = pipeline.ExternalReference(combatSceneDescriptionPath(t, []dnd.SceneDescription{{ + ID: "chunk-000001", + SourceRef: source.SourceRef{SourceID: "npc-session", StartUnitID: 2, EndUnitID: 5}, + Kind: dnd.SceneKindCombat, + Title: "Mismatched combat encounter", + Summary: "The classification range does not match the accepted chunk.", + }})) + configValue.Pipelines["dnd-combat"] = profile + catalog := moduleCatalog(registries) + effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-combat", Catalog: catalog}) + if err != nil { + t.Fatalf("Resolve() error = %v", err) + } + materialized, warnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{}) + if err != nil || len(warnings) != 0 { + t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings) + } + client := &fakeCombatLLMClient{} + prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client}) + if err != nil { + t.Fatalf("Prepare() error = %v", err) + } + output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{ + Prepared: prepared, + RawInput: readNPCFixture(t), + ExtractWorkers: 1, + }) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + if len(client.requests) != 0 || len(output.Rejected) != 0 || !hasCombatWarning(output.Warnings, "scene_classification_unavailable") { + t.Fatalf("requests/rejected/warnings = %#v / %#v / %#v, want accepted unavailable-scene skip without a combat request", client.requests, output.Rejected, output.Warnings) + } + if len(output.NormalizeOutputs) != 1 { + t.Fatalf("normalized outputs = %#v, want accepted empty combat lane", output.NormalizeOutputs) + } + value, err := combatcodec.New().Decode(output.NormalizeOutputs[0].Artifact.Content) + if err != nil { + t.Fatalf("Decode(combat output) error = %v", err) + } + if value.CombatTurns == nil || len(value.CombatTurns) != 0 { + t.Fatalf("mismatched scene combat output = %#v, want non-nil empty turns", value) + } +} + func TestCombatNormalizerRejectsCampaignReferenceBinding(t *testing.T) { registries := productionNPCRegistries(t) catalog := moduleCatalog(registries) diff --git a/internal/modules/integration/dnd_npc_grounded_test.go b/internal/modules/integration/dnd_npc_grounded_test.go index 711828c..2455752 100644 --- a/internal/modules/integration/dnd_npc_grounded_test.go +++ b/internal/modules/integration/dnd_npc_grounded_test.go @@ -5,6 +5,8 @@ import ( "crypto/sha256" "encoding/json" "fmt" + "os" + "path/filepath" "strings" "sync" "testing" @@ -28,6 +30,7 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T registries := productionNPCRegistries(t) catalog := moduleCatalog(registries) configValue := loadGroundedPipelineConfig(t) + campaignInputs := configureGroundedCampaignReferences(t, &configValue) effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-grounded", Catalog: catalog}) if err != nil { t.Fatalf("Resolve() error = %v", err) @@ -63,10 +66,12 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T } { assertCombatFingerprint(t, prepared.CheckpointFingerprints(), name) } + checkpoint := newGeneratedReferenceCheckpointLoader() output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{ Prepared: prepared, RawInput: readNPCFixture(t), ExtractWorkers: 1, + Checkpoint: checkpoint, }) if err != nil { t.Fatalf("Run() error = %v", err) @@ -94,16 +99,26 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T } } - var npcPayload []byte + var npcPayload, scenePayload []byte + seenSteps := map[string]string{} for _, serialized := range output.NormalizeOutputs { - if serialized.LaneID == "npcs" { + seenSteps[serialized.LaneID] = serialized.StepID + switch serialized.LaneID { + case "npcs": npcPayload = append([]byte(nil), serialized.Artifact.Content...) - break + case "scene-descriptions": + scenePayload = append([]byte(nil), serialized.Artifact.Content...) } } + if seenSteps["npcs"] != "identify-npcs" || seenSteps["scene-descriptions"] != "identify-npcs" || seenSteps["spells"] != "grounded-events" || seenSteps["combat"] != "grounded-events" { + t.Fatalf("normalized output steps = %#v, want ordered producer and consumer steps", seenSteps) + } if len(npcPayload) == 0 { t.Fatal("NPC producer did not publish a canonical payload") } + if len(scenePayload) == 0 { + t.Fatal("scene producer did not publish a canonical payload") + } npcValue, err := npccodec.New().Decode(npcPayload) if err != nil { t.Fatalf("decode NPC producer payload: %v", err) @@ -113,6 +128,15 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T t.Fatalf("encode canonical NPC producer payload: %v", err) } canonicalDigest := digestBytes(npcPayload) + sceneValue, err := scenecodec.New().Decode(scenePayload) + if err != nil { + t.Fatalf("decode scene producer payload: %v", err) + } + scenePayload, err = scenecodec.New().Encode(sceneValue) + if err != nil { + t.Fatalf("encode canonical scene producer payload: %v", err) + } + sceneDigest := digestBytes(scenePayload) projection := []byte(`{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}`) projectionDigest := digestBytes(projection) seenConsumers := map[string]bool{} @@ -124,11 +148,29 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T if input.MediaType != npccodec.MediaType || input.Digest != projectionDigest || string(input.Content) != string(projection) || input.OriginURI != "" { t.Fatalf("%s NPC prompt input = %#v, want names-only generated registry", request.PromptID, input) } + if request.PromptID == combatextract.PromptID { + transcript, ok := request.Inputs["transcript"] + if !ok || !strings.Contains(string(transcript.Content), "Mira Thorn asks the party") { + t.Fatalf("combat transcript input = %#v, want configured transcript content", transcript) + } + for name, want := range campaignInputs { + input, ok := request.Inputs[name] + if !ok || string(input.Content) != want { + t.Fatalf("combat %s input = %#v, want configured campaign reference", name, input) + } + } + if _, ok := request.Inputs["scene_descriptions"]; ok { + t.Fatalf("combat prompt inputs = %#v, must not contain scene descriptions", request.Inputs) + } + } seenConsumers[request.PromptID] = true } if !seenConsumers[spells.PromptID] || !seenConsumers[combatextract.PromptID] { t.Fatalf("consumer prompt IDs = %#v, want spell and combat requests", seenConsumers) } + if combatCompletionCount(client.requestsSnapshot()) != 1 { + t.Fatalf("combat completion requests = %#v, want exactly one for the exact combat scene", client.requestsSnapshot()) + } provenanceCount := 0 for _, reference := range output.Manifest.References { @@ -147,21 +189,34 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T for _, reference := range output.Manifest.References { if reference.SlotName == "scene_descriptions" { sceneProvenanceCount++ + if reference.Stage != string(pipeline.StageExtract) || reference.StepID != "grounded-events" || reference.LaneID != "combat" || reference.OriginType != "generated" || reference.Digest != sceneDigest || reference.ArtifactKind != string(dnd.SceneDescriptionListKind) || reference.SchemaID != scenecodec.SchemaID || reference.SchemaName != scenecodec.SchemaName || reference.SchemaVersion != scenecodec.SchemaVersion || reference.MediaType != scenecodec.MediaType || reference.ProducerPipeline != "dnd-npc-grounded" || reference.ProducerStep != "identify-npcs" || reference.ProducerLane != "scene-descriptions" || reference.ProducerModule != "dnd/scene-descriptions" { + t.Fatalf("scene generated provenance = %#v, want canonical combat handoff identity", reference) + } } } if sceneProvenanceCount != 1 { t.Fatalf("scene generated provenance count = %d, want combat extractor handoff", sceneProvenanceCount) } + assertGeneratedSceneDependency(t, checkpoint.extractDependencies("combat")) manifestContent, err := json.Marshal(output.Manifest) if err != nil { t.Fatal(err) } - for _, forbidden := range []string{"Mira Thorn", "Hooded Guard", "For every NPC record", "Extract Dungeons & Dragons"} { + for _, forbidden := range []string{"Mira Thorn", "Hooded Guard", "For every NPC record", "Extract Dungeons & Dragons", "A combat encounter", "The party faces an active encounter.", "Mira Thorn asks the party", `"scenes":`} { if strings.Contains(string(manifestContent), forbidden) { t.Fatalf("manifest exposes prompt or reference payload content %q", forbidden) } } for _, lane := range output.Manifest.ArtifactLanes { + componentContent, err := json.Marshal(lane.Metadata) + if err != nil { + t.Fatalf("marshal %s component metadata: %v", lane.ID, err) + } + for _, forbidden := range []string{"A combat encounter", "The party faces an active encounter.", "Mira Thorn asks the party", `"scenes":`} { + if strings.Contains(string(componentContent), forbidden) { + t.Fatalf("%s component metadata exposes scene or transcript content %q", lane.ID, forbidden) + } + } for _, component := range []string{"extractor", "normalizer"} { metadata, ok := lane.Metadata[component].(map[string]any) if ok && (metadata["npc_registry_digest"] != nil || metadata["scene_eligibility_digest"] != nil) { @@ -193,6 +248,38 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T assertCurrentEvidence(t, combatValue.CombatTurns[0].SourceRefs) } +func TestGroundedPipelineSkipsCombatForExactNarrativeScene(t *testing.T) { + registries := productionNPCRegistries(t) + configValue := loadGroundedPipelineConfig(t) + profile := configValue.Pipelines["dnd-npc-grounded"] + combat := profile.Steps[1].Artifacts["combat"] + combat.Extract.Retries = 2 + profile.Steps[1].Artifacts["combat"] = combat + configValue.Pipelines["dnd-npc-grounded"] = profile + + client := &groundedDNDLLMClient{sceneKind: dnd.SceneKindNarrative} + output := runGroundedPipeline(t, configValue, registries, client, nil) + if combatCompletionCount(client.requestsSnapshot()) != 0 { + t.Fatalf("combat completion requests = %#v, want none for an exact narrative scene", client.requestsSnapshot()) + } + if len(output.Rejected) != 0 || hasCombatWarning(output.Warnings, "scene_classification_unavailable") { + t.Fatalf("narrative output rejected/warnings = %#v / %#v, want accepted skip without unavailable warning", output.Rejected, output.Warnings) + } + combatValue := normalizedCombatOutput(t, output) + if combatValue.CombatTurns == nil || len(combatValue.CombatTurns) != 0 { + t.Fatalf("narrative combat output = %#v, want an accepted empty combat list", combatValue) + } +} + +func TestGeneratedSceneReferenceChangesCombatCheckpointDependency(t *testing.T) { + registries := productionNPCRegistries(t) + first := generatedSceneDependency(t, registries, "A combat encounter") + second := generatedSceneDependency(t, registries, "A revised combat encounter") + if first == second { + t.Fatalf("generated combat dependencies = %q and %q, want canonical scene artifact change to invalidate reuse", first, second) + } +} + func assertFingerprintValue(t *testing.T, fingerprints []pipeline.CheckpointFingerprint, name, want string) { t.Helper() for _, fingerprint := range fingerprints { @@ -225,9 +312,128 @@ func loadGroundedPipelineConfig(t *testing.T) config.Config { return cfg } +func configureGroundedCampaignReferences(t *testing.T, cfg *config.Config) map[string]string { + t.Helper() + values := map[string]string{ + "party": "Mira Thorn: ally\n", + "glossary": "Greencloak: Mira Thorn's title\n", + } + profile := cfg.Pipelines["dnd-npc-grounded"] + profile.References = make(map[string]pipeline.ReferenceSource, len(values)) + for name, content := range values { + path := filepath.Join(t.TempDir(), name+".txt") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("write %s reference: %v", name, err) + } + profile.References[name] = pipeline.ExternalReference(path) + } + cfg.Pipelines["dnd-npc-grounded"] = profile + return values +} + +func runGroundedPipeline(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient, checkpoint pipeline.CheckpointLoader) pipeline.RunOutput { + t.Helper() + catalog := moduleCatalog(registries) + effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-grounded", Catalog: catalog}) + if err != nil { + t.Fatalf("Resolve() error = %v", err) + } + materialized, warnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{}) + if err != nil || len(warnings) != 0 { + t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings) + } + prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client}) + if err != nil { + t.Fatalf("Prepare() error = %v", err) + } + output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{ + Prepared: prepared, + RawInput: readNPCFixture(t), + ExtractWorkers: 1, + Checkpoint: checkpoint, + }) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + return output +} + +func normalizedCombatOutput(t *testing.T, output pipeline.RunOutput) dnd.CombatTurnList { + t.Helper() + for _, serialized := range output.NormalizeOutputs { + if serialized.LaneID != "combat" { + continue + } + value, err := combatcodec.New().Decode(serialized.Artifact.Content) + if err != nil { + t.Fatalf("decode combat output: %v", err) + } + return value + } + t.Fatalf("normalized outputs = %#v, want combat lane", output.NormalizeOutputs) + return dnd.CombatTurnList{} +} + +func combatCompletionCount(requests []contracts.StructuredCompletionRequest) int { + count := 0 + for _, request := range requests { + if request.PromptID == combatextract.PromptID { + count++ + } + } + return count +} + +func generatedSceneDependency(t *testing.T, registries pipeline.Registries, sceneTitle string) string { + t.Helper() + client := &groundedDNDLLMClient{sceneTitle: sceneTitle} + checkpoint := newGeneratedReferenceCheckpointLoader() + runGroundedPipeline(t, loadGroundedPipelineConfig(t), registries, client, checkpoint) + return assertGeneratedSceneDependency(t, checkpoint.extractDependencies("combat")) +} + +func assertGeneratedSceneDependency(t *testing.T, dependencies []pipeline.CheckpointFingerprint) string { + t.Helper() + for _, dependency := range dependencies { + if dependency.Name == "generated-reference:scene_descriptions:0" && strings.HasPrefix(dependency.Value, "sha256:") { + return dependency.Value + } + } + t.Fatalf("combat extract checkpoint dependencies = %#v, want generated scene reference fingerprint", dependencies) + return "" +} + +type generatedReferenceCheckpointLoader struct { + pipeline.CheckpointLoader + extract map[string][][]pipeline.CheckpointFingerprint +} + +func newGeneratedReferenceCheckpointLoader() *generatedReferenceCheckpointLoader { + return &generatedReferenceCheckpointLoader{ + CheckpointLoader: pipeline.NoopCheckpointLoader(), + extract: make(map[string][][]pipeline.CheckpointFingerprint), + } +} + +func (loader *generatedReferenceCheckpointLoader) Enabled() bool { return true } + +func (loader *generatedReferenceCheckpointLoader) Extract(laneID, _ string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) { + loader.extract[laneID] = append(loader.extract[laneID], append([]pipeline.CheckpointFingerprint(nil), dependencies...)) + return pipeline.ExtractCheckpoint{}, pipeline.NewCheckpointDecision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonMissing) +} + +func (loader *generatedReferenceCheckpointLoader) extractDependencies(laneID string) []pipeline.CheckpointFingerprint { + if len(loader.extract[laneID]) != 1 { + return nil + } + return loader.extract[laneID][0] +} + type groundedDNDLLMClient struct { - mu sync.Mutex - requests []contracts.StructuredCompletionRequest + mu sync.Mutex + requests []contracts.StructuredCompletionRequest + sceneKind dnd.SceneKind + sceneTitle string } func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { @@ -250,7 +456,15 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ }, }} case sceneextract.PromptID: - payload = map[string]any{"kind": "combat", "title": "A combat encounter", "summary": "The party faces an active encounter."} + kind := client.sceneKind + if kind == "" { + kind = dnd.SceneKindCombat + } + title := client.sceneTitle + if title == "" { + title = "A combat encounter" + } + payload = map[string]any{"kind": kind, "title": title, "summary": "The party faces an active encounter."} case spells.PromptID: payload = map[string]any{"spell_casts": []any{map[string]any{ "caster": "Mira Thorn", "spell": "Cure Wounds",