From 63de44c347e213aa1324da774ffdf2073dc45c01 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 25 Jul 2026 22:10:53 +0000 Subject: [PATCH] Add D&D item event integration coverage --- examples/dnd-complete.config.yml | 8 +- internal/cli/example_contract_test.go | 11 +- internal/cli/production_contract_test.go | 8 +- .../integration/dnd_itemevents_runner_test.go | 170 ++++++++++++++++++ .../seriatim_item_events_session.json | 12 ++ 5 files changed, 205 insertions(+), 4 deletions(-) create mode 100644 internal/modules/integration/dnd_itemevents_runner_test.go create mode 100644 internal/modules/integration/testdata/seriatim_item_events_session.json diff --git a/examples/dnd-complete.config.yml b/examples/dnd-complete.config.yml index 586a869..78453cb 100644 --- a/examples/dnd-complete.config.yml +++ b/examples/dnd-complete.config.yml @@ -29,9 +29,15 @@ pipelines: options: include_chunk_map: true steps: - # Establish session-wide reference artifacts before extracting events. + # Establish session-wide reference artifacts alongside independent item events. - id: describe-session artifacts: + item-events: + extract: + module: dnd/item-events + retries: 2 + merge: appendorder + normalize: dnd/item-events npcs: extract: module: dnd/npcs diff --git a/internal/cli/example_contract_test.go b/internal/cli/example_contract_test.go index 8062135..277a978 100644 --- a/internal/cli/example_contract_test.go +++ b/internal/cli/example_contract_test.go @@ -34,7 +34,7 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) { t.Fatalf("materialize maintained example references for %q: %v", pipelineID, err) } if example.name == "complete" { - if got := exampleStepLaneIDs(materialized); strings.Join(got, "|") != "describe-session:npcs,scene-descriptions|extract-events:combat-turns,npc-interactions,spells" { + if got := exampleStepLaneIDs(materialized); strings.Join(got, "|") != "describe-session:item-events,npcs,scene-descriptions|extract-events:combat-turns,npc-interactions,spells" { t.Fatalf("complete example steps and lanes = %v, want every D&D extractor in the documented two-step composition", got) } spellLane := referenceContractLane(t, materialized, "spells") @@ -42,6 +42,15 @@ func TestMaintainedExamplesLoadResolveAndList(t *testing.T) { len(spellLane.NormalizeReferences.ReferenceSet.Slots["spell_catalog"].Items) != 1 { t.Fatalf("complete example spell catalog reference was not materialized: %#v", spellLane) } + itemEventLane := referenceContractLane(t, materialized, "item-events") + for _, references := range []pipeline.ResolvedReferenceTarget{itemEventLane.ExtractReferences, itemEventLane.NormalizeReferences} { + if _, found := references.ReferenceSet.Slots["npcs"]; found { + t.Fatalf("item event lane unexpectedly depends on generated NPCs: %#v", itemEventLane) + } + if _, found := references.ReferenceSet.Slots["scene_descriptions"]; found { + t.Fatalf("item event lane unexpectedly depends on generated scene descriptions: %#v", itemEventLane) + } + } } } var stdout, stderr strings.Builder diff --git a/internal/cli/production_contract_test.go b/internal/cli/production_contract_test.go index 21f83bc..a487a30 100644 --- a/internal/cli/production_contract_test.go +++ b/internal/cli/production_contract_test.go @@ -561,8 +561,8 @@ func TestProductionSceneRunRecordsAnnotationFreeChunkPlanAndProvenance(t *testin if len(warnings.Warnings) != 0 { t.Fatalf("warnings = %#v, want none", warnings.Warnings) } - if len(fake.requestsFor(scenes.PromptID)) != 1 || len(fake.requestsFor(spells.PromptID)) != 1 { - t.Fatalf("fake prompt requests = %#v, want one scene and one spell request", fake.requestPrompts()) + if len(fake.requestsFor(scenes.PromptID)) != 1 || len(fake.requestsFor(spells.PromptID)) != 1 || len(fake.requestsFor(itemeventextract.PromptID)) != 1 { + t.Fatalf("fake prompt requests = %#v, want one scene, spell, and item-event request", fake.requestPrompts()) } } @@ -699,6 +699,8 @@ pipelines: artifacts: spells: extract: dnd/spells + item-events: + extract: dnd/item-events `, outputRoot, filepath.Join(filepath.Dir(outputRoot), "debug"), chunkModule) } @@ -797,6 +799,8 @@ func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, r } else { content = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`) } + case itemeventextract.PromptID: + content = []byte(`{"events":[{"name":"Cure Wounds","kind":"acquired","to":"party","source_refs":[{"start_segment":1,"end_segment":1}]}]}`) default: return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID) } diff --git a/internal/modules/integration/dnd_itemevents_runner_test.go b/internal/modules/integration/dnd_itemevents_runner_test.go new file mode 100644 index 0000000..b615cc7 --- /dev/null +++ b/internal/modules/integration/dnd_itemevents_runner_test.go @@ -0,0 +1,170 @@ +package integration_test + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/config" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + itemeventcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemevents" + itemeventextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/itemevents" + itemeventnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/itemevents" + itemeventshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemevents/shape" + itemeventrefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemevents/source_refs" + "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript" +) + +func TestProductionItemEventPipelineProducesNormalizedDurableArtifact(t *testing.T) { + raw := readItemEventFixture(t) + registries := productionNPCRegistries(t) + resolved := resolveItemEventPipeline(t, registries) + client := &itemEventProductionLLMClient{responses: [][]byte{[]byte(`{"events":[ + {"name":"Healing Potion","kind":"transferred","quantity":1,"from":"Aria","to":"Borin","source_refs":[{"start_segment":3,"end_segment":3}]}, + {"name":" Gold Pieces ","kind":"acquired","quantity":25,"to":" party ","source_refs":[{"start_segment":2,"end_segment":2}]}, + {"name":"Missing Relic","kind":"discovered","source_refs":[{"start_segment":4,"end_segment":4}]}, + {"name":"Ancient Coin","kind":"discovered","source_refs":[{"start_segment":1,"end_segment":1}]}, + {"name":"Gold Pieces","kind":"acquired","quantity":25,"to":"party","source_refs":[{"start_segment":2,"end_segment":2}]} + ]}`)}} + + output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: raw}) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 { + t.Fatalf("run output = %#v, rejected = %#v", output.NormalizeOutputs, output.Rejected) + } + serialized := normalizedLane(t, output, "item-events") + if serialized.NormalizerKey != itemeventnormalize.Key || serialized.Artifact.Schema.ID != itemeventcodec.SchemaID || serialized.Artifact.Schema.Version != itemeventcodec.SchemaVersion { + t.Fatalf("serialized item events = %#v", serialized) + } + events, err := itemeventcodec.New().Decode(serialized.Artifact.Content) + if err != nil { + t.Fatalf("Decode(item event output) error = %v", err) + } + if len(events.Events) != 4 { + t.Fatalf("item events = %#v, want one exact duplicate removed", events) + } + first, second, third, fourth := events.Events[0], events.Events[1], events.Events[2], events.Events[3] + if first.Name != "Ancient Coin" || second.Name != "Gold Pieces" || second.Kind != dnd.ItemEventKindAcquired || second.Quantity == nil || *second.Quantity != 25 || second.To != "party" || third.Name != "Healing Potion" || third.Kind != dnd.ItemEventKindTransferred || third.From != "Aria" || third.To != "Borin" || fourth.Name != "Missing Relic" { + t.Fatalf("normalized item event sequence = %#v", events.Events) + } + for _, event := range events.Events { + for _, ref := range event.SourceRefs { + if ref.SourceID != "item-events-session" { + t.Fatalf("source reference = %#v, want current transcript source", ref) + } + } + } + if !hasItemEventWarning(output.Warnings, "item_event_source_unrelated") || !hasItemEventWarning(output.Warnings, "duplicate_item_event_collapsed") { + t.Fatalf("warnings = %#v, want advisory relatedness and duplicate-collapse warnings", output.Warnings) + } + if output.Manifest.ValidationStatus != "approved" || len(output.Manifest.ArtifactLanes) != 1 { + t.Fatalf("manifest = %#v", output.Manifest) + } + lane := output.Manifest.ArtifactLanes[0] + if lane.ID != "item-events" || lane.Extractor != itemeventextract.Key || lane.Merger != pipeline.DefaultMergeModule || lane.Normalizer != itemeventnormalize.Key { + t.Fatalf("manifest lane = %#v", lane) + } + if len(client.requests) != 1 || client.requests[0].PromptID != itemeventextract.PromptID { + t.Fatalf("LLM requests = %#v", client.requests) + } +} + +func TestProductionItemEventPipelineRetriesBlockingCandidates(t *testing.T) { + registries := productionNPCRegistries(t) + resolved := resolveItemEventPipeline(t, registries) + for _, test := range []struct { + name string + response string + reasonCode string + validatorName string + }{ + {name: "shape", response: `{"events":[{"name":"","kind":"discovered","source_refs":[{"start_segment":1,"end_segment":1}]}]}`, reasonCode: itemeventshape.ReasonCode, validatorName: itemeventshape.Key}, + {name: "source references", response: `{"events":[{"name":"Ancient Coin","kind":"discovered","source_refs":[{"start_segment":99,"end_segment":99}]}]}`, reasonCode: itemeventrefs.ReasonCode, validatorName: itemeventrefs.Key}, + } { + t.Run(test.name, func(t *testing.T) { + client := &itemEventProductionLLMClient{responses: [][]byte{[]byte(test.response), []byte(test.response), []byte(test.response)}} + output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: readItemEventFixture(t)}) + if err != nil { + t.Fatalf("Run() error = %v; want rejected candidate outcome", err) + } + if len(output.NormalizeOutputs) != 0 || len(output.Rejected) != 1 || len(client.requests) != 3 { + t.Fatalf("output=%#v rejected=%#v requests=%d", output.NormalizeOutputs, output.Rejected, len(client.requests)) + } + rejection := output.Rejected[0] + if rejection.ReasonCode != test.reasonCode || rejection.ValidatorName != test.validatorName || rejection.AttemptCount != 3 { + t.Fatalf("rejection = %#v", rejection) + } + }) + } +} + +func resolveItemEventPipeline(t *testing.T, registries pipeline.Registries) pipeline.ResolvedPipeline { + t.Helper() + configValue := config.Default() + configValue.Pipelines["dnd-item-events-fixture"] = pipeline.PipelineProfile{ + Input: pipeline.Binding(transcript.Key), + Chunk: pipeline.ModuleBinding{Module: pipeline.DefaultChunkModule, Options: map[string]any{"max_units": 100}}, + Artifacts: map[string]pipeline.ArtifactLaneProfile{ + "item-events": { + Extract: pipeline.ModuleBinding{Module: itemeventextract.Key, Retries: 2}, + Normalize: pipeline.Binding(itemeventnormalize.Key), + }, + }, + } + effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-item-events-fixture", Catalog: moduleCatalog(registries)}) + if err != nil { + t.Fatalf("Resolve() error = %v", err) + } + return effective.ResolvedPipeline +} + +func readItemEventFixture(t *testing.T) []byte { + t.Helper() + raw, err := os.ReadFile("testdata/seriatim_item_events_session.json") + if err != nil { + t.Fatal(err) + } + return raw +} + +func hasItemEventWarning(warnings []contracts.Warning, reasonCode string) bool { + for _, warning := range warnings { + if warning.ReasonCode == reasonCode && strings.HasPrefix(warning.Scope, "events[") { + return true + } + } + return false +} + +type itemEventProductionLLMClient struct { + responses [][]byte + requests []contracts.StructuredCompletionRequest +} + +func (client *itemEventProductionLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { + if err := ctx.Err(); err != nil { + return contracts.StructuredCompletionResponse{}, err + } + if request.PromptID != itemeventextract.PromptID { + return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", request.PromptID) + } + index := len(client.requests) + if index >= len(client.responses) { + return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing item event response %d", index) + } + content := append([]byte(nil), client.responses[index]...) + if err := json.Unmarshal(content, out); err != nil { + return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate item event structured target: %w", err) + } + client.requests = append(client.requests, request) + return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "item-events-fake"}, nil +} + +var _ contracts.StructuredLLMClient = (*itemEventProductionLLMClient)(nil) diff --git a/internal/modules/integration/testdata/seriatim_item_events_session.json b/internal/modules/integration/testdata/seriatim_item_events_session.json new file mode 100644 index 0000000..3da707c --- /dev/null +++ b/internal/modules/integration/testdata/seriatim_item_events_session.json @@ -0,0 +1,12 @@ +{ + "metadata": { + "id": "item-events-session", + "title": "Synthetic D&D item-event session" + }, + "segments": [ + {"id": 1, "start": 0, "end": 4, "speaker": "DM", "text": "An ancient coin rests in the chest."}, + {"id": 2, "start": 4, "end": 8, "speaker": "DM", "text": "The party acquires 25 gold pieces."}, + {"id": 3, "start": 8, "end": 12, "speaker": "DM", "text": "Aria gives Borin a healing potion."}, + {"id": 4, "start": 12, "end": 16, "speaker": "DM", "text": "The party continues down the road."} + ] +}