eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Publish evidence context in JSON output bundles

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-27 18:23:48 +00:00
parent e61e522662
commit 256cc98ddb
6 changed files with 276 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
internal/modules/integration/dnd_npc_grounded_test.go
View File

@@ -7,6 +7,7 @@ import (
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"sync"
"testing"
@@ -14,6 +15,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/evidencecontext"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
@@ -254,6 +256,65 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
assertCurrentEvidence(t, combatValue.CombatTurns[0].SourceRefs)
}
func TestProductionDNDOutputPublishesSelectedEvidenceContext(t *testing.T) {
registries := productionNPCRegistries(t)
configValue := loadGroundedPipelineConfig(t)
profile := configValue.Pipelines["dnd-npc-grounded"]
profile.Output.Options = map[string]any{"evidence_context": map[string]any{
"enabled": true, "window_units": 0, "lanes": []any{"npcs", "spells", "combat"},
}}
configValue.Pipelines["dnd-npc-grounded"] = profile
raw := strings.NewReplacer(
`"id": 1`, `"id": 10`,
`"id": 2`, `"id": 30`,
`"id": 3`, `"id": 20`,
`"id": 4`, `"id": 50`,
`"id": 5`, `"id": 40`,
).Replace(string(readNPCFixture(t)))
output := runGroundedPipelineWithRaw(t, configValue, registries, &groundedDNDLLMClient{firstUnitID: 10, thirdUnitID: 20}, nil, []byte(raw))
value, err := evidencecontext.New().Decode(outputFileContent(t, output.OutputFiles, "evidence-context.json"))
if err != nil {
t.Fatalf("Decode(evidence context) error = %v", err)
}
if !reflect.DeepEqual(value.SelectedLanes, []string{"combat", "npcs", "spells"}) {
t.Fatalf("selected lanes = %#v, want configured production lanes without scene descriptions", value.SelectedLanes)
}
if len(value.Contexts) != 2 || len(value.Contexts[0].Units) != 1 || len(value.Contexts[1].Units) != 1 || value.Contexts[0].Units[0].ID != 10 || value.Contexts[1].Units[0].ID != 20 {
t.Fatalf("evidence contexts = %#v, want source-position union with non-monotonic unit IDs", value.Contexts)
}
firstRefs := value.Contexts[0].EvidenceRefs
if len(firstRefs) != 3 || firstRefs[0].LaneID != "combat" || firstRefs[1].LaneID != "npcs" || firstRefs[2].LaneID != "spells" {
t.Fatalf("first context evidence = %#v, want overlapping selected lane references", firstRefs)
}
for _, context := range value.Contexts {
for _, reference := range context.EvidenceRefs {
if reference.LaneID == "scene-descriptions" {
t.Fatalf("evidence refs = %#v, want scene descriptions excluded by allowlist", value.Contexts)
}
}
}
}
func TestProductionDNDOutputCanExplicitlySelectSceneDescriptionEvidence(t *testing.T) {
registries := productionNPCRegistries(t)
configValue := loadGroundedPipelineConfig(t)
profile := configValue.Pipelines["dnd-npc-grounded"]
profile.Output.Options = map[string]any{"evidence_context": map[string]any{
"enabled": true, "lanes": []any{"scene-descriptions"},
}}
configValue.Pipelines["dnd-npc-grounded"] = profile
output := runGroundedPipeline(t, configValue, registries, &groundedDNDLLMClient{}, nil)
value, err := evidencecontext.New().Decode(outputFileContent(t, output.OutputFiles, "evidence-context.json"))
if err != nil {
t.Fatalf("Decode(evidence context) error = %v", err)
}
if !reflect.DeepEqual(value.SelectedLanes, []string{"scene-descriptions"}) || len(value.Contexts) == 0 || len(value.Contexts[0].EvidenceRefs) == 0 || value.Contexts[0].EvidenceRefs[0].LaneID != "scene-descriptions" {
t.Fatalf("evidence context = %#v, want explicitly selected scene-description evidence", value)
}
}
func TestGroundedPipelineSkipsCombatForExactNarrativeScene(t *testing.T) {
registries := productionNPCRegistries(t)
configValue := loadGroundedPipelineConfig(t)
@@ -338,6 +399,10 @@ func configureGroundedCampaignReferences(t *testing.T, cfg *config.Config) map[s
}
func runGroundedPipeline(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient, checkpoint pipeline.CheckpointLoader) pipeline.RunOutput {
return runGroundedPipelineWithRaw(t, configValue, registries, client, checkpoint, readNPCFixture(t))
}
func runGroundedPipelineWithRaw(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient, checkpoint pipeline.CheckpointLoader, raw []byte) pipeline.RunOutput {
t.Helper()
catalog := moduleCatalog(registries)
effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-grounded", Catalog: catalog})
@@ -354,7 +419,7 @@ func runGroundedPipeline(t *testing.T, configValue config.Config, registries pip
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readNPCFixture(t),
RawInput: append([]byte(nil), raw...),
ExtractWorkers: 1,
Checkpoint: checkpoint,
})
@@ -364,6 +429,17 @@ func runGroundedPipeline(t *testing.T, configValue config.Config, registries pip
return output
}
func outputFileContent(t *testing.T, files []contracts.OutputFile, name string) []byte {
t.Helper()
for _, file := range files {
if file.Name == name {
return append([]byte(nil), file.Bytes...)
}
}
t.Fatalf("output files = %#v, missing %q", files, name)
return nil
}
func normalizedCombatOutput(t *testing.T, output pipeline.RunOutput) dnd.CombatTurnList {
t.Helper()
for _, serialized := range output.NormalizeOutputs {
@@ -436,10 +512,12 @@ func (loader *generatedReferenceCheckpointLoader) extractDependencies(laneID str
}
type groundedDNDLLMClient struct {
mu sync.Mutex
requests []contracts.StructuredCompletionRequest
sceneKind dnd.SceneKind
sceneTitle string
mu sync.Mutex
requests []contracts.StructuredCompletionRequest
sceneKind dnd.SceneKind
sceneTitle string
firstUnitID int
thirdUnitID int
}
func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
@@ -451,14 +529,22 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ
client.mu.Unlock()
var payload any
firstUnitID := client.firstUnitID
if firstUnitID == 0 {
firstUnitID = 1
}
thirdUnitID := client.thirdUnitID
if thirdUnitID == 0 {
thirdUnitID = 3
}
switch request.PromptID {
case npcs.PromptID:
payload = map[string]any{"npcs": []any{
map[string]any{
"name": "Mira Thorn", "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}},
"name": "Mira Thorn", "source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}},
},
map[string]any{
"name": "Hooded Guard", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}},
"name": "Hooded Guard", "source_refs": []any{map[string]int{"start_unit_id": thirdUnitID, "end_unit_id": thirdUnitID}},
},
}}
case npcnormalize.PromptID:
@@ -476,13 +562,13 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ
case spells.PromptID:
payload = map[string]any{"spell_casts": []any{map[string]any{
"caster": "Mira Thorn", "spell": "Cure Wounds",
"source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}},
"source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}},
}}}
case combatextract.PromptID:
payload = map[string]any{"combat_turns": []any{map[string]any{
"actor": "Mira Thorn",
"turn_kind": "turn",
"source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}},
"source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}},
}}}
default:
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected grounded prompt %q", request.PromptID)