eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Gate combat extraction on scene descriptions

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-25 19:23:19 +00:00
parent 7e35915b3e
commit 989f2c220b
8 changed files with 433 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
examples/dnd-complete.config.yml
View File

@@ -52,6 +52,10 @@ pipelines:
artifact:
step: describe-session
lane: npcs
scene_descriptions:
artifact:
step: describe-session
lane: scene-descriptions
artifacts:
spells:
extract:

48
internal/cli/dnd_combat_contract_test.go
View File

@@ -48,8 +48,12 @@ func TestProductionCombatConfigurationResolvesTypedLane(t *testing.T) {
if !ok || codecSpec.Schema.ID != "notarius.dnd.combat_turns" || codecSpec.Schema.Version != "v1" {
t.Fatalf("combat codec spec = %#v, want compatible durable schema", codecSpec)
}
if !hasReferenceSlot(extractSpec.ReferenceSlots, "npcs") || !hasReferenceSlot(normalizeSpec.ReferenceSlots, "npcs") {
t.Fatalf("combat reference slots = %#v / %#v, want stage-local NPC slots", extractSpec.ReferenceSlots, normalizeSpec.ReferenceSlots)
if !hasReferenceSlot(extractSpec.ReferenceSlots, "npcs") || !hasReferenceSlot(extractSpec.ReferenceSlots, "scene_descriptions") || !hasReferenceSlot(normalizeSpec.ReferenceSlots, "npcs") {
t.Fatalf("combat reference slots = %#v / %#v, want extraction scene and NPC slots plus normalization NPC slot", extractSpec.ReferenceSlots, normalizeSpec.ReferenceSlots)
}
sceneSlot := referenceSlot(extractSpec.ReferenceSlots, "scene_descriptions")
if !sceneSlot.Required || !reflect.DeepEqual(sceneSlot.AcceptedMediaTypes, []string{"application/json"}) || !reflect.DeepEqual(sceneSlot.AcceptedArtifactKinds, []contracts.ArtifactKind{dnd.SceneDescriptionListKind}) || sceneSlot.MaxBytes != 1048576 {
t.Fatalf("scene description slot = %#v, want required approved scene artifact", sceneSlot)
}
wantExtractChain := []pipeline.ModuleBinding{
@@ -89,8 +93,19 @@ func TestProductionCombatConfigurationResolvesTypedLane(t *testing.T) {
t.Fatalf("Resolve(bound references) error = %v, want nil", err)
}
boundLane := bound.ResolvedPipeline.Steps[0].ArtifactLanes[0]
if len(boundLane.ExtractReferences.Bindings) != 1 || len(boundLane.NormalizeReferences.Bindings) != 1 || boundLane.ExtractReferences.Bindings[0].SlotName != "npcs" || boundLane.NormalizeReferences.Bindings[0].SlotName != "npcs" {
t.Fatalf("bound combat references = %#v / %#v, want one independent NPC binding per stage", boundLane.ExtractReferences, boundLane.NormalizeReferences)
if len(boundLane.ExtractReferences.Bindings) != 2 || len(boundLane.NormalizeReferences.Bindings) != 1 || !hasReferenceBinding(boundLane.ExtractReferences.Bindings, "npcs") || !hasReferenceBinding(boundLane.ExtractReferences.Bindings, "scene_descriptions") || !hasReferenceBinding(boundLane.NormalizeReferences.Bindings, "npcs") {
t.Fatalf("bound combat references = %#v / %#v, want extraction scene and NPC bindings plus normalization NPC binding", boundLane.ExtractReferences, boundLane.NormalizeReferences)
}
}
func TestProductionCombatConfigurationRequiresSceneDescriptions(t *testing.T) {
components := productionTestComponents(t)
cfg := productionCombatContractConfig()
profile := cfg.Pipelines["dnd-combat"]
profile.References = nil
cfg.Pipelines["dnd-combat"] = profile
if _, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-combat", Catalog: catalogFromRegistries(components.registries)}); err == nil || !strings.Contains(err.Error(), "scene_descriptions") || !strings.Contains(err.Error(), "required") {
t.Fatalf("Resolve() error = %v, want required scene reference failure", err)
}
}
@@ -151,9 +166,10 @@ func TestProductionCombatConfigurationResolvesTypedUnconditionalValidators(t *te
func productionCombatContractConfig() config.Config {
cfg := config.Default()
cfg.Pipelines["dnd-combat"] = pipeline.PipelineProfile{
ID: "dnd-combat",
Input: pipeline.Binding("seriatim"),
Chunk: pipeline.Binding(pipeline.DefaultChunkModule),
ID: "dnd-combat",
Input: pipeline.Binding("seriatim"),
Chunk: pipeline.Binding(pipeline.DefaultChunkModule),
References: map[string]pipeline.ReferenceSource{"scene_descriptions": pipeline.ExternalReference("scenes.json")},
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"combat": {
Extract: pipeline.ModuleBinding{Module: combatextract.Key, Retries: 2},
@@ -172,3 +188,21 @@ func hasReferenceSlot(slots []contracts.ReferenceSlot, name string) bool {
}
return false
}
func hasReferenceBinding(bindings []pipeline.ReferenceBinding, name string) bool {
for _, binding := range bindings {
if binding.SlotName == name {
return true
}
}
return false
}
func referenceSlot(slots []contracts.ReferenceSlot, name string) contracts.ReferenceSlot {
for _, slot := range slots {
if slot.Name == name {
return slot
}
}
return contracts.ReferenceSlot{}
}

58
internal/modules/dnd/extract/combatturns/extractor.go
View File

@@ -9,17 +9,21 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
npcregistry "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/registry"
sceneregistry "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/scenedescriptions/registry"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
)
const (
Key = "dnd/combat-turns"
mappingPolicy = "dnd.combat_turns.extract_mapping.v2"
Key = "dnd/combat-turns"
mappingPolicy = "dnd.combat_turns.extract_mapping.v2"
sceneGatePolicy = "dnd.combat_turns.scene_gate.v1"
)
const (
NPCRegistryReferenceSlot = npcregistry.ReferenceSlot
NPCRegistryMaxBytes = npcregistry.MaxBytes
NPCRegistryReferenceSlot = npcregistry.ReferenceSlot
NPCRegistryMaxBytes = npcregistry.MaxBytes
SceneDescriptionReferenceSlot = sceneregistry.ReferenceSlot
SceneDescriptionMaxBytes = sceneregistry.MaxBytes
)
var requiredCapabilities = []string{
@@ -47,6 +51,14 @@ func referenceSlots() []contracts.ReferenceSlot {
AcceptedArtifactKinds: []contracts.ArtifactKind{dnd.NPCListKind},
MaxBytes: NPCRegistryMaxBytes,
})
slots = append(slots, contracts.ReferenceSlot{
Name: SceneDescriptionReferenceSlot,
Description: "Required scene descriptions that gate combat eligibility and are not combat evidence.",
Required: true,
AcceptedMediaTypes: []string{"application/json"},
AcceptedArtifactKinds: []contracts.ArtifactKind{dnd.SceneDescriptionListKind},
MaxBytes: SceneDescriptionMaxBytes,
})
sort.Slice(slots, func(i, j int) bool { return slots[i].Name < slots[j].Name })
return slots
}
@@ -60,6 +72,7 @@ type Options struct{}
type Extractor struct {
llm contracts.StructuredLLMClient
npcResolver *npcregistry.Resolver
sceneResolver *sceneregistry.Resolver
promptSHA string
responseSchemaSHA string
}
@@ -79,6 +92,10 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr
if err != nil {
return nil, extractorErrorf("prepare NPC registry prompt input: %w", err)
}
sceneResolver, err := sceneregistry.NewResolver(referenceSet)
if err != nil {
return nil, extractorErrorf("prepare scene eligibility: %w", err)
}
promptSHA, err := scriptoriumPromptMetadata()
if err != nil {
return nil, extractorErrorf("load prompt metadata: %w", err)
@@ -90,6 +107,7 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr
return &Extractor{
llm: llmClient,
npcResolver: npcResolver,
sceneResolver: sceneResolver,
promptSHA: promptSHA,
responseSchemaSHA: responseSchema.SHA256,
}, nil
@@ -108,6 +126,7 @@ func (e *Extractor) ManifestMetadata() map[string]any {
"prompt_version": SchemaVersion,
"prompt_sha256": e.promptSHA,
"mapping_policy": mappingPolicy,
"scene_gate_policy": sceneGatePolicy,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
@@ -119,6 +138,11 @@ func (e *Extractor) ManifestMetadata() map[string]any {
metadata["npc_registry_digest"] = seeded.Digest()
metadata["npc_count"] = seeded.Count()
}
seededScenes := e.sceneResolver.Seeded()
if seededScenes.Bound() {
metadata["scene_eligibility_digest"] = seededScenes.EligibilityDigest()
metadata["scene_description_count"] = seededScenes.Count()
}
return metadata
}
@@ -130,9 +154,12 @@ func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
{Name: "prompt", Value: e.promptSHA},
{Name: "response_schema", Value: e.responseSchemaSHA},
{Name: "mapping_policy", Value: mappingPolicy},
{Name: "scene_gate_policy", Value: sceneGatePolicy},
}
seeded := e.npcResolver.Seeded()
fingerprints = append(fingerprints, pipeline.CheckpointFingerprint{Name: "npc_registry", Value: seeded.ProjectionDigest()})
seededScenes := e.sceneResolver.Seeded()
fingerprints = append(fingerprints, pipeline.CheckpointFingerprint{Name: "scene_eligibility", Value: seededScenes.EligibilityDigest()})
return fingerprints
}
@@ -147,6 +174,29 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
if err != nil {
return contracts.TypedExtractionResult[dnd.CombatTurnList]{}, extractorErrorf("%w", err)
}
if _, ok := req.References.Slots[SceneDescriptionReferenceSlot]; !ok {
return contracts.TypedExtractionResult[dnd.CombatTurnList]{}, extractorErrorf("required scene descriptions reference is missing")
}
scenes, err := e.sceneResolver.Resolve(req.References)
if err != nil {
return contracts.TypedExtractionResult[dnd.CombatTurnList]{}, extractorErrorf("resolve scene eligibility: %w", err)
}
match := scenes.Match(req.Chunk)
switch match.State {
case sceneregistry.MatchExact:
if match.Kind != dnd.SceneKindCombat {
return contracts.TypedExtractionResult[dnd.CombatTurnList]{Value: dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{}}}, nil
}
case sceneregistry.MatchMissing, sceneregistry.MatchMismatched:
return contracts.TypedExtractionResult[dnd.CombatTurnList]{
Value: dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{}},
Warnings: []contracts.Warning{{
Scope: SceneDescriptionReferenceSlot,
ReasonCode: "scene_classification_unavailable",
Message: "No exact scene classification was available; combat extraction was skipped.",
}},
}, nil
}
order := shared.NewSourceRefOrder(req.Source)
npcRegistry, err := e.npcResolver.Resolve(req.References)
if err != nil {

253
internal/modules/dnd/extract/combatturns/extractor_test.go
View File

@@ -13,6 +13,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcs"
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
)
@@ -34,7 +35,7 @@ func TestExtractMapsAndOrdersCombatTurnsBySourcePosition(t *testing.T) {
},
}}}
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest(t))
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
@@ -57,7 +58,7 @@ func TestExtractMapsAndOrdersCombatTurnsBySourcePosition(t *testing.T) {
t.Fatalf("LLM request identity = %#v", request)
}
transcript := request.Inputs["transcript"]
if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:chunk" || transcript.OriginURI != "file:///session-alpha.json" || !reflect.DeepEqual(transcript.Content, extractionRequest().Chunk.Content) {
if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:chunk" || transcript.OriginURI != "file:///session-alpha.json" || !reflect.DeepEqual(transcript.Content, extractionRequest(t).Chunk.Content) {
t.Fatalf("transcript input = %#v, want chunk-scoped source input", transcript)
}
}
@@ -69,7 +70,7 @@ func TestExtractPreservesInvalidCandidatesForValidators(t *testing.T) {
SourceRefs: []combatSourceRefResponse{{StartUnitID: 99, EndUnitID: 0}},
},
}}}
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest(t))
if err != nil {
t.Fatalf("Extract() error = %v, want nil for candidate values", err)
}
@@ -93,10 +94,11 @@ func TestExtractUsesDocumentOrderForReferencesAndTurns(t *testing.T) {
}},
{Actor: "Second", TurnKind: "other", SourceRefs: []combatSourceRefResponse{{StartUnitID: 30, EndUnitID: 30}}},
}}}
req := extractionRequest()
req := extractionRequest(t)
req.Source.Units = []source.SourceUnit{{ID: 30}, {ID: 10}}
req.Chunk.Units = append([]source.SourceUnit(nil), req.Source.Units...)
req.Chunk.Ref = source.SourceRef{SourceID: req.Source.ID, StartUnitID: 30, EndUnitID: 10}
req.References = sceneReferences(t, req.Chunk, dnd.SceneKindCombat)
result, err := newExtractor(t, client).Extract(context.Background(), req)
if err != nil {
@@ -127,7 +129,10 @@ func TestExtractPassesReferencesAndNPCGroundingWithoutUsingItAsEvidence(t *testi
"glossary": {Slot: contracts.ReferenceSlot{Name: "glossary"}, Items: []contracts.ReferenceItem{{SlotName: "glossary", Content: []byte("ogre: a large foe")}}},
NPCRegistryReferenceSlot: {Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot}, Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: "application/json", Content: npcRegistryJSON(t)}}},
}}
req := extractionRequest()
req := extractionRequest(t)
for name, slot := range req.References.Slots {
references.Slots[name] = slot
}
req.References = references
if _, err := newExtractor(t, client, references).Extract(context.Background(), req); err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
@@ -143,14 +148,20 @@ func TestExtractPassesReferencesAndNPCGroundingWithoutUsingItAsEvidence(t *testi
if strings.Contains(string(inputs["transcript"].Content), "Aria: cleric") {
t.Fatal("transcript input contains reference content")
}
if _, ok := inputs[SceneDescriptionReferenceSlot]; ok {
t.Fatalf("prompt inputs include scene descriptions: %#v", inputs)
}
metadata := newExtractor(t, &fakeCombatTurnsLLMClient{}, references).ManifestMetadata()
if metadata["npc_count"] != 1 || !strings.HasPrefix(metadata["npc_registry_digest"].(string), "sha256:") {
t.Fatalf("bound registry metadata = %#v, want digest and count", metadata)
}
if metadata["scene_description_count"] != 1 || !strings.HasPrefix(metadata["scene_eligibility_digest"].(string), "sha256:") {
t.Fatalf("bound scene metadata = %#v, want digest and count", metadata)
}
fingerprints := newExtractor(t, &fakeCombatTurnsLLMClient{}, references).CheckpointFingerprints()
if len(fingerprints) != 4 || fingerprints[3].Name != "npc_registry" {
t.Fatalf("bound fingerprints = %#v, want local identities plus NPC registry", fingerprints)
if len(fingerprints) != 6 || fingerprints[3].Name != "scene_gate_policy" || fingerprints[4].Name != "npc_registry" || fingerprints[5].Name != "scene_eligibility" {
t.Fatalf("bound fingerprints = %#v, want local identities plus scene gate and NPC registry", fingerprints)
}
encoded, err := json.Marshal(metadata)
if err != nil {
@@ -164,7 +175,7 @@ func TestExtractPassesReferencesAndNPCGroundingWithoutUsingItAsEvidence(t *testi
func TestExtractUnboundRegistryUsesExactEmptyPromptAndOmitsIdentity(t *testing.T) {
client := &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{}}}
extractor := newExtractor(t, client)
if _, err := extractor.Extract(context.Background(), extractionRequest()); err != nil {
if _, err := extractor.Extract(context.Background(), extractionRequest(t)); err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
input := client.requests[0].Inputs[NPCRegistryReferenceSlot]
@@ -175,7 +186,7 @@ func TestExtractUnboundRegistryUsesExactEmptyPromptAndOmitsIdentity(t *testing.T
if _, ok := metadata["npc_registry_digest"]; ok {
t.Fatalf("unbound metadata has registry digest: %#v", metadata)
}
if fingerprints := extractor.CheckpointFingerprints(); len(fingerprints) != 4 || fingerprints[3].Name != "npc_registry" || fingerprints[3].Value != input.Digest {
if fingerprints := extractor.CheckpointFingerprints(); len(fingerprints) != 6 || fingerprints[4].Name != "npc_registry" || fingerprints[4].Value != input.Digest || fingerprints[5].Name != "scene_eligibility" || fingerprints[5].Value == "" {
t.Fatalf("unbound fingerprints = %#v, want empty-projection identity", fingerprints)
}
}
@@ -184,13 +195,11 @@ func TestExtractorResolvesOperationNPCOverrideWithoutSingletonMetadata(t *testin
content := npcRegistryJSON(t)
client := &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{}}}
extractor := newExtractor(t, client)
request := extractionRequest()
request.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
NPCRegistryReferenceSlot: {
Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot},
Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: npccodec.MediaType, Content: append([]byte(nil), content...), Origin: contracts.ReferenceOrigin{Type: "generated"}}},
},
}}
request := extractionRequest(t)
request.References.Slots[NPCRegistryReferenceSlot] = contracts.ResolvedReferenceSlot{
Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot},
Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: npccodec.MediaType, Content: append([]byte(nil), content...), Origin: contracts.ReferenceOrigin{Type: "generated"}}},
}
if _, err := extractor.Extract(context.Background(), request); err != nil {
t.Fatalf("Extract() error = %v", err)
}
@@ -198,11 +207,151 @@ func TestExtractorResolvesOperationNPCOverrideWithoutSingletonMetadata(t *testin
if input.Digest == "" || string(input.Content) != `{"npcs":[{"name":"Mira Thorn"}]}` || input.OriginURI != "" {
t.Fatalf("operation NPC input = %#v, want generated canonical grounding without provenance", input)
}
if metadata := extractor.ManifestMetadata(); metadata["npc_registry_digest"] != nil || metadata["npc_count"] != nil {
if metadata := extractor.ManifestMetadata(); metadata["npc_registry_digest"] != nil || metadata["npc_count"] != nil || metadata["scene_eligibility_digest"] != nil || metadata["scene_description_count"] != nil {
t.Fatalf("singleton metadata = %#v, want no operation-varying NPC identity", metadata)
}
}
func TestExtractAppliesSceneEligibilityBeforePromptConstruction(t *testing.T) {
for _, test := range []struct {
name string
scenes []dnd.SceneDescription
wantWarning string
wantError string
wantCalls int
wantEmptyList bool
}{
{
name: "exact combat",
scenes: []dnd.SceneDescription{sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindCombat)},
wantCalls: 1,
},
{
name: "exact narrative",
scenes: []dnd.SceneDescription{sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindNarrative)},
wantEmptyList: true,
},
{
name: "exact recap",
scenes: []dnd.SceneDescription{sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindRecap)},
wantEmptyList: true,
},
{
name: "exact meta",
scenes: []dnd.SceneDescription{sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindMeta)},
wantEmptyList: true,
},
{
name: "missing ID",
scenes: []dnd.SceneDescription{func() dnd.SceneDescription {
value := sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindCombat)
value.ID = "other-chunk"
return value
}()},
wantWarning: "scene_classification_unavailable",
wantEmptyList: true,
},
{
name: "source mismatch",
scenes: []dnd.SceneDescription{func() dnd.SceneDescription {
value := sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindCombat)
value.SourceRef.SourceID = "other-source"
return value
}()},
wantWarning: "scene_classification_unavailable",
wantEmptyList: true,
},
{
name: "start mismatch",
scenes: []dnd.SceneDescription{func() dnd.SceneDescription {
value := sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindCombat)
value.SourceRef.StartUnitID = 2
return value
}()},
wantWarning: "scene_classification_unavailable",
wantEmptyList: true,
},
{
name: "end mismatch",
scenes: []dnd.SceneDescription{func() dnd.SceneDescription {
value := sceneDescription(extractionRequest(t).Chunk, dnd.SceneKindCombat)
value.SourceRef.EndUnitID = 10
return value
}()},
wantWarning: "scene_classification_unavailable",
wantEmptyList: true,
},
{
name: "missing required reference",
wantError: "required scene descriptions reference is missing",
},
} {
t.Run(test.name, func(t *testing.T) {
client := &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{}}}
req := extractionRequest(t)
if test.wantError != "" {
req.References = contracts.ReferenceSet{}
} else {
req.References = sceneListReferences(t, test.scenes...)
}
result, err := newExtractor(t, client).Extract(context.Background(), req)
if test.wantError != "" {
if err == nil || !strings.Contains(err.Error(), test.wantError) {
t.Fatalf("Extract() error = %v, want %q", err, test.wantError)
}
} else if err != nil {
t.Fatalf("Extract() error = %v", err)
}
if len(client.requests) != test.wantCalls {
t.Fatalf("LLM calls = %d, want %d", len(client.requests), test.wantCalls)
}
if test.wantEmptyList && result.Value.CombatTurns == nil {
t.Fatal("CombatTurns = nil, want accepted non-nil empty list")
}
if test.wantWarning != "" {
if len(result.Warnings) != 1 || result.Warnings[0].Scope != SceneDescriptionReferenceSlot || result.Warnings[0].ReasonCode != test.wantWarning {
t.Fatalf("warnings = %#v", result.Warnings)
}
} else if len(result.Warnings) != 0 {
t.Fatalf("warnings = %#v, want none", result.Warnings)
}
})
}
}
func TestSceneEligibilityMetadataAndFingerprintsTrackGatingValues(t *testing.T) {
chunk := extractionRequest(t).Chunk
base := sceneDescription(chunk, dnd.SceneKindCombat)
baseExtractor := newExtractor(t, &fakeCombatTurnsLLMClient{}, sceneListReferences(t, base))
metadata := baseExtractor.ManifestMetadata()
if metadata["scene_gate_policy"] != sceneGatePolicy || metadata["scene_description_count"] != 1 {
t.Fatalf("metadata = %#v, want scene gate policy and count", metadata)
}
baseDigest := fingerprintValue(t, baseExtractor.CheckpointFingerprints(), "scene_eligibility")
proseChanged := base
proseChanged.Title = "Different title"
proseChanged.Summary = "Different summary."
if got := fingerprintValue(t, newExtractor(t, &fakeCombatTurnsLLMClient{}, sceneListReferences(t, proseChanged)).CheckpointFingerprints(), "scene_eligibility"); got != baseDigest {
t.Fatalf("prose-only eligibility digest = %q, want %q", got, baseDigest)
}
for _, test := range []struct {
name string
change func(*dnd.SceneDescription)
}{
{"kind", func(value *dnd.SceneDescription) { value.Kind = dnd.SceneKindNarrative }},
{"range", func(value *dnd.SceneDescription) { value.SourceRef.StartUnitID = 2 }},
} {
t.Run(test.name, func(t *testing.T) {
changed := base
test.change(&changed)
if got := fingerprintValue(t, newExtractor(t, &fakeCombatTurnsLLMClient{}, sceneListReferences(t, changed)).CheckpointFingerprints(), "scene_eligibility"); got == baseDigest {
t.Fatal("eligibility digest did not change")
}
})
}
}
func TestNewRejectsMalformedNPCRegistryBeforeLLMCallWithoutContent(t *testing.T) {
client := &fakeCombatTurnsLLMClient{}
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
@@ -220,8 +369,25 @@ func TestNewRejectsMalformedNPCRegistryBeforeLLMCallWithoutContent(t *testing.T)
}
}
func TestNewRejectsInvalidSceneReferenceBeforeLLMCallWithoutContent(t *testing.T) {
client := &fakeCombatTurnsLLMClient{}
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
SceneDescriptionReferenceSlot: {
Slot: contracts.ReferenceSlot{Name: SceneDescriptionReferenceSlot},
Items: []contracts.ReferenceItem{{SlotName: SceneDescriptionReferenceSlot, MediaType: scenecodec.MediaType, Content: []byte(`{"scenes":[]}`)}},
},
}}
_, err := New(client, Options{}, references)
if err == nil || !strings.Contains(err.Error(), "prepare scene eligibility") || strings.Contains(err.Error(), `{"scenes":[]}`) {
t.Fatalf("New() error = %v, want bounded content-free scene failure", err)
}
if len(client.requests) != 0 {
t.Fatalf("LLM calls = %d, want none during failed construction", len(client.requests))
}
}
func TestExtractRejectsInvalidRequestsAndWrapsProviderFailures(t *testing.T) {
validReq := extractionRequest()
validReq := extractionRequest(t)
validExtractor := newExtractor(t, &fakeCombatTurnsLLMClient{response: extractionResponse{CombatTurns: []combatTurnResponse{}}})
var nilExtractor *Extractor
tests := []struct {
@@ -243,7 +409,7 @@ func TestExtractRejectsInvalidRequestsAndWrapsProviderFailures(t *testing.T) {
}
})
}
_, err := newExtractor(t, &fakeCombatTurnsLLMClient{err: errors.New("provider unavailable")}).Extract(context.Background(), extractionRequest())
_, err := newExtractor(t, &fakeCombatTurnsLLMClient{err: errors.New("provider unavailable")}).Extract(context.Background(), extractionRequest(t))
if err == nil || !strings.Contains(err.Error(), "dnd combat turns") || !strings.Contains(err.Error(), "provider unavailable") {
t.Fatalf("provider error = %v, want contextual wrapped error", err)
}
@@ -267,7 +433,7 @@ func TestExtractorManifestMetadataAndFingerprints(t *testing.T) {
t.Fatalf("metadata[%q] = %#v, want digest", key, metadata[key])
}
}
wantNames := map[string]struct{}{"prompt": {}, "response_schema": {}, "mapping_policy": {}, "npc_registry": {}}
wantNames := map[string]struct{}{"prompt": {}, "response_schema": {}, "mapping_policy": {}, "scene_gate_policy": {}, "npc_registry": {}, "scene_eligibility": {}}
for _, fingerprint := range extractor.CheckpointFingerprints() {
if _, ok := wantNames[fingerprint.Name]; !ok {
t.Fatalf("unexpected fingerprint = %#v", fingerprint)
@@ -304,7 +470,8 @@ func TestModuleSpecAndRegistration(t *testing.T) {
}
}
func extractionRequest() contracts.TypedExtractionRequest {
func extractionRequest(t *testing.T) contracts.TypedExtractionRequest {
t.Helper()
doc := combatSourceDocument()
chunk := &source.Chunk{
ID: "session-alpha:chunk:0",
@@ -315,13 +482,15 @@ func extractionRequest() contracts.TypedExtractionRequest {
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), doc.Units...),
}
return contracts.TypedExtractionRequest{
request := contracts.TypedExtractionRequest{
Source: doc,
Chunk: chunk,
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"),
SessionID: "session-123",
LLMProfile: "profile-combat",
}
request.References = sceneReferences(t, chunk, dnd.SceneKindCombat)
return request
}
func combatSourceDocument() *source.SourceDocument {
@@ -358,6 +527,46 @@ func npcRegistryJSON(t *testing.T) []byte {
return content
}
func sceneReferences(t *testing.T, chunk *source.Chunk, kind dnd.SceneKind) contracts.ReferenceSet {
t.Helper()
return sceneListReferences(t, sceneDescription(chunk, kind))
}
func sceneListReferences(t *testing.T, scenes ...dnd.SceneDescription) contracts.ReferenceSet {
t.Helper()
content, err := scenecodec.New().Encode(dnd.SceneDescriptionList{Scenes: scenes})
if err != nil {
t.Fatalf("encode scene descriptions: %v", err)
}
return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
SceneDescriptionReferenceSlot: {
Slot: contracts.ReferenceSlot{Name: SceneDescriptionReferenceSlot},
Items: []contracts.ReferenceItem{{SlotName: SceneDescriptionReferenceSlot, MediaType: scenecodec.MediaType, Content: content}},
},
}}
}
func sceneDescription(chunk *source.Chunk, kind dnd.SceneKind) dnd.SceneDescription {
return dnd.SceneDescription{
ID: chunk.ID,
SourceRef: chunk.Ref,
Kind: kind,
Title: "Combat eligibility",
Summary: "A valid scene description for combat extraction.",
}
}
func fingerprintValue(t *testing.T, fingerprints []pipeline.CheckpointFingerprint, name string) string {
t.Helper()
for _, fingerprint := range fingerprints {
if fingerprint.Name == name {
return fingerprint.Value
}
}
t.Fatalf("fingerprints = %#v, missing %q", fingerprints, name)
return ""
}
type fakeCombatTurnsLLMClient struct {
response extractionResponse
err error

41
internal/modules/integration/dnd_combat_runner_test.go
View File

@@ -18,6 +18,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcs"
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
combatextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/combatturns"
combatnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/combatturns"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
@@ -59,6 +60,7 @@ func TestProductionCombatPipelineRetriesMergesNormalizesAndWritesJSON(t *testing
}
fingerprints := prepared.CheckpointFingerprints()
assertCombatFingerprint(t, fingerprints, "extract:combat:"+combatextract.Key+":npc_registry")
assertCombatFingerprint(t, fingerprints, "extract:combat:"+combatextract.Key+":scene_eligibility")
assertCombatFingerprint(t, fingerprints, "normalize:combat:"+combatnormalize.Key+":npc_registry")
if len(fingerprints) == 0 {
t.Fatal("checkpoint fingerprints = empty, want production combat identities")
@@ -99,16 +101,16 @@ func TestProductionCombatPipelineRetriesMergesNormalizesAndWritesJSON(t *testing
if !hasCombatWarning(output.Warnings, combatnormalize.ReasonCodeActorCanonicalized) {
t.Fatalf("warnings = %#v, want registry normalization warning", output.Warnings)
}
if len(output.Manifest.References) != 2 {
t.Fatalf("manifest references = %#v, want separate extract and normalize provenance", output.Manifest.References)
if len(output.Manifest.References) != 3 {
t.Fatalf("manifest references = %#v, want scene eligibility plus extract and normalize NPC provenance", output.Manifest.References)
}
lane := output.Manifest.ArtifactLanes[0]
if lane.ID != "combat" || lane.Extractor != combatextract.Key || lane.Merger != pipeline.DefaultMergeModule || lane.Normalizer != combatnormalize.Key {
t.Fatalf("manifest lane = %#v, want complete combat composition", lane)
}
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
if !ok || extractorMetadata["npc_count"] != 2 || extractorMetadata["npc_registry_digest"] == "" {
t.Fatalf("extractor metadata = %#v, want registry digest and count", lane.Metadata)
if !ok || extractorMetadata["npc_count"] != 2 || extractorMetadata["npc_registry_digest"] == "" || extractorMetadata["scene_description_count"] != 3 || extractorMetadata["scene_eligibility_digest"] == "" || extractorMetadata["scene_gate_policy"] != "dnd.combat_turns.scene_gate.v1" {
t.Fatalf("extractor metadata = %#v, want registry and scene eligibility metadata", lane.Metadata)
}
normalizerMetadata, ok := lane.Metadata["normalizer"].(map[string]any)
if !ok || normalizerMetadata["npc_count"] != 2 || normalizerMetadata["normalization_policy"] != combatnormalize.NormalizationPolicy {
@@ -131,6 +133,13 @@ func TestProductionCombatPipelineAttributesExhaustedInvalidEnumsToShapeValidatio
configValue := combatOnlyConfig()
profile := configValue.Pipelines["dnd-combat-fixture"]
profile.Chunk.Options["max_units"] = 100
profile.References["scene_descriptions"] = pipeline.ExternalReference(combatSceneDescriptionPath(t, []dnd.SceneDescription{{
ID: "chunk-000001",
SourceRef: source.SourceRef{SourceID: "npc-session", StartUnitID: 1, EndUnitID: 5},
Kind: dnd.SceneKindCombat,
Title: "Complete combat encounter",
Summary: "The accepted chunk contains active combat.",
}}))
configValue.Pipelines["dnd-combat-fixture"] = profile
effective, err := configValue.Resolve(config.ResolveInput{
PipelineID: "dnd-combat-fixture",
@@ -144,7 +153,11 @@ func TestProductionCombatPipelineAttributesExhaustedInvalidEnumsToShapeValidatio
combatTestInvalidEnumResponse("invalid"),
combatTestInvalidEnumResponse("unknown"),
}}
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: client})
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, moduleCatalog(registries), pipeline.ReferenceMaterializationOptions{})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v", err)
}
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client})
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
@@ -267,11 +280,25 @@ func combatTestNPCPayload(t *testing.T) combatNPCPayload {
return combatNPCPayload{path: path}
}
func combatSceneDescriptionPath(t *testing.T, scenes []dnd.SceneDescription) string {
t.Helper()
content, err := scenecodec.New().Encode(dnd.SceneDescriptionList{Scenes: scenes})
if err != nil {
t.Fatalf("Encode(scene descriptions) error = %v", err)
}
path := filepath.Join(t.TempDir(), "scene-descriptions.json")
if err := os.WriteFile(path, content, 0o600); err != nil {
t.Fatal(err)
}
return path
}
func combatOnlyConfig() config.Config {
cfg := config.Default()
cfg.Pipelines["dnd-combat"] = pipeline.PipelineProfile{
Input: pipeline.Binding(transcript.Key),
Chunk: pipeline.ModuleBinding{Module: pipeline.DefaultChunkModule, Options: map[string]any{"max_units": 2}},
Input: pipeline.Binding(transcript.Key),
Chunk: pipeline.ModuleBinding{Module: pipeline.DefaultChunkModule, Options: map[string]any{"max_units": 2}},
References: map[string]pipeline.ReferenceSource{"scene_descriptions": pipeline.ExternalReference(repositoryPathForIntegration("internal", "modules", "integration", "testdata", "combat_scene_descriptions.json"))},
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"combat": {
Extract: pipeline.ModuleBinding{Module: combatextract.Key, Retries: 2},

34
internal/modules/integration/dnd_npc_grounded_test.go
View File

@@ -16,9 +16,11 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcs"
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
combatextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/combatturns"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcs"
sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
)
@@ -41,8 +43,9 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
t.Fatalf("Prepare() error = %v", err)
}
for name, value := range map[string]string{
"extract:npcs:dnd/npcs:mapping_policy": "dnd.npcs.extract_mapping.v2",
"extract:spells:dnd/spells:mapping_policy": "dnd.spells.extract_mapping.v2",
"extract:npcs:dnd/npcs:mapping_policy": "dnd.npcs.extract_mapping.v2",
"extract:spells:dnd/spells:mapping_policy": "dnd.spells.extract_mapping.v2",
"extract:combat:dnd/combat-turns:scene_gate_policy": "dnd.combat_turns.scene_gate.v1",
} {
assertFingerprintValue(t, prepared.CheckpointFingerprints(), name, value)
}
@@ -54,6 +57,7 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
"extract:spells:dnd/spells:npc_registry",
"extract:combat:dnd/combat-turns:prompt",
"extract:combat:dnd/combat-turns:response_schema",
"extract:combat:dnd/combat-turns:scene_eligibility",
"extract:combat:dnd/combat-turns:npc_registry",
"normalize:combat:dnd/combat-turns:npc_registry",
} {
@@ -67,19 +71,20 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 3 {
t.Fatalf("run outputs = %#v rejected = %#v, want NPC, spell, and combat outputs", output.NormalizeOutputs, output.Rejected)
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 4 {
t.Fatalf("run outputs = %#v rejected = %#v, want NPC, scene, spell, and combat outputs", output.NormalizeOutputs, output.Rejected)
}
wantSchemas := map[string]string{"npcs": npccodec.SchemaID, "spells": spellcodec.SchemaID, "combat": combatcodec.SchemaID}
wantSchemas := map[string]string{"npcs": npccodec.SchemaID, "scene-descriptions": scenecodec.SchemaID, "spells": spellcodec.SchemaID, "combat": combatcodec.SchemaID}
for _, serialized := range output.NormalizeOutputs {
if serialized.Artifact.Schema.ID != wantSchemas[serialized.LaneID] || serialized.Artifact.Schema.Version != "v1" {
t.Fatalf("%s artifact schema = %#v, want minimal v1 identity", serialized.LaneID, serialized.Artifact.Schema)
}
}
wantExtractorIdentity := map[string]struct{ promptID, schemaID string }{
"npcs": {npcs.PromptID, npcs.ResponseSchemaID},
"spells": {spells.PromptID, spells.ResponseSchemaID},
"combat": {combatextract.PromptID, combatextract.ResponseSchemaID},
"npcs": {npcs.PromptID, npcs.ResponseSchemaID},
"scene-descriptions": {sceneextract.PromptID, sceneextract.ResponseSchemaID},
"spells": {spells.PromptID, spells.ResponseSchemaID},
"combat": {combatextract.PromptID, combatextract.ResponseSchemaID},
}
for _, lane := range output.Manifest.ArtifactLanes {
want, ok := wantExtractorIdentity[lane.ID]
@@ -138,6 +143,15 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
if provenanceCount != 3 {
t.Fatalf("NPC generated provenance count = %d, want spell extract plus combat extract/normalize", provenanceCount)
}
sceneProvenanceCount := 0
for _, reference := range output.Manifest.References {
if reference.SlotName == "scene_descriptions" {
sceneProvenanceCount++
}
}
if sceneProvenanceCount != 1 {
t.Fatalf("scene generated provenance count = %d, want combat extractor handoff", sceneProvenanceCount)
}
manifestContent, err := json.Marshal(output.Manifest)
if err != nil {
t.Fatal(err)
@@ -150,7 +164,7 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
for _, lane := range output.Manifest.ArtifactLanes {
for _, component := range []string{"extractor", "normalizer"} {
metadata, ok := lane.Metadata[component].(map[string]any)
if ok && metadata["npc_registry_digest"] != nil {
if ok && (metadata["npc_registry_digest"] != nil || metadata["scene_eligibility_digest"] != nil) {
t.Fatalf("%s %s metadata = %#v, want generated identity only in framework provenance", lane.ID, component, metadata)
}
}
@@ -235,6 +249,8 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ
"name": "Hooded Guard", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}},
},
}}
case sceneextract.PromptID:
payload = map[string]any{"kind": "combat", "title": "A combat encounter", "summary": "The party faces an active encounter."}
case spells.PromptID:
payload = map[string]any{"spell_casts": []any{map[string]any{
"caster": "Mira Thorn", "spell": "Cure Wounds",

37
internal/modules/integration/testdata/combat_scene_descriptions.json vendored Normal file
View File

@@ -0,0 +1,37 @@
{
"scenes": [
{
"id": "chunk-000001",
"source_ref": {
"source_id": "npc-session",
"start_unit_id": 1,
"end_unit_id": 2
},
"kind": "combat",
"title": "First combat exchange",
"summary": "The first accepted chunk contains active combat."
},
{
"id": "chunk-000002",
"source_ref": {
"source_id": "npc-session",
"start_unit_id": 3,
"end_unit_id": 4
},
"kind": "combat",
"title": "Second combat exchange",
"summary": "The second accepted chunk contains active combat."
},
{
"id": "chunk-000003",
"source_ref": {
"source_id": "npc-session",
"start_unit_id": 5,
"end_unit_id": 5
},
"kind": "combat",
"title": "Final combat exchange",
"summary": "The final accepted chunk contains active combat."
}
]
}

7
internal/modules/integration/testdata/dnd_npc_grounded_pipeline.yml vendored
View File

@@ -16,12 +16,19 @@ pipelines:
npcs:
extract: dnd/npcs
normalize: dnd/npcs
scene-descriptions:
extract: dnd/scene-descriptions
normalize: dnd/scene-descriptions
- id: grounded-events
references:
npcs:
artifact:
step: identify-npcs
lane: npcs
scene_descriptions:
artifact:
step: identify-npcs
lane: scene-descriptions
artifacts:
spells:
extract: dnd/spells