Order spell and NPC extraction by document position

internal/modules/dnd/extract/spells/canonicalize.go

@@ -5,18 +5,19 @@ import (
 
 	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
 	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
+	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
 )
 
-func canonicalizeResponse(response *extractionResponse) {
+func canonicalizeResponse(response *extractionResponse, order shared.SourceRefOrder, sourceID string) {
 	if response == nil {
 		return
 	}
 	for index := range response.SpellCasts {
-		canonicalizeSpellCast(&response.SpellCasts[index])
+		canonicalizeSpellCast(&response.SpellCasts[index], order, sourceID)
 	}
 	sort.SliceStable(response.SpellCasts, func(i, j int) bool {
-		left, leftOK := earliestSourceUnit(response.SpellCasts[i])
-		right, rightOK := earliestSourceUnit(response.SpellCasts[j])
+		left, leftOK := order.EarliestValid(spellSourceRefs(response.SpellCasts[i].SourceRefs, sourceID))
+		right, rightOK := order.EarliestValid(spellSourceRefs(response.SpellCasts[j].SourceRefs, sourceID))
 		if leftOK != rightOK {
 			return leftOK
 		}
@@ -27,53 +28,33 @@ func canonicalizeResponse(response *extractionResponse) {
 	})
 }
 
-func canonicalizeSpellCast(spell *spellCastResponse) {
-	sort.SliceStable(spell.SourceRefs, func(i, j int) bool {
-		left := spell.SourceRefs[i]
-		right := spell.SourceRefs[j]
-		if left.StartUnitID != right.StartUnitID {
-			return unitSortValue(left.StartUnitID) < unitSortValue(right.StartUnitID)
-		}
-		return unitSortValue(left.EndUnitID) < unitSortValue(right.EndUnitID)
-	})
-	spell.SourceRefs = dedupeSourceRefs(spell.SourceRefs)
+func canonicalizeSpellCast(spell *spellCastResponse, order shared.SourceRefOrder, sourceID string) {
+	if spell == nil {
+		return
+	}
+	spell.SourceRefs = spellResponseRefs(order.Canonicalize(spellSourceRefs(spell.SourceRefs, sourceID)))
 }
 
-func dedupeSourceRefs(refs []spellSourceRefResponse) []spellSourceRefResponse {
-	if len(refs) < 2 {
-		return refs
+func spellSourceRefs(refs []spellSourceRefResponse, sourceID string) []source.SourceRef {
+	if refs == nil {
+		return nil
 	}
-	out := refs[:0]
-	var previous spellSourceRefResponse
+	values := make([]source.SourceRef, len(refs))
 	for index, ref := range refs {
-		if index > 0 && sameSourceRef(previous, ref) {
-			continue
-		}
-		out = append(out, ref)
-		previous = ref
+		values[index] = source.SourceRef{SourceID: sourceID, StartUnitID: ref.StartUnitID, EndUnitID: ref.EndUnitID}
 	}
-	return out
+	return values
 }
 
-func sameSourceRef(left spellSourceRefResponse, right spellSourceRefResponse) bool {
-	return left.StartUnitID == right.StartUnitID && left.EndUnitID == right.EndUnitID
-}
-
-func earliestSourceUnit(spell spellCastResponse) (int, bool) {
-	for _, ref := range spell.SourceRefs {
-		start := ref.StartUnitID
-		if start > 0 {
-			return start, true
-		}
+func spellResponseRefs(refs []source.SourceRef) []spellSourceRefResponse {
+	if refs == nil {
+		return nil
 	}
-	return 0, false
-}
-
-func unitSortValue(value int) int {
-	if value <= 0 {
-		return int(^uint(0) >> 1)
+	values := make([]spellSourceRefResponse, len(refs))
+	for index, ref := range refs {
+		values[index] = spellSourceRefResponse{StartUnitID: ref.StartUnitID, EndUnitID: ref.EndUnitID}
 	}
-	return value
+	return values
 }
 
 func canonicalSpellList(response extractionResponse, sourceID string) dnd.SpellList {

internal/modules/dnd/extract/spells/extractor.go

@@ -17,7 +17,7 @@ const Key = "dnd/spells"
 const ArtifactType = "dnd.spell_cast"
 const SchemaVersion = "v1"
 
-const mappingPolicy = "dnd.spells.extract_mapping.v1"
+const mappingPolicy = "dnd.spells.extract_mapping.v2"
 
 const (
 	NPCRegistryReferenceSlot = npcregistry.ReferenceSlot
@@ -185,6 +185,7 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
 	if err != nil {
 		return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("%w", err)
 	}
+	order := shared.NewSourceRefOrder(req.Source)
 	npcRegistry, err := e.npcResolver.Resolve(req.References)
 	if err != nil {
 		return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("resolve NPC registry: %w", err)
@@ -204,7 +205,7 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
 	}, &response); err != nil {
 		return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("complete structured output: %w", err)
 	}
-	canonicalizeResponse(&response)
+	canonicalizeResponse(&response, order, req.Source.ID)
 	return contracts.TypedExtractionResult[dnd.SpellList]{Value: canonicalSpellList(response, req.Source.ID)}, nil
 }
 

internal/modules/dnd/extract/spells/extractor_test.go

@@ -284,6 +284,57 @@ func TestExtractOrdersAndDeduplicatesEvidence(t *testing.T) {
 	}
 }
 
+func TestExtractUsesDocumentOrderForReferencesAndSpellCasts(t *testing.T) {
+	client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{
+		{Caster: "Later", Spell: "Fire Bolt", SourceRefs: responseSourceRefs(10, 10)},
+		{Caster: "Earlier", Spell: "Cure Wounds", SourceRefs: []spellSourceRefResponse{
+			{StartUnitID: 10, EndUnitID: 10},
+			{StartUnitID: 30, EndUnitID: 30},
+			{StartUnitID: 30, EndUnitID: 30},
+			{StartUnitID: 999, EndUnitID: 0},
+		}},
+		{Caster: "Unavailable", Spell: "Healing Word", SourceRefs: []spellSourceRefResponse{{StartUnitID: 999, EndUnitID: 0}}},
+	}}}
+	req := extractionRequest()
+	req.Source.Units = []source.SourceUnit{{ID: 30}, {ID: 10}}
+	req.Chunk.Units = append([]source.SourceUnit(nil), req.Source.Units...)
+	req.Chunk.Ref = source.SourceRef{SourceID: req.Source.ID, StartUnitID: 30, EndUnitID: 10}
+
+	result, err := newExtractor(t, client).Extract(context.Background(), req)
+	if err != nil {
+		t.Fatalf("Extract() error = %v", err)
+	}
+	if got := []string{result.Value.SpellCasts[0].Spell, result.Value.SpellCasts[1].Spell, result.Value.SpellCasts[2].Spell}; !reflect.DeepEqual(got, []string{"Cure Wounds", "Fire Bolt", "Healing Word"}) {
+		t.Fatalf("spell order = %#v, want document chronology followed by invalid evidence", got)
+	}
+	refs := result.Value.SpellCasts[0].SourceRefs
+	if got := []int{refs[0].StartUnitID, refs[1].StartUnitID, refs[2].StartUnitID}; !reflect.DeepEqual(got, []int{30, 10, 999}) {
+		t.Fatalf("source refs = %#v, want document order with exact duplicate removed", refs)
+	}
+	refs[0].StartUnitID = 777
+	for _, spell := range client.response.SpellCasts {
+		for _, ref := range spell.SourceRefs {
+			if ref.StartUnitID == 777 {
+				t.Fatal("result source references alias the model response")
+			}
+		}
+	}
+}
+
+func TestExtractPreservesStableSpellOrderForEqualEvidence(t *testing.T) {
+	client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{
+		{Caster: "First", Spell: "Cure Wounds", SourceRefs: responseSourceRefs(2, 2)},
+		{Caster: "Second", Spell: "Fire Bolt", SourceRefs: responseSourceRefs(2, 2)},
+	}}}
+	result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
+	if err != nil {
+		t.Fatalf("Extract() error = %v", err)
+	}
+	if got := []string{result.Value.SpellCasts[0].Caster, result.Value.SpellCasts[1].Caster}; !reflect.DeepEqual(got, []string{"First", "Second"}) {
+		t.Fatalf("equal-evidence order = %#v, want stable response order", got)
+	}
+}
+
 func TestExtractPreservesInvalidEvidenceForValidators(t *testing.T) {
 	client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{{
 		Caster: "Aria", Spell: "Cure Wounds",