eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden item occurrence grounding

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-06 13:37:21 +00:00
parent 8cf03a2a44
commit 4192aa8584
16 changed files with 152 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
internal/modules/dnd/extract/itemoccurrences/canonicalize.go
View File

@@ -1,6 +1,7 @@
package itemoccurrences
import (
"fmt"
"sort"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
@@ -15,12 +16,15 @@ type orderedItemOccurrenceResponse struct {
hasEvidence bool
}
func canonicalizeResponse(response *extractionResponse, order shared.SourceRefOrder, sourceID string) {
func canonicalizeResponse(response *extractionResponse, order shared.SourceRefOrder, sourceID string, registry *itemregistry.Registry) error {
if response == nil {
return
return nil
}
ordered := make([]orderedItemOccurrenceResponse, len(response.Occurrences))
for index := range response.Occurrences {
if err := validateRegistryPair(index, response.Occurrences[index], registry); err != nil {
return err
}
earliest, hasEvidence := canonicalizeItemOccurrence(&response.Occurrences[index], order, sourceID)
ordered[index] = orderedItemOccurrenceResponse{
value: response.Occurrences[index],
@@ -40,6 +44,7 @@ func canonicalizeResponse(response *extractionResponse, order shared.SourceRefOr
for index := range ordered {
response.Occurrences[index] = ordered[index].value
}
return nil
}
func canonicalizeItemOccurrence(occurrence *itemOccurrenceResponse, order shared.SourceRefOrder, sourceID string) (int, bool) {
@@ -51,16 +56,23 @@ func canonicalizeItemOccurrence(occurrence *itemOccurrenceResponse, order shared
return order.EarliestValid(refs)
}
func canonicalItemOccurrenceList(response extractionResponse, sourceID string, registry *itemregistry.Registry) dnd.ItemOccurrenceList {
func validateRegistryPair(index int, occurrence itemOccurrenceResponse, registry *itemregistry.Registry) error {
item, found := registry.LookupID(occurrence.ItemID)
if !found {
return fmt.Errorf("occurrences[%d].item_id is not in the item registry", index)
}
if item.Name != occurrence.Name {
return fmt.Errorf("occurrences[%d].name does not match item_id", index)
}
return nil
}
func canonicalItemOccurrenceList(response extractionResponse, sourceID string) dnd.ItemOccurrenceList {
if response.Occurrences == nil {
return dnd.ItemOccurrenceList{}
}
occurrences := make([]dnd.ItemOccurrence, 0, len(response.Occurrences))
for _, occurrence := range response.Occurrences {
item, found := registry.LookupID(occurrence.ItemID)
if !found || item.Name != occurrence.Name {
continue
}
occurrences = append(occurrences, dnd.ItemOccurrence{
ItemID: occurrence.ItemID,
Name: occurrence.Name,

8
internal/modules/dnd/extract/itemoccurrences/extractor.go
View File

@@ -109,6 +109,7 @@ func (e *Extractor) ManifestMetadata() map[string]any {
"response_schema_version": SchemaVersion,
"response_schema_sha256": e.responseSchemaSHA,
"mapping_policy": mappingPolicy,
"comparison_policy": shared.TextComparisonPolicy,
}
seeded := e.itemResolver.Seeded()
if seeded.Bound() {
@@ -126,6 +127,7 @@ func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
{Name: "prompt", Value: e.promptSHA},
{Name: "response_schema", Value: e.responseSchemaSHA},
{Name: "mapping_policy", Value: mappingPolicy},
{Name: "comparison_policy", Value: shared.TextComparisonPolicy},
{Name: "item_registry", Value: e.itemResolver.Seeded().ProjectionDigest()},
}
}
@@ -159,8 +161,10 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
}, &response); err != nil {
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("complete structured output: %w", err)
}
canonicalizeResponse(&response, order, req.Source.ID)
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{Value: canonicalItemOccurrenceList(response, req.Source.ID, registry)}, nil
if err := canonicalizeResponse(&response, order, req.Source.ID, registry); err != nil {
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("map item occurrence response: %w", err)
}
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{Value: canonicalItemOccurrenceList(response, req.Source.ID)}, nil
}
func ModuleSpec() pipeline.ModuleSpec {

53
internal/modules/dnd/extract/itemoccurrences/extractor_test.go
View File

@@ -2,6 +2,7 @@ package itemoccurrences
import (
"context"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -12,8 +13,6 @@ func TestExtractGroundsOccurrencesInRequiredRegistry(t *testing.T) {
id := itemidentity.DeriveID("Torch")
client := &fakeItemOccurrencesLLMClient{response: extractionResponse{Occurrences: []itemOccurrenceResponse{
{ItemID: id, Name: "Torch", Kind: "lost", From: "party", SourceRefs: responseRefs(1, 1)},
{ItemID: "unknown", Name: "Unknown", Kind: "lost", From: "party", SourceRefs: responseRefs(2, 2)},
{ItemID: "torch", Name: "Lantern", Kind: "lost", From: "party", SourceRefs: responseRefs(3, 3)},
}}}
req := extractionRequest()
req.References = itemRegistryReferences(t)
@@ -30,6 +29,56 @@ func TestExtractGroundsOccurrencesInRequiredRegistry(t *testing.T) {
}
}
func TestExtractRejectsInvalidRegistryPairs(t *testing.T) {
id := itemidentity.DeriveID("Torch")
for _, test := range []struct {
name string
occurrence itemOccurrenceResponse
wantError string
}{
{
name: "unknown item ID",
occurrence: itemOccurrenceResponse{ItemID: "unknown", Name: "Torch", Kind: "lost", From: "party", SourceRefs: responseRefs(1, 1)},
wantError: "occurrences[0].item_id is not in the item registry",
},
{
name: "mismatched item name",
occurrence: itemOccurrenceResponse{ItemID: id, Name: "Lantern", Kind: "lost", From: "party", SourceRefs: responseRefs(1, 1)},
wantError: "occurrences[0].name does not match item_id",
},
} {
t.Run(test.name, func(t *testing.T) {
client := &fakeItemOccurrencesLLMClient{response: extractionResponse{Occurrences: []itemOccurrenceResponse{test.occurrence}}}
req := extractionRequest()
req.References = itemRegistryReferences(t)
result, err := newExtractor(t, client, req.References).Extract(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), test.wantError) {
t.Fatalf("Extract() error = %v, want %q", err, test.wantError)
}
if len(result.Value.Occurrences) != 0 {
t.Fatalf("Extract() returned partial occurrences: %#v", result.Value.Occurrences)
}
})
}
}
func TestExtractRejectsResponseWithInvalidRegistryPairAfterValidOccurrence(t *testing.T) {
id := itemidentity.DeriveID("Torch")
client := &fakeItemOccurrencesLLMClient{response: extractionResponse{Occurrences: []itemOccurrenceResponse{
{ItemID: id, Name: "Torch", Kind: "lost", From: "party", SourceRefs: responseRefs(1, 1)},
{ItemID: "unknown", Name: "Unknown", Kind: "lost", From: "party", SourceRefs: responseRefs(2, 2)},
}}}
req := extractionRequest()
req.References = itemRegistryReferences(t)
result, err := newExtractor(t, client, req.References).Extract(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "occurrences[1].item_id") {
t.Fatalf("Extract() error = %v, want occurrence index and item ID", err)
}
if len(result.Value.Occurrences) != 0 {
t.Fatalf("Extract() returned partial occurrences: %#v", result.Value.Occurrences)
}
}
func TestExtractRequiresItemRegistry(t *testing.T) {
_, err := newExtractor(t, &fakeItemOccurrencesLLMClient{}).Extract(context.Background(), extractionRequest())
if err == nil {