Add item registry extraction and validation

This commit is contained in:
2026-08-05 19:27:49 +00:00
parent ce04387dbc
commit ad9ee076b5
19 changed files with 1204 additions and 0 deletions

View File

@@ -0,0 +1,87 @@
package itemregistry
import (
"sort"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
)
type orderedItemResponse struct {
value itemResponse
earliest int
hasEvidence bool
}
func canonicalizeResponse(response *extractionResponse, order shared.SourceRefOrder, sourceID string) {
if response == nil {
return
}
ordered := make([]orderedItemResponse, len(response.Items))
for index := range response.Items {
earliest, hasEvidence := canonicalizeItem(&response.Items[index], order, sourceID)
ordered[index] = orderedItemResponse{value: response.Items[index], earliest: earliest, hasEvidence: hasEvidence}
}
sort.SliceStable(ordered, func(left, right int) bool {
if ordered[left].hasEvidence != ordered[right].hasEvidence {
return ordered[left].hasEvidence
}
if !ordered[left].hasEvidence {
return false
}
return ordered[left].earliest < ordered[right].earliest
})
for index := range ordered {
response.Items[index] = ordered[index].value
}
}
func canonicalizeItem(item *itemResponse, order shared.SourceRefOrder, sourceID string) (int, bool) {
if item == nil {
return 0, false
}
item.Name = identity.NormalizeDisplay(item.Name)
refs := order.Canonicalize(canonicalSourceRefs(item.SourceRefs, sourceID))
item.SourceRefs = itemResponseRefs(refs)
return order.EarliestValid(refs)
}
func canonicalItemRegistry(response extractionResponse, sourceID string) dnd.ItemRegistry {
if response.Items == nil {
return dnd.ItemRegistry{Items: nil}
}
items := make([]dnd.Item, len(response.Items))
for index, item := range response.Items {
refs := canonicalSourceRefs(item.SourceRefs, sourceID)
items[index] = dnd.Item{
ID: identity.DeriveID(item.Name),
Name: item.Name,
SourceRefs: refs,
}
}
return dnd.ItemRegistry{Items: items}
}
func canonicalSourceRefs(values []itemSourceRefResponse, sourceID string) []source.SourceRef {
if values == nil {
return nil
}
refs := make([]source.SourceRef, len(values))
for index, value := range values {
refs[index] = source.SourceRef{SourceID: sourceID, StartUnitID: value.StartUnitID, EndUnitID: value.EndUnitID}
}
return refs
}
func itemResponseRefs(values []source.SourceRef) []itemSourceRefResponse {
if values == nil {
return nil
}
refs := make([]itemSourceRefResponse, len(values))
for index, value := range values {
refs[index] = itemSourceRefResponse{StartUnitID: value.StartUnitID, EndUnitID: value.EndUnitID}
}
return refs
}

View File

@@ -0,0 +1,152 @@
// Package itemregistry extracts source-grounded D&D item-registry candidates.
package itemregistry
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
)
const (
Key = "dnd/item-registry"
mappingPolicy = "dnd.item_registry.extract_mapping.v1"
)
var requiredCapabilities = []string{"chunks", "source.transcript"}
var providedCapabilities = []string{"dnd.item_registry"}
var referenceSlotDescriptions = shared.ReferenceSlotDescriptions{
Glossary: "Optional campaign glossary reference material used only for item disambiguation.",
Party: "Optional party roster reference material used only for item disambiguation.",
Players: "Optional player list reference material used only for item disambiguation.",
Roster: "Deprecated alias for party roster reference material used only for item disambiguation.",
}
func referenceSlots() []contracts.ReferenceSlot {
return shared.ReferenceSlots(referenceSlotDescriptions)
}
var _ contracts.Extractor[dnd.ItemRegistry] = (*Extractor)(nil)
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
type Options struct{}
type Extractor struct {
llm contracts.StructuredLLMClient
promptSHA string
responseSchemaSHA string
}
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
if llmClient == nil {
return nil, extractorErrorf("LLM client must not be nil")
}
if len(references) > 1 {
return nil, extractorErrorf("at most one reference set may be supplied")
}
promptSHA, err := promptAssetMetadata()
if err != nil {
return nil, extractorErrorf("load prompt metadata: %w", err)
}
responseSchema, err := loadResponseSchema()
if err != nil {
return nil, extractorErrorf("load response schema: %w", err)
}
return &Extractor{llm: llmClient, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil
}
func (e *Extractor) Key() string { return Key }
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
func (e *Extractor) ManifestMetadata() map[string]any {
if e == nil {
return nil
}
return map[string]any{
"prompt_id": PromptID,
"prompt_version": SchemaVersion,
"prompt_sha256": e.promptSHA,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
"response_schema_version": SchemaVersion,
"response_schema_sha256": e.responseSchemaSHA,
"identity_policy": identity.Policy,
"mapping_policy": mappingPolicy,
}
}
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
if e == nil {
return nil
}
return []pipeline.CheckpointFingerprint{
{Name: "prompt", Value: e.promptSHA},
{Name: "response_schema", Value: e.responseSchemaSHA},
{Name: "identity_policy", Value: identity.Policy},
{Name: "mapping_policy", Value: mappingPolicy},
}
}
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.ItemRegistry], error) {
if e == nil {
return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("extractor must not be nil")
}
if e.llm == nil {
return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("LLM client must not be nil")
}
sourceInput, err := shared.PrepareChunkExtraction(ctx, req)
if err != nil {
return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("%w", err)
}
order := shared.NewSourceRefOrder(req.Source)
var response extractionResponse
if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion,
ProfileID: req.LLMProfile, SessionID: req.SessionID,
Inputs: shared.PromptInputs(sourceInput, req.References),
}, &response); err != nil {
return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("complete structured output: %w", err)
}
canonicalizeResponse(&response, order, req.Source.ID)
return contracts.TypedExtractionResult[dnd.ItemRegistry]{Value: canonicalItemRegistry(response, req.Source.ID)}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked,
Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...),
ArtifactKind: dnd.ItemRegistryKind, ReferenceSlots: referenceSlots(),
}
}
func Register(registry *pipeline.ExtractorRegistry) error {
return pipeline.RegisterExtractorBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Extractor[dnd.ItemRegistry], error) {
options, err := DecodeOptions(request.Options)
if err != nil {
return nil, err
}
return New(request.Dependencies.LLM, options, request.References)
})
}
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
func DecodeOptions(options map[string]any) (Options, error) {
if err := pipeline.RejectUnknownOptions(options); err != nil {
return Options{}, extractorErrorf("%w", err)
}
return Options{}, nil
}
func extractorErrorf(format string, args ...any) error {
return fmt.Errorf("dnd item registry extractor: "+format, args...)
}

View File

@@ -0,0 +1,108 @@
package itemregistry
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
)
func TestExtractMapsItemsWithOwnedEvidenceAndDeterministicOrder(t *testing.T) {
client := &fakeItemsLLMClient{response: extractionResponse{Items: []itemResponse{
{Name: " Gold Pieces ", SourceRefs: responseSourceRefs(3, 3)},
{Name: "Star Compass", SourceRefs: []itemSourceRefResponse{{StartUnitID: 2, EndUnitID: 2}, {StartUnitID: 1, EndUnitID: 1}, {StartUnitID: 1, EndUnitID: 1}}},
}}}
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
refs := []source.SourceRef{{SourceID: "session-items", StartUnitID: 1, EndUnitID: 1}, {SourceID: "session-items", StartUnitID: 2, EndUnitID: 2}}
want := dnd.ItemRegistry{Items: []dnd.Item{
{ID: identity.DeriveID("Star Compass"), Name: "Star Compass", SourceRefs: refs},
{ID: identity.DeriveID("Gold Pieces"), Name: "Gold Pieces", SourceRefs: []source.SourceRef{{SourceID: "session-items", StartUnitID: 3, EndUnitID: 3}}},
}}
if !reflect.DeepEqual(result.Value, want) {
t.Fatalf("Value = %#v, want %#v", result.Value, want)
}
result.Value.Items[0].SourceRefs[0].StartUnitID = 99
for _, item := range client.response.Items {
for _, ref := range item.SourceRefs {
if ref.StartUnitID == 99 {
t.Fatal("result source references alias the model response")
}
}
}
}
func TestExtractKeepsAliasCandidatesSeparate(t *testing.T) {
client := &fakeItemsLLMClient{response: extractionResponse{Items: []itemResponse{
{Name: "Star Compass", SourceRefs: responseSourceRefs(1, 1)},
{Name: "Compass of the Stars", SourceRefs: responseSourceRefs(2, 2)},
}}}
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
if err != nil || len(result.Value.Items) != 2 || result.Value.Items[0].ID == result.Value.Items[1].ID {
t.Fatalf("Extract() = %#v, %v; want separate alias candidates", result, err)
}
}
func TestExtractPreservesInvalidCandidatesForValidators(t *testing.T) {
client := &fakeItemsLLMClient{content: []byte(`{"items":[{"name":"","source_refs":[{"start_unit_id":0,"end_unit_id":-1}]}]}`)}
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
item := result.Value.Items[0]
if item.ID != "" || item.Name != "" || !reflect.DeepEqual(item.SourceRefs, []source.SourceRef{{SourceID: "session-items", StartUnitID: 0, EndUnitID: -1}}) {
t.Fatalf("item = %#v, want invalid candidate preserved", item)
}
}
func TestExtractPassesReferencesWithoutTreatingThemAsEvidence(t *testing.T) {
client := &fakeItemsLLMClient{response: extractionResponse{Items: []itemResponse{}}}
req := extractionRequest()
req.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"glossary": {Slot: contracts.ReferenceSlot{Name: "glossary"}, Items: []contracts.ReferenceItem{{SlotName: "glossary", Content: []byte("Star Compass: heirloom")}}},
}}
if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil {
t.Fatalf("Extract() error = %v", err)
}
inputs := client.requests[0].Inputs
if string(inputs["glossary"].Content) != "Star Compass: heirloom" || strings.Contains(string(inputs["transcript"].Content), "heirloom") {
t.Fatalf("prompt inputs = %#v, want separated reference material", inputs)
}
}
func TestExtractHandlesCancellationAndFailures(t *testing.T) {
request := extractionRequest()
canceled, cancel := context.WithCancel(context.Background())
cancel()
if _, err := newExtractor(t, &fakeItemsLLMClient{}).Extract(canceled, request); err == nil || !errors.Is(err, context.Canceled) || !strings.Contains(err.Error(), "dnd item registry") {
t.Fatalf("canceled Extract() error = %v, want contextual cancellation", err)
}
for _, test := range []struct {
name string
extractor *Extractor
req contracts.TypedExtractionRequest
want string
}{
{name: "nil extractor", extractor: nil, req: request, want: "extractor"},
{name: "nil client", extractor: &Extractor{}, req: request, want: "LLM client"},
{name: "preflight", extractor: newExtractor(t, &fakeItemsLLMClient{}), req: mismatchedSourceInputRequest(request), want: "must match chunk"},
} {
t.Run(test.name, func(t *testing.T) {
if _, err := test.extractor.Extract(context.Background(), test.req); err == nil || !strings.Contains(err.Error(), "dnd item registry") || !strings.Contains(err.Error(), test.want) {
t.Fatalf("Extract() error = %v, want local context", err)
}
})
}
_, err := newExtractor(t, &fakeItemsLLMClient{err: errors.New("provider unavailable")}).Extract(context.Background(), request)
if err == nil || !strings.Contains(err.Error(), "dnd item registry") || !strings.Contains(err.Error(), "provider unavailable") {
t.Fatalf("provider error = %v, want contextual provider error", err)
}
}

View File

@@ -0,0 +1,15 @@
package itemregistry
type extractionResponse struct {
Items []itemResponse `json:"items"`
}
type itemResponse struct {
Name string `json:"name"`
SourceRefs []itemSourceRefResponse `json:"source_refs"`
}
type itemSourceRefResponse struct {
StartUnitID int `json:"start_unit_id"`
EndUnitID int `json:"end_unit_id"`
}

View File

@@ -0,0 +1,74 @@
package itemregistry
import (
"fmt"
"io/fs"
"sync"
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
)
const promptAssetRoot = "assets/prompts"
var promptAssetManifest = shared.PromptAssetManifest{
ModuleDir: PromptID,
ModuleFiles: []promptfs.ModulePromptFile{
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
{Name: "instructions.md", Path: "prompts/instructions.md"},
},
SharedFiles: []string{
"common-dnd-system.md",
"common-dnd-extraction-evidence.md",
"common-dnd-identity.md",
"common-dnd-transcript-chunk.md",
"common-dnd-references.md",
},
}
func moduleAssetFS() (fs.FS, error) {
assets, err := fs.Sub(rootassets.FS(), "dnd/item-registry/extract")
if err != nil {
return nil, fmt.Errorf("scope item extraction assets: %w", err)
}
return assets, nil
}
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
assets, err := moduleAssetFS()
if err != nil {
return err
}
promptFS, err := promptAssetManifest.PromptFS(assets)
if err != nil {
return fmt.Errorf("prepare item prompt assets: %w", err)
}
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
return err
}
schemas, err := fs.Sub(assets, "schemas")
if err != nil {
return fmt.Errorf("scope item extraction schemas: %w", err)
}
return registry.RegisterSchemaFS(schemas, ".")
}
func promptAssetMetadata() (string, error) {
promptAssetHashOnce.Do(func() {
assets, err := moduleAssetFS()
if err != nil {
promptAssetHashErr = err
return
}
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
})
return promptAssetHash, promptAssetHashErr
}
var (
promptAssetHashOnce sync.Once
promptAssetHash string
promptAssetHashErr error
)

View File

@@ -0,0 +1,46 @@
package itemregistry
import (
"context"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestRegisterPromptAssetsPreparesItemPrompt(t *testing.T) {
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v", err)
}
options, err := registry.PromptKitOptions()
if err != nil {
t.Fatalf("PromptKitOptions() error = %v", err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ID: "item-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "item-test-model"})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-test-profile", Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"units":[{"sentinel":"item-transcript"}]}`), "players": promptkit.Inline("item-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "),
}})
if err != nil {
t.Fatalf("Prepare() error = %v", err)
}
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_item_registry_llm.v1.json" {
t.Fatalf("prepared prompt = %#v, want item prompt identity and schema wiring", prepared)
}
content := make([]string, len(prepared.Messages))
for index, message := range prepared.Messages {
content[index] = message.Content
}
rendered := strings.Join(content, "\n")
for _, policy := range []string{"named unique items", "concrete reusable item types", "currency denomination", "loot", "generic weapons", "Do not use capitalization as an eligibility test"} {
if !strings.Contains(rendered, policy) {
t.Fatalf("rendered prompt omits item eligibility policy %q", policy)
}
}
}

View File

@@ -0,0 +1,60 @@
package itemregistry
import (
"encoding/json"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
)
func TestModuleRegistrationMetadataAndRedaction(t *testing.T) {
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
t.Fatalf("New(nil) error = %v, want client rejection", err)
}
if _, err := New(&fakeItemsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one") {
t.Fatalf("New() error = %v, want reference-set rejection", err)
}
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.item_registry"}, ArtifactKind: dnd.ItemRegistryKind, ReferenceSlots: referenceSlots()}
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
}
registry := pipeline.NewExtractorRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v", err)
}
if got, ok := registry.Spec(Key); !ok || !reflect.DeepEqual(got, want) {
t.Fatalf("registry spec = %#v, present = %t", got, ok)
}
if err := Register(nil); err == nil || !strings.Contains(err.Error(), "extractor registry") {
t.Fatalf("Register(nil) error = %v, want registry rejection", err)
}
if _, err := DecodeOptions(map[string]any{"unknown": true}); err == nil {
t.Fatal("DecodeOptions() accepted unknown option")
}
references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"glossary": {Items: []contracts.ReferenceItem{{Content: []byte("secret-campaign-context")}}},
}}
metadata := newExtractor(t, &fakeItemsLLMClient{}, references).ManifestMetadata()
for key, value := range map[string]string{
"prompt_id": PromptID, "prompt_version": SchemaVersion,
"response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion,
"identity_policy": identity.Policy, "mapping_policy": mappingPolicy,
} {
if metadata[key] != value {
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], value)
}
}
encoded, err := json.Marshal(metadata)
if err != nil || strings.Contains(string(encoded), "secret-campaign-context") || strings.Contains(string(encoded), "session-items") {
t.Fatalf("manifest metadata = %s, want static redacted provenance only", encoded)
}
if got := newExtractor(t, &fakeItemsLLMClient{}).CheckpointFingerprints(); len(got) != 4 || got[0].Name != "prompt" || got[1].Name != "response_schema" || got[2].Value != identity.Policy || got[3].Value != mappingPolicy {
t.Fatalf("CheckpointFingerprints() = %#v", got)
}
}

View File

@@ -0,0 +1,25 @@
package itemregistry
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
const (
PromptID = "dnd.item_registry"
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_item_registry_llm")
ResponseSchemaID = "notarius.dnd.item_registry.llm"
ResponseSchemaName = "notarius_dnd_item_registry_llm_v1"
SchemaVersion = "v1"
)
func loadResponseSchema() (llm.ResponseSchema, error) {
assets, err := moduleAssetFS()
if err != nil {
return llm.ResponseSchema{}, err
}
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
Key: ResponseSchemaKey,
ID: ResponseSchemaID,
Version: SchemaVersion,
Name: ResponseSchemaName,
AssetPath: "schemas/dnd_item_registry_llm.v1.json",
})
}

View File

@@ -0,0 +1,25 @@
package itemregistry
import (
"encoding/json"
"strings"
"testing"
)
func TestItemResponseSchemaIsPrivateAndStructural(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v", err)
}
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
t.Fatalf("schema = %#v, want item response schema", schema)
}
var document map[string]any
if err := json.Unmarshal(schema.JSONSchema, &document); err != nil {
t.Fatal(err)
}
encoded, err := json.Marshal(document)
if err != nil || strings.Contains(string(encoded), `"id"`) || !strings.Contains(string(encoded), `"additionalProperties":false`) {
t.Fatalf("schema = %s, want strict private response without durable ID", encoded)
}
}

View File

@@ -0,0 +1,92 @@
package itemregistry
import (
"context"
"encoding/json"
"errors"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func extractionRequest() contracts.TypedExtractionRequest {
doc := sourceDocument()
chunk := &source.Chunk{
ID: "session-items:chunk:0", SourceID: doc.ID, Index: 0,
Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 3},
Content: []byte(`{"units":[1,2,3]}`), MediaType: "application/json",
Units: append([]source.SourceUnit(nil), doc.Units...),
}
return contracts.TypedExtractionRequest{
Source: doc, Chunk: chunk,
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-items.json"),
SessionID: "item-session", LLMProfile: "item-profile",
}
}
func sourceDocument() *source.SourceDocument {
return &source.SourceDocument{ID: "session-items", Kind: "transcript", Format: "application/json", Digest: "sha256:test", Units: []source.SourceUnit{
{ID: 1, Kind: "transcript_segment", Text: "The party finds a rope."},
{ID: 2, Kind: "transcript_segment", Text: "They recover the Star Compass."},
{ID: 3, Kind: "transcript_segment", Text: "The chest contains gold pieces."},
}}
}
func responseSourceRefs(startUnitID, endUnitID int) []itemSourceRefResponse {
return []itemSourceRefResponse{{StartUnitID: startUnitID, EndUnitID: endUnitID}}
}
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
t.Helper()
extractor, err := New(client, Options{}, references...)
if err != nil {
t.Fatalf("New() error = %v, want nil", err)
}
return extractor
}
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest {
req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "file:///other.json")
return req
}
type fakeItemsLLMClient struct {
response extractionResponse
content []byte
err error
requests []contracts.StructuredCompletionRequest
}
func (client *fakeItemsLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
if err := ctx.Err(); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
if client.err != nil {
return contracts.StructuredCompletionResponse{}, client.err
}
target, ok := out.(*extractionResponse)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
}
content := append([]byte(nil), client.content...)
if len(content) != 0 {
if err := json.Unmarshal(content, target); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
} else {
*target = client.response
var err error
content, err = json.Marshal(client.response)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
}
}
return contracts.StructuredCompletionResponse{Content: content}, nil
}
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
req.Inputs = req.Inputs.Clone()
return req
}

View File

@@ -0,0 +1,97 @@
// Package shape validates the required extracted item fields.
package shape
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
)
const (
Key = "extract/dnd/item-registry/shape"
ReasonCode = "invalid_item_shape"
policy = "dnd.item_registry.validator.shape.v1"
)
type Options struct{}
type Validator struct{}
var _ contracts.TypedValidator[dnd.ItemRegistry] = (*Validator)(nil)
var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil)
func New(Options) *Validator { return &Validator{} }
func (v *Validator) Name() string { return Key }
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
return []pipeline.CheckpointFingerprint{{Name: "policy", Value: policy}}
}
func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.ItemRegistry]) (contracts.ValidationResult, error) {
issues := issuesFor(req.Value)
if len(issues) > 0 {
return rejection(diagnostics.Aggregate("invalid item shape", issues)), nil
}
return contracts.ValidationResult{Approved: true}, nil
}
func Validate(value dnd.ItemRegistry) error {
issues := issuesFor(value)
if len(issues) == 0 {
return nil
}
return fmt.Errorf("%s", diagnostics.Aggregate("invalid item shape", issues))
}
func issuesFor(value dnd.ItemRegistry) []string {
if value.Items == nil {
return []string{"items must be present"}
}
issues := make([]string, 0)
for index, item := range value.Items {
prefix := fmt.Sprintf("items[%d]", index)
if strings.TrimSpace(item.ID) == "" {
issues = append(issues, prefix+".id must not be empty")
}
if strings.TrimSpace(item.Name) == "" {
issues = append(issues, prefix+".name must not be empty")
}
if len(item.SourceRefs) == 0 {
issues = append(issues, prefix+".source_refs must not be empty")
}
}
return issues
}
func Spec() pipeline.ValidatorSpec {
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
}
func Register(registry *pipeline.ValidatorRegistry) error {
return pipeline.RegisterTypedValidatorBuilder(registry, dnd.ItemRegistryKind, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.TypedValidator[dnd.ItemRegistry], error) {
options, err := DecodeOptions(request.Options)
if err != nil {
return nil, err
}
return New(options), nil
})
}
func DecodeOptions(options map[string]any) (Options, error) {
if err := pipeline.RejectUnknownOptions(options); err != nil {
return Options{}, err
}
return Options{}, nil
}
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
func rejection(message string) contracts.ValidationResult {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message}
}

View File

@@ -0,0 +1,44 @@
package shape
import (
"context"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
)
func TestValidatorRejectsMalformedItemsWithoutMutation(t *testing.T) {
value := dnd.ItemRegistry{Items: []dnd.Item{{Name: "", SourceRefs: []source.SourceRef{}}}}
before := value
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Value: value})
if err != nil || result.Approved || result.ReasonCode != ReasonCode || !strings.Contains(result.Message, "id must not be empty") || !strings.Contains(result.Message, "source_refs must not be empty") || !reflect.DeepEqual(value, before) {
t.Fatalf("Validate() = %#v, %v; want non-mutating shape rejection", result, err)
}
result, err = New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Value: dnd.ItemRegistry{}})
if err != nil || result.Approved || !strings.Contains(result.Message, "items must be present") {
t.Fatalf("missing items = %#v, %v; want rejection", result, err)
}
}
func TestValidatorApprovesAndRegisters(t *testing.T) {
value := dnd.ItemRegistry{Items: []dnd.Item{{ID: "item", Name: "Rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}}}}
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Value: value})
if err != nil || !result.Approved {
t.Fatalf("Validate() = %#v, %v; want approval", result, err)
}
registry := pipeline.NewValidatorRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v", err)
}
if got, ok := registry.Spec(Key); !ok || got != Spec() || got.ExecutionClass != contracts.ExecutionClassDeterministic {
t.Fatalf("registered spec = %#v, ok = %t", got, ok)
}
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
t.Fatal("DecodeOptions() accepted unknown options")
}
}

View File

@@ -0,0 +1,101 @@
// Package sourcerefs validates item citations against the current source.
package sourcerefs
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
itemshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/shape"
)
const (
Key = "extract/dnd/item-registry/source_refs"
ReasonCode = "invalid_item_source_refs"
policy = "dnd.item_registry.validator.source_refs.v1"
)
type Options struct{}
type Validator struct{}
var _ contracts.TypedValidator[dnd.ItemRegistry] = (*Validator)(nil)
var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil)
func New(Options) *Validator { return &Validator{} }
func (v *Validator) Name() string { return Key }
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
return []pipeline.CheckpointFingerprint{{Name: "policy", Value: policy}}
}
func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.ItemRegistry]) (contracts.ValidationResult, error) {
if req.Stage == string(pipeline.StageExtract) && req.Chunk == nil {
return contracts.ValidationResult{}, fmt.Errorf("item source-reference validator requires the current extraction chunk")
}
if err := itemshape.Validate(req.Value); err != nil {
return contracts.ValidationResult{Approved: true}, nil
}
index := source.NewDocumentIndex(req.Source)
issues := make([]string, 0)
for itemIndex, item := range req.Value.Items {
for refIndex, ref := range item.SourceRefs {
if err := index.ValidateRef(ref); err != nil {
issues = append(issues, fmt.Sprintf("items[%d].source_refs[%d]: %s", itemIndex, refIndex, diagnostics.Truncate(err.Error())))
continue
}
if req.Stage == string(pipeline.StageExtract) && !chunkContainsRef(req.Chunk, ref) {
issues = append(issues, fmt.Sprintf("items[%d].source_refs[%d]: source reference is outside the current extraction chunk", itemIndex, refIndex))
}
}
}
if len(issues) == 0 {
return contracts.ValidationResult{Approved: true}, nil
}
return rejection(diagnostics.Aggregate("invalid item source references", issues)), nil
}
func chunkContainsRef(chunk *source.Chunk, ref source.SourceRef) bool {
if chunk == nil || ref.SourceID != chunk.SourceID {
return false
}
startFound := false
endFound := false
for _, unit := range chunk.Units {
startFound = startFound || unit.ID == ref.StartUnitID
endFound = endFound || unit.ID == ref.EndUnitID
}
return startFound && endFound
}
func Spec() pipeline.ValidatorSpec {
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
}
func Register(registry *pipeline.ValidatorRegistry) error {
return pipeline.RegisterTypedValidatorBuilder(registry, dnd.ItemRegistryKind, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.TypedValidator[dnd.ItemRegistry], error) {
options, err := DecodeOptions(request.Options)
if err != nil {
return nil, err
}
return New(options), nil
})
}
func DecodeOptions(options map[string]any) (Options, error) {
if err := pipeline.RejectUnknownOptions(options); err != nil {
return Options{}, err
}
return Options{}, nil
}
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
func rejection(message string) contracts.ValidationResult {
return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message}
}

View File

@@ -0,0 +1,60 @@
package sourcerefs
import (
"context"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
)
func TestValidatorRejectsInvalidAndOutOfChunkReferencesWithoutMutation(t *testing.T) {
value := validItemRegistry()
value.Items[0].SourceRefs = []source.SourceRef{{SourceID: "foreign", StartUnitID: 1, EndUnitID: 1}}
before := value
result, err := New(Options{}).Validate(context.Background(), request(validDocument(), value))
if err != nil || result.Approved || result.ReasonCode != ReasonCode || !strings.Contains(result.Message, "items[0].source_refs[0]") || !reflect.DeepEqual(value, before) {
t.Fatalf("Validate() = %#v, %v; want source-reference rejection", result, err)
}
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1}, {ID: 2}}}
value = validItemRegistry()
value.Items[0].SourceRefs[0].EndUnitID = 2
result, err = New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Stage: string(pipeline.StageExtract), Source: doc, Chunk: &source.Chunk{SourceID: "session", Units: []source.SourceUnit{{ID: 1}}}, Value: value})
if err != nil || result.Approved || !strings.Contains(result.Message, "outside the current extraction chunk") {
t.Fatalf("out-of-chunk evidence = %#v, %v", result, err)
}
}
func TestValidatorDefersMalformedShapeAndRegisters(t *testing.T) {
result, err := New(Options{}).Validate(context.Background(), request(nil, dnd.ItemRegistry{Items: []dnd.Item{{Name: "Missing"}}}))
if err != nil || !result.Approved {
t.Fatalf("malformed shape = %#v, %v; want deferral", result, err)
}
_, err = New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Stage: string(pipeline.StageExtract), Source: validDocument(), Value: validItemRegistry()})
if err == nil || !strings.Contains(err.Error(), "requires the current extraction chunk") {
t.Fatalf("missing chunk error = %v", err)
}
registry := pipeline.NewValidatorRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v", err)
}
if got, ok := registry.Spec(Key); !ok || got != Spec() || got.ExecutionClass != contracts.ExecutionClassDeterministic {
t.Fatalf("registered spec = %#v, ok = %t", got, ok)
}
}
func request(doc *source.SourceDocument, value dnd.ItemRegistry) contracts.TypedValidationRequest[dnd.ItemRegistry] {
return contracts.TypedValidationRequest[dnd.ItemRegistry]{Source: doc, Value: value}
}
func validDocument() *source.SourceDocument {
return &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Text: "The party finds rope."}}}
}
func validItemRegistry() dnd.ItemRegistry {
return dnd.ItemRegistry{Items: []dnd.Item{{ID: "item", Name: "Rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}}}}
}

View File

@@ -0,0 +1,88 @@
// Package sourcerelatedness warns when cited source text does not mention an item.
package sourcerelatedness
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
itemshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/shape"
)
const (
Key = "extract/dnd/item-registry/source_relatedness"
WarningReasonCode = "item_not_near_source"
OmittedReasonCode = "item_relatedness_warnings_omitted"
policy = "dnd.item_registry.validator.source_relatedness.v1"
)
type Options struct{}
type Validator struct{}
var _ contracts.TypedValidator[dnd.ItemRegistry] = (*Validator)(nil)
var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil)
func New(Options) *Validator { return &Validator{} }
func (v *Validator) Name() string { return Key }
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
return []pipeline.CheckpointFingerprint{{Name: "policy", Value: policy}}
}
func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.ItemRegistry]) (contracts.ValidationResult, error) {
if err := itemshape.Validate(req.Value); err != nil {
return contracts.ValidationResult{Approved: true}, nil
}
resolver, err := shared.NewCitationResolver(req.Source)
if err != nil {
return contracts.ValidationResult{Approved: true}, nil
}
citedTexts := make([]string, len(req.Value.Items))
for itemIndex, item := range req.Value.Items {
citedText, err := resolver.CitedText(item.SourceRefs)
if err != nil {
return contracts.ValidationResult{Approved: true}, nil
}
citedTexts[itemIndex] = citedText
}
warnings := make([]contracts.Warning, 0)
for itemIndex, item := range req.Value.Items {
if shared.ContainsTokenSequence(citedTexts[itemIndex], item.Name) {
continue
}
warnings = append(warnings, contracts.Warning{
Scope: fmt.Sprintf("items[%d]", itemIndex), ReasonCode: WarningReasonCode,
Message: fmt.Sprintf("Item %s was not found in cited source text", diagnostics.Quote(item.Name)),
})
}
return contracts.ValidationResult{Approved: true, Warnings: diagnostics.LimitWarnings(warnings, "items", OmittedReasonCode)}, nil
}
func Spec() pipeline.ValidatorSpec {
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic}
}
func Register(registry *pipeline.ValidatorRegistry) error {
return pipeline.RegisterTypedValidatorBuilder(registry, dnd.ItemRegistryKind, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.TypedValidator[dnd.ItemRegistry], error) {
options, err := DecodeOptions(request.Options)
if err != nil {
return nil, err
}
return New(options), nil
})
}
func DecodeOptions(options map[string]any) (Options, error) {
if err := pipeline.RejectUnknownOptions(options); err != nil {
return Options{}, err
}
return Options{}, nil
}
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }

View File

@@ -0,0 +1,46 @@
package sourcerelatedness
import (
"context"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
)
func TestValidatorUsesOnlyCitedTranscriptText(t *testing.T) {
value := dnd.ItemRegistry{Items: []dnd.Item{{ID: "item", Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}}}}
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Text: "They recover the star compass."}, {ID: 2, Text: "Unrelated text."}}}
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Source: doc, Value: value})
if err != nil || !result.Approved || len(result.Warnings) != 0 {
t.Fatalf("cited match = %#v, %v; want approval without warnings", result, err)
}
value.Items[0].Name = "Glossary Relic"
before := value
result, err = New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Source: doc, References: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"glossary": {Items: []contracts.ReferenceItem{{Content: []byte("Glossary Relic")}}}}}, Value: value})
if err != nil || !result.Approved || len(result.Warnings) != 1 || result.Warnings[0].ReasonCode != WarningReasonCode || !reflect.DeepEqual(value, before) {
t.Fatalf("reference-only match = %#v, %v; want advisory warning", result, err)
}
}
func TestValidatorBoundsWarningsAndRegisters(t *testing.T) {
items := make([]dnd.Item, diagnostics.MaxWarnings+1)
for index := range items {
items[index] = dnd.Item{ID: "item", Name: "Missing", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}}
}
result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Source: &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Text: "Nothing here."}}}, Value: dnd.ItemRegistry{Items: items}})
if err != nil || !result.Approved || len(result.Warnings) != diagnostics.MaxWarnings || result.Warnings[len(result.Warnings)-1].ReasonCode != OmittedReasonCode {
t.Fatalf("bounded warnings = %#v, %v", result, err)
}
registry := pipeline.NewValidatorRegistry()
if err := Register(registry); err != nil {
t.Fatalf("Register() error = %v", err)
}
if got, ok := registry.Spec(Key); !ok || got != Spec() || got.ExecutionClass != contracts.ExecutionClassDeterministic {
t.Fatalf("registered spec = %#v, ok = %t", got, ok)
}
}