diff --git a/assets/dnd/item-registry/extract/prompts/instructions.md b/assets/dnd/item-registry/extract/prompts/instructions.md new file mode 100644 index 0000000..a5b676f --- /dev/null +++ b/assets/dnd/item-registry/extract/prompts/instructions.md @@ -0,0 +1,12 @@ +Extract only items established by the provided Dungeons & Dragons transcript. + +Include named unique items, concrete reusable item types, and stable unique +designations. Record each currency denomination separately when it is +established, such as copper pieces, silver pieces, gold pieces, or platinum +pieces. Do not use capitalization as an eligibility test. Keep distinct names +and designations as separate candidates; do not merge aliases or invent +qualifiers. + +Do not record vague categories such as "loot", "treasure", or "some gear"; +generic weapons; inferred properties; quantities; or inferred uniqueness. Omit +uncertain or unsupported items. diff --git a/assets/dnd/item-registry/extract/prompts/prompt.yaml b/assets/dnd/item-registry/extract/prompts/prompt.yaml new file mode 100644 index 0000000..c380ba4 --- /dev/null +++ b/assets/dnd/item-registry/extract/prompts/prompt.yaml @@ -0,0 +1,40 @@ +id: dnd.item_registry +version: "v1" +default_profile: dnd-extraction +inputs: + - name: transcript + required: true + content_type: application/json + - name: players + required: false + content_type: text/plain + - name: party + required: false + content_type: text/plain + - name: glossary + required: false + content_type: text/plain +messages: + - role: system + content_file: ./sharedassets/common-dnd-system.md + - role: user + content_file: ./sharedassets/common-dnd-identity.md + - role: user + content_file: ./sharedassets/common-dnd-references.md + cache_control: + type: ephemeral + - role: user + content_file: ./sharedassets/common-dnd-transcript-chunk.md + cache_control: + type: ephemeral + - role: user + content_file: ./sharedassets/common-dnd-extraction-evidence.md + - role: user + content_file: ./instructions.md + cache_control: + type: ephemeral +output: + format: json + validation_mode: json_schema + schema_path: dnd_item_registry_llm.v1.json + repair_attempts: 0 diff --git a/assets/dnd/item-registry/extract/schemas/dnd_item_registry_llm.v1.json b/assets/dnd/item-registry/extract/schemas/dnd_item_registry_llm.v1.json new file mode 100644 index 0000000..33187b1 --- /dev/null +++ b/assets/dnd/item-registry/extract/schemas/dnd_item_registry_llm.v1.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "notarius.dnd.item_registry.llm", + "type": "object", + "additionalProperties": false, + "required": ["items"], + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["name", "source_refs"], + "properties": { + "name": {"type": "string"}, + "source_refs": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["start_unit_id", "end_unit_id"], + "properties": { + "start_unit_id": {"type": "integer"}, + "end_unit_id": {"type": "integer"} + } + } + } + } + } + } + } +} diff --git a/internal/modules/dnd/extract/itemregistry/canonicalize.go b/internal/modules/dnd/extract/itemregistry/canonicalize.go new file mode 100644 index 0000000..cf4905c --- /dev/null +++ b/internal/modules/dnd/extract/itemregistry/canonicalize.go @@ -0,0 +1,87 @@ +package itemregistry + +import ( + "sort" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" +) + +type orderedItemResponse struct { + value itemResponse + earliest int + hasEvidence bool +} + +func canonicalizeResponse(response *extractionResponse, order shared.SourceRefOrder, sourceID string) { + if response == nil { + return + } + ordered := make([]orderedItemResponse, len(response.Items)) + for index := range response.Items { + earliest, hasEvidence := canonicalizeItem(&response.Items[index], order, sourceID) + ordered[index] = orderedItemResponse{value: response.Items[index], earliest: earliest, hasEvidence: hasEvidence} + } + sort.SliceStable(ordered, func(left, right int) bool { + if ordered[left].hasEvidence != ordered[right].hasEvidence { + return ordered[left].hasEvidence + } + if !ordered[left].hasEvidence { + return false + } + return ordered[left].earliest < ordered[right].earliest + }) + for index := range ordered { + response.Items[index] = ordered[index].value + } +} + +func canonicalizeItem(item *itemResponse, order shared.SourceRefOrder, sourceID string) (int, bool) { + if item == nil { + return 0, false + } + item.Name = identity.NormalizeDisplay(item.Name) + refs := order.Canonicalize(canonicalSourceRefs(item.SourceRefs, sourceID)) + item.SourceRefs = itemResponseRefs(refs) + return order.EarliestValid(refs) +} + +func canonicalItemRegistry(response extractionResponse, sourceID string) dnd.ItemRegistry { + if response.Items == nil { + return dnd.ItemRegistry{Items: nil} + } + items := make([]dnd.Item, len(response.Items)) + for index, item := range response.Items { + refs := canonicalSourceRefs(item.SourceRefs, sourceID) + items[index] = dnd.Item{ + ID: identity.DeriveID(item.Name), + Name: item.Name, + SourceRefs: refs, + } + } + return dnd.ItemRegistry{Items: items} +} + +func canonicalSourceRefs(values []itemSourceRefResponse, sourceID string) []source.SourceRef { + if values == nil { + return nil + } + refs := make([]source.SourceRef, len(values)) + for index, value := range values { + refs[index] = source.SourceRef{SourceID: sourceID, StartUnitID: value.StartUnitID, EndUnitID: value.EndUnitID} + } + return refs +} + +func itemResponseRefs(values []source.SourceRef) []itemSourceRefResponse { + if values == nil { + return nil + } + refs := make([]itemSourceRefResponse, len(values)) + for index, value := range values { + refs[index] = itemSourceRefResponse{StartUnitID: value.StartUnitID, EndUnitID: value.EndUnitID} + } + return refs +} diff --git a/internal/modules/dnd/extract/itemregistry/extractor.go b/internal/modules/dnd/extract/itemregistry/extractor.go new file mode 100644 index 0000000..ca6a57e --- /dev/null +++ b/internal/modules/dnd/extract/itemregistry/extractor.go @@ -0,0 +1,152 @@ +// Package itemregistry extracts source-grounded D&D item-registry candidates. +package itemregistry + +import ( + "context" + "fmt" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" +) + +const ( + Key = "dnd/item-registry" + mappingPolicy = "dnd.item_registry.extract_mapping.v1" +) + +var requiredCapabilities = []string{"chunks", "source.transcript"} +var providedCapabilities = []string{"dnd.item_registry"} + +var referenceSlotDescriptions = shared.ReferenceSlotDescriptions{ + Glossary: "Optional campaign glossary reference material used only for item disambiguation.", + Party: "Optional party roster reference material used only for item disambiguation.", + Players: "Optional player list reference material used only for item disambiguation.", + Roster: "Deprecated alias for party roster reference material used only for item disambiguation.", +} + +func referenceSlots() []contracts.ReferenceSlot { + return shared.ReferenceSlots(referenceSlotDescriptions) +} + +var _ contracts.Extractor[dnd.ItemRegistry] = (*Extractor)(nil) +var _ contracts.ManifestMetadataProvider = (*Extractor)(nil) +var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil) + +type Options struct{} + +type Extractor struct { + llm contracts.StructuredLLMClient + promptSHA string + responseSchemaSHA string +} + +func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { + if llmClient == nil { + return nil, extractorErrorf("LLM client must not be nil") + } + if len(references) > 1 { + return nil, extractorErrorf("at most one reference set may be supplied") + } + promptSHA, err := promptAssetMetadata() + if err != nil { + return nil, extractorErrorf("load prompt metadata: %w", err) + } + responseSchema, err := loadResponseSchema() + if err != nil { + return nil, extractorErrorf("load response schema: %w", err) + } + return &Extractor{llm: llmClient, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil +} + +func (e *Extractor) Key() string { return Key } + +func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() } + +func (e *Extractor) ManifestMetadata() map[string]any { + if e == nil { + return nil + } + return map[string]any{ + "prompt_id": PromptID, + "prompt_version": SchemaVersion, + "prompt_sha256": e.promptSHA, + "response_schema_key": string(ResponseSchemaKey), + "response_schema_id": ResponseSchemaID, + "response_schema_name": ResponseSchemaName, + "response_schema_version": SchemaVersion, + "response_schema_sha256": e.responseSchemaSHA, + "identity_policy": identity.Policy, + "mapping_policy": mappingPolicy, + } +} + +func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint { + if e == nil { + return nil + } + return []pipeline.CheckpointFingerprint{ + {Name: "prompt", Value: e.promptSHA}, + {Name: "response_schema", Value: e.responseSchemaSHA}, + {Name: "identity_policy", Value: identity.Policy}, + {Name: "mapping_policy", Value: mappingPolicy}, + } +} + +func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.ItemRegistry], error) { + if e == nil { + return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("extractor must not be nil") + } + if e.llm == nil { + return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("LLM client must not be nil") + } + sourceInput, err := shared.PrepareChunkExtraction(ctx, req) + if err != nil { + return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("%w", err) + } + order := shared.NewSourceRefOrder(req.Source) + + var response extractionResponse + if _, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ + StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion, + ProfileID: req.LLMProfile, SessionID: req.SessionID, + Inputs: shared.PromptInputs(sourceInput, req.References), + }, &response); err != nil { + return contracts.TypedExtractionResult[dnd.ItemRegistry]{}, extractorErrorf("complete structured output: %w", err) + } + canonicalizeResponse(&response, order, req.Source.ID) + return contracts.TypedExtractionResult[dnd.ItemRegistry]{Value: canonicalItemRegistry(response, req.Source.ID)}, nil +} + +func ModuleSpec() pipeline.ModuleSpec { + return pipeline.ModuleSpec{ + Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, + Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), + ArtifactKind: dnd.ItemRegistryKind, ReferenceSlots: referenceSlots(), + } +} + +func Register(registry *pipeline.ExtractorRegistry) error { + return pipeline.RegisterExtractorBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Extractor[dnd.ItemRegistry], error) { + options, err := DecodeOptions(request.Options) + if err != nil { + return nil, err + } + return New(request.Dependencies.LLM, options, request.References) + }) +} + +func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err } + +func DecodeOptions(options map[string]any) (Options, error) { + if err := pipeline.RejectUnknownOptions(options); err != nil { + return Options{}, extractorErrorf("%w", err) + } + return Options{}, nil +} + +func extractorErrorf(format string, args ...any) error { + return fmt.Errorf("dnd item registry extractor: "+format, args...) +} diff --git a/internal/modules/dnd/extract/itemregistry/extractor_test.go b/internal/modules/dnd/extract/itemregistry/extractor_test.go new file mode 100644 index 0000000..ea30e7c --- /dev/null +++ b/internal/modules/dnd/extract/itemregistry/extractor_test.go @@ -0,0 +1,108 @@ +package itemregistry + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity" +) + +func TestExtractMapsItemsWithOwnedEvidenceAndDeterministicOrder(t *testing.T) { + client := &fakeItemsLLMClient{response: extractionResponse{Items: []itemResponse{ + {Name: " Gold Pieces ", SourceRefs: responseSourceRefs(3, 3)}, + {Name: "Star Compass", SourceRefs: []itemSourceRefResponse{{StartUnitID: 2, EndUnitID: 2}, {StartUnitID: 1, EndUnitID: 1}, {StartUnitID: 1, EndUnitID: 1}}}, + }}} + result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest()) + if err != nil { + t.Fatalf("Extract() error = %v, want nil", err) + } + refs := []source.SourceRef{{SourceID: "session-items", StartUnitID: 1, EndUnitID: 1}, {SourceID: "session-items", StartUnitID: 2, EndUnitID: 2}} + want := dnd.ItemRegistry{Items: []dnd.Item{ + {ID: identity.DeriveID("Star Compass"), Name: "Star Compass", SourceRefs: refs}, + {ID: identity.DeriveID("Gold Pieces"), Name: "Gold Pieces", SourceRefs: []source.SourceRef{{SourceID: "session-items", StartUnitID: 3, EndUnitID: 3}}}, + }} + if !reflect.DeepEqual(result.Value, want) { + t.Fatalf("Value = %#v, want %#v", result.Value, want) + } + result.Value.Items[0].SourceRefs[0].StartUnitID = 99 + for _, item := range client.response.Items { + for _, ref := range item.SourceRefs { + if ref.StartUnitID == 99 { + t.Fatal("result source references alias the model response") + } + } + } +} + +func TestExtractKeepsAliasCandidatesSeparate(t *testing.T) { + client := &fakeItemsLLMClient{response: extractionResponse{Items: []itemResponse{ + {Name: "Star Compass", SourceRefs: responseSourceRefs(1, 1)}, + {Name: "Compass of the Stars", SourceRefs: responseSourceRefs(2, 2)}, + }}} + result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest()) + if err != nil || len(result.Value.Items) != 2 || result.Value.Items[0].ID == result.Value.Items[1].ID { + t.Fatalf("Extract() = %#v, %v; want separate alias candidates", result, err) + } +} + +func TestExtractPreservesInvalidCandidatesForValidators(t *testing.T) { + client := &fakeItemsLLMClient{content: []byte(`{"items":[{"name":"","source_refs":[{"start_unit_id":0,"end_unit_id":-1}]}]}`)} + result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest()) + if err != nil { + t.Fatalf("Extract() error = %v, want nil", err) + } + item := result.Value.Items[0] + if item.ID != "" || item.Name != "" || !reflect.DeepEqual(item.SourceRefs, []source.SourceRef{{SourceID: "session-items", StartUnitID: 0, EndUnitID: -1}}) { + t.Fatalf("item = %#v, want invalid candidate preserved", item) + } +} + +func TestExtractPassesReferencesWithoutTreatingThemAsEvidence(t *testing.T) { + client := &fakeItemsLLMClient{response: extractionResponse{Items: []itemResponse{}}} + req := extractionRequest() + req.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ + "glossary": {Slot: contracts.ReferenceSlot{Name: "glossary"}, Items: []contracts.ReferenceItem{{SlotName: "glossary", Content: []byte("Star Compass: heirloom")}}}, + }} + if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil { + t.Fatalf("Extract() error = %v", err) + } + inputs := client.requests[0].Inputs + if string(inputs["glossary"].Content) != "Star Compass: heirloom" || strings.Contains(string(inputs["transcript"].Content), "heirloom") { + t.Fatalf("prompt inputs = %#v, want separated reference material", inputs) + } +} + +func TestExtractHandlesCancellationAndFailures(t *testing.T) { + request := extractionRequest() + canceled, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := newExtractor(t, &fakeItemsLLMClient{}).Extract(canceled, request); err == nil || !errors.Is(err, context.Canceled) || !strings.Contains(err.Error(), "dnd item registry") { + t.Fatalf("canceled Extract() error = %v, want contextual cancellation", err) + } + for _, test := range []struct { + name string + extractor *Extractor + req contracts.TypedExtractionRequest + want string + }{ + {name: "nil extractor", extractor: nil, req: request, want: "extractor"}, + {name: "nil client", extractor: &Extractor{}, req: request, want: "LLM client"}, + {name: "preflight", extractor: newExtractor(t, &fakeItemsLLMClient{}), req: mismatchedSourceInputRequest(request), want: "must match chunk"}, + } { + t.Run(test.name, func(t *testing.T) { + if _, err := test.extractor.Extract(context.Background(), test.req); err == nil || !strings.Contains(err.Error(), "dnd item registry") || !strings.Contains(err.Error(), test.want) { + t.Fatalf("Extract() error = %v, want local context", err) + } + }) + } + _, err := newExtractor(t, &fakeItemsLLMClient{err: errors.New("provider unavailable")}).Extract(context.Background(), request) + if err == nil || !strings.Contains(err.Error(), "dnd item registry") || !strings.Contains(err.Error(), "provider unavailable") { + t.Fatalf("provider error = %v, want contextual provider error", err) + } +} diff --git a/internal/modules/dnd/extract/itemregistry/model.go b/internal/modules/dnd/extract/itemregistry/model.go new file mode 100644 index 0000000..eaa675f --- /dev/null +++ b/internal/modules/dnd/extract/itemregistry/model.go @@ -0,0 +1,15 @@ +package itemregistry + +type extractionResponse struct { + Items []itemResponse `json:"items"` +} + +type itemResponse struct { + Name string `json:"name"` + SourceRefs []itemSourceRefResponse `json:"source_refs"` +} + +type itemSourceRefResponse struct { + StartUnitID int `json:"start_unit_id"` + EndUnitID int `json:"end_unit_id"` +} diff --git a/internal/modules/dnd/extract/itemregistry/prompt_assets.go b/internal/modules/dnd/extract/itemregistry/prompt_assets.go new file mode 100644 index 0000000..b5ab9d7 --- /dev/null +++ b/internal/modules/dnd/extract/itemregistry/prompt_assets.go @@ -0,0 +1,74 @@ +package itemregistry + +import ( + "fmt" + "io/fs" + "sync" + + rootassets "gitea.maximumdirect.net/eric/notarius/assets" + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" +) + +const promptAssetRoot = "assets/prompts" + +var promptAssetManifest = shared.PromptAssetManifest{ + ModuleDir: PromptID, + ModuleFiles: []promptfs.ModulePromptFile{ + {Name: "prompt.yaml", Path: "prompts/prompt.yaml"}, + {Name: "instructions.md", Path: "prompts/instructions.md"}, + }, + SharedFiles: []string{ + "common-dnd-system.md", + "common-dnd-extraction-evidence.md", + "common-dnd-identity.md", + "common-dnd-transcript-chunk.md", + "common-dnd-references.md", + }, +} + +func moduleAssetFS() (fs.FS, error) { + assets, err := fs.Sub(rootassets.FS(), "dnd/item-registry/extract") + if err != nil { + return nil, fmt.Errorf("scope item extraction assets: %w", err) + } + return assets, nil +} + +func RegisterPromptAssets(registry *llm.AssetRegistry) error { + assets, err := moduleAssetFS() + if err != nil { + return err + } + promptFS, err := promptAssetManifest.PromptFS(assets) + if err != nil { + return fmt.Errorf("prepare item prompt assets: %w", err) + } + if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil { + return err + } + schemas, err := fs.Sub(assets, "schemas") + if err != nil { + return fmt.Errorf("scope item extraction schemas: %w", err) + } + return registry.RegisterSchemaFS(schemas, ".") +} + +func promptAssetMetadata() (string, error) { + promptAssetHashOnce.Do(func() { + assets, err := moduleAssetFS() + if err != nil { + promptAssetHashErr = err + return + } + promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets) + }) + return promptAssetHash, promptAssetHashErr +} + +var ( + promptAssetHashOnce sync.Once + promptAssetHash string + promptAssetHashErr error +) diff --git a/internal/modules/dnd/extract/itemregistry/prompt_assets_test.go b/internal/modules/dnd/extract/itemregistry/prompt_assets_test.go new file mode 100644 index 0000000..3c99739 --- /dev/null +++ b/internal/modules/dnd/extract/itemregistry/prompt_assets_test.go @@ -0,0 +1,46 @@ +package itemregistry + +import ( + "context" + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + "gitea.maximumdirect.net/eric/promptkit" +) + +func TestRegisterPromptAssetsPreparesItemPrompt(t *testing.T) { + registry := llm.NewAssetRegistry() + if err := RegisterPromptAssets(registry); err != nil { + t.Fatalf("RegisterPromptAssets() error = %v", err) + } + options, err := registry.PromptKitOptions() + if err != nil { + t.Fatalf("PromptKitOptions() error = %v", err) + } + options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ID: "item-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "item-test-model"}))) + engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...) + if err != nil { + t.Fatalf("NewEngine() error = %v", err) + } + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-test-profile", Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline(`{"units":[{"sentinel":"item-transcript"}]}`), "players": promptkit.Inline("item-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "), + }}) + if err != nil { + t.Fatalf("Prepare() error = %v", err) + } + if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_item_registry_llm.v1.json" { + t.Fatalf("prepared prompt = %#v, want item prompt identity and schema wiring", prepared) + } + content := make([]string, len(prepared.Messages)) + for index, message := range prepared.Messages { + content[index] = message.Content + } + rendered := strings.Join(content, "\n") + for _, policy := range []string{"named unique items", "concrete reusable item types", "currency denomination", "loot", "generic weapons", "Do not use capitalization as an eligibility test"} { + if !strings.Contains(rendered, policy) { + t.Fatalf("rendered prompt omits item eligibility policy %q", policy) + } + } +} diff --git a/internal/modules/dnd/extract/itemregistry/registry_test.go b/internal/modules/dnd/extract/itemregistry/registry_test.go new file mode 100644 index 0000000..cd05241 --- /dev/null +++ b/internal/modules/dnd/extract/itemregistry/registry_test.go @@ -0,0 +1,60 @@ +package itemregistry + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity" +) + +func TestModuleRegistrationMetadataAndRedaction(t *testing.T) { + if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") { + t.Fatalf("New(nil) error = %v, want client rejection", err) + } + if _, err := New(&fakeItemsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one") { + t.Fatalf("New() error = %v, want reference-set rejection", err) + } + want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.item_registry"}, ArtifactKind: dnd.ItemRegistryKind, ReferenceSlots: referenceSlots()} + if got := ModuleSpec(); !reflect.DeepEqual(got, want) { + t.Fatalf("ModuleSpec() = %#v, want %#v", got, want) + } + registry := pipeline.NewExtractorRegistry() + if err := Register(registry); err != nil { + t.Fatalf("Register() error = %v", err) + } + if got, ok := registry.Spec(Key); !ok || !reflect.DeepEqual(got, want) { + t.Fatalf("registry spec = %#v, present = %t", got, ok) + } + if err := Register(nil); err == nil || !strings.Contains(err.Error(), "extractor registry") { + t.Fatalf("Register(nil) error = %v, want registry rejection", err) + } + if _, err := DecodeOptions(map[string]any{"unknown": true}); err == nil { + t.Fatal("DecodeOptions() accepted unknown option") + } + references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ + "glossary": {Items: []contracts.ReferenceItem{{Content: []byte("secret-campaign-context")}}}, + }} + metadata := newExtractor(t, &fakeItemsLLMClient{}, references).ManifestMetadata() + for key, value := range map[string]string{ + "prompt_id": PromptID, "prompt_version": SchemaVersion, + "response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID, + "response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion, + "identity_policy": identity.Policy, "mapping_policy": mappingPolicy, + } { + if metadata[key] != value { + t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], value) + } + } + encoded, err := json.Marshal(metadata) + if err != nil || strings.Contains(string(encoded), "secret-campaign-context") || strings.Contains(string(encoded), "session-items") { + t.Fatalf("manifest metadata = %s, want static redacted provenance only", encoded) + } + if got := newExtractor(t, &fakeItemsLLMClient{}).CheckpointFingerprints(); len(got) != 4 || got[0].Name != "prompt" || got[1].Name != "response_schema" || got[2].Value != identity.Policy || got[3].Value != mappingPolicy { + t.Fatalf("CheckpointFingerprints() = %#v", got) + } +} diff --git a/internal/modules/dnd/extract/itemregistry/schema.go b/internal/modules/dnd/extract/itemregistry/schema.go new file mode 100644 index 0000000..e2addf3 --- /dev/null +++ b/internal/modules/dnd/extract/itemregistry/schema.go @@ -0,0 +1,25 @@ +package itemregistry + +import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + +const ( + PromptID = "dnd.item_registry" + ResponseSchemaKey = llm.ResponseSchemaKey("dnd_item_registry_llm") + ResponseSchemaID = "notarius.dnd.item_registry.llm" + ResponseSchemaName = "notarius_dnd_item_registry_llm_v1" + SchemaVersion = "v1" +) + +func loadResponseSchema() (llm.ResponseSchema, error) { + assets, err := moduleAssetFS() + if err != nil { + return llm.ResponseSchema{}, err + } + return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{ + Key: ResponseSchemaKey, + ID: ResponseSchemaID, + Version: SchemaVersion, + Name: ResponseSchemaName, + AssetPath: "schemas/dnd_item_registry_llm.v1.json", + }) +} diff --git a/internal/modules/dnd/extract/itemregistry/schema_test.go b/internal/modules/dnd/extract/itemregistry/schema_test.go new file mode 100644 index 0000000..35da5fa --- /dev/null +++ b/internal/modules/dnd/extract/itemregistry/schema_test.go @@ -0,0 +1,25 @@ +package itemregistry + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestItemResponseSchemaIsPrivateAndStructural(t *testing.T) { + schema, err := loadResponseSchema() + if err != nil { + t.Fatalf("loadResponseSchema() error = %v", err) + } + if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) { + t.Fatalf("schema = %#v, want item response schema", schema) + } + var document map[string]any + if err := json.Unmarshal(schema.JSONSchema, &document); err != nil { + t.Fatal(err) + } + encoded, err := json.Marshal(document) + if err != nil || strings.Contains(string(encoded), `"id"`) || !strings.Contains(string(encoded), `"additionalProperties":false`) { + t.Fatalf("schema = %s, want strict private response without durable ID", encoded) + } +} diff --git a/internal/modules/dnd/extract/itemregistry/test_helpers_test.go b/internal/modules/dnd/extract/itemregistry/test_helpers_test.go new file mode 100644 index 0000000..21c8cc3 --- /dev/null +++ b/internal/modules/dnd/extract/itemregistry/test_helpers_test.go @@ -0,0 +1,92 @@ +package itemregistry + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" +) + +func extractionRequest() contracts.TypedExtractionRequest { + doc := sourceDocument() + chunk := &source.Chunk{ + ID: "session-items:chunk:0", SourceID: doc.ID, Index: 0, + Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 3}, + Content: []byte(`{"units":[1,2,3]}`), MediaType: "application/json", + Units: append([]source.SourceUnit(nil), doc.Units...), + } + return contracts.TypedExtractionRequest{ + Source: doc, Chunk: chunk, + SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-items.json"), + SessionID: "item-session", LLMProfile: "item-profile", + } +} + +func sourceDocument() *source.SourceDocument { + return &source.SourceDocument{ID: "session-items", Kind: "transcript", Format: "application/json", Digest: "sha256:test", Units: []source.SourceUnit{ + {ID: 1, Kind: "transcript_segment", Text: "The party finds a rope."}, + {ID: 2, Kind: "transcript_segment", Text: "They recover the Star Compass."}, + {ID: 3, Kind: "transcript_segment", Text: "The chest contains gold pieces."}, + }} +} + +func responseSourceRefs(startUnitID, endUnitID int) []itemSourceRefResponse { + return []itemSourceRefResponse{{StartUnitID: startUnitID, EndUnitID: endUnitID}} +} + +func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor { + t.Helper() + extractor, err := New(client, Options{}, references...) + if err != nil { + t.Fatalf("New() error = %v, want nil", err) + } + return extractor +} + +func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest { + req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "file:///other.json") + return req +} + +type fakeItemsLLMClient struct { + response extractionResponse + content []byte + err error + requests []contracts.StructuredCompletionRequest +} + +func (client *fakeItemsLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { + client.requests = append(client.requests, cloneStructuredCompletionRequest(req)) + if err := ctx.Err(); err != nil { + return contracts.StructuredCompletionResponse{}, err + } + if client.err != nil { + return contracts.StructuredCompletionResponse{}, client.err + } + target, ok := out.(*extractionResponse) + if !ok { + return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target") + } + content := append([]byte(nil), client.content...) + if len(content) != 0 { + if err := json.Unmarshal(content, target); err != nil { + return contracts.StructuredCompletionResponse{}, err + } + } else { + *target = client.response + var err error + content, err = json.Marshal(client.response) + if err != nil { + return contracts.StructuredCompletionResponse{}, err + } + } + return contracts.StructuredCompletionResponse{Content: content}, nil +} + +func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest { + req.Inputs = req.Inputs.Clone() + return req +} diff --git a/internal/modules/dnd/validate/itemregistry/shape/validator.go b/internal/modules/dnd/validate/itemregistry/shape/validator.go new file mode 100644 index 0000000..cb7cdd2 --- /dev/null +++ b/internal/modules/dnd/validate/itemregistry/shape/validator.go @@ -0,0 +1,97 @@ +// Package shape validates the required extracted item fields. +package shape + +import ( + "context" + "fmt" + "strings" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics" +) + +const ( + Key = "extract/dnd/item-registry/shape" + ReasonCode = "invalid_item_shape" + policy = "dnd.item_registry.validator.shape.v1" +) + +type Options struct{} +type Validator struct{} + +var _ contracts.TypedValidator[dnd.ItemRegistry] = (*Validator)(nil) +var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil) + +func New(Options) *Validator { return &Validator{} } +func (v *Validator) Name() string { return Key } +func (v *Validator) ExecutionClass() contracts.ExecutionClass { + return contracts.ExecutionClassDeterministic +} +func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint { + return []pipeline.CheckpointFingerprint{{Name: "policy", Value: policy}} +} + +func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.ItemRegistry]) (contracts.ValidationResult, error) { + issues := issuesFor(req.Value) + if len(issues) > 0 { + return rejection(diagnostics.Aggregate("invalid item shape", issues)), nil + } + return contracts.ValidationResult{Approved: true}, nil +} + +func Validate(value dnd.ItemRegistry) error { + issues := issuesFor(value) + if len(issues) == 0 { + return nil + } + return fmt.Errorf("%s", diagnostics.Aggregate("invalid item shape", issues)) +} + +func issuesFor(value dnd.ItemRegistry) []string { + if value.Items == nil { + return []string{"items must be present"} + } + issues := make([]string, 0) + for index, item := range value.Items { + prefix := fmt.Sprintf("items[%d]", index) + if strings.TrimSpace(item.ID) == "" { + issues = append(issues, prefix+".id must not be empty") + } + if strings.TrimSpace(item.Name) == "" { + issues = append(issues, prefix+".name must not be empty") + } + if len(item.SourceRefs) == 0 { + issues = append(issues, prefix+".source_refs must not be empty") + } + } + return issues +} + +func Spec() pipeline.ValidatorSpec { + return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic} +} + +func Register(registry *pipeline.ValidatorRegistry) error { + return pipeline.RegisterTypedValidatorBuilder(registry, dnd.ItemRegistryKind, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.TypedValidator[dnd.ItemRegistry], error) { + options, err := DecodeOptions(request.Options) + if err != nil { + return nil, err + } + return New(options), nil + }) +} + +func DecodeOptions(options map[string]any) (Options, error) { + if err := pipeline.RejectUnknownOptions(options); err != nil { + return Options{}, err + } + return Options{}, nil +} + +func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err } + +func rejection(message string) contracts.ValidationResult { + return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message} +} diff --git a/internal/modules/dnd/validate/itemregistry/shape/validator_test.go b/internal/modules/dnd/validate/itemregistry/shape/validator_test.go new file mode 100644 index 0000000..24c1418 --- /dev/null +++ b/internal/modules/dnd/validate/itemregistry/shape/validator_test.go @@ -0,0 +1,44 @@ +package shape + +import ( + "context" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" +) + +func TestValidatorRejectsMalformedItemsWithoutMutation(t *testing.T) { + value := dnd.ItemRegistry{Items: []dnd.Item{{Name: "", SourceRefs: []source.SourceRef{}}}} + before := value + result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Value: value}) + if err != nil || result.Approved || result.ReasonCode != ReasonCode || !strings.Contains(result.Message, "id must not be empty") || !strings.Contains(result.Message, "source_refs must not be empty") || !reflect.DeepEqual(value, before) { + t.Fatalf("Validate() = %#v, %v; want non-mutating shape rejection", result, err) + } + result, err = New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Value: dnd.ItemRegistry{}}) + if err != nil || result.Approved || !strings.Contains(result.Message, "items must be present") { + t.Fatalf("missing items = %#v, %v; want rejection", result, err) + } +} + +func TestValidatorApprovesAndRegisters(t *testing.T) { + value := dnd.ItemRegistry{Items: []dnd.Item{{ID: "item", Name: "Rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}}}} + result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Value: value}) + if err != nil || !result.Approved { + t.Fatalf("Validate() = %#v, %v; want approval", result, err) + } + registry := pipeline.NewValidatorRegistry() + if err := Register(registry); err != nil { + t.Fatalf("Register() error = %v", err) + } + if got, ok := registry.Spec(Key); !ok || got != Spec() || got.ExecutionClass != contracts.ExecutionClassDeterministic { + t.Fatalf("registered spec = %#v, ok = %t", got, ok) + } + if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil { + t.Fatal("DecodeOptions() accepted unknown options") + } +} diff --git a/internal/modules/dnd/validate/itemregistry/source_refs/validator.go b/internal/modules/dnd/validate/itemregistry/source_refs/validator.go new file mode 100644 index 0000000..93236a6 --- /dev/null +++ b/internal/modules/dnd/validate/itemregistry/source_refs/validator.go @@ -0,0 +1,101 @@ +// Package sourcerefs validates item citations against the current source. +package sourcerefs + +import ( + "context" + "fmt" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics" + itemshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/shape" +) + +const ( + Key = "extract/dnd/item-registry/source_refs" + ReasonCode = "invalid_item_source_refs" + policy = "dnd.item_registry.validator.source_refs.v1" +) + +type Options struct{} +type Validator struct{} + +var _ contracts.TypedValidator[dnd.ItemRegistry] = (*Validator)(nil) +var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil) + +func New(Options) *Validator { return &Validator{} } +func (v *Validator) Name() string { return Key } +func (v *Validator) ExecutionClass() contracts.ExecutionClass { + return contracts.ExecutionClassDeterministic +} +func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint { + return []pipeline.CheckpointFingerprint{{Name: "policy", Value: policy}} +} + +func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.ItemRegistry]) (contracts.ValidationResult, error) { + if req.Stage == string(pipeline.StageExtract) && req.Chunk == nil { + return contracts.ValidationResult{}, fmt.Errorf("item source-reference validator requires the current extraction chunk") + } + if err := itemshape.Validate(req.Value); err != nil { + return contracts.ValidationResult{Approved: true}, nil + } + index := source.NewDocumentIndex(req.Source) + issues := make([]string, 0) + for itemIndex, item := range req.Value.Items { + for refIndex, ref := range item.SourceRefs { + if err := index.ValidateRef(ref); err != nil { + issues = append(issues, fmt.Sprintf("items[%d].source_refs[%d]: %s", itemIndex, refIndex, diagnostics.Truncate(err.Error()))) + continue + } + if req.Stage == string(pipeline.StageExtract) && !chunkContainsRef(req.Chunk, ref) { + issues = append(issues, fmt.Sprintf("items[%d].source_refs[%d]: source reference is outside the current extraction chunk", itemIndex, refIndex)) + } + } + } + if len(issues) == 0 { + return contracts.ValidationResult{Approved: true}, nil + } + return rejection(diagnostics.Aggregate("invalid item source references", issues)), nil +} + +func chunkContainsRef(chunk *source.Chunk, ref source.SourceRef) bool { + if chunk == nil || ref.SourceID != chunk.SourceID { + return false + } + startFound := false + endFound := false + for _, unit := range chunk.Units { + startFound = startFound || unit.ID == ref.StartUnitID + endFound = endFound || unit.ID == ref.EndUnitID + } + return startFound && endFound +} + +func Spec() pipeline.ValidatorSpec { + return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic} +} + +func Register(registry *pipeline.ValidatorRegistry) error { + return pipeline.RegisterTypedValidatorBuilder(registry, dnd.ItemRegistryKind, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.TypedValidator[dnd.ItemRegistry], error) { + options, err := DecodeOptions(request.Options) + if err != nil { + return nil, err + } + return New(options), nil + }) +} + +func DecodeOptions(options map[string]any) (Options, error) { + if err := pipeline.RejectUnknownOptions(options); err != nil { + return Options{}, err + } + return Options{}, nil +} + +func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err } + +func rejection(message string) contracts.ValidationResult { + return contracts.ValidationResult{Approved: false, ReasonCode: ReasonCode, Message: message} +} diff --git a/internal/modules/dnd/validate/itemregistry/source_refs/validator_test.go b/internal/modules/dnd/validate/itemregistry/source_refs/validator_test.go new file mode 100644 index 0000000..b1edc31 --- /dev/null +++ b/internal/modules/dnd/validate/itemregistry/source_refs/validator_test.go @@ -0,0 +1,60 @@ +package sourcerefs + +import ( + "context" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" +) + +func TestValidatorRejectsInvalidAndOutOfChunkReferencesWithoutMutation(t *testing.T) { + value := validItemRegistry() + value.Items[0].SourceRefs = []source.SourceRef{{SourceID: "foreign", StartUnitID: 1, EndUnitID: 1}} + before := value + result, err := New(Options{}).Validate(context.Background(), request(validDocument(), value)) + if err != nil || result.Approved || result.ReasonCode != ReasonCode || !strings.Contains(result.Message, "items[0].source_refs[0]") || !reflect.DeepEqual(value, before) { + t.Fatalf("Validate() = %#v, %v; want source-reference rejection", result, err) + } + doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1}, {ID: 2}}} + value = validItemRegistry() + value.Items[0].SourceRefs[0].EndUnitID = 2 + result, err = New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Stage: string(pipeline.StageExtract), Source: doc, Chunk: &source.Chunk{SourceID: "session", Units: []source.SourceUnit{{ID: 1}}}, Value: value}) + if err != nil || result.Approved || !strings.Contains(result.Message, "outside the current extraction chunk") { + t.Fatalf("out-of-chunk evidence = %#v, %v", result, err) + } +} + +func TestValidatorDefersMalformedShapeAndRegisters(t *testing.T) { + result, err := New(Options{}).Validate(context.Background(), request(nil, dnd.ItemRegistry{Items: []dnd.Item{{Name: "Missing"}}})) + if err != nil || !result.Approved { + t.Fatalf("malformed shape = %#v, %v; want deferral", result, err) + } + _, err = New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Stage: string(pipeline.StageExtract), Source: validDocument(), Value: validItemRegistry()}) + if err == nil || !strings.Contains(err.Error(), "requires the current extraction chunk") { + t.Fatalf("missing chunk error = %v", err) + } + registry := pipeline.NewValidatorRegistry() + if err := Register(registry); err != nil { + t.Fatalf("Register() error = %v", err) + } + if got, ok := registry.Spec(Key); !ok || got != Spec() || got.ExecutionClass != contracts.ExecutionClassDeterministic { + t.Fatalf("registered spec = %#v, ok = %t", got, ok) + } +} + +func request(doc *source.SourceDocument, value dnd.ItemRegistry) contracts.TypedValidationRequest[dnd.ItemRegistry] { + return contracts.TypedValidationRequest[dnd.ItemRegistry]{Source: doc, Value: value} +} + +func validDocument() *source.SourceDocument { + return &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Text: "The party finds rope."}}} +} + +func validItemRegistry() dnd.ItemRegistry { + return dnd.ItemRegistry{Items: []dnd.Item{{ID: "item", Name: "Rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}}}} +} diff --git a/internal/modules/dnd/validate/itemregistry/source_relatedness/validator.go b/internal/modules/dnd/validate/itemregistry/source_relatedness/validator.go new file mode 100644 index 0000000..ce6adf3 --- /dev/null +++ b/internal/modules/dnd/validate/itemregistry/source_relatedness/validator.go @@ -0,0 +1,88 @@ +// Package sourcerelatedness warns when cited source text does not mention an item. +package sourcerelatedness + +import ( + "context" + "fmt" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics" + itemshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/shape" +) + +const ( + Key = "extract/dnd/item-registry/source_relatedness" + WarningReasonCode = "item_not_near_source" + OmittedReasonCode = "item_relatedness_warnings_omitted" + policy = "dnd.item_registry.validator.source_relatedness.v1" +) + +type Options struct{} +type Validator struct{} + +var _ contracts.TypedValidator[dnd.ItemRegistry] = (*Validator)(nil) +var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil) + +func New(Options) *Validator { return &Validator{} } +func (v *Validator) Name() string { return Key } +func (v *Validator) ExecutionClass() contracts.ExecutionClass { + return contracts.ExecutionClassDeterministic +} +func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint { + return []pipeline.CheckpointFingerprint{{Name: "policy", Value: policy}} +} + +func (v *Validator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.ItemRegistry]) (contracts.ValidationResult, error) { + if err := itemshape.Validate(req.Value); err != nil { + return contracts.ValidationResult{Approved: true}, nil + } + resolver, err := shared.NewCitationResolver(req.Source) + if err != nil { + return contracts.ValidationResult{Approved: true}, nil + } + citedTexts := make([]string, len(req.Value.Items)) + for itemIndex, item := range req.Value.Items { + citedText, err := resolver.CitedText(item.SourceRefs) + if err != nil { + return contracts.ValidationResult{Approved: true}, nil + } + citedTexts[itemIndex] = citedText + } + warnings := make([]contracts.Warning, 0) + for itemIndex, item := range req.Value.Items { + if shared.ContainsTokenSequence(citedTexts[itemIndex], item.Name) { + continue + } + warnings = append(warnings, contracts.Warning{ + Scope: fmt.Sprintf("items[%d]", itemIndex), ReasonCode: WarningReasonCode, + Message: fmt.Sprintf("Item %s was not found in cited source text", diagnostics.Quote(item.Name)), + }) + } + return contracts.ValidationResult{Approved: true, Warnings: diagnostics.LimitWarnings(warnings, "items", OmittedReasonCode)}, nil +} + +func Spec() pipeline.ValidatorSpec { + return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassDeterministic} +} + +func Register(registry *pipeline.ValidatorRegistry) error { + return pipeline.RegisterTypedValidatorBuilder(registry, dnd.ItemRegistryKind, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.TypedValidator[dnd.ItemRegistry], error) { + options, err := DecodeOptions(request.Options) + if err != nil { + return nil, err + } + return New(options), nil + }) +} + +func DecodeOptions(options map[string]any) (Options, error) { + if err := pipeline.RejectUnknownOptions(options); err != nil { + return Options{}, err + } + return Options{}, nil +} + +func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err } diff --git a/internal/modules/dnd/validate/itemregistry/source_relatedness/validator_test.go b/internal/modules/dnd/validate/itemregistry/source_relatedness/validator_test.go new file mode 100644 index 0000000..147d839 --- /dev/null +++ b/internal/modules/dnd/validate/itemregistry/source_relatedness/validator_test.go @@ -0,0 +1,46 @@ +package sourcerelatedness + +import ( + "context" + "reflect" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics" +) + +func TestValidatorUsesOnlyCitedTranscriptText(t *testing.T) { + value := dnd.ItemRegistry{Items: []dnd.Item{{ID: "item", Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}}}} + doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Text: "They recover the star compass."}, {ID: 2, Text: "Unrelated text."}}} + result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Source: doc, Value: value}) + if err != nil || !result.Approved || len(result.Warnings) != 0 { + t.Fatalf("cited match = %#v, %v; want approval without warnings", result, err) + } + value.Items[0].Name = "Glossary Relic" + before := value + result, err = New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Source: doc, References: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"glossary": {Items: []contracts.ReferenceItem{{Content: []byte("Glossary Relic")}}}}}, Value: value}) + if err != nil || !result.Approved || len(result.Warnings) != 1 || result.Warnings[0].ReasonCode != WarningReasonCode || !reflect.DeepEqual(value, before) { + t.Fatalf("reference-only match = %#v, %v; want advisory warning", result, err) + } +} + +func TestValidatorBoundsWarningsAndRegisters(t *testing.T) { + items := make([]dnd.Item, diagnostics.MaxWarnings+1) + for index := range items { + items[index] = dnd.Item{ID: "item", Name: "Missing", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}} + } + result, err := New(Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Source: &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Text: "Nothing here."}}}, Value: dnd.ItemRegistry{Items: items}}) + if err != nil || !result.Approved || len(result.Warnings) != diagnostics.MaxWarnings || result.Warnings[len(result.Warnings)-1].ReasonCode != OmittedReasonCode { + t.Fatalf("bounded warnings = %#v, %v", result, err) + } + registry := pipeline.NewValidatorRegistry() + if err := Register(registry); err != nil { + t.Fatalf("Register() error = %v", err) + } + if got, ok := registry.Spec(Key); !ok || got != Spec() || got.ExecutionClass != contracts.ExecutionClassDeterministic { + t.Fatalf("registered spec = %#v, ok = %t", got, ok) + } +}