package npcoccurrences import ( "context" "encoding/json" "errors" "reflect" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" ) func TestExtractMapsEveryKindAndOrdersBySourcePosition(t *testing.T) { client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{ {Name: "Other", Kind: "other", SourceRefs: occurrenceRefs(30, 30)}, {Name: "Opponent", Kind: "combat_opponent", SourceRefs: occurrenceRefs(20, 20)}, {Name: "Ally", Kind: "combat_ally", SourceRefs: occurrenceRefs(5, 5)}, {Name: "Speaker", Kind: "dialogue", SourceRefs: append(occurrenceRefs(2, 2), occurrenceRefs(2, 2)...)}, {Name: "Present", Kind: "noncombat_presence", SourceRefs: occurrenceRefs(7, 7)}, {Name: "Mentioned", Kind: "mentioned", SourceRefs: occurrenceRefs(10, 10)}, {Name: "Invalid", Kind: "unsupported", SourceRefs: occurrenceRefs(0, 0)}, }}} references := requiredRegistryReferences(t, "Mentioned", "Speaker", "Present", "Ally", "Opponent", "Other", "Invalid") req := extractionRequest() req.References = references result, err := newExtractor(t, client, references).Extract(context.Background(), req) if err != nil { t.Fatalf("Extract() error = %v", err) } if got := occurrenceNames(result.Value); !reflect.DeepEqual(got, []string{"Mentioned", "Speaker", "Present", "Ally", "Opponent", "Other", "Invalid"}) { t.Fatalf("occurrence order = %#v", got) } if got := occurrenceKinds(result.Value); !reflect.DeepEqual(got, []dnd.NPCOccurrenceKind{ dnd.NPCOccurrenceKindMentioned, dnd.NPCOccurrenceKindDialogue, dnd.NPCOccurrenceKindNoncombatPresence, dnd.NPCOccurrenceKindCombatAlly, dnd.NPCOccurrenceKindCombatOpponent, dnd.NPCOccurrenceKindOther, "unsupported", }) { t.Fatalf("occurrence kinds = %#v", got) } if refs := result.Value.Occurrences[1].SourceRefs; !reflect.DeepEqual(refs, []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}}) { t.Fatalf("canonical source refs = %#v", refs) } if id := result.Value.Occurrences[0].NPCID; id != identity.DeriveID("Mentioned") { t.Fatalf("durable NPC ID = %q, want registry identity", id) } if invalid := result.Value.Occurrences[6]; invalid.Name != "Invalid" || invalid.Kind != "unsupported" || !reflect.DeepEqual(invalid.SourceRefs, []source.SourceRef{{SourceID: "session-alpha"}}) { t.Fatalf("invalid candidate = %#v, want preserved values with current source identity", invalid) } if len(client.requests) != 1 { t.Fatalf("LLM calls = %d, want 1", len(client.requests)) } } func TestExtractUsesDocumentOrderForReferencesAndOccurrences(t *testing.T) { client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{ {Name: "Later", Kind: "dialogue", SourceRefs: occurrenceRefs(10, 10)}, {Name: "First", Kind: "mentioned", SourceRefs: []occurrenceSourceRefResponse{ {StartUnitID: 10, EndUnitID: 10}, {StartUnitID: 30, EndUnitID: 30}, {StartUnitID: 30, EndUnitID: 30}, {StartUnitID: 999, EndUnitID: 0}, }}, {Name: "Second", Kind: "other", SourceRefs: occurrenceRefs(30, 30)}, }}} references := requiredRegistryReferences(t, "Later", "First", "Second") req := extractionRequest() req.References = references req.Source.Units = []source.SourceUnit{{ID: 30}, {ID: 10}} req.Chunk.Units = append([]source.SourceUnit(nil), req.Source.Units...) req.Chunk.Ref = source.SourceRef{SourceID: req.Source.ID, StartUnitID: 30, EndUnitID: 10} result, err := newExtractor(t, client, references).Extract(context.Background(), req) if err != nil { t.Fatalf("Extract() error = %v", err) } if got := occurrenceNames(result.Value); !reflect.DeepEqual(got, []string{"First", "Second", "Later"}) { t.Fatalf("occurrence order = %#v, want document chronology with stable equal-evidence ties", got) } refs := result.Value.Occurrences[0].SourceRefs if got := []int{refs[0].StartUnitID, refs[1].StartUnitID, refs[2].StartUnitID}; !reflect.DeepEqual(got, []int{30, 10, 999}) { t.Fatalf("source refs = %#v, want document order with exact duplicate removed", refs) } refs[0].StartUnitID = 777 for _, occurrence := range client.response.Occurrences { for _, ref := range occurrence.SourceRefs { if ref.StartUnitID == 777 { t.Fatal("result source references alias the model response") } } } } func TestNewRequiresLLMAndRejectsAmbiguousReferenceSets(t *testing.T) { if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") { t.Fatalf("New(nil) error = %v", err) } if _, err := New(&fakeOccurrencesLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") { t.Fatalf("New() error = %v", err) } } func TestExtractUsesNamesOnlyRegistryAndCurrentTranscriptEvidence(t *testing.T) { client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{{ Name: "Mira Thorn", Kind: "dialogue", SourceRefs: occurrenceRefs(10, 10), }}}} references := requiredRegistryReferences(t, "Mira Thorn", "Hooded Guard") req := extractionRequest() req.References = references if _, err := newExtractor(t, client, references).Extract(context.Background(), req); err != nil { t.Fatalf("Extract() error = %v", err) } request := client.requests[0] registry := request.Inputs[NPCRegistryReferenceSlot] if registry.Name != NPCRegistryReferenceSlot || registry.MediaType != "application/json" || string(registry.Content) != `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}` { t.Fatalf("registry prompt input = %#v, want names-only projection", registry) } for _, forbidden := range []string{"npc:sha256:", "other-session", "start_unit_id"} { if strings.Contains(string(registry.Content), forbidden) { t.Fatalf("registry prompt input leaked %q: %s", forbidden, registry.Content) } } if strings.Contains(string(request.Inputs["transcript"].Content), "other-session") { t.Fatal("transcript input contains registry evidence") } metadata, err := json.Marshal(newExtractor(t, &fakeOccurrencesLLMClient{}, references).ManifestMetadata()) if err != nil { t.Fatal(err) } if strings.Contains(string(metadata), "Mira Thorn") || strings.Contains(string(metadata), "other-session") { t.Fatalf("manifest metadata leaked registry content: %s", metadata) } } func TestExtractRequiresBoundRegistryBeforeLLMCall(t *testing.T) { client := &fakeOccurrencesLLMClient{} if _, err := newExtractor(t, client).Extract(context.Background(), extractionRequest()); err == nil || !strings.Contains(err.Error(), "NPC registry reference is required") { t.Fatalf("Extract() error = %v, want required registry failure", err) } if len(client.requests) != 0 { t.Fatalf("LLM calls = %d, want no call", len(client.requests)) } malformed := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ NPCRegistryReferenceSlot: { Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot}, Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: "application/json", Content: []byte(`{"secret":"private source"}`)}}, }, }} if _, err := New(client, Options{}, malformed); err == nil || !strings.Contains(err.Error(), "prepare NPC registry") || strings.Contains(err.Error(), "private source") { t.Fatalf("New() error = %v, want content-safe malformed registry error", err) } if len(client.requests) != 0 { t.Fatalf("LLM calls = %d, want no call", len(client.requests)) } } func TestExtractRejectsUnknownNamesWithoutPartialResult(t *testing.T) { references := requiredRegistryReferences(t, "Mira Thorn") client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{ {Name: "Mira Thorn", Kind: "dialogue", SourceRefs: occurrenceRefs(10, 10)}, {Name: "Unknown NPC", Kind: "mentioned", SourceRefs: occurrenceRefs(20, 20)}, }}} req := extractionRequest() req.References = references result, err := newExtractor(t, client, references).Extract(context.Background(), req) if err == nil || !strings.Contains(err.Error(), "name is not in the NPC registry") { t.Fatalf("Extract() error = %v, want unknown name failure", err) } if len(result.Value.Occurrences) != 0 { t.Fatalf("Extract() returned partial result = %#v", result.Value) } } func TestExtractCanonicalizesComparisonEquivalentRegistryNames(t *testing.T) { references := requiredRegistryReferences(t, "Mira Thorn") client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{{ Name: " mIRA\u2003thorn ", Kind: "dialogue", SourceRefs: occurrenceRefs(10, 10), }}}} req := extractionRequest() req.References = references result, err := newExtractor(t, client, references).Extract(context.Background(), req) if err != nil { t.Fatalf("Extract() error = %v", err) } occurrence := result.Value.Occurrences[0] if occurrence.Name != "Mira Thorn" || occurrence.NPCID != identity.DeriveID("Mira Thorn") { t.Fatalf("canonical occurrence = %#v", occurrence) } } func TestExtractResolvesGeneratedRegistryAtOperationTime(t *testing.T) { client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{}}} references := requiredRegistryReferences(t, "Mira Thorn") req := extractionRequest() req.References = references extractor := newExtractor(t, client) if _, err := extractor.Extract(context.Background(), req); err != nil { t.Fatalf("Extract() error = %v", err) } if input := client.requests[0].Inputs[NPCRegistryReferenceSlot]; string(input.Content) != `{"npcs":[{"name":"Mira Thorn"}]}` || input.OriginURI != "" { t.Fatalf("generated registry prompt input = %#v", input) } metadata := extractor.ManifestMetadata() if _, ok := metadata["npc_registry_digest"]; ok { t.Fatalf("operation-time registry leaked into static metadata: %#v", metadata) } } func TestExtractAcceptsEmptyBoundRegistryAndEmptyResponse(t *testing.T) { content, err := npccodec.New().Encode(dnd.NPCRegistry{NPCs: []dnd.NPC{}}) if err != nil { t.Fatal(err) } references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ NPCRegistryReferenceSlot: { Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot}, Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: npccodec.MediaType, Content: content}}, }, }} client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{}}} req := extractionRequest() req.References = references result, err := newExtractor(t, client, references).Extract(context.Background(), req) if err != nil || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 { t.Fatalf("Extract() = %#v, %v; want present empty occurrences", result, err) } } func TestExtractRejectsNonemptyResponseForEmptyBoundRegistry(t *testing.T) { content, err := npccodec.New().Encode(dnd.NPCRegistry{NPCs: []dnd.NPC{}}) if err != nil { t.Fatal(err) } references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{NPCRegistryReferenceSlot: { Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot}, Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: npccodec.MediaType, Content: content}}, }}} client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{{Name: "Mira Thorn", Kind: "dialogue", SourceRefs: occurrenceRefs(10, 10)}}}} req := extractionRequest() req.References = references result, err := newExtractor(t, client, references).Extract(context.Background(), req) if err == nil || !strings.Contains(err.Error(), "name is not in the NPC registry") || len(result.Value.Occurrences) != 0 { t.Fatalf("Extract() = %#v, %v; want no accepted occurrences", result, err) } } func TestExtractRejectsInvalidRequestsAndProviderFailures(t *testing.T) { references := requiredRegistryReferences(t, "Mira Thorn") valid := extractionRequest() valid.References = references extractor := newExtractor(t, &fakeOccurrencesLLMClient{}, references) var nilExtractor *Extractor for _, test := range []struct { name string extractor *Extractor ctx context.Context req contracts.TypedExtractionRequest want string }{ {"nil extractor", nilExtractor, context.Background(), valid, "extractor"}, {"nil LLM client", &Extractor{}, context.Background(), valid, "LLM client"}, {"wrapped preflight failure", extractor, context.Background(), mismatchedSourceInputRequest(valid), "must match chunk"}, } { t.Run(test.name, func(t *testing.T) { if _, err := test.extractor.Extract(test.ctx, test.req); err == nil || !strings.Contains(err.Error(), test.want) { t.Fatalf("Extract() error = %v, want %q", err, test.want) } }) } if _, err := newExtractor(t, &fakeOccurrencesLLMClient{err: errors.New("provider unavailable")}, references).Extract(context.Background(), valid); err == nil || !strings.Contains(err.Error(), "provider unavailable") { t.Fatalf("provider error = %v", err) } } func TestModuleSpecRegistrationMetadataAndFingerprints(t *testing.T) { got := ModuleSpec() if got.Key != Key || got.Stage != pipeline.StageExtract || got.ArtifactKind != dnd.NPCOccurrenceListKind || !reflect.DeepEqual(got.Requires, []string{"chunks", "source.transcript"}) || !reflect.DeepEqual(got.Provides, []string{"dnd.npc_occurrences"}) { t.Fatalf("ModuleSpec() = %#v", got) } var registrySlot contracts.ReferenceSlot for _, slot := range got.ReferenceSlots { if slot.Name == NPCRegistryReferenceSlot { registrySlot = slot } } if !registrySlot.Required || !reflect.DeepEqual(registrySlot.AcceptedArtifactKinds, []contracts.ArtifactKind{dnd.NPCRegistryKind}) || registrySlot.MaxBytes != NPCRegistryMaxBytes { t.Fatalf("NPC registry slot = %#v", registrySlot) } got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed" if ModuleSpec().ReferenceSlots[0].AcceptedMediaTypes[0] == "changed" { t.Fatal("ModuleSpec() returned mutable reference slots") } extractorSlots := newExtractor(t, &fakeOccurrencesLLMClient{}).ReferenceSlots() extractorSlots[0].AcceptedMediaTypes[0] = "changed" if newExtractor(t, &fakeOccurrencesLLMClient{}).ReferenceSlots()[0].AcceptedMediaTypes[0] == "changed" { t.Fatal("ReferenceSlots() returned mutable reference slots") } registry := pipeline.NewExtractorRegistry() if err := Register(registry); err != nil { t.Fatal(err) } if _, ok := registry.Spec(Key); !ok { t.Fatalf("registry missing %q", Key) } if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil { t.Fatal("DecodeOptions() accepted unknown option") } references := requiredRegistryReferences(t, "Mira Thorn") extractor := newExtractor(t, &fakeOccurrencesLLMClient{}, references) metadata := extractor.ManifestMetadata() for key, want := range map[string]string{ "prompt_id": PromptID, "prompt_version": SchemaVersion, "mapping_policy": mappingPolicy, "response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID, "response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion, } { if metadata[key] != want { t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want) } } for _, key := range []string{"prompt_sha256", "response_schema_sha256", "npc_registry_digest"} { if value, ok := metadata[key].(string); !ok || !strings.HasPrefix(value, "sha256:") { t.Fatalf("metadata[%q] = %#v, want digest", key, metadata[key]) } } wantFingerprints := map[string]struct{}{"prompt": {}, "response_schema": {}, "mapping_policy": {}, "npc_registry": {}} for _, fingerprint := range extractor.CheckpointFingerprints() { delete(wantFingerprints, fingerprint.Name) } if len(wantFingerprints) != 0 { t.Fatalf("missing fingerprints = %#v", wantFingerprints) } } func extractionRequest() contracts.TypedExtractionRequest { doc := sourceDocument() chunk := &source.Chunk{ ID: "session-alpha:chunk:0", SourceID: doc.ID, Index: 0, Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 30}, Content: []byte(`{"units":[10,2,7,5,20,30]}`), MediaType: "application/json", Units: append([]source.SourceUnit(nil), doc.Units...), } return contracts.TypedExtractionRequest{ Source: doc, Chunk: chunk, SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"), SessionID: "session-123", LLMProfile: "profile-npc-occurrences", } } func sourceDocument() *source.SourceDocument { return &source.SourceDocument{ ID: "session-alpha", Kind: "transcript", Format: "application/vnd.seriatim.minimal+json", Digest: "sha256:source", Units: []source.SourceUnit{ {ID: 10, Kind: "transcript_segment", Text: "Mira Thorn is mentioned."}, {ID: 2, Kind: "transcript_segment", Text: "Mira Thorn speaks to the party."}, {ID: 7, Kind: "transcript_segment", Text: "Mira Thorn watches nearby."}, {ID: 5, Kind: "transcript_segment", Text: "Mira Thorn joins the party in combat."}, {ID: 20, Kind: "transcript_segment", Text: "Mira Thorn attacks the party."}, {ID: 30, Kind: "transcript_segment", Text: "Mira Thorn performs a ritual."}, }, } } func requiredRegistryReferences(t *testing.T, names ...string) contracts.ReferenceSet { t.Helper() npcs := make([]dnd.NPC, len(names)) for index, name := range names { npcs[index] = dnd.NPC{ ID: identity.DeriveID(name), Name: name, SourceRefs: []source.SourceRef{{SourceID: "other-session", StartUnitID: index + 1, EndUnitID: index + 1}}, } } content, err := npccodec.New().Encode(dnd.NPCRegistry{NPCs: npcs}) if err != nil { t.Fatal(err) } return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ NPCRegistryReferenceSlot: { Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot}, Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: npccodec.MediaType, Content: content, Origin: contracts.ReferenceOrigin{Type: "generated"}}}, }, }} } func occurrenceRefs(start, end int) []occurrenceSourceRefResponse { return []occurrenceSourceRefResponse{{StartUnitID: start, EndUnitID: end}} } func occurrenceNames(value dnd.NPCOccurrenceList) []string { names := make([]string, len(value.Occurrences)) for index, occurrence := range value.Occurrences { names[index] = occurrence.Name } return names } func occurrenceKinds(value dnd.NPCOccurrenceList) []dnd.NPCOccurrenceKind { kinds := make([]dnd.NPCOccurrenceKind, len(value.Occurrences)) for index, occurrence := range value.Occurrences { kinds[index] = occurrence.Kind } return kinds } func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest { req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "") return req } func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor { t.Helper() extractor, err := New(client, Options{}, references...) if err != nil { t.Fatalf("New() error = %v", err) } return extractor } type fakeOccurrencesLLMClient struct { response extractionResponse err error requests []contracts.StructuredCompletionRequest } func (client *fakeOccurrencesLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { client.requests = append(client.requests, cloneStructuredCompletionRequest(req)) if client.err != nil { return contracts.StructuredCompletionResponse{}, client.err } target, ok := out.(*extractionResponse) if !ok { return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target") } *target = client.response content, err := json.Marshal(client.response) if err != nil { return contracts.StructuredCompletionResponse{}, err } return contracts.StructuredCompletionResponse{Content: content}, nil } func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest { req.Inputs = req.Inputs.Clone() return req }