package itemregistry import ( "context" "encoding/json" "errors" "fmt" "reflect" "strconv" "strings" "testing" "time" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics" identityvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/identity" "gitea.maximumdirect.net/eric/promptkit" ) func TestModuleContractAndMetadata(t *testing.T) { want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.ItemRegistryKind} if got := ModuleSpec(); !reflect.DeepEqual(got, want) { t.Fatalf("ModuleSpec() = %#v, want %#v", got, want) } registry := pipeline.NewNormalizerRegistry() if err := Register(registry); err != nil { t.Fatal(err) } if _, err := New(nil, Options{}); err == nil { t.Fatal("New() accepted nil client") } metadata := newNormalizer(t, &recordingNormalizerClient{}).ManifestMetadata() limits, ok := metadata["semantic_reconciliation_limits"].(map[string]any) if !ok || metadata["identity_policy"] != identity.Policy || metadata["normalization_policy"] != normalizationPolicy || metadata["prompt_id"] != PromptID || metadata["prompt_version"] != PromptVersion || metadata["response_schema_key"] != string(semanticreconcile.ResponseSchemaKey) || metadata["response_schema_id"] != semanticreconcile.ResponseSchemaID || metadata["response_schema_name"] != semanticreconcile.ResponseSchemaName || metadata["semantic_reconciliation_policy"] != semanticreconcile.Policy || len(limits) != 3 { t.Fatalf("metadata = %#v", metadata) } for _, name := range []string{"prompt", "response_schema", "semantic_reconciliation_policy", "semantic_reconciliation_limits", "identity_policy", "normalization_policy"} { if !hasFingerprint(newNormalizer(t, &recordingNormalizerClient{}).CheckpointFingerprints(), name) { t.Fatalf("fingerprints missing %q", name) } } } func TestNormalizeConsolidatesEqualNamesAcrossEvidenceWithoutMutation(t *testing.T) { doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{ {ID: 1, Text: "The rope is secured."}, {ID: 2, Text: "The party takes the rope."}, {ID: 3, Text: "The rope is packed away."}, {ID: 4, Text: "A lantern lights the path."}, }} input := dnd.ItemRegistry{Items: []dnd.Item{ {Name: " Rope ", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 3, EndUnitID: 3}, {SourceID: "session", StartUnitID: 1, EndUnitID: 1}}}, {Name: "rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}}}, {Name: "Lantern", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 4, EndUnitID: 4}}}, }} before := dnd.ItemRegistry{Items: []dnd.Item{ {Name: " Rope ", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 3, EndUnitID: 3}, {SourceID: "session", StartUnitID: 1, EndUnitID: 1}}}, {Name: "rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}}}, {Name: "Lantern", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 4, EndUnitID: 4}}}, }} client := &recordingNormalizerClient{} result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) if err != nil || len(result.Value.Items) != 2 || !reflect.DeepEqual(input, before) || len(client.requests) != 1 { t.Fatalf("Normalize() = %#v, %v; want deterministic non-mutating consolidation", result, err) } rope := result.Value.Items[0] wantRefs := []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}, {SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}} if rope.Name != "Rope" || rope.ID != identity.DeriveID("Rope") || !reflect.DeepEqual(rope.SourceRefs, wantRefs) || result.Value.Items[1].Name != "Lantern" || !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) { t.Fatalf("items = %#v, warnings = %#v; want earliest display name, canonical evidence union, and stable placement", result.Value.Items, result.Warnings) } validation, validationErr := identityvalidator.New(identityvalidator.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Value: result.Value}) if validationErr != nil || !validation.Approved { t.Fatalf("default item-registry identity validation = %#v, %v; want normalized cross-chunk candidates accepted", validation, validationErr) } var candidates struct { Candidates []struct { Label string `json:"label"` } `json:"candidates"` } if err := json.Unmarshal(client.requests[0].Inputs["candidates"].Content, &candidates); err != nil || len(candidates.Candidates) != 2 || candidates.Candidates[0].Label != "Rope" || candidates.Candidates[1].Label != "Lantern" { t.Fatalf("semantic candidates = %#v, %v; want one candidate per comparison name", candidates, err) } repeated, repeatErr := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(result.Value, doc)) if repeatErr != nil || !reflect.DeepEqual(repeated.Value, result.Value) { t.Fatalf("repeated normalization = %#v, %v; want stable output", repeated, repeatErr) } } func BenchmarkComparisonNameGroupsManyDistinct(b *testing.B) { records := make([]normalizedRecord, 1_000) for index := range records { records[index] = normalizedRecord{item: dnd.Item{Name: "Item " + strconv.Itoa(index)}} } b.ReportAllocs() b.ResetTimer() for iteration := 0; iteration < b.N; iteration++ { if groups := comparisonNameGroups(records); len(groups) != len(records) { b.Fatalf("group count = %d, want %d", len(groups), len(records)) } } } func TestNormalizeAppliesSafeAliasProposal(t *testing.T) { doc := semanticDocument() input := dnd.ItemRegistry{Items: []dnd.Item{ {Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Compass of the Stars", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}, {Name: "Gold Pieces", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}}, }} client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2}]}`} result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) if err != nil || result.Retry != nil || len(result.Value.Items) != 2 { t.Fatalf("Normalize() = %#v, %v", result, err) } merged := result.Value.Items[0] if merged.Name != "Compass of the Stars" || merged.ID != identity.DeriveID(merged.Name) || len(merged.SourceRefs) != 2 || !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) { t.Fatalf("merged item = %#v, warnings = %#v", merged, result.Warnings) } encoded := string(client.requests[0].Inputs["candidates"].Content) + string(client.requests[0].Inputs["transcript"].Content) if strings.Contains(encoded, doc.ID) || strings.Contains(encoded, "candidate-") || strings.Contains(encoded, merged.ID) || !strings.Contains(encoded, `"source_refs"`) { t.Fatalf("private inputs = %s", encoded) } } func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) { doc := semanticDocument() ref := func(unitID int) []source.SourceRef { return []source.SourceRef{{SourceID: doc.ID, StartUnitID: unitID, EndUnitID: unitID}} } for _, test := range []struct { name string items []dnd.Item response string wantNames []string wantRefCounts []int wantRetry bool warning string }{ { name: "same denomination aliases", items: []dnd.Item{ {Name: "GP", SourceRefs: ref(10)}, {Name: "Gold Piece", SourceRefs: ref(20)}, {Name: "Gold Pieces", SourceRefs: ref(30)}, }, response: `{"duplicate_groups":[{"candidate_ids":[1,2,3],"canonical_candidate_id":2}]}`, wantNames: []string{"Gold Piece"}, wantRefCounts: []int{3}, warning: ReasonCodeDuplicateItemCollapsed, }, { name: "different denominations", items: []dnd.Item{ {Name: "Gold Pieces", SourceRefs: ref(10)}, {Name: "Silver Pieces", SourceRefs: ref(20)}, }, response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":1}]}`, wantNames: []string{"Gold Pieces", "Silver Pieces"}, wantRefCounts: []int{1, 1}, wantRetry: true, warning: ReasonCodeItemSemanticProposalInvalid, }, { name: "currency plus ordinary item", items: []dnd.Item{ {Name: "Gold Pieces", SourceRefs: ref(10)}, {Name: "Longsword", SourceRefs: ref(20)}, }, response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":1}]}`, wantNames: []string{"Gold Pieces", "Longsword"}, wantRefCounts: []int{1, 1}, wantRetry: true, warning: ReasonCodeItemSemanticProposalInvalid, }, { name: "ordinary items", items: []dnd.Item{ {Name: "Star Compass", SourceRefs: ref(10)}, {Name: "Compass of the Stars", SourceRefs: ref(20)}, }, response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2}]}`, wantNames: []string{"Compass of the Stars"}, wantRefCounts: []int{2}, warning: ReasonCodeDuplicateItemCollapsed, }, { name: "currency plus ordinary canonical item", items: []dnd.Item{ {Name: "Gold Pieces", SourceRefs: ref(10)}, {Name: "Longsword", SourceRefs: ref(20)}, }, response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2}]}`, wantNames: []string{"Gold Pieces", "Longsword"}, wantRefCounts: []int{1, 1}, wantRetry: true, warning: ReasonCodeItemSemanticProposalInvalid, }, } { t.Run(test.name, func(t *testing.T) { input := dnd.ItemRegistry{Items: make([]dnd.Item, len(test.items))} for index, item := range test.items { input.Items[index] = cloneItem(item) } before := dnd.ItemRegistry{Items: make([]dnd.Item, len(input.Items))} for index, item := range input.Items { before.Items[index] = cloneItem(item) } result, err := newNormalizer(t, &recordingNormalizerClient{response: test.response}).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) if err != nil || (result.Retry != nil) != test.wantRetry || !reflect.DeepEqual(input, before) || !hasWarning(result.Warnings, test.warning) { t.Fatalf("Normalize() = %#v, %v", result, err) } if len(result.Value.Items) != len(test.wantNames) { t.Fatalf("items = %#v, want names %#v", result.Value.Items, test.wantNames) } for index, item := range result.Value.Items { if item.Name != test.wantNames[index] || len(item.SourceRefs) != test.wantRefCounts[index] { t.Fatalf("item %d = %#v, want name %q with %d source refs", index, item, test.wantNames[index], test.wantRefCounts[index]) } } if test.wantRetry && (len(result.Retry.FallbackWarnings) != 1 || result.Retry.FallbackWarnings[0].ReasonCode != ReasonCodeItemSemanticReconciliationExhausted) { t.Fatalf("retry = %#v, want preserved-group fallback", result.Retry) } }) } } func TestNormalizePreservesCandidatesForUnsafeProposalGroups(t *testing.T) { doc := semanticDocument() input := dnd.ItemRegistry{Items: []dnd.Item{ {Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}, {Name: "Rope", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}}, }} client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":1},{"candidate_ids":[2,3],"canonical_candidate_id":3},{"candidate_ids":[1,3],"canonical_candidate_id":99}]}`} result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) if err != nil || result.Retry == nil || len(result.Value.Items) != 3 || !strings.Contains(result.Retry.Message, "overlapping_member") || !strings.Contains(result.Retry.Message, "canonical_unknown") || strings.Contains(result.Retry.Message, "Star Compass") || len(result.Retry.Message) > 4096 { t.Fatalf("Normalize() = %#v, %v; want deterministic retry fallback", result, err) } } func TestNormalizeAppliesIndependentGroupAndCountsAllOmissions(t *testing.T) { doc := semanticDocument() ref := func(unitID int) []source.SourceRef { return []source.SourceRef{{SourceID: doc.ID, StartUnitID: unitID, EndUnitID: unitID}} } input := dnd.ItemRegistry{Items: []dnd.Item{ {Name: "Star Compass", SourceRefs: ref(10)}, {Name: "Compass of the Stars", SourceRefs: ref(20)}, {Name: "Gold Pieces", SourceRefs: ref(30)}, {Name: "Silver Pieces", SourceRefs: ref(30)}, {Name: "Rope", SourceRefs: ref(10)}, }} client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2},{"candidate_ids":[3,4],"canonical_candidate_id":3},{"candidate_ids":[5,99],"canonical_candidate_id":5}]}`} result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) if err != nil || result.Retry == nil { t.Fatalf("Normalize() = %#v, %v; want retry with independently accepted output", result, err) } wantNames := []string{"Compass of the Stars", "Gold Pieces", "Silver Pieces", "Rope"} if len(result.Value.Items) != len(wantNames) { t.Fatalf("items = %#v, want %v", result.Value.Items, wantNames) } for index, name := range wantNames { if result.Value.Items[index].Name != name { t.Fatalf("item %d = %#v, want %q", index, result.Value.Items[index], name) } } if !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) || !hasWarning(result.Warnings, ReasonCodeItemSemanticProposalInvalid) { t.Fatalf("warnings = %#v, want accepted and guarded-group diagnostics", result.Warnings) } if len(result.Retry.FallbackWarnings) != 1 || !strings.Contains(result.Retry.FallbackWarnings[0].Message, "2 proposal group(s)") { t.Fatalf("retry = %#v, want one guarded and one malformed group counted", result.Retry) } } func TestNormalizeLimitSkipDoesNotCallLLMAndAddsBoundedFallbackWarning(t *testing.T) { client := &recordingNormalizerClient{} doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Kind: "narration", Text: "A crowded storeroom"}}} limit := semanticreconcile.DefaultLimits().MaximumCandidates input := dnd.ItemRegistry{Items: make([]dnd.Item, limit+1)} for index := range input.Items { input.Items[index] = dnd.Item{Name: fmt.Sprintf("Item %d", index), SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}} } result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) if err != nil || result.Retry != nil { t.Fatalf("Normalize() = %#v, %v; want deterministic limit fallback", result, err) } if len(client.requests) != 0 || len(result.Value.Items) != limit+1 { t.Fatalf("completion calls = %d, items = %d; want no call and all records", len(client.requests), len(result.Value.Items)) } if !hasWarning(result.Warnings, ReasonCodeItemSemanticReconciliationExhausted) || len(result.Warnings) > diagnostics.MaxWarnings { t.Fatalf("warnings = %#v, want bounded reconciliation fallback", result.Warnings) } } func TestNormalizeRetryFallbackErrorsWarningsAndIdempotence(t *testing.T) { doc := semanticDocument() input := dnd.ItemRegistry{Items: []dnd.Item{{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}}} invalid, err := newNormalizer(t, &recordingNormalizerClient{err: contracts.ErrInvalidStructuredOutput}).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) if err != nil || invalid.Retry == nil || invalid.Retry.ReasonCode != ReasonCodeItemSemanticProposalInvalid { t.Fatalf("invalid result = %#v, %v", invalid, err) } _, err = newNormalizer(t, &recordingNormalizerClient{err: errors.New("provider unavailable")}).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) if err == nil || !strings.Contains(err.Error(), "provider unavailable") { t.Fatalf("provider error = %v", err) } if bounded := limitWarningsForRetry(make([]contracts.Warning, diagnostics.MaxWarnings+5)); len(bounded) != diagnostics.MaxWarnings-1 || bounded[len(bounded)-1].ReasonCode != ReasonCodeItemNormalizationWarningsOmitted { t.Fatalf("retry warning limit = %#v", bounded) } first, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) second, secondErr := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(first.Value, doc)) if err != nil || secondErr != nil || !reflect.DeepEqual(first.Value, second.Value) { t.Fatalf("normalization must be idempotent: first=%#v err=%v second=%#v err=%v", first, err, second, secondErr) } } func TestRegisterPromptAssetsPreparesItemNormalizationPrompt(t *testing.T) { registry := llm.NewAssetRegistry() if err := semanticreconcile.RegisterAssets(registry); err != nil { t.Fatal(err) } if err := RegisterPromptAssets(registry); err != nil { t.Fatal(err) } options, err := registry.PromptKitOptions() if err != nil { t.Fatal(err) } options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ID: "item-normalize-test", Endpoint: "http://127.0.0.1:1/v1", Model: "test"}))) engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...) if err != nil { t.Fatal(err) } prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "item-normalize-test", Inputs: map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidates":[{"candidate_id":1,"label":"Rope","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`)}}) if err != nil { t.Fatal(err) } if prepared.OutputContract.SchemaPath != "semantic_reconciliation_llm.v1.json" || !strings.Contains(prepared.Messages[1].Content, "candidate_id") || !strings.Contains(prepared.Messages[1].Content, "integer") || !strings.Contains(prepared.Messages[2].Content, "currency denominations") || !strings.Contains(prepared.Messages[2].Content, "materially different item") || prepared.Messages[2].CacheControl == nil || prepared.Messages[2].CacheControl.Type != promptkit.CacheControlEphemeral { t.Fatalf("prepared prompt = %#v", prepared) } if prepared.Messages[4].CacheControl == nil || prepared.Messages[4].CacheControl.Type != promptkit.CacheControlEphemeral || !strings.Contains(prepared.Messages[3].Content, `"Rope"`) || strings.Contains(prepared.Messages[3].Content, `"windows"`) || !strings.Contains(prepared.Messages[4].Content, `"windows"`) || strings.Contains(prepared.Messages[4].Content, `"Rope"`) { t.Fatalf("prepared prompt = %#v, want isolated candidate and transcript presentation", prepared) } } type recordingNormalizerClient struct { response string err error requests []contracts.StructuredCompletionRequest } func (c *recordingNormalizerClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, output any) (contracts.StructuredCompletionResponse, error) { c.requests = append(c.requests, request) if c.err != nil { return contracts.StructuredCompletionResponse{}, c.err } response := c.response if response == "" { response = `{"duplicate_groups":[]}` } content := []byte(response) if err := json.Unmarshal(content, output); err != nil { return contracts.StructuredCompletionResponse{}, err } return contracts.StructuredCompletionResponse{Content: content}, nil } func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normalizer { t.Helper() normalizer, err := New(client, Options{}) if err != nil { t.Fatalf("New() error = %v", err) } return normalizer } func normalizeRequest(value dnd.ItemRegistry) contracts.TypedNormalizeRequest[dnd.ItemRegistry] { return contracts.TypedNormalizeRequest[dnd.ItemRegistry]{MergeOutput: contracts.MergeArtifact[dnd.ItemRegistry]{Value: value}} } func normalizeRequestWithSource(value dnd.ItemRegistry, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.ItemRegistry] { request := normalizeRequest(value) request.Source = doc return request } func semanticDocument() *source.SourceDocument { return &source.SourceDocument{ID: "item-session", Units: []source.SourceUnit{{ID: 10, Text: "The Star Compass points north."}, {ID: 20, Text: "The compass of the stars glows."}, {ID: 30, Text: "The chest holds gold pieces."}}} } func hasWarning(warnings []contracts.Warning, reason string) bool { for _, warning := range warnings { if warning.ReasonCode == reason { return true } } return false } func hasFingerprint(fingerprints []pipeline.CheckpointFingerprint, name string) bool { for _, fingerprint := range fingerprints { if fingerprint.Name == name && fingerprint.Value != "" { return true } } return false }