Register item registry normalization
This commit is contained in:
193
internal/modules/dnd/normalize/itemregistry/normalizer_test.go
Normal file
193
internal/modules/dnd/normalize/itemregistry/normalizer_test.go
Normal file
@@ -0,0 +1,193 @@
|
||||
package itemregistry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/entityreconcile"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestModuleContractAndMetadata(t *testing.T) {
|
||||
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.ItemRegistryKind}
|
||||
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
||||
}
|
||||
registry := pipeline.NewNormalizerRegistry()
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := New(nil, Options{}); err == nil {
|
||||
t.Fatal("New() accepted nil client")
|
||||
}
|
||||
metadata := newNormalizer(t, &recordingNormalizerClient{}).ManifestMetadata()
|
||||
if metadata["identity_policy"] != identity.Policy || metadata["response_schema_id"] != entityreconcile.ResponseSchemaID || metadata["normalization_policy"] != normalizationPolicy || metadata["semantic_context_radius"] != semanticContextRadius {
|
||||
t.Fatalf("metadata = %#v", metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizePreprocessesExactDuplicatesWithoutMutation(t *testing.T) {
|
||||
input := dnd.ItemRegistry{Items: []dnd.Item{
|
||||
{Name: " Rope ", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}},
|
||||
{Name: "rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}},
|
||||
{Name: "Rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 2, EndUnitID: 2}}},
|
||||
}}
|
||||
before := dnd.ItemRegistry{Items: append([]dnd.Item(nil), input.Items...)}
|
||||
result, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequest(input))
|
||||
if err != nil || len(result.Value.Items) != 2 || !reflect.DeepEqual(input, before) {
|
||||
t.Fatalf("Normalize() = %#v, %v; want non-mutating exact deduplication", result, err)
|
||||
}
|
||||
if result.Value.Items[0].ID != result.Value.Items[1].ID || !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) {
|
||||
t.Fatalf("items = %#v, warnings = %#v; want type identity and duplicate warning", result.Value.Items, result.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeAppliesSafeAliasProposalAndPreservesCurrency(t *testing.T) {
|
||||
doc := semanticDocument()
|
||||
input := dnd.ItemRegistry{Items: []dnd.Item{
|
||||
{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}},
|
||||
{Name: "Compass of the Stars", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
||||
{Name: "Gold Pieces", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||||
}}
|
||||
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000002"}]}`}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry != nil || len(result.Value.Items) != 2 {
|
||||
t.Fatalf("Normalize() = %#v, %v", result, err)
|
||||
}
|
||||
merged := result.Value.Items[0]
|
||||
if merged.Name != "Compass of the Stars" || merged.ID != identity.DeriveID(merged.Name) || len(merged.SourceRefs) != 2 || !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) {
|
||||
t.Fatalf("merged item = %#v, warnings = %#v", merged, result.Warnings)
|
||||
}
|
||||
encoded := string(client.requests[0].Inputs["candidates"].Content) + string(client.requests[0].Inputs["transcript"].Content)
|
||||
if strings.Contains(encoded, doc.ID) || !strings.Contains(encoded, "candidate-000001") || strings.Contains(encoded, merged.ID) {
|
||||
t.Fatalf("private inputs = %s", encoded)
|
||||
}
|
||||
currencyClient := &recordingNormalizerClient{response: `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000001"}]}`}
|
||||
currency := dnd.ItemRegistry{Items: []dnd.Item{{Name: "Gold Pieces", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Silver Pieces", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}}}
|
||||
result, err = newNormalizer(t, currencyClient).Normalize(context.Background(), normalizeRequestWithSource(currency, doc))
|
||||
if err != nil || result.Retry == nil || len(result.Value.Items) != 2 || !hasWarning(result.Warnings, ReasonCodeItemSemanticProposalInvalid) {
|
||||
t.Fatalf("currency result = %#v, %v; want denominations retained", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizePreservesCandidatesForUnsafeProposalGroups(t *testing.T) {
|
||||
doc := semanticDocument()
|
||||
input := dnd.ItemRegistry{Items: []dnd.Item{
|
||||
{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}},
|
||||
{Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
||||
{Name: "Rope", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||||
}}
|
||||
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000001"},{"members":["candidate-000002","candidate-000003"],"canonical":"candidate-000003"},{"members":["candidate-000001","candidate-000003"],"canonical":"candidate-000099"}]}`}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry == nil || len(result.Value.Items) != 3 || !strings.Contains(result.Retry.Message, "overlapping_member") || !strings.Contains(result.Retry.Message, "canonical_unknown") || strings.Contains(result.Retry.Message, "Star Compass") || len(result.Retry.Message) > 4096 {
|
||||
t.Fatalf("Normalize() = %#v, %v; want deterministic retry fallback", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeRetryFallbackErrorsWarningsAndIdempotence(t *testing.T) {
|
||||
doc := semanticDocument()
|
||||
input := dnd.ItemRegistry{Items: []dnd.Item{{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}}}
|
||||
invalid, err := newNormalizer(t, &recordingNormalizerClient{err: contracts.ErrInvalidStructuredOutput}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || invalid.Retry == nil || invalid.Retry.ReasonCode != ReasonCodeItemSemanticProposalInvalid {
|
||||
t.Fatalf("invalid result = %#v, %v", invalid, err)
|
||||
}
|
||||
_, err = newNormalizer(t, &recordingNormalizerClient{err: errors.New("provider unavailable")}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err == nil || !strings.Contains(err.Error(), "provider unavailable") {
|
||||
t.Fatalf("provider error = %v", err)
|
||||
}
|
||||
if bounded := limitWarningsForRetry(make([]contracts.Warning, diagnostics.MaxWarnings+5)); len(bounded) != diagnostics.MaxWarnings-1 || bounded[len(bounded)-1].ReasonCode != ReasonCodeItemNormalizationWarningsOmitted {
|
||||
t.Fatalf("retry warning limit = %#v", bounded)
|
||||
}
|
||||
first, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
second, secondErr := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(first.Value, doc))
|
||||
if err != nil || secondErr != nil || !reflect.DeepEqual(first.Value, second.Value) {
|
||||
t.Fatalf("normalization must be idempotent: first=%#v err=%v second=%#v err=%v", first, err, second, secondErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterPromptAssetsPreparesItemNormalizationPrompt(t *testing.T) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := entityreconcile.RegisterSchemaAssets(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
options, err := registry.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ID: "item-normalize-test", Endpoint: "http://127.0.0.1:1/v1", Model: "test"})))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: entityreconcile.SchemaVersion, ProfileID: "item-normalize-test", Inputs: map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidates":[{"key":"candidate-000001","name":"Rope","source_refs":[]}]}`), "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if prepared.OutputContract.SchemaPath != "dnd_entity_reconcile_llm.v1.json" || !strings.Contains(prepared.Messages[1].Content, "currency denominations") || !strings.Contains(prepared.Messages[1].Content, "materially different item types") {
|
||||
t.Fatalf("prepared prompt = %#v", prepared)
|
||||
}
|
||||
}
|
||||
|
||||
type recordingNormalizerClient struct {
|
||||
response string
|
||||
err error
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
}
|
||||
|
||||
func (c *recordingNormalizerClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, output any) (contracts.StructuredCompletionResponse, error) {
|
||||
c.requests = append(c.requests, request)
|
||||
if c.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, c.err
|
||||
}
|
||||
response := c.response
|
||||
if response == "" {
|
||||
response = `{"duplicate_groups":[]}`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(response), output); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: json.RawMessage(response)}, nil
|
||||
}
|
||||
|
||||
func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normalizer {
|
||||
t.Helper()
|
||||
normalizer, err := New(client, Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
return normalizer
|
||||
}
|
||||
func normalizeRequest(value dnd.ItemRegistry) contracts.TypedNormalizeRequest[dnd.ItemRegistry] {
|
||||
return contracts.TypedNormalizeRequest[dnd.ItemRegistry]{MergeOutput: contracts.MergeArtifact[dnd.ItemRegistry]{Value: value}}
|
||||
}
|
||||
func normalizeRequestWithSource(value dnd.ItemRegistry, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.ItemRegistry] {
|
||||
request := normalizeRequest(value)
|
||||
request.Source = doc
|
||||
return request
|
||||
}
|
||||
func semanticDocument() *source.SourceDocument {
|
||||
return &source.SourceDocument{ID: "item-session", Units: []source.SourceUnit{{ID: 10, Text: "The Star Compass points north."}, {ID: 20, Text: "The compass of the stars glows."}, {ID: 30, Text: "The chest holds gold pieces."}}}
|
||||
}
|
||||
func hasWarning(warnings []contracts.Warning, reason string) bool {
|
||||
for _, warning := range warnings {
|
||||
if warning.ReasonCode == reason {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in New Issue
Block a user