194 lines
11 KiB
Go
194 lines
11 KiB
Go
package itemregistry
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/entityreconcile"
|
|
"gitea.maximumdirect.net/eric/promptkit"
|
|
)
|
|
|
|
func TestModuleContractAndMetadata(t *testing.T) {
|
|
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.ItemRegistryKind}
|
|
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
|
}
|
|
registry := pipeline.NewNormalizerRegistry()
|
|
if err := Register(registry); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := New(nil, Options{}); err == nil {
|
|
t.Fatal("New() accepted nil client")
|
|
}
|
|
metadata := newNormalizer(t, &recordingNormalizerClient{}).ManifestMetadata()
|
|
if metadata["identity_policy"] != identity.Policy || metadata["response_schema_id"] != entityreconcile.ResponseSchemaID || metadata["normalization_policy"] != normalizationPolicy || metadata["semantic_context_radius"] != semanticContextRadius {
|
|
t.Fatalf("metadata = %#v", metadata)
|
|
}
|
|
}
|
|
|
|
func TestNormalizePreprocessesExactDuplicatesWithoutMutation(t *testing.T) {
|
|
input := dnd.ItemRegistry{Items: []dnd.Item{
|
|
{Name: " Rope ", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}},
|
|
{Name: "rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}}},
|
|
{Name: "Rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 2, EndUnitID: 2}}},
|
|
}}
|
|
before := dnd.ItemRegistry{Items: append([]dnd.Item(nil), input.Items...)}
|
|
result, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequest(input))
|
|
if err != nil || len(result.Value.Items) != 2 || !reflect.DeepEqual(input, before) {
|
|
t.Fatalf("Normalize() = %#v, %v; want non-mutating exact deduplication", result, err)
|
|
}
|
|
if result.Value.Items[0].ID != result.Value.Items[1].ID || !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) {
|
|
t.Fatalf("items = %#v, warnings = %#v; want type identity and duplicate warning", result.Value.Items, result.Warnings)
|
|
}
|
|
}
|
|
|
|
func TestNormalizeAppliesSafeAliasProposalAndPreservesCurrency(t *testing.T) {
|
|
doc := semanticDocument()
|
|
input := dnd.ItemRegistry{Items: []dnd.Item{
|
|
{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}},
|
|
{Name: "Compass of the Stars", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
|
{Name: "Gold Pieces", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
|
}}
|
|
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000002"}]}`}
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err != nil || result.Retry != nil || len(result.Value.Items) != 2 {
|
|
t.Fatalf("Normalize() = %#v, %v", result, err)
|
|
}
|
|
merged := result.Value.Items[0]
|
|
if merged.Name != "Compass of the Stars" || merged.ID != identity.DeriveID(merged.Name) || len(merged.SourceRefs) != 2 || !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) {
|
|
t.Fatalf("merged item = %#v, warnings = %#v", merged, result.Warnings)
|
|
}
|
|
encoded := string(client.requests[0].Inputs["candidates"].Content) + string(client.requests[0].Inputs["transcript"].Content)
|
|
if strings.Contains(encoded, doc.ID) || !strings.Contains(encoded, "candidate-000001") || strings.Contains(encoded, merged.ID) {
|
|
t.Fatalf("private inputs = %s", encoded)
|
|
}
|
|
currencyClient := &recordingNormalizerClient{response: `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000001"}]}`}
|
|
currency := dnd.ItemRegistry{Items: []dnd.Item{{Name: "Gold Pieces", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Silver Pieces", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}}}
|
|
result, err = newNormalizer(t, currencyClient).Normalize(context.Background(), normalizeRequestWithSource(currency, doc))
|
|
if err != nil || result.Retry == nil || len(result.Value.Items) != 2 || !hasWarning(result.Warnings, ReasonCodeItemSemanticProposalInvalid) {
|
|
t.Fatalf("currency result = %#v, %v; want denominations retained", result, err)
|
|
}
|
|
}
|
|
|
|
func TestNormalizePreservesCandidatesForUnsafeProposalGroups(t *testing.T) {
|
|
doc := semanticDocument()
|
|
input := dnd.ItemRegistry{Items: []dnd.Item{
|
|
{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}},
|
|
{Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
|
{Name: "Rope", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
|
}}
|
|
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000001"},{"members":["candidate-000002","candidate-000003"],"canonical":"candidate-000003"},{"members":["candidate-000001","candidate-000003"],"canonical":"candidate-000099"}]}`}
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err != nil || result.Retry == nil || len(result.Value.Items) != 3 || !strings.Contains(result.Retry.Message, "overlapping_member") || !strings.Contains(result.Retry.Message, "canonical_unknown") || strings.Contains(result.Retry.Message, "Star Compass") || len(result.Retry.Message) > 4096 {
|
|
t.Fatalf("Normalize() = %#v, %v; want deterministic retry fallback", result, err)
|
|
}
|
|
}
|
|
|
|
func TestNormalizeRetryFallbackErrorsWarningsAndIdempotence(t *testing.T) {
|
|
doc := semanticDocument()
|
|
input := dnd.ItemRegistry{Items: []dnd.Item{{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}}}
|
|
invalid, err := newNormalizer(t, &recordingNormalizerClient{err: contracts.ErrInvalidStructuredOutput}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err != nil || invalid.Retry == nil || invalid.Retry.ReasonCode != ReasonCodeItemSemanticProposalInvalid {
|
|
t.Fatalf("invalid result = %#v, %v", invalid, err)
|
|
}
|
|
_, err = newNormalizer(t, &recordingNormalizerClient{err: errors.New("provider unavailable")}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err == nil || !strings.Contains(err.Error(), "provider unavailable") {
|
|
t.Fatalf("provider error = %v", err)
|
|
}
|
|
if bounded := limitWarningsForRetry(make([]contracts.Warning, diagnostics.MaxWarnings+5)); len(bounded) != diagnostics.MaxWarnings-1 || bounded[len(bounded)-1].ReasonCode != ReasonCodeItemNormalizationWarningsOmitted {
|
|
t.Fatalf("retry warning limit = %#v", bounded)
|
|
}
|
|
first, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
second, secondErr := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(first.Value, doc))
|
|
if err != nil || secondErr != nil || !reflect.DeepEqual(first.Value, second.Value) {
|
|
t.Fatalf("normalization must be idempotent: first=%#v err=%v second=%#v err=%v", first, err, second, secondErr)
|
|
}
|
|
}
|
|
|
|
func TestRegisterPromptAssetsPreparesItemNormalizationPrompt(t *testing.T) {
|
|
registry := llm.NewAssetRegistry()
|
|
if err := entityreconcile.RegisterSchemaAssets(registry); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := RegisterPromptAssets(registry); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
options, err := registry.PromptKitOptions()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ID: "item-normalize-test", Endpoint: "http://127.0.0.1:1/v1", Model: "test"})))
|
|
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: entityreconcile.SchemaVersion, ProfileID: "item-normalize-test", Inputs: map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidates":[{"key":"candidate-000001","name":"Rope","source_refs":[]}]}`), "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`)}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if prepared.OutputContract.SchemaPath != "dnd_entity_reconcile_llm.v1.json" || !strings.Contains(prepared.Messages[1].Content, "currency denominations") || !strings.Contains(prepared.Messages[1].Content, "materially different item types") {
|
|
t.Fatalf("prepared prompt = %#v", prepared)
|
|
}
|
|
}
|
|
|
|
type recordingNormalizerClient struct {
|
|
response string
|
|
err error
|
|
requests []contracts.StructuredCompletionRequest
|
|
}
|
|
|
|
func (c *recordingNormalizerClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, output any) (contracts.StructuredCompletionResponse, error) {
|
|
c.requests = append(c.requests, request)
|
|
if c.err != nil {
|
|
return contracts.StructuredCompletionResponse{}, c.err
|
|
}
|
|
response := c.response
|
|
if response == "" {
|
|
response = `{"duplicate_groups":[]}`
|
|
}
|
|
if err := json.Unmarshal([]byte(response), output); err != nil {
|
|
return contracts.StructuredCompletionResponse{}, err
|
|
}
|
|
return contracts.StructuredCompletionResponse{Content: json.RawMessage(response)}, nil
|
|
}
|
|
|
|
func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normalizer {
|
|
t.Helper()
|
|
normalizer, err := New(client, Options{})
|
|
if err != nil {
|
|
t.Fatalf("New() error = %v", err)
|
|
}
|
|
return normalizer
|
|
}
|
|
func normalizeRequest(value dnd.ItemRegistry) contracts.TypedNormalizeRequest[dnd.ItemRegistry] {
|
|
return contracts.TypedNormalizeRequest[dnd.ItemRegistry]{MergeOutput: contracts.MergeArtifact[dnd.ItemRegistry]{Value: value}}
|
|
}
|
|
func normalizeRequestWithSource(value dnd.ItemRegistry, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.ItemRegistry] {
|
|
request := normalizeRequest(value)
|
|
request.Source = doc
|
|
return request
|
|
}
|
|
func semanticDocument() *source.SourceDocument {
|
|
return &source.SourceDocument{ID: "item-session", Units: []source.SourceUnit{{ID: 10, Text: "The Star Compass points north."}, {ID: 20, Text: "The compass of the stars glows."}, {ID: 30, Text: "The chest holds gold pieces."}}}
|
|
}
|
|
func hasWarning(warnings []contracts.Warning, reason string) bool {
|
|
for _, warning := range warnings {
|
|
if warning.ReasonCode == reason {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|