224 lines
13 KiB
Go
224 lines
13 KiB
Go
package itemregistry
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/entityreconcile"
|
|
identityvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/identity"
|
|
"gitea.maximumdirect.net/eric/promptkit"
|
|
)
|
|
|
|
func TestModuleContractAndMetadata(t *testing.T) {
|
|
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.ItemRegistryKind}
|
|
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
|
}
|
|
registry := pipeline.NewNormalizerRegistry()
|
|
if err := Register(registry); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := New(nil, Options{}); err == nil {
|
|
t.Fatal("New() accepted nil client")
|
|
}
|
|
metadata := newNormalizer(t, &recordingNormalizerClient{}).ManifestMetadata()
|
|
if metadata["identity_policy"] != identity.Policy || metadata["response_schema_id"] != entityreconcile.ResponseSchemaID || metadata["normalization_policy"] != normalizationPolicy || metadata["semantic_context_radius"] != semanticContextRadius {
|
|
t.Fatalf("metadata = %#v", metadata)
|
|
}
|
|
}
|
|
|
|
func TestNormalizeConsolidatesEqualNamesAcrossEvidenceWithoutMutation(t *testing.T) {
|
|
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{
|
|
{ID: 1, Text: "The rope is secured."},
|
|
{ID: 2, Text: "The party takes the rope."},
|
|
{ID: 3, Text: "The rope is packed away."},
|
|
{ID: 4, Text: "A lantern lights the path."},
|
|
}}
|
|
input := dnd.ItemRegistry{Items: []dnd.Item{
|
|
{Name: " Rope ", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 3, EndUnitID: 3}, {SourceID: "session", StartUnitID: 1, EndUnitID: 1}}},
|
|
{Name: "rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}}},
|
|
{Name: "Lantern", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 4, EndUnitID: 4}}},
|
|
}}
|
|
before := dnd.ItemRegistry{Items: []dnd.Item{
|
|
{Name: " Rope ", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 3, EndUnitID: 3}, {SourceID: "session", StartUnitID: 1, EndUnitID: 1}}},
|
|
{Name: "rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}}},
|
|
{Name: "Lantern", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 4, EndUnitID: 4}}},
|
|
}}
|
|
client := &recordingNormalizerClient{}
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err != nil || len(result.Value.Items) != 2 || !reflect.DeepEqual(input, before) || len(client.requests) != 1 {
|
|
t.Fatalf("Normalize() = %#v, %v; want deterministic non-mutating consolidation", result, err)
|
|
}
|
|
rope := result.Value.Items[0]
|
|
wantRefs := []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}, {SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}}
|
|
if rope.Name != "Rope" || rope.ID != identity.DeriveID("Rope") || !reflect.DeepEqual(rope.SourceRefs, wantRefs) || result.Value.Items[1].Name != "Lantern" || !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) {
|
|
t.Fatalf("items = %#v, warnings = %#v; want earliest display name, canonical evidence union, and stable placement", result.Value.Items, result.Warnings)
|
|
}
|
|
validation, validationErr := identityvalidator.New(identityvalidator.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Value: result.Value})
|
|
if validationErr != nil || !validation.Approved {
|
|
t.Fatalf("default item-registry identity validation = %#v, %v; want normalized cross-chunk candidates accepted", validation, validationErr)
|
|
}
|
|
var candidates struct {
|
|
Candidates []struct {
|
|
Name string `json:"name"`
|
|
} `json:"candidates"`
|
|
}
|
|
if err := json.Unmarshal(client.requests[0].Inputs["candidates"].Content, &candidates); err != nil || len(candidates.Candidates) != 2 || candidates.Candidates[0].Name != "Rope" || candidates.Candidates[1].Name != "Lantern" {
|
|
t.Fatalf("semantic candidates = %#v, %v; want one candidate per comparison name", candidates, err)
|
|
}
|
|
repeated, repeatErr := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(result.Value, doc))
|
|
if repeatErr != nil || !reflect.DeepEqual(repeated.Value, result.Value) {
|
|
t.Fatalf("repeated normalization = %#v, %v; want stable output", repeated, repeatErr)
|
|
}
|
|
}
|
|
|
|
func TestNormalizeAppliesSafeAliasProposalAndPreservesCurrency(t *testing.T) {
|
|
doc := semanticDocument()
|
|
input := dnd.ItemRegistry{Items: []dnd.Item{
|
|
{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}},
|
|
{Name: "Compass of the Stars", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
|
{Name: "Gold Pieces", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
|
}}
|
|
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000002"}]}`}
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err != nil || result.Retry != nil || len(result.Value.Items) != 2 {
|
|
t.Fatalf("Normalize() = %#v, %v", result, err)
|
|
}
|
|
merged := result.Value.Items[0]
|
|
if merged.Name != "Compass of the Stars" || merged.ID != identity.DeriveID(merged.Name) || len(merged.SourceRefs) != 2 || !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) {
|
|
t.Fatalf("merged item = %#v, warnings = %#v", merged, result.Warnings)
|
|
}
|
|
encoded := string(client.requests[0].Inputs["candidates"].Content) + string(client.requests[0].Inputs["transcript"].Content)
|
|
if strings.Contains(encoded, doc.ID) || !strings.Contains(encoded, "candidate-000001") || strings.Contains(encoded, merged.ID) {
|
|
t.Fatalf("private inputs = %s", encoded)
|
|
}
|
|
currencyClient := &recordingNormalizerClient{response: `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000001"}]}`}
|
|
currency := dnd.ItemRegistry{Items: []dnd.Item{{Name: "Gold Pieces", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Silver Pieces", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}}}
|
|
result, err = newNormalizer(t, currencyClient).Normalize(context.Background(), normalizeRequestWithSource(currency, doc))
|
|
if err != nil || result.Retry == nil || len(result.Value.Items) != 2 || !hasWarning(result.Warnings, ReasonCodeItemSemanticProposalInvalid) {
|
|
t.Fatalf("currency result = %#v, %v; want denominations retained", result, err)
|
|
}
|
|
}
|
|
|
|
func TestNormalizePreservesCandidatesForUnsafeProposalGroups(t *testing.T) {
|
|
doc := semanticDocument()
|
|
input := dnd.ItemRegistry{Items: []dnd.Item{
|
|
{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}},
|
|
{Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
|
{Name: "Rope", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
|
}}
|
|
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000001"},{"members":["candidate-000002","candidate-000003"],"canonical":"candidate-000003"},{"members":["candidate-000001","candidate-000003"],"canonical":"candidate-000099"}]}`}
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err != nil || result.Retry == nil || len(result.Value.Items) != 3 || !strings.Contains(result.Retry.Message, "overlapping_member") || !strings.Contains(result.Retry.Message, "canonical_unknown") || strings.Contains(result.Retry.Message, "Star Compass") || len(result.Retry.Message) > 4096 {
|
|
t.Fatalf("Normalize() = %#v, %v; want deterministic retry fallback", result, err)
|
|
}
|
|
}
|
|
|
|
func TestNormalizeRetryFallbackErrorsWarningsAndIdempotence(t *testing.T) {
|
|
doc := semanticDocument()
|
|
input := dnd.ItemRegistry{Items: []dnd.Item{{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}}}
|
|
invalid, err := newNormalizer(t, &recordingNormalizerClient{err: contracts.ErrInvalidStructuredOutput}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err != nil || invalid.Retry == nil || invalid.Retry.ReasonCode != ReasonCodeItemSemanticProposalInvalid {
|
|
t.Fatalf("invalid result = %#v, %v", invalid, err)
|
|
}
|
|
_, err = newNormalizer(t, &recordingNormalizerClient{err: errors.New("provider unavailable")}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err == nil || !strings.Contains(err.Error(), "provider unavailable") {
|
|
t.Fatalf("provider error = %v", err)
|
|
}
|
|
if bounded := limitWarningsForRetry(make([]contracts.Warning, diagnostics.MaxWarnings+5)); len(bounded) != diagnostics.MaxWarnings-1 || bounded[len(bounded)-1].ReasonCode != ReasonCodeItemNormalizationWarningsOmitted {
|
|
t.Fatalf("retry warning limit = %#v", bounded)
|
|
}
|
|
first, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
second, secondErr := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(first.Value, doc))
|
|
if err != nil || secondErr != nil || !reflect.DeepEqual(first.Value, second.Value) {
|
|
t.Fatalf("normalization must be idempotent: first=%#v err=%v second=%#v err=%v", first, err, second, secondErr)
|
|
}
|
|
}
|
|
|
|
func TestRegisterPromptAssetsPreparesItemNormalizationPrompt(t *testing.T) {
|
|
registry := llm.NewAssetRegistry()
|
|
if err := entityreconcile.RegisterSchemaAssets(registry); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := RegisterPromptAssets(registry); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
options, err := registry.PromptKitOptions()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ID: "item-normalize-test", Endpoint: "http://127.0.0.1:1/v1", Model: "test"})))
|
|
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: entityreconcile.SchemaVersion, ProfileID: "item-normalize-test", Inputs: map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidates":[{"key":"candidate-000001","name":"Rope","source_refs":[]}]}`), "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`)}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if prepared.OutputContract.SchemaPath != "dnd_entity_reconcile_llm.v1.json" || !strings.Contains(prepared.Messages[1].Content, "currency denominations") || !strings.Contains(prepared.Messages[1].Content, "materially different item types") {
|
|
t.Fatalf("prepared prompt = %#v", prepared)
|
|
}
|
|
}
|
|
|
|
type recordingNormalizerClient struct {
|
|
response string
|
|
err error
|
|
requests []contracts.StructuredCompletionRequest
|
|
}
|
|
|
|
func (c *recordingNormalizerClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, output any) (contracts.StructuredCompletionResponse, error) {
|
|
c.requests = append(c.requests, request)
|
|
if c.err != nil {
|
|
return contracts.StructuredCompletionResponse{}, c.err
|
|
}
|
|
response := c.response
|
|
if response == "" {
|
|
response = `{"duplicate_groups":[]}`
|
|
}
|
|
if err := json.Unmarshal([]byte(response), output); err != nil {
|
|
return contracts.StructuredCompletionResponse{}, err
|
|
}
|
|
return contracts.StructuredCompletionResponse{Content: json.RawMessage(response)}, nil
|
|
}
|
|
|
|
func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normalizer {
|
|
t.Helper()
|
|
normalizer, err := New(client, Options{})
|
|
if err != nil {
|
|
t.Fatalf("New() error = %v", err)
|
|
}
|
|
return normalizer
|
|
}
|
|
func normalizeRequest(value dnd.ItemRegistry) contracts.TypedNormalizeRequest[dnd.ItemRegistry] {
|
|
return contracts.TypedNormalizeRequest[dnd.ItemRegistry]{MergeOutput: contracts.MergeArtifact[dnd.ItemRegistry]{Value: value}}
|
|
}
|
|
func normalizeRequestWithSource(value dnd.ItemRegistry, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.ItemRegistry] {
|
|
request := normalizeRequest(value)
|
|
request.Source = doc
|
|
return request
|
|
}
|
|
func semanticDocument() *source.SourceDocument {
|
|
return &source.SourceDocument{ID: "item-session", Units: []source.SourceUnit{{ID: 10, Text: "The Star Compass points north."}, {ID: 20, Text: "The compass of the stars glows."}, {ID: 30, Text: "The chest holds gold pieces."}}}
|
|
}
|
|
func hasWarning(warnings []contracts.Warning, reason string) bool {
|
|
for _, warning := range warnings {
|
|
if warning.ReasonCode == reason {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|