Migrate item reconciliation to shared engine
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -14,10 +15,10 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/entityreconcile"
|
||||
identityvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/identity"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
@@ -35,9 +36,15 @@ func TestModuleContractAndMetadata(t *testing.T) {
|
||||
t.Fatal("New() accepted nil client")
|
||||
}
|
||||
metadata := newNormalizer(t, &recordingNormalizerClient{}).ManifestMetadata()
|
||||
if metadata["identity_policy"] != identity.Policy || metadata["response_schema_id"] != entityreconcile.ResponseSchemaID || metadata["normalization_policy"] != normalizationPolicy || metadata["semantic_context_radius"] != semanticContextRadius {
|
||||
limits, ok := metadata["semantic_reconciliation_limits"].(map[string]any)
|
||||
if !ok || metadata["identity_policy"] != identity.Policy || metadata["normalization_policy"] != normalizationPolicy || metadata["prompt_id"] != PromptID || metadata["prompt_version"] != PromptVersion || metadata["response_schema_key"] != string(semanticreconcile.ResponseSchemaKey) || metadata["response_schema_id"] != semanticreconcile.ResponseSchemaID || metadata["response_schema_name"] != semanticreconcile.ResponseSchemaName || metadata["semantic_reconciliation_policy"] != semanticreconcile.Policy || len(limits) != 3 {
|
||||
t.Fatalf("metadata = %#v", metadata)
|
||||
}
|
||||
for _, name := range []string{"prompt", "response_schema", "semantic_reconciliation_policy", "semantic_reconciliation_limits", "identity_policy", "normalization_policy"} {
|
||||
if !hasFingerprint(newNormalizer(t, &recordingNormalizerClient{}).CheckpointFingerprints(), name) {
|
||||
t.Fatalf("fingerprints missing %q", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeConsolidatesEqualNamesAcrossEvidenceWithoutMutation(t *testing.T) {
|
||||
@@ -73,10 +80,10 @@ func TestNormalizeConsolidatesEqualNamesAcrossEvidenceWithoutMutation(t *testing
|
||||
}
|
||||
var candidates struct {
|
||||
Candidates []struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
} `json:"candidates"`
|
||||
}
|
||||
if err := json.Unmarshal(client.requests[0].Inputs["candidates"].Content, &candidates); err != nil || len(candidates.Candidates) != 2 || candidates.Candidates[0].Name != "Rope" || candidates.Candidates[1].Name != "Lantern" {
|
||||
if err := json.Unmarshal(client.requests[0].Inputs["candidates"].Content, &candidates); err != nil || len(candidates.Candidates) != 2 || candidates.Candidates[0].Label != "Rope" || candidates.Candidates[1].Label != "Lantern" {
|
||||
t.Fatalf("semantic candidates = %#v, %v; want one candidate per comparison name", candidates, err)
|
||||
}
|
||||
repeated, repeatErr := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(result.Value, doc))
|
||||
@@ -106,7 +113,7 @@ func TestNormalizeAppliesSafeAliasProposal(t *testing.T) {
|
||||
{Name: "Compass of the Stars", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
||||
{Name: "Gold Pieces", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||||
}}
|
||||
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000002"}]}`}
|
||||
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2}]}`}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry != nil || len(result.Value.Items) != 2 {
|
||||
t.Fatalf("Normalize() = %#v, %v", result, err)
|
||||
@@ -142,7 +149,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
{Name: "Gold Piece", SourceRefs: ref(20)},
|
||||
{Name: "Gold Pieces", SourceRefs: ref(30)},
|
||||
},
|
||||
response: `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002","candidate-000003"],"canonical":"candidate-000002"}]}`,
|
||||
response: `{"duplicate_groups":[{"candidate_ids":[1,2,3],"canonical_candidate_id":2}]}`,
|
||||
wantNames: []string{"Gold Piece"},
|
||||
wantRefCounts: []int{3},
|
||||
warning: ReasonCodeDuplicateItemCollapsed,
|
||||
@@ -153,7 +160,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
{Name: "Gold Pieces", SourceRefs: ref(10)},
|
||||
{Name: "Silver Pieces", SourceRefs: ref(20)},
|
||||
},
|
||||
response: `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000001"}]}`,
|
||||
response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":1}]}`,
|
||||
wantNames: []string{"Gold Pieces", "Silver Pieces"},
|
||||
wantRefCounts: []int{1, 1},
|
||||
wantRetry: true,
|
||||
@@ -165,7 +172,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
{Name: "Gold Pieces", SourceRefs: ref(10)},
|
||||
{Name: "Longsword", SourceRefs: ref(20)},
|
||||
},
|
||||
response: `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000001"}]}`,
|
||||
response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":1}]}`,
|
||||
wantNames: []string{"Gold Pieces", "Longsword"},
|
||||
wantRefCounts: []int{1, 1},
|
||||
wantRetry: true,
|
||||
@@ -177,7 +184,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
{Name: "Star Compass", SourceRefs: ref(10)},
|
||||
{Name: "Compass of the Stars", SourceRefs: ref(20)},
|
||||
},
|
||||
response: `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000002"}]}`,
|
||||
response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2}]}`,
|
||||
wantNames: []string{"Compass of the Stars"},
|
||||
wantRefCounts: []int{2},
|
||||
warning: ReasonCodeDuplicateItemCollapsed,
|
||||
@@ -188,7 +195,7 @@ func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
||||
{Name: "Gold Pieces", SourceRefs: ref(10)},
|
||||
{Name: "Longsword", SourceRefs: ref(20)},
|
||||
},
|
||||
response: `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000002"}]}`,
|
||||
response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2}]}`,
|
||||
wantNames: []string{"Gold Pieces", "Longsword"},
|
||||
wantRefCounts: []int{1, 1},
|
||||
wantRetry: true,
|
||||
@@ -231,13 +238,67 @@ func TestNormalizePreservesCandidatesForUnsafeProposalGroups(t *testing.T) {
|
||||
{Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
||||
{Name: "Rope", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
||||
}}
|
||||
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000001"},{"members":["candidate-000002","candidate-000003"],"canonical":"candidate-000003"},{"members":["candidate-000001","candidate-000003"],"canonical":"candidate-000099"}]}`}
|
||||
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":1},{"candidate_ids":[2,3],"canonical_candidate_id":3},{"candidate_ids":[1,3],"canonical_candidate_id":99}]}`}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry == nil || len(result.Value.Items) != 3 || !strings.Contains(result.Retry.Message, "overlapping_member") || !strings.Contains(result.Retry.Message, "canonical_unknown") || strings.Contains(result.Retry.Message, "Star Compass") || len(result.Retry.Message) > 4096 {
|
||||
t.Fatalf("Normalize() = %#v, %v; want deterministic retry fallback", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeAppliesIndependentGroupAndCountsAllOmissions(t *testing.T) {
|
||||
doc := semanticDocument()
|
||||
ref := func(unitID int) []source.SourceRef {
|
||||
return []source.SourceRef{{SourceID: doc.ID, StartUnitID: unitID, EndUnitID: unitID}}
|
||||
}
|
||||
input := dnd.ItemRegistry{Items: []dnd.Item{
|
||||
{Name: "Star Compass", SourceRefs: ref(10)},
|
||||
{Name: "Compass of the Stars", SourceRefs: ref(20)},
|
||||
{Name: "Gold Pieces", SourceRefs: ref(30)},
|
||||
{Name: "Silver Pieces", SourceRefs: ref(30)},
|
||||
{Name: "Rope", SourceRefs: ref(10)},
|
||||
}}
|
||||
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2},{"candidate_ids":[3,4],"canonical_candidate_id":3},{"candidate_ids":[5,99],"canonical_candidate_id":5}]}`}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry == nil {
|
||||
t.Fatalf("Normalize() = %#v, %v; want retry with independently accepted output", result, err)
|
||||
}
|
||||
wantNames := []string{"Compass of the Stars", "Gold Pieces", "Silver Pieces", "Rope"}
|
||||
if len(result.Value.Items) != len(wantNames) {
|
||||
t.Fatalf("items = %#v, want %v", result.Value.Items, wantNames)
|
||||
}
|
||||
for index, name := range wantNames {
|
||||
if result.Value.Items[index].Name != name {
|
||||
t.Fatalf("item %d = %#v, want %q", index, result.Value.Items[index], name)
|
||||
}
|
||||
}
|
||||
if !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) || !hasWarning(result.Warnings, ReasonCodeItemSemanticProposalInvalid) {
|
||||
t.Fatalf("warnings = %#v, want accepted and guarded-group diagnostics", result.Warnings)
|
||||
}
|
||||
if len(result.Retry.FallbackWarnings) != 1 || !strings.Contains(result.Retry.FallbackWarnings[0].Message, "2 proposal group(s)") {
|
||||
t.Fatalf("retry = %#v, want one guarded and one malformed group counted", result.Retry)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeLimitSkipDoesNotCallLLMAndAddsBoundedFallbackWarning(t *testing.T) {
|
||||
client := &recordingNormalizerClient{}
|
||||
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Kind: "narration", Text: "A crowded storeroom"}}}
|
||||
limit := semanticreconcile.DefaultLimits().MaximumCandidates
|
||||
input := dnd.ItemRegistry{Items: make([]dnd.Item, limit+1)}
|
||||
for index := range input.Items {
|
||||
input.Items[index] = dnd.Item{Name: fmt.Sprintf("Item %d", index), SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
|
||||
}
|
||||
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
||||
if err != nil || result.Retry != nil {
|
||||
t.Fatalf("Normalize() = %#v, %v; want deterministic limit fallback", result, err)
|
||||
}
|
||||
if len(client.requests) != 0 || len(result.Value.Items) != limit+1 {
|
||||
t.Fatalf("completion calls = %d, items = %d; want no call and all records", len(client.requests), len(result.Value.Items))
|
||||
}
|
||||
if !hasWarning(result.Warnings, ReasonCodeItemSemanticReconciliationExhausted) || len(result.Warnings) > diagnostics.MaxWarnings {
|
||||
t.Fatalf("warnings = %#v, want bounded reconciliation fallback", result.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeRetryFallbackErrorsWarningsAndIdempotence(t *testing.T) {
|
||||
doc := semanticDocument()
|
||||
input := dnd.ItemRegistry{Items: []dnd.Item{{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}}}
|
||||
@@ -261,7 +322,7 @@ func TestNormalizeRetryFallbackErrorsWarningsAndIdempotence(t *testing.T) {
|
||||
|
||||
func TestRegisterPromptAssetsPreparesItemNormalizationPrompt(t *testing.T) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := entityreconcile.RegisterSchemaAssets(registry); err != nil {
|
||||
if err := semanticreconcile.RegisterAssets(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
@@ -276,13 +337,16 @@ func TestRegisterPromptAssetsPreparesItemNormalizationPrompt(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: entityreconcile.SchemaVersion, ProfileID: "item-normalize-test", Inputs: map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidates":[{"name":"Rope","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`)}})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "item-normalize-test", Inputs: map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidates":[{"candidate_id":1,"label":"Rope","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if prepared.OutputContract.SchemaPath != "dnd_entity_reconcile_llm.v1.json" || !strings.Contains(prepared.Messages[1].Content, "currency denominations") || !strings.Contains(prepared.Messages[1].Content, "materially different item types") {
|
||||
if prepared.OutputContract.SchemaPath != "semantic_reconciliation_llm.v1.json" || !strings.Contains(prepared.Messages[1].Content, "candidate_id") || !strings.Contains(prepared.Messages[1].Content, "integer") || !strings.Contains(prepared.Messages[2].Content, "currency denominations") || !strings.Contains(prepared.Messages[2].Content, "materially different item") || prepared.Messages[2].CacheControl == nil || prepared.Messages[2].CacheControl.Type != promptkit.CacheControlEphemeral {
|
||||
t.Fatalf("prepared prompt = %#v", prepared)
|
||||
}
|
||||
if prepared.Messages[4].CacheControl == nil || prepared.Messages[4].CacheControl.Type != promptkit.CacheControlEphemeral || !strings.Contains(prepared.Messages[3].Content, `"Rope"`) || strings.Contains(prepared.Messages[3].Content, `"windows"`) || !strings.Contains(prepared.Messages[4].Content, `"windows"`) || strings.Contains(prepared.Messages[4].Content, `"Rope"`) {
|
||||
t.Fatalf("prepared prompt = %#v, want isolated candidate and transcript presentation", prepared)
|
||||
}
|
||||
}
|
||||
|
||||
type recordingNormalizerClient struct {
|
||||
@@ -300,53 +364,13 @@ func (c *recordingNormalizerClient) CompleteStructured(_ context.Context, reques
|
||||
if response == "" {
|
||||
response = `{"duplicate_groups":[]}`
|
||||
}
|
||||
content, err := contextualProposalResponse(response, request.Inputs["candidates"].Content)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
content := []byte(response)
|
||||
if err := json.Unmarshal(content, output); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
}
|
||||
|
||||
func contextualProposalResponse(response string, candidateContent []byte) ([]byte, error) {
|
||||
if !strings.Contains(response, "candidate-") {
|
||||
return []byte(response), nil
|
||||
}
|
||||
var selection struct {
|
||||
DuplicateGroups []struct {
|
||||
Members []string `json:"members"`
|
||||
Canonical string `json:"canonical"`
|
||||
} `json:"duplicate_groups"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(response), &selection); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var candidates struct {
|
||||
Candidates []entityreconcile.Selector `json:"candidates"`
|
||||
}
|
||||
if err := json.Unmarshal(candidateContent, &candidates); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
selector := func(key string) entityreconcile.Selector {
|
||||
index, err := strconv.Atoi(strings.TrimPrefix(key, "candidate-"))
|
||||
if err != nil || index < 1 || index > len(candidates.Candidates) {
|
||||
return entityreconcile.Selector{Name: key, SourceRefs: []entityreconcile.SourceRange{}}
|
||||
}
|
||||
return candidates.Candidates[index-1].Clone()
|
||||
}
|
||||
proposal := entityreconcile.ProposalResponse{DuplicateGroups: make([]entityreconcile.DuplicateGroup, len(selection.DuplicateGroups))}
|
||||
for index, group := range selection.DuplicateGroups {
|
||||
members := make([]entityreconcile.Selector, len(group.Members))
|
||||
for memberIndex, key := range group.Members {
|
||||
members[memberIndex] = selector(key)
|
||||
}
|
||||
proposal.DuplicateGroups[index] = entityreconcile.DuplicateGroup{Members: members, Canonical: selector(group.Canonical)}
|
||||
}
|
||||
return json.Marshal(proposal)
|
||||
}
|
||||
|
||||
func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normalizer {
|
||||
t.Helper()
|
||||
normalizer, err := New(client, Options{})
|
||||
@@ -374,3 +398,12 @@ func hasWarning(warnings []contracts.Warning, reason string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasFingerprint(fingerprints []pipeline.CheckpointFingerprint, name string) bool {
|
||||
for _, fingerprint := range fingerprints {
|
||||
if fingerprint.Name == name && fingerprint.Value != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user