410 lines
22 KiB
Go
410 lines
22 KiB
Go
package itemregistry
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"reflect"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
|
|
identityvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/identity"
|
|
"gitea.maximumdirect.net/eric/promptkit"
|
|
)
|
|
|
|
func TestModuleContractAndMetadata(t *testing.T) {
|
|
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.ItemRegistryKind}
|
|
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
|
}
|
|
registry := pipeline.NewNormalizerRegistry()
|
|
if err := Register(registry); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := New(nil, Options{}); err == nil {
|
|
t.Fatal("New() accepted nil client")
|
|
}
|
|
metadata := newNormalizer(t, &recordingNormalizerClient{}).ManifestMetadata()
|
|
limits, ok := metadata["semantic_reconciliation_limits"].(map[string]any)
|
|
if !ok || metadata["identity_policy"] != identity.Policy || metadata["normalization_policy"] != normalizationPolicy || metadata["prompt_id"] != PromptID || metadata["prompt_version"] != PromptVersion || metadata["response_schema_key"] != string(semanticreconcile.ResponseSchemaKey) || metadata["response_schema_id"] != semanticreconcile.ResponseSchemaID || metadata["response_schema_name"] != semanticreconcile.ResponseSchemaName || metadata["semantic_reconciliation_policy"] != semanticreconcile.Policy || len(limits) != 3 {
|
|
t.Fatalf("metadata = %#v", metadata)
|
|
}
|
|
for _, name := range []string{"prompt", "response_schema", "semantic_reconciliation_policy", "semantic_reconciliation_limits", "identity_policy", "normalization_policy"} {
|
|
if !hasFingerprint(newNormalizer(t, &recordingNormalizerClient{}).CheckpointFingerprints(), name) {
|
|
t.Fatalf("fingerprints missing %q", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNormalizeConsolidatesEqualNamesAcrossEvidenceWithoutMutation(t *testing.T) {
|
|
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{
|
|
{ID: 1, Text: "The rope is secured."},
|
|
{ID: 2, Text: "The party takes the rope."},
|
|
{ID: 3, Text: "The rope is packed away."},
|
|
{ID: 4, Text: "A lantern lights the path."},
|
|
}}
|
|
input := dnd.ItemRegistry{Items: []dnd.Item{
|
|
{Name: " Rope ", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 3, EndUnitID: 3}, {SourceID: "session", StartUnitID: 1, EndUnitID: 1}}},
|
|
{Name: "rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}}},
|
|
{Name: "Lantern", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 4, EndUnitID: 4}}},
|
|
}}
|
|
before := dnd.ItemRegistry{Items: []dnd.Item{
|
|
{Name: " Rope ", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 3, EndUnitID: 3}, {SourceID: "session", StartUnitID: 1, EndUnitID: 1}}},
|
|
{Name: "rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}}},
|
|
{Name: "Lantern", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 4, EndUnitID: 4}}},
|
|
}}
|
|
client := &recordingNormalizerClient{}
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err != nil || len(result.Value.Items) != 2 || !reflect.DeepEqual(input, before) || len(client.requests) != 1 {
|
|
t.Fatalf("Normalize() = %#v, %v; want deterministic non-mutating consolidation", result, err)
|
|
}
|
|
rope := result.Value.Items[0]
|
|
wantRefs := []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}, {SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}}
|
|
if rope.Name != "Rope" || rope.ID != identity.DeriveID("Rope") || !reflect.DeepEqual(rope.SourceRefs, wantRefs) || result.Value.Items[1].Name != "Lantern" || !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) {
|
|
t.Fatalf("items = %#v, warnings = %#v; want earliest display name, canonical evidence union, and stable placement", result.Value.Items, result.Warnings)
|
|
}
|
|
validation, validationErr := identityvalidator.New(identityvalidator.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Value: result.Value})
|
|
if validationErr != nil || !validation.Approved {
|
|
t.Fatalf("default item-registry identity validation = %#v, %v; want normalized cross-chunk candidates accepted", validation, validationErr)
|
|
}
|
|
var candidates struct {
|
|
Candidates []struct {
|
|
Label string `json:"label"`
|
|
} `json:"candidates"`
|
|
}
|
|
if err := json.Unmarshal(client.requests[0].Inputs["candidates"].Content, &candidates); err != nil || len(candidates.Candidates) != 2 || candidates.Candidates[0].Label != "Rope" || candidates.Candidates[1].Label != "Lantern" {
|
|
t.Fatalf("semantic candidates = %#v, %v; want one candidate per comparison name", candidates, err)
|
|
}
|
|
repeated, repeatErr := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(result.Value, doc))
|
|
if repeatErr != nil || !reflect.DeepEqual(repeated.Value, result.Value) {
|
|
t.Fatalf("repeated normalization = %#v, %v; want stable output", repeated, repeatErr)
|
|
}
|
|
}
|
|
|
|
func BenchmarkComparisonNameGroupsManyDistinct(b *testing.B) {
|
|
records := make([]normalizedRecord, 1_000)
|
|
for index := range records {
|
|
records[index] = normalizedRecord{item: dnd.Item{Name: "Item " + strconv.Itoa(index)}}
|
|
}
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for iteration := 0; iteration < b.N; iteration++ {
|
|
if groups := comparisonNameGroups(records); len(groups) != len(records) {
|
|
b.Fatalf("group count = %d, want %d", len(groups), len(records))
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNormalizeAppliesSafeAliasProposal(t *testing.T) {
|
|
doc := semanticDocument()
|
|
input := dnd.ItemRegistry{Items: []dnd.Item{
|
|
{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}},
|
|
{Name: "Compass of the Stars", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
|
{Name: "Gold Pieces", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
|
}}
|
|
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2}]}`}
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err != nil || result.Retry != nil || len(result.Value.Items) != 2 {
|
|
t.Fatalf("Normalize() = %#v, %v", result, err)
|
|
}
|
|
merged := result.Value.Items[0]
|
|
if merged.Name != "Compass of the Stars" || merged.ID != identity.DeriveID(merged.Name) || len(merged.SourceRefs) != 2 || !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) {
|
|
t.Fatalf("merged item = %#v, warnings = %#v", merged, result.Warnings)
|
|
}
|
|
encoded := string(client.requests[0].Inputs["candidates"].Content) + string(client.requests[0].Inputs["transcript"].Content)
|
|
if strings.Contains(encoded, doc.ID) || strings.Contains(encoded, "candidate-") || strings.Contains(encoded, merged.ID) || !strings.Contains(encoded, `"source_refs"`) {
|
|
t.Fatalf("private inputs = %s", encoded)
|
|
}
|
|
}
|
|
|
|
func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
|
|
doc := semanticDocument()
|
|
ref := func(unitID int) []source.SourceRef {
|
|
return []source.SourceRef{{SourceID: doc.ID, StartUnitID: unitID, EndUnitID: unitID}}
|
|
}
|
|
for _, test := range []struct {
|
|
name string
|
|
items []dnd.Item
|
|
response string
|
|
wantNames []string
|
|
wantRefCounts []int
|
|
wantRetry bool
|
|
warning string
|
|
}{
|
|
{
|
|
name: "same denomination aliases",
|
|
items: []dnd.Item{
|
|
{Name: "GP", SourceRefs: ref(10)},
|
|
{Name: "Gold Piece", SourceRefs: ref(20)},
|
|
{Name: "Gold Pieces", SourceRefs: ref(30)},
|
|
},
|
|
response: `{"duplicate_groups":[{"candidate_ids":[1,2,3],"canonical_candidate_id":2}]}`,
|
|
wantNames: []string{"Gold Piece"},
|
|
wantRefCounts: []int{3},
|
|
warning: ReasonCodeDuplicateItemCollapsed,
|
|
},
|
|
{
|
|
name: "different denominations",
|
|
items: []dnd.Item{
|
|
{Name: "Gold Pieces", SourceRefs: ref(10)},
|
|
{Name: "Silver Pieces", SourceRefs: ref(20)},
|
|
},
|
|
response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":1}]}`,
|
|
wantNames: []string{"Gold Pieces", "Silver Pieces"},
|
|
wantRefCounts: []int{1, 1},
|
|
wantRetry: true,
|
|
warning: ReasonCodeItemSemanticProposalInvalid,
|
|
},
|
|
{
|
|
name: "currency plus ordinary item",
|
|
items: []dnd.Item{
|
|
{Name: "Gold Pieces", SourceRefs: ref(10)},
|
|
{Name: "Longsword", SourceRefs: ref(20)},
|
|
},
|
|
response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":1}]}`,
|
|
wantNames: []string{"Gold Pieces", "Longsword"},
|
|
wantRefCounts: []int{1, 1},
|
|
wantRetry: true,
|
|
warning: ReasonCodeItemSemanticProposalInvalid,
|
|
},
|
|
{
|
|
name: "ordinary items",
|
|
items: []dnd.Item{
|
|
{Name: "Star Compass", SourceRefs: ref(10)},
|
|
{Name: "Compass of the Stars", SourceRefs: ref(20)},
|
|
},
|
|
response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2}]}`,
|
|
wantNames: []string{"Compass of the Stars"},
|
|
wantRefCounts: []int{2},
|
|
warning: ReasonCodeDuplicateItemCollapsed,
|
|
},
|
|
{
|
|
name: "currency plus ordinary canonical item",
|
|
items: []dnd.Item{
|
|
{Name: "Gold Pieces", SourceRefs: ref(10)},
|
|
{Name: "Longsword", SourceRefs: ref(20)},
|
|
},
|
|
response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2}]}`,
|
|
wantNames: []string{"Gold Pieces", "Longsword"},
|
|
wantRefCounts: []int{1, 1},
|
|
wantRetry: true,
|
|
warning: ReasonCodeItemSemanticProposalInvalid,
|
|
},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
input := dnd.ItemRegistry{Items: make([]dnd.Item, len(test.items))}
|
|
for index, item := range test.items {
|
|
input.Items[index] = cloneItem(item)
|
|
}
|
|
before := dnd.ItemRegistry{Items: make([]dnd.Item, len(input.Items))}
|
|
for index, item := range input.Items {
|
|
before.Items[index] = cloneItem(item)
|
|
}
|
|
|
|
result, err := newNormalizer(t, &recordingNormalizerClient{response: test.response}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err != nil || (result.Retry != nil) != test.wantRetry || !reflect.DeepEqual(input, before) || !hasWarning(result.Warnings, test.warning) {
|
|
t.Fatalf("Normalize() = %#v, %v", result, err)
|
|
}
|
|
if len(result.Value.Items) != len(test.wantNames) {
|
|
t.Fatalf("items = %#v, want names %#v", result.Value.Items, test.wantNames)
|
|
}
|
|
for index, item := range result.Value.Items {
|
|
if item.Name != test.wantNames[index] || len(item.SourceRefs) != test.wantRefCounts[index] {
|
|
t.Fatalf("item %d = %#v, want name %q with %d source refs", index, item, test.wantNames[index], test.wantRefCounts[index])
|
|
}
|
|
}
|
|
if test.wantRetry && (len(result.Retry.FallbackWarnings) != 1 || result.Retry.FallbackWarnings[0].ReasonCode != ReasonCodeItemSemanticReconciliationExhausted) {
|
|
t.Fatalf("retry = %#v, want preserved-group fallback", result.Retry)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNormalizePreservesCandidatesForUnsafeProposalGroups(t *testing.T) {
|
|
doc := semanticDocument()
|
|
input := dnd.ItemRegistry{Items: []dnd.Item{
|
|
{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}},
|
|
{Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
|
|
{Name: "Rope", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
|
|
}}
|
|
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":1},{"candidate_ids":[2,3],"canonical_candidate_id":3},{"candidate_ids":[1,3],"canonical_candidate_id":99}]}`}
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err != nil || result.Retry == nil || len(result.Value.Items) != 3 || !strings.Contains(result.Retry.Message, "overlapping_member") || !strings.Contains(result.Retry.Message, "canonical_unknown") || strings.Contains(result.Retry.Message, "Star Compass") || len(result.Retry.Message) > 4096 {
|
|
t.Fatalf("Normalize() = %#v, %v; want deterministic retry fallback", result, err)
|
|
}
|
|
}
|
|
|
|
func TestNormalizeAppliesIndependentGroupAndCountsAllOmissions(t *testing.T) {
|
|
doc := semanticDocument()
|
|
ref := func(unitID int) []source.SourceRef {
|
|
return []source.SourceRef{{SourceID: doc.ID, StartUnitID: unitID, EndUnitID: unitID}}
|
|
}
|
|
input := dnd.ItemRegistry{Items: []dnd.Item{
|
|
{Name: "Star Compass", SourceRefs: ref(10)},
|
|
{Name: "Compass of the Stars", SourceRefs: ref(20)},
|
|
{Name: "Gold Pieces", SourceRefs: ref(30)},
|
|
{Name: "Silver Pieces", SourceRefs: ref(30)},
|
|
{Name: "Rope", SourceRefs: ref(10)},
|
|
}}
|
|
client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_ids":[1,2],"canonical_candidate_id":2},{"candidate_ids":[3,4],"canonical_candidate_id":3},{"candidate_ids":[5,99],"canonical_candidate_id":5}]}`}
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err != nil || result.Retry == nil {
|
|
t.Fatalf("Normalize() = %#v, %v; want retry with independently accepted output", result, err)
|
|
}
|
|
wantNames := []string{"Compass of the Stars", "Gold Pieces", "Silver Pieces", "Rope"}
|
|
if len(result.Value.Items) != len(wantNames) {
|
|
t.Fatalf("items = %#v, want %v", result.Value.Items, wantNames)
|
|
}
|
|
for index, name := range wantNames {
|
|
if result.Value.Items[index].Name != name {
|
|
t.Fatalf("item %d = %#v, want %q", index, result.Value.Items[index], name)
|
|
}
|
|
}
|
|
if !hasWarning(result.Warnings, ReasonCodeDuplicateItemCollapsed) || !hasWarning(result.Warnings, ReasonCodeItemSemanticProposalInvalid) {
|
|
t.Fatalf("warnings = %#v, want accepted and guarded-group diagnostics", result.Warnings)
|
|
}
|
|
if len(result.Retry.FallbackWarnings) != 1 || !strings.Contains(result.Retry.FallbackWarnings[0].Message, "2 proposal group(s)") {
|
|
t.Fatalf("retry = %#v, want one guarded and one malformed group counted", result.Retry)
|
|
}
|
|
}
|
|
|
|
func TestNormalizeLimitSkipDoesNotCallLLMAndAddsBoundedFallbackWarning(t *testing.T) {
|
|
client := &recordingNormalizerClient{}
|
|
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Kind: "narration", Text: "A crowded storeroom"}}}
|
|
limit := semanticreconcile.DefaultLimits().MaximumCandidates
|
|
input := dnd.ItemRegistry{Items: make([]dnd.Item, limit+1)}
|
|
for index := range input.Items {
|
|
input.Items[index] = dnd.Item{Name: fmt.Sprintf("Item %d", index), SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
|
|
}
|
|
result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err != nil || result.Retry != nil {
|
|
t.Fatalf("Normalize() = %#v, %v; want deterministic limit fallback", result, err)
|
|
}
|
|
if len(client.requests) != 0 || len(result.Value.Items) != limit+1 {
|
|
t.Fatalf("completion calls = %d, items = %d; want no call and all records", len(client.requests), len(result.Value.Items))
|
|
}
|
|
if !hasWarning(result.Warnings, ReasonCodeItemSemanticReconciliationExhausted) || len(result.Warnings) > diagnostics.MaxWarnings {
|
|
t.Fatalf("warnings = %#v, want bounded reconciliation fallback", result.Warnings)
|
|
}
|
|
}
|
|
|
|
func TestNormalizeRetryFallbackErrorsWarningsAndIdempotence(t *testing.T) {
|
|
doc := semanticDocument()
|
|
input := dnd.ItemRegistry{Items: []dnd.Item{{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}}}
|
|
invalid, err := newNormalizer(t, &recordingNormalizerClient{err: contracts.ErrInvalidStructuredOutput}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err != nil || invalid.Retry == nil || invalid.Retry.ReasonCode != ReasonCodeItemSemanticProposalInvalid {
|
|
t.Fatalf("invalid result = %#v, %v", invalid, err)
|
|
}
|
|
_, err = newNormalizer(t, &recordingNormalizerClient{err: errors.New("provider unavailable")}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
if err == nil || !strings.Contains(err.Error(), "provider unavailable") {
|
|
t.Fatalf("provider error = %v", err)
|
|
}
|
|
if bounded := limitWarningsForRetry(make([]contracts.Warning, diagnostics.MaxWarnings+5)); len(bounded) != diagnostics.MaxWarnings-1 || bounded[len(bounded)-1].ReasonCode != ReasonCodeItemNormalizationWarningsOmitted {
|
|
t.Fatalf("retry warning limit = %#v", bounded)
|
|
}
|
|
first, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
|
|
second, secondErr := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(first.Value, doc))
|
|
if err != nil || secondErr != nil || !reflect.DeepEqual(first.Value, second.Value) {
|
|
t.Fatalf("normalization must be idempotent: first=%#v err=%v second=%#v err=%v", first, err, second, secondErr)
|
|
}
|
|
}
|
|
|
|
func TestRegisterPromptAssetsPreparesItemNormalizationPrompt(t *testing.T) {
|
|
registry := llm.NewAssetRegistry()
|
|
if err := semanticreconcile.RegisterAssets(registry); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := RegisterPromptAssets(registry); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
options, err := registry.PromptKitOptions()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ID: "item-normalize-test", Endpoint: "http://127.0.0.1:1/v1", Model: "test"})))
|
|
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "item-normalize-test", Inputs: map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidates":[{"candidate_id":1,"label":"Rope","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`)}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if prepared.OutputContract.SchemaPath != "semantic_reconciliation_llm.v1.json" || !strings.Contains(prepared.Messages[1].Content, "candidate_id") || !strings.Contains(prepared.Messages[1].Content, "integer") || !strings.Contains(prepared.Messages[2].Content, "currency denominations") || !strings.Contains(prepared.Messages[2].Content, "materially different item") || prepared.Messages[2].CacheControl == nil || prepared.Messages[2].CacheControl.Type != promptkit.CacheControlEphemeral {
|
|
t.Fatalf("prepared prompt = %#v", prepared)
|
|
}
|
|
if prepared.Messages[4].CacheControl == nil || prepared.Messages[4].CacheControl.Type != promptkit.CacheControlEphemeral || !strings.Contains(prepared.Messages[3].Content, `"Rope"`) || strings.Contains(prepared.Messages[3].Content, `"windows"`) || !strings.Contains(prepared.Messages[4].Content, `"windows"`) || strings.Contains(prepared.Messages[4].Content, `"Rope"`) {
|
|
t.Fatalf("prepared prompt = %#v, want isolated candidate and transcript presentation", prepared)
|
|
}
|
|
}
|
|
|
|
type recordingNormalizerClient struct {
|
|
response string
|
|
err error
|
|
requests []contracts.StructuredCompletionRequest
|
|
}
|
|
|
|
func (c *recordingNormalizerClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, output any) (contracts.StructuredCompletionResponse, error) {
|
|
c.requests = append(c.requests, request)
|
|
if c.err != nil {
|
|
return contracts.StructuredCompletionResponse{}, c.err
|
|
}
|
|
response := c.response
|
|
if response == "" {
|
|
response = `{"duplicate_groups":[]}`
|
|
}
|
|
content := []byte(response)
|
|
if err := json.Unmarshal(content, output); err != nil {
|
|
return contracts.StructuredCompletionResponse{}, err
|
|
}
|
|
return contracts.StructuredCompletionResponse{Content: content}, nil
|
|
}
|
|
|
|
func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normalizer {
|
|
t.Helper()
|
|
normalizer, err := New(client, Options{})
|
|
if err != nil {
|
|
t.Fatalf("New() error = %v", err)
|
|
}
|
|
return normalizer
|
|
}
|
|
func normalizeRequest(value dnd.ItemRegistry) contracts.TypedNormalizeRequest[dnd.ItemRegistry] {
|
|
return contracts.TypedNormalizeRequest[dnd.ItemRegistry]{MergeOutput: contracts.MergeArtifact[dnd.ItemRegistry]{Value: value}}
|
|
}
|
|
func normalizeRequestWithSource(value dnd.ItemRegistry, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.ItemRegistry] {
|
|
request := normalizeRequest(value)
|
|
request.Source = doc
|
|
return request
|
|
}
|
|
func semanticDocument() *source.SourceDocument {
|
|
return &source.SourceDocument{ID: "item-session", Units: []source.SourceUnit{{ID: 10, Text: "The Star Compass points north."}, {ID: 20, Text: "The compass of the stars glows."}, {ID: 30, Text: "The chest holds gold pieces."}}}
|
|
}
|
|
func hasWarning(warnings []contracts.Warning, reason string) bool {
|
|
for _, warning := range warnings {
|
|
if warning.ReasonCode == reason {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func hasFingerprint(fingerprints []pipeline.CheckpointFingerprint, name string) bool {
|
|
for _, fingerprint := range fingerprints {
|
|
if fingerprint.Name == name && fingerprint.Value != "" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|