diff --git a/internal/modules/dnd/normalize/npcs/assets.go b/internal/modules/dnd/normalize/npcs/assets.go index 2990279..7423e70 100644 --- a/internal/modules/dnd/normalize/npcs/assets.go +++ b/internal/modules/dnd/normalize/npcs/assets.go @@ -2,5 +2,5 @@ package npcs import "embed" -//go:embed assets/schemas/dnd_npcs_normalize_llm.v1.json assets/prompts/*.yaml assets/prompts/*.md +//go:embed assets/prompts/*.yaml assets/prompts/*.md var embeddedAssets embed.FS diff --git a/internal/modules/dnd/normalize/npcs/assets/prompts/dnd.npcs.normalize.yaml b/internal/modules/dnd/normalize/npcs/assets/prompts/dnd.npcs.normalize.yaml index 036e0e2..af2c96a 100644 --- a/internal/modules/dnd/normalize/npcs/assets/prompts/dnd.npcs.normalize.yaml +++ b/internal/modules/dnd/normalize/npcs/assets/prompts/dnd.npcs.normalize.yaml @@ -14,7 +14,7 @@ messages: - role: user content_file: ./task.md - role: user - content_file: ./instructions.md + content_file: ./sharedassets/common-dnd-entity-reconciliation.md cache_control: type: ephemeral - role: user @@ -26,5 +26,5 @@ messages: output: format: json validation_mode: json_schema - schema_path: dnd_npcs_normalize_llm.v1.json + schema_path: dnd_entity_reconcile_llm.v1.json repair_attempts: 0 diff --git a/internal/modules/dnd/normalize/npcs/assets/prompts/instructions.md b/internal/modules/dnd/normalize/npcs/assets/prompts/instructions.md deleted file mode 100644 index d6a4896..0000000 --- a/internal/modules/dnd/normalize/npcs/assets/prompts/instructions.md +++ /dev/null @@ -1,10 +0,0 @@ -Return duplicate groups only when the transcript context clearly establishes a -single individual. Prefer no group when identity is ambiguous. - -Copy supplied display names into each group's members. Choose canonical_name -from that same group's supplied members. Prefer a complete stable proper name -over an abbreviation, but prefer an unadorned proper name over that name plus a -contextual class, role, title, or relationship descriptor unless the descriptor -is established as part of the name. - -Do not invent names, source references, replacement records, or explanations. diff --git a/internal/modules/dnd/normalize/npcs/assets/prompts/task.md b/internal/modules/dnd/normalize/npcs/assets/prompts/task.md index b8e6576..6063b29 100644 --- a/internal/modules/dnd/normalize/npcs/assets/prompts/task.md +++ b/internal/modules/dnd/normalize/npcs/assets/prompts/task.md @@ -1,2 +1,8 @@ -Identify only supplied NPC display names that clearly refer to the same -individual in the supplied transcript context. +Review NPC candidates and their cited transcript context to identify aliases +that refer to the same individual. Propose only groups supported by the +transcript, and preserve distinct individuals even when their names are +similar. + +For every accepted group, choose as canonical the supplied candidate whose +display name is the most complete and clear NPC name. Do not invent, edit, or +combine display names. diff --git a/internal/modules/dnd/normalize/npcs/assets/schemas/dnd_npcs_normalize_llm.v1.json b/internal/modules/dnd/normalize/npcs/assets/schemas/dnd_npcs_normalize_llm.v1.json deleted file mode 100644 index 40de207..0000000 --- a/internal/modules/dnd/normalize/npcs/assets/schemas/dnd_npcs_normalize_llm.v1.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "notarius.dnd.npcs.normalize.llm", - "type": "object", - "additionalProperties": false, - "required": ["duplicate_groups"], - "properties": { - "duplicate_groups": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "required": ["members", "canonical_name"], - "properties": { - "members": { - "type": "array", - "items": {"type": "string"} - }, - "canonical_name": {"type": "string"} - } - } - } - } -} diff --git a/internal/modules/dnd/normalize/npcs/context_material.go b/internal/modules/dnd/normalize/npcs/context_material.go deleted file mode 100644 index f6ce328..0000000 --- a/internal/modules/dnd/normalize/npcs/context_material.go +++ /dev/null @@ -1,214 +0,0 @@ -package npcs - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "sort" - - "gitea.maximumdirect.net/eric/notarius/internal/core/source" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" -) - -const semanticContextRadius = 2 - -type normalizeContextMaterials struct { - Candidates contracts.LLMInputMaterial - Transcript contracts.LLMInputMaterial - candidatePositions []int -} - -type normalizeCandidateInput struct { - NPCs []normalizeCandidate `json:"npcs"` -} - -type normalizeCandidate struct { - Name string `json:"name"` - SourceRefs []normalizeCandidateSourceRef `json:"source_refs"` -} - -type normalizeCandidateSourceRef struct { - StartUnitID int `json:"start_unit_id"` - EndUnitID int `json:"end_unit_id"` -} - -type normalizeTranscriptInput struct { - Windows []normalizeTranscriptWindow `json:"windows"` -} - -type normalizeTranscriptWindow struct { - Units []normalizeTranscriptUnit `json:"units"` -} - -type normalizeTranscriptUnit struct { - ID int `json:"id"` - Kind string `json:"kind"` - Text string `json:"text"` - Metadata map[string]any `json:"metadata,omitempty"` - Cited bool `json:"cited"` -} - -type normalizeProposalResponse struct { - DuplicateGroups []normalizeProposalGroup `json:"duplicate_groups"` -} - -type normalizeProposalGroup struct { - Members []string `json:"members"` - CanonicalName string `json:"canonical_name"` -} - -type sourceInterval struct { - start int - end int -} - -func buildDefaultNormalizeContextMaterials(doc *source.SourceDocument, records []dnd.NPC) (normalizeContextMaterials, bool, error) { - return buildNormalizeContextMaterials(doc, records, semanticContextRadius) -} - -// buildNormalizeContextMaterials prepares the owned prompt inputs for a -// document-level normalization proposal. A false ready value means semantic -// normalization has no comparison-distinct eligible candidates to consider. -func buildNormalizeContextMaterials(doc *source.SourceDocument, records []dnd.NPC, radius int) (materials normalizeContextMaterials, ready bool, err error) { - if doc == nil { - return normalizeContextMaterials{}, false, nil - } - if radius < 0 { - return normalizeContextMaterials{}, false, fmt.Errorf("build NPC normalization context: radius must not be negative") - } - - index := source.NewDocumentIndex(doc) - candidates := make([]normalizeCandidate, 0, len(records)) - candidatePositions := make([]int, 0, len(records)) - intervals := make([]sourceInterval, 0) - cited := make([]bool, len(doc.Units)) - seenKeys := make(map[string]struct{}, len(records)) - - for position, record := range records { - key := identity.ComparisonKey(record.Name) - if key == "" || len(record.SourceRefs) == 0 { - continue - } - if _, exists := seenKeys[key]; exists { - continue - } - - references, recordIntervals, valid := normalizeRecordReferences(index, record.SourceRefs) - if !valid { - continue - } - seenKeys[key] = struct{}{} - candidates = append(candidates, normalizeCandidate{Name: record.Name, SourceRefs: references}) - candidatePositions = append(candidatePositions, position) - for _, interval := range recordIntervals { - for position := interval.start; position <= interval.end; position++ { - cited[position] = true - } - intervals = append(intervals, sourceInterval{ - start: maxInt(0, interval.start-radius), - end: minInt(len(doc.Units)-1, interval.end+radius), - }) - } - } - if len(candidates) < 2 { - return normalizeContextMaterials{}, false, nil - } - - windows, err := normalizeContextWindows(doc.Units, coalesceIntervals(intervals), cited) - if err != nil { - return normalizeContextMaterials{}, false, fmt.Errorf("build NPC normalization context: invalid source metadata") - } - candidateContent, err := json.Marshal(normalizeCandidateInput{NPCs: candidates}) - if err != nil { - return normalizeContextMaterials{}, false, fmt.Errorf("build NPC normalization context: invalid candidate material") - } - transcriptContent, err := json.Marshal(normalizeTranscriptInput{Windows: windows}) - if err != nil { - return normalizeContextMaterials{}, false, fmt.Errorf("build NPC normalization context: invalid transcript material") - } - return normalizeContextMaterials{ - Candidates: newNormalizeInputMaterial("candidates", candidateContent), - Transcript: newNormalizeInputMaterial("transcript", transcriptContent), - candidatePositions: candidatePositions, - }, true, nil -} - -func normalizeRecordReferences(index source.DocumentIndex, refs []source.SourceRef) ([]normalizeCandidateSourceRef, []sourceInterval, bool) { - references := make([]normalizeCandidateSourceRef, 0, len(refs)) - intervals := make([]sourceInterval, 0, len(refs)) - for _, ref := range refs { - if err := index.ValidateRef(ref); err != nil { - return nil, nil, false - } - start, _ := index.Position(ref.StartUnitID) - end, _ := index.Position(ref.EndUnitID) - references = append(references, normalizeCandidateSourceRef{StartUnitID: ref.StartUnitID, EndUnitID: ref.EndUnitID}) - intervals = append(intervals, sourceInterval{start: start, end: end}) - } - return references, intervals, true -} - -func coalesceIntervals(intervals []sourceInterval) []sourceInterval { - if len(intervals) == 0 { - return nil - } - ordered := append([]sourceInterval(nil), intervals...) - sort.Slice(ordered, func(i, j int) bool { - if ordered[i].start != ordered[j].start { - return ordered[i].start < ordered[j].start - } - return ordered[i].end < ordered[j].end - }) - coalesced := make([]sourceInterval, 0, len(ordered)) - for _, interval := range ordered { - if len(coalesced) == 0 || interval.start > coalesced[len(coalesced)-1].end+1 { - coalesced = append(coalesced, interval) - continue - } - if interval.end > coalesced[len(coalesced)-1].end { - coalesced[len(coalesced)-1].end = interval.end - } - } - return coalesced -} - -func normalizeContextWindows(units []source.SourceUnit, intervals []sourceInterval, cited []bool) ([]normalizeTranscriptWindow, error) { - windows := make([]normalizeTranscriptWindow, 0, len(intervals)) - for _, interval := range intervals { - window := normalizeTranscriptWindow{Units: make([]normalizeTranscriptUnit, 0, interval.end-interval.start+1)} - for position := interval.start; position <= interval.end; position++ { - unit := units[position] - metadata, err := source.CloneMetadata(unit.Metadata) - if err != nil { - return nil, err - } - window.Units = append(window.Units, normalizeTranscriptUnit{ - ID: unit.ID, Kind: unit.Kind, Text: unit.Text, Metadata: metadata, Cited: cited[position], - }) - } - windows = append(windows, window) - } - return windows, nil -} - -func newNormalizeInputMaterial(name string, content []byte) contracts.LLMInputMaterial { - digest := sha256.Sum256(content) - return contracts.NewLLMInputMaterial(name, "application/json", content, "sha256:"+hex.EncodeToString(digest[:]), "") -} - -func minInt(left, right int) int { - if left < right { - return left - } - return right -} - -func maxInt(left, right int) int { - if left > right { - return left - } - return right -} diff --git a/internal/modules/dnd/normalize/npcs/context_material_test.go b/internal/modules/dnd/normalize/npcs/context_material_test.go deleted file mode 100644 index d7afa1f..0000000 --- a/internal/modules/dnd/normalize/npcs/context_material_test.go +++ /dev/null @@ -1,132 +0,0 @@ -package npcs - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "reflect" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/core/source" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" -) - -func TestBuildNormalizeContextMaterialsUsesDocumentOrderAndOwnedInputs(t *testing.T) { - doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{ - {ID: 40, Kind: "narration", Text: "zero"}, - {ID: 10, Kind: "speech", Text: "one", Metadata: map[string]any{"speaker": map[string]any{"name": "Mira"}}}, - {ID: 70, Kind: "speech", Text: "two"}, - {ID: 20, Kind: "narration", Text: "three"}, - {ID: 90, Kind: "speech", Text: "four"}, - {ID: 30, Kind: "narration", Text: "five"}, - }} - records := []dnd.NPC{ - {Name: "Mira Thorn", ID: "npc:sha256:internal", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 20}}}, - {Name: "Captain Vale", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 90, EndUnitID: 90}}}, - {Name: "Broken", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 10}}}, - } - before := append([]dnd.NPC(nil), records...) - materials, ready, err := buildNormalizeContextMaterials(doc, records, 1) - if err != nil || !ready { - t.Fatalf("buildNormalizeContextMaterials() = %#v, %t, %v; want ready materials", materials, ready, err) - } - if !reflect.DeepEqual(records, before) { - t.Fatalf("records mutated to %#v", records) - } - for _, material := range []struct { - name string - data []byte - }{ - {name: "candidates", data: materials.Candidates.Content}, - {name: "transcript", data: materials.Transcript.Content}, - } { - if !json.Valid(material.data) || string(material.data) == "" { - t.Fatalf("%s content = %q, want JSON", material.name, material.data) - } - digest := sha256.Sum256(material.data) - wantDigest := "sha256:" + hex.EncodeToString(digest[:]) - got := materials.Candidates - if material.name == "transcript" { - got = materials.Transcript - } - if got.Name != material.name || got.MediaType != "application/json" || got.OriginURI != "" || got.Digest != wantDigest { - t.Fatalf("%s material = %#v, want owned JSON material", material.name, got) - } - } - encoded := string(materials.Candidates.Content) + string(materials.Transcript.Content) - if strings.Contains(encoded, "npc:sha256:internal") || strings.Contains(encoded, doc.ID) { - t.Fatalf("model material leaked private identifier or source id: %s", encoded) - } - - var candidates normalizeCandidateInput - if err := json.Unmarshal(materials.Candidates.Content, &candidates); err != nil { - t.Fatal(err) - } - if len(candidates.NPCs) != 2 || candidates.NPCs[0].Name != "Mira Thorn" || candidates.NPCs[0].SourceRefs[0] != (normalizeCandidateSourceRef{StartUnitID: 10, EndUnitID: 20}) { - t.Fatalf("candidates = %#v, want two valid current-order candidates", candidates) - } - var transcript normalizeTranscriptInput - if err := json.Unmarshal(materials.Transcript.Content, &transcript); err != nil { - t.Fatal(err) - } - if len(transcript.Windows) != 1 || len(transcript.Windows[0].Units) != 6 { - t.Fatalf("transcript = %#v, want one coalesced window", transcript) - } - units := transcript.Windows[0].Units - for index, wantID := range []int{40, 10, 70, 20, 90, 30} { - if units[index].ID != wantID { - t.Fatalf("unit %d id = %d, want source-order id %d", index, units[index].ID, wantID) - } - } - if units[0].Cited || !units[1].Cited || !units[2].Cited || !units[3].Cited || !units[4].Cited || units[5].Cited { - t.Fatalf("citation markers = %#v, want original ranges only", units) - } - if units[1].Metadata["speaker"].(map[string]any)["name"] != "Mira" { - t.Fatalf("metadata = %#v, want copied generic metadata", units[1].Metadata) - } - - windows, err := normalizeContextWindows(doc.Units, []sourceInterval{{start: 1, end: 1}}, make([]bool, len(doc.Units))) - if err != nil { - t.Fatal(err) - } - windows[0].Units[0].Metadata["speaker"].(map[string]any)["name"] = "changed" - if doc.Units[1].Metadata["speaker"].(map[string]any)["name"] != "Mira" { - t.Fatal("copied metadata aliases source document") - } -} - -func TestBuildNormalizeContextMaterialsExcludesInvalidReferencesAndCoalescesAdjacentWindows(t *testing.T) { - doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{ - {ID: 9}, {ID: 3}, {ID: 8}, {ID: 1}, {ID: 7}, {ID: 2}, {ID: 6}, - }} - records := []dnd.NPC{ - {Name: "One", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 3, EndUnitID: 3}}}, - {Name: "Two", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 8, EndUnitID: 8}}}, - {Name: "Blank"}, - {Name: "Missing", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 99, EndUnitID: 99}}}, - {Name: "Foreign", SourceRefs: []source.SourceRef{{SourceID: "other", StartUnitID: 1, EndUnitID: 1}}}, - {Name: "Reversed", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 6, EndUnitID: 3}}}, - } - materials, ready, err := buildNormalizeContextMaterials(doc, records, 0) - if err != nil || !ready { - t.Fatalf("buildNormalizeContextMaterials() error = %v, ready = %t", err, ready) - } - var candidates normalizeCandidateInput - if err := json.Unmarshal(materials.Candidates.Content, &candidates); err != nil { - t.Fatal(err) - } - if got := []string{candidates.NPCs[0].Name, candidates.NPCs[1].Name}; !reflect.DeepEqual(got, []string{"One", "Two"}) { - t.Fatalf("candidate names = %#v, want only valid records", got) - } - var transcript normalizeTranscriptInput - if err := json.Unmarshal(materials.Transcript.Content, &transcript); err != nil { - t.Fatal(err) - } - if len(transcript.Windows) != 1 || len(transcript.Windows[0].Units) != 2 { - t.Fatalf("windows = %#v, want adjacent cited units coalesced", transcript.Windows) - } - if transcript.Windows[0].Units[0].ID != 3 || transcript.Windows[0].Units[1].ID != 8 { - t.Fatalf("window units = %#v, want document-order adjacent units", transcript.Windows[0].Units) - } -} diff --git a/internal/modules/dnd/normalize/npcs/normalizer.go b/internal/modules/dnd/normalize/npcs/normalizer.go index cb2f269..13e3ba9 100644 --- a/internal/modules/dnd/normalize/npcs/normalizer.go +++ b/internal/modules/dnd/normalize/npcs/normalizer.go @@ -17,12 +17,14 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/entityreconcile" ) const ( Key = "dnd/npcs" + PromptID = "dnd.npcs.normalize" normalizationPolicy = "dnd.npcs.normalize.v3" - semanticContextPolicy = "dnd.npcs.semantic_context.v1" + semanticContextPolicy = "dnd.entity_reconcile.context.v1" NormalizationPolicy = normalizationPolicy ReasonCodeNPCFieldsNormalized = "npc_fields_normalized" @@ -57,7 +59,7 @@ func New(llmClient contracts.StructuredLLMClient, _ Options) (*Normalizer, error if err != nil { return nil, normalizerErrorf("load prompt metadata: %w", err) } - responseSchema, err := loadResponseSchema() + responseSchema, err := entityreconcile.LoadResponseSchema() if err != nil { return nil, normalizerErrorf("load response schema: %w", err) } @@ -73,12 +75,12 @@ func (n *Normalizer) ManifestMetadata() map[string]any { } return map[string]any{ "prompt_id": PromptID, - "prompt_version": SchemaVersion, + "prompt_version": entityreconcile.SchemaVersion, "prompt_sha256": n.promptSHA, - "response_schema_key": string(ResponseSchemaKey), - "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, - "response_schema_version": SchemaVersion, + "response_schema_key": string(entityreconcile.ResponseSchemaKey), + "response_schema_id": entityreconcile.ResponseSchemaID, + "response_schema_name": entityreconcile.ResponseSchemaName, + "response_schema_version": entityreconcile.SchemaVersion, "response_schema_sha256": n.responseSchemaSHA, "identity_policy": identity.Policy, "normalization_policy": normalizationPolicy, @@ -117,7 +119,7 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize order := shared.NewSourceRefOrder(req.Source) records, warnings := preprocessRecords(req.MergeOutput.Value, order) deterministic := recordList(records) - materials, ready, err := buildDefaultNormalizeContextMaterials(req.Source, recordValues(records)) + materials, ready, err := entityreconcile.BuildContext(req.Source, reconciliationCandidates(records), semanticContextRadius) if err != nil { return contracts.TypedNormalizeResult[dnd.NPCList]{}, normalizerErrorf("build semantic context: %w", err) } @@ -125,9 +127,9 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize return contracts.TypedNormalizeResult[dnd.NPCList]{Value: deterministic, Warnings: limitWarnings(warnings)}, nil } - var response normalizeProposalResponse + var response entityreconcile.ProposalResponse if _, err := n.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ - StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion, + StageName: Key, PromptID: PromptID, PromptVersion: entityreconcile.SchemaVersion, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: contracts.LLMInputSet{"candidates": materials.Candidates, "transcript": materials.Transcript}, }, &response); err != nil { @@ -137,10 +139,10 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize return contracts.TypedNormalizeResult[dnd.NPCList]{}, normalizerErrorf("complete structured output: %w", err) } - assessment := assessProposal(response, records, materials.candidatePositions) - applied, semanticWarnings := applySafeGroups(records, assessment.safeGroups, order) + assessment := materials.Assess(response) + applied, semanticWarnings := applySafeGroups(records, reconciliationGroups(assessment, materials.CandidateKeys()), order) warnings = append(warnings, semanticWarnings...) - if assessment.discardedGroups == 0 { + if assessment.DiscardedGroups() == 0 { return contracts.TypedNormalizeResult[dnd.NPCList]{Value: recordList(applied), Warnings: limitWarnings(warnings)}, nil } return retryResult(recordList(applied), warnings, assessment), nil @@ -158,14 +160,14 @@ func (n *Normalizer) invalidStructuredResult(value dnd.NPCList, warnings []contr } } -func retryResult(value dnd.NPCList, warnings []contracts.Warning, assessment proposalAssessment) contracts.TypedNormalizeResult[dnd.NPCList] { +func retryResult(value dnd.NPCList, warnings []contracts.Warning, assessment entityreconcile.Assessment) contracts.TypedNormalizeResult[dnd.NPCList] { return contracts.TypedNormalizeResult[dnd.NPCList]{ Value: value, Warnings: limitWarningsForRetry(warnings), Retry: &contracts.NormalizeRetry{ ReasonCode: ReasonCodeNPCSemanticProposalInvalid, - Message: diagnostics.Aggregate("semantic proposal requires retry", assessment.issues), - FallbackWarnings: []contracts.Warning{semanticFallbackWarning(assessment.discardedGroups)}, + Message: diagnostics.Aggregate("semantic proposal requires retry", reconciliationIssues(assessment)), + FallbackWarnings: []contracts.Warning{semanticFallbackWarning(assessment.DiscardedGroups())}, }, } } diff --git a/internal/modules/dnd/normalize/npcs/normalizer_test.go b/internal/modules/dnd/normalize/npcs/normalizer_test.go index 7d57b7c..a9fc6e0 100644 --- a/internal/modules/dnd/normalize/npcs/normalizer_test.go +++ b/internal/modules/dnd/normalize/npcs/normalizer_test.go @@ -12,6 +12,7 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/entityreconcile" ) func TestModuleContractAndIdentity(t *testing.T) { @@ -33,7 +34,7 @@ func TestModuleContractAndIdentity(t *testing.T) { t.Fatal("New(nil, Options{}) error = nil, want nil client rejection") } normalizer := newNormalizer(t, &recordingNPCNormalizerClient{}) - if metadata := normalizer.ManifestMetadata(); metadata["identity_policy"] != identity.Policy || metadata["normalization_policy"] != normalizationPolicy || metadata["prompt_id"] != PromptID || metadata["semantic_context_policy"] != semanticContextPolicy || metadata["semantic_context_radius"] != semanticContextRadius { + if metadata := normalizer.ManifestMetadata(); metadata["identity_policy"] != identity.Policy || metadata["normalization_policy"] != normalizationPolicy || metadata["prompt_id"] != PromptID || metadata["response_schema_key"] != string(entityreconcile.ResponseSchemaKey) || metadata["response_schema_id"] != entityreconcile.ResponseSchemaID || metadata["response_schema_name"] != entityreconcile.ResponseSchemaName || metadata["semantic_context_policy"] != semanticContextPolicy || metadata["semantic_context_radius"] != semanticContextRadius { t.Fatalf("metadata = %#v", metadata) } wantFingerprints := []pipeline.CheckpointFingerprint{{Name: "prompt", Value: normalizer.promptSHA}, {Name: "response_schema", Value: normalizer.responseSchemaSHA}, {Name: "identity_policy", Value: identity.Policy}, {Name: "normalization_policy", Value: normalizationPolicy}, {Name: "semantic_context_policy", Value: semanticContextPolicy + ":2"}} diff --git a/internal/modules/dnd/normalize/npcs/prompt_assets.go b/internal/modules/dnd/normalize/npcs/prompt_assets.go index 28779dc..2494382 100644 --- a/internal/modules/dnd/normalize/npcs/prompt_assets.go +++ b/internal/modules/dnd/normalize/npcs/prompt_assets.go @@ -16,11 +16,11 @@ var promptAssetManifest = shared.PromptAssetManifest{ ModuleFiles: []promptfs.ModulePromptFile{ {Name: "dnd.npcs.normalize.yaml", Path: "assets/prompts/dnd.npcs.normalize.yaml"}, {Name: "task.md", Path: "assets/prompts/task.md"}, - {Name: "instructions.md", Path: "assets/prompts/instructions.md"}, {Name: "candidates.md", Path: "assets/prompts/candidates.md"}, }, SharedFiles: []string{ "common-dnd-system.md", + "common-dnd-entity-reconciliation.md", "common-dnd-transcript.md", }, } @@ -30,10 +30,7 @@ func RegisterPromptAssets(registry *llm.AssetRegistry) error { if err != nil { return fmt.Errorf("prepare NPC normalization prompt assets: %w", err) } - if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil { - return err - } - return registry.RegisterSchemaFS(embeddedAssets, "assets/schemas") + return registry.RegisterPromptFS(promptFS, promptAssetRoot) } func promptAssetMetadata() (string, error) { diff --git a/internal/modules/dnd/normalize/npcs/prompt_assets_test.go b/internal/modules/dnd/normalize/npcs/prompt_assets_test.go index 654b93f..592ec4b 100644 --- a/internal/modules/dnd/normalize/npcs/prompt_assets_test.go +++ b/internal/modules/dnd/normalize/npcs/prompt_assets_test.go @@ -8,17 +8,21 @@ import ( "time" "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/entityreconcile" "gitea.maximumdirect.net/eric/promptkit" ) func TestRegisterPromptAssetsPreparesNormalizationPrompt(t *testing.T) { - if want := []string{"common-dnd-system.md", "common-dnd-transcript.md"}; !reflect.DeepEqual(promptAssetManifest.SharedFiles, want) { + if want := []string{"common-dnd-system.md", "common-dnd-entity-reconciliation.md", "common-dnd-transcript.md"}; !reflect.DeepEqual(promptAssetManifest.SharedFiles, want) { t.Fatalf("shared prompt assets = %#v, want %#v", promptAssetManifest.SharedFiles, want) } if promptHash, err := promptAssetMetadata(); err != nil || promptHash == "" { t.Fatalf("promptAssetMetadata() = %q, %v; want prompt fingerprint", promptHash, err) } registry := llm.NewAssetRegistry() + if err := entityreconcile.RegisterSchemaAssets(registry); err != nil { + t.Fatalf("RegisterSchemaAssets() error = %v", err) + } if err := RegisterPromptAssets(registry); err != nil { t.Fatalf("RegisterPromptAssets() error = %v", err) } @@ -34,16 +38,16 @@ func TestRegisterPromptAssetsPreparesNormalizationPrompt(t *testing.T) { t.Fatalf("NewEngine() error = %v", err) } prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "normalize-test-profile", + PromptID: PromptID, PromptVersion: entityreconcile.SchemaVersion, ProfileID: "normalize-test-profile", Inputs: map[string]promptkit.ArtifactRef{ - "candidates": promptkit.Inline(`{"npcs":[{"name":"Mira","source_refs":[]}]}`), + "candidates": promptkit.Inline(`{"candidates":[{"key":"candidate-000001","name":"Mira","source_refs":[]}]}`), "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`), }, }) if err != nil { t.Fatalf("Prepare() error = %v", err) } - if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npcs_normalize_llm.v1.json" { + if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_entity_reconcile_llm.v1.json" { t.Fatalf("prepared prompt = %#v, want normalization prompt identity and schema", prepared) } if len(prepared.Messages) != 5 { diff --git a/internal/modules/dnd/normalize/npcs/proposal.go b/internal/modules/dnd/normalize/npcs/proposal.go deleted file mode 100644 index 8b1663d..0000000 --- a/internal/modules/dnd/normalize/npcs/proposal.go +++ /dev/null @@ -1,195 +0,0 @@ -package npcs - -import ( - "fmt" - "sort" - "strconv" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics" -) - -type proposalAssessment struct { - safeGroups []safeProposalGroup - discardedGroups int - issues []string -} - -type safeProposalGroup struct { - members []int - canonical int -} - -type assessedProposalGroup struct { - members []int - canonical int - locallyValid bool - conflicting bool -} - -func assessProposal(response normalizeProposalResponse, records []normalizedRecord, candidatePositions []int) proposalAssessment { - positionsByKey := make(map[string][]int, len(candidatePositions)) - for _, position := range candidatePositions { - if position < 0 || position >= len(records) { - continue - } - key := identity.ComparisonKey(records[position].npc.Name) - if key != "" { - positionsByKey[key] = append(positionsByKey[key], position) - } - } - groups := make([]assessedProposalGroup, len(response.DuplicateGroups)) - issues := make([]string, 0) - owners := make(map[int][]int) - - for groupIndex, proposal := range response.DuplicateGroups { - group, groupIssues := assessProposalGroup(proposal, positionsByKey) - groups[groupIndex] = group - for _, position := range group.members { - owners[position] = append(owners[position], groupIndex) - } - for _, issue := range groupIssues { - issues = append(issues, proposalIssue(groupIndex, issue)) - } - } - for groupIndex := range groups { - for _, position := range groups[groupIndex].members { - if len(owners[position]) > 1 { - groups[groupIndex].conflicting = true - break - } - } - if groups[groupIndex].conflicting { - issues = append(issues, proposalIssue(groupIndex, "overlapping_member")) - } - } - - assessment := proposalAssessment{issues: issues} - for _, group := range groups { - if !group.locallyValid || group.conflicting { - assessment.discardedGroups++ - continue - } - assessment.safeGroups = append(assessment.safeGroups, safeProposalGroup{members: group.members, canonical: group.canonical}) - } - return assessment -} - -func assessProposalGroup(proposal normalizeProposalGroup, positionsByKey map[string][]int) (assessedProposalGroup, []string) { - issues := make([]string, 0) - members := make([]int, 0, len(proposal.Members)) - seenMembers := make(map[int]struct{}, len(proposal.Members)) - for _, name := range proposal.Members { - position, issue := resolveCandidate(name, positionsByKey) - if issue != "" { - issues = append(issues, "member_"+issue) - continue - } - if _, exists := seenMembers[position]; exists { - issues = append(issues, "repeated_member") - continue - } - seenMembers[position] = struct{}{} - members = append(members, position) - } - canonical, canonicalIssue := resolveCandidate(proposal.CanonicalName, positionsByKey) - if canonicalIssue != "" { - issues = append(issues, "canonical_"+canonicalIssue) - } - if len(members) < 2 { - issues = append(issues, "fewer_than_two_members") - } - if canonicalIssue == "" && !containsPosition(members, canonical) { - issues = append(issues, "canonical_not_member") - } - sort.Ints(members) - return assessedProposalGroup{ - members: members, canonical: canonical, locallyValid: len(issues) == 0, - }, issues -} - -func resolveCandidate(name string, positionsByKey map[string][]int) (position int, issue string) { - key := identity.ComparisonKey(name) - if key == "" { - return 0, "blank" - } - positions := positionsByKey[key] - if len(positions) == 0 { - return 0, "unknown" - } - if len(positions) != 1 { - return 0, "ambiguous" - } - return positions[0], "" -} - -func containsPosition(positions []int, want int) bool { - for _, position := range positions { - if position == want { - return true - } - } - return false -} - -func proposalIssue(groupIndex int, category string) string { - return "group " + strconv.Itoa(groupIndex) + ": " + category -} - -func applySafeGroups(records []normalizedRecord, groups []safeProposalGroup, order shared.SourceRefOrder) ([]normalizedRecord, []contracts.Warning) { - byMember := make(map[int]safeProposalGroup, len(groups)*2) - for _, group := range groups { - for _, member := range group.members { - byMember[member] = group - } - } - output := make([]normalizedRecord, 0, len(records)-len(groups)) - warnings := make([]contracts.Warning, 0, len(groups)) - for index, record := range records { - group, grouped := byMember[index] - if !grouped { - output = append(output, cloneRecord(record)) - continue - } - if group.members[0] != index { - continue - } - consolidated := consolidateSemanticGroup(records, group, order) - output = append(output, consolidated) - warnings = append(warnings, semanticDuplicateWarning(consolidated, records[group.canonical])) - } - return output, warnings -} - -func consolidateSemanticGroup(records []normalizedRecord, group safeProposalGroup, order shared.SourceRefOrder) normalizedRecord { - output := cloneRecord(records[group.members[0]]) - output.npc.Name = records[group.canonical].npc.Name - for _, member := range group.members[1:] { - output.npc.SourceRefs = append(output.npc.SourceRefs, records[member].npc.SourceRefs...) - output.inputIndexes = append(output.inputIndexes, records[member].inputIndexes...) - if records[member].earliest < output.earliest { - output.earliest = records[member].earliest - } - } - output.inputIndexes = sortedUniqueIndexes(output.inputIndexes) - output.npc.SourceRefs = order.Canonicalize(output.npc.SourceRefs) - output.npc.ID = identity.DeriveID(output.npc.Name) - return output -} - -func semanticDuplicateWarning(record normalizedRecord, canonical normalizedRecord) contracts.Warning { - details := make([]string, 0, len(record.inputIndexes)+1) - for _, inputIndex := range record.inputIndexes { - details = append(details, fmt.Sprintf("input index %d", inputIndex)) - } - if canonical.earliest != record.earliest { - details = append(details, fmt.Sprintf("canonical display name from input index %d", canonical.earliest)) - } - return contracts.Warning{ - Scope: npcScope(record.earliest), - ReasonCode: ReasonCodeDuplicateNPCCollapsed, - Message: diagnostics.Aggregate("semantic duplicate consolidation", details), - } -} diff --git a/internal/modules/dnd/normalize/npcs/reconciliation.go b/internal/modules/dnd/normalize/npcs/reconciliation.go new file mode 100644 index 0000000..dd6cf27 --- /dev/null +++ b/internal/modules/dnd/normalize/npcs/reconciliation.go @@ -0,0 +1,122 @@ +package npcs + +import ( + "fmt" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/entityreconcile" +) + +const semanticContextRadius = 2 + +type safeReconciliationGroup struct { + members []int + canonical int +} + +func reconciliationCandidates(records []normalizedRecord) []entityreconcile.Candidate { + candidates := make([]entityreconcile.Candidate, len(records)) + for index, record := range records { + candidates[index] = entityreconcile.Candidate{ + Name: record.npc.Name, + SourceRefs: cloneSourceRefs(record.npc.SourceRefs), + } + } + return candidates +} + +func reconciliationGroups(assessment entityreconcile.Assessment, candidateKeys []string) []safeReconciliationGroup { + positions := make(map[string]int, len(candidateKeys)) + for index, key := range candidateKeys { + positions[key] = index + } + safeGroups := assessment.SafeGroups() + groups := make([]safeReconciliationGroup, 0, len(safeGroups)) + for _, group := range safeGroups { + members := group.Members() + memberPositions := make([]int, len(members)) + valid := true + for index, key := range members { + position, ok := positions[key] + if !ok { + valid = false + break + } + memberPositions[index] = position + } + canonical, ok := positions[group.Canonical()] + if !valid || !ok { + continue + } + groups = append(groups, safeReconciliationGroup{members: memberPositions, canonical: canonical}) + } + return groups +} + +func reconciliationIssues(assessment entityreconcile.Assessment) []string { + issues := assessment.Issues() + details := make([]string, len(issues)) + for index, issue := range issues { + details[index] = fmt.Sprintf("group %d: %s", issue.GroupIndex, issue.Category) + } + return details +} + +func applySafeGroups(records []normalizedRecord, groups []safeReconciliationGroup, order shared.SourceRefOrder) ([]normalizedRecord, []contracts.Warning) { + byMember := make(map[int]safeReconciliationGroup, len(groups)*2) + for _, group := range groups { + for _, member := range group.members { + byMember[member] = group + } + } + output := make([]normalizedRecord, 0, len(records)-len(groups)) + warnings := make([]contracts.Warning, 0, len(groups)) + for index, record := range records { + group, grouped := byMember[index] + if !grouped { + output = append(output, cloneRecord(record)) + continue + } + if group.members[0] != index { + continue + } + consolidated := consolidateSemanticGroup(records, group, order) + output = append(output, consolidated) + warnings = append(warnings, semanticDuplicateWarning(consolidated, records[group.canonical])) + } + return output, warnings +} + +func consolidateSemanticGroup(records []normalizedRecord, group safeReconciliationGroup, order shared.SourceRefOrder) normalizedRecord { + output := cloneRecord(records[group.members[0]]) + output.npc.Name = records[group.canonical].npc.Name + for _, member := range group.members[1:] { + output.npc.SourceRefs = append(output.npc.SourceRefs, records[member].npc.SourceRefs...) + output.inputIndexes = append(output.inputIndexes, records[member].inputIndexes...) + if records[member].earliest < output.earliest { + output.earliest = records[member].earliest + } + } + output.inputIndexes = sortedUniqueIndexes(output.inputIndexes) + output.npc.SourceRefs = order.Canonicalize(output.npc.SourceRefs) + output.npc.ID = identity.DeriveID(output.npc.Name) + return output +} + +func semanticDuplicateWarning(record normalizedRecord, canonical normalizedRecord) contracts.Warning { + details := make([]string, 0, len(record.inputIndexes)+1) + for _, inputIndex := range record.inputIndexes { + details = append(details, fmt.Sprintf("input index %d", inputIndex)) + } + if canonical.earliest != record.earliest { + details = append(details, fmt.Sprintf("canonical display name from input index %d", canonical.earliest)) + } + return contracts.Warning{ + Scope: npcScope(record.earliest), + ReasonCode: ReasonCodeDuplicateNPCCollapsed, + Message: diagnostics.Aggregate("semantic duplicate consolidation", details), + } +} diff --git a/internal/modules/dnd/normalize/npcs/schema.go b/internal/modules/dnd/normalize/npcs/schema.go deleted file mode 100644 index f196a7c..0000000 --- a/internal/modules/dnd/normalize/npcs/schema.go +++ /dev/null @@ -1,21 +0,0 @@ -package npcs - -import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - -const ( - PromptID = "dnd.npcs.normalize" - ResponseSchemaKey = llm.ResponseSchemaKey("dnd_npcs_normalize_llm") - ResponseSchemaID = "notarius.dnd.npcs.normalize.llm" - ResponseSchemaName = "notarius_dnd_npcs_normalize_llm_v1" - SchemaVersion = "v1" -) - -func loadResponseSchema() (llm.ResponseSchema, error) { - return llm.LoadResponseSchema(embeddedAssets, llm.ResponseSchemaDefinition{ - Key: ResponseSchemaKey, - ID: ResponseSchemaID, - Version: SchemaVersion, - Name: ResponseSchemaName, - AssetPath: "assets/schemas/dnd_npcs_normalize_llm.v1.json", - }) -} diff --git a/internal/modules/dnd/normalize/npcs/schema_test.go b/internal/modules/dnd/normalize/npcs/schema_test.go deleted file mode 100644 index 3e8e03c..0000000 --- a/internal/modules/dnd/normalize/npcs/schema_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package npcs - -import ( - "bytes" - "encoding/json" - "strings" - "testing" - - "github.com/santhosh-tekuri/jsonschema/v6" -) - -func TestNormalizeResponseSchemaIsStrictlyStructural(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatalf("loadResponseSchema() error = %v", err) - } - if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) { - t.Fatalf("schema = %#v, want private normalization schema identity", schema) - } - - for _, test := range []struct { - name string - value any - valid bool - }{ - {name: "empty groups", value: map[string]any{"duplicate_groups": []any{}}, valid: true}, - {name: "semantically invalid group", value: map[string]any{"duplicate_groups": []any{map[string]any{"members": []any{"", "unknown"}, "canonical_name": ""}}}, valid: true}, - {name: "missing groups", value: map[string]any{}}, - {name: "unknown top level field", value: map[string]any{"duplicate_groups": []any{}, "extra": true}}, - {name: "unknown group field", value: map[string]any{"duplicate_groups": []any{map[string]any{"members": []any{}, "canonical_name": "Mira", "extra": true}}}}, - {name: "wrong groups type", value: map[string]any{"duplicate_groups": "no"}}, - {name: "wrong member type", value: map[string]any{"duplicate_groups": []any{map[string]any{"members": []any{1}, "canonical_name": "Mira"}}}}, - } { - t.Run(test.name, func(t *testing.T) { - content, err := json.Marshal(test.value) - if err != nil { - t.Fatal(err) - } - err = validateNormalizeSchema(content, schema.JSONSchema) - if (err == nil) != test.valid { - t.Fatalf("validateNormalizeSchema() error = %v, want valid=%t", err, test.valid) - } - }) - } -} - -func validateNormalizeSchema(instanceContent, schemaContent []byte) error { - instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent)) - if err != nil { - return err - } - document, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent)) - if err != nil { - return err - } - compiler := jsonschema.NewCompiler() - if err := compiler.AddResource("schema.json", document); err != nil { - return err - } - compiled, err := compiler.Compile("schema.json") - if err != nil { - return err - } - return compiled.Validate(instance) -} diff --git a/internal/modules/dnd/normalize/npcs/semantic_normalizer_test.go b/internal/modules/dnd/normalize/npcs/semantic_normalizer_test.go index 8b89ac2..b64f638 100644 --- a/internal/modules/dnd/normalize/npcs/semantic_normalizer_test.go +++ b/internal/modules/dnd/normalize/npcs/semantic_normalizer_test.go @@ -12,6 +12,7 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/entityreconcile" ) func TestNormalizeSkipsSemanticCompletionWithoutTwoEligibleCandidates(t *testing.T) { @@ -29,7 +30,7 @@ func TestNormalizeSkipsSemanticCompletionWithoutTwoEligibleCandidates(t *testing } func TestNormalizeAppliesSafeProposalAndUsesPrivateInputs(t *testing.T) { - client := &recordingNPCNormalizerClient{response: `{"duplicate_groups":[{"members":[" MIRA THORN ","Mira"],"canonical_name":"Mira Thorn"}]}`} + client := &recordingNPCNormalizerClient{response: `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000002"}]}`} normalizer := newNormalizer(t, client) doc := semanticDocument() input := dnd.NPCList{NPCs: []dnd.NPC{ @@ -62,7 +63,7 @@ func TestNormalizeAppliesSafeProposalAndUsesPrivateInputs(t *testing.T) { t.Fatalf("completion calls = %d, want one", len(client.requests)) } completion := client.requests[0] - if completion.StageName != Key || completion.PromptID != PromptID || completion.PromptVersion != SchemaVersion || completion.ProfileID != request.LLMProfile || completion.SessionID != request.SessionID || len(completion.Inputs) != 2 { + if completion.StageName != Key || completion.PromptID != PromptID || completion.PromptVersion != entityreconcile.SchemaVersion || completion.ProfileID != request.LLMProfile || completion.SessionID != request.SessionID || len(completion.Inputs) != 2 { t.Fatalf("completion request = %#v, want normalize request identity and exactly two inputs", completion) } encoded := string(completion.Inputs["candidates"].Content) + string(completion.Inputs["transcript"].Content) @@ -72,7 +73,7 @@ func TestNormalizeAppliesSafeProposalAndUsesPrivateInputs(t *testing.T) { } func TestNormalizeUnsafeProposalReturnsSafeRetryFallback(t *testing.T) { - client := &recordingNPCNormalizerClient{response: `{"duplicate_groups":[{"members":["Mira","Mira Thorn"],"canonical_name":"Mira Thorn"},{"members":["Captain Vale","Unknown"],"canonical_name":"Captain Vale"}]}`} + client := &recordingNPCNormalizerClient{response: `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000002"},{"members":["candidate-000003","unknown"],"canonical":"candidate-000003"}]}`} normalizer := newNormalizer(t, client) doc := semanticDocument() input := dnd.NPCList{NPCs: []dnd.NPC{ @@ -95,30 +96,6 @@ func TestNormalizeUnsafeProposalReturnsSafeRetryFallback(t *testing.T) { } } -func TestNormalizeRejectsOverlapsWithoutResponseOrderDependence(t *testing.T) { - records := []normalizedRecord{ - {npc: dnd.NPC{Name: "Alpha"}}, {npc: dnd.NPC{Name: "Bravo"}}, {npc: dnd.NPC{Name: "Charlie"}}, {npc: dnd.NPC{Name: "Delta"}}, - } - for index := range records { - records[index].inputIndexes = []int{index} - records[index].earliest = index - } - response := normalizeProposalResponse{DuplicateGroups: []normalizeProposalGroup{ - {Members: []string{"Alpha", "Bravo"}, CanonicalName: "Alpha"}, - {Members: []string{"Bravo", "Charlie"}, CanonicalName: "Bravo"}, - {Members: []string{"Charlie", "Delta"}, CanonicalName: "Charlie"}, - }} - assessment := assessProposal(response, records, []int{0, 1, 2, 3}) - if assessment.discardedGroups != 3 || len(assessment.safeGroups) != 0 { - t.Fatalf("assessment = %#v, want chained conflicts all discarded", assessment) - } - for _, issue := range []string{"group 0: overlapping_member", "group 1: overlapping_member", "group 2: overlapping_member"} { - if !containsString(assessment.issues, issue) { - t.Fatalf("issues = %#v, want %q", assessment.issues, issue) - } - } -} - func TestNormalizeInvalidStructuredOutputAndOperationalErrorsRemainDistinct(t *testing.T) { doc := semanticDocument() input := dnd.NPCList{NPCs: []dnd.NPC{ @@ -159,7 +136,7 @@ func TestNormalizeRedactsContextMaterialFailures(t *testing.T) { request := normalizeRequestWithSource(input, doc) request.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(transcript), "sha256:test", originPath) _, err := normalizer.Normalize(context.Background(), request) - if err == nil || !strings.Contains(err.Error(), "build NPC normalization context: invalid source metadata") { + if err == nil || !strings.Contains(err.Error(), "build entity reconciliation context: invalid source metadata") { t.Fatalf("Normalize() error = %v; want content-safe context-material failure", err) } for _, forbidden := range []string{metadataKey, metadataValue, transcript, firstName, secondName, sourceID, originPath, "float64", "non-finite"} { @@ -174,8 +151,8 @@ func TestNormalizeRedactsContextMaterialFailures(t *testing.T) { func TestNormalizeDoesNotAccumulateSafeGroupsAcrossAttempts(t *testing.T) { client := &recordingNPCNormalizerClient{responses: []string{ - `{"duplicate_groups":[{"members":["Mira","Mira Thorn"],"canonical_name":"Mira Thorn"},{"members":["Captain Vale","Unknown"],"canonical_name":"Captain Vale"}]}`, - `{"duplicate_groups":[{"members":["Mira","Captain Vale"],"canonical_name":"Captain Vale"}]}`, + `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000002"},{"members":["candidate-000003","unknown"],"canonical":"candidate-000003"}]}`, + `{"duplicate_groups":[{"members":["candidate-000001","candidate-000003"],"canonical":"candidate-000003"}]}`, }} normalizer := newNormalizer(t, client) doc := semanticDocument() @@ -198,7 +175,7 @@ func TestNormalizeDoesNotAccumulateSafeGroupsAcrossAttempts(t *testing.T) { } func TestNormalizeExcludesIneligibleRecordsFromSemanticGroups(t *testing.T) { - client := &recordingNPCNormalizerClient{response: `{"duplicate_groups":[{"members":["Mira","Broken"],"canonical_name":"Mira"}]}`} + client := &recordingNPCNormalizerClient{response: `{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000001"}]}`} normalizer := newNormalizer(t, client) doc := semanticDocument() input := dnd.NPCList{NPCs: []dnd.NPC{ @@ -210,42 +187,31 @@ func TestNormalizeExcludesIneligibleRecordsFromSemanticGroups(t *testing.T) { if err != nil || result.Retry == nil || len(result.Value.NPCs) != 3 { t.Fatalf("Normalize() = %#v, %v; want unchanged retry fallback", result, err) } - if !strings.Contains(result.Retry.Message, "member_unknown") || result.Value.NPCs[1].Name != "Broken" { + if !strings.Contains(result.Retry.Message, "member_ineligible") || result.Value.NPCs[1].Name != "Broken" { t.Fatalf("result = %#v, want ineligible record excluded but preserved", result) } } -func TestProposalValidationRejectsUnsafeCategories(t *testing.T) { +func TestReconciliationCandidatesKeepEqualDisplayNamesDistinct(t *testing.T) { + doc := semanticDocument() records := []normalizedRecord{ - {npc: dnd.NPC{Name: "Mira"}, inputIndexes: []int{0}, earliest: 0}, - {npc: dnd.NPC{Name: "Mira Thorn"}, inputIndexes: []int{1}, earliest: 1}, - {npc: dnd.NPC{Name: "Captain Vale"}, inputIndexes: []int{2}, earliest: 2}, + {npc: dnd.NPC{Name: "The Guard", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}}, + {npc: dnd.NPC{Name: "The Guard", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}}, } - for _, proposal := range []normalizeProposalGroup{ - {Members: []string{"Mira"}, CanonicalName: "Mira"}, - {Members: []string{"Mira", "Mira"}, CanonicalName: "Mira"}, - {Members: []string{"Mira", "Mira Thorn"}, CanonicalName: "Unknown"}, - {Members: []string{"Mira", "Mira Thorn"}, CanonicalName: " "}, - {Members: []string{" ", "Mira Thorn"}, CanonicalName: "Mira Thorn"}, - {Members: []string{"Mira", "Mira Thorn"}, CanonicalName: "Captain Vale"}, - } { - assessment := assessProposal(normalizeProposalResponse{DuplicateGroups: []normalizeProposalGroup{proposal}}, records, []int{0, 1, 2}) - if assessment.discardedGroups != 1 || len(assessment.safeGroups) != 0 { - t.Fatalf("assessment for %#v = %#v, want discarded unsafe group", proposal, assessment) - } + materials, ready, err := entityreconcile.BuildContext(doc, reconciliationCandidates(records), semanticContextRadius) + if err != nil || !ready { + t.Fatalf("BuildContext() = %#v, %t, %v; want ready keyed candidates", materials, ready, err) } -} - -func TestProposalResolutionUsesOnlyExistingComparisonKeyEquivalences(t *testing.T) { - records := []normalizedRecord{ - {npc: dnd.NPC{Name: "O'Neill"}, inputIndexes: []int{0}, earliest: 0}, - {npc: dnd.NPC{Name: "Mira Thorn"}, inputIndexes: []int{1}, earliest: 1}, + keys := materials.CandidateKeys() + if !reflect.DeepEqual(keys, []string{"candidate-000001", "candidate-000002"}) || strings.Count(string(materials.Candidates.Content), `"The Guard"`) != 2 { + t.Fatalf("candidate keys and inputs = %#v, %s; want distinct equal-display candidates", keys, materials.Candidates.Content) } - assessment := assessProposal(normalizeProposalResponse{DuplicateGroups: []normalizeProposalGroup{{ - Members: []string{" O’NEILL ", "Mira Thorn"}, CanonicalName: "Mira Thorn", - }}}, records, []int{0, 1}) - if assessment.discardedGroups != 0 || len(assessment.safeGroups) != 1 || assessment.safeGroups[0].canonical != 1 { - t.Fatalf("assessment = %#v, want comparison-key-only resolution", assessment) + assessment := materials.Assess(entityreconcile.ProposalResponse{DuplicateGroups: []entityreconcile.DuplicateGroup{{ + Members: keys, Canonical: keys[1], + }}}) + groups := reconciliationGroups(assessment, keys) + if assessment.DiscardedGroups() != 0 || len(groups) != 1 || groups[0].canonical != 1 { + t.Fatalf("reconciliation = %#v, %#v; want distinct keyed group", assessment, groups) } } @@ -276,12 +242,3 @@ func cloneNPCList(input dnd.NPCList) dnd.NPCList { } return output } - -func containsString(values []string, want string) bool { - for _, value := range values { - if value == want { - return true - } - } - return false -} diff --git a/internal/modules/dnd/register/modules.go b/internal/modules/dnd/register/modules.go index 47ee441..c5d1c9b 100644 --- a/internal/modules/dnd/register/modules.go +++ b/internal/modules/dnd/register/modules.go @@ -26,6 +26,7 @@ import ( npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcs" scenedescriptionnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions" spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/entityreconcile" "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder" "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop" ) @@ -104,6 +105,7 @@ func registerModules(registries pipeline.Registries) error { func registerPromptAssets(assets *llm.AssetRegistry) error { return runRegistrations([]registration{ + {name: "entity reconciliation schema assets", register: func() error { return entityreconcile.RegisterSchemaAssets(assets) }}, {name: "scenes prompt assets", register: func() error { return scenes.RegisterPromptAssets(assets) }}, {name: "spells prompt assets", register: func() error { return spellextract.RegisterPromptAssets(assets) }}, {name: "npcs prompt assets", register: func() error { return npcextract.RegisterPromptAssets(assets) }}, diff --git a/internal/modules/dnd/register/register_test.go b/internal/modules/dnd/register/register_test.go index 3b6cb71..844c9ce 100644 --- a/internal/modules/dnd/register/register_test.go +++ b/internal/modules/dnd/register/register_test.go @@ -71,8 +71,8 @@ func TestRegisterAddsDNDFamily(t *testing.T) { if err != nil { t.Fatalf("SchemaFS() error = %v", err) } - if _, err := fs.ReadFile(schemaFS, "dnd_npcs_normalize_llm.v1.json"); err != nil { - t.Fatalf("normalization schema asset = %v, want registered private schema", err) + if _, err := fs.ReadFile(schemaFS, "dnd_entity_reconcile_llm.v1.json"); err != nil { + t.Fatalf("entity reconciliation schema asset = %v, want registered shared schema", err) } assertContainsKeys(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"dnd/scenes"}) assertContainsKeys(t, "extractors", registries.Extractors.RegisteredKeys(), []string{"dnd/spells", npcextract.Key, combatextract.Key, enemyeventextract.Key, itemeventextract.Key, interactionextract.Key, scenedescriptionextract.Key}) @@ -289,6 +289,12 @@ func TestRegisterAddsDNDFamily(t *testing.T) { "dnd.npcs/sharedassets/common-dnd-system.md", "dnd.npcs/sharedassets/common-dnd-transcript.md", "dnd.npcs/task.md", + "dnd.npcs.normalize/candidates.md", + "dnd.npcs.normalize/dnd.npcs.normalize.yaml", + "dnd.npcs.normalize/sharedassets/common-dnd-entity-reconciliation.md", + "dnd.npcs.normalize/sharedassets/common-dnd-system.md", + "dnd.npcs.normalize/sharedassets/common-dnd-transcript.md", + "dnd.npcs.normalize/task.md", "dnd.combat_turns/dnd.combat_turns.yaml", "dnd.combat_turns/instructions.md", "dnd.combat_turns/sharedassets/common-dnd-references.md", diff --git a/internal/modules/integration/dnd_npc_grounded_test.go b/internal/modules/integration/dnd_npc_grounded_test.go index 9cda74b..08f549a 100644 --- a/internal/modules/integration/dnd_npc_grounded_test.go +++ b/internal/modules/integration/dnd_npc_grounded_test.go @@ -52,7 +52,7 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T "extract:npcs:dnd/npcs:mapping_policy": "dnd.npcs.extract_mapping.v2", "normalize:npcs:dnd/npcs:identity_policy": "dnd.npcs.identity.v1", "normalize:npcs:dnd/npcs:normalization_policy": "dnd.npcs.normalize.v3", - "normalize:npcs:dnd/npcs:semantic_context_policy": "dnd.npcs.semantic_context.v1:2", + "normalize:npcs:dnd/npcs:semantic_context_policy": "dnd.entity_reconcile.context.v1:2", "extract:spells:dnd/spells:mapping_policy": "dnd.spells.extract_mapping.v2", "extract:combat:dnd/combat-turns:scene_gate_policy": "dnd.combat_turns.scene_gate.v1", } { diff --git a/internal/modules/integration/dnd_npc_interactions_runner_test.go b/internal/modules/integration/dnd_npc_interactions_runner_test.go index ee2b2ec..ecc7c15 100644 --- a/internal/modules/integration/dnd_npc_interactions_runner_test.go +++ b/internal/modules/integration/dnd_npc_interactions_runner_test.go @@ -289,7 +289,7 @@ func (client *semanticNPCInteractionClient) CompleteStructured(_ context.Context } payload = map[string]any{"npcs": []any{map[string]any{"name": name, "source_refs": []any{map[string]int{"start_unit_id": client.npcCalls, "end_unit_id": client.npcCalls}}}}} case npcnormalize.PromptID: - payload = map[string]any{"duplicate_groups": []any{map[string]any{"members": []string{"Mira Thorn", "Mira Thorn, the Greencloak"}, "canonical_name": "Mira Thorn"}}} + payload = map[string]any{"duplicate_groups": []any{map[string]any{"members": []string{"candidate-000001", "candidate-000002"}, "canonical": "candidate-000001"}}} case interactionextract.PromptID: payload = map[string]any{"interactions": []any{map[string]any{"name": "Mira Thorn", "kind": "dialogue", "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}}}}} default: diff --git a/internal/modules/integration/dnd_npcs_runner_test.go b/internal/modules/integration/dnd_npcs_runner_test.go index 781fca1..539bb02 100644 --- a/internal/modules/integration/dnd_npcs_runner_test.go +++ b/internal/modules/integration/dnd_npcs_runner_test.go @@ -17,6 +17,7 @@ import ( npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcs" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" dndregister "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/register" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/entityreconcile" npcshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcs/shape" npcsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcs/source_refs" genericregister "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/register" @@ -96,7 +97,7 @@ func TestRunnerProcessesSeriatimInputWithProductionDNDNPCPipeline(t *testing.T) t.Fatalf("manifest lane = %#v, want NPC production composition", lane) } normalizerMetadata, ok := lane.Metadata["normalizer"].(map[string]any) - if !ok || normalizerMetadata["identity_policy"] != identity.Policy || normalizerMetadata["normalization_policy"] != npcnormalize.NormalizationPolicy || normalizerMetadata["prompt_id"] != npcnormalize.PromptID || normalizerMetadata["response_schema_id"] != npcnormalize.ResponseSchemaID { + if !ok || normalizerMetadata["identity_policy"] != identity.Policy || normalizerMetadata["normalization_policy"] != npcnormalize.NormalizationPolicy || normalizerMetadata["prompt_id"] != npcnormalize.PromptID || normalizerMetadata["response_schema_id"] != entityreconcile.ResponseSchemaID { t.Fatalf("normalizer metadata = %#v, want identity and normalization policies", lane.Metadata) } var npcOutputFile *contracts.OutputFile @@ -178,8 +179,8 @@ func TestProductionNPCNormalizationRetryUsesFinalSafeProposal(t *testing.T) { {Name: "Mira Thorn", SourceRefs: []npcProductionSourceRef{{StartUnitID: 2, EndUnitID: 2}}}, {Name: "Hooded Guard", SourceRefs: []npcProductionSourceRef{{StartUnitID: 3, EndUnitID: 3}}}, }} - partial := []byte(`{"duplicate_groups":[{"members":["Mira","Mira Thorn"],"canonical_name":"Mira Thorn"},{"members":["Hooded Guard","Unknown"],"canonical_name":"Hooded Guard"}]}`) - safe := []byte(`{"duplicate_groups":[{"members":["Mira","Mira Thorn"],"canonical_name":"Mira Thorn"}]}`) + partial := []byte(`{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000002"},{"members":["candidate-000003","unknown"],"canonical":"candidate-000003"}]}`) + safe := []byte(`{"duplicate_groups":[{"members":["candidate-000001","candidate-000002"],"canonical":"candidate-000002"}]}`) for _, test := range []struct { name string