diff --git a/internal/modules/dnd/shared/assets.go b/internal/modules/dnd/shared/assets.go index 1e7483f..cafc407 100644 --- a/internal/modules/dnd/shared/assets.go +++ b/internal/modules/dnd/shared/assets.go @@ -23,12 +23,13 @@ type PromptAssetManifest struct { } var sharedPromptPaths = map[string]string{ - "common-dnd-system.md": "assets/prompts/common-dnd-system.md", - "common-dnd-extraction-evidence.md": "assets/prompts/common-dnd-extraction-evidence.md", - "common-dnd-identity.md": "assets/prompts/common-dnd-identity.md", - "common-dnd-transcript.md": "assets/prompts/common-dnd-transcript.md", - "common-dnd-references.md": "assets/prompts/common-dnd-references.md", - "common-dnd-npcs.md": "assets/prompts/common-dnd-npcs.md", + "common-dnd-system.md": "assets/prompts/common-dnd-system.md", + "common-dnd-extraction-evidence.md": "assets/prompts/common-dnd-extraction-evidence.md", + "common-dnd-identity.md": "assets/prompts/common-dnd-identity.md", + "common-dnd-transcript.md": "assets/prompts/common-dnd-transcript.md", + "common-dnd-references.md": "assets/prompts/common-dnd-references.md", + "common-dnd-npcs.md": "assets/prompts/common-dnd-npcs.md", + "common-dnd-entity-reconciliation.md": "assets/prompts/common-dnd-entity-reconciliation.md", } func (manifest PromptAssetManifest) PromptFS(moduleFS fs.FS) (fs.FS, error) { diff --git a/internal/modules/dnd/shared/assets/prompts/common-dnd-entity-reconciliation.md b/internal/modules/dnd/shared/assets/prompts/common-dnd-entity-reconciliation.md new file mode 100644 index 0000000..04f60df --- /dev/null +++ b/internal/modules/dnd/shared/assets/prompts/common-dnd-entity-reconciliation.md @@ -0,0 +1,6 @@ +Identify only well-supported duplicate groups among the supplied candidates. + +Candidate keys are opaque identifiers. Copy each selected key exactly. A group +must contain at least two supplied keys, and its `canonical` key must be one of +its members. Do not create keys, records, names, source references, evidence, +or replacement values. Omit any uncertain or unsafe group. diff --git a/internal/modules/dnd/shared/entityreconcile/assets.go b/internal/modules/dnd/shared/entityreconcile/assets.go new file mode 100644 index 0000000..90269cc --- /dev/null +++ b/internal/modules/dnd/shared/entityreconcile/assets.go @@ -0,0 +1,6 @@ +package entityreconcile + +import "embed" + +//go:embed assets/schemas/dnd_entity_reconcile_llm.v1.json +var embeddedAssets embed.FS diff --git a/internal/modules/dnd/shared/entityreconcile/assets/schemas/dnd_entity_reconcile_llm.v1.json b/internal/modules/dnd/shared/entityreconcile/assets/schemas/dnd_entity_reconcile_llm.v1.json new file mode 100644 index 0000000..49fe3e7 --- /dev/null +++ b/internal/modules/dnd/shared/entityreconcile/assets/schemas/dnd_entity_reconcile_llm.v1.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "notarius.dnd.entity_reconcile.llm", + "type": "object", + "additionalProperties": false, + "required": ["duplicate_groups"], + "properties": { + "duplicate_groups": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["members", "canonical"], + "properties": { + "members": { + "type": "array", + "items": {"type": "string"} + }, + "canonical": {"type": "string"} + } + } + } + } +} diff --git a/internal/modules/dnd/shared/entityreconcile/context.go b/internal/modules/dnd/shared/entityreconcile/context.go new file mode 100644 index 0000000..3840cd1 --- /dev/null +++ b/internal/modules/dnd/shared/entityreconcile/context.go @@ -0,0 +1,226 @@ +// Package entityreconcile provides safe, D&D-specific duplicate proposal +// materials shared by entity normalizers. +package entityreconcile + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "sort" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" +) + +const candidateKeyFormat = "candidate-%06d" + +// Candidate is one domain-neutral entity candidate supplied by a normalizer. +// BuildContext never retains or mutates its source references. +type Candidate struct { + Name string + SourceRefs []source.SourceRef +} + +// Materials contains owned prompt inputs and opaque candidate-key mappings. +type Materials struct { + Candidates contracts.LLMInputMaterial + Transcript contracts.LLMInputMaterial + + candidateKeys []string + eligible map[string]struct{} +} + +// CandidateKeys returns all deterministic keys in candidate input order. +func (m Materials) CandidateKeys() []string { + return append([]string(nil), m.candidateKeys...) +} + +// EligibleCandidateKeys returns only candidates whose evidence safely produced +// transcript context, preserving candidate input order. +func (m Materials) EligibleCandidateKeys() []string { + keys := make([]string, 0, len(m.eligible)) + for _, key := range m.candidateKeys { + if _, ok := m.eligible[key]; ok { + keys = append(keys, key) + } + } + return keys +} + +type candidateInput struct { + Candidates []candidateView `json:"candidates"` +} + +type candidateView struct { + Key string `json:"key"` + Name string `json:"name"` + SourceRefs []candidateSourceRef `json:"source_refs"` +} + +type candidateSourceRef struct { + StartUnitID int `json:"start_unit_id"` + EndUnitID int `json:"end_unit_id"` +} + +type transcriptInput struct { + Windows []transcriptWindow `json:"windows"` +} + +type transcriptWindow struct { + Units []transcriptUnit `json:"units"` +} + +type transcriptUnit struct { + ID int `json:"id"` + Kind string `json:"kind"` + Text string `json:"text"` + Metadata map[string]any `json:"metadata,omitempty"` + Cited bool `json:"cited"` +} + +type sourceInterval struct { + start int + end int +} + +// BuildContext constructs bounded, source-ordered prompt inputs. It returns +// ready=false when fewer than two candidates have safe context. +func BuildContext(doc *source.SourceDocument, candidates []Candidate, radius int) (Materials, bool, error) { + if radius < 0 { + return Materials{}, false, fmt.Errorf("build entity reconciliation context: radius must not be negative") + } + materials := Materials{ + candidateKeys: make([]string, len(candidates)), + eligible: make(map[string]struct{}), + } + for index := range candidates { + key := fmt.Sprintf(candidateKeyFormat, index+1) + materials.candidateKeys[index] = key + } + if doc == nil { + return materials, false, nil + } + + index := source.NewDocumentIndex(doc) + views := make([]candidateView, 0, len(candidates)) + intervals := make([]sourceInterval, 0) + cited := make([]bool, len(doc.Units)) + for candidateIndex, candidate := range candidates { + references, candidateIntervals, valid := candidateReferences(index, candidate.SourceRefs) + if !valid { + continue + } + key := materials.candidateKeys[candidateIndex] + materials.eligible[key] = struct{}{} + views = append(views, candidateView{Key: key, Name: candidate.Name, SourceRefs: references}) + for _, interval := range candidateIntervals { + for position := interval.start; position <= interval.end; position++ { + cited[position] = true + } + intervals = append(intervals, sourceInterval{ + start: maxInt(0, interval.start-radius), + end: minInt(len(doc.Units)-1, interval.end+radius), + }) + } + } + if len(views) < 2 { + return materials, false, nil + } + + windows, err := contextWindows(doc.Units, coalesceIntervals(intervals), cited) + if err != nil { + return Materials{}, false, fmt.Errorf("build entity reconciliation context: invalid source metadata") + } + candidateContent, err := json.Marshal(candidateInput{Candidates: views}) + if err != nil { + return Materials{}, false, fmt.Errorf("build entity reconciliation context: invalid candidate material") + } + transcriptContent, err := json.Marshal(transcriptInput{Windows: windows}) + if err != nil { + return Materials{}, false, fmt.Errorf("build entity reconciliation context: invalid transcript material") + } + materials.Candidates = newInputMaterial("candidates", candidateContent) + materials.Transcript = newInputMaterial("transcript", transcriptContent) + return materials, true, nil +} + +func candidateReferences(index source.DocumentIndex, refs []source.SourceRef) ([]candidateSourceRef, []sourceInterval, bool) { + if len(refs) == 0 { + return nil, nil, false + } + references := make([]candidateSourceRef, 0, len(refs)) + intervals := make([]sourceInterval, 0, len(refs)) + for _, ref := range refs { + if err := index.ValidateRef(ref); err != nil { + return nil, nil, false + } + start, _ := index.Position(ref.StartUnitID) + end, _ := index.Position(ref.EndUnitID) + references = append(references, candidateSourceRef{StartUnitID: ref.StartUnitID, EndUnitID: ref.EndUnitID}) + intervals = append(intervals, sourceInterval{start: start, end: end}) + } + return references, intervals, true +} + +func coalesceIntervals(intervals []sourceInterval) []sourceInterval { + if len(intervals) == 0 { + return nil + } + ordered := append([]sourceInterval(nil), intervals...) + sort.Slice(ordered, func(left, right int) bool { + if ordered[left].start != ordered[right].start { + return ordered[left].start < ordered[right].start + } + return ordered[left].end < ordered[right].end + }) + coalesced := make([]sourceInterval, 0, len(ordered)) + for _, interval := range ordered { + if len(coalesced) == 0 || interval.start > coalesced[len(coalesced)-1].end+1 { + coalesced = append(coalesced, interval) + continue + } + if interval.end > coalesced[len(coalesced)-1].end { + coalesced[len(coalesced)-1].end = interval.end + } + } + return coalesced +} + +func contextWindows(units []source.SourceUnit, intervals []sourceInterval, cited []bool) ([]transcriptWindow, error) { + windows := make([]transcriptWindow, 0, len(intervals)) + for _, interval := range intervals { + window := transcriptWindow{Units: make([]transcriptUnit, 0, interval.end-interval.start+1)} + for position := interval.start; position <= interval.end; position++ { + unit := units[position] + metadata, err := source.CloneMetadata(unit.Metadata) + if err != nil { + return nil, err + } + window.Units = append(window.Units, transcriptUnit{ + ID: unit.ID, Kind: unit.Kind, Text: unit.Text, Metadata: metadata, Cited: cited[position], + }) + } + windows = append(windows, window) + } + return windows, nil +} + +func newInputMaterial(name string, content []byte) contracts.LLMInputMaterial { + digest := sha256.Sum256(content) + return contracts.NewLLMInputMaterial(name, "application/json", content, "sha256:"+hex.EncodeToString(digest[:]), "") +} + +func minInt(left, right int) int { + if left < right { + return left + } + return right +} + +func maxInt(left, right int) int { + if left > right { + return left + } + return right +} diff --git a/internal/modules/dnd/shared/entityreconcile/entityreconcile_test.go b/internal/modules/dnd/shared/entityreconcile/entityreconcile_test.go new file mode 100644 index 0000000..5620c1d --- /dev/null +++ b/internal/modules/dnd/shared/entityreconcile/entityreconcile_test.go @@ -0,0 +1,275 @@ +package entityreconcile + +import ( + "bytes" + "encoding/json" + "io/fs" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + "github.com/santhosh-tekuri/jsonschema/v6" +) + +func TestBuildContextUsesOpaqueKeysSourceOrderAndOwnedData(t *testing.T) { + doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{ + {ID: 40, Kind: "narration", Text: "zero"}, + {ID: 10, Kind: "speech", Text: "one", Metadata: map[string]any{"speaker": map[string]any{"name": "Mira"}}}, + {ID: 70, Kind: "speech", Text: "two"}, + {ID: 20, Kind: "narration", Text: "three"}, + {ID: 90, Kind: "speech", Text: "four"}, + }} + candidates := []Candidate{ + {Name: "The Tavern", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 20}}}, + {Name: "The Tavern", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 90, EndUnitID: 90}}}, + {Name: "Broken", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 10}}}, + } + before := cloneCandidates(candidates) + + materials, ready, err := BuildContext(doc, candidates, 1) + if err != nil || !ready { + t.Fatalf("BuildContext() = %#v, %t, %v; want ready materials", materials, ready, err) + } + if !reflect.DeepEqual(candidates, before) { + t.Fatalf("BuildContext() mutated candidates: %#v", candidates) + } + if got, want := materials.CandidateKeys(), []string{"candidate-000001", "candidate-000002", "candidate-000003"}; !reflect.DeepEqual(got, want) { + t.Fatalf("CandidateKeys() = %#v, want %#v", got, want) + } + if got, want := materials.EligibleCandidateKeys(), []string{"candidate-000001", "candidate-000002"}; !reflect.DeepEqual(got, want) { + t.Fatalf("EligibleCandidateKeys() = %#v, want %#v", got, want) + } + if !json.Valid(materials.Candidates.Content) || !json.Valid(materials.Transcript.Content) { + t.Fatalf("prompt materials are not JSON: %#v", materials) + } + if strings.Contains(string(materials.Candidates.Content), doc.ID) { + t.Fatalf("candidate material leaked source identity: %s", materials.Candidates.Content) + } + + var candidatePayload candidateInput + if err := json.Unmarshal(materials.Candidates.Content, &candidatePayload); err != nil { + t.Fatal(err) + } + if len(candidatePayload.Candidates) != 2 || candidatePayload.Candidates[0].Key != "candidate-000001" || candidatePayload.Candidates[1].Key != "candidate-000002" || candidatePayload.Candidates[0].Name != candidatePayload.Candidates[1].Name { + t.Fatalf("candidate payload = %#v, want distinct opaque keys for equal names", candidatePayload) + } + if got := candidatePayload.Candidates[0].SourceRefs[0]; got != (candidateSourceRef{StartUnitID: 10, EndUnitID: 20}) { + t.Fatalf("candidate reference = %#v", got) + } + + var transcript transcriptInput + if err := json.Unmarshal(materials.Transcript.Content, &transcript); err != nil { + t.Fatal(err) + } + if len(transcript.Windows) != 1 || len(transcript.Windows[0].Units) != 5 { + t.Fatalf("windows = %#v, want one bounded coalesced window", transcript.Windows) + } + units := transcript.Windows[0].Units + for index, wantID := range []int{40, 10, 70, 20, 90} { + if units[index].ID != wantID { + t.Fatalf("window unit %d = %d, want source-order %d", index, units[index].ID, wantID) + } + } + if units[0].Cited || !units[1].Cited || !units[2].Cited || !units[3].Cited || !units[4].Cited { + t.Fatalf("citation flags = %#v", units) + } + + windows, err := contextWindows(doc.Units, []sourceInterval{{start: 1, end: 1}}, make([]bool, len(doc.Units))) + if err != nil { + t.Fatal(err) + } + windows[0].Units[0].Metadata["speaker"].(map[string]any)["name"] = "changed" + if doc.Units[1].Metadata["speaker"].(map[string]any)["name"] != "Mira" { + t.Fatal("context metadata aliases source document") + } +} + +func TestBuildContextExcludesUnsafeReferencesAndCoalescesAdjacentWindows(t *testing.T) { + doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 9}, {ID: 3}, {ID: 8}, {ID: 1}, {ID: 7}}} + candidates := []Candidate{ + {Name: "One", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 3, EndUnitID: 3}}}, + {Name: "Two", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 8, EndUnitID: 8}}}, + {Name: "Missing", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 99, EndUnitID: 99}}}, + {Name: "Foreign", SourceRefs: []source.SourceRef{{SourceID: "other", StartUnitID: 1, EndUnitID: 1}}}, + {Name: "Blank"}, + } + materials, ready, err := BuildContext(doc, candidates, 0) + if err != nil || !ready { + t.Fatalf("BuildContext() error = %v, ready = %t", err, ready) + } + if got, want := materials.EligibleCandidateKeys(), []string{"candidate-000001", "candidate-000002"}; !reflect.DeepEqual(got, want) { + t.Fatalf("EligibleCandidateKeys() = %#v, want %#v", got, want) + } + var transcript transcriptInput + if err := json.Unmarshal(materials.Transcript.Content, &transcript); err != nil { + t.Fatal(err) + } + if len(transcript.Windows) != 1 || len(transcript.Windows[0].Units) != 2 || transcript.Windows[0].Units[0].ID != 3 || transcript.Windows[0].Units[1].ID != 8 { + t.Fatalf("windows = %#v, want adjacent document-order units coalesced", transcript.Windows) + } + if _, ready, err := BuildContext(doc, candidates, -1); err == nil || ready { + t.Fatalf("BuildContext(radius=-1) = ready %t, err %v", ready, err) + } +} + +func TestAssessmentRejectsEveryUnsafeProposalCategory(t *testing.T) { + materials := preparedMaterials(t, 4, true) + keys := materials.CandidateKeys() + unsafe := []struct { + name string + response ProposalResponse + category string + }{ + {"blank member", ProposalResponse{DuplicateGroups: []DuplicateGroup{{Members: []string{"", keys[1]}, Canonical: keys[1]}}}, "member_blank"}, + {"unknown member", ProposalResponse{DuplicateGroups: []DuplicateGroup{{Members: []string{"candidate-999999", keys[1]}, Canonical: keys[1]}}}, "member_unknown"}, + {"ineligible member", ProposalResponse{DuplicateGroups: []DuplicateGroup{{Members: []string{keys[0], keys[3]}, Canonical: keys[0]}}}, "member_ineligible"}, + {"repeated member", ProposalResponse{DuplicateGroups: []DuplicateGroup{{Members: []string{keys[0], keys[0]}, Canonical: keys[0]}}}, "repeated_member"}, + {"too small", ProposalResponse{DuplicateGroups: []DuplicateGroup{{Members: []string{keys[0]}, Canonical: keys[0]}}}, "fewer_than_two_members"}, + {"canonical blank", ProposalResponse{DuplicateGroups: []DuplicateGroup{{Members: []string{keys[0], keys[1]}, Canonical: ""}}}, "canonical_blank"}, + {"canonical not member", ProposalResponse{DuplicateGroups: []DuplicateGroup{{Members: []string{keys[0], keys[1]}, Canonical: keys[2]}}}, "canonical_not_member"}, + {"overlapping", ProposalResponse{DuplicateGroups: []DuplicateGroup{ + {Members: []string{keys[0], keys[1]}, Canonical: keys[0]}, + {Members: []string{keys[1], keys[2]}, Canonical: keys[2]}, + }}, "overlapping_member"}, + } + for _, test := range unsafe { + t.Run(test.name, func(t *testing.T) { + assessment := materials.Assess(test.response) + if len(assessment.SafeGroups()) != 0 || assessment.DiscardedGroups() != len(test.response.DuplicateGroups) || !hasIssue(assessment.Issues(), test.category) { + t.Fatalf("Assess() = groups %#v discarded %d issues %#v; want %q rejection", assessment.SafeGroups(), assessment.DiscardedGroups(), assessment.Issues(), test.category) + } + }) + } +} + +func TestAssessmentReturnsNonOverlappingSafeGroupsAndDefensiveCopies(t *testing.T) { + materials := preparedMaterials(t, 4, false) + keys := materials.CandidateKeys() + assessment := materials.Assess(ProposalResponse{DuplicateGroups: []DuplicateGroup{ + {Members: []string{keys[1], keys[0]}, Canonical: keys[1]}, + {Members: []string{keys[3], keys[2]}, Canonical: keys[2]}, + }}) + groups := assessment.SafeGroups() + if assessment.DiscardedGroups() != 0 || len(assessment.Issues()) != 0 || len(groups) != 2 { + t.Fatalf("assessment = %#v, %d, %#v", groups, assessment.DiscardedGroups(), assessment.Issues()) + } + if got, want := groups[0].Members(), []string{keys[0], keys[1]}; !reflect.DeepEqual(got, want) || groups[0].Canonical() != keys[1] { + t.Fatalf("first safe group = %#v / %q", got, groups[0].Canonical()) + } + keys[0] = "changed" + if materials.CandidateKeys()[0] == "changed" { + t.Fatal("CandidateKeys() exposed retained keys") + } + members := groups[0].Members() + members[0] = "changed" + if groups[0].Members()[0] == "changed" || assessment.SafeGroups()[0].Members()[0] == "changed" { + t.Fatal("SafeGroups() exposed retained members") + } +} + +func TestSharedResponseSchemaIsPrivateStrictAndRegisterableOnce(t *testing.T) { + schema, err := LoadResponseSchema() + if err != nil { + t.Fatal(err) + } + if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) { + t.Fatalf("schema = %#v", schema) + } + for _, test := range []struct { + name string + value any + valid bool + }{ + {"empty groups", map[string]any{"duplicate_groups": []any{}}, true}, + {"semantic proposal problem", map[string]any{"duplicate_groups": []any{map[string]any{"members": []any{""}, "canonical": ""}}}, true}, + {"missing groups", map[string]any{}, false}, + {"unknown top level", map[string]any{"duplicate_groups": []any{}, "extra": true}, false}, + {"replacement name", map[string]any{"duplicate_groups": []any{map[string]any{"members": []any{}, "canonical": "candidate-000001", "name": "replacement"}}}, false}, + {"replacement evidence", map[string]any{"duplicate_groups": []any{map[string]any{"members": []any{}, "canonical": "candidate-000001", "source_refs": []any{}}}}, false}, + {"wrong key type", map[string]any{"duplicate_groups": []any{map[string]any{"members": []any{1}, "canonical": "candidate-000001"}}}, false}, + } { + t.Run(test.name, func(t *testing.T) { + content, err := json.Marshal(test.value) + if err != nil { + t.Fatal(err) + } + err = validateSchema(content, schema.JSONSchema) + if (err == nil) != test.valid { + t.Fatalf("validateSchema() error = %v, want valid=%t", err, test.valid) + } + }) + } + first := schema.JSONSchema + first[0] = '[' + second, err := LoadResponseSchema() + if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first, second.JSONSchema) { + t.Fatalf("LoadResponseSchema() returned shared content: %s, %v", second.JSONSchema, err) + } + registry := llm.NewAssetRegistry() + if err := RegisterSchemaAssets(registry); err != nil { + t.Fatalf("RegisterSchemaAssets() error = %v", err) + } + if schemaFS, err := registry.SchemaFS(); err != nil { + t.Fatalf("SchemaFS() error = %v", err) + } else if content, err := fs.ReadFile(schemaFS, "dnd_entity_reconcile_llm.v1.json"); err != nil || !json.Valid(content) { + t.Fatalf("shared schema asset = %s, %v", content, err) + } +} + +func preparedMaterials(t *testing.T, count int, includeIneligible bool) Materials { + t.Helper() + doc := &source.SourceDocument{ID: "session", Units: make([]source.SourceUnit, count)} + candidates := make([]Candidate, count) + for index := range candidates { + doc.Units[index] = source.SourceUnit{ID: index + 1, Text: "unit"} + candidates[index] = Candidate{Name: "same display name", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: index + 1, EndUnitID: index + 1}}} + } + if includeIneligible && count > 3 { + candidates[3].SourceRefs = []source.SourceRef{{SourceID: "other", StartUnitID: 1, EndUnitID: 1}} + } + materials, ready, err := BuildContext(doc, candidates, 0) + if err != nil || !ready { + t.Fatalf("BuildContext() = %#v, %t, %v", materials, ready, err) + } + return materials +} + +func cloneCandidates(input []Candidate) []Candidate { + output := make([]Candidate, len(input)) + copy(output, input) + for index := range output { + output[index].SourceRefs = append([]source.SourceRef(nil), input[index].SourceRefs...) + } + return output +} + +func hasIssue(issues []Issue, want string) bool { + for _, issue := range issues { + if issue.Category == want { + return true + } + } + return false +} + +func validateSchema(instanceContent, schemaContent []byte) error { + instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent)) + if err != nil { + return err + } + document, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent)) + if err != nil { + return err + } + compiler := jsonschema.NewCompiler() + if err := compiler.AddResource("schema.json", document); err != nil { + return err + } + compiled, err := compiler.Compile("schema.json") + if err != nil { + return err + } + return compiled.Validate(instance) +} diff --git a/internal/modules/dnd/shared/entityreconcile/proposal.go b/internal/modules/dnd/shared/entityreconcile/proposal.go new file mode 100644 index 0000000..b3dcd8a --- /dev/null +++ b/internal/modules/dnd/shared/entityreconcile/proposal.go @@ -0,0 +1,168 @@ +package entityreconcile + +import ( + "sort" + "strings" +) + +// ProposalResponse is the private structured response exchanged with the +// reconciliation prompt. It identifies candidates only by opaque keys. +type ProposalResponse struct { + DuplicateGroups []DuplicateGroup `json:"duplicate_groups"` +} + +// DuplicateGroup proposes candidate keys that might denote one entity. +type DuplicateGroup struct { + Members []string `json:"members"` + Canonical string `json:"canonical"` +} + +// Issue identifies one unsafe proposal category without prescribing a warning +// message or retry policy to a consuming normalizer. +type Issue struct { + GroupIndex int + Category string +} + +// SafeGroup identifies one validated, non-overlapping duplicate group. +type SafeGroup struct { + members []string + canonical string +} + +// Members returns an owned copy of the group's candidate keys. +func (g SafeGroup) Members() []string { return append([]string(nil), g.members...) } + +// Canonical returns the selected canonical candidate key. +func (g SafeGroup) Canonical() string { return g.canonical } + +// Assessment contains only safe groups. Its accessors return owned copies so +// callers cannot mutate retained assessment data. +type Assessment struct { + safeGroups []SafeGroup + discardedGroups int + issues []Issue +} + +// SafeGroups returns validated non-overlapping groups in proposal order. +func (a Assessment) SafeGroups() []SafeGroup { + groups := make([]SafeGroup, len(a.safeGroups)) + for index, group := range a.safeGroups { + groups[index] = SafeGroup{members: append([]string(nil), group.members...), canonical: group.canonical} + } + return groups +} + +// DiscardedGroups returns the number of rejected proposal groups. +func (a Assessment) DiscardedGroups() int { return a.discardedGroups } + +// Issues returns the deterministic rejection categories in proposal order. +func (a Assessment) Issues() []Issue { return append([]Issue(nil), a.issues...) } + +// Assess validates a proposal against the opaque keys created by BuildContext. +func (m Materials) Assess(response ProposalResponse) Assessment { + all := make(map[string]struct{}, len(m.candidateKeys)) + for _, key := range m.candidateKeys { + all[key] = struct{}{} + } + groups := make([]assessedGroup, len(response.DuplicateGroups)) + issues := make([]Issue, 0) + for groupIndex, proposal := range response.DuplicateGroups { + groups[groupIndex] = assessGroup(proposal, all, m.eligible) + for _, category := range groups[groupIndex].issues { + issues = append(issues, Issue{GroupIndex: groupIndex, Category: category}) + } + } + + owners := make(map[string][]int) + for groupIndex, group := range groups { + if !group.locallyValid { + continue + } + for _, key := range group.members { + owners[key] = append(owners[key], groupIndex) + } + } + for groupIndex := range groups { + if !groups[groupIndex].locallyValid { + continue + } + for _, key := range groups[groupIndex].members { + if len(owners[key]) > 1 { + groups[groupIndex].conflicting = true + issues = append(issues, Issue{GroupIndex: groupIndex, Category: "overlapping_member"}) + break + } + } + } + + assessment := Assessment{issues: issues} + for _, group := range groups { + if !group.locallyValid || group.conflicting { + assessment.discardedGroups++ + continue + } + assessment.safeGroups = append(assessment.safeGroups, SafeGroup{members: append([]string(nil), group.members...), canonical: group.canonical}) + } + return assessment +} + +type assessedGroup struct { + members []string + canonical string + issues []string + locallyValid bool + conflicting bool +} + +func assessGroup(proposal DuplicateGroup, all, eligible map[string]struct{}) assessedGroup { + issues := make([]string, 0) + members := make([]string, 0, len(proposal.Members)) + seen := make(map[string]struct{}, len(proposal.Members)) + for _, key := range proposal.Members { + if category := keyCategory(key, all, eligible); category != "" { + issues = append(issues, "member_"+category) + continue + } + if _, exists := seen[key]; exists { + issues = append(issues, "repeated_member") + continue + } + seen[key] = struct{}{} + members = append(members, key) + } + canonicalCategory := keyCategory(proposal.Canonical, all, eligible) + if canonicalCategory != "" { + issues = append(issues, "canonical_"+canonicalCategory) + } + if len(members) < 2 { + issues = append(issues, "fewer_than_two_members") + } + if canonicalCategory == "" && !contains(members, proposal.Canonical) { + issues = append(issues, "canonical_not_member") + } + sort.Strings(members) + return assessedGroup{members: members, canonical: proposal.Canonical, issues: issues, locallyValid: len(issues) == 0} +} + +func keyCategory(key string, all, eligible map[string]struct{}) string { + if strings.TrimSpace(key) == "" { + return "blank" + } + if _, ok := all[key]; !ok { + return "unknown" + } + if _, ok := eligible[key]; !ok { + return "ineligible" + } + return "" +} + +func contains(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} diff --git a/internal/modules/dnd/shared/entityreconcile/schema.go b/internal/modules/dnd/shared/entityreconcile/schema.go new file mode 100644 index 0000000..d4dacd9 --- /dev/null +++ b/internal/modules/dnd/shared/entityreconcile/schema.go @@ -0,0 +1,31 @@ +package entityreconcile + +import ( + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" +) + +const ( + ResponseSchemaKey = llm.ResponseSchemaKey("dnd_entity_reconcile_llm") + ResponseSchemaID = "notarius.dnd.entity_reconcile.llm" + ResponseSchemaName = "notarius_dnd_entity_reconcile_llm_v1" + SchemaVersion = "v1" + SchemaAssetPath = "assets/schemas/dnd_entity_reconcile_llm.v1.json" +) + +// LoadResponseSchema returns the shared private duplicate-group response +// contract. It is intentionally separate from durable artifact schemas. +func LoadResponseSchema() (llm.ResponseSchema, error) { + return llm.LoadResponseSchema(embeddedAssets, llm.ResponseSchemaDefinition{ + Key: ResponseSchemaKey, + ID: ResponseSchemaID, + Version: SchemaVersion, + Name: ResponseSchemaName, + AssetPath: SchemaAssetPath, + }) +} + +// RegisterSchemaAssets makes the shared private response schema available to +// prompt preparation. A family registrar can register it once for all consumers. +func RegisterSchemaAssets(registry *llm.AssetRegistry) error { + return registry.RegisterSchemaFS(embeddedAssets, "assets/schemas") +}