diff --git a/assets/dnd/entity-reconciliation/schemas/dnd_entity_reconcile_llm.v1.json b/assets/dnd/entity-reconciliation/schemas/dnd_entity_reconcile_llm.v1.json deleted file mode 100644 index a251924..0000000 --- a/assets/dnd/entity-reconciliation/schemas/dnd_entity_reconcile_llm.v1.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "notarius.dnd.entity_reconcile.llm", - "type": "object", - "additionalProperties": false, - "required": ["duplicate_groups"], - "properties": { - "duplicate_groups": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "required": ["members", "canonical"], - "properties": { - "members": { - "type": "array", - "items": {"$ref": "#/$defs/selector"} - }, - "canonical": {"$ref": "#/$defs/selector"} - } - } - } - }, - "$defs": { - "selector": { - "type": "object", - "additionalProperties": false, - "required": ["name", "source_refs"], - "properties": { - "name": {"type": "string", "minLength": 1}, - "source_refs": { - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "additionalProperties": false, - "required": ["start_unit_id", "end_unit_id"], - "properties": { - "start_unit_id": {"type": "integer", "minimum": 1}, - "end_unit_id": {"type": "integer", "minimum": 1} - } - } - } - } - } - } -} diff --git a/assets/dnd/shared/prompts/common-dnd-entity-reconciliation.md b/assets/dnd/shared/prompts/common-dnd-entity-reconciliation.md deleted file mode 100644 index c45ff9b..0000000 --- a/assets/dnd/shared/prompts/common-dnd-entity-reconciliation.md +++ /dev/null @@ -1,7 +0,0 @@ -Identify only well-supported duplicate groups among the supplied candidates. - -Return each selected candidate's supplied contextual descriptor exactly: its -`name` and complete ordered `source_refs`. A group must contain at least two -supplied descriptors, and its `canonical` descriptor must be one of its -members. Do not invent names, ranges, records, evidence, or replacement values. -Omit any uncertain or unsafe group. diff --git a/internal/cli/production_contract_test.go b/internal/cli/production_contract_test.go index db142c4..8ae2cf2 100644 --- a/internal/cli/production_contract_test.go +++ b/internal/cli/production_contract_test.go @@ -25,6 +25,7 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes" combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns" @@ -311,6 +312,47 @@ func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) { if _, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil { t.Fatalf("prepare production scene and spell modules: %v", err) } + + schemaFS, err := components.assets.SchemaFS() + if err != nil { + t.Fatalf("production schema assets: %v", err) + } + if _, err := fs.ReadFile(schemaFS, filepath.Base(semanticreconcile.SchemaAssetPath)); err != nil { + t.Fatalf("generic reconciliation schema asset: %v", err) + } + options, err := components.assets.PromptKitOptions() + if err != nil { + t.Fatalf("production PromptKit options: %v", err) + } + options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ + ID: "assembled-prompt-test", Endpoint: "http://127.0.0.1:1/v1", Model: "test", + }))) + engine, err := promptkit.NewEngine(promptkit.Config{}, options...) + if err != nil { + t.Fatalf("production prompt engine: %v", err) + } + inputs := map[string]promptkit.ArtifactRef{ + "candidates": promptkit.Inline(`{"candidates":[{"candidate_id":1,"label":"Alias","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), + "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`), + } + for _, prompt := range []struct { + id string + version string + }{ + {id: npcnormalize.PromptID, version: npcnormalize.PromptVersion}, + {id: itemregistrynormalize.PromptID, version: itemregistrynormalize.PromptVersion}, + {id: locationnormalize.PromptID, version: locationnormalize.PromptVersion}, + } { + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: prompt.id, PromptVersion: prompt.version, ProfileID: "assembled-prompt-test", Inputs: inputs, + }) + if err != nil { + t.Fatalf("prepare production prompt %q: %v", prompt.id, err) + } + if prepared.OutputContract.SchemaPath != filepath.Base(semanticreconcile.SchemaAssetPath) { + t.Fatalf("prompt %q schema = %q, want generic reconciliation schema", prompt.id, prepared.OutputContract.SchemaPath) + } + } } func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) { diff --git a/internal/framework/semanticreconcile/proposal.go b/internal/framework/semanticreconcile/proposal.go index 0f0b6f9..f44d5b0 100644 --- a/internal/framework/semanticreconcile/proposal.go +++ b/internal/framework/semanticreconcile/proposal.go @@ -1,6 +1,9 @@ package semanticreconcile -import "sort" +import ( + "fmt" + "sort" +) // ProposalResponse is the complete private structured response contract. type ProposalResponse struct { @@ -35,6 +38,16 @@ type Issue struct { Category IssueCategory } +// IssueDetails renders stable, domain-neutral proposal diagnostics for an +// adapter's retry message. +func IssueDetails(issues []Issue) []string { + details := make([]string, len(issues)) + for index, issue := range issues { + details[index] = fmt.Sprintf("group %d: %s", issue.GroupIndex, issue.Category) + } + return details +} + // PlanGroup identifies one validated group using original candidate positions. type PlanGroup struct { memberPositions []int diff --git a/internal/framework/semanticreconcile/proposal_test.go b/internal/framework/semanticreconcile/proposal_test.go index dba67c7..ae76b62 100644 --- a/internal/framework/semanticreconcile/proposal_test.go +++ b/internal/framework/semanticreconcile/proposal_test.go @@ -39,6 +39,17 @@ func TestAssessProducesAStableOriginalPositionPlan(t *testing.T) { } } +func TestIssueDetailsPreservesIssueOrder(t *testing.T) { + issues := []Issue{ + {GroupIndex: 3, Category: IssueCanonicalUnknown}, + {GroupIndex: 1, Category: IssueMemberUnknown}, + } + want := []string{"group 3: canonical_unknown", "group 1: member_unknown"} + if got := IssueDetails(issues); !reflect.DeepEqual(got, want) { + t.Fatalf("IssueDetails() = %#v, want %#v", got, want) + } +} + func TestAssessRejectsEveryUnsafeLocalGroupShape(t *testing.T) { preparation := proposalPreparation(t) tests := []struct { diff --git a/internal/modules/dnd/extract/itemoccurrences/prompt_assets_test.go b/internal/modules/dnd/extract/itemoccurrences/prompt_assets_test.go index cd9bf7f..a98f08f 100644 --- a/internal/modules/dnd/extract/itemoccurrences/prompt_assets_test.go +++ b/internal/modules/dnd/extract/itemoccurrences/prompt_assets_test.go @@ -48,13 +48,15 @@ func TestPromptAssetsPrepareItemOccurrencePrompt(t *testing.T) { rendered[index] = message.Content } content := strings.Join(rendered, "\n") - for _, field := range []string{"start_unit_id", "end_unit_id", "source_id"} { + for _, field := range []string{"start_unit_id", "end_unit_id"} { if !strings.Contains(content, field) { t.Fatalf("prepared prompt does not include shared evidence field %q", field) } } - if strings.Contains(content, "start_segment") || strings.Contains(content, "end_segment") { - t.Fatalf("prepared prompt contains obsolete segment evidence fields: %s", content) + for _, obsolete := range []string{"source_id", "start_segment", "end_segment"} { + if strings.Contains(content, obsolete) { + t.Fatalf("prepared prompt contains obsolete evidence field %q: %s", obsolete, content) + } } } diff --git a/internal/modules/dnd/normalize/itemregistry/normalizer.go b/internal/modules/dnd/normalize/itemregistry/normalizer.go index c8a1b8d..57f2fc1 100644 --- a/internal/modules/dnd/normalize/itemregistry/normalizer.go +++ b/internal/modules/dnd/normalize/itemregistry/normalizer.go @@ -154,7 +154,7 @@ func (n *Normalizer) invalidStructuredResult(value dnd.ItemRegistry, warnings [] } func retryResult(value dnd.ItemRegistry, warnings []contracts.Warning, reconciliation semanticreconcile.Result, rejectedGroups int) contracts.TypedNormalizeResult[dnd.ItemRegistry] { - details := reconciliationIssues(reconciliation.Issues()) + details := semanticreconcile.IssueDetails(reconciliation.Issues()) if rejectedGroups > 0 { details = append(details, "currency may only be consolidated with aliases of one denomination") } diff --git a/internal/modules/dnd/normalize/itemregistry/reconciliation.go b/internal/modules/dnd/normalize/itemregistry/reconciliation.go index 5cbe6c3..308a5b6 100644 --- a/internal/modules/dnd/normalize/itemregistry/reconciliation.go +++ b/internal/modules/dnd/normalize/itemregistry/reconciliation.go @@ -133,14 +133,6 @@ func currencyDenomination(name string) string { } } -func reconciliationIssues(issues []semanticreconcile.Issue) []string { - details := make([]string, len(issues)) - for index, issue := range issues { - details[index] = fmt.Sprintf("group %d: %s", issue.GroupIndex, issue.Category) - } - return details -} - func semanticDuplicateWarning(provenance semanticreconcile.GroupProvenance, canonical normalizedRecord) contracts.Warning { inputIndexes := provenance.OriginalInputIndexes() details := make([]string, 0, len(inputIndexes)+1) diff --git a/internal/modules/dnd/normalize/locationregistry/normalizer.go b/internal/modules/dnd/normalize/locationregistry/normalizer.go index ffbfc53..b75a13c 100644 --- a/internal/modules/dnd/normalize/locationregistry/normalizer.go +++ b/internal/modules/dnd/normalize/locationregistry/normalizer.go @@ -155,7 +155,7 @@ func (n *Normalizer) invalidStructuredResult(value dnd.LocationRegistry, warning func retryResult(value dnd.LocationRegistry, warnings []contracts.Warning, reconciliation semanticreconcile.Result) contracts.TypedNormalizeResult[dnd.LocationRegistry] { return contracts.TypedNormalizeResult[dnd.LocationRegistry]{Value: value, Warnings: limitWarningsForRetry(warnings), Retry: &contracts.NormalizeRetry{ - ReasonCode: ReasonCodeLocationSemanticProposalInvalid, Message: diagnostics.Aggregate("semantic proposal requires retry", reconciliationIssues(reconciliation.Issues())), + ReasonCode: ReasonCodeLocationSemanticProposalInvalid, Message: diagnostics.Aggregate("semantic proposal requires retry", semanticreconcile.IssueDetails(reconciliation.Issues())), FallbackWarnings: []contracts.Warning{semanticFallbackWarning(reconciliation.DiscardedGroupCount())}, }} } diff --git a/internal/modules/dnd/normalize/locationregistry/reconciliation.go b/internal/modules/dnd/normalize/locationregistry/reconciliation.go index 24a10c7..ebeaa8b 100644 --- a/internal/modules/dnd/normalize/locationregistry/reconciliation.go +++ b/internal/modules/dnd/normalize/locationregistry/reconciliation.go @@ -63,14 +63,6 @@ func applyReconciliationPlan(plan semanticreconcile.Plan, records []normalizedRe return output, warnings, nil } -func reconciliationIssues(issues []semanticreconcile.Issue) []string { - details := make([]string, len(issues)) - for index, issue := range issues { - details[index] = fmt.Sprintf("group %d: %s", issue.GroupIndex, issue.Category) - } - return details -} - func semanticDuplicateWarning(provenance semanticreconcile.GroupProvenance, canonical normalizedRecord) contracts.Warning { inputIndexes := provenance.OriginalInputIndexes() details := make([]string, 0, len(inputIndexes)+1) diff --git a/internal/modules/dnd/normalize/npcregistry/normalizer.go b/internal/modules/dnd/normalize/npcregistry/normalizer.go index 63a41cd..a4ce414 100644 --- a/internal/modules/dnd/normalize/npcregistry/normalizer.go +++ b/internal/modules/dnd/normalize/npcregistry/normalizer.go @@ -163,7 +163,7 @@ func retryResult(value dnd.NPCRegistry, warnings []contracts.Warning, reconcilia Warnings: limitWarningsForRetry(warnings), Retry: &contracts.NormalizeRetry{ ReasonCode: ReasonCodeNPCSemanticProposalInvalid, - Message: diagnostics.Aggregate("semantic proposal requires retry", reconciliationIssues(reconciliation.Issues())), + Message: diagnostics.Aggregate("semantic proposal requires retry", semanticreconcile.IssueDetails(reconciliation.Issues())), FallbackWarnings: []contracts.Warning{semanticFallbackWarning(reconciliation.DiscardedGroupCount())}, }, } diff --git a/internal/modules/dnd/normalize/npcregistry/reconciliation.go b/internal/modules/dnd/normalize/npcregistry/reconciliation.go index e9c43b3..a2232f1 100644 --- a/internal/modules/dnd/normalize/npcregistry/reconciliation.go +++ b/internal/modules/dnd/normalize/npcregistry/reconciliation.go @@ -63,14 +63,6 @@ func applyReconciliationPlan(plan semanticreconcile.Plan, records []normalizedRe return output, warnings, nil } -func reconciliationIssues(issues []semanticreconcile.Issue) []string { - details := make([]string, len(issues)) - for index, issue := range issues { - details[index] = fmt.Sprintf("group %d: %s", issue.GroupIndex, issue.Category) - } - return details -} - func semanticDuplicateWarning(provenance semanticreconcile.GroupProvenance, canonical normalizedRecord) contracts.Warning { inputIndexes := provenance.OriginalInputIndexes() details := make([]string, 0, len(inputIndexes)+1) diff --git a/internal/modules/dnd/register/modules.go b/internal/modules/dnd/register/modules.go index 99fe932..fdf1450 100644 --- a/internal/modules/dnd/register/modules.go +++ b/internal/modules/dnd/register/modules.go @@ -35,7 +35,6 @@ import ( npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry" scenedescriptionnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions" spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/entityreconcile" "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder" "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop" ) @@ -147,7 +146,6 @@ func registerModules(registries pipeline.Registries) error { func registerPromptAssets(assets *llm.AssetRegistry) error { return runRegistrations([]registration{ - {name: "entity reconciliation schema assets", register: func() error { return entityreconcile.RegisterSchemaAssets(assets) }}, {name: "scenes prompt assets", register: func() error { return scenes.RegisterPromptAssets(assets) }}, {name: "spells prompt assets", register: func() error { return spellextract.RegisterPromptAssets(assets) }}, {name: "npc registry prompt assets", register: func() error { return npcextract.RegisterPromptAssets(assets) }}, diff --git a/internal/modules/dnd/register/register_test.go b/internal/modules/dnd/register/register_test.go index b725945..90212e9 100644 --- a/internal/modules/dnd/register/register_test.go +++ b/internal/modules/dnd/register/register_test.go @@ -77,13 +77,6 @@ func TestRegisterAddsDNDFamily(t *testing.T) { if _, err := fs.ReadFile(fallbackFS, "dnd-extraction.yaml"); err != nil { t.Fatalf("fallback profile asset = %v, want registered D&D profile", err) } - schemaFS, err := assets.SchemaFS() - if err != nil { - t.Fatalf("SchemaFS() error = %v", err) - } - if _, err := fs.ReadFile(schemaFS, "dnd_entity_reconcile_llm.v1.json"); err != nil { - t.Fatalf("entity reconciliation schema asset = %v, want registered shared schema", err) - } assertContainsKeys(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"dnd/scenes"}) assertContainsKeys(t, "extractors", registries.Extractors.RegisteredKeys(), []string{"dnd/spells", npcextract.Key, combatextract.Key, enemyeventextract.Key, itemoccurrenceextract.Key, itemregistryextract.Key, occurrenceextract.Key, scenedescriptionextract.Key, locationextract.Key, locationoccurrenceextract.Key}) assertContainsKeys(t, "normalizers", registries.Normalizers.RegisteredKeys(), []string{spellnormalize.Key, npcnormalize.Key, combatnormalize.Key, enemyeventnormalize.Key, itemoccurrencenormalize.Key, itemregistrynormalize.Key, occurrencenormalize.Key, scenedescriptionnormalize.Key, locationnormalize.Key, locationoccurrencenormalize.Key, pipeline.DefaultNormalizeModule}) diff --git a/internal/modules/dnd/shared/assets.go b/internal/modules/dnd/shared/assets.go index fb94959..a7e22f8 100644 --- a/internal/modules/dnd/shared/assets.go +++ b/internal/modules/dnd/shared/assets.go @@ -22,15 +22,14 @@ type PromptAssetManifest struct { } var sharedPromptPaths = map[string]string{ - "common-dnd-system.md": "prompts/common-dnd-system.md", - "common-dnd-extraction-evidence.md": "prompts/common-dnd-extraction-evidence.md", - "common-dnd-identity.md": "prompts/common-dnd-identity.md", - "common-dnd-transcript-full.md": "prompts/common-dnd-transcript-full.md", - "common-dnd-transcript-chunk.md": "prompts/common-dnd-transcript-chunk.md", - "common-dnd-transcript-windows.md": "prompts/common-dnd-transcript-windows.md", - "common-dnd-references.md": "prompts/common-dnd-references.md", - "common-dnd-npc-registry.md": "prompts/common-dnd-npc-registry.md", - "common-dnd-entity-reconciliation.md": "prompts/common-dnd-entity-reconciliation.md", + "common-dnd-system.md": "prompts/common-dnd-system.md", + "common-dnd-extraction-evidence.md": "prompts/common-dnd-extraction-evidence.md", + "common-dnd-identity.md": "prompts/common-dnd-identity.md", + "common-dnd-transcript-full.md": "prompts/common-dnd-transcript-full.md", + "common-dnd-transcript-chunk.md": "prompts/common-dnd-transcript-chunk.md", + "common-dnd-transcript-windows.md": "prompts/common-dnd-transcript-windows.md", + "common-dnd-references.md": "prompts/common-dnd-references.md", + "common-dnd-npc-registry.md": "prompts/common-dnd-npc-registry.md", } func sharedAssetFS() (fs.FS, error) { diff --git a/internal/modules/dnd/shared/entityreconcile/context.go b/internal/modules/dnd/shared/entityreconcile/context.go deleted file mode 100644 index 34e2ca6..0000000 --- a/internal/modules/dnd/shared/entityreconcile/context.go +++ /dev/null @@ -1,292 +0,0 @@ -// Package entityreconcile provides safe, D&D-specific duplicate proposal -// materials shared by entity normalizers. -package entityreconcile - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "sort" - - "gitea.maximumdirect.net/eric/notarius/internal/core/source" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" -) - -const candidateKeyFormat = "candidate-%06d" - -// Candidate is one domain-neutral entity candidate supplied by a normalizer. -// BuildContext never retains or mutates its source references. -type Candidate struct { - Name string - SourceRefs []source.SourceRef -} - -// Selector identifies one candidate through its canonical name and source-free -// evidence ranges. It is the complete model-facing candidate descriptor. -type Selector struct { - Name string `json:"name"` - SourceRefs []SourceRange `json:"source_refs"` -} - -// Clone returns an owned copy of the selector. -func (s Selector) Clone() Selector { - s.SourceRefs = cloneSourceRanges(s.SourceRefs) - return s -} - -// SourceRange is a source-free evidence coordinate used in a selector. -type SourceRange struct { - StartUnitID int `json:"start_unit_id"` - EndUnitID int `json:"end_unit_id"` -} - -// Materials contains owned prompt inputs and opaque candidate-key mappings. -type Materials struct { - Candidates contracts.LLMInputMaterial - Transcript contracts.LLMInputMaterial - - candidateKeys []string - eligible map[string]struct{} - keyBySelector map[string]string - collidedSelectors map[string]struct{} -} - -// CandidateKeys returns all deterministic keys in candidate input order. -func (m Materials) CandidateKeys() []string { - return append([]string(nil), m.candidateKeys...) -} - -// EligibleCandidateKeys returns only candidates whose evidence safely produced -// transcript context, preserving candidate input order. -func (m Materials) EligibleCandidateKeys() []string { - keys := make([]string, 0, len(m.eligible)) - for _, key := range m.candidateKeys { - if _, ok := m.eligible[key]; ok { - keys = append(keys, key) - } - } - return keys -} - -type candidateInput struct { - Candidates []Selector `json:"candidates"` -} - -type transcriptInput struct { - Windows []transcriptWindow `json:"windows"` -} - -type transcriptWindow struct { - Units []transcriptUnit `json:"units"` -} - -type transcriptUnit struct { - ID int `json:"id"` - Kind string `json:"kind"` - Text string `json:"text"` - Metadata map[string]any `json:"metadata,omitempty"` - Cited bool `json:"cited"` -} - -type sourceInterval struct { - start int - end int -} - -// BuildContext constructs bounded, source-ordered prompt inputs. It returns -// ready=false when fewer than two candidates have safe context. -func BuildContext(doc *source.SourceDocument, candidates []Candidate, radius int) (Materials, bool, error) { - if radius < 0 { - return Materials{}, false, fmt.Errorf("build entity reconciliation context: radius must not be negative") - } - materials := Materials{ - candidateKeys: make([]string, len(candidates)), - eligible: make(map[string]struct{}), - keyBySelector: make(map[string]string), - collidedSelectors: make(map[string]struct{}), - } - for index := range candidates { - key := fmt.Sprintf(candidateKeyFormat, index+1) - materials.candidateKeys[index] = key - } - if doc == nil { - return materials, false, nil - } - - index := source.NewDocumentIndex(doc) - type preparedCandidate struct { - key string - selector Selector - intervals []sourceInterval - lookupKey string - } - prepared := make([]preparedCandidate, 0, len(candidates)) - selectorCounts := make(map[string]int, len(candidates)) - views := make([]Selector, 0, len(candidates)) - intervals := make([]sourceInterval, 0) - cited := make([]bool, len(doc.Units)) - for candidateIndex, candidate := range candidates { - references, candidateIntervals, valid := candidateReferences(index, candidate.SourceRefs) - if !valid { - continue - } - key := materials.candidateKeys[candidateIndex] - selector := Selector{Name: candidate.Name, SourceRefs: references} - lookupKey, err := selectorLookupKey(selector) - if err != nil { - return Materials{}, false, fmt.Errorf("build entity reconciliation context: invalid candidate material") - } - prepared = append(prepared, preparedCandidate{key: key, selector: selector, intervals: candidateIntervals, lookupKey: lookupKey}) - selectorCounts[lookupKey]++ - } - for _, candidate := range prepared { - if selectorCounts[candidate.lookupKey] != 1 { - materials.collidedSelectors[candidate.lookupKey] = struct{}{} - continue - } - materials.eligible[candidate.key] = struct{}{} - materials.keyBySelector[candidate.lookupKey] = candidate.key - views = append(views, candidate.selector.Clone()) - for _, interval := range candidate.intervals { - for position := interval.start; position <= interval.end; position++ { - cited[position] = true - } - intervals = append(intervals, sourceInterval{ - start: maxInt(0, interval.start-radius), - end: minInt(len(doc.Units)-1, interval.end+radius), - }) - } - } - if len(views) < 2 { - return materials, false, nil - } - - windows, err := contextWindows(doc.Units, coalesceIntervals(intervals), cited) - if err != nil { - return Materials{}, false, fmt.Errorf("build entity reconciliation context: invalid source metadata") - } - candidateContent, err := json.Marshal(candidateInput{Candidates: views}) - if err != nil { - return Materials{}, false, fmt.Errorf("build entity reconciliation context: invalid candidate material") - } - transcriptContent, err := json.Marshal(transcriptInput{Windows: windows}) - if err != nil { - return Materials{}, false, fmt.Errorf("build entity reconciliation context: invalid transcript material") - } - materials.Candidates = newInputMaterial("candidates", candidateContent) - materials.Transcript = newInputMaterial("transcript", transcriptContent) - return materials, true, nil -} - -func candidateReferences(index source.DocumentIndex, refs []source.SourceRef) ([]SourceRange, []sourceInterval, bool) { - if len(refs) == 0 { - return nil, nil, false - } - type referencedInterval struct { - reference SourceRange - interval sourceInterval - } - prepared := make([]referencedInterval, 0, len(refs)) - for _, ref := range refs { - if err := index.ValidateRef(ref); err != nil { - return nil, nil, false - } - start, _ := index.Position(ref.StartUnitID) - end, _ := index.Position(ref.EndUnitID) - prepared = append(prepared, referencedInterval{reference: SourceRange{StartUnitID: ref.StartUnitID, EndUnitID: ref.EndUnitID}, interval: sourceInterval{start: start, end: end}}) - } - sort.Slice(prepared, func(left, right int) bool { - if prepared[left].interval.start != prepared[right].interval.start { - return prepared[left].interval.start < prepared[right].interval.start - } - return prepared[left].interval.end < prepared[right].interval.end - }) - references := make([]SourceRange, 0, len(prepared)) - intervals := make([]sourceInterval, 0, len(prepared)) - for _, item := range prepared { - if len(references) > 0 && references[len(references)-1] == item.reference { - continue - } - references = append(references, item.reference) - intervals = append(intervals, item.interval) - } - return references, intervals, true -} - -func selectorLookupKey(selector Selector) (string, error) { - content, err := json.Marshal(selector.Clone()) - if err != nil { - return "", err - } - return string(content), nil -} - -func cloneSourceRanges(values []SourceRange) []SourceRange { - if len(values) == 0 { - return []SourceRange{} - } - return append([]SourceRange(nil), values...) -} - -func coalesceIntervals(intervals []sourceInterval) []sourceInterval { - if len(intervals) == 0 { - return nil - } - ordered := append([]sourceInterval(nil), intervals...) - sort.Slice(ordered, func(left, right int) bool { - if ordered[left].start != ordered[right].start { - return ordered[left].start < ordered[right].start - } - return ordered[left].end < ordered[right].end - }) - coalesced := make([]sourceInterval, 0, len(ordered)) - for _, interval := range ordered { - if len(coalesced) == 0 || interval.start > coalesced[len(coalesced)-1].end+1 { - coalesced = append(coalesced, interval) - continue - } - if interval.end > coalesced[len(coalesced)-1].end { - coalesced[len(coalesced)-1].end = interval.end - } - } - return coalesced -} - -func contextWindows(units []source.SourceUnit, intervals []sourceInterval, cited []bool) ([]transcriptWindow, error) { - windows := make([]transcriptWindow, 0, len(intervals)) - for _, interval := range intervals { - window := transcriptWindow{Units: make([]transcriptUnit, 0, interval.end-interval.start+1)} - for position := interval.start; position <= interval.end; position++ { - unit := units[position] - metadata, err := source.CloneMetadata(unit.Metadata) - if err != nil { - return nil, err - } - window.Units = append(window.Units, transcriptUnit{ - ID: unit.ID, Kind: unit.Kind, Text: unit.Text, Metadata: metadata, Cited: cited[position], - }) - } - windows = append(windows, window) - } - return windows, nil -} - -func newInputMaterial(name string, content []byte) contracts.LLMInputMaterial { - digest := sha256.Sum256(content) - return contracts.NewLLMInputMaterial(name, "application/json", content, "sha256:"+hex.EncodeToString(digest[:]), "") -} - -func minInt(left, right int) int { - if left < right { - return left - } - return right -} - -func maxInt(left, right int) int { - if left > right { - return left - } - return right -} diff --git a/internal/modules/dnd/shared/entityreconcile/entityreconcile_test.go b/internal/modules/dnd/shared/entityreconcile/entityreconcile_test.go deleted file mode 100644 index 767a764..0000000 --- a/internal/modules/dnd/shared/entityreconcile/entityreconcile_test.go +++ /dev/null @@ -1,328 +0,0 @@ -package entityreconcile - -import ( - "bytes" - "encoding/json" - "io/fs" - "reflect" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/core/source" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "github.com/santhosh-tekuri/jsonschema/v6" -) - -func TestBuildContextUsesContextualSelectorsSourceOrderAndOwnedData(t *testing.T) { - doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{ - {ID: 40, Kind: "narration", Text: "zero"}, - {ID: 10, Kind: "speech", Text: "one", Metadata: map[string]any{"speaker": map[string]any{"name": "Mira"}}}, - {ID: 70, Kind: "speech", Text: "two"}, - {ID: 20, Kind: "narration", Text: "three"}, - {ID: 90, Kind: "speech", Text: "four"}, - }} - candidates := []Candidate{ - {Name: "The Tavern", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 20}}}, - {Name: "The Tavern", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 90, EndUnitID: 90}}}, - {Name: "Broken", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 10}}}, - } - before := cloneCandidates(candidates) - - materials, ready, err := BuildContext(doc, candidates, 1) - if err != nil || !ready { - t.Fatalf("BuildContext() = %#v, %t, %v; want ready materials", materials, ready, err) - } - if !reflect.DeepEqual(candidates, before) { - t.Fatalf("BuildContext() mutated candidates: %#v", candidates) - } - if got, want := materials.CandidateKeys(), []string{"candidate-000001", "candidate-000002", "candidate-000003"}; !reflect.DeepEqual(got, want) { - t.Fatalf("CandidateKeys() = %#v, want %#v", got, want) - } - if got, want := materials.EligibleCandidateKeys(), []string{"candidate-000001", "candidate-000002"}; !reflect.DeepEqual(got, want) { - t.Fatalf("EligibleCandidateKeys() = %#v, want %#v", got, want) - } - if !json.Valid(materials.Candidates.Content) || !json.Valid(materials.Transcript.Content) { - t.Fatalf("prompt materials are not JSON: %#v", materials) - } - if strings.Contains(string(materials.Candidates.Content), doc.ID) { - t.Fatalf("candidate material leaked source identity: %s", materials.Candidates.Content) - } - - var candidatePayload candidateInput - if err := json.Unmarshal(materials.Candidates.Content, &candidatePayload); err != nil { - t.Fatal(err) - } - if len(candidatePayload.Candidates) != 2 || candidatePayload.Candidates[0].Name != candidatePayload.Candidates[1].Name || strings.Contains(string(materials.Candidates.Content), "candidate-") { - t.Fatalf("candidate payload = %#v, want contextual descriptors without keys", candidatePayload) - } - if got := candidatePayload.Candidates[0].SourceRefs[0]; got != (SourceRange{StartUnitID: 10, EndUnitID: 20}) { - t.Fatalf("candidate reference = %#v", got) - } - - var transcript transcriptInput - if err := json.Unmarshal(materials.Transcript.Content, &transcript); err != nil { - t.Fatal(err) - } - if len(transcript.Windows) != 1 || len(transcript.Windows[0].Units) != 5 { - t.Fatalf("windows = %#v, want one bounded coalesced window", transcript.Windows) - } - units := transcript.Windows[0].Units - for index, wantID := range []int{40, 10, 70, 20, 90} { - if units[index].ID != wantID { - t.Fatalf("window unit %d = %d, want source-order %d", index, units[index].ID, wantID) - } - } - if units[0].Cited || !units[1].Cited || !units[2].Cited || !units[3].Cited || !units[4].Cited { - t.Fatalf("citation flags = %#v", units) - } - - windows, err := contextWindows(doc.Units, []sourceInterval{{start: 1, end: 1}}, make([]bool, len(doc.Units))) - if err != nil { - t.Fatal(err) - } - windows[0].Units[0].Metadata["speaker"].(map[string]any)["name"] = "changed" - if doc.Units[1].Metadata["speaker"].(map[string]any)["name"] != "Mira" { - t.Fatal("context metadata aliases source document") - } -} - -func TestBuildContextExcludesCollidingDescriptors(t *testing.T) { - doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1}, {ID: 2}, {ID: 3}}} - candidates := []Candidate{ - {Name: "The Tavern", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}, - {Name: "The Tavern", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}, - {Name: "The Market", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 3, EndUnitID: 3}}}, - } - materials, ready, err := BuildContext(doc, candidates, 0) - if err != nil || ready || len(materials.EligibleCandidateKeys()) != 1 || strings.Contains(string(materials.Candidates.Content), "The Tavern") { - t.Fatalf("BuildContext() = %#v, %t, %v", materials, ready, err) - } - assessment := materials.Assess(ProposalResponse{DuplicateGroups: []DuplicateGroup{{ - Members: []Selector{{Name: "The Tavern", SourceRefs: []SourceRange{{StartUnitID: 1, EndUnitID: 1}}}, {Name: "The Market", SourceRefs: []SourceRange{{StartUnitID: 3, EndUnitID: 3}}}}, - Canonical: Selector{Name: "The Market", SourceRefs: []SourceRange{{StartUnitID: 3, EndUnitID: 3}}}, - }}}) - if !hasIssue(assessment.Issues(), "member_ineligible") { - t.Fatalf("Assess() issues = %#v, want collided descriptor rejection", assessment.Issues()) - } -} - -func TestAssessmentRejectsPartialAndReorderedDescriptors(t *testing.T) { - doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 10}, {ID: 20}, {ID: 30}}} - materials, ready, err := BuildContext(doc, []Candidate{ - {Name: "The Tavern", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}, {SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, - {Name: "The Market", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}}, - }, 0) - if err != nil || !ready { - t.Fatalf("BuildContext() = %#v, %t, %v", materials, ready, err) - } - selectors := materialSelectors(t, materials) - for _, refs := range [][]SourceRange{ - {{StartUnitID: 10, EndUnitID: 10}}, - {{StartUnitID: 20, EndUnitID: 20}, {StartUnitID: 10, EndUnitID: 10}}, - } { - assessment := materials.Assess(ProposalResponse{DuplicateGroups: []DuplicateGroup{{ - Members: []Selector{{Name: "The Tavern", SourceRefs: refs}, selectors[1]}, - Canonical: selectors[1], - }}}) - if !hasIssue(assessment.Issues(), "member_unknown") { - t.Fatalf("Assess(%#v) issues = %#v, want descriptor mismatch rejection", refs, assessment.Issues()) - } - } -} - -func TestBuildContextExcludesUnsafeReferencesAndCoalescesAdjacentWindows(t *testing.T) { - doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 9}, {ID: 3}, {ID: 8}, {ID: 1}, {ID: 7}}} - candidates := []Candidate{ - {Name: "One", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 3, EndUnitID: 3}}}, - {Name: "Two", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 8, EndUnitID: 8}}}, - {Name: "Missing", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 99, EndUnitID: 99}}}, - {Name: "Foreign", SourceRefs: []source.SourceRef{{SourceID: "other", StartUnitID: 1, EndUnitID: 1}}}, - {Name: "Blank"}, - } - materials, ready, err := BuildContext(doc, candidates, 0) - if err != nil || !ready { - t.Fatalf("BuildContext() error = %v, ready = %t", err, ready) - } - if got, want := materials.EligibleCandidateKeys(), []string{"candidate-000001", "candidate-000002"}; !reflect.DeepEqual(got, want) { - t.Fatalf("EligibleCandidateKeys() = %#v, want %#v", got, want) - } - var transcript transcriptInput - if err := json.Unmarshal(materials.Transcript.Content, &transcript); err != nil { - t.Fatal(err) - } - if len(transcript.Windows) != 1 || len(transcript.Windows[0].Units) != 2 || transcript.Windows[0].Units[0].ID != 3 || transcript.Windows[0].Units[1].ID != 8 { - t.Fatalf("windows = %#v, want adjacent document-order units coalesced", transcript.Windows) - } - if _, ready, err := BuildContext(doc, candidates, -1); err == nil || ready { - t.Fatalf("BuildContext(radius=-1) = ready %t, err %v", ready, err) - } -} - -func TestAssessmentRejectsEveryUnsafeProposalCategory(t *testing.T) { - materials := preparedMaterials(t, 4, true) - selectors := materialSelectors(t, materials) - unsafe := []struct { - name string - response ProposalResponse - category string - }{ - {"blank member", ProposalResponse{DuplicateGroups: []DuplicateGroup{{Members: []Selector{{}, selectors[1]}, Canonical: selectors[1]}}}, "member_blank"}, - {"unknown member", ProposalResponse{DuplicateGroups: []DuplicateGroup{{Members: []Selector{{Name: "unknown", SourceRefs: []SourceRange{{StartUnitID: 99, EndUnitID: 99}}}, selectors[1]}, Canonical: selectors[1]}}}, "member_unknown"}, - {"repeated member", ProposalResponse{DuplicateGroups: []DuplicateGroup{{Members: []Selector{selectors[0], selectors[0]}, Canonical: selectors[0]}}}, "repeated_member"}, - {"too small", ProposalResponse{DuplicateGroups: []DuplicateGroup{{Members: []Selector{selectors[0]}, Canonical: selectors[0]}}}, "fewer_than_two_members"}, - {"canonical blank", ProposalResponse{DuplicateGroups: []DuplicateGroup{{Members: []Selector{selectors[0], selectors[1]}, Canonical: Selector{}}}}, "canonical_blank"}, - {"canonical not member", ProposalResponse{DuplicateGroups: []DuplicateGroup{{Members: []Selector{selectors[0], selectors[1]}, Canonical: selectors[2]}}}, "canonical_not_member"}, - {"overlapping", ProposalResponse{DuplicateGroups: []DuplicateGroup{ - {Members: []Selector{selectors[0], selectors[1]}, Canonical: selectors[0]}, - {Members: []Selector{selectors[1], selectors[2]}, Canonical: selectors[2]}, - }}, "overlapping_member"}, - } - for _, test := range unsafe { - t.Run(test.name, func(t *testing.T) { - assessment := materials.Assess(test.response) - if len(assessment.SafeGroups()) != 0 || assessment.DiscardedGroups() != len(test.response.DuplicateGroups) || !hasIssue(assessment.Issues(), test.category) { - t.Fatalf("Assess() = groups %#v discarded %d issues %#v; want %q rejection", assessment.SafeGroups(), assessment.DiscardedGroups(), assessment.Issues(), test.category) - } - }) - } -} - -func TestAssessmentReturnsNonOverlappingSafeGroupsAndDefensiveCopies(t *testing.T) { - materials := preparedMaterials(t, 4, false) - keys := materials.CandidateKeys() - selectors := materialSelectors(t, materials) - assessment := materials.Assess(ProposalResponse{DuplicateGroups: []DuplicateGroup{ - {Members: []Selector{selectors[1], selectors[0]}, Canonical: selectors[1]}, - {Members: []Selector{selectors[3], selectors[2]}, Canonical: selectors[2]}, - }}) - groups := assessment.SafeGroups() - if assessment.DiscardedGroups() != 0 || len(assessment.Issues()) != 0 || len(groups) != 2 { - t.Fatalf("assessment = %#v, %d, %#v", groups, assessment.DiscardedGroups(), assessment.Issues()) - } - if got, want := groups[0].Members(), []string{keys[0], keys[1]}; !reflect.DeepEqual(got, want) || groups[0].Canonical() != keys[1] { - t.Fatalf("first safe group = %#v / %q", got, groups[0].Canonical()) - } - keys[0] = "changed" - if materials.CandidateKeys()[0] == "changed" { - t.Fatal("CandidateKeys() exposed retained keys") - } - members := groups[0].Members() - members[0] = "changed" - if groups[0].Members()[0] == "changed" || assessment.SafeGroups()[0].Members()[0] == "changed" { - t.Fatal("SafeGroups() exposed retained members") - } -} - -func TestSharedResponseSchemaIsPrivateStrictAndRegisterableOnce(t *testing.T) { - schema, err := LoadResponseSchema() - if err != nil { - t.Fatal(err) - } - if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) { - t.Fatalf("schema = %#v", schema) - } - for _, test := range []struct { - name string - value any - valid bool - }{ - {"empty groups", map[string]any{"duplicate_groups": []any{}}, true}, - {"semantic proposal problem", map[string]any{"duplicate_groups": []any{map[string]any{"members": []any{map[string]any{"name": "Mira", "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1}}}}, "canonical": map[string]any{"name": "Mira", "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1}}}}}}, true}, - {"missing groups", map[string]any{}, false}, - {"unknown top level", map[string]any{"duplicate_groups": []any{}, "extra": true}, false}, - {"replacement name", map[string]any{"duplicate_groups": []any{map[string]any{"members": []any{}, "canonical": map[string]any{"name": "Mira", "source_refs": []any{}}, "name": "replacement"}}}, false}, - {"missing selector evidence", map[string]any{"duplicate_groups": []any{map[string]any{"members": []any{}, "canonical": map[string]any{"name": "Mira"}}}}, false}, - {"invalid range", map[string]any{"duplicate_groups": []any{map[string]any{"members": []any{map[string]any{"name": "Mira", "source_refs": []any{map[string]any{"start_unit_id": 0, "end_unit_id": 1}}}}, "canonical": map[string]any{"name": "Mira", "source_refs": []any{}}}}}, false}, - } { - t.Run(test.name, func(t *testing.T) { - content, err := json.Marshal(test.value) - if err != nil { - t.Fatal(err) - } - err = validateSchema(content, schema.JSONSchema) - if (err == nil) != test.valid { - t.Fatalf("validateSchema() error = %v, want valid=%t", err, test.valid) - } - }) - } - first := schema.JSONSchema - first[0] = '[' - second, err := LoadResponseSchema() - if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first, second.JSONSchema) { - t.Fatalf("LoadResponseSchema() returned shared content: %s, %v", second.JSONSchema, err) - } - registry := llm.NewAssetRegistry() - if err := RegisterSchemaAssets(registry); err != nil { - t.Fatalf("RegisterSchemaAssets() error = %v", err) - } - if schemaFS, err := registry.SchemaFS(); err != nil { - t.Fatalf("SchemaFS() error = %v", err) - } else if content, err := fs.ReadFile(schemaFS, "dnd_entity_reconcile_llm.v1.json"); err != nil || !json.Valid(content) { - t.Fatalf("shared schema asset = %s, %v", content, err) - } -} - -func preparedMaterials(t *testing.T, count int, includeIneligible bool) Materials { - t.Helper() - doc := &source.SourceDocument{ID: "session", Units: make([]source.SourceUnit, count)} - candidates := make([]Candidate, count) - for index := range candidates { - doc.Units[index] = source.SourceUnit{ID: index + 1, Text: "unit"} - candidates[index] = Candidate{Name: "same display name", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: index + 1, EndUnitID: index + 1}}} - } - if includeIneligible && count > 3 { - candidates[3].SourceRefs = []source.SourceRef{{SourceID: "other", StartUnitID: 1, EndUnitID: 1}} - } - materials, ready, err := BuildContext(doc, candidates, 0) - if err != nil || !ready { - t.Fatalf("BuildContext() = %#v, %t, %v", materials, ready, err) - } - return materials -} - -func materialSelectors(t *testing.T, materials Materials) []Selector { - t.Helper() - var input candidateInput - if err := json.Unmarshal(materials.Candidates.Content, &input); err != nil { - t.Fatal(err) - } - return input.Candidates -} - -func cloneCandidates(input []Candidate) []Candidate { - output := make([]Candidate, len(input)) - copy(output, input) - for index := range output { - output[index].SourceRefs = append([]source.SourceRef(nil), input[index].SourceRefs...) - } - return output -} - -func hasIssue(issues []Issue, want string) bool { - for _, issue := range issues { - if issue.Category == want { - return true - } - } - return false -} - -func validateSchema(instanceContent, schemaContent []byte) error { - instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent)) - if err != nil { - return err - } - document, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent)) - if err != nil { - return err - } - compiler := jsonschema.NewCompiler() - if err := compiler.AddResource("schema.json", document); err != nil { - return err - } - compiled, err := compiler.Compile("schema.json") - if err != nil { - return err - } - return compiled.Validate(instance) -} diff --git a/internal/modules/dnd/shared/entityreconcile/proposal.go b/internal/modules/dnd/shared/entityreconcile/proposal.go deleted file mode 100644 index 829e7fe..0000000 --- a/internal/modules/dnd/shared/entityreconcile/proposal.go +++ /dev/null @@ -1,175 +0,0 @@ -package entityreconcile - -import ( - "sort" - "strings" -) - -// ProposalResponse is the private structured response exchanged with the -// reconciliation prompt. It identifies candidates by contextual selectors. -type ProposalResponse struct { - DuplicateGroups []DuplicateGroup `json:"duplicate_groups"` -} - -// DuplicateGroup proposes contextual candidate descriptors that might denote -// one entity. -type DuplicateGroup struct { - Members []Selector `json:"members"` - Canonical Selector `json:"canonical"` -} - -// Issue identifies one unsafe proposal category without prescribing a warning -// message or retry policy to a consuming normalizer. -type Issue struct { - GroupIndex int - Category string -} - -// SafeGroup identifies one validated, non-overlapping duplicate group. -type SafeGroup struct { - members []string - canonical string -} - -// Members returns an owned copy of the group's candidate keys. -func (g SafeGroup) Members() []string { return append([]string(nil), g.members...) } - -// Canonical returns the selected canonical candidate key. -func (g SafeGroup) Canonical() string { return g.canonical } - -// Assessment contains only safe groups. Its accessors return owned copies so -// callers cannot mutate retained assessment data. -type Assessment struct { - safeGroups []SafeGroup - discardedGroups int - issues []Issue -} - -// SafeGroups returns validated non-overlapping groups in proposal order. -func (a Assessment) SafeGroups() []SafeGroup { - groups := make([]SafeGroup, len(a.safeGroups)) - for index, group := range a.safeGroups { - groups[index] = SafeGroup{members: append([]string(nil), group.members...), canonical: group.canonical} - } - return groups -} - -// DiscardedGroups returns the number of rejected proposal groups. -func (a Assessment) DiscardedGroups() int { return a.discardedGroups } - -// Issues returns the deterministic rejection categories in proposal order. -func (a Assessment) Issues() []Issue { return append([]Issue(nil), a.issues...) } - -// Assess resolves contextual descriptors to internal candidate keys, then -// validates the proposal without exposing those keys to the model. -func (m Materials) Assess(response ProposalResponse) Assessment { - groups := make([]assessedGroup, len(response.DuplicateGroups)) - issues := make([]Issue, 0) - for groupIndex, proposal := range response.DuplicateGroups { - groups[groupIndex] = m.assessGroup(proposal) - for _, category := range groups[groupIndex].issues { - issues = append(issues, Issue{GroupIndex: groupIndex, Category: category}) - } - } - - owners := make(map[string][]int) - for groupIndex, group := range groups { - if !group.locallyValid { - continue - } - for _, key := range group.members { - owners[key] = append(owners[key], groupIndex) - } - } - for groupIndex := range groups { - if !groups[groupIndex].locallyValid { - continue - } - for _, key := range groups[groupIndex].members { - if len(owners[key]) > 1 { - groups[groupIndex].conflicting = true - issues = append(issues, Issue{GroupIndex: groupIndex, Category: "overlapping_member"}) - break - } - } - } - - assessment := Assessment{issues: issues} - for _, group := range groups { - if !group.locallyValid || group.conflicting { - assessment.discardedGroups++ - continue - } - assessment.safeGroups = append(assessment.safeGroups, SafeGroup{members: append([]string(nil), group.members...), canonical: group.canonical}) - } - return assessment -} - -type assessedGroup struct { - members []string - canonical string - issues []string - locallyValid bool - conflicting bool -} - -func (m Materials) assessGroup(proposal DuplicateGroup) assessedGroup { - issues := make([]string, 0) - members := make([]string, 0, len(proposal.Members)) - seen := make(map[string]struct{}, len(proposal.Members)) - for _, selector := range proposal.Members { - key, category := m.selectorKey(selector) - if category != "" { - issues = append(issues, "member_"+category) - continue - } - if _, exists := seen[key]; exists { - issues = append(issues, "repeated_member") - continue - } - seen[key] = struct{}{} - members = append(members, key) - } - canonical, canonicalCategory := m.selectorKey(proposal.Canonical) - if canonicalCategory != "" { - issues = append(issues, "canonical_"+canonicalCategory) - } - if len(members) < 2 { - issues = append(issues, "fewer_than_two_members") - } - if canonicalCategory == "" && !contains(members, canonical) { - issues = append(issues, "canonical_not_member") - } - sort.Strings(members) - return assessedGroup{members: members, canonical: canonical, issues: issues, locallyValid: len(issues) == 0} -} - -func (m Materials) selectorKey(selector Selector) (string, string) { - if strings.TrimSpace(selector.Name) == "" { - return "", "blank" - } - lookupKey, err := selectorLookupKey(selector) - if err != nil { - return "", "unknown" - } - if _, collided := m.collidedSelectors[lookupKey]; collided { - return "", "ineligible" - } - key, ok := m.keyBySelector[lookupKey] - if !ok { - return "", "unknown" - } - if _, eligible := m.eligible[key]; !eligible { - return "", "ineligible" - } - return key, "" -} - -func contains(values []string, want string) bool { - for _, value := range values { - if value == want { - return true - } - } - return false -} diff --git a/internal/modules/dnd/shared/entityreconcile/schema.go b/internal/modules/dnd/shared/entityreconcile/schema.go deleted file mode 100644 index 0039874..0000000 --- a/internal/modules/dnd/shared/entityreconcile/schema.go +++ /dev/null @@ -1,55 +0,0 @@ -package entityreconcile - -import ( - "fmt" - "io/fs" - - rootassets "gitea.maximumdirect.net/eric/notarius/assets" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" -) - -const ( - ResponseSchemaKey = llm.ResponseSchemaKey("dnd_entity_reconcile_llm") - ResponseSchemaID = "notarius.dnd.entity_reconcile.llm" - ResponseSchemaName = "notarius_dnd_entity_reconcile_llm_v1" - SchemaVersion = "v1" - SchemaAssetPath = "schemas/dnd_entity_reconcile_llm.v1.json" -) - -func schemaAssetFS() (fs.FS, error) { - assets, err := fs.Sub(rootassets.FS(), "dnd/entity-reconciliation") - if err != nil { - return nil, fmt.Errorf("scope entity reconciliation assets: %w", err) - } - return assets, nil -} - -// LoadResponseSchema returns the shared private duplicate-group response -// contract. It is intentionally separate from durable artifact schemas. -func LoadResponseSchema() (llm.ResponseSchema, error) { - assets, err := schemaAssetFS() - if err != nil { - return llm.ResponseSchema{}, err - } - return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{ - Key: ResponseSchemaKey, - ID: ResponseSchemaID, - Version: SchemaVersion, - Name: ResponseSchemaName, - AssetPath: SchemaAssetPath, - }) -} - -// RegisterSchemaAssets makes the shared private response schema available to -// prompt preparation. A family registrar can register it once for all consumers. -func RegisterSchemaAssets(registry *llm.AssetRegistry) error { - assets, err := schemaAssetFS() - if err != nil { - return err - } - schemas, err := fs.Sub(assets, "schemas") - if err != nil { - return fmt.Errorf("scope entity reconciliation schemas: %w", err) - } - return registry.RegisterSchemaFS(schemas, ".") -}