From d1c48db4bc59f58048d5b3703f91f5f8d6b89140 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 26 Jul 2026 01:36:32 +0000 Subject: [PATCH] Add LLM-assisted NPC normalization --- .../dnd/normalize/npcs/context_material.go | 14 +- .../modules/dnd/normalize/npcs/normalizer.go | 236 ++++++++++++++-- .../dnd/normalize/npcs/normalizer_test.go | 58 +++- .../modules/dnd/normalize/npcs/proposal.go | 195 +++++++++++++ .../npcs/semantic_normalizer_test.go | 265 ++++++++++++++++++ 5 files changed, 726 insertions(+), 42 deletions(-) create mode 100644 internal/modules/dnd/normalize/npcs/proposal.go create mode 100644 internal/modules/dnd/normalize/npcs/semantic_normalizer_test.go diff --git a/internal/modules/dnd/normalize/npcs/context_material.go b/internal/modules/dnd/normalize/npcs/context_material.go index 6320d31..2e5e4c1 100644 --- a/internal/modules/dnd/normalize/npcs/context_material.go +++ b/internal/modules/dnd/normalize/npcs/context_material.go @@ -16,8 +16,9 @@ import ( const semanticContextRadius = 2 type normalizeContextMaterials struct { - Candidates contracts.LLMInputMaterial - Transcript contracts.LLMInputMaterial + Candidates contracts.LLMInputMaterial + Transcript contracts.LLMInputMaterial + candidatePositions []int } type normalizeCandidateInput struct { @@ -81,11 +82,12 @@ func buildNormalizeContextMaterials(doc *source.SourceDocument, records []dnd.NP index := source.NewDocumentIndex(doc) candidates := make([]normalizeCandidate, 0, len(records)) + candidatePositions := make([]int, 0, len(records)) intervals := make([]sourceInterval, 0) cited := make([]bool, len(doc.Units)) seenKeys := make(map[string]struct{}, len(records)) - for _, record := range records { + for position, record := range records { key := identity.ComparisonKey(record.Name) if key == "" || len(record.SourceRefs) == 0 { continue @@ -100,6 +102,7 @@ func buildNormalizeContextMaterials(doc *source.SourceDocument, records []dnd.NP } seenKeys[key] = struct{}{} candidates = append(candidates, normalizeCandidate{Name: record.Name, SourceRefs: references}) + candidatePositions = append(candidatePositions, position) for _, interval := range recordIntervals { for position := interval.start; position <= interval.end; position++ { cited[position] = true @@ -127,8 +130,9 @@ func buildNormalizeContextMaterials(doc *source.SourceDocument, records []dnd.NP return normalizeContextMaterials{}, false, fmt.Errorf("build NPC normalization context: encode transcript: %w", err) } return normalizeContextMaterials{ - Candidates: newNormalizeInputMaterial("candidates", candidateContent), - Transcript: newNormalizeInputMaterial("transcript", transcriptContent), + Candidates: newNormalizeInputMaterial("candidates", candidateContent), + Transcript: newNormalizeInputMaterial("transcript", transcriptContent), + candidatePositions: candidatePositions, }, true, nil } diff --git a/internal/modules/dnd/normalize/npcs/normalizer.go b/internal/modules/dnd/normalize/npcs/normalizer.go index 89d3c73..d3ef3fe 100644 --- a/internal/modules/dnd/normalize/npcs/normalizer.go +++ b/internal/modules/dnd/normalize/npcs/normalizer.go @@ -3,8 +3,10 @@ package npcs import ( "context" + "errors" "fmt" "reflect" + "sort" "strconv" "strings" @@ -18,14 +20,18 @@ import ( ) const ( - Key = "dnd/npcs" - normalizationPolicy = "dnd.npcs.normalize.v2" - NormalizationPolicy = normalizationPolicy + Key = "dnd/npcs" + normalizationPolicy = "dnd.npcs.normalize.v3" + semanticContextPolicy = "dnd.npcs.semantic_context.v1" + NormalizationPolicy = normalizationPolicy - ReasonCodeNPCFieldsNormalized = "npc_fields_normalized" - ReasonCodeNPCIDRecomputed = "npc_id_recomputed" - ReasonCodeSourceReferencesNormalized = "source_references_normalized" - ReasonCodeDuplicateNPCCollapsed = "duplicate_npc_collapsed" + ReasonCodeNPCFieldsNormalized = "npc_fields_normalized" + ReasonCodeNPCIDRecomputed = "npc_id_recomputed" + ReasonCodeSourceReferencesNormalized = "source_references_normalized" + ReasonCodeDuplicateNPCCollapsed = "duplicate_npc_collapsed" + ReasonCodeNPCSemanticProposalInvalid = "npc_semantic_proposal_invalid" + ReasonCodeNPCSemanticReconciliationExhausted = "npc_semantic_reconciliation_exhausted" + ReasonCodeNPCNormalizationWarningsOmitted = "npc_normalization_warnings_omitted" ) var requiredCapabilities = []string{"merged"} @@ -36,9 +42,28 @@ var _ contracts.ManifestMetadataProvider = (*Normalizer)(nil) var _ pipeline.CheckpointFingerprintProvider = (*Normalizer)(nil) type Options struct{} -type Normalizer struct{} -func New(Options) *Normalizer { return &Normalizer{} } +type Normalizer struct { + llm contracts.StructuredLLMClient + promptSHA string + responseSchemaSHA string +} + +func New(llmClient contracts.StructuredLLMClient, _ Options) (*Normalizer, error) { + if llmClient == nil { + return nil, normalizerErrorf("LLM client must not be nil") + } + promptSHA, err := scriptoriumPromptMetadata() + if err != nil { + return nil, normalizerErrorf("load prompt metadata: %w", err) + } + responseSchema, err := loadResponseSchema() + if err != nil { + return nil, normalizerErrorf("load response schema: %w", err) + } + return &Normalizer{llm: llmClient, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil +} + func (n *Normalizer) Key() string { return Key } func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil } @@ -46,7 +71,20 @@ func (n *Normalizer) ManifestMetadata() map[string]any { if n == nil { return nil } - return map[string]any{"identity_policy": identity.Policy, "normalization_policy": normalizationPolicy} + return map[string]any{ + "prompt_id": PromptID, + "prompt_version": SchemaVersion, + "prompt_sha256": n.promptSHA, + "response_schema_key": string(ResponseSchemaKey), + "response_schema_id": ResponseSchemaID, + "response_schema_name": ResponseSchemaName, + "response_schema_version": SchemaVersion, + "response_schema_sha256": n.responseSchemaSHA, + "identity_policy": identity.Policy, + "normalization_policy": normalizationPolicy, + "semantic_context_policy": semanticContextPolicy, + "semantic_context_radius": semanticContextRadius, + } } func (n *Normalizer) CheckpointFingerprints() []pipeline.CheckpointFingerprint { @@ -54,8 +92,11 @@ func (n *Normalizer) CheckpointFingerprints() []pipeline.CheckpointFingerprint { return nil } return []pipeline.CheckpointFingerprint{ + {Name: "prompt", Value: n.promptSHA}, + {Name: "response_schema", Value: n.responseSchemaSHA}, {Name: "identity_policy", Value: identity.Policy}, {Name: "normalization_policy", Value: normalizationPolicy}, + {Name: "semantic_context_policy", Value: fmt.Sprintf("%s:%d", semanticContextPolicy, semanticContextRadius)}, } } @@ -63,30 +104,114 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize if n == nil { return contracts.TypedNormalizeResult[dnd.NPCList]{}, normalizerErrorf("normalizer must not be nil") } + if n.llm == nil { + return contracts.TypedNormalizeResult[dnd.NPCList]{}, normalizerErrorf("LLM client must not be nil") + } if ctx == nil { return contracts.TypedNormalizeResult[dnd.NPCList]{}, normalizerErrorf("context must not be nil") } if err := ctx.Err(); err != nil { return contracts.TypedNormalizeResult[dnd.NPCList]{}, normalizerErrorf("context error before normalize: %w", err) } + order := shared.NewSourceRefOrder(req.Source) - value, warnings := normalizeList(req.MergeOutput.Value, order) - return contracts.TypedNormalizeResult[dnd.NPCList]{Value: value, Warnings: warnings}, nil + records, warnings := preprocessRecords(req.MergeOutput.Value, order) + deterministic := recordList(records) + materials, ready, err := buildDefaultNormalizeContextMaterials(req.Source, recordValues(records)) + if err != nil { + return contracts.TypedNormalizeResult[dnd.NPCList]{}, normalizerErrorf("build semantic context: %w", err) + } + if !ready { + return contracts.TypedNormalizeResult[dnd.NPCList]{Value: deterministic, Warnings: limitWarnings(warnings)}, nil + } + + var response normalizeProposalResponse + if _, err := n.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ + StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion, + ProfileID: req.LLMProfile, SessionID: req.SessionID, + Inputs: contracts.LLMInputSet{"candidates": materials.Candidates, "transcript": materials.Transcript}, + }, &response); err != nil { + if errors.Is(err, contracts.ErrInvalidStructuredOutput) { + return n.invalidStructuredResult(deterministic, warnings), nil + } + return contracts.TypedNormalizeResult[dnd.NPCList]{}, normalizerErrorf("complete structured output: %w", err) + } + + assessment := assessProposal(response, records, materials.candidatePositions) + applied, semanticWarnings := applySafeGroups(records, assessment.safeGroups, order) + warnings = append(warnings, semanticWarnings...) + if assessment.discardedGroups == 0 { + return contracts.TypedNormalizeResult[dnd.NPCList]{Value: recordList(applied), Warnings: limitWarnings(warnings)}, nil + } + return retryResult(recordList(applied), warnings, assessment), nil +} + +func (n *Normalizer) invalidStructuredResult(value dnd.NPCList, warnings []contracts.Warning) contracts.TypedNormalizeResult[dnd.NPCList] { + return contracts.TypedNormalizeResult[dnd.NPCList]{ + Value: value, + Warnings: limitWarningsForRetry(warnings), + Retry: &contracts.NormalizeRetry{ + ReasonCode: ReasonCodeNPCSemanticProposalInvalid, + Message: "semantic proposal requires retry: invalid structured output", + FallbackWarnings: []contracts.Warning{semanticFallbackWarning(-1)}, + }, + } +} + +func retryResult(value dnd.NPCList, warnings []contracts.Warning, assessment proposalAssessment) contracts.TypedNormalizeResult[dnd.NPCList] { + return contracts.TypedNormalizeResult[dnd.NPCList]{ + Value: value, + Warnings: limitWarningsForRetry(warnings), + Retry: &contracts.NormalizeRetry{ + ReasonCode: ReasonCodeNPCSemanticProposalInvalid, + Message: diagnostics.Aggregate("semantic proposal requires retry", assessment.issues), + FallbackWarnings: []contracts.Warning{semanticFallbackWarning(assessment.discardedGroups)}, + }, + } +} + +func semanticFallbackWarning(discardedGroups int) contracts.Warning { + message := "semantic proposal could not be applied" + if discardedGroups >= 0 { + message = fmt.Sprintf("%d proposal group(s) omitted after semantic proposal retry exhaustion", discardedGroups) + } + return contracts.Warning{Scope: "npcs", ReasonCode: ReasonCodeNPCSemanticReconciliationExhausted, Message: message} +} + +func limitWarnings(warnings []contracts.Warning) []contracts.Warning { + return diagnostics.LimitWarnings(warnings, "npcs", ReasonCodeNPCNormalizationWarningsOmitted) +} + +func limitWarningsForRetry(warnings []contracts.Warning) []contracts.Warning { + if warnings == nil { + return nil + } + if len(warnings) < diagnostics.MaxWarnings { + return append([]contracts.Warning(nil), warnings...) + } + displayed := diagnostics.MaxWarnings - 2 + bounded := append([]contracts.Warning(nil), warnings[:displayed]...) + return append(bounded, contracts.Warning{ + Scope: "npcs", ReasonCode: ReasonCodeNPCNormalizationWarningsOmitted, + Message: fmt.Sprintf("%d additional warning(s) omitted", len(warnings)-displayed), + }) } type normalizedRecord struct { - npc dnd.NPC + npc dnd.NPC + inputIndexes []int + earliest int } -func normalizeList(input dnd.NPCList, order shared.SourceRefOrder) (dnd.NPCList, []contracts.Warning) { +func preprocessRecords(input dnd.NPCList, order shared.SourceRefOrder) ([]normalizedRecord, []contracts.Warning) { if input.NPCs == nil { - return dnd.NPCList{}, nil + return nil, nil } records := make([]normalizedRecord, len(input.NPCs)) warnings := make([]contracts.Warning, 0) for index, inputNPC := range input.NPCs { npc, fieldsChanged, referencesChanged := normalizeRecord(inputNPC, order) - records[index] = normalizedRecord{npc: npc} + records[index] = normalizedRecord{npc: npc, inputIndexes: []int{index}, earliest: index} if fieldsChanged { warnings = append(warnings, contracts.Warning{Scope: npcScope(index), ReasonCode: ReasonCodeNPCFieldsNormalized, Message: fmt.Sprintf("input index %d: NPC name normalized for %s", index, diagnostics.Quote(inputNPC.Name))}) } @@ -99,16 +224,16 @@ func normalizeList(input dnd.NPCList, order shared.SourceRefOrder) (dnd.NPCList, } groups := canonicalNameGroups(records) - output := dnd.NPCList{NPCs: make([]dnd.NPC, 0, len(groups))} + output := make([]normalizedRecord, 0, len(groups)) for _, members := range groups { consolidated, referencesChanged := consolidate(records, members, order) - retainedIndex := members[0] - output.NPCs = append(output.NPCs, consolidated) + output = append(output, consolidated) + retainedIndex := consolidated.earliest if referencesChanged { - warnings = append(warnings, contracts.Warning{Scope: npcScope(retainedIndex), ReasonCode: ReasonCodeSourceReferencesNormalized, Message: fmt.Sprintf("input index %d: source references normalized during identity consolidation (final count %d)", retainedIndex, len(consolidated.SourceRefs))}) + warnings = append(warnings, contracts.Warning{Scope: npcScope(retainedIndex), ReasonCode: ReasonCodeSourceReferencesNormalized, Message: fmt.Sprintf("input index %d: source references normalized during identity consolidation (final count %d)", retainedIndex, len(consolidated.npc.SourceRefs))}) } if len(members) > 1 { - warnings = append(warnings, duplicateWarning(retainedIndex, members[1:])) + warnings = append(warnings, duplicateWarning(retainedIndex, memberInputIndexes(records, members[1:]))) } } return output, warnings @@ -144,15 +269,68 @@ func canonicalNameGroups(records []normalizedRecord) [][]int { return groups } -func consolidate(records []normalizedRecord, members []int, order shared.SourceRefOrder) (dnd.NPC, bool) { - output := cloneNPC(records[members[0]].npc) - originalRefs := cloneSourceRefs(output.SourceRefs) +func consolidate(records []normalizedRecord, members []int, order shared.SourceRefOrder) (normalizedRecord, bool) { + output := cloneRecord(records[members[0]]) + originalRefs := cloneSourceRefs(output.npc.SourceRefs) for _, member := range members[1:] { - output.SourceRefs = append(output.SourceRefs, records[member].npc.SourceRefs...) + output.npc.SourceRefs = append(output.npc.SourceRefs, records[member].npc.SourceRefs...) + output.inputIndexes = append(output.inputIndexes, records[member].inputIndexes...) + if records[member].earliest < output.earliest { + output.earliest = records[member].earliest + } } - output.SourceRefs = order.Canonicalize(output.SourceRefs) - output.ID = identity.DeriveID(output.Name) - return output, !reflect.DeepEqual(originalRefs, output.SourceRefs) + output.inputIndexes = sortedUniqueIndexes(output.inputIndexes) + output.npc.SourceRefs = order.Canonicalize(output.npc.SourceRefs) + output.npc.ID = identity.DeriveID(output.npc.Name) + return output, !reflect.DeepEqual(originalRefs, output.npc.SourceRefs) +} + +func cloneRecord(input normalizedRecord) normalizedRecord { + input.npc = cloneNPC(input.npc) + input.inputIndexes = append([]int(nil), input.inputIndexes...) + return input +} + +func memberInputIndexes(records []normalizedRecord, members []int) []int { + indexes := make([]int, 0, len(members)) + for _, member := range members { + indexes = append(indexes, records[member].inputIndexes...) + } + return sortedUniqueIndexes(indexes) +} + +func sortedUniqueIndexes(indexes []int) []int { + if len(indexes) == 0 { + return nil + } + out := append([]int(nil), indexes...) + sort.Ints(out) + write := 1 + for _, index := range out[1:] { + if index != out[write-1] { + out[write] = index + write++ + } + } + return out[:write] +} + +func recordValues(records []normalizedRecord) []dnd.NPC { + if records == nil { + return nil + } + values := make([]dnd.NPC, len(records)) + for index, record := range records { + values[index] = cloneNPC(record.npc) + } + return values +} + +func recordList(records []normalizedRecord) dnd.NPCList { + if records == nil { + return dnd.NPCList{} + } + return dnd.NPCList{NPCs: recordValues(records)} } func cloneSourceRefs(input []source.SourceRef) []source.SourceRef { @@ -191,7 +369,7 @@ func Register(registry *pipeline.NormalizerRegistry) error { if err != nil { return nil, err } - return New(options), nil + return New(request.Dependencies.LLM, options) }) } diff --git a/internal/modules/dnd/normalize/npcs/normalizer_test.go b/internal/modules/dnd/normalize/npcs/normalizer_test.go index 98f2dfa..d4a5c79 100644 --- a/internal/modules/dnd/normalize/npcs/normalizer_test.go +++ b/internal/modules/dnd/normalize/npcs/normalizer_test.go @@ -2,6 +2,7 @@ package npcs import ( "context" + "encoding/json" "reflect" "strings" "testing" @@ -28,11 +29,14 @@ func TestModuleContractAndIdentity(t *testing.T) { if err := Register(registry); err != nil { t.Fatalf("Register() error = %v", err) } - normalizer := New(Options{}) - if metadata := normalizer.ManifestMetadata(); metadata["identity_policy"] != identity.Policy || metadata["normalization_policy"] != normalizationPolicy { + if _, err := New(nil, Options{}); err == nil { + t.Fatal("New(nil, Options{}) error = nil, want nil client rejection") + } + normalizer := newNormalizer(t, &recordingNPCNormalizerClient{}) + if metadata := normalizer.ManifestMetadata(); metadata["identity_policy"] != identity.Policy || metadata["normalization_policy"] != normalizationPolicy || metadata["prompt_id"] != PromptID || metadata["semantic_context_policy"] != semanticContextPolicy || metadata["semantic_context_radius"] != semanticContextRadius { t.Fatalf("metadata = %#v", metadata) } - wantFingerprints := []pipeline.CheckpointFingerprint{{Name: "identity_policy", Value: identity.Policy}, {Name: "normalization_policy", Value: normalizationPolicy}} + wantFingerprints := []pipeline.CheckpointFingerprint{{Name: "prompt", Value: normalizer.promptSHA}, {Name: "response_schema", Value: normalizer.responseSchemaSHA}, {Name: "identity_policy", Value: identity.Policy}, {Name: "normalization_policy", Value: normalizationPolicy}, {Name: "semantic_context_policy", Value: semanticContextPolicy + ":2"}} if got := normalizer.CheckpointFingerprints(); !reflect.DeepEqual(got, wantFingerprints) { t.Fatalf("fingerprints = %#v, want %#v", got, wantFingerprints) } @@ -46,7 +50,7 @@ func TestNormalizeNamesEvidenceAndIDs(t *testing.T) { {SourceID: "b", StartUnitID: 2, EndUnitID: 3}, }, }}} - result, err := New(Options{}).Normalize(context.Background(), normalizeRequest(input)) + result, err := newNormalizer(t, &recordingNPCNormalizerClient{}).Normalize(context.Background(), normalizeRequest(input)) if err != nil { t.Fatalf("Normalize() error = %v", err) } @@ -68,7 +72,7 @@ func TestNormalizeOrdersEvidenceBySourceDocumentPosition(t *testing.T) { {SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}, {SourceID: doc.ID, StartUnitID: 999, EndUnitID: 999}, }}}} - result, err := New(Options{}).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) + result, err := newNormalizer(t, &recordingNPCNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(input, doc)) if err != nil { t.Fatalf("Normalize() error = %v", err) } @@ -84,7 +88,7 @@ func TestNormalizeConsolidatesCanonicalNamesOnlyAndUnionsEvidence(t *testing.T) {Name: "captain vale", SourceRefs: []source.SourceRef{{SourceID: "b", StartUnitID: 2, EndUnitID: 2}}}, {Name: "The Captain", SourceRefs: []source.SourceRef{{SourceID: "c", StartUnitID: 3, EndUnitID: 3}}}, }} - result, err := New(Options{}).Normalize(context.Background(), normalizeRequest(input)) + result, err := newNormalizer(t, &recordingNPCNormalizerClient{}).Normalize(context.Background(), normalizeRequest(input)) if err != nil { t.Fatalf("Normalize() error = %v", err) } @@ -102,7 +106,7 @@ func TestNormalizeConsolidatesCanonicalNamesOnlyAndUnionsEvidence(t *testing.T) func TestNormalizePreservesInvalidCandidatesAndDoesNotAliasInput(t *testing.T) { input := dnd.NPCList{NPCs: []dnd.NPC{{Name: " ", SourceRefs: []source.SourceRef{{SourceID: "source", StartUnitID: 0, EndUnitID: -1}}}}} before := dnd.NPCList{NPCs: []dnd.NPC{{Name: " ", SourceRefs: []source.SourceRef{{SourceID: "source", StartUnitID: 0, EndUnitID: -1}}}}} - result, err := New(Options{}).Normalize(context.Background(), normalizeRequest(input)) + result, err := newNormalizer(t, &recordingNPCNormalizerClient{}).Normalize(context.Background(), normalizeRequest(input)) if err != nil || !reflect.DeepEqual(input, before) { t.Fatalf("Normalize() = %#v, %v; input mutated to %#v", result, err, input) } @@ -116,7 +120,7 @@ func TestNormalizePreservesInvalidCandidatesAndDoesNotAliasInput(t *testing.T) { } func TestNormalizeHandlesNilAndCanceledCalls(t *testing.T) { - normalizer := New(Options{}) + normalizer := newNormalizer(t, &recordingNPCNormalizerClient{}) result, err := normalizer.Normalize(context.Background(), normalizeRequest(dnd.NPCList{NPCs: nil})) if err != nil || result.Value.NPCs != nil { t.Fatalf("nil list result = %#v, error = %v", result.Value, err) @@ -128,6 +132,44 @@ func TestNormalizeHandlesNilAndCanceledCalls(t *testing.T) { } } +type recordingNPCNormalizerClient struct { + response string + responses []string + err error + requests []contracts.StructuredCompletionRequest +} + +func (c *recordingNPCNormalizerClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, output any) (contracts.StructuredCompletionResponse, error) { + c.requests = append(c.requests, request) + if c.err != nil { + return contracts.StructuredCompletionResponse{}, c.err + } + response := c.response + if len(c.responses) > 0 { + responseIndex := len(c.requests) - 1 + if responseIndex >= len(c.responses) { + responseIndex = len(c.responses) - 1 + } + response = c.responses[responseIndex] + } + if response == "" { + response = `{"duplicate_groups":[]}` + } + if err := json.Unmarshal([]byte(response), output); err != nil { + return contracts.StructuredCompletionResponse{}, err + } + return contracts.StructuredCompletionResponse{Content: json.RawMessage(response)}, nil +} + +func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normalizer { + t.Helper() + normalizer, err := New(client, Options{}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + return normalizer +} + func normalizeRequest(value dnd.NPCList) contracts.TypedNormalizeRequest[dnd.NPCList] { return contracts.TypedNormalizeRequest[dnd.NPCList]{MergeOutput: contracts.MergeArtifact[dnd.NPCList]{Value: value}} } diff --git a/internal/modules/dnd/normalize/npcs/proposal.go b/internal/modules/dnd/normalize/npcs/proposal.go new file mode 100644 index 0000000..8b1663d --- /dev/null +++ b/internal/modules/dnd/normalize/npcs/proposal.go @@ -0,0 +1,195 @@ +package npcs + +import ( + "fmt" + "sort" + "strconv" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics" +) + +type proposalAssessment struct { + safeGroups []safeProposalGroup + discardedGroups int + issues []string +} + +type safeProposalGroup struct { + members []int + canonical int +} + +type assessedProposalGroup struct { + members []int + canonical int + locallyValid bool + conflicting bool +} + +func assessProposal(response normalizeProposalResponse, records []normalizedRecord, candidatePositions []int) proposalAssessment { + positionsByKey := make(map[string][]int, len(candidatePositions)) + for _, position := range candidatePositions { + if position < 0 || position >= len(records) { + continue + } + key := identity.ComparisonKey(records[position].npc.Name) + if key != "" { + positionsByKey[key] = append(positionsByKey[key], position) + } + } + groups := make([]assessedProposalGroup, len(response.DuplicateGroups)) + issues := make([]string, 0) + owners := make(map[int][]int) + + for groupIndex, proposal := range response.DuplicateGroups { + group, groupIssues := assessProposalGroup(proposal, positionsByKey) + groups[groupIndex] = group + for _, position := range group.members { + owners[position] = append(owners[position], groupIndex) + } + for _, issue := range groupIssues { + issues = append(issues, proposalIssue(groupIndex, issue)) + } + } + for groupIndex := range groups { + for _, position := range groups[groupIndex].members { + if len(owners[position]) > 1 { + groups[groupIndex].conflicting = true + break + } + } + if groups[groupIndex].conflicting { + issues = append(issues, proposalIssue(groupIndex, "overlapping_member")) + } + } + + assessment := proposalAssessment{issues: issues} + for _, group := range groups { + if !group.locallyValid || group.conflicting { + assessment.discardedGroups++ + continue + } + assessment.safeGroups = append(assessment.safeGroups, safeProposalGroup{members: group.members, canonical: group.canonical}) + } + return assessment +} + +func assessProposalGroup(proposal normalizeProposalGroup, positionsByKey map[string][]int) (assessedProposalGroup, []string) { + issues := make([]string, 0) + members := make([]int, 0, len(proposal.Members)) + seenMembers := make(map[int]struct{}, len(proposal.Members)) + for _, name := range proposal.Members { + position, issue := resolveCandidate(name, positionsByKey) + if issue != "" { + issues = append(issues, "member_"+issue) + continue + } + if _, exists := seenMembers[position]; exists { + issues = append(issues, "repeated_member") + continue + } + seenMembers[position] = struct{}{} + members = append(members, position) + } + canonical, canonicalIssue := resolveCandidate(proposal.CanonicalName, positionsByKey) + if canonicalIssue != "" { + issues = append(issues, "canonical_"+canonicalIssue) + } + if len(members) < 2 { + issues = append(issues, "fewer_than_two_members") + } + if canonicalIssue == "" && !containsPosition(members, canonical) { + issues = append(issues, "canonical_not_member") + } + sort.Ints(members) + return assessedProposalGroup{ + members: members, canonical: canonical, locallyValid: len(issues) == 0, + }, issues +} + +func resolveCandidate(name string, positionsByKey map[string][]int) (position int, issue string) { + key := identity.ComparisonKey(name) + if key == "" { + return 0, "blank" + } + positions := positionsByKey[key] + if len(positions) == 0 { + return 0, "unknown" + } + if len(positions) != 1 { + return 0, "ambiguous" + } + return positions[0], "" +} + +func containsPosition(positions []int, want int) bool { + for _, position := range positions { + if position == want { + return true + } + } + return false +} + +func proposalIssue(groupIndex int, category string) string { + return "group " + strconv.Itoa(groupIndex) + ": " + category +} + +func applySafeGroups(records []normalizedRecord, groups []safeProposalGroup, order shared.SourceRefOrder) ([]normalizedRecord, []contracts.Warning) { + byMember := make(map[int]safeProposalGroup, len(groups)*2) + for _, group := range groups { + for _, member := range group.members { + byMember[member] = group + } + } + output := make([]normalizedRecord, 0, len(records)-len(groups)) + warnings := make([]contracts.Warning, 0, len(groups)) + for index, record := range records { + group, grouped := byMember[index] + if !grouped { + output = append(output, cloneRecord(record)) + continue + } + if group.members[0] != index { + continue + } + consolidated := consolidateSemanticGroup(records, group, order) + output = append(output, consolidated) + warnings = append(warnings, semanticDuplicateWarning(consolidated, records[group.canonical])) + } + return output, warnings +} + +func consolidateSemanticGroup(records []normalizedRecord, group safeProposalGroup, order shared.SourceRefOrder) normalizedRecord { + output := cloneRecord(records[group.members[0]]) + output.npc.Name = records[group.canonical].npc.Name + for _, member := range group.members[1:] { + output.npc.SourceRefs = append(output.npc.SourceRefs, records[member].npc.SourceRefs...) + output.inputIndexes = append(output.inputIndexes, records[member].inputIndexes...) + if records[member].earliest < output.earliest { + output.earliest = records[member].earliest + } + } + output.inputIndexes = sortedUniqueIndexes(output.inputIndexes) + output.npc.SourceRefs = order.Canonicalize(output.npc.SourceRefs) + output.npc.ID = identity.DeriveID(output.npc.Name) + return output +} + +func semanticDuplicateWarning(record normalizedRecord, canonical normalizedRecord) contracts.Warning { + details := make([]string, 0, len(record.inputIndexes)+1) + for _, inputIndex := range record.inputIndexes { + details = append(details, fmt.Sprintf("input index %d", inputIndex)) + } + if canonical.earliest != record.earliest { + details = append(details, fmt.Sprintf("canonical display name from input index %d", canonical.earliest)) + } + return contracts.Warning{ + Scope: npcScope(record.earliest), + ReasonCode: ReasonCodeDuplicateNPCCollapsed, + Message: diagnostics.Aggregate("semantic duplicate consolidation", details), + } +} diff --git a/internal/modules/dnd/normalize/npcs/semantic_normalizer_test.go b/internal/modules/dnd/normalize/npcs/semantic_normalizer_test.go new file mode 100644 index 0000000..2cbdd12 --- /dev/null +++ b/internal/modules/dnd/normalize/npcs/semantic_normalizer_test.go @@ -0,0 +1,265 @@ +package npcs + +import ( + "context" + "errors" + "math" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" +) + +func TestNormalizeSkipsSemanticCompletionWithoutTwoEligibleCandidates(t *testing.T) { + client := &recordingNPCNormalizerClient{} + normalizer := newNormalizer(t, client) + doc := semanticDocument() + input := dnd.NPCList{NPCs: []dnd.NPC{{Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}}} + result, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc)) + if err != nil || len(client.requests) != 0 || result.Retry != nil { + t.Fatalf("Normalize() = %#v, %v; calls = %d, want deterministic no-call result", result, err, len(client.requests)) + } + if result.Value.NPCs[0].Name != "Mira Thorn" { + t.Fatalf("NPCs = %#v, want deterministic record", result.Value.NPCs) + } +} + +func TestNormalizeAppliesSafeProposalAndUsesPrivateInputs(t *testing.T) { + client := &recordingNPCNormalizerClient{response: `{"duplicate_groups":[{"members":[" MIRA THORN ","Mira"],"canonical_name":"Mira Thorn"}]}`} + normalizer := newNormalizer(t, client) + doc := semanticDocument() + input := dnd.NPCList{NPCs: []dnd.NPC{ + {ID: "npc:sha256:short", Name: "Mira", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, + {ID: "npc:sha256:long", Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}, + {ID: "npc:sha256:captain", Name: "Captain Vale", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}}, + }} + before := cloneNPCList(input) + request := normalizeRequestWithSource(input, doc) + request.LLMProfile = "normalizer-profile" + request.SessionID = "normalizer-session" + result, err := normalizer.Normalize(context.Background(), request) + if err != nil || result.Retry != nil { + t.Fatalf("Normalize() = %#v, %v; want accepted semantic result", result, err) + } + if !reflect.DeepEqual(input, before) { + t.Fatalf("Normalize() mutated input to %#v", input) + } + if len(result.Value.NPCs) != 2 || result.Value.NPCs[0].Name != "Mira Thorn" || result.Value.NPCs[1].Name != "Captain Vale" { + t.Fatalf("NPCs = %#v, want canonical record at earliest position", result.Value.NPCs) + } + merged := result.Value.NPCs[0] + if merged.ID != identity.DeriveID("Mira Thorn") || !reflect.DeepEqual(merged.SourceRefs, []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}, {SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}) { + t.Fatalf("merged NPC = %#v, want canonical ID and original evidence union", merged) + } + if !hasWarning(result.Warnings, ReasonCodeDuplicateNPCCollapsed, "npcs[0]") { + t.Fatalf("warnings = %#v, want semantic collapse warning", result.Warnings) + } + if len(client.requests) != 1 { + t.Fatalf("completion calls = %d, want one", len(client.requests)) + } + completion := client.requests[0] + if completion.StageName != Key || completion.PromptID != PromptID || completion.PromptVersion != SchemaVersion || completion.ProfileID != request.LLMProfile || completion.SessionID != request.SessionID || len(completion.Inputs) != 2 { + t.Fatalf("completion request = %#v, want normalize request identity and exactly two inputs", completion) + } + encoded := string(completion.Inputs["candidates"].Content) + string(completion.Inputs["transcript"].Content) + if strings.Contains(encoded, "npc:sha256:") || strings.Contains(encoded, doc.ID) { + t.Fatalf("completion inputs leaked private identifiers: %s", encoded) + } +} + +func TestNormalizeUnsafeProposalReturnsSafeRetryFallback(t *testing.T) { + client := &recordingNPCNormalizerClient{response: `{"duplicate_groups":[{"members":["Mira","Mira Thorn"],"canonical_name":"Mira Thorn"},{"members":["Captain Vale","Unknown"],"canonical_name":"Captain Vale"}]}`} + normalizer := newNormalizer(t, client) + doc := semanticDocument() + input := dnd.NPCList{NPCs: []dnd.NPC{ + {Name: "Mira", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, + {Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}, + {Name: "Captain Vale", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}}, + }} + result, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc)) + if err != nil || result.Retry == nil { + t.Fatalf("Normalize() = %#v, %v; want retryable safe fallback", result, err) + } + if result.Retry.ReasonCode != ReasonCodeNPCSemanticProposalInvalid || !strings.Contains(result.Retry.Message, "group 1: member_unknown") { + t.Fatalf("retry = %#v, want bounded invalid-proposal diagnostics", result.Retry) + } + if len(result.Value.NPCs) != 2 || result.Value.NPCs[0].Name != "Mira Thorn" || result.Value.NPCs[1].Name != "Captain Vale" { + t.Fatalf("fallback NPCs = %#v, want independently safe group applied", result.Value.NPCs) + } + if len(result.Retry.FallbackWarnings) != 1 || result.Retry.FallbackWarnings[0].ReasonCode != ReasonCodeNPCSemanticReconciliationExhausted || !strings.Contains(result.Retry.FallbackWarnings[0].Message, "1 proposal group") { + t.Fatalf("fallback warnings = %#v, want exact omitted-group warning", result.Retry.FallbackWarnings) + } +} + +func TestNormalizeRejectsOverlapsWithoutResponseOrderDependence(t *testing.T) { + records := []normalizedRecord{ + {npc: dnd.NPC{Name: "Alpha"}}, {npc: dnd.NPC{Name: "Bravo"}}, {npc: dnd.NPC{Name: "Charlie"}}, {npc: dnd.NPC{Name: "Delta"}}, + } + for index := range records { + records[index].inputIndexes = []int{index} + records[index].earliest = index + } + response := normalizeProposalResponse{DuplicateGroups: []normalizeProposalGroup{ + {Members: []string{"Alpha", "Bravo"}, CanonicalName: "Alpha"}, + {Members: []string{"Bravo", "Charlie"}, CanonicalName: "Bravo"}, + {Members: []string{"Charlie", "Delta"}, CanonicalName: "Charlie"}, + }} + assessment := assessProposal(response, records, []int{0, 1, 2, 3}) + if assessment.discardedGroups != 3 || len(assessment.safeGroups) != 0 { + t.Fatalf("assessment = %#v, want chained conflicts all discarded", assessment) + } + for _, issue := range []string{"group 0: overlapping_member", "group 1: overlapping_member", "group 2: overlapping_member"} { + if !containsString(assessment.issues, issue) { + t.Fatalf("issues = %#v, want %q", assessment.issues, issue) + } + } +} + +func TestNormalizeInvalidStructuredOutputAndOperationalErrorsRemainDistinct(t *testing.T) { + doc := semanticDocument() + input := dnd.NPCList{NPCs: []dnd.NPC{ + {Name: "Mira", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, + {Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}, + }} + invalid := newNormalizer(t, &recordingNPCNormalizerClient{err: contracts.ErrInvalidStructuredOutput}) + result, err := invalid.Normalize(context.Background(), normalizeRequestWithSource(input, doc)) + if err != nil || result.Retry == nil || result.Retry.ReasonCode != ReasonCodeNPCSemanticProposalInvalid || len(result.Value.NPCs) != 2 { + t.Fatalf("invalid structured result = %#v, %v; want deterministic retry fallback", result, err) + } + operational := errors.New("provider unavailable") + if _, err := newNormalizer(t, &recordingNPCNormalizerClient{err: operational}).Normalize(context.Background(), normalizeRequestWithSource(input, doc)); !errors.Is(err, operational) || errors.Is(err, contracts.ErrInvalidStructuredOutput) { + t.Fatalf("operational completion error = %v, want ordinary error", err) + } +} + +func TestNormalizeRejectsContextEncodingFailuresWithoutLeakingContent(t *testing.T) { + client := &recordingNPCNormalizerClient{} + normalizer := newNormalizer(t, client) + doc := semanticDocument() + doc.Units[0].Metadata = map[string]any{"invalid": math.NaN()} + input := dnd.NPCList{NPCs: []dnd.NPC{ + {Name: "Mira", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, + {Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}, + }} + if _, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc)); err == nil || !strings.Contains(err.Error(), "build semantic context") || strings.Contains(err.Error(), doc.Units[0].Text) || len(client.requests) != 0 { + t.Fatalf("Normalize() error = %v, calls = %d; want safe preparation error before completion", err, len(client.requests)) + } +} + +func TestNormalizeDoesNotAccumulateSafeGroupsAcrossAttempts(t *testing.T) { + client := &recordingNPCNormalizerClient{responses: []string{ + `{"duplicate_groups":[{"members":["Mira","Mira Thorn"],"canonical_name":"Mira Thorn"},{"members":["Captain Vale","Unknown"],"canonical_name":"Captain Vale"}]}`, + `{"duplicate_groups":[{"members":["Mira","Captain Vale"],"canonical_name":"Captain Vale"}]}`, + }} + normalizer := newNormalizer(t, client) + doc := semanticDocument() + input := dnd.NPCList{NPCs: []dnd.NPC{ + {Name: "Mira", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, + {Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}, + {Name: "Captain Vale", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}}, + }} + first, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc)) + if err != nil || first.Retry == nil || len(first.Value.NPCs) != 2 { + t.Fatalf("first Normalize() = %#v, %v; want partial retry fallback", first, err) + } + second, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc)) + if err != nil || second.Retry != nil { + t.Fatalf("second Normalize() = %#v, %v; want accepted independent retry result", second, err) + } + if got := []string{second.Value.NPCs[0].Name, second.Value.NPCs[1].Name}; !reflect.DeepEqual(got, []string{"Captain Vale", "Mira Thorn"}) { + t.Fatalf("second NPCs = %#v, want proposal applied to original merge output", second.Value.NPCs) + } +} + +func TestNormalizeExcludesIneligibleRecordsFromSemanticGroups(t *testing.T) { + client := &recordingNPCNormalizerClient{response: `{"duplicate_groups":[{"members":["Mira","Broken"],"canonical_name":"Mira"}]}`} + normalizer := newNormalizer(t, client) + doc := semanticDocument() + input := dnd.NPCList{NPCs: []dnd.NPC{ + {Name: "Mira", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, + {Name: "Broken", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 999, EndUnitID: 999}}}, + {Name: "Captain Vale", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}}, + }} + result, err := normalizer.Normalize(context.Background(), normalizeRequestWithSource(input, doc)) + if err != nil || result.Retry == nil || len(result.Value.NPCs) != 3 { + t.Fatalf("Normalize() = %#v, %v; want unchanged retry fallback", result, err) + } + if !strings.Contains(result.Retry.Message, "member_unknown") || result.Value.NPCs[1].Name != "Broken" { + t.Fatalf("result = %#v, want ineligible record excluded but preserved", result) + } +} + +func TestProposalValidationRejectsUnsafeCategories(t *testing.T) { + records := []normalizedRecord{ + {npc: dnd.NPC{Name: "Mira"}, inputIndexes: []int{0}, earliest: 0}, + {npc: dnd.NPC{Name: "Mira Thorn"}, inputIndexes: []int{1}, earliest: 1}, + {npc: dnd.NPC{Name: "Captain Vale"}, inputIndexes: []int{2}, earliest: 2}, + } + for _, proposal := range []normalizeProposalGroup{ + {Members: []string{"Mira"}, CanonicalName: "Mira"}, + {Members: []string{"Mira", "Mira"}, CanonicalName: "Mira"}, + {Members: []string{"Mira", "Mira Thorn"}, CanonicalName: "Unknown"}, + {Members: []string{"Mira", "Mira Thorn"}, CanonicalName: " "}, + {Members: []string{" ", "Mira Thorn"}, CanonicalName: "Mira Thorn"}, + {Members: []string{"Mira", "Mira Thorn"}, CanonicalName: "Captain Vale"}, + } { + assessment := assessProposal(normalizeProposalResponse{DuplicateGroups: []normalizeProposalGroup{proposal}}, records, []int{0, 1, 2}) + if assessment.discardedGroups != 1 || len(assessment.safeGroups) != 0 { + t.Fatalf("assessment for %#v = %#v, want discarded unsafe group", proposal, assessment) + } + } +} + +func TestProposalResolutionUsesOnlyExistingComparisonKeyEquivalences(t *testing.T) { + records := []normalizedRecord{ + {npc: dnd.NPC{Name: "O'Neill"}, inputIndexes: []int{0}, earliest: 0}, + {npc: dnd.NPC{Name: "Mira Thorn"}, inputIndexes: []int{1}, earliest: 1}, + } + assessment := assessProposal(normalizeProposalResponse{DuplicateGroups: []normalizeProposalGroup{{ + Members: []string{" O’NEILL ", "Ｍｉｒａ　Ｔｈｏｒｎ"}, CanonicalName: "Ｍｉｒａ　Ｔｈｏｒｎ", + }}}, records, []int{0, 1}) + if assessment.discardedGroups != 0 || len(assessment.safeGroups) != 1 || assessment.safeGroups[0].canonical != 1 { + t.Fatalf("assessment = %#v, want comparison-key-only resolution", assessment) + } +} + +func TestRetryWarningLimitReservesExhaustionWarningPosition(t *testing.T) { + warnings := make([]contracts.Warning, 0, 25) + for index := 0; index < 25; index++ { + warnings = append(warnings, contracts.Warning{Scope: "npcs", ReasonCode: "test", Message: "warning"}) + } + bounded := limitWarningsForRetry(warnings) + if len(bounded) != 19 || bounded[len(bounded)-1].ReasonCode != ReasonCodeNPCNormalizationWarningsOmitted || !strings.Contains(bounded[len(bounded)-1].Message, "7 additional") { + t.Fatalf("retry warnings = %#v, want 18 warnings plus accurate omission summary", bounded) + } +} + +func semanticDocument() *source.SourceDocument { + return &source.SourceDocument{ID: "session", Units: []source.SourceUnit{ + {ID: 10, Kind: "speech", Text: "Mira speaks"}, + {ID: 15, Kind: "narration", Text: "surrounding context"}, + {ID: 20, Kind: "speech", Text: "Mira Thorn replies"}, + {ID: 30, Kind: "speech", Text: "Captain Vale watches"}, + }} +} + +func cloneNPCList(input dnd.NPCList) dnd.NPCList { + output := dnd.NPCList{NPCs: make([]dnd.NPC, len(input.NPCs))} + for index, npc := range input.NPCs { + output.NPCs[index] = cloneNPC(npc) + } + return output +} + +func containsString(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +}