// Package npcs normalizes merged D&D non-player character records. package npcs import ( "context" "errors" "fmt" "reflect" "sort" "strconv" "strings" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics" ) const ( Key = "dnd/npcs" normalizationPolicy = "dnd.npcs.normalize.v3" semanticContextPolicy = "dnd.npcs.semantic_context.v1" NormalizationPolicy = normalizationPolicy ReasonCodeNPCFieldsNormalized = "npc_fields_normalized" ReasonCodeNPCIDRecomputed = "npc_id_recomputed" ReasonCodeSourceReferencesNormalized = "source_references_normalized" ReasonCodeDuplicateNPCCollapsed = "duplicate_npc_collapsed" ReasonCodeNPCSemanticProposalInvalid = "npc_semantic_proposal_invalid" ReasonCodeNPCSemanticReconciliationExhausted = "npc_semantic_reconciliation_exhausted" ReasonCodeNPCNormalizationWarningsOmitted = "npc_normalization_warnings_omitted" ) var requiredCapabilities = []string{"merged"} var providedCapabilities = []string{"normalized"} var _ contracts.Normalizer[dnd.NPCList] = (*Normalizer)(nil) var _ contracts.ManifestMetadataProvider = (*Normalizer)(nil) var _ pipeline.CheckpointFingerprintProvider = (*Normalizer)(nil) type Options struct{} type Normalizer struct { llm contracts.StructuredLLMClient promptSHA string responseSchemaSHA string } func New(llmClient contracts.StructuredLLMClient, _ Options) (*Normalizer, error) { if llmClient == nil { return nil, normalizerErrorf("LLM client must not be nil") } promptSHA, err := promptAssetMetadata() if err != nil { return nil, normalizerErrorf("load prompt metadata: %w", err) } responseSchema, err := loadResponseSchema() if err != nil { return nil, normalizerErrorf("load response schema: %w", err) } return &Normalizer{llm: llmClient, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil } func (n *Normalizer) Key() string { return Key } func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil } func (n *Normalizer) ManifestMetadata() map[string]any { if n == nil { return nil } return map[string]any{ "prompt_id": PromptID, "prompt_version": SchemaVersion, "prompt_sha256": n.promptSHA, "response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID, "response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion, "response_schema_sha256": n.responseSchemaSHA, "identity_policy": identity.Policy, "normalization_policy": normalizationPolicy, "semantic_context_policy": semanticContextPolicy, "semantic_context_radius": semanticContextRadius, } } func (n *Normalizer) CheckpointFingerprints() []pipeline.CheckpointFingerprint { if n == nil { return nil } return []pipeline.CheckpointFingerprint{ {Name: "prompt", Value: n.promptSHA}, {Name: "response_schema", Value: n.responseSchemaSHA}, {Name: "identity_policy", Value: identity.Policy}, {Name: "normalization_policy", Value: normalizationPolicy}, {Name: "semantic_context_policy", Value: fmt.Sprintf("%s:%d", semanticContextPolicy, semanticContextRadius)}, } } func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalizeRequest[dnd.NPCList]) (contracts.TypedNormalizeResult[dnd.NPCList], error) { if n == nil { return contracts.TypedNormalizeResult[dnd.NPCList]{}, normalizerErrorf("normalizer must not be nil") } if n.llm == nil { return contracts.TypedNormalizeResult[dnd.NPCList]{}, normalizerErrorf("LLM client must not be nil") } if ctx == nil { return contracts.TypedNormalizeResult[dnd.NPCList]{}, normalizerErrorf("context must not be nil") } if err := ctx.Err(); err != nil { return contracts.TypedNormalizeResult[dnd.NPCList]{}, normalizerErrorf("context error before normalize: %w", err) } order := shared.NewSourceRefOrder(req.Source) records, warnings := preprocessRecords(req.MergeOutput.Value, order) deterministic := recordList(records) materials, ready, err := buildDefaultNormalizeContextMaterials(req.Source, recordValues(records)) if err != nil { return contracts.TypedNormalizeResult[dnd.NPCList]{}, normalizerErrorf("build semantic context: %w", err) } if !ready { return contracts.TypedNormalizeResult[dnd.NPCList]{Value: deterministic, Warnings: limitWarnings(warnings)}, nil } var response normalizeProposalResponse if _, err := n.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: contracts.LLMInputSet{"candidates": materials.Candidates, "transcript": materials.Transcript}, }, &response); err != nil { if errors.Is(err, contracts.ErrInvalidStructuredOutput) { return n.invalidStructuredResult(deterministic, warnings), nil } return contracts.TypedNormalizeResult[dnd.NPCList]{}, normalizerErrorf("complete structured output: %w", err) } assessment := assessProposal(response, records, materials.candidatePositions) applied, semanticWarnings := applySafeGroups(records, assessment.safeGroups, order) warnings = append(warnings, semanticWarnings...) if assessment.discardedGroups == 0 { return contracts.TypedNormalizeResult[dnd.NPCList]{Value: recordList(applied), Warnings: limitWarnings(warnings)}, nil } return retryResult(recordList(applied), warnings, assessment), nil } func (n *Normalizer) invalidStructuredResult(value dnd.NPCList, warnings []contracts.Warning) contracts.TypedNormalizeResult[dnd.NPCList] { return contracts.TypedNormalizeResult[dnd.NPCList]{ Value: value, Warnings: limitWarningsForRetry(warnings), Retry: &contracts.NormalizeRetry{ ReasonCode: ReasonCodeNPCSemanticProposalInvalid, Message: "semantic proposal requires retry: invalid structured output", FallbackWarnings: []contracts.Warning{semanticFallbackWarning(-1)}, }, } } func retryResult(value dnd.NPCList, warnings []contracts.Warning, assessment proposalAssessment) contracts.TypedNormalizeResult[dnd.NPCList] { return contracts.TypedNormalizeResult[dnd.NPCList]{ Value: value, Warnings: limitWarningsForRetry(warnings), Retry: &contracts.NormalizeRetry{ ReasonCode: ReasonCodeNPCSemanticProposalInvalid, Message: diagnostics.Aggregate("semantic proposal requires retry", assessment.issues), FallbackWarnings: []contracts.Warning{semanticFallbackWarning(assessment.discardedGroups)}, }, } } func semanticFallbackWarning(discardedGroups int) contracts.Warning { message := "semantic proposal could not be applied" if discardedGroups >= 0 { message = fmt.Sprintf("%d proposal group(s) omitted after semantic proposal retry exhaustion", discardedGroups) } return contracts.Warning{Scope: "npcs", ReasonCode: ReasonCodeNPCSemanticReconciliationExhausted, Message: message} } func limitWarnings(warnings []contracts.Warning) []contracts.Warning { return diagnostics.LimitWarnings(warnings, "npcs", ReasonCodeNPCNormalizationWarningsOmitted) } func limitWarningsForRetry(warnings []contracts.Warning) []contracts.Warning { if warnings == nil { return nil } if len(warnings) < diagnostics.MaxWarnings { return append([]contracts.Warning(nil), warnings...) } displayed := diagnostics.MaxWarnings - 2 bounded := append([]contracts.Warning(nil), warnings[:displayed]...) return append(bounded, contracts.Warning{ Scope: "npcs", ReasonCode: ReasonCodeNPCNormalizationWarningsOmitted, Message: fmt.Sprintf("%d additional warning(s) omitted", len(warnings)-displayed), }) } type normalizedRecord struct { npc dnd.NPC inputIndexes []int earliest int } func preprocessRecords(input dnd.NPCList, order shared.SourceRefOrder) ([]normalizedRecord, []contracts.Warning) { if input.NPCs == nil { return nil, nil } records := make([]normalizedRecord, len(input.NPCs)) warnings := make([]contracts.Warning, 0) for index, inputNPC := range input.NPCs { npc, fieldsChanged, referencesChanged := normalizeRecord(inputNPC, order) records[index] = normalizedRecord{npc: npc, inputIndexes: []int{index}, earliest: index} if fieldsChanged { warnings = append(warnings, contracts.Warning{Scope: npcScope(index), ReasonCode: ReasonCodeNPCFieldsNormalized, Message: fmt.Sprintf("input index %d: NPC name normalized for %s", index, diagnostics.Quote(inputNPC.Name))}) } if referencesChanged { warnings = append(warnings, contracts.Warning{Scope: npcScope(index), ReasonCode: ReasonCodeSourceReferencesNormalized, Message: fmt.Sprintf("input index %d: source references normalized (original count %d, final count %d)", index, len(inputNPC.SourceRefs), len(npc.SourceRefs))}) } if inputNPC.ID != npc.ID { warnings = append(warnings, contracts.Warning{Scope: npcScope(index), ReasonCode: ReasonCodeNPCIDRecomputed, Message: fmt.Sprintf("input index %d: NPC ID recomputed from %s", index, diagnostics.Quote(npc.Name))}) } } groups := canonicalNameGroups(records) output := make([]normalizedRecord, 0, len(groups)) for _, members := range groups { consolidated, referencesChanged := consolidate(records, members, order) output = append(output, consolidated) retainedIndex := consolidated.earliest if referencesChanged { warnings = append(warnings, contracts.Warning{Scope: npcScope(retainedIndex), ReasonCode: ReasonCodeSourceReferencesNormalized, Message: fmt.Sprintf("input index %d: source references normalized during identity consolidation (final count %d)", retainedIndex, len(consolidated.npc.SourceRefs))}) } if len(members) > 1 { warnings = append(warnings, duplicateWarning(retainedIndex, memberInputIndexes(records, members[1:]))) } } return output, warnings } func normalizeRecord(input dnd.NPC, order shared.SourceRefOrder) (dnd.NPC, bool, bool) { output := cloneNPC(input) output.Name = identity.NormalizeDisplay(input.Name) output.SourceRefs = order.Canonicalize(input.SourceRefs) output.ID = identity.DeriveID(output.Name) return output, input.Name != output.Name, !reflect.DeepEqual(input.SourceRefs, output.SourceRefs) } func cloneNPC(input dnd.NPC) dnd.NPC { input.SourceRefs = cloneSourceRefs(input.SourceRefs) return input } func canonicalNameGroups(records []normalizedRecord) [][]int { groups := make([][]int, 0, len(records)) ownerByKey := make(map[string]int, len(records)) for index, record := range records { key := identity.ComparisonKey(record.npc.Name) if key != "" { if groupIndex, ok := ownerByKey[key]; ok { groups[groupIndex] = append(groups[groupIndex], index) continue } ownerByKey[key] = len(groups) } groups = append(groups, []int{index}) } return groups } func consolidate(records []normalizedRecord, members []int, order shared.SourceRefOrder) (normalizedRecord, bool) { output := cloneRecord(records[members[0]]) originalRefs := cloneSourceRefs(output.npc.SourceRefs) for _, member := range members[1:] { output.npc.SourceRefs = append(output.npc.SourceRefs, records[member].npc.SourceRefs...) output.inputIndexes = append(output.inputIndexes, records[member].inputIndexes...) if records[member].earliest < output.earliest { output.earliest = records[member].earliest } } output.inputIndexes = sortedUniqueIndexes(output.inputIndexes) output.npc.SourceRefs = order.Canonicalize(output.npc.SourceRefs) output.npc.ID = identity.DeriveID(output.npc.Name) return output, !reflect.DeepEqual(originalRefs, output.npc.SourceRefs) } func cloneRecord(input normalizedRecord) normalizedRecord { input.npc = cloneNPC(input.npc) input.inputIndexes = append([]int(nil), input.inputIndexes...) return input } func memberInputIndexes(records []normalizedRecord, members []int) []int { indexes := make([]int, 0, len(members)) for _, member := range members { indexes = append(indexes, records[member].inputIndexes...) } return sortedUniqueIndexes(indexes) } func sortedUniqueIndexes(indexes []int) []int { if len(indexes) == 0 { return nil } out := append([]int(nil), indexes...) sort.Ints(out) write := 1 for _, index := range out[1:] { if index != out[write-1] { out[write] = index write++ } } return out[:write] } func recordValues(records []normalizedRecord) []dnd.NPC { if records == nil { return nil } values := make([]dnd.NPC, len(records)) for index, record := range records { values[index] = cloneNPC(record.npc) } return values } func recordList(records []normalizedRecord) dnd.NPCList { if records == nil { return dnd.NPCList{} } return dnd.NPCList{NPCs: recordValues(records)} } func cloneSourceRefs(input []source.SourceRef) []source.SourceRef { if input == nil { return nil } return append([]source.SourceRef(nil), input...) } func duplicateWarning(retainedIndex int, removed []int) contracts.Warning { const maxDisplayedIndices = 20 displayed := removed if len(displayed) > maxDisplayedIndices { displayed = displayed[:maxDisplayedIndices] } indices := make([]string, len(displayed)) for index, removedIndex := range displayed { indices[index] = strconv.Itoa(removedIndex) } message := fmt.Sprintf("retained input index %d; removed input indices [%s]", retainedIndex, strings.Join(indices, ", ")) if omitted := len(removed) - len(displayed); omitted > 0 { message += fmt.Sprintf("; %d additional removed input indices omitted", omitted) } return contracts.Warning{Scope: npcScope(retainedIndex), ReasonCode: ReasonCodeDuplicateNPCCollapsed, Message: message} } func npcScope(index int) string { return fmt.Sprintf("npcs[%d]", index) } func ModuleSpec() pipeline.ModuleSpec { return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.NPCListKind} } func Register(registry *pipeline.NormalizerRegistry) error { return pipeline.RegisterNormalizerBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Normalizer[dnd.NPCList], error) { options, err := DecodeOptions(request.Options) if err != nil { return nil, err } return New(request.Dependencies.LLM, options) }) } func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err } func DecodeOptions(options map[string]any) (Options, error) { if err := pipeline.RejectUnknownOptions(options); err != nil { return Options{}, normalizerErrorf("%w", err) } return Options{}, nil } func normalizerErrorf(format string, args ...any) error { return fmt.Errorf("dnd npcs normalizer: "+format, args...) }