From 3ba2bfd7f6686541f3e8ea4cbf6ee1524678f685 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 21 Jul 2026 14:58:26 +0000 Subject: [PATCH] Finalize D&D prompt integration and documentation --- docs/internal/llm.md | 42 ++-- docs/internal/modules.md | 30 +-- docs/roadmap/dnd.md | 201 +++--------------- .../chunk/scenes/scriptorium_assets_test.go | 42 ++-- .../dnd/normalize/combatturns/normalizer.go | 7 +- .../normalize/combatturns/normalizer_test.go | 2 +- .../modules/dnd/normalize/npcs/normalizer.go | 7 +- .../dnd/normalize/npcs/normalizer_test.go | 10 +- 8 files changed, 105 insertions(+), 236 deletions(-) diff --git a/docs/internal/llm.md b/docs/internal/llm.md index c542d37..fc0948e 100644 --- a/docs/internal/llm.md +++ b/docs/internal/llm.md @@ -98,30 +98,36 @@ D&D prompt content. `internal/framework/promptfs` provides the domain-neutral filesystem composition helper used to combine module-owned files with shared domain prompt fragments. -The D&D spell, NPC, and combat-turn extractors each declare an ordered prompt -asset manifest. The manifest lists the package-owned YAML and Markdown files, -then the exact shared fragments rendered by that prompt; the same ordered list -drives both filesystem mounting and the prompt fingerprint. Shared extraction -evidence and in-world identity messages precede the cacheable transcript and -campaign-reference messages. Spell and combat prompts then render immediate -resolution and the cacheable NPC registry before their lane-specific material; -the NPC prompt renders its task and instructions instead. The identity, -transcript, and campaign-reference messages are ephemeral in all three prompts, -and the NPC registry is ephemeral where it is used by spell and combat prompts. -Unused shared assets are neither mounted nor fingerprinted. +The D&D scene chunker and spell, NPC, and combat-turn extractors each declare an +ordered prompt asset manifest. The manifest lists the package-owned YAML and +Markdown files, then the exact shared fragments rendered by that prompt; the +same ordered list drives both filesystem mounting and the prompt fingerprint. +The three extraction prompts share system, extraction evidence, in-world +identity, transcript, and campaign-reference messages. Spell and combat prompts +then render immediate resolution and the ephemeral NPC registry before their +lane-specific material; the NPC prompt renders its task and instructions +instead. The scene prompt uses system, transcript, and campaign-reference +messages before its task and instructions. Identity, transcript, and +campaign-reference messages are ephemeral in the extraction prompts; the scene +prompt marks transcript and campaign references ephemeral, and the NPC registry +is ephemeral where spell and combat prompts use it. Unused shared assets are +neither mounted nor fingerprinted. Schema helpers load embedded JSON Schema with identity and digest metadata, return defensive copies, and expose a diagnostics map that omits schema bytes. The small framework registry contains only generic test schemas; production schemas remain package-owned. -The spell and combat extractors' package-owned prompts declare their structured -JSON inputs and private response schemas. The combat private schema owns the -transport envelope—required fields, JSON types, nullability, and unknown-field -rejection—while its deterministic validators own semantic constraints such as -enum membership, non-empty values and collections, and positive numbers. The -spell extractor's prompt declares a required `application/json` `spell_catalog` -input and an optional `application/json` `npcs` input. The extractor generates +The spell, NPC, and combat extractors' package-owned prompts declare their +structured JSON inputs and private response schemas. Each private response +schema remains separate from its durable artifact codec schema; this work does +not use shared schema fragments or schema generation. The combat private schema +owns the transport envelope—required fields, JSON types, nullability, and +unknown-field rejection—while its deterministic validators own semantic +constraints such as enum membership, non-empty values and collections, and +positive numbers. The spell extractor's prompt declares a required +`application/json` `spell_catalog` input and an optional `application/json` +`npcs` input. The extractor generates the catalog input from its prepared effective catalog as `{"spell_names":[...]}` using sorted canonical names only. The shared D&D prompt assets include a generic NPC grounding fragment directly diff --git a/docs/internal/modules.md b/docs/internal/modules.md index 2129def..96833b0 100644 --- a/docs/internal/modules.md +++ b/docs/internal/modules.md @@ -48,15 +48,18 @@ package-local embedded assets. Shared filesystem composition belongs in `internal/framework/promptfs`; reusable D&D prompt fragments, reference declarations, prompt-input assembly, and source-unit/citation helpers belong in `internal/modules/dnd/shared`, which also owns bounded D&D diagnostics. The -spell, NPC, and combat-turn extractors use ordered package-local prompt -manifests for both rendering and prompt fingerprinting, so only the shared -fragments each prompt actually renders participate in either operation. Their -common prompt order is system, extraction evidence, in-world identity, -transcript, and campaign references, with ephemeral cache control on identity, -transcript, and references. Spell and combat prompts append immediate -resolution and an ephemeral NPC registry before lane-specific material; the -NPC prompt appends its task and instructions. Stage contracts expose only -Notarius structured-completion types, not Scriptorium public types. +D&D scene chunker and spell, NPC, and combat-turn extractors use ordered +package-local prompt manifests for both rendering and prompt fingerprinting, so +only the shared fragments each prompt actually renders participate in either +operation. The three extraction prompts share the order system, extraction +evidence, in-world identity, transcript, and campaign references, with +ephemeral cache control on identity, transcript, and references. Spell and +combat prompts append immediate resolution and an ephemeral NPC registry before +lane-specific material; the NPC prompt appends its task and instructions. The +scene prompt uses system, transcript, and campaign references before its +lane-specific task and instructions, with ephemeral cache control on transcript +and references. Stage contracts expose only Notarius structured-completion +types, not Scriptorium public types. The shared `ChunkPromptMaterial` helper owns common transcript material preparation for the spell, NPC, and combat-turn extractors. It clones supplied @@ -361,8 +364,9 @@ reject the complete result with bounded, stable index/name diagnostics. The source-reference validator defers malformed shapes, validates every cited range, and reports all range defects through a bounded aggregate while preserving `invalid_source_refs`. The relatedness validator resolves all cited -ranges through the shared document-order traversal, then warns when a -case-insensitive spell name is absent from the cited source text. Invalid shape +ranges through the shared document-order traversal, then warns when a normalized +consecutive spell-name token sequence is absent from the cited source text. +Invalid shape or cited ranges produce no relatedness warnings; the shape and source-reference validators own those defects. @@ -416,7 +420,9 @@ that order, then exposes the matching catalog for resolution. The generic and Seriatim registrars own their production leaf registrations. The D&D registrar owns D&D leaf registrations, typed spell, NPC, and combat default-validator chains, typed append-order specializations, and D&D prompt/schema asset -collection. +collection. Its registration helpers group module, validator, prompt-asset, and +chain composition while retaining artifact-specific merge and clone behavior in +the registrar. Concrete implementation packages do not import generic implementation packages directly. A concrete family's `register` package is its composition diff --git a/docs/roadmap/dnd.md b/docs/roadmap/dnd.md index af80f01..b3510f7 100644 --- a/docs/roadmap/dnd.md +++ b/docs/roadmap/dnd.md @@ -1,180 +1,35 @@ # D&D Module Harmonization And Prompt Reuse -This roadmap records proposed improvements to the spell, NPC, and combat-turn -pipeline lanes. Current implemented behavior remains documented in the -[module internals](../internal/modules.md) and -[LLM runtime internals](../internal/llm.md). The work below is not yet an -implemented contract. +This roadmap tracked harmonization of the spell, NPC, and combat-turn lanes. +The implemented behavior is now owned by [Module Internals](../internal/modules.md) +and [LLM Runtime Internals](../internal/llm.md); this document records status +and the one remaining external prerequisite rather than duplicating those +current-behavior references. -The three lanes already share the same overall decomposition: a typed -extractor, durable codec, merger, normalizer, shape and provenance validators, -domain-specific validation, embedded prompt assets, and central production -registration. Spell catalog resolution, NPC identity and registry support, and -combat normalization invariants are intentional domain differences. The goal -is to remove mechanical drift without hiding those differences behind a broad -generic abstraction. +## Implementation Status -## Target State +The offline/runtime work is complete: -### Prompt Reuse Is A Rendered-Request Invariant +- shared prompt assets have explicit per-prompt manifests that drive both + mounting and prompt fingerprints; +- shared prompt inputs, deterministic chunk material preparation, and cited + source traversal are centralized; +- prompt ordering and cache-control policy are aligned across the D&D + extraction prompts, while scene chunking retains its distinct prompt shape; +- validator prerequisite, bounded-diagnostic, and checkpoint policies are + aligned without changing compatibility-sensitive identifiers; +- production registration remains explicit and typed, with grouped composition, + central default-chain ownership, and artifact-specific merge behavior; +- private LLM response schemas remain package-owned and separate from durable + codec schemas; and +- focused tests and repository-wide test, vet, build, and diff checks pass. -Provider prompt caching depends on the request prefix being byte-for-byte -identical. Similar prose, duplicated files, or equivalent structured values are -not sufficient. Treat message role, content bytes, ordering, cache-control -metadata, input rendering, and any provider-visible separators as part of the -cache identity. +## Deferred evaluation -The target prompt layout has the longest valid common prefix before any -module-specific message: - -- Put shared static system, extraction-evidence, and in-world-identity policy - before dynamic transcript and campaign-reference messages so reuse does not - depend on identical transcript content. -- Keep the shared transcript and campaign-reference messages in the same roles - and order in every D&D extractor. -- Keep cache-control declarations identical on corresponding messages. -- Define one canonical ordering and serialization for shared named inputs. - Continue resolving the deprecated `roster` input into the canonical `party` - input before prompt rendering. -- Keep genuinely universal extraction rules in shared message assets. These - rules include using only transcript units as evidence, treating references as - disambiguation, returning schema-conforming JSON only, using integer - `start_unit_id` and `end_unit_id` values, omitting `source_id` for the mapper - to assign, citing all factual claims, and preferring narrow ranges over broad - ranges that bridge unrelated conversation. -- Use one shared in-world identity message for lanes whose artifacts name - actors or participants. It contains only rules that spells, NPCs, and combat - turns can follow verbatim; artifact-specific identity, alias, target, and - relationship rules remain local. -- Keep rules shared by only a subset in subset-specific shared assets. The - immediate-declaration-and-resolution boundary and NPC-registry grounding are - shared by spells and combat. The NPC extractor does not consume a prior NPC - registry. -- Place subset-specific and module-specific messages only after the longest - useful all-lane prefix. Place schema-specific task instructions last. -- Avoid inserting a nominally shared message when its rendered input or - wording differs by lane. Factor the common bytes into one message and leave - the differences in later messages instead. - -Shared messages are embedded once and referenced by every applicable prompt. -This makes reuse of the canonical asset the easiest way to preserve cache -identity and avoids parallel module-specific copies that can drift. - -### Shared Prompt Ownership And Input Behavior Are Protected - -Focused tests protect shared asset composition and input behavior without -snapshotting the length or exact boundary of the common message prefix. - -- Verify that each applicable prompt mounts and renders the canonical shared - assets rather than module-specific copies. -- Verify common input material identity, including name, media type, content, - digest, origin URI, size, empty-value representation, reference ordering, - and `roster` fallback behavior. -- Test that every prompt fingerprint includes exactly the assets actually - rendered by that prompt. The current grouped reference hash helper should be - replaced or refined so a module does not fingerprint an unused shared asset, - while no used asset is omitted. -- Document the expected shared message ordering and cache-control policy so - later changes are reviewed against the caching intent. -- Prefer behavioral and ownership assertions over message-count, prefix-length, - or full-prompt snapshots. - -### Stable Extraction Preparation Is Centralized - -Move the three identical chunk-input preparation implementations into a D&D -shared helper. The helper should clone the supplied material, fall back to the -chunk content, verify byte equality, and fill the canonical name, media type, -and size without retaining mutable request data. - -Nil context, cancellation, source, chunk, and empty-unit checks remain local so -typed result handling and module error context stay explicit. Do not wrap the -complete LLM call, response DTO mapping, catalog preparation, registry -preparation, or manifest metadata in a generic extractor framework. - -Use the shared helper as the single source of the `transcript` prompt material -so identical input requests cannot drift between lanes. - -### Validator Checkpoint And Composition Policy Is Aligned - -- Add explicit versioned policy checkpoint fingerprints to the spell shape, - source-reference, and source-relatedness validators. Bump a policy version - whenever acceptance, rejection, warning, or diagnostic-selection behavior - changes. -- Shape validators own malformed artifact shape; later validators defer when - shape is invalid. -- Make all source-reference validators collect bounded diagnostics through the - shared D&D diagnostics package rather than mixing first-error and aggregate - behavior. -- Use artifact-qualified reason codes consistently unless a reason code is - intentionally a stable cross-artifact contract. -- Add a compact cross-lane validator contract test covering deterministic - execution class, strict empty options, registration, policy fingerprinting, - prerequisite behavior, and diagnostic bounds. - -### Source-Evidence Traversal Is Shared - -Extract a D&D helper that validates source ranges and returns cited units or -text once, in source-document order, without repeating units covered by -overlapping ranges. Use it from spell, NPC, and combat relatedness validators. - -Keep matching policy artifact-specific: - -- spell matching may use canonical catalog names and aliases; -- NPC matching may use NPC comparison keys and aliases; and -- combat matching may apply separate actor and declaration heuristics. - -Review Unicode normalization, apostrophe handling, word boundaries, short-name -false positives, multiword identities, and overlapping ranges with shared -table-driven fixtures. Relatedness remains a warning heuristic and should not -be presented as proof that every semantic claim is supported. - -## Structural Cleanup - -### Lane Registration - -Reduce repetition in the central D&D registrar with focused registration -helpers grouped by modules, validators, prompt assets, and default chains. -Retain artifact-specific append and deep-clone behavior in the registrar, -central ownership of validator ordering, and explicit typed registration. -Avoid reflection and heterogeneous erased lane descriptors. - -### Naming And Package Conventions - -Adopt consistent conventions for package aliases, module keys, artifact kinds, -prompt IDs, schema IDs, reason codes, policy IDs, metadata fields, and -checkpoint fingerprint names. Compatibility-sensitive identifiers should -change only through an explicit migration; internal aliases can be harmonized -independently. - -### Schemas And Assets - -- Keep private LLM response schemas distinct from durable artifact codec - schemas. They represent different trust and compatibility boundaries. -- Keep source-reference schema definitions package-owned in this work; do not - add schema composition or generation machinery solely to deduplicate them. -- Prefer one embedded shared asset over synchronized copies whenever content - must be identical for prompt caching. -- Make prompt fingerprints derive from an explicit prompt asset manifest, or - from the prepared prompt definition, so message composition and provenance - cannot drift independently. - -Prompt factoring must not be accepted solely because cache reuse improves. -Retain or restore module-specific wording when evaluation shows a meaningful -quality regression. Record cache observations using non-secret request and -usage metadata rather than prompt or transcript payloads. - -## Completion Criteria - -This roadmap is complete when: - -- the three lanes expose the same stable structural conventions while keeping - documented domain differences local; -- shared provider-visible messages are produced from canonical shared assets, - and shared inputs have one deterministic implementation; -- the common prompt ordering is deliberate, documented, and as cache-friendly - as extraction quality permits; -- every validator policy that affects reusable results participates in - checkpoint identity; -- source-reference traversal and diagnostics no longer drift between lanes; -- registration remains explicit and type-safe; and -- focused D&D tests plus repository-wide tests, vetting, and the CLI build pass. +An optional live before/after provider comparison remains deferred because it +requires credentials and a maintained human-reviewed transcript or fixture +set. It is not part of the default test suite or merge gate. If those +prerequisites become available, record only aggregate extraction-review +results, prompt token counts, cache-hit/cache-write metrics, and non-secret +prompt hashes. Never commit transcript content, rendered prompts, credentials, +endpoints, or private reference material. diff --git a/internal/modules/dnd/chunk/scenes/scriptorium_assets_test.go b/internal/modules/dnd/chunk/scenes/scriptorium_assets_test.go index eca2c51..583ca24 100644 --- a/internal/modules/dnd/chunk/scenes/scriptorium_assets_test.go +++ b/internal/modules/dnd/chunk/scenes/scriptorium_assets_test.go @@ -18,29 +18,29 @@ func TestScriptoriumPromptPreparesTranscriptAndTaskMessages(t *testing.T) { if prepared.PromptID != PromptID { t.Fatalf("prompt id = %q, want %q", prepared.PromptID, PromptID) } - if got := len(prepared.Messages); got != 5 { - t.Fatalf("message count = %d, want 5", got) + transcriptMessages := 0 + referenceMessageFound := false + for _, message := range prepared.Messages { + if strings.Contains(message.Content, string(transcript)) { + transcriptMessages++ + if message.Role != "user" || message.CacheControl == nil { + t.Fatalf("transcript message did not render as cacheable user message: %#v", message) + } + } + if strings.Contains(message.Content, "Alice: Aria") && + strings.Contains(message.Content, "Aria: cleric") && + strings.Contains(message.Content, "Brightmantle: temple") { + referenceMessageFound = true + if message.Role != "user" || message.CacheControl == nil { + t.Fatalf("reference message did not render as cacheable user message: %#v", message) + } + } } - if prepared.Messages[1].Role != "user" || prepared.Messages[1].CacheControl == nil { - t.Fatalf("transcript message did not render as cacheable user message: %#v", prepared.Messages[1]) + if transcriptMessages != 1 { + t.Fatalf("raw transcript rendered in %d messages, want exactly one", transcriptMessages) } - if !strings.Contains(prepared.Messages[1].Content, string(transcript)) { - t.Fatalf("transcript message did not include source input") - } - if prepared.Messages[2].CacheControl == nil { - t.Fatalf("reference message did not render as cacheable user message: %#v", prepared.Messages[2]) - } - if !strings.Contains(prepared.Messages[2].Content, "Alice: Aria") { - t.Fatalf("reference message missing player content") - } - if !strings.Contains(prepared.Messages[2].Content, "Aria: cleric") { - t.Fatalf("reference message missing party content") - } - if !strings.Contains(prepared.Messages[2].Content, "Brightmantle: temple") { - t.Fatalf("reference message missing glossary content") - } - if strings.Contains(prepared.Messages[3].Content, string(transcript)) { - t.Fatalf("task message leaked transcript bytes") + if !referenceMessageFound { + t.Fatalf("reference message did not render all supplied reference material") } } diff --git a/internal/modules/dnd/normalize/combatturns/normalizer.go b/internal/modules/dnd/normalize/combatturns/normalizer.go index ec770c1..28ef8e9 100644 --- a/internal/modules/dnd/normalize/combatturns/normalizer.go +++ b/internal/modules/dnd/normalize/combatturns/normalizer.go @@ -19,7 +19,8 @@ import ( const ( Key = "dnd/combat-turns" - NormalizationPolicy = "dnd.combat_turns.normalize.v1" + normalizationPolicy = "dnd.combat_turns.normalize.v1" + NormalizationPolicy = normalizationPolicy ReasonCodeFieldsNormalized = "combat_turn_fields_normalized" ReasonCodeActorCanonicalized = "combat_actor_canonicalized" @@ -71,7 +72,7 @@ func (n *Normalizer) ManifestMetadata() map[string]any { return nil } metadata := map[string]any{ - "normalization_policy": NormalizationPolicy, + "normalization_policy": normalizationPolicy, "identity_policy": identity.Policy, } if n.npcRegistry.Bound() { @@ -86,7 +87,7 @@ func (n *Normalizer) CheckpointFingerprints() []pipeline.CheckpointFingerprint { return nil } fingerprints := []pipeline.CheckpointFingerprint{ - {Name: "normalization_policy", Value: NormalizationPolicy}, + {Name: "normalization_policy", Value: normalizationPolicy}, {Name: "identity_policy", Value: identity.Policy}, } if n.npcRegistry.Bound() { diff --git a/internal/modules/dnd/normalize/combatturns/normalizer_test.go b/internal/modules/dnd/normalize/combatturns/normalizer_test.go index f14a4a8..6030d1d 100644 --- a/internal/modules/dnd/normalize/combatturns/normalizer_test.go +++ b/internal/modules/dnd/normalize/combatturns/normalizer_test.go @@ -189,7 +189,7 @@ func TestNormalizerPreparationMetadataFingerprintsAndModuleContract(t *testing.T if err != nil { t.Fatalf("New() error = %v", err) } - if metadata := unbound.ManifestMetadata(); metadata["npc_registry_digest"] != nil || metadata["npc_count"] != nil || metadata["normalization_policy"] != NormalizationPolicy || metadata["identity_policy"] != identity.Policy { + if metadata := unbound.ManifestMetadata(); metadata["npc_registry_digest"] != nil || metadata["npc_count"] != nil || metadata["normalization_policy"] != normalizationPolicy || metadata["identity_policy"] != identity.Policy { t.Fatalf("unbound metadata = %#v", metadata) } if got := unbound.CheckpointFingerprints(); len(got) != 2 || got[0].Name != "normalization_policy" || got[1].Name != "identity_policy" { diff --git a/internal/modules/dnd/normalize/npcs/normalizer.go b/internal/modules/dnd/normalize/npcs/normalizer.go index d36f2ee..8dbbc8c 100644 --- a/internal/modules/dnd/normalize/npcs/normalizer.go +++ b/internal/modules/dnd/normalize/npcs/normalizer.go @@ -19,7 +19,8 @@ import ( const ( Key = "dnd/npcs" - NormalizationPolicy = "dnd.npcs.normalize.v1" + normalizationPolicy = "dnd.npcs.normalize.v1" + NormalizationPolicy = normalizationPolicy ReasonCodeNPCFieldsNormalized = "npc_fields_normalized" ReasonCodeNPCIDRecomputed = "npc_id_recomputed" @@ -51,7 +52,7 @@ func (n *Normalizer) ManifestMetadata() map[string]any { } return map[string]any{ "identity_policy": identity.Policy, - "normalization_policy": NormalizationPolicy, + "normalization_policy": normalizationPolicy, } } @@ -61,7 +62,7 @@ func (n *Normalizer) CheckpointFingerprints() []pipeline.CheckpointFingerprint { } return []pipeline.CheckpointFingerprint{ {Name: "identity_policy", Value: identity.Policy}, - {Name: "normalization_policy", Value: NormalizationPolicy}, + {Name: "normalization_policy", Value: normalizationPolicy}, } } diff --git a/internal/modules/dnd/normalize/npcs/normalizer_test.go b/internal/modules/dnd/normalize/npcs/normalizer_test.go index 8618c7b..722265f 100644 --- a/internal/modules/dnd/normalize/npcs/normalizer_test.go +++ b/internal/modules/dnd/normalize/npcs/normalizer_test.go @@ -10,7 +10,7 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" - domainidentity "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" + identity "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" ) func TestModuleContractAndIdentity(t *testing.T) { @@ -48,13 +48,13 @@ func TestModuleContractAndIdentity(t *testing.T) { normalizer := New(Options{}) metadata := normalizer.ManifestMetadata() - if metadata["identity_policy"] != domainidentity.Policy || metadata["normalization_policy"] != NormalizationPolicy { + if metadata["identity_policy"] != identity.Policy || metadata["normalization_policy"] != normalizationPolicy { t.Fatalf("metadata = %#v, want identity and normalization policies", metadata) } fingerprints := normalizer.CheckpointFingerprints() wantFingerprints := []pipeline.CheckpointFingerprint{ - {Name: "identity_policy", Value: domainidentity.Policy}, - {Name: "normalization_policy", Value: NormalizationPolicy}, + {Name: "identity_policy", Value: identity.Policy}, + {Name: "normalization_policy", Value: normalizationPolicy}, } if !reflect.DeepEqual(fingerprints, wantFingerprints) { t.Fatalf("fingerprints = %#v, want %#v", fingerprints, wantFingerprints) @@ -104,7 +104,7 @@ func TestNormalizePerRecordFieldsAndEvidence(t *testing.T) { if !reflect.DeepEqual(got.SourceRefs, wantRefs) { t.Fatalf("source refs = %#v, want %#v", got.SourceRefs, wantRefs) } - if got.ID != domainidentity.DeriveID("Lady Ash") { + if got.ID != identity.DeriveID("Lady Ash") { t.Fatalf("ID = %q, want derived ID", got.ID) } if !hasWarning(result.Warnings, ReasonCodeNPCFieldsNormalized, "npcs[0]") ||