Bugfix in the built-in prompt references definition

This commit is contained in:
2026-07-05 18:48:45 -05:00
parent 7d4c027d09
commit 3df686f474
13 changed files with 274 additions and 46 deletions

View File

@@ -252,15 +252,17 @@ The `generic` chunker accepts:
`max_units`.
The `dnd/scenes` chunker requires transcript source capabilities, calls the
configured structured LLM runtime, and does not accept module options.
configured structured LLM runtime, and does not accept module options. It
declares optional `roster` and `glossary` references for scene disambiguation.
The `dnd/spells` extractor declares optional text reference slots:
The `dnd/spells` extractor declares optional reference slots:
- `roster`
- `glossary`
The extractor uses these references only as supporting disambiguation material;
spell casts still must be present in the source transcript.
Both modules accept UTF-8 plain text, Markdown, YAML, or JSON reference files.
The extractor uses references only as supporting disambiguation material; spell
casts still must be present in the source transcript.
## Diagnostics

View File

@@ -157,10 +157,10 @@ metadata under `artifact_lanes[].metadata.extractor`. Durable artifact payload
details belong in the
[D&D spell artifact contract](../integrations/dnd-spell-artifacts.md).
The extractor declares optional `roster` and `glossary` reference slots
accepting UTF-8 text without narrowing accepted media types. Its prompt frames
references as supporting disambiguation material only; spell-cast artifacts must
still be grounded in the source transcript.
The `dnd/scenes` chunker and `dnd/spells` extractor declare optional `roster`
and `glossary` reference slots accepting UTF-8 plain text, Markdown, YAML, or
JSON. Their prompts frame references as supporting disambiguation material only;
spell-cast artifacts must still be grounded in the source transcript.
## D&D Spell Validators

View File

@@ -119,8 +119,8 @@ Symptoms include:
Fix:
- Confirm the selected chunker, extractor, or normalizer declares the slot. The
implemented `dnd/spells` extractor declares optional `roster` and `glossary`
slots.
implemented `dnd/scenes` chunker and `dnd/spells` extractor declare optional
`roster` and `glossary` slots.
- Use a specific selector when more than one selected target declares the same
slot. Examples include `chunk.context=./context.txt`,
`spells.extract.context=./extract-context.txt`, and
@@ -136,7 +136,8 @@ Fix:
- Ensure the file is readable UTF-8 text and within any byte limit declared by
the declaring module.
- If the declaring module narrows accepted media types, use a file extension that
infers an accepted type such as `text/markdown` or `application/json`.
infers an accepted type such as `text/markdown`, `application/yaml`, or
`application/json`.
Unknown extensions infer `application/octet-stream`.
- If diagnostics are retained, inspect `resolved-pipeline.json`,
`resolved-references.json`, and `error.log`.

View File

@@ -175,6 +175,9 @@ func referenceMediaTypeForPath(path string) string {
if extension == ".md" || extension == ".markdown" {
return "text/markdown"
}
if extension == ".yaml" || extension == ".yml" {
return "application/yaml"
}
return unknownMediaType
}
return canonicalMediaType(mediaType)

View File

@@ -249,6 +249,25 @@ func TestMaterializeReferencesAcceptsDeclaredJSONMediaType(t *testing.T) {
}
}
func TestMaterializeReferencesAcceptsDeclaredYAMLMediaType(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "roster.yaml")
writeReferenceFile(t, path, []byte("aria: cleric\n"))
slot := contracts.ReferenceSlot{Name: "roster", AcceptedMediaTypes: []string{"application/yaml"}}
resolved := resolvedPipelineWithReference(t, "roster", "roster.yaml", contracts.ReferenceBindingSourceConfig, slot)
materialized, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
ConfigPath: filepath.Join(configDir, "config.yml"),
})
if err != nil {
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
}
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
if item.MediaType != "application/yaml" {
t.Fatalf("MediaType = %q, want application/yaml", item.MediaType)
}
}
func TestMaterializeReferencesRejectsUnacceptedMediaType(t *testing.T) {
configDir := t.TempDir()
path := filepath.Join(configDir, "roster.json")

View File

@@ -1,10 +1,16 @@
id: dnd.scenes
version: "v1"
default_profile: mistral-small-3
default_profile: gemini-2-flash
inputs:
- name: transcript
required: true
content_type: application/json
- name: roster
required: false
content_type: text/plain
- name: glossary
required: false
content_type: text/plain
messages:
- role: system
content_file: ./shared/system.md
@@ -12,6 +18,10 @@ messages:
content_file: ./shared/transcript.md
cache_control:
type: ephemeral
- role: user
content_file: ./shared/references.md
cache_control:
type: ephemeral
- role: user
content_file: ./dnd/scenes/task.md
- role: user

View File

@@ -21,6 +21,27 @@ var providedCapabilities = []string{
"chunks.scenes",
}
var acceptedReferenceMediaTypes = []string{
"application/json",
"application/x-yaml",
"application/yaml",
"text/markdown",
"text/plain",
}
var referenceSlots = []contracts.ReferenceSlot{
{
Name: "glossary",
Description: "Optional campaign glossary reference material used only for scene disambiguation.",
AcceptedMediaTypes: append([]string(nil), acceptedReferenceMediaTypes...),
},
{
Name: "roster",
Description: "Optional campaign roster or player-character reference material used only for scene disambiguation.",
AcceptedMediaTypes: append([]string(nil), acceptedReferenceMediaTypes...),
},
}
var _ contracts.Chunker = (*Chunker)(nil)
var _ contracts.ManifestMetadataProvider = (*Chunker)(nil)
@@ -35,7 +56,7 @@ func (c *Chunker) Key() string {
}
func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
return cloneReferenceSlots(referenceSlots)
}
func (c *Chunker) ManifestMetadata() map[string]any {
@@ -91,9 +112,7 @@ func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contra
PromptVersion: ResponseSchemaVersion,
ProfileID: req.LLMProfile,
SessionID: req.SessionID,
Inputs: contracts.LLMInputSet{
"transcript": transcriptPromptInput(req.SourceInput),
},
Inputs: promptInputs(req),
}, &response); err != nil {
return contracts.ChunkResult{}, chunkerErrorf("complete structured output: %w", err)
}
@@ -112,18 +131,13 @@ func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contra
}, nil
}
func transcriptPromptInput(material contracts.LLMInputMaterial) contracts.LLMInputMaterial {
out := material.Clone()
out.Name = "transcript"
return out
}
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageChunk,
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
Key: Key,
Stage: pipeline.StageChunk,
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
ReferenceSlots: cloneReferenceSlots(referenceSlots),
}
}
@@ -314,3 +328,15 @@ func cloneMetadata(metadata map[string]any) map[string]any {
func chunkerErrorf(format string, args ...any) error {
return fmt.Errorf("dnd scenes chunker: "+format, args...)
}
func cloneReferenceSlots(slots []contracts.ReferenceSlot) []contracts.ReferenceSlot {
if len(slots) == 0 {
return nil
}
out := make([]contracts.ReferenceSlot, len(slots))
for i, slot := range slots {
slot.AcceptedMediaTypes = append([]string(nil), slot.AcceptedMediaTypes...)
out[i] = slot
}
return out
}

View File

@@ -23,10 +23,11 @@ func TestNewModuleSpecAndRegister(t *testing.T) {
}
want := pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageChunk,
Requires: []string{"source.transcript"},
Provides: []string{"chunks", "chunks.scenes"},
Key: Key,
Stage: pipeline.StageChunk,
Requires: []string{"source.transcript"},
Provides: []string{"chunks", "chunks.scenes"},
ReferenceSlots: wantReferenceSlots(),
}
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
@@ -34,6 +35,7 @@ func TestNewModuleSpecAndRegister(t *testing.T) {
got := ModuleSpec()
got.Requires[0] = "changed"
got.Provides[0] = "changed"
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
if again := ModuleSpec(); !reflect.DeepEqual(again, want) {
t.Fatalf("ModuleSpec() after caller mutation = %#v, want %#v", again, want)
}
@@ -56,8 +58,8 @@ func TestNewModuleSpecAndRegister(t *testing.T) {
if built.Key() != Key {
t.Fatalf("built Key() = %q, want %q", built.Key(), Key)
}
if slots := built.ReferenceSlots(); len(slots) != 0 {
t.Fatalf("ReferenceSlots() = %#v, want none", slots)
if slots := built.ReferenceSlots(); !reflect.DeepEqual(slots, want.ReferenceSlots) {
t.Fatalf("ReferenceSlots() = %#v, want %#v", slots, want.ReferenceSlots)
}
}
@@ -71,6 +73,22 @@ func TestRegisterNilRegistryReturnsError(t *testing.T) {
}
}
func wantReferenceSlots() []contracts.ReferenceSlot {
accepted := []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}
return []contracts.ReferenceSlot{
{
Name: "glossary",
Description: "Optional campaign glossary reference material used only for scene disambiguation.",
AcceptedMediaTypes: append([]string(nil), accepted...),
},
{
Name: "roster",
Description: "Optional campaign roster or player-character reference material used only for scene disambiguation.",
AcceptedMediaTypes: append([]string(nil), accepted...),
},
}
}
func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
client := &fakeScenesLLMClient{
response: chunkResponse{
@@ -128,6 +146,12 @@ func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
if got := string(transcript.Content); got != sceneTranscriptJSON {
t.Fatalf("transcript content = %q, want original source input", got)
}
if got := string(req.Inputs["roster"].Content); got != " " {
t.Fatalf("roster input = %q, want empty reference placeholder", got)
}
if got := string(req.Inputs["glossary"].Content); got != " " {
t.Fatalf("glossary input = %q, want empty reference placeholder", got)
}
if got := chunkIDs(result.Chunks); !reflect.DeepEqual(got, []string{"scene-000001", "scene-000002"}) {
t.Fatalf("chunk IDs = %#v, want deterministic scene IDs", got)
@@ -162,6 +186,54 @@ func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
}
}
func TestChunkPassesReferencesAsPromptInputs(t *testing.T) {
client := &fakeScenesLLMClient{response: chunkResponse{
Scenes: []sceneResponse{
{
StartUnitID: "seg-001",
EndUnitID: "seg-004",
ShortTitle: "Ambush",
PrimaryMode: "Combat",
MainParticipants: []string{"Aria"},
Summary: "The party is ambushed.",
BoundaryNote: "One scene covers the short fixture.",
BoundaryConfidence: "High",
},
},
}}
req := chunkRequestWithClient(client)
req.References = contracts.ReferenceSet{
Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {
Slot: contracts.ReferenceSlot{Name: "roster"},
Items: []contracts.ReferenceItem{
{SlotName: "roster", Content: []byte("Aria: cleric")},
},
},
"glossary": {
Slot: contracts.ReferenceSlot{Name: "glossary"},
Items: []contracts.ReferenceItem{
{SlotName: "glossary", Content: []byte("Brightmantle: local temple")},
},
},
},
}
if _, err := New().Chunk(context.Background(), req); err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
}
request := client.requests[0]
if got := string(request.Inputs["roster"].Content); got != "Aria: cleric" {
t.Fatalf("roster input = %q, want reference content", got)
}
if got := string(request.Inputs["glossary"].Content); got != "Brightmantle: local temple" {
t.Fatalf("glossary input = %q, want reference content", got)
}
if strings.Contains(string(request.Inputs["transcript"].Content), "Aria: cleric") {
t.Fatalf("transcript input contains reference content")
}
}
func TestChunkRejectsWhitespaceOnlyBoundaryCaveats(t *testing.T) {
client := &fakeScenesLLMClient{
response: chunkResponse{

View File

@@ -1,8 +1,12 @@
package scenes
import (
"bytes"
"fmt"
"sort"
"sync"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/promptassets"
)
@@ -16,18 +20,88 @@ func RegisterPromptAssets(registry *llm.AssetRegistry) error {
return registry.RegisterSchemaFS(embeddedAssets, "assets/schemas")
}
func promptInputs(req contracts.ChunkRequest) contracts.LLMInputSet {
return contracts.LLMInputSet{
"transcript": transcriptPromptInput(req.SourceInput),
"roster": referencePromptMaterial("roster", req.References.Slots["roster"]),
"glossary": referencePromptMaterial("glossary", req.References.Slots["glossary"]),
}
}
func transcriptPromptInput(material contracts.LLMInputMaterial) contracts.LLMInputMaterial {
out := material.Clone()
out.Name = "transcript"
return out
}
func referencePromptMaterial(name string, slot contracts.ResolvedReferenceSlot) contracts.LLMInputMaterial {
body := referencePromptInput(slot)
digest := ""
originURI := ""
if len(slot.Items) == 1 {
digest = slot.Items[0].Digest
originURI = slot.Items[0].Origin.URI
}
return contracts.NewLLMInputMaterial(name, "text/plain", body, digest, originURI)
}
func scriptoriumPromptMetadata() (string, error) {
scriptoriumPromptHashOnce.Do(func() {
parts := append([]llm.AssetHashPart{
{FS: embeddedAssets, Path: "assets/scriptorium/prompts/dnd.scenes.yaml"},
{FS: embeddedAssets, Path: "assets/scriptorium/prompts/dnd/scenes/task.md"},
{FS: embeddedAssets, Path: "assets/scriptorium/prompts/dnd/scenes/instructions.md"},
}, promptassets.CommonHashParts()...)
}, append(promptassets.CommonHashParts(), promptassets.ReferenceHashParts()...)...)
scriptoriumPromptHash, scriptoriumPromptHashErr = llm.HashAssets(parts)
})
return scriptoriumPromptHash, scriptoriumPromptHashErr
}
func referencePromptInput(slot contracts.ResolvedReferenceSlot) []byte {
if len(slot.Items) == 0 {
return []byte(" ")
}
items := append([]contracts.ReferenceItem(nil), slot.Items...)
sort.SliceStable(items, func(i, j int) bool {
if items[i].Origin.URI != items[j].Origin.URI {
return items[i].Origin.URI < items[j].Origin.URI
}
if items[i].Digest != items[j].Digest {
return items[i].Digest < items[j].Digest
}
return string(items[i].Content) < string(items[j].Content)
})
if len(items) == 1 {
return append([]byte(nil), items[0].Content...)
}
var b bytes.Buffer
for i, item := range items {
if i > 0 {
b.WriteString("\n\n")
}
fmt.Fprintf(&b, "Reference %d\n", i+1)
if item.Origin.Type != "" {
fmt.Fprintf(&b, "Origin-Type: %s\n", item.Origin.Type)
}
if item.Origin.URI != "" {
fmt.Fprintf(&b, "Origin-URI: %s\n", item.Origin.URI)
}
if item.Digest != "" {
fmt.Fprintf(&b, "Digest: %s\n", item.Digest)
}
if item.MediaType != "" {
fmt.Fprintf(&b, "Media-Type: %s\n", item.MediaType)
}
if item.SizeBytes > 0 {
fmt.Fprintf(&b, "Size-Bytes: %d\n", item.SizeBytes)
}
b.WriteString("\n")
b.Write(item.Content)
}
return b.Bytes()
}
var (
scriptoriumPromptHashOnce sync.Once
scriptoriumPromptHash string

View File

@@ -14,13 +14,13 @@ import (
func TestScriptoriumPromptPreparesTranscriptAndTaskMessages(t *testing.T) {
transcript := []byte(`{"id":"session-1","segments":[{"id":"u1","text":"We enter the crypt."}]}`)
prepared := prepareScenesPrompt(t, transcript)
prepared := prepareScenesPrompt(t, transcript, "Aria: cleric", "Brightmantle: temple")
if prepared.PromptID != PromptID {
t.Fatalf("prompt id = %q, want %q", prepared.PromptID, PromptID)
}
if got := len(prepared.Messages); got != 4 {
t.Fatalf("message count = %d, want 4", got)
if got := len(prepared.Messages); got != 5 {
t.Fatalf("message count = %d, want 5", got)
}
if prepared.Messages[1].Role != "user" || prepared.Messages[1].CacheControl == nil {
t.Fatalf("transcript message did not render as cacheable user message: %#v", prepared.Messages[1])
@@ -29,17 +29,26 @@ func TestScriptoriumPromptPreparesTranscriptAndTaskMessages(t *testing.T) {
if prepared.Messages[1].Content != wantTranscript {
t.Fatalf("transcript message = %q, want byte-identical shared transcript body", prepared.Messages[1].Content)
}
if !strings.Contains(prepared.Messages[2].Content, "Divide the provided transcript") {
t.Fatalf("task message missing scene task text: %q", prepared.Messages[2].Content)
if prepared.Messages[2].CacheControl == nil {
t.Fatalf("reference message did not render as cacheable user message: %#v", prepared.Messages[2])
}
if strings.Contains(prepared.Messages[2].Content, string(transcript)) {
if !strings.Contains(prepared.Messages[2].Content, "Roster reference:\nAria: cleric") {
t.Fatalf("reference message missing roster content: %q", prepared.Messages[2].Content)
}
if !strings.Contains(prepared.Messages[2].Content, "Glossary reference:\nBrightmantle: temple") {
t.Fatalf("reference message missing glossary content: %q", prepared.Messages[2].Content)
}
if !strings.Contains(prepared.Messages[3].Content, "Divide the provided transcript") {
t.Fatalf("task message missing scene task text: %q", prepared.Messages[3].Content)
}
if strings.Contains(prepared.Messages[3].Content, string(transcript)) {
t.Fatalf("task message leaked transcript bytes")
}
}
func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) {
transcript := []byte(`{"secret":"source text"}`)
prepared := prepareScenesPrompt(t, transcript)
prepared := prepareScenesPrompt(t, transcript, "private roster note", "private glossary note")
metadata := New().ManifestMetadata()
payload, err := json.Marshal(map[string]any{
@@ -61,6 +70,8 @@ func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) {
diagnostics := string(payload)
for _, forbidden := range []string{
"source text",
"private roster note",
"private glossary note",
"Divide the provided transcript",
`"properties"`,
"start_unit_id",
@@ -77,7 +88,7 @@ func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) {
}
}
func prepareScenesPrompt(t *testing.T, transcript []byte) *scriptorium.PreparedRun {
func prepareScenesPrompt(t *testing.T, transcript []byte, roster string, glossary string) *scriptorium.PreparedRun {
t.Helper()
registry := llm.NewAssetRegistry()
if err := promptassets.Register(registry); err != nil {
@@ -93,6 +104,8 @@ func prepareScenesPrompt(t *testing.T, transcript []byte) *scriptorium.PreparedR
ProfileID: "scene-test-profile",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///session.json", string(transcript)),
"roster": scriptorium.Inline(roster),
"glossary": scriptorium.Inline(glossary),
},
})
if err != nil {

View File

@@ -1,6 +1,6 @@
id: dnd.spells
version: "v1"
default_profile: mistral-small-3
default_profile: gemini-2-flash
inputs:
- name: transcript
required: true

View File

@@ -25,16 +25,24 @@ var providedCapabilities = []string{
"dnd.spell_casts",
}
var acceptedReferenceMediaTypes = []string{
"application/json",
"application/x-yaml",
"application/yaml",
"text/markdown",
"text/plain",
}
var referenceSlots = []contracts.ReferenceSlot{
{
Name: "glossary",
Description: "Optional campaign glossary reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"text/plain; charset=utf-8"},
AcceptedMediaTypes: append([]string(nil), acceptedReferenceMediaTypes...),
},
{
Name: "roster",
Description: "Optional campaign roster or player-character reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"text/plain; charset=utf-8"},
AcceptedMediaTypes: append([]string(nil), acceptedReferenceMediaTypes...),
},
}

View File

@@ -41,12 +41,12 @@ func TestModuleSpec(t *testing.T) {
{
Name: "glossary",
Description: "Optional campaign glossary reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"text/plain; charset=utf-8"},
AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"},
},
{
Name: "roster",
Description: "Optional campaign roster or player-character reference material used only for disambiguation.",
AcceptedMediaTypes: []string{"text/plain; charset=utf-8"},
AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"},
},
},
}