diff --git a/internal/cli/catalog.go b/internal/cli/catalog.go index d349918..ae75d7e 100644 --- a/internal/cli/catalog.go +++ b/internal/cli/catalog.go @@ -8,9 +8,11 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes" "gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/promptassets" "gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells" "gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim" "gitea.maximumdirect.net/eric/notarius/internal/modules/merge/appendorder" @@ -60,6 +62,20 @@ func productionCatalog() (pipeline.ModuleCatalog, error) { return catalogFromRegistries(registries), nil } +func productionPromptAssets() (*llm.AssetRegistry, error) { + registry := llm.NewAssetRegistry() + if err := promptassets.Register(registry); err != nil { + return nil, fmt.Errorf("register shared dnd prompt assets: %w", err) + } + if err := scenes.RegisterPromptAssets(registry); err != nil { + return nil, fmt.Errorf("register dnd scenes prompt assets: %w", err) + } + if err := spells.RegisterPromptAssets(registry); err != nil { + return nil, fmt.Errorf("register dnd spells prompt assets: %w", err) + } + return registry, nil +} + func effectiveCatalog(opts Options) (pipeline.ModuleCatalog, error) { if !isEmptyCatalog(opts.Catalog) { return opts.Catalog, nil diff --git a/internal/framework/llm/asset_registry.go b/internal/framework/llm/asset_registry.go new file mode 100644 index 0000000..da9343c --- /dev/null +++ b/internal/framework/llm/asset_registry.go @@ -0,0 +1,333 @@ +package llm + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/fs" + "path" + "sort" + "strings" + "time" + + "gitea.maximumdirect.net/eric/scriptorium" +) + +type AssetSource struct { + FS fs.FS + Root string +} + +type AssetRegistry struct { + prompts []AssetSource + schemas []AssetSource +} + +type AssetHashPart struct { + FS fs.FS + Path string +} + +func NewAssetRegistry() *AssetRegistry { + return &AssetRegistry{} +} + +func (r *AssetRegistry) RegisterPromptFS(fsys fs.FS, root string) error { + if r == nil { + return fmt.Errorf("asset registry must not be nil") + } + source, err := newAssetSource(fsys, root) + if err != nil { + return fmt.Errorf("register prompt assets: %w", err) + } + r.prompts = append(r.prompts, source) + return nil +} + +func (r *AssetRegistry) RegisterSchemaFS(fsys fs.FS, root string) error { + if r == nil { + return fmt.Errorf("asset registry must not be nil") + } + source, err := newAssetSource(fsys, root) + if err != nil { + return fmt.Errorf("register schema assets: %w", err) + } + r.schemas = append(r.schemas, source) + return nil +} + +func (r *AssetRegistry) PromptFS() (fs.FS, error) { + if r == nil { + return nil, fmt.Errorf("asset registry must not be nil") + } + return flattenAssetSources(r.prompts) +} + +func (r *AssetRegistry) SchemaFS() (fs.FS, error) { + if r == nil { + return nil, fmt.Errorf("asset registry must not be nil") + } + return flattenAssetSources(r.schemas) +} + +func (r *AssetRegistry) ScriptoriumOptions() ([]scriptorium.Option, error) { + promptFS, err := r.PromptFS() + if err != nil { + return nil, fmt.Errorf("prepare prompt assets: %w", err) + } + schemaFS, err := r.SchemaFS() + if err != nil { + return nil, fmt.Errorf("prepare schema assets: %w", err) + } + return []scriptorium.Option{ + scriptorium.WithPromptFS(promptFS, "."), + scriptorium.WithSchemaFS(schemaFS, "."), + }, nil +} + +func HashAssets(parts []AssetHashPart) (string, error) { + if len(parts) == 0 { + return "", fmt.Errorf("asset hash requires at least one part") + } + hash := sha256.New() + for _, part := range parts { + cleanPath, err := cleanAssetPath(part.Path) + if err != nil { + return "", fmt.Errorf("hash asset %q: %w", part.Path, err) + } + data, err := fs.ReadFile(part.FS, cleanPath) + if err != nil { + return "", fmt.Errorf("read hash asset %s: %w", cleanPath, err) + } + if _, err := io.WriteString(hash, cleanPath); err != nil { + return "", err + } + if _, err := hash.Write([]byte{0}); err != nil { + return "", err + } + if _, err := hash.Write(data); err != nil { + return "", err + } + if _, err := hash.Write([]byte{0}); err != nil { + return "", err + } + } + return "sha256:" + hex.EncodeToString(hash.Sum(nil)), nil +} + +func newAssetSource(fsys fs.FS, root string) (AssetSource, error) { + if fsys == nil { + return AssetSource{}, fmt.Errorf("filesystem must not be nil") + } + cleanRoot, err := cleanAssetRoot(root) + if err != nil { + return AssetSource{}, err + } + return AssetSource{FS: fsys, Root: cleanRoot}, nil +} + +func flattenAssetSources(sources []AssetSource) (fs.FS, error) { + out := assetMapFS{} + for _, source := range sources { + if err := fs.WalkDir(source.FS, source.Root, func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + return nil + } + rel := name + if source.Root != "." { + rel = strings.TrimPrefix(name, source.Root+"/") + } + rel, err := cleanAssetPath(rel) + if err != nil { + return err + } + if _, exists := out[rel]; exists { + return fmt.Errorf("duplicate asset path %q", rel) + } + data, err := fs.ReadFile(source.FS, name) + if err != nil { + return err + } + out[rel] = append([]byte(nil), data...) + return nil + }); err != nil { + return nil, fmt.Errorf("walk asset root %s: %w", source.Root, err) + } + } + return out, nil +} + +func cleanAssetRoot(root string) (string, error) { + trimmed := strings.TrimSpace(root) + if trimmed == "" || trimmed == "." { + return ".", nil + } + return cleanAssetPath(trimmed) +} + +func cleanAssetPath(name string) (string, error) { + trimmed := strings.TrimSpace(name) + if trimmed == "" { + return "", fmt.Errorf("path must not be empty") + } + cleaned := path.Clean(strings.TrimPrefix(trimmed, "/")) + if cleaned == "." || !fs.ValidPath(cleaned) { + return "", fmt.Errorf("invalid path %q", name) + } + return cleaned, nil +} + +type assetMapFS map[string][]byte + +func (m assetMapFS) Open(name string) (fs.File, error) { + cleaned, err := cleanOpenPath(name) + if err != nil { + return nil, &fs.PathError{Op: "open", Path: name, Err: err} + } + if data, ok := m[cleaned]; ok { + return &assetFile{ + reader: bytes.NewReader(data), + info: assetFileInfo{name: path.Base(cleaned), size: int64(len(data))}, + }, nil + } + entries := m.dirEntries(cleaned) + if entries != nil { + return &assetDir{name: path.Base(cleaned), entries: entries}, nil + } + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} +} + +func (m assetMapFS) ReadFile(name string) ([]byte, error) { + cleaned, err := cleanOpenPath(name) + if err != nil { + return nil, &fs.PathError{Op: "readfile", Path: name, Err: err} + } + data, ok := m[cleaned] + if !ok { + return nil, &fs.PathError{Op: "readfile", Path: name, Err: fs.ErrNotExist} + } + return append([]byte(nil), data...), nil +} + +func (m assetMapFS) ReadDir(name string) ([]fs.DirEntry, error) { + cleaned, err := cleanOpenPath(name) + if err != nil { + return nil, &fs.PathError{Op: "readdir", Path: name, Err: err} + } + entries := m.dirEntries(cleaned) + if entries == nil { + return nil, &fs.PathError{Op: "readdir", Path: name, Err: fs.ErrNotExist} + } + return entries, nil +} + +func (m assetMapFS) dirEntries(dir string) []fs.DirEntry { + children := map[string]assetDirEntry{} + prefix := "" + if dir != "." { + prefix = dir + "/" + } + for name, data := range m { + if !strings.HasPrefix(name, prefix) { + continue + } + rest := strings.TrimPrefix(name, prefix) + if rest == "" { + continue + } + childName, _, hasSlash := strings.Cut(rest, "/") + entry := assetDirEntry{name: childName, dir: hasSlash} + if !hasSlash { + entry.size = int64(len(data)) + } + children[childName] = entry + } + if len(children) == 0 { + return nil + } + names := make([]string, 0, len(children)) + for name := range children { + names = append(names, name) + } + sort.Strings(names) + entries := make([]fs.DirEntry, 0, len(names)) + for _, name := range names { + entries = append(entries, children[name]) + } + return entries +} + +func cleanOpenPath(name string) (string, error) { + if name == "." { + return ".", nil + } + return cleanAssetPath(name) +} + +type assetFile struct { + reader *bytes.Reader + info assetFileInfo +} + +func (f *assetFile) Stat() (fs.FileInfo, error) { return f.info, nil } +func (f *assetFile) Read(p []byte) (int, error) { return f.reader.Read(p) } +func (f *assetFile) Close() error { return nil } + +type assetDir struct { + name string + offset int + entries []fs.DirEntry +} + +func (d *assetDir) Stat() (fs.FileInfo, error) { return assetFileInfo{name: d.name, dir: true}, nil } +func (d *assetDir) Read([]byte) (int, error) { return 0, fmt.Errorf("cannot read directory") } +func (d *assetDir) Close() error { return nil } +func (d *assetDir) ReadDir(n int) ([]fs.DirEntry, error) { + if d.offset >= len(d.entries) { + return nil, io.EOF + } + end := len(d.entries) + if n > 0 && d.offset+n < end { + end = d.offset + n + } + out := append([]fs.DirEntry(nil), d.entries[d.offset:end]...) + d.offset = end + return out, nil +} + +type assetDirEntry struct { + name string + dir bool + size int64 +} + +func (e assetDirEntry) Name() string { return e.name } +func (e assetDirEntry) IsDir() bool { return e.dir } +func (e assetDirEntry) Type() fs.FileMode { return e.InfoMode().Type() } +func (e assetDirEntry) Info() (fs.FileInfo, error) { + return assetFileInfo{name: e.name, dir: e.dir, size: e.size}, nil +} +func (e assetDirEntry) InfoMode() fs.FileMode { + if e.dir { + return fs.ModeDir | 0o555 + } + return 0o444 +} + +type assetFileInfo struct { + name string + dir bool + size int64 +} + +func (i assetFileInfo) Name() string { return i.name } +func (i assetFileInfo) Size() int64 { return i.size } +func (i assetFileInfo) Mode() fs.FileMode { return assetDirEntry{dir: i.dir}.InfoMode() } +func (i assetFileInfo) ModTime() time.Time { return time.Time{} } +func (i assetFileInfo) IsDir() bool { return i.dir } +func (i assetFileInfo) Sys() any { return nil } diff --git a/internal/framework/llm/asset_registry_test.go b/internal/framework/llm/asset_registry_test.go new file mode 100644 index 0000000..183ebbe --- /dev/null +++ b/internal/framework/llm/asset_registry_test.go @@ -0,0 +1,167 @@ +package llm + +import ( + "context" + "strings" + "testing" + "testing/fstest" + "time" + + "gitea.maximumdirect.net/eric/scriptorium" +) + +func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) { + registry := NewAssetRegistry() + mustRegisterPromptFS(t, registry, fstest.MapFS{ + "prompts/test.yaml": {Data: []byte(validPromptYAML("schemas/out.json"))}, + "prompts/messages/user.tmpl": {Data: []byte(`Input: {{ input "transcript" }}`)}, + "prompts/messages/task.tmpl": {Data: []byte("Return JSON.")}, + "schemas/ignored/schema.json": {Data: []byte(`{"type":"object"}`)}, + }, "prompts") + mustRegisterSchemaFS(t, registry, fstest.MapFS{ + "root/schemas/out.json": {Data: []byte(`{"type":"object"}`)}, + }, "root") + + engine := newAssetTestEngine(t, registry) + prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ + PromptID: "asset.test", + ProfileID: "asset-test-profile", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.Inline(`{"ok":true}`), + }, + }) + if err != nil { + t.Fatalf("Prepare() error = %v, want nil", err) + } + if got := len(prepared.Messages); got != 2 { + t.Fatalf("message count = %d, want 2", got) + } + if prepared.OutputContract.SchemaPath != "schemas/out.json" { + t.Fatalf("schema path = %q, want schemas/out.json", prepared.OutputContract.SchemaPath) + } +} + +func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) { + registry := NewAssetRegistry() + mustRegisterPromptFS(t, registry, fstest.MapFS{ + "test.yaml": {Data: []byte(validPromptYAML("out.json"))}, + }, ".") + mustRegisterSchemaFS(t, registry, fstest.MapFS{ + "out.json": {Data: []byte(`{"type":"object"}`)}, + }, ".") + + engine := newAssetTestEngine(t, registry) + _, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ + PromptID: "asset.test", + ProfileID: "asset-test-profile", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.Inline(`{"ok":true}`), + }, + }) + if err == nil || !strings.Contains(err.Error(), "content_file") { + t.Fatalf("Prepare() error = %v, want missing content_file error", err) + } +} + +func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) { + registry := NewAssetRegistry() + mustRegisterPromptFS(t, registry, fstest.MapFS{ + "test.yaml": {Data: []byte(validPromptYAML("missing.json"))}, + "messages/user.tmpl": {Data: []byte(`Input: {{ input "transcript" }}`)}, + "messages/task.tmpl": {Data: []byte("Return JSON.")}, + }, ".") + mustRegisterSchemaFS(t, registry, fstest.MapFS{ + "present.json": {Data: []byte(`{"type":"object"}`)}, + }, ".") + + engine := newAssetTestEngine(t, registry) + _, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ + PromptID: "asset.test", + ProfileID: "asset-test-profile", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.Inline(`{"ok":true}`), + }, + }) + if err == nil || !strings.Contains(err.Error(), "missing.json") { + t.Fatalf("Prepare() error = %v, want missing schema error", err) + } +} + +func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) { + registry := NewAssetRegistry() + mustRegisterPromptFS(t, registry, fstest.MapFS{"one/prompt.yaml": {Data: []byte("id: one")}}, "one") + mustRegisterPromptFS(t, registry, fstest.MapFS{"two/prompt.yaml": {Data: []byte("id: two")}}, "two") + + _, err := registry.PromptFS() + if err == nil || !strings.Contains(err.Error(), "duplicate asset path") { + t.Fatalf("PromptFS() error = %v, want duplicate path error", err) + } +} + +func TestHashAssetsOmitsRawAssetContent(t *testing.T) { + hash, err := HashAssets([]AssetHashPart{{ + FS: fstest.MapFS{"prompt.md": {Data: []byte("secret prompt text")}}, + Path: "prompt.md", + }}) + if err != nil { + t.Fatalf("HashAssets() error = %v, want nil", err) + } + if !strings.HasPrefix(hash, "sha256:") { + t.Fatalf("hash = %q, want sha256-prefixed value", hash) + } + if strings.Contains(hash, "secret prompt text") { + t.Fatalf("hash leaked asset content") + } +} + +func newAssetTestEngine(t *testing.T, registry *AssetRegistry) *scriptorium.Engine { + t.Helper() + options, err := registry.ScriptoriumOptions() + if err != nil { + t.Fatalf("ScriptoriumOptions() error = %v, want nil", err) + } + options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{ + ID: "asset-test-profile", + Endpoint: "http://127.0.0.1:1/v1", + Model: "asset-test-model", + }))) + engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...) + if err != nil { + t.Fatalf("NewEngine() error = %v, want nil", err) + } + return engine +} + +func mustRegisterPromptFS(t *testing.T, registry *AssetRegistry, fsys fstest.MapFS, root string) { + t.Helper() + if err := registry.RegisterPromptFS(fsys, root); err != nil { + t.Fatalf("RegisterPromptFS() error = %v, want nil", err) + } +} + +func mustRegisterSchemaFS(t *testing.T, registry *AssetRegistry, fsys fstest.MapFS, root string) { + t.Helper() + if err := registry.RegisterSchemaFS(fsys, root); err != nil { + t.Fatalf("RegisterSchemaFS() error = %v, want nil", err) + } +} + +func validPromptYAML(schemaPath string) string { + return `id: asset.test +version: "v1" +inputs: + - name: transcript + required: true + content_type: application/json +messages: + - role: user + content_file: ./messages/user.tmpl + - role: user + content_file: ./messages/task.tmpl +output: + format: json + validation_mode: json_schema + schema_path: ` + schemaPath + ` + repair_attempts: 0 +` +} diff --git a/internal/modules/chunk/dnd/scenes/assets.go b/internal/modules/chunk/dnd/scenes/assets.go index 6a0779b..1ef7a1e 100644 --- a/internal/modules/chunk/dnd/scenes/assets.go +++ b/internal/modules/chunk/dnd/scenes/assets.go @@ -2,5 +2,5 @@ package scenes import "embed" -//go:embed assets/prompts/*.md assets/schemas/*.json +//go:embed assets/prompts/*.md assets/schemas/*.json assets/scriptorium/prompts/*.yaml assets/scriptorium/prompts/dnd/scenes/*.md var embeddedAssets embed.FS diff --git a/internal/modules/chunk/dnd/scenes/assets/scriptorium/prompts/dnd.scenes.yaml b/internal/modules/chunk/dnd/scenes/assets/scriptorium/prompts/dnd.scenes.yaml new file mode 100644 index 0000000..08f7ef5 --- /dev/null +++ b/internal/modules/chunk/dnd/scenes/assets/scriptorium/prompts/dnd.scenes.yaml @@ -0,0 +1,22 @@ +id: dnd.scenes +version: "v1" +inputs: + - name: transcript + required: true + content_type: application/json +messages: + - role: system + content_file: ./shared/system.md + - role: user + content_file: ./shared/transcript.md + cache_control: + type: ephemeral + - role: user + content_file: ./dnd/scenes/task.md + - role: user + content_file: ./dnd/scenes/instructions.md +output: + format: json + validation_mode: json_schema + schema_path: dnd_scenes.v1.json + repair_attempts: 0 diff --git a/internal/modules/chunk/dnd/scenes/assets/scriptorium/prompts/dnd/scenes/instructions.md b/internal/modules/chunk/dnd/scenes/assets/scriptorium/prompts/dnd/scenes/instructions.md new file mode 100644 index 0000000..78afb03 --- /dev/null +++ b/internal/modules/chunk/dnd/scenes/assets/scriptorium/prompts/dnd/scenes/instructions.md @@ -0,0 +1,52 @@ +Good reasons to start a new scene include: +- the party moves to a new location; +- a combat encounter begins or ends; +- combat changes into a substantially different phase; +- the party shifts between combat, exploration, social interaction, discussion, + planning, travel, rest, or downtime; +- a new NPC, faction, threat, or objective becomes central; +- the party completes one immediate goal and begins another; +- a major table-level rules discussion interrupts and materially changes play. + +Do not start a new scene merely because: +- the speaker changes; +- a new combat round begins; +- a player asks a brief rules question; +- there is a joke, aside, or short table comment; +- a character takes a routine turn; +- the same encounter continues without a meaningful change in situation. + +dnd/scenes boundary policy: +- cover the full provided transcript from the first source unit to the last + source unit; +- return sequential scenes with no gaps; +- do not overlap scenes; +- preserve source-unit order; +- use exact source-unit IDs from the transcript; +- each scene must have start_unit_id and end_unit_id; +- do not include final chunk IDs or chunk indexes. + +For each scene: +- short_title should be brief and factual; +- primary_mode must be Recap, Discussion, Combat, or Narrative; +- main_participants should include only principal characters, NPCs, factions, or + groups involved; +- summary should be factual and compact, usually one to three sentences; +- boundary_note should explain why the scene begins at start_unit_id and ends at + end_unit_id; +- boundary_confidence must be High, Medium, or Low. + +Primary mode guidance: +- Use Recap for opening recap, initiative setup, session framing, or immediate + continuation from prior events. +- Use Discussion when the party is primarily discussing options or choosing a + course of action. +- Use Combat when active combat or combat-resolution mechanics dominate. +- Use Narrative for all other non-combat gameplay, including exploration, social + interactions, shopping, preparation, travel, rest, and downtime. + +In boundary_caveats, list overall caveats about scene divisions. Include scenes +that could reasonably be split differently, combat phases that were kept +together, gradual transitions, or places where map context would have helped. + +Return exactly one JSON object and no explanatory text. diff --git a/internal/modules/chunk/dnd/scenes/assets/scriptorium/prompts/dnd/scenes/task.md b/internal/modules/chunk/dnd/scenes/assets/scriptorium/prompts/dnd/scenes/task.md new file mode 100644 index 0000000..aede806 --- /dev/null +++ b/internal/modules/chunk/dnd/scenes/assets/scriptorium/prompts/dnd/scenes/task.md @@ -0,0 +1,5 @@ +Divide the provided transcript into coherent Dungeons & Dragons scenes for the +dnd/scenes chunk module. + +A scene is a coherent unit of play. Start a new scene when there is a meaningful +change in location, objective, threat, activity, encounter, or mode of play. diff --git a/internal/modules/chunk/dnd/scenes/chunker.go b/internal/modules/chunk/dnd/scenes/chunker.go index 0db7498..7b4194b 100644 --- a/internal/modules/chunk/dnd/scenes/chunker.go +++ b/internal/modules/chunk/dnd/scenes/chunker.go @@ -39,11 +39,14 @@ func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot { } func (c *Chunker) ManifestMetadata() map[string]any { - promptMetadata := scenesPromptBundle.Metadata() + promptSHA, err := scriptoriumPromptMetadata() + if err != nil { + promptSHA = scenesPromptBundle.Metadata().SHA256 + } metadata := map[string]any{ "prompt_id": PromptID, - "prompt_version": promptMetadata.PromptVersion, - "prompt_sha256": promptMetadata.SHA256, + "prompt_version": ResponseSchemaVersion, + "prompt_sha256": promptSHA, "response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID, "response_schema_name": ResponseSchemaName, diff --git a/internal/modules/chunk/dnd/scenes/scriptorium_assets.go b/internal/modules/chunk/dnd/scenes/scriptorium_assets.go new file mode 100644 index 0000000..f635e01 --- /dev/null +++ b/internal/modules/chunk/dnd/scenes/scriptorium_assets.go @@ -0,0 +1,35 @@ +package scenes + +import ( + "sync" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/promptassets" +) + +const scriptoriumPromptRoot = "assets/scriptorium/prompts" + +func RegisterPromptAssets(registry *llm.AssetRegistry) error { + if err := registry.RegisterPromptFS(embeddedAssets, scriptoriumPromptRoot); err != nil { + return err + } + return registry.RegisterSchemaFS(embeddedAssets, "assets/schemas") +} + +func scriptoriumPromptMetadata() (string, error) { + scriptoriumPromptHashOnce.Do(func() { + parts := append([]llm.AssetHashPart{ + {FS: embeddedAssets, Path: "assets/scriptorium/prompts/dnd.scenes.yaml"}, + {FS: embeddedAssets, Path: "assets/scriptorium/prompts/dnd/scenes/task.md"}, + {FS: embeddedAssets, Path: "assets/scriptorium/prompts/dnd/scenes/instructions.md"}, + }, promptassets.CommonHashParts()...) + scriptoriumPromptHash, scriptoriumPromptHashErr = llm.HashAssets(parts) + }) + return scriptoriumPromptHash, scriptoriumPromptHashErr +} + +var ( + scriptoriumPromptHashOnce sync.Once + scriptoriumPromptHash string + scriptoriumPromptHashErr error +) diff --git a/internal/modules/chunk/dnd/scenes/scriptorium_assets_test.go b/internal/modules/chunk/dnd/scenes/scriptorium_assets_test.go new file mode 100644 index 0000000..6f0323a --- /dev/null +++ b/internal/modules/chunk/dnd/scenes/scriptorium_assets_test.go @@ -0,0 +1,120 @@ +package scenes + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/promptassets" + "gitea.maximumdirect.net/eric/scriptorium" +) + +func TestScriptoriumPromptPreparesTranscriptAndTaskMessages(t *testing.T) { + transcript := []byte(`{"id":"session-1","segments":[{"id":"u1","text":"We enter the crypt."}]}`) + prepared := prepareScenesPrompt(t, transcript) + + if prepared.PromptID != PromptID { + t.Fatalf("prompt id = %q, want %q", prepared.PromptID, PromptID) + } + if got := len(prepared.Messages); got != 4 { + t.Fatalf("message count = %d, want 4", got) + } + if prepared.Messages[1].Role != "user" || prepared.Messages[1].CacheControl == nil { + t.Fatalf("transcript message did not render as cacheable user message: %#v", prepared.Messages[1]) + } + wantTranscript := "A transcript of a Dungeons & Dragons gameplay session is provided below.\n\n" + string(transcript) + "\n" + if prepared.Messages[1].Content != wantTranscript { + t.Fatalf("transcript message = %q, want byte-identical shared transcript body", prepared.Messages[1].Content) + } + if !strings.Contains(prepared.Messages[2].Content, "Divide the provided transcript") { + t.Fatalf("task message missing scene task text: %q", prepared.Messages[2].Content) + } + if strings.Contains(prepared.Messages[2].Content, string(transcript)) { + t.Fatalf("task message leaked transcript bytes") + } +} + +func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) { + transcript := []byte(`{"secret":"source text"}`) + prepared := prepareScenesPrompt(t, transcript) + metadata := New().ManifestMetadata() + + payload, err := json.Marshal(map[string]any{ + "prepared": map[string]any{ + "prompt_id": prepared.PromptID, + "prompt_version": prepared.PromptVersion, + "prompt_hash": prepared.PromptHash, + "rendered_prompt_hash": prepared.RenderedPromptHash, + "selected_profile_id": prepared.SelectedProfileID, + "output_contract": prepared.OutputContract, + "input_hashes": prepared.InputHashes, + "effective_model_params": prepared.EffectiveModelParams, + }, + "manifest": metadata, + }) + if err != nil { + t.Fatalf("marshal diagnostics: %v", err) + } + diagnostics := string(payload) + for _, forbidden := range []string{ + "source text", + "Divide the provided transcript", + `"properties"`, + "start_unit_id", + } { + if strings.Contains(diagnostics, forbidden) { + t.Fatalf("diagnostics leaked %q: %s", forbidden, diagnostics) + } + } + if metadata["prompt_id"] != PromptID || metadata["prompt_version"] != ResponseSchemaVersion { + t.Fatalf("manifest prompt metadata = %#v", metadata) + } + if !strings.HasPrefix(metadata["prompt_sha256"].(string), "sha256:") { + t.Fatalf("manifest prompt hash = %#v, want sha256-prefixed", metadata["prompt_sha256"]) + } +} + +func prepareScenesPrompt(t *testing.T, transcript []byte) *scriptorium.PreparedRun { + t.Helper() + registry := llm.NewAssetRegistry() + if err := promptassets.Register(registry); err != nil { + t.Fatalf("register shared prompt assets: %v", err) + } + if err := RegisterPromptAssets(registry); err != nil { + t.Fatalf("register scene prompt assets: %v", err) + } + engine := newScenesScriptoriumEngine(t, registry) + prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ + PromptID: PromptID, + PromptVersion: ResponseSchemaVersion, + ProfileID: "scene-test-profile", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.InlineWithURI("file:///session.json", string(transcript)), + }, + }) + if err != nil { + t.Fatalf("Prepare() error = %v, want nil", err) + } + return prepared +} + +func newScenesScriptoriumEngine(t *testing.T, registry *llm.AssetRegistry) *scriptorium.Engine { + t.Helper() + options, err := registry.ScriptoriumOptions() + if err != nil { + t.Fatalf("ScriptoriumOptions() error = %v, want nil", err) + } + options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{ + ID: "scene-test-profile", + Endpoint: "http://127.0.0.1:1/v1", + Model: "scene-test-model", + }))) + engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...) + if err != nil { + t.Fatalf("NewEngine() error = %v, want nil", err) + } + return engine +} diff --git a/internal/modules/dnd/promptassets/assets.go b/internal/modules/dnd/promptassets/assets.go new file mode 100644 index 0000000..31a4a22 --- /dev/null +++ b/internal/modules/dnd/promptassets/assets.go @@ -0,0 +1,27 @@ +package promptassets + +import ( + "embed" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" +) + +//go:embed assets/prompts/shared/*.md +var embeddedAssets embed.FS + +func Register(registry *llm.AssetRegistry) error { + return registry.RegisterPromptFS(embeddedAssets, "assets/prompts") +} + +func CommonHashParts() []llm.AssetHashPart { + return []llm.AssetHashPart{ + {FS: embeddedAssets, Path: "assets/prompts/shared/system.md"}, + {FS: embeddedAssets, Path: "assets/prompts/shared/transcript.md"}, + } +} + +func ReferenceHashParts() []llm.AssetHashPart { + return []llm.AssetHashPart{ + {FS: embeddedAssets, Path: "assets/prompts/shared/references.md"}, + } +} diff --git a/internal/modules/dnd/promptassets/assets/prompts/shared/references.md b/internal/modules/dnd/promptassets/assets/prompts/shared/references.md new file mode 100644 index 0000000..d6e0b8b --- /dev/null +++ b/internal/modules/dnd/promptassets/assets/prompts/shared/references.md @@ -0,0 +1,9 @@ +Optional reference material for this Dungeons & Dragons campaign is provided +below. Use it only to disambiguate names, aliases, speakers, campaign terms, or +spell names already present in the transcript. + +Roster reference: +{{ input "roster" }} + +Glossary reference: +{{ input "glossary" }} diff --git a/internal/modules/dnd/promptassets/assets/prompts/shared/system.md b/internal/modules/dnd/promptassets/assets/prompts/shared/system.md new file mode 100644 index 0000000..ec3c575 --- /dev/null +++ b/internal/modules/dnd/promptassets/assets/prompts/shared/system.md @@ -0,0 +1,8 @@ +You work with Dungeons & Dragons gameplay transcripts. + +Use only the provided transcript and reference material. Source text may contain +transcription errors, repeated lines, incomplete sentences, and misheard proper +nouns. Reference material, when present, is supporting context only and must not +be treated as a source of extracted events by itself. + +Return only valid JSON matching the configured response schema. diff --git a/internal/modules/dnd/promptassets/assets/prompts/shared/transcript.md b/internal/modules/dnd/promptassets/assets/prompts/shared/transcript.md new file mode 100644 index 0000000..bd15f3e --- /dev/null +++ b/internal/modules/dnd/promptassets/assets/prompts/shared/transcript.md @@ -0,0 +1,3 @@ +A transcript of a Dungeons & Dragons gameplay session is provided below. + +{{ input "transcript" }} diff --git a/internal/modules/extract/dnd/spells/assets.go b/internal/modules/extract/dnd/spells/assets.go index 22e668e..6324bc9 100644 --- a/internal/modules/extract/dnd/spells/assets.go +++ b/internal/modules/extract/dnd/spells/assets.go @@ -2,5 +2,5 @@ package spells import "embed" -//go:embed assets/prompts/*.md assets/schemas/*.json +//go:embed assets/prompts/*.md assets/schemas/*.json assets/scriptorium/prompts/*.yaml assets/scriptorium/prompts/dnd/spells/*.md var embeddedAssets embed.FS diff --git a/internal/modules/extract/dnd/spells/assets/scriptorium/prompts/dnd.spells.yaml b/internal/modules/extract/dnd/spells/assets/scriptorium/prompts/dnd.spells.yaml new file mode 100644 index 0000000..ae1fe44 --- /dev/null +++ b/internal/modules/extract/dnd/spells/assets/scriptorium/prompts/dnd.spells.yaml @@ -0,0 +1,32 @@ +id: dnd.spells +version: "v1" +inputs: + - name: transcript + required: true + content_type: application/json + - name: roster + required: false + content_type: text/plain + - name: glossary + required: false + content_type: text/plain +messages: + - role: system + content_file: ./shared/system.md + - role: user + content_file: ./shared/transcript.md + cache_control: + type: ephemeral + - role: user + content_file: ./shared/references.md + cache_control: + type: ephemeral + - role: user + content_file: ./dnd/spells/task.md + - role: user + content_file: ./dnd/spells/instructions.md +output: + format: json + validation_mode: json_schema + schema_path: dnd_spells.v1.json + repair_attempts: 0 diff --git a/internal/modules/extract/dnd/spells/assets/scriptorium/prompts/dnd/spells/instructions.md b/internal/modules/extract/dnd/spells/assets/scriptorium/prompts/dnd/spells/instructions.md new file mode 100644 index 0000000..15c5958 --- /dev/null +++ b/internal/modules/extract/dnd/spells/assets/scriptorium/prompts/dnd/spells/instructions.md @@ -0,0 +1,11 @@ +Source references must use the source-unit IDs exactly as provided. + +Return only D&D spell-cast artifacts. For each spell cast, identify the in-world +caster, spell name, effect, narrative description, and source references using +source_id, start_unit_id, and end_unit_id. + +Use roster and glossary reference material only to clarify source text. Do not +return spells, casters, or effects that are mentioned only in reference +material. + +Return exactly one JSON object and no explanatory text. diff --git a/internal/modules/extract/dnd/spells/assets/scriptorium/prompts/dnd/spells/task.md b/internal/modules/extract/dnd/spells/assets/scriptorium/prompts/dnd/spells/task.md new file mode 100644 index 0000000..31fd382 --- /dev/null +++ b/internal/modules/extract/dnd/spells/assets/scriptorium/prompts/dnd/spells/task.md @@ -0,0 +1,5 @@ +Extract Dungeons & Dragons spell-cast artifacts from the provided transcript. + +Extract only spell casts that are supported by the transcript. Do not infer +spells from general D&D knowledge or from table chatter that does not identify a +spell being cast. diff --git a/internal/modules/extract/dnd/spells/extractor.go b/internal/modules/extract/dnd/spells/extractor.go index cc96a93..4e12cf4 100644 --- a/internal/modules/extract/dnd/spells/extractor.go +++ b/internal/modules/extract/dnd/spells/extractor.go @@ -63,11 +63,14 @@ func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { } func (e *Extractor) ManifestMetadata() map[string]any { - promptMetadata := spellsPromptBundle.Metadata() + promptSHA, err := scriptoriumPromptMetadata() + if err != nil { + promptSHA = spellsPromptBundle.Metadata().SHA256 + } metadata := map[string]any{ "prompt_id": PromptID, - "prompt_version": promptMetadata.PromptVersion, - "prompt_sha256": promptMetadata.SHA256, + "prompt_version": SchemaVersion, + "prompt_sha256": promptSHA, "response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID, "response_schema_name": ResponseSchemaName, diff --git a/internal/modules/extract/dnd/spells/scriptorium_assets.go b/internal/modules/extract/dnd/spells/scriptorium_assets.go new file mode 100644 index 0000000..9d511ad --- /dev/null +++ b/internal/modules/extract/dnd/spells/scriptorium_assets.go @@ -0,0 +1,84 @@ +package spells + +import ( + "bytes" + "fmt" + "sort" + "sync" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/promptassets" +) + +const scriptoriumPromptRoot = "assets/scriptorium/prompts" + +func RegisterPromptAssets(registry *llm.AssetRegistry) error { + if err := registry.RegisterPromptFS(embeddedAssets, scriptoriumPromptRoot); err != nil { + return err + } + return registry.RegisterSchemaFS(embeddedAssets, "assets/schemas") +} + +func scriptoriumPromptMetadata() (string, error) { + scriptoriumPromptHashOnce.Do(func() { + parts := append([]llm.AssetHashPart{ + {FS: embeddedAssets, Path: "assets/scriptorium/prompts/dnd.spells.yaml"}, + {FS: embeddedAssets, Path: "assets/scriptorium/prompts/dnd/spells/task.md"}, + {FS: embeddedAssets, Path: "assets/scriptorium/prompts/dnd/spells/instructions.md"}, + }, append(promptassets.CommonHashParts(), promptassets.ReferenceHashParts()...)...) + scriptoriumPromptHash, scriptoriumPromptHashErr = llm.HashAssets(parts) + }) + return scriptoriumPromptHash, scriptoriumPromptHashErr +} + +func referencePromptInput(slot contracts.ResolvedReferenceSlot) []byte { + if len(slot.Items) == 0 { + return []byte(" ") + } + items := append([]contracts.ReferenceItem(nil), slot.Items...) + sort.SliceStable(items, func(i, j int) bool { + if items[i].Origin.URI != items[j].Origin.URI { + return items[i].Origin.URI < items[j].Origin.URI + } + if items[i].Digest != items[j].Digest { + return items[i].Digest < items[j].Digest + } + return string(items[i].Content) < string(items[j].Content) + }) + if len(items) == 1 { + return append([]byte(nil), items[0].Content...) + } + + var b bytes.Buffer + for i, item := range items { + if i > 0 { + b.WriteString("\n\n") + } + fmt.Fprintf(&b, "Reference %d\n", i+1) + if item.Origin.Type != "" { + fmt.Fprintf(&b, "Origin-Type: %s\n", item.Origin.Type) + } + if item.Origin.URI != "" { + fmt.Fprintf(&b, "Origin-URI: %s\n", item.Origin.URI) + } + if item.Digest != "" { + fmt.Fprintf(&b, "Digest: %s\n", item.Digest) + } + if item.MediaType != "" { + fmt.Fprintf(&b, "Media-Type: %s\n", item.MediaType) + } + if item.SizeBytes > 0 { + fmt.Fprintf(&b, "Size-Bytes: %d\n", item.SizeBytes) + } + b.WriteString("\n") + b.Write(item.Content) + } + return b.Bytes() +} + +var ( + scriptoriumPromptHashOnce sync.Once + scriptoriumPromptHash string + scriptoriumPromptHashErr error +) diff --git a/internal/modules/extract/dnd/spells/scriptorium_assets_test.go b/internal/modules/extract/dnd/spells/scriptorium_assets_test.go new file mode 100644 index 0000000..005990f --- /dev/null +++ b/internal/modules/extract/dnd/spells/scriptorium_assets_test.go @@ -0,0 +1,180 @@ +package spells + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/promptassets" + "gitea.maximumdirect.net/eric/scriptorium" +) + +func TestScriptoriumPromptPreparesTranscriptReferencesAndTaskMessages(t *testing.T) { + transcript := []byte(`{"id":"session-1","segments":[{"id":"u1","text":"Mira casts shield."}]}`) + prepared := prepareSpellsPrompt(t, transcript, "Mira: wizard", "Shield: abjuration") + + if prepared.PromptID != PromptID { + t.Fatalf("prompt id = %q, want %q", prepared.PromptID, PromptID) + } + if got := len(prepared.Messages); got != 5 { + t.Fatalf("message count = %d, want 5", got) + } + wantTranscript := "A transcript of a Dungeons & Dragons gameplay session is provided below.\n\n" + string(transcript) + "\n" + if prepared.Messages[1].Content != wantTranscript { + t.Fatalf("transcript message = %q, want byte-identical shared transcript body", prepared.Messages[1].Content) + } + if prepared.Messages[1].CacheControl == nil || prepared.Messages[2].CacheControl == nil { + t.Fatalf("expected transcript and reference messages to be cacheable: %#v", prepared.Messages) + } + if !strings.Contains(prepared.Messages[2].Content, "Roster reference:\nMira: wizard") { + t.Fatalf("reference message missing roster content: %q", prepared.Messages[2].Content) + } + if !strings.Contains(prepared.Messages[2].Content, "Glossary reference:\nShield: abjuration") { + t.Fatalf("reference message missing glossary content: %q", prepared.Messages[2].Content) + } + if !strings.Contains(prepared.Messages[3].Content, "Extract Dungeons & Dragons spell-cast artifacts") { + t.Fatalf("task message missing spell task text: %q", prepared.Messages[3].Content) + } + if strings.Contains(prepared.Messages[3].Content, string(transcript)) { + t.Fatalf("task message leaked transcript bytes") + } +} + +func TestScriptoriumPromptPreparesWithMissingOptionalReferences(t *testing.T) { + transcript := []byte(`{"id":"session-1","segments":[]}`) + prepared := prepareSpellsPrompt(t, transcript, " ", " ") + + if !strings.Contains(prepared.Messages[2].Content, "Roster reference:\n ") { + t.Fatalf("reference message did not include empty roster input: %q", prepared.Messages[2].Content) + } + if !strings.Contains(prepared.Messages[2].Content, "Glossary reference:\n ") { + t.Fatalf("reference message did not include empty glossary input: %q", prepared.Messages[2].Content) + } +} + +func TestReferencePromptInputRenderingIsDeterministic(t *testing.T) { + slot := contracts.ResolvedReferenceSlot{ + Items: []contracts.ReferenceItem{ + { + SlotName: "roster", + MediaType: "text/plain", + Content: []byte("second"), + Digest: "sha256:bbb", + Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///b.txt"}, + SizeBytes: 6, + }, + { + SlotName: "roster", + MediaType: "text/plain", + Content: []byte("first"), + Digest: "sha256:aaa", + Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///a.txt"}, + SizeBytes: 5, + }, + }, + } + first := string(referencePromptInput(slot)) + second := string(referencePromptInput(slot)) + if first != second { + t.Fatalf("reference rendering was not deterministic:\nfirst=%q\nsecond=%q", first, second) + } + if !strings.Contains(first, "Reference 1\nOrigin-Type: file\nOrigin-URI: file:///a.txt\nDigest: sha256:aaa") { + t.Fatalf("first reference heading was not stable: %q", first) + } + if strings.Index(first, "first") > strings.Index(first, "second") { + t.Fatalf("references were not sorted deterministically: %q", first) + } +} + +func TestSingleReferencePromptInputKeepsContentOnly(t *testing.T) { + got := string(referencePromptInput(contracts.ResolvedReferenceSlot{ + Items: []contracts.ReferenceItem{{Content: []byte("single reference")}}, + })) + if got != "single reference" { + t.Fatalf("single reference rendering = %q, want raw content only", got) + } +} + +func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) { + transcript := []byte(`{"secret":"source text"}`) + reference := "private roster note" + prepared := prepareSpellsPrompt(t, transcript, reference, " ") + metadata := New().ManifestMetadata() + + payload, err := json.Marshal(map[string]any{ + "prepared": map[string]any{ + "prompt_id": prepared.PromptID, + "prompt_version": prepared.PromptVersion, + "prompt_hash": prepared.PromptHash, + "rendered_prompt_hash": prepared.RenderedPromptHash, + "selected_profile_id": prepared.SelectedProfileID, + "output_contract": prepared.OutputContract, + "input_hashes": prepared.InputHashes, + "effective_model_params": prepared.EffectiveModelParams, + }, + "manifest": metadata, + }) + if err != nil { + t.Fatalf("marshal diagnostics: %v", err) + } + diagnostics := string(payload) + for _, forbidden := range []string{ + "source text", + reference, + "Extract Dungeons & Dragons spell-cast artifacts", + `"properties"`, + "spell_casts", + } { + if strings.Contains(diagnostics, forbidden) { + t.Fatalf("diagnostics leaked %q: %s", forbidden, diagnostics) + } + } + if metadata["prompt_id"] != PromptID || metadata["prompt_version"] != SchemaVersion { + t.Fatalf("manifest prompt metadata = %#v", metadata) + } + if !strings.HasPrefix(metadata["prompt_sha256"].(string), "sha256:") { + t.Fatalf("manifest prompt hash = %#v, want sha256-prefixed", metadata["prompt_sha256"]) + } +} + +func prepareSpellsPrompt(t *testing.T, transcript []byte, roster string, glossary string) *scriptorium.PreparedRun { + t.Helper() + registry := llm.NewAssetRegistry() + if err := promptassets.Register(registry); err != nil { + t.Fatalf("register shared prompt assets: %v", err) + } + if err := RegisterPromptAssets(registry); err != nil { + t.Fatalf("register spell prompt assets: %v", err) + } + options, err := registry.ScriptoriumOptions() + if err != nil { + t.Fatalf("ScriptoriumOptions() error = %v, want nil", err) + } + options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{ + ID: "spell-test-profile", + Endpoint: "http://127.0.0.1:1/v1", + Model: "spell-test-model", + }))) + engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...) + if err != nil { + t.Fatalf("NewEngine() error = %v, want nil", err) + } + prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ + PromptID: PromptID, + PromptVersion: SchemaVersion, + ProfileID: "spell-test-profile", + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.InlineWithURI("file:///session.json", string(transcript)), + "roster": scriptorium.Inline(roster), + "glossary": scriptorium.Inline(glossary), + }, + }) + if err != nil { + t.Fatalf("Prepare() error = %v, want nil", err) + } + return prepared +}