Add Scriptorium prompt assets
This commit is contained in:
@@ -8,9 +8,11 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/promptassets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/merge/appendorder"
|
||||
@@ -60,6 +62,20 @@ func productionCatalog() (pipeline.ModuleCatalog, error) {
|
||||
return catalogFromRegistries(registries), nil
|
||||
}
|
||||
|
||||
func productionPromptAssets() (*llm.AssetRegistry, error) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := promptassets.Register(registry); err != nil {
|
||||
return nil, fmt.Errorf("register shared dnd prompt assets: %w", err)
|
||||
}
|
||||
if err := scenes.RegisterPromptAssets(registry); err != nil {
|
||||
return nil, fmt.Errorf("register dnd scenes prompt assets: %w", err)
|
||||
}
|
||||
if err := spells.RegisterPromptAssets(registry); err != nil {
|
||||
return nil, fmt.Errorf("register dnd spells prompt assets: %w", err)
|
||||
}
|
||||
return registry, nil
|
||||
}
|
||||
|
||||
func effectiveCatalog(opts Options) (pipeline.ModuleCatalog, error) {
|
||||
if !isEmptyCatalog(opts.Catalog) {
|
||||
return opts.Catalog, nil
|
||||
|
||||
333
internal/framework/llm/asset_registry.go
Normal file
333
internal/framework/llm/asset_registry.go
Normal file
@@ -0,0 +1,333 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
type AssetSource struct {
|
||||
FS fs.FS
|
||||
Root string
|
||||
}
|
||||
|
||||
type AssetRegistry struct {
|
||||
prompts []AssetSource
|
||||
schemas []AssetSource
|
||||
}
|
||||
|
||||
type AssetHashPart struct {
|
||||
FS fs.FS
|
||||
Path string
|
||||
}
|
||||
|
||||
func NewAssetRegistry() *AssetRegistry {
|
||||
return &AssetRegistry{}
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) RegisterPromptFS(fsys fs.FS, root string) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("asset registry must not be nil")
|
||||
}
|
||||
source, err := newAssetSource(fsys, root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("register prompt assets: %w", err)
|
||||
}
|
||||
r.prompts = append(r.prompts, source)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) RegisterSchemaFS(fsys fs.FS, root string) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("asset registry must not be nil")
|
||||
}
|
||||
source, err := newAssetSource(fsys, root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("register schema assets: %w", err)
|
||||
}
|
||||
r.schemas = append(r.schemas, source)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) PromptFS() (fs.FS, error) {
|
||||
if r == nil {
|
||||
return nil, fmt.Errorf("asset registry must not be nil")
|
||||
}
|
||||
return flattenAssetSources(r.prompts)
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) SchemaFS() (fs.FS, error) {
|
||||
if r == nil {
|
||||
return nil, fmt.Errorf("asset registry must not be nil")
|
||||
}
|
||||
return flattenAssetSources(r.schemas)
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) ScriptoriumOptions() ([]scriptorium.Option, error) {
|
||||
promptFS, err := r.PromptFS()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare prompt assets: %w", err)
|
||||
}
|
||||
schemaFS, err := r.SchemaFS()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare schema assets: %w", err)
|
||||
}
|
||||
return []scriptorium.Option{
|
||||
scriptorium.WithPromptFS(promptFS, "."),
|
||||
scriptorium.WithSchemaFS(schemaFS, "."),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func HashAssets(parts []AssetHashPart) (string, error) {
|
||||
if len(parts) == 0 {
|
||||
return "", fmt.Errorf("asset hash requires at least one part")
|
||||
}
|
||||
hash := sha256.New()
|
||||
for _, part := range parts {
|
||||
cleanPath, err := cleanAssetPath(part.Path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("hash asset %q: %w", part.Path, err)
|
||||
}
|
||||
data, err := fs.ReadFile(part.FS, cleanPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read hash asset %s: %w", cleanPath, err)
|
||||
}
|
||||
if _, err := io.WriteString(hash, cleanPath); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := hash.Write([]byte{0}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := hash.Write(data); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := hash.Write([]byte{0}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return "sha256:" + hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func newAssetSource(fsys fs.FS, root string) (AssetSource, error) {
|
||||
if fsys == nil {
|
||||
return AssetSource{}, fmt.Errorf("filesystem must not be nil")
|
||||
}
|
||||
cleanRoot, err := cleanAssetRoot(root)
|
||||
if err != nil {
|
||||
return AssetSource{}, err
|
||||
}
|
||||
return AssetSource{FS: fsys, Root: cleanRoot}, nil
|
||||
}
|
||||
|
||||
func flattenAssetSources(sources []AssetSource) (fs.FS, error) {
|
||||
out := assetMapFS{}
|
||||
for _, source := range sources {
|
||||
if err := fs.WalkDir(source.FS, source.Root, func(name string, entry fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
rel := name
|
||||
if source.Root != "." {
|
||||
rel = strings.TrimPrefix(name, source.Root+"/")
|
||||
}
|
||||
rel, err := cleanAssetPath(rel)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, exists := out[rel]; exists {
|
||||
return fmt.Errorf("duplicate asset path %q", rel)
|
||||
}
|
||||
data, err := fs.ReadFile(source.FS, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out[rel] = append([]byte(nil), data...)
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("walk asset root %s: %w", source.Root, err)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func cleanAssetRoot(root string) (string, error) {
|
||||
trimmed := strings.TrimSpace(root)
|
||||
if trimmed == "" || trimmed == "." {
|
||||
return ".", nil
|
||||
}
|
||||
return cleanAssetPath(trimmed)
|
||||
}
|
||||
|
||||
func cleanAssetPath(name string) (string, error) {
|
||||
trimmed := strings.TrimSpace(name)
|
||||
if trimmed == "" {
|
||||
return "", fmt.Errorf("path must not be empty")
|
||||
}
|
||||
cleaned := path.Clean(strings.TrimPrefix(trimmed, "/"))
|
||||
if cleaned == "." || !fs.ValidPath(cleaned) {
|
||||
return "", fmt.Errorf("invalid path %q", name)
|
||||
}
|
||||
return cleaned, nil
|
||||
}
|
||||
|
||||
type assetMapFS map[string][]byte
|
||||
|
||||
func (m assetMapFS) Open(name string) (fs.File, error) {
|
||||
cleaned, err := cleanOpenPath(name)
|
||||
if err != nil {
|
||||
return nil, &fs.PathError{Op: "open", Path: name, Err: err}
|
||||
}
|
||||
if data, ok := m[cleaned]; ok {
|
||||
return &assetFile{
|
||||
reader: bytes.NewReader(data),
|
||||
info: assetFileInfo{name: path.Base(cleaned), size: int64(len(data))},
|
||||
}, nil
|
||||
}
|
||||
entries := m.dirEntries(cleaned)
|
||||
if entries != nil {
|
||||
return &assetDir{name: path.Base(cleaned), entries: entries}, nil
|
||||
}
|
||||
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
|
||||
}
|
||||
|
||||
func (m assetMapFS) ReadFile(name string) ([]byte, error) {
|
||||
cleaned, err := cleanOpenPath(name)
|
||||
if err != nil {
|
||||
return nil, &fs.PathError{Op: "readfile", Path: name, Err: err}
|
||||
}
|
||||
data, ok := m[cleaned]
|
||||
if !ok {
|
||||
return nil, &fs.PathError{Op: "readfile", Path: name, Err: fs.ErrNotExist}
|
||||
}
|
||||
return append([]byte(nil), data...), nil
|
||||
}
|
||||
|
||||
func (m assetMapFS) ReadDir(name string) ([]fs.DirEntry, error) {
|
||||
cleaned, err := cleanOpenPath(name)
|
||||
if err != nil {
|
||||
return nil, &fs.PathError{Op: "readdir", Path: name, Err: err}
|
||||
}
|
||||
entries := m.dirEntries(cleaned)
|
||||
if entries == nil {
|
||||
return nil, &fs.PathError{Op: "readdir", Path: name, Err: fs.ErrNotExist}
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func (m assetMapFS) dirEntries(dir string) []fs.DirEntry {
|
||||
children := map[string]assetDirEntry{}
|
||||
prefix := ""
|
||||
if dir != "." {
|
||||
prefix = dir + "/"
|
||||
}
|
||||
for name, data := range m {
|
||||
if !strings.HasPrefix(name, prefix) {
|
||||
continue
|
||||
}
|
||||
rest := strings.TrimPrefix(name, prefix)
|
||||
if rest == "" {
|
||||
continue
|
||||
}
|
||||
childName, _, hasSlash := strings.Cut(rest, "/")
|
||||
entry := assetDirEntry{name: childName, dir: hasSlash}
|
||||
if !hasSlash {
|
||||
entry.size = int64(len(data))
|
||||
}
|
||||
children[childName] = entry
|
||||
}
|
||||
if len(children) == 0 {
|
||||
return nil
|
||||
}
|
||||
names := make([]string, 0, len(children))
|
||||
for name := range children {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
entries := make([]fs.DirEntry, 0, len(names))
|
||||
for _, name := range names {
|
||||
entries = append(entries, children[name])
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func cleanOpenPath(name string) (string, error) {
|
||||
if name == "." {
|
||||
return ".", nil
|
||||
}
|
||||
return cleanAssetPath(name)
|
||||
}
|
||||
|
||||
type assetFile struct {
|
||||
reader *bytes.Reader
|
||||
info assetFileInfo
|
||||
}
|
||||
|
||||
func (f *assetFile) Stat() (fs.FileInfo, error) { return f.info, nil }
|
||||
func (f *assetFile) Read(p []byte) (int, error) { return f.reader.Read(p) }
|
||||
func (f *assetFile) Close() error { return nil }
|
||||
|
||||
type assetDir struct {
|
||||
name string
|
||||
offset int
|
||||
entries []fs.DirEntry
|
||||
}
|
||||
|
||||
func (d *assetDir) Stat() (fs.FileInfo, error) { return assetFileInfo{name: d.name, dir: true}, nil }
|
||||
func (d *assetDir) Read([]byte) (int, error) { return 0, fmt.Errorf("cannot read directory") }
|
||||
func (d *assetDir) Close() error { return nil }
|
||||
func (d *assetDir) ReadDir(n int) ([]fs.DirEntry, error) {
|
||||
if d.offset >= len(d.entries) {
|
||||
return nil, io.EOF
|
||||
}
|
||||
end := len(d.entries)
|
||||
if n > 0 && d.offset+n < end {
|
||||
end = d.offset + n
|
||||
}
|
||||
out := append([]fs.DirEntry(nil), d.entries[d.offset:end]...)
|
||||
d.offset = end
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type assetDirEntry struct {
|
||||
name string
|
||||
dir bool
|
||||
size int64
|
||||
}
|
||||
|
||||
func (e assetDirEntry) Name() string { return e.name }
|
||||
func (e assetDirEntry) IsDir() bool { return e.dir }
|
||||
func (e assetDirEntry) Type() fs.FileMode { return e.InfoMode().Type() }
|
||||
func (e assetDirEntry) Info() (fs.FileInfo, error) {
|
||||
return assetFileInfo{name: e.name, dir: e.dir, size: e.size}, nil
|
||||
}
|
||||
func (e assetDirEntry) InfoMode() fs.FileMode {
|
||||
if e.dir {
|
||||
return fs.ModeDir | 0o555
|
||||
}
|
||||
return 0o444
|
||||
}
|
||||
|
||||
type assetFileInfo struct {
|
||||
name string
|
||||
dir bool
|
||||
size int64
|
||||
}
|
||||
|
||||
func (i assetFileInfo) Name() string { return i.name }
|
||||
func (i assetFileInfo) Size() int64 { return i.size }
|
||||
func (i assetFileInfo) Mode() fs.FileMode { return assetDirEntry{dir: i.dir}.InfoMode() }
|
||||
func (i assetFileInfo) ModTime() time.Time { return time.Time{} }
|
||||
func (i assetFileInfo) IsDir() bool { return i.dir }
|
||||
func (i assetFileInfo) Sys() any { return nil }
|
||||
167
internal/framework/llm/asset_registry_test.go
Normal file
167
internal/framework/llm/asset_registry_test.go
Normal file
@@ -0,0 +1,167 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{
|
||||
"prompts/test.yaml": {Data: []byte(validPromptYAML("schemas/out.json"))},
|
||||
"prompts/messages/user.tmpl": {Data: []byte(`Input: {{ input "transcript" }}`)},
|
||||
"prompts/messages/task.tmpl": {Data: []byte("Return JSON.")},
|
||||
"schemas/ignored/schema.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, "prompts")
|
||||
mustRegisterSchemaFS(t, registry, fstest.MapFS{
|
||||
"root/schemas/out.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, "root")
|
||||
|
||||
engine := newAssetTestEngine(t, registry)
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "asset.test",
|
||||
ProfileID: "asset-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline(`{"ok":true}`),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
if got := len(prepared.Messages); got != 2 {
|
||||
t.Fatalf("message count = %d, want 2", got)
|
||||
}
|
||||
if prepared.OutputContract.SchemaPath != "schemas/out.json" {
|
||||
t.Fatalf("schema path = %q, want schemas/out.json", prepared.OutputContract.SchemaPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{
|
||||
"test.yaml": {Data: []byte(validPromptYAML("out.json"))},
|
||||
}, ".")
|
||||
mustRegisterSchemaFS(t, registry, fstest.MapFS{
|
||||
"out.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, ".")
|
||||
|
||||
engine := newAssetTestEngine(t, registry)
|
||||
_, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "asset.test",
|
||||
ProfileID: "asset-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline(`{"ok":true}`),
|
||||
},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "content_file") {
|
||||
t.Fatalf("Prepare() error = %v, want missing content_file error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{
|
||||
"test.yaml": {Data: []byte(validPromptYAML("missing.json"))},
|
||||
"messages/user.tmpl": {Data: []byte(`Input: {{ input "transcript" }}`)},
|
||||
"messages/task.tmpl": {Data: []byte("Return JSON.")},
|
||||
}, ".")
|
||||
mustRegisterSchemaFS(t, registry, fstest.MapFS{
|
||||
"present.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, ".")
|
||||
|
||||
engine := newAssetTestEngine(t, registry)
|
||||
_, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "asset.test",
|
||||
ProfileID: "asset-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline(`{"ok":true}`),
|
||||
},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "missing.json") {
|
||||
t.Fatalf("Prepare() error = %v, want missing schema error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{"one/prompt.yaml": {Data: []byte("id: one")}}, "one")
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{"two/prompt.yaml": {Data: []byte("id: two")}}, "two")
|
||||
|
||||
_, err := registry.PromptFS()
|
||||
if err == nil || !strings.Contains(err.Error(), "duplicate asset path") {
|
||||
t.Fatalf("PromptFS() error = %v, want duplicate path error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHashAssetsOmitsRawAssetContent(t *testing.T) {
|
||||
hash, err := HashAssets([]AssetHashPart{{
|
||||
FS: fstest.MapFS{"prompt.md": {Data: []byte("secret prompt text")}},
|
||||
Path: "prompt.md",
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("HashAssets() error = %v, want nil", err)
|
||||
}
|
||||
if !strings.HasPrefix(hash, "sha256:") {
|
||||
t.Fatalf("hash = %q, want sha256-prefixed value", hash)
|
||||
}
|
||||
if strings.Contains(hash, "secret prompt text") {
|
||||
t.Fatalf("hash leaked asset content")
|
||||
}
|
||||
}
|
||||
|
||||
func newAssetTestEngine(t *testing.T, registry *AssetRegistry) *scriptorium.Engine {
|
||||
t.Helper()
|
||||
options, err := registry.ScriptoriumOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
ID: "asset-test-profile",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
Model: "asset-test-model",
|
||||
})))
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func mustRegisterPromptFS(t *testing.T, registry *AssetRegistry, fsys fstest.MapFS, root string) {
|
||||
t.Helper()
|
||||
if err := registry.RegisterPromptFS(fsys, root); err != nil {
|
||||
t.Fatalf("RegisterPromptFS() error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRegisterSchemaFS(t *testing.T, registry *AssetRegistry, fsys fstest.MapFS, root string) {
|
||||
t.Helper()
|
||||
if err := registry.RegisterSchemaFS(fsys, root); err != nil {
|
||||
t.Fatalf("RegisterSchemaFS() error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func validPromptYAML(schemaPath string) string {
|
||||
return `id: asset.test
|
||||
version: "v1"
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./messages/user.tmpl
|
||||
- role: user
|
||||
content_file: ./messages/task.tmpl
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: ` + schemaPath + `
|
||||
repair_attempts: 0
|
||||
`
|
||||
}
|
||||
@@ -2,5 +2,5 @@ package scenes
|
||||
|
||||
import "embed"
|
||||
|
||||
//go:embed assets/prompts/*.md assets/schemas/*.json
|
||||
//go:embed assets/prompts/*.md assets/schemas/*.json assets/scriptorium/prompts/*.yaml assets/scriptorium/prompts/dnd/scenes/*.md
|
||||
var embeddedAssets embed.FS
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
id: dnd.scenes
|
||||
version: "v1"
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./shared/system.md
|
||||
- role: user
|
||||
content_file: ./shared/transcript.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./dnd/scenes/task.md
|
||||
- role: user
|
||||
content_file: ./dnd/scenes/instructions.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_scenes.v1.json
|
||||
repair_attempts: 0
|
||||
@@ -0,0 +1,52 @@
|
||||
Good reasons to start a new scene include:
|
||||
- the party moves to a new location;
|
||||
- a combat encounter begins or ends;
|
||||
- combat changes into a substantially different phase;
|
||||
- the party shifts between combat, exploration, social interaction, discussion,
|
||||
planning, travel, rest, or downtime;
|
||||
- a new NPC, faction, threat, or objective becomes central;
|
||||
- the party completes one immediate goal and begins another;
|
||||
- a major table-level rules discussion interrupts and materially changes play.
|
||||
|
||||
Do not start a new scene merely because:
|
||||
- the speaker changes;
|
||||
- a new combat round begins;
|
||||
- a player asks a brief rules question;
|
||||
- there is a joke, aside, or short table comment;
|
||||
- a character takes a routine turn;
|
||||
- the same encounter continues without a meaningful change in situation.
|
||||
|
||||
dnd/scenes boundary policy:
|
||||
- cover the full provided transcript from the first source unit to the last
|
||||
source unit;
|
||||
- return sequential scenes with no gaps;
|
||||
- do not overlap scenes;
|
||||
- preserve source-unit order;
|
||||
- use exact source-unit IDs from the transcript;
|
||||
- each scene must have start_unit_id and end_unit_id;
|
||||
- do not include final chunk IDs or chunk indexes.
|
||||
|
||||
For each scene:
|
||||
- short_title should be brief and factual;
|
||||
- primary_mode must be Recap, Discussion, Combat, or Narrative;
|
||||
- main_participants should include only principal characters, NPCs, factions, or
|
||||
groups involved;
|
||||
- summary should be factual and compact, usually one to three sentences;
|
||||
- boundary_note should explain why the scene begins at start_unit_id and ends at
|
||||
end_unit_id;
|
||||
- boundary_confidence must be High, Medium, or Low.
|
||||
|
||||
Primary mode guidance:
|
||||
- Use Recap for opening recap, initiative setup, session framing, or immediate
|
||||
continuation from prior events.
|
||||
- Use Discussion when the party is primarily discussing options or choosing a
|
||||
course of action.
|
||||
- Use Combat when active combat or combat-resolution mechanics dominate.
|
||||
- Use Narrative for all other non-combat gameplay, including exploration, social
|
||||
interactions, shopping, preparation, travel, rest, and downtime.
|
||||
|
||||
In boundary_caveats, list overall caveats about scene divisions. Include scenes
|
||||
that could reasonably be split differently, combat phases that were kept
|
||||
together, gradual transitions, or places where map context would have helped.
|
||||
|
||||
Return exactly one JSON object and no explanatory text.
|
||||
@@ -0,0 +1,5 @@
|
||||
Divide the provided transcript into coherent Dungeons & Dragons scenes for the
|
||||
dnd/scenes chunk module.
|
||||
|
||||
A scene is a coherent unit of play. Start a new scene when there is a meaningful
|
||||
change in location, objective, threat, activity, encounter, or mode of play.
|
||||
@@ -39,11 +39,14 @@ func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
}
|
||||
|
||||
func (c *Chunker) ManifestMetadata() map[string]any {
|
||||
promptMetadata := scenesPromptBundle.Metadata()
|
||||
promptSHA, err := scriptoriumPromptMetadata()
|
||||
if err != nil {
|
||||
promptSHA = scenesPromptBundle.Metadata().SHA256
|
||||
}
|
||||
metadata := map[string]any{
|
||||
"prompt_id": PromptID,
|
||||
"prompt_version": promptMetadata.PromptVersion,
|
||||
"prompt_sha256": promptMetadata.SHA256,
|
||||
"prompt_version": ResponseSchemaVersion,
|
||||
"prompt_sha256": promptSHA,
|
||||
"response_schema_key": string(ResponseSchemaKey),
|
||||
"response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName,
|
||||
|
||||
35
internal/modules/chunk/dnd/scenes/scriptorium_assets.go
Normal file
35
internal/modules/chunk/dnd/scenes/scriptorium_assets.go
Normal file
@@ -0,0 +1,35 @@
|
||||
package scenes
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/promptassets"
|
||||
)
|
||||
|
||||
const scriptoriumPromptRoot = "assets/scriptorium/prompts"
|
||||
|
||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
if err := registry.RegisterPromptFS(embeddedAssets, scriptoriumPromptRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
return registry.RegisterSchemaFS(embeddedAssets, "assets/schemas")
|
||||
}
|
||||
|
||||
func scriptoriumPromptMetadata() (string, error) {
|
||||
scriptoriumPromptHashOnce.Do(func() {
|
||||
parts := append([]llm.AssetHashPart{
|
||||
{FS: embeddedAssets, Path: "assets/scriptorium/prompts/dnd.scenes.yaml"},
|
||||
{FS: embeddedAssets, Path: "assets/scriptorium/prompts/dnd/scenes/task.md"},
|
||||
{FS: embeddedAssets, Path: "assets/scriptorium/prompts/dnd/scenes/instructions.md"},
|
||||
}, promptassets.CommonHashParts()...)
|
||||
scriptoriumPromptHash, scriptoriumPromptHashErr = llm.HashAssets(parts)
|
||||
})
|
||||
return scriptoriumPromptHash, scriptoriumPromptHashErr
|
||||
}
|
||||
|
||||
var (
|
||||
scriptoriumPromptHashOnce sync.Once
|
||||
scriptoriumPromptHash string
|
||||
scriptoriumPromptHashErr error
|
||||
)
|
||||
120
internal/modules/chunk/dnd/scenes/scriptorium_assets_test.go
Normal file
120
internal/modules/chunk/dnd/scenes/scriptorium_assets_test.go
Normal file
@@ -0,0 +1,120 @@
|
||||
package scenes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/promptassets"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
func TestScriptoriumPromptPreparesTranscriptAndTaskMessages(t *testing.T) {
|
||||
transcript := []byte(`{"id":"session-1","segments":[{"id":"u1","text":"We enter the crypt."}]}`)
|
||||
prepared := prepareScenesPrompt(t, transcript)
|
||||
|
||||
if prepared.PromptID != PromptID {
|
||||
t.Fatalf("prompt id = %q, want %q", prepared.PromptID, PromptID)
|
||||
}
|
||||
if got := len(prepared.Messages); got != 4 {
|
||||
t.Fatalf("message count = %d, want 4", got)
|
||||
}
|
||||
if prepared.Messages[1].Role != "user" || prepared.Messages[1].CacheControl == nil {
|
||||
t.Fatalf("transcript message did not render as cacheable user message: %#v", prepared.Messages[1])
|
||||
}
|
||||
wantTranscript := "A transcript of a Dungeons & Dragons gameplay session is provided below.\n\n" + string(transcript) + "\n"
|
||||
if prepared.Messages[1].Content != wantTranscript {
|
||||
t.Fatalf("transcript message = %q, want byte-identical shared transcript body", prepared.Messages[1].Content)
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[2].Content, "Divide the provided transcript") {
|
||||
t.Fatalf("task message missing scene task text: %q", prepared.Messages[2].Content)
|
||||
}
|
||||
if strings.Contains(prepared.Messages[2].Content, string(transcript)) {
|
||||
t.Fatalf("task message leaked transcript bytes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) {
|
||||
transcript := []byte(`{"secret":"source text"}`)
|
||||
prepared := prepareScenesPrompt(t, transcript)
|
||||
metadata := New().ManifestMetadata()
|
||||
|
||||
payload, err := json.Marshal(map[string]any{
|
||||
"prepared": map[string]any{
|
||||
"prompt_id": prepared.PromptID,
|
||||
"prompt_version": prepared.PromptVersion,
|
||||
"prompt_hash": prepared.PromptHash,
|
||||
"rendered_prompt_hash": prepared.RenderedPromptHash,
|
||||
"selected_profile_id": prepared.SelectedProfileID,
|
||||
"output_contract": prepared.OutputContract,
|
||||
"input_hashes": prepared.InputHashes,
|
||||
"effective_model_params": prepared.EffectiveModelParams,
|
||||
},
|
||||
"manifest": metadata,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal diagnostics: %v", err)
|
||||
}
|
||||
diagnostics := string(payload)
|
||||
for _, forbidden := range []string{
|
||||
"source text",
|
||||
"Divide the provided transcript",
|
||||
`"properties"`,
|
||||
"start_unit_id",
|
||||
} {
|
||||
if strings.Contains(diagnostics, forbidden) {
|
||||
t.Fatalf("diagnostics leaked %q: %s", forbidden, diagnostics)
|
||||
}
|
||||
}
|
||||
if metadata["prompt_id"] != PromptID || metadata["prompt_version"] != ResponseSchemaVersion {
|
||||
t.Fatalf("manifest prompt metadata = %#v", metadata)
|
||||
}
|
||||
if !strings.HasPrefix(metadata["prompt_sha256"].(string), "sha256:") {
|
||||
t.Fatalf("manifest prompt hash = %#v, want sha256-prefixed", metadata["prompt_sha256"])
|
||||
}
|
||||
}
|
||||
|
||||
func prepareScenesPrompt(t *testing.T, transcript []byte) *scriptorium.PreparedRun {
|
||||
t.Helper()
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := promptassets.Register(registry); err != nil {
|
||||
t.Fatalf("register shared prompt assets: %v", err)
|
||||
}
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("register scene prompt assets: %v", err)
|
||||
}
|
||||
engine := newScenesScriptoriumEngine(t, registry)
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: PromptID,
|
||||
PromptVersion: ResponseSchemaVersion,
|
||||
ProfileID: "scene-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.InlineWithURI("file:///session.json", string(transcript)),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
return prepared
|
||||
}
|
||||
|
||||
func newScenesScriptoriumEngine(t *testing.T, registry *llm.AssetRegistry) *scriptorium.Engine {
|
||||
t.Helper()
|
||||
options, err := registry.ScriptoriumOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
ID: "scene-test-profile",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
Model: "scene-test-model",
|
||||
})))
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
27
internal/modules/dnd/promptassets/assets.go
Normal file
27
internal/modules/dnd/promptassets/assets.go
Normal file
@@ -0,0 +1,27 @@
|
||||
package promptassets
|
||||
|
||||
import (
|
||||
"embed"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
)
|
||||
|
||||
//go:embed assets/prompts/shared/*.md
|
||||
var embeddedAssets embed.FS
|
||||
|
||||
func Register(registry *llm.AssetRegistry) error {
|
||||
return registry.RegisterPromptFS(embeddedAssets, "assets/prompts")
|
||||
}
|
||||
|
||||
func CommonHashParts() []llm.AssetHashPart {
|
||||
return []llm.AssetHashPart{
|
||||
{FS: embeddedAssets, Path: "assets/prompts/shared/system.md"},
|
||||
{FS: embeddedAssets, Path: "assets/prompts/shared/transcript.md"},
|
||||
}
|
||||
}
|
||||
|
||||
func ReferenceHashParts() []llm.AssetHashPart {
|
||||
return []llm.AssetHashPart{
|
||||
{FS: embeddedAssets, Path: "assets/prompts/shared/references.md"},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
Optional reference material for this Dungeons & Dragons campaign is provided
|
||||
below. Use it only to disambiguate names, aliases, speakers, campaign terms, or
|
||||
spell names already present in the transcript.
|
||||
|
||||
Roster reference:
|
||||
{{ input "roster" }}
|
||||
|
||||
Glossary reference:
|
||||
{{ input "glossary" }}
|
||||
@@ -0,0 +1,8 @@
|
||||
You work with Dungeons & Dragons gameplay transcripts.
|
||||
|
||||
Use only the provided transcript and reference material. Source text may contain
|
||||
transcription errors, repeated lines, incomplete sentences, and misheard proper
|
||||
nouns. Reference material, when present, is supporting context only and must not
|
||||
be treated as a source of extracted events by itself.
|
||||
|
||||
Return only valid JSON matching the configured response schema.
|
||||
@@ -0,0 +1,3 @@
|
||||
A transcript of a Dungeons & Dragons gameplay session is provided below.
|
||||
|
||||
{{ input "transcript" }}
|
||||
@@ -2,5 +2,5 @@ package spells
|
||||
|
||||
import "embed"
|
||||
|
||||
//go:embed assets/prompts/*.md assets/schemas/*.json
|
||||
//go:embed assets/prompts/*.md assets/schemas/*.json assets/scriptorium/prompts/*.yaml assets/scriptorium/prompts/dnd/spells/*.md
|
||||
var embeddedAssets embed.FS
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
id: dnd.spells
|
||||
version: "v1"
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
- name: roster
|
||||
required: false
|
||||
content_type: text/plain
|
||||
- name: glossary
|
||||
required: false
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ./shared/system.md
|
||||
- role: user
|
||||
content_file: ./shared/transcript.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./shared/references.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
content_file: ./dnd/spells/task.md
|
||||
- role: user
|
||||
content_file: ./dnd/spells/instructions.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd_spells.v1.json
|
||||
repair_attempts: 0
|
||||
@@ -0,0 +1,11 @@
|
||||
Source references must use the source-unit IDs exactly as provided.
|
||||
|
||||
Return only D&D spell-cast artifacts. For each spell cast, identify the in-world
|
||||
caster, spell name, effect, narrative description, and source references using
|
||||
source_id, start_unit_id, and end_unit_id.
|
||||
|
||||
Use roster and glossary reference material only to clarify source text. Do not
|
||||
return spells, casters, or effects that are mentioned only in reference
|
||||
material.
|
||||
|
||||
Return exactly one JSON object and no explanatory text.
|
||||
@@ -0,0 +1,5 @@
|
||||
Extract Dungeons & Dragons spell-cast artifacts from the provided transcript.
|
||||
|
||||
Extract only spell casts that are supported by the transcript. Do not infer
|
||||
spells from general D&D knowledge or from table chatter that does not identify a
|
||||
spell being cast.
|
||||
@@ -63,11 +63,14 @@ func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
}
|
||||
|
||||
func (e *Extractor) ManifestMetadata() map[string]any {
|
||||
promptMetadata := spellsPromptBundle.Metadata()
|
||||
promptSHA, err := scriptoriumPromptMetadata()
|
||||
if err != nil {
|
||||
promptSHA = spellsPromptBundle.Metadata().SHA256
|
||||
}
|
||||
metadata := map[string]any{
|
||||
"prompt_id": PromptID,
|
||||
"prompt_version": promptMetadata.PromptVersion,
|
||||
"prompt_sha256": promptMetadata.SHA256,
|
||||
"prompt_version": SchemaVersion,
|
||||
"prompt_sha256": promptSHA,
|
||||
"response_schema_key": string(ResponseSchemaKey),
|
||||
"response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName,
|
||||
|
||||
84
internal/modules/extract/dnd/spells/scriptorium_assets.go
Normal file
84
internal/modules/extract/dnd/spells/scriptorium_assets.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package spells
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/promptassets"
|
||||
)
|
||||
|
||||
const scriptoriumPromptRoot = "assets/scriptorium/prompts"
|
||||
|
||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
if err := registry.RegisterPromptFS(embeddedAssets, scriptoriumPromptRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
return registry.RegisterSchemaFS(embeddedAssets, "assets/schemas")
|
||||
}
|
||||
|
||||
func scriptoriumPromptMetadata() (string, error) {
|
||||
scriptoriumPromptHashOnce.Do(func() {
|
||||
parts := append([]llm.AssetHashPart{
|
||||
{FS: embeddedAssets, Path: "assets/scriptorium/prompts/dnd.spells.yaml"},
|
||||
{FS: embeddedAssets, Path: "assets/scriptorium/prompts/dnd/spells/task.md"},
|
||||
{FS: embeddedAssets, Path: "assets/scriptorium/prompts/dnd/spells/instructions.md"},
|
||||
}, append(promptassets.CommonHashParts(), promptassets.ReferenceHashParts()...)...)
|
||||
scriptoriumPromptHash, scriptoriumPromptHashErr = llm.HashAssets(parts)
|
||||
})
|
||||
return scriptoriumPromptHash, scriptoriumPromptHashErr
|
||||
}
|
||||
|
||||
func referencePromptInput(slot contracts.ResolvedReferenceSlot) []byte {
|
||||
if len(slot.Items) == 0 {
|
||||
return []byte(" ")
|
||||
}
|
||||
items := append([]contracts.ReferenceItem(nil), slot.Items...)
|
||||
sort.SliceStable(items, func(i, j int) bool {
|
||||
if items[i].Origin.URI != items[j].Origin.URI {
|
||||
return items[i].Origin.URI < items[j].Origin.URI
|
||||
}
|
||||
if items[i].Digest != items[j].Digest {
|
||||
return items[i].Digest < items[j].Digest
|
||||
}
|
||||
return string(items[i].Content) < string(items[j].Content)
|
||||
})
|
||||
if len(items) == 1 {
|
||||
return append([]byte(nil), items[0].Content...)
|
||||
}
|
||||
|
||||
var b bytes.Buffer
|
||||
for i, item := range items {
|
||||
if i > 0 {
|
||||
b.WriteString("\n\n")
|
||||
}
|
||||
fmt.Fprintf(&b, "Reference %d\n", i+1)
|
||||
if item.Origin.Type != "" {
|
||||
fmt.Fprintf(&b, "Origin-Type: %s\n", item.Origin.Type)
|
||||
}
|
||||
if item.Origin.URI != "" {
|
||||
fmt.Fprintf(&b, "Origin-URI: %s\n", item.Origin.URI)
|
||||
}
|
||||
if item.Digest != "" {
|
||||
fmt.Fprintf(&b, "Digest: %s\n", item.Digest)
|
||||
}
|
||||
if item.MediaType != "" {
|
||||
fmt.Fprintf(&b, "Media-Type: %s\n", item.MediaType)
|
||||
}
|
||||
if item.SizeBytes > 0 {
|
||||
fmt.Fprintf(&b, "Size-Bytes: %d\n", item.SizeBytes)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
b.Write(item.Content)
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
var (
|
||||
scriptoriumPromptHashOnce sync.Once
|
||||
scriptoriumPromptHash string
|
||||
scriptoriumPromptHashErr error
|
||||
)
|
||||
180
internal/modules/extract/dnd/spells/scriptorium_assets_test.go
Normal file
180
internal/modules/extract/dnd/spells/scriptorium_assets_test.go
Normal file
@@ -0,0 +1,180 @@
|
||||
package spells
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/promptassets"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
func TestScriptoriumPromptPreparesTranscriptReferencesAndTaskMessages(t *testing.T) {
|
||||
transcript := []byte(`{"id":"session-1","segments":[{"id":"u1","text":"Mira casts shield."}]}`)
|
||||
prepared := prepareSpellsPrompt(t, transcript, "Mira: wizard", "Shield: abjuration")
|
||||
|
||||
if prepared.PromptID != PromptID {
|
||||
t.Fatalf("prompt id = %q, want %q", prepared.PromptID, PromptID)
|
||||
}
|
||||
if got := len(prepared.Messages); got != 5 {
|
||||
t.Fatalf("message count = %d, want 5", got)
|
||||
}
|
||||
wantTranscript := "A transcript of a Dungeons & Dragons gameplay session is provided below.\n\n" + string(transcript) + "\n"
|
||||
if prepared.Messages[1].Content != wantTranscript {
|
||||
t.Fatalf("transcript message = %q, want byte-identical shared transcript body", prepared.Messages[1].Content)
|
||||
}
|
||||
if prepared.Messages[1].CacheControl == nil || prepared.Messages[2].CacheControl == nil {
|
||||
t.Fatalf("expected transcript and reference messages to be cacheable: %#v", prepared.Messages)
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[2].Content, "Roster reference:\nMira: wizard") {
|
||||
t.Fatalf("reference message missing roster content: %q", prepared.Messages[2].Content)
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[2].Content, "Glossary reference:\nShield: abjuration") {
|
||||
t.Fatalf("reference message missing glossary content: %q", prepared.Messages[2].Content)
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[3].Content, "Extract Dungeons & Dragons spell-cast artifacts") {
|
||||
t.Fatalf("task message missing spell task text: %q", prepared.Messages[3].Content)
|
||||
}
|
||||
if strings.Contains(prepared.Messages[3].Content, string(transcript)) {
|
||||
t.Fatalf("task message leaked transcript bytes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptoriumPromptPreparesWithMissingOptionalReferences(t *testing.T) {
|
||||
transcript := []byte(`{"id":"session-1","segments":[]}`)
|
||||
prepared := prepareSpellsPrompt(t, transcript, " ", " ")
|
||||
|
||||
if !strings.Contains(prepared.Messages[2].Content, "Roster reference:\n ") {
|
||||
t.Fatalf("reference message did not include empty roster input: %q", prepared.Messages[2].Content)
|
||||
}
|
||||
if !strings.Contains(prepared.Messages[2].Content, "Glossary reference:\n ") {
|
||||
t.Fatalf("reference message did not include empty glossary input: %q", prepared.Messages[2].Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferencePromptInputRenderingIsDeterministic(t *testing.T) {
|
||||
slot := contracts.ResolvedReferenceSlot{
|
||||
Items: []contracts.ReferenceItem{
|
||||
{
|
||||
SlotName: "roster",
|
||||
MediaType: "text/plain",
|
||||
Content: []byte("second"),
|
||||
Digest: "sha256:bbb",
|
||||
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///b.txt"},
|
||||
SizeBytes: 6,
|
||||
},
|
||||
{
|
||||
SlotName: "roster",
|
||||
MediaType: "text/plain",
|
||||
Content: []byte("first"),
|
||||
Digest: "sha256:aaa",
|
||||
Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///a.txt"},
|
||||
SizeBytes: 5,
|
||||
},
|
||||
},
|
||||
}
|
||||
first := string(referencePromptInput(slot))
|
||||
second := string(referencePromptInput(slot))
|
||||
if first != second {
|
||||
t.Fatalf("reference rendering was not deterministic:\nfirst=%q\nsecond=%q", first, second)
|
||||
}
|
||||
if !strings.Contains(first, "Reference 1\nOrigin-Type: file\nOrigin-URI: file:///a.txt\nDigest: sha256:aaa") {
|
||||
t.Fatalf("first reference heading was not stable: %q", first)
|
||||
}
|
||||
if strings.Index(first, "first") > strings.Index(first, "second") {
|
||||
t.Fatalf("references were not sorted deterministically: %q", first)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingleReferencePromptInputKeepsContentOnly(t *testing.T) {
|
||||
got := string(referencePromptInput(contracts.ResolvedReferenceSlot{
|
||||
Items: []contracts.ReferenceItem{{Content: []byte("single reference")}},
|
||||
}))
|
||||
if got != "single reference" {
|
||||
t.Fatalf("single reference rendering = %q, want raw content only", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptoriumPromptDiagnosticsOmitRawMaterials(t *testing.T) {
|
||||
transcript := []byte(`{"secret":"source text"}`)
|
||||
reference := "private roster note"
|
||||
prepared := prepareSpellsPrompt(t, transcript, reference, " ")
|
||||
metadata := New().ManifestMetadata()
|
||||
|
||||
payload, err := json.Marshal(map[string]any{
|
||||
"prepared": map[string]any{
|
||||
"prompt_id": prepared.PromptID,
|
||||
"prompt_version": prepared.PromptVersion,
|
||||
"prompt_hash": prepared.PromptHash,
|
||||
"rendered_prompt_hash": prepared.RenderedPromptHash,
|
||||
"selected_profile_id": prepared.SelectedProfileID,
|
||||
"output_contract": prepared.OutputContract,
|
||||
"input_hashes": prepared.InputHashes,
|
||||
"effective_model_params": prepared.EffectiveModelParams,
|
||||
},
|
||||
"manifest": metadata,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal diagnostics: %v", err)
|
||||
}
|
||||
diagnostics := string(payload)
|
||||
for _, forbidden := range []string{
|
||||
"source text",
|
||||
reference,
|
||||
"Extract Dungeons & Dragons spell-cast artifacts",
|
||||
`"properties"`,
|
||||
"spell_casts",
|
||||
} {
|
||||
if strings.Contains(diagnostics, forbidden) {
|
||||
t.Fatalf("diagnostics leaked %q: %s", forbidden, diagnostics)
|
||||
}
|
||||
}
|
||||
if metadata["prompt_id"] != PromptID || metadata["prompt_version"] != SchemaVersion {
|
||||
t.Fatalf("manifest prompt metadata = %#v", metadata)
|
||||
}
|
||||
if !strings.HasPrefix(metadata["prompt_sha256"].(string), "sha256:") {
|
||||
t.Fatalf("manifest prompt hash = %#v, want sha256-prefixed", metadata["prompt_sha256"])
|
||||
}
|
||||
}
|
||||
|
||||
func prepareSpellsPrompt(t *testing.T, transcript []byte, roster string, glossary string) *scriptorium.PreparedRun {
|
||||
t.Helper()
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := promptassets.Register(registry); err != nil {
|
||||
t.Fatalf("register shared prompt assets: %v", err)
|
||||
}
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("register spell prompt assets: %v", err)
|
||||
}
|
||||
options, err := registry.ScriptoriumOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
ID: "spell-test-profile",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
Model: "spell-test-model",
|
||||
})))
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: "spell-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.InlineWithURI("file:///session.json", string(transcript)),
|
||||
"roster": scriptorium.Inline(roster),
|
||||
"glossary": scriptorium.Inline(glossary),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
return prepared
|
||||
}
|
||||
Reference in New Issue
Block a user