Cut modules over to Scriptorium prompts
This commit is contained in:
@@ -2,5 +2,5 @@ package spells
|
||||
|
||||
import "embed"
|
||||
|
||||
//go:embed assets/prompts/*.md assets/schemas/*.json assets/scriptorium/prompts/*.yaml assets/scriptorium/prompts/dnd/spells/*.md
|
||||
//go:embed assets/schemas/*.json assets/scriptorium/prompts/*.yaml assets/scriptorium/prompts/dnd/spells/*.md
|
||||
var embeddedAssets embed.FS
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
You extract D&D spell-cast artifacts from source units.
|
||||
|
||||
{{ hardening }}
|
||||
|
||||
Extract only spell casts that are supported by the provided source text. Do not
|
||||
infer spells from general D&D knowledge or from table chatter that does not
|
||||
identify a spell being cast.
|
||||
|
||||
Reference material, when present, is supporting context only. Use it only to
|
||||
disambiguate names, aliases, speakers, campaign terms, or spell names already
|
||||
present in the source text. Do not extract a spell cast solely because it appears
|
||||
in reference material.
|
||||
|
||||
Source references must use the source-unit IDs exactly as provided.
|
||||
@@ -1,34 +0,0 @@
|
||||
Source document ID: {{ .SourceID }}
|
||||
{{ if .HasChunk }}
|
||||
Chunk ID: {{ .ChunkID }}
|
||||
Chunk index: {{ .ChunkIndex }}
|
||||
{{ end }}
|
||||
|
||||
Source units:
|
||||
{{ range .Units }}
|
||||
- Unit ID: {{ .ID }}
|
||||
Text: {{ .Text }}
|
||||
{{ if .Metadata }}
|
||||
Metadata:
|
||||
{{ range .Metadata }}
|
||||
- {{ .Key }}: {{ .Value }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
|
||||
{{ if hasreference "roster" }}
|
||||
Roster reference material:
|
||||
{{ reference "roster" }}
|
||||
|
||||
{{ end }}
|
||||
{{ if hasreference "glossary" }}
|
||||
Glossary reference material:
|
||||
{{ reference "glossary" }}
|
||||
|
||||
{{ end }}
|
||||
Return only D&D spell-cast artifacts. For each spell cast, identify the in-world
|
||||
caster, spell name, effect, narrative description, and source references using
|
||||
source_id, start_unit_id, and end_unit_id.
|
||||
|
||||
Use roster and glossary reference material only to clarify source text. Do not
|
||||
return spells, casters, or effects that are mentioned only in reference material.
|
||||
@@ -65,7 +65,7 @@ func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
func (e *Extractor) ManifestMetadata() map[string]any {
|
||||
promptSHA, err := scriptoriumPromptMetadata()
|
||||
if err != nil {
|
||||
promptSHA = spellsPromptBundle.Metadata().SHA256
|
||||
promptSHA = ""
|
||||
}
|
||||
metadata := map[string]any{
|
||||
"prompt_id": PromptID,
|
||||
@@ -112,24 +112,14 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.ExtractionRequest
|
||||
return contracts.ExtractionResult{}, extractorErrorf("LLM client must not be nil")
|
||||
}
|
||||
|
||||
system, user, _, err := renderPrompt(req)
|
||||
if err != nil {
|
||||
return contracts.ExtractionResult{}, extractorErrorf("render prompt: %w", err)
|
||||
}
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
return contracts.ExtractionResult{}, extractorErrorf("load response schema %q: %w", ResponseSchemaKey, err)
|
||||
}
|
||||
|
||||
var response extractionResponse
|
||||
if _, err := req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key,
|
||||
Messages: []contracts.LLMMessage{
|
||||
{Role: "system", Content: system},
|
||||
{Role: "user", Content: user},
|
||||
},
|
||||
ResponseSchemaName: schema.Name,
|
||||
ResponseSchema: schema.JSONSchema,
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
Inputs: promptInputs(req),
|
||||
}, &response); err != nil {
|
||||
return contracts.ExtractionResult{}, extractorErrorf("complete structured output: %w", err)
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package spells
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -41,29 +40,21 @@ func TestExtractReturnsSpellCandidateFromStructuredOutput(t *testing.T) {
|
||||
if req.StageName != Key {
|
||||
t.Fatalf("StageName = %q, want %q", req.StageName, Key)
|
||||
}
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||
if req.PromptID != PromptID || req.PromptVersion != SchemaVersion {
|
||||
t.Fatalf("prompt = %q/%q, want %q/%q", req.PromptID, req.PromptVersion, PromptID, SchemaVersion)
|
||||
}
|
||||
if req.ResponseSchemaName != schema.Name {
|
||||
t.Fatalf("ResponseSchemaName = %q, want %q", req.ResponseSchemaName, schema.Name)
|
||||
if req.SessionID != "session-123" || req.ProfileID != "profile-spells" {
|
||||
t.Fatalf("session/profile = %q/%q, want session-123/profile-spells", req.SessionID, req.ProfileID)
|
||||
}
|
||||
if !bytes.Equal(req.ResponseSchema, schema.JSONSchema) {
|
||||
t.Fatal("ResponseSchema does not match registered D&D spells schema")
|
||||
if len(req.Messages) != 0 || req.ResponseSchemaName != "" || len(req.ResponseSchema) != 0 {
|
||||
t.Fatalf("legacy prompt fields set: messages=%#v schema=%q/%s", req.Messages, req.ResponseSchemaName, req.ResponseSchema)
|
||||
}
|
||||
if len(req.Messages) != 2 {
|
||||
t.Fatalf("len(Messages) = %d, want 2", len(req.Messages))
|
||||
transcript := req.Inputs["transcript"]
|
||||
if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:transcript" || transcript.OriginURI != "file:///session-alpha.json" {
|
||||
t.Fatalf("transcript metadata = %#v", transcript)
|
||||
}
|
||||
if req.Messages[0].Role != "system" || req.Messages[1].Role != "user" {
|
||||
t.Fatalf("Messages roles = %#v, want system then user", req.Messages)
|
||||
}
|
||||
if !strings.Contains(req.Messages[0].Content, "D&D spell-cast") {
|
||||
t.Fatalf("system message = %q, want D&D spell context", req.Messages[0].Content)
|
||||
}
|
||||
for _, want := range []string{"session-alpha", "session-alpha:chunk:0", "seg-001", "Cure Wounds"} {
|
||||
if !strings.Contains(req.Messages[1].Content, want) {
|
||||
t.Fatalf("user message = %q, want substring %q", req.Messages[1].Content, want)
|
||||
}
|
||||
if got := string(transcript.Content); got != spellTranscriptJSON {
|
||||
t.Fatalf("transcript content = %q, want original source input", got)
|
||||
}
|
||||
|
||||
if len(result.Candidates) != 1 {
|
||||
@@ -116,7 +107,7 @@ func TestExtractorManifestMetadataIncludesPromptAndSchemaProvenance(t *testing.T
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractIncludesReferencesInPrompt(t *testing.T) {
|
||||
func TestExtractPassesReferencesAsPromptInputs(t *testing.T) {
|
||||
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
|
||||
req := extractionRequestWithClient(client)
|
||||
req.References = contracts.ReferenceSet{
|
||||
@@ -143,26 +134,18 @@ func TestExtractIncludesReferencesInPrompt(t *testing.T) {
|
||||
if len(client.requests) != 1 {
|
||||
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
|
||||
}
|
||||
system := client.requests[0].Messages[0].Content
|
||||
for _, want := range []string{
|
||||
"Reference material, when present, is supporting context only.",
|
||||
"in reference material.",
|
||||
} {
|
||||
if !strings.Contains(system, want) {
|
||||
t.Fatalf("system prompt = %q, want substring %q", system, want)
|
||||
}
|
||||
request := client.requests[0]
|
||||
if len(request.Messages) != 0 {
|
||||
t.Fatalf("Messages = %#v, want no locally rendered prompt", request.Messages)
|
||||
}
|
||||
user := client.requests[0].Messages[1].Content
|
||||
for _, want := range []string{
|
||||
"Roster reference material:",
|
||||
"Aria Brightmantle: party cleric",
|
||||
"Glossary reference material:",
|
||||
"Brightmantle: local temple name",
|
||||
"mentioned only in reference material.",
|
||||
} {
|
||||
if !strings.Contains(user, want) {
|
||||
t.Fatalf("user prompt = %q, want substring %q", user, want)
|
||||
}
|
||||
if got := string(request.Inputs["roster"].Content); got != "Aria Brightmantle: party cleric" {
|
||||
t.Fatalf("roster input = %q, want reference content", got)
|
||||
}
|
||||
if got := string(request.Inputs["glossary"].Content); got != "Brightmantle: local temple name" {
|
||||
t.Fatalf("glossary input = %q, want reference content", got)
|
||||
}
|
||||
if strings.Contains(string(request.Inputs["transcript"].Content), "Aria Brightmantle: party cleric") {
|
||||
t.Fatalf("transcript input contains reference content")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -308,9 +291,18 @@ func TestExtractCopiesCandidateSourceRefs(t *testing.T) {
|
||||
func extractionRequestWithClient(client contracts.StructuredLLMClient) contracts.ExtractionRequest {
|
||||
req := promptExtractionRequest()
|
||||
req.LLMClient = client
|
||||
req.SourceInput = spellSourceInput()
|
||||
req.SessionID = "session-123"
|
||||
req.LLMProfile = "profile-spells"
|
||||
return req
|
||||
}
|
||||
|
||||
const spellTranscriptJSON = `{"id":"session-alpha","segments":[{"id":"seg-001","text":"Aria raises her hand and casts Cure Wounds."}]}`
|
||||
|
||||
func spellSourceInput() contracts.LLMInputMaterial {
|
||||
return contracts.NewLLMInputMaterial("source", "application/json", []byte(spellTranscriptJSON), "sha256:transcript", "file:///session-alpha.json")
|
||||
}
|
||||
|
||||
func emptyChunkRequest(req contracts.ExtractionRequest) contracts.ExtractionRequest {
|
||||
req.Chunk = &contracts.SourceChunk{
|
||||
ID: req.Chunk.ID,
|
||||
@@ -327,13 +319,7 @@ type fakeSpellsLLMClient struct {
|
||||
}
|
||||
|
||||
func (client *fakeSpellsLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.requests = append(client.requests, contracts.StructuredCompletionRequest{
|
||||
StageName: req.StageName,
|
||||
Messages: append([]contracts.LLMMessage(nil), req.Messages...),
|
||||
Model: req.Model,
|
||||
ResponseSchemaName: req.ResponseSchemaName,
|
||||
ResponseSchema: append(json.RawMessage(nil), req.ResponseSchema...),
|
||||
})
|
||||
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
|
||||
if client.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, client.err
|
||||
}
|
||||
@@ -349,3 +335,10 @@ func (client *fakeSpellsLLMClient) CompleteStructured(ctx context.Context, req c
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
}
|
||||
|
||||
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
|
||||
req.Messages = append([]contracts.LLMMessage(nil), req.Messages...)
|
||||
req.ResponseSchema = append(json.RawMessage(nil), req.ResponseSchema...)
|
||||
req.Inputs = req.Inputs.Clone()
|
||||
return req
|
||||
}
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
package spells
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/prompt"
|
||||
)
|
||||
|
||||
type promptData struct {
|
||||
SourceID string
|
||||
HasChunk bool
|
||||
ChunkID string
|
||||
ChunkIndex int
|
||||
Units []promptUnit
|
||||
}
|
||||
|
||||
type promptUnit struct {
|
||||
ID string
|
||||
Text string
|
||||
Metadata []promptMetadata
|
||||
}
|
||||
|
||||
type promptMetadata struct {
|
||||
Key string
|
||||
Value string
|
||||
}
|
||||
|
||||
var spellsPromptBundle = mustLoadPromptBundle()
|
||||
|
||||
func mustLoadPromptBundle() *prompt.Bundle {
|
||||
bundle, err := prompt.LoadBundle(embeddedAssets, prompt.Definition{
|
||||
PromptID: PromptID,
|
||||
Version: SchemaVersion,
|
||||
EmbeddedPath: "assets/prompts",
|
||||
SystemPath: "assets/prompts/system.md",
|
||||
UserPath: "assets/prompts/user.md",
|
||||
ReferenceSlots: cloneReferenceSlots(referenceSlots),
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return bundle
|
||||
}
|
||||
|
||||
func buildPromptData(req contracts.ExtractionRequest) (promptData, error) {
|
||||
if req.Source == nil {
|
||||
return promptData{}, fmt.Errorf("dnd spells prompt: source must not be nil")
|
||||
}
|
||||
if req.Chunk == nil {
|
||||
return promptData{}, fmt.Errorf("dnd spells prompt: chunk must not be nil")
|
||||
}
|
||||
|
||||
data := promptData{
|
||||
SourceID: req.Source.ID,
|
||||
HasChunk: true,
|
||||
ChunkID: req.Chunk.ID,
|
||||
ChunkIndex: req.Chunk.Index,
|
||||
Units: make([]promptUnit, 0, len(req.Chunk.Units)),
|
||||
}
|
||||
for _, unit := range req.Chunk.Units {
|
||||
data.Units = append(data.Units, promptUnit{
|
||||
ID: unit.ID,
|
||||
Text: unit.Text,
|
||||
Metadata: selectedMetadata(unit),
|
||||
})
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func renderPrompt(req contracts.ExtractionRequest) (system string, user string, metadata prompt.Metadata, err error) {
|
||||
data, err := buildPromptData(req)
|
||||
if err != nil {
|
||||
return "", "", prompt.Metadata{}, err
|
||||
}
|
||||
system, user, metadata, err = spellsPromptBundle.RenderUserSystemWithReferences(data, req.References)
|
||||
if err != nil {
|
||||
return "", "", prompt.Metadata{}, fmt.Errorf("dnd spells prompt: %w", err)
|
||||
}
|
||||
return system, user, metadata, nil
|
||||
}
|
||||
|
||||
func selectedMetadata(unit source.SourceUnit) []promptMetadata {
|
||||
if len(unit.Metadata) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
keys := []string{"speaker", "start", "end"}
|
||||
metadata := make([]promptMetadata, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
value, ok := unit.Metadata[key]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
rendered := strings.TrimSpace(fmt.Sprint(value))
|
||||
if rendered == "" {
|
||||
continue
|
||||
}
|
||||
metadata = append(metadata, promptMetadata{
|
||||
Key: key,
|
||||
Value: rendered,
|
||||
})
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
@@ -1,209 +0,0 @@
|
||||
package spells
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/prompt"
|
||||
)
|
||||
|
||||
func TestBuildPromptDataFromGenericSourceChunk(t *testing.T) {
|
||||
req := promptExtractionRequest()
|
||||
|
||||
data, err := buildPromptData(req)
|
||||
if err != nil {
|
||||
t.Fatalf("buildPromptData() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if data.SourceID != "session-alpha" {
|
||||
t.Fatalf("SourceID = %q, want session-alpha", data.SourceID)
|
||||
}
|
||||
if !data.HasChunk || data.ChunkID != "session-alpha:chunk:0" || data.ChunkIndex != 0 {
|
||||
t.Fatalf("chunk data = %#v, want fixture chunk", data)
|
||||
}
|
||||
if len(data.Units) != 2 {
|
||||
t.Fatalf("len(Units) = %d, want 2", len(data.Units))
|
||||
}
|
||||
first := data.Units[0]
|
||||
if first.ID != "seg-001" || first.Text != "Aria raises her hand and casts Cure Wounds." {
|
||||
t.Fatalf("first unit = %#v, want source unit data", first)
|
||||
}
|
||||
wantMetadata := []promptMetadata{
|
||||
{Key: "speaker", Value: "Alice"},
|
||||
{Key: "start", Value: "1.25"},
|
||||
{Key: "end", Value: "3.5"},
|
||||
}
|
||||
if !reflect.DeepEqual(first.Metadata, wantMetadata) {
|
||||
t.Fatalf("first.Metadata = %#v, want %#v", first.Metadata, wantMetadata)
|
||||
}
|
||||
if len(data.Units[1].Metadata) != 0 {
|
||||
t.Fatalf("second.Metadata = %#v, want no selected metadata", data.Units[1].Metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPromptDataDoesNotMutateRequest(t *testing.T) {
|
||||
req := promptExtractionRequest()
|
||||
beforeSource := mustJSON(t, req.Source)
|
||||
beforeChunk := mustJSON(t, req.Chunk)
|
||||
beforeRequest := mustJSON(t, req)
|
||||
|
||||
if _, err := buildPromptData(req); err != nil {
|
||||
t.Fatalf("buildPromptData() error = %v, want nil", err)
|
||||
}
|
||||
afterSource := mustJSON(t, req.Source)
|
||||
afterChunk := mustJSON(t, req.Chunk)
|
||||
afterRequest := mustJSON(t, req)
|
||||
if beforeSource != afterSource || beforeChunk != afterChunk || beforeRequest != afterRequest {
|
||||
t.Fatalf(
|
||||
"request mutated:\nsource before: %s\nsource after: %s\nchunk before: %s\nchunk after: %s\nrequest before: %s\nrequest after: %s",
|
||||
beforeSource,
|
||||
afterSource,
|
||||
beforeChunk,
|
||||
afterChunk,
|
||||
beforeRequest,
|
||||
afterRequest,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderPromptIncludesSourceContext(t *testing.T) {
|
||||
system, user, metadata, err := renderPrompt(promptExtractionRequest())
|
||||
if err != nil {
|
||||
t.Fatalf("renderPrompt() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(system, prompt.HardeningText()) {
|
||||
t.Fatalf("system prompt = %q, want hardening text", system)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"session-alpha",
|
||||
"session-alpha:chunk:0",
|
||||
"seg-001",
|
||||
"Aria raises her hand and casts Cure Wounds.",
|
||||
"speaker: Alice",
|
||||
"start: 1.25",
|
||||
"end: 3.5",
|
||||
} {
|
||||
if !strings.Contains(user, want) {
|
||||
t.Fatalf("user prompt = %q, want substring %q", user, want)
|
||||
}
|
||||
}
|
||||
if metadata.PromptID != PromptID {
|
||||
t.Fatalf("metadata.PromptID = %q, want %q", metadata.PromptID, PromptID)
|
||||
}
|
||||
if metadata.PromptVersion != SchemaVersion {
|
||||
t.Fatalf("metadata.PromptVersion = %q, want %q", metadata.PromptVersion, SchemaVersion)
|
||||
}
|
||||
if metadata.EmbeddedPath != "assets/prompts" {
|
||||
t.Fatalf("metadata.EmbeddedPath = %q, want assets/prompts", metadata.EmbeddedPath)
|
||||
}
|
||||
if strings.Contains(user, "Roster reference material") || strings.Contains(user, "Glossary reference material") {
|
||||
t.Fatalf("user prompt = %q, want no optional reference sections without bindings", user)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderPromptIncludesBoundReferences(t *testing.T) {
|
||||
req := promptExtractionRequest()
|
||||
req.References = contracts.ReferenceSet{
|
||||
Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||
"roster": {
|
||||
Slot: contracts.ReferenceSlot{Name: "roster"},
|
||||
Items: []contracts.ReferenceItem{
|
||||
{SlotName: "roster", Content: []byte("Aria: cleric, also known as Sister Aria")},
|
||||
},
|
||||
},
|
||||
"glossary": {
|
||||
Slot: contracts.ReferenceSlot{Name: "glossary"},
|
||||
Items: []contracts.ReferenceItem{
|
||||
{SlotName: "glossary", Content: []byte("Cure Wounds: healing spell")},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_, user, metadata, err := renderPrompt(req)
|
||||
if err != nil {
|
||||
t.Fatalf("renderPrompt() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
for _, want := range []string{
|
||||
"Roster reference material:",
|
||||
"Aria: cleric, also known as Sister Aria",
|
||||
"Glossary reference material:",
|
||||
"Cure Wounds: healing spell",
|
||||
"Use roster and glossary reference material only to clarify source text.",
|
||||
"mentioned only in reference material.",
|
||||
} {
|
||||
if !strings.Contains(user, want) {
|
||||
t.Fatalf("user prompt = %q, want substring %q", user, want)
|
||||
}
|
||||
}
|
||||
if metadata.SHA256 != spellsPromptBundle.Metadata().SHA256 {
|
||||
t.Fatalf("metadata.SHA256 = %q, want template hash %q", metadata.SHA256, spellsPromptBundle.Metadata().SHA256)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPromptDataRejectsMissingSourceContext(t *testing.T) {
|
||||
if _, err := buildPromptData(contracts.ExtractionRequest{}); err == nil || !strings.Contains(err.Error(), "source") {
|
||||
t.Fatalf("buildPromptData() error = %v, want source error", err)
|
||||
}
|
||||
if _, err := buildPromptData(contracts.ExtractionRequest{Source: promptSourceDocument()}); err == nil || !strings.Contains(err.Error(), "chunk") {
|
||||
t.Fatalf("buildPromptData() error = %v, want chunk error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func promptExtractionRequest() contracts.ExtractionRequest {
|
||||
doc := promptSourceDocument()
|
||||
chunk := &contracts.SourceChunk{
|
||||
ID: "session-alpha:chunk:0",
|
||||
SourceID: doc.ID,
|
||||
Index: 0,
|
||||
Units: append([]source.SourceUnit(nil), doc.Units...),
|
||||
Metadata: map[string]any{"ignored": "chunk metadata"},
|
||||
}
|
||||
return contracts.ExtractionRequest{
|
||||
Source: doc,
|
||||
Chunk: chunk,
|
||||
}
|
||||
}
|
||||
|
||||
func promptSourceDocument() *source.SourceDocument {
|
||||
return &source.SourceDocument{
|
||||
ID: "session-alpha",
|
||||
Kind: "transcript",
|
||||
Format: "application/vnd.seriatim.minimal+json",
|
||||
Digest: "sha256:test",
|
||||
Units: []source.SourceUnit{
|
||||
{
|
||||
ID: "seg-001",
|
||||
Kind: "transcript_segment",
|
||||
Text: "Aria raises her hand and casts Cure Wounds.",
|
||||
Metadata: map[string]any{
|
||||
"speaker": "Alice",
|
||||
"start": json.Number("1.25"),
|
||||
"end": json.Number("3.5"),
|
||||
"ignored": "not rendered",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "seg-002",
|
||||
Kind: "transcript_segment",
|
||||
Text: "The fighter's wounds begin to close.",
|
||||
Metadata: map[string]any{"ignored": "not rendered"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func mustJSON(t *testing.T, value any) string {
|
||||
t.Helper()
|
||||
encoded, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v, want nil", err)
|
||||
}
|
||||
return string(encoded)
|
||||
}
|
||||
@@ -152,16 +152,15 @@ func TestRunnerPassesRosterAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T)
|
||||
if len(llmClient.requests) != 1 {
|
||||
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
|
||||
}
|
||||
user := llmClient.requests[0].Messages[1].Content
|
||||
for _, want := range []string{
|
||||
"Roster reference material:",
|
||||
"Aria: party cleric",
|
||||
"Glossary reference material:",
|
||||
"Fire Bolt: evocation cantrip",
|
||||
} {
|
||||
if !strings.Contains(user, want) {
|
||||
t.Fatalf("user prompt = %q, want substring %q", user, want)
|
||||
}
|
||||
request := llmClient.requests[0]
|
||||
if len(request.Messages) != 0 {
|
||||
t.Fatalf("Messages = %#v, want no locally rendered prompt", request.Messages)
|
||||
}
|
||||
if got := string(request.Inputs["roster"].Content); got != "Aria: party cleric\nBorin: fighter" {
|
||||
t.Fatalf("roster input = %q, want reference text", got)
|
||||
}
|
||||
if got := string(request.Inputs["glossary"].Content); got != "Fire Bolt: evocation cantrip" {
|
||||
t.Fatalf("glossary input = %q, want reference text", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -191,9 +190,12 @@ func TestRunnerDoesNotExtractSpellMentionedOnlyInRoster(t *testing.T) {
|
||||
if len(llmClient.requests) != 1 {
|
||||
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
|
||||
}
|
||||
user := llmClient.requests[0].Messages[1].Content
|
||||
if !strings.Contains(user, "Lightning Bolt") {
|
||||
t.Fatalf("user prompt = %q, want roster-only spell in reference section", user)
|
||||
request := llmClient.requests[0]
|
||||
if len(request.Messages) != 0 {
|
||||
t.Fatalf("Messages = %#v, want no locally rendered prompt", request.Messages)
|
||||
}
|
||||
if got := string(request.Inputs["roster"].Content); !strings.Contains(got, "Lightning Bolt") {
|
||||
t.Fatalf("roster input = %q, want roster-only spell in reference input", got)
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "approved" {
|
||||
t.Fatalf("ValidationStatus = %q, want approved empty extraction", output.Manifest.ValidationStatus)
|
||||
|
||||
@@ -20,6 +20,31 @@ func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
return registry.RegisterSchemaFS(embeddedAssets, "assets/schemas")
|
||||
}
|
||||
|
||||
func promptInputs(req contracts.ExtractionRequest) contracts.LLMInputSet {
|
||||
return contracts.LLMInputSet{
|
||||
"transcript": transcriptPromptInput(req.SourceInput),
|
||||
"roster": referencePromptMaterial("roster", req.References.Slots["roster"]),
|
||||
"glossary": referencePromptMaterial("glossary", req.References.Slots["glossary"]),
|
||||
}
|
||||
}
|
||||
|
||||
func transcriptPromptInput(material contracts.LLMInputMaterial) contracts.LLMInputMaterial {
|
||||
out := material.Clone()
|
||||
out.Name = "transcript"
|
||||
return out
|
||||
}
|
||||
|
||||
func referencePromptMaterial(name string, slot contracts.ResolvedReferenceSlot) contracts.LLMInputMaterial {
|
||||
body := referencePromptInput(slot)
|
||||
digest := ""
|
||||
originURI := ""
|
||||
if len(slot.Items) == 1 {
|
||||
digest = slot.Items[0].Digest
|
||||
originURI = slot.Items[0].Origin.URI
|
||||
}
|
||||
return contracts.NewLLMInputMaterial(name, "text/plain", body, digest, originURI)
|
||||
}
|
||||
|
||||
func scriptoriumPromptMetadata() (string, error) {
|
||||
scriptoriumPromptHashOnce.Do(func() {
|
||||
parts := append([]llm.AssetHashPart{
|
||||
|
||||
61
internal/modules/extract/dnd/spells/test_helpers_test.go
Normal file
61
internal/modules/extract/dnd/spells/test_helpers_test.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package spells
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func promptExtractionRequest() contracts.ExtractionRequest {
|
||||
doc := promptSourceDocument()
|
||||
chunk := &contracts.SourceChunk{
|
||||
ID: "session-alpha:chunk:0",
|
||||
SourceID: doc.ID,
|
||||
Index: 0,
|
||||
Units: append([]source.SourceUnit(nil), doc.Units...),
|
||||
Metadata: map[string]any{"ignored": "chunk metadata"},
|
||||
}
|
||||
return contracts.ExtractionRequest{
|
||||
Source: doc,
|
||||
Chunk: chunk,
|
||||
}
|
||||
}
|
||||
|
||||
func promptSourceDocument() *source.SourceDocument {
|
||||
return &source.SourceDocument{
|
||||
ID: "session-alpha",
|
||||
Kind: "transcript",
|
||||
Format: "application/vnd.seriatim.minimal+json",
|
||||
Digest: "sha256:test",
|
||||
Units: []source.SourceUnit{
|
||||
{
|
||||
ID: "seg-001",
|
||||
Kind: "transcript_segment",
|
||||
Text: "Aria raises her hand and casts Cure Wounds.",
|
||||
Metadata: map[string]any{
|
||||
"speaker": "Alice",
|
||||
"start": json.Number("1.25"),
|
||||
"end": json.Number("3.5"),
|
||||
"ignored": "not rendered",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "seg-002",
|
||||
Kind: "transcript_segment",
|
||||
Text: "The fighter's wounds begin to close.",
|
||||
Metadata: map[string]any{"ignored": "not rendered"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func mustJSON(t *testing.T, value any) string {
|
||||
t.Helper()
|
||||
encoded, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v, want nil", err)
|
||||
}
|
||||
return string(encoded)
|
||||
}
|
||||
Reference in New Issue
Block a user