Render captured prompt templates without reloading sources
This commit is contained in:
177
frozen_rendering_contract_test.go
Normal file
177
frozen_rendering_contract_test.go
Normal file
@@ -0,0 +1,177 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
const capturedPromptDocument = `
|
||||
prompts:
|
||||
captured:
|
||||
"1":
|
||||
default_profile: offline
|
||||
session_id: '{{.session}}'
|
||||
messages:
|
||||
- role: user
|
||||
content_file: body.tmpl
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`
|
||||
|
||||
func capturedPromptEngine(t *testing.T, source fs.FS) *promptkit.Engine {
|
||||
t.Helper()
|
||||
inputs, err := promptkit.FilesystemInputs("application prompts", source, "config")
|
||||
if err != nil {
|
||||
t.Fatalf("select prompt inputs: %v", err)
|
||||
}
|
||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{
|
||||
{Inputs: inputs},
|
||||
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
|
||||
"offline": {Endpoint: &endpoint, Model: &model},
|
||||
})}}},
|
||||
},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func TestEngineCapturesPromptContentUntilReconstruction(t *testing.T) {
|
||||
files := fstest.MapFS{
|
||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
||||
"config/body.tmpl": &fstest.MapFile{Data: []byte("before {{.name}}")},
|
||||
}
|
||||
first := capturedPromptEngine(t, files)
|
||||
files["config/body.tmpl"] = &fstest.MapFile{Data: []byte("after {{.name}}")}
|
||||
request := promptkit.RunRequest{PromptID: "captured", Vars: map[string]string{"name": "Ada", "session": "session-1"}}
|
||||
oldPrepared, err := first.Prepare(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with existing engine: %v", err)
|
||||
}
|
||||
second := capturedPromptEngine(t, files)
|
||||
newPrepared, err := second.Prepare(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with rebuilt engine: %v", err)
|
||||
}
|
||||
if oldPrepared.Messages[0].Content != "before Ada" || newPrepared.Messages[0].Content != "after Ada" || oldPrepared.PromptHash == newPrepared.PromptHash {
|
||||
t.Fatalf("captured content or hash did not follow engine lifetime: old=%+v new=%+v", oldPrepared, newPrepared)
|
||||
}
|
||||
}
|
||||
|
||||
type unreadableAfterConstructionFS struct {
|
||||
fs.FS
|
||||
closed atomic.Bool
|
||||
reads atomic.Int64
|
||||
}
|
||||
|
||||
func (source *unreadableAfterConstructionFS) Open(name string) (fs.File, error) {
|
||||
if source.closed.Load() {
|
||||
source.reads.Add(1)
|
||||
return nil, fs.ErrPermission
|
||||
}
|
||||
return source.FS.Open(name)
|
||||
}
|
||||
|
||||
func TestCapturedTemplatesBindInputsIndependentlyAcrossConcurrentPreparations(t *testing.T) {
|
||||
source := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
||||
"config/body.tmpl": &fstest.MapFile{Data: []byte(`{{.name}}: {{input "note"}}`)},
|
||||
}}
|
||||
engine := capturedPromptEngine(t, source)
|
||||
source.closed.Store(true)
|
||||
|
||||
const calls = 24
|
||||
var wait sync.WaitGroup
|
||||
errs := make(chan error, calls)
|
||||
for index := 0; index < calls; index++ {
|
||||
wait.Add(1)
|
||||
go func(index int) {
|
||||
defer wait.Done()
|
||||
name := fmt.Sprintf("caller-%d", index)
|
||||
body := fmt.Sprintf("body-%d", index)
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "captured", SessionID: "direct-session",
|
||||
Vars: map[string]string{"name": name},
|
||||
Inputs: map[string]promptkit.ArtifactRef{"note": promptkit.Inline(body)},
|
||||
})
|
||||
if err == nil && (prepared.Messages[0].Content != name+": "+body || prepared.SessionID != "direct-session") {
|
||||
err = fmt.Errorf("unexpected prepared content: %+v", prepared)
|
||||
}
|
||||
errs <- err
|
||||
}(index)
|
||||
}
|
||||
wait.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if source.reads.Load() != 0 {
|
||||
t.Fatalf("preparation reopened captured source %d times", source.reads.Load())
|
||||
}
|
||||
|
||||
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "captured", Vars: map[string]string{"name": "Ada"},
|
||||
Inputs: map[string]promptkit.ArtifactRef{"note": promptkit.Inline("body")},
|
||||
})
|
||||
if !errors.Is(err, promptkit.ErrPromptRender) {
|
||||
t.Fatalf("missing session variable = %v, want runtime render error", err)
|
||||
}
|
||||
_, err = engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "captured", SessionID: "direct-session", Vars: map[string]string{"name": "Ada"},
|
||||
})
|
||||
if !errors.Is(err, promptkit.ErrPromptRender) {
|
||||
t.Fatalf("missing input helper artifact = %v, want runtime render error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *testing.T) {
|
||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||
schemas := fstest.MapFS{}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", map[string]promptkit.ProfileConfig{
|
||||
"offline": {Endpoint: &endpoint, Model: &model},
|
||||
})}}}},
|
||||
Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{
|
||||
"prompts": map[string]any{"structured": map[string]any{"1": map[string]any{
|
||||
"default_profile": "offline",
|
||||
"inputs": []any{map[string]any{"name": "note", "required": true}},
|
||||
"messages": []any{map[string]any{"role": "user", "content": `{{input "note"}}`}},
|
||||
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"},
|
||||
}}},
|
||||
})},
|
||||
}}, promptkit.WithSchemaFS(schemas, "."))
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine before request resources exist: %v", err)
|
||||
}
|
||||
schemas["schema.json"] = &fstest.MapFile{Data: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object"}`)}
|
||||
artifactPath := filepath.Join(t.TempDir(), "note.txt")
|
||||
if err := os.WriteFile(artifactPath, []byte("request artifact"), 0o600); err != nil {
|
||||
t.Fatalf("write request artifact: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "structured", Inputs: map[string]promptkit.ArtifactRef{
|
||||
"note": promptkit.File(artifactPath),
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with resources supplied after construction: %v", err)
|
||||
}
|
||||
if len(prepared.Messages) != 1 || prepared.Messages[0].Content != "request artifact" || prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil {
|
||||
t.Fatalf("runtime artifact or schema was not prepared: %+v", prepared)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user