142 lines
7.7 KiB
Go
142 lines
7.7 KiB
Go
package promptkit_test
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
"go.maximumdirect.net/configkit"
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
func contentHashEngine(t *testing.T, promptID, profileID, backendID, content string, output map[string]any, cache bool, opts ...promptkit.Option) *promptkit.Engine {
|
|
t.Helper()
|
|
message := map[string]any{"role": "user", "content": content}
|
|
if cache {
|
|
message["cache_control"] = map[string]any{"type": "ephemeral"}
|
|
}
|
|
config := promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.Mapping("hash test", map[string]any{
|
|
"backends": []any{map[string]any{"id": backendID, "endpoint": "https://example.test/v1"}},
|
|
"profiles": []any{map[string]any{"id": profileID, "backend": backendID, "model": "model"}},
|
|
"prompts": []any{map[string]any{"id": promptID, "default_profile": profileID, "inputs": []any{map[string]any{"name": "input", "required": false}}, "messages": []any{message}, "output": output}},
|
|
})}}}
|
|
engine, err := promptkit.NewEngine(config, opts...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return engine
|
|
}
|
|
|
|
func prepareContentHash(t *testing.T, engine *promptkit.Engine, request promptkit.RunRequest) *promptkit.PreparedRun {
|
|
t.Helper()
|
|
prepared, err := engine.Prepare(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if prepared.MessageContentHash == "" {
|
|
t.Fatal("missing content hash")
|
|
}
|
|
return prepared
|
|
}
|
|
|
|
func TestMessageContentHashIgnoresPreparationMetadata(t *testing.T) {
|
|
textOutput := map[string]any{"format": "text", "validation_mode": "none"}
|
|
baseEngine := contentHashEngine(t, "first", "alpha", "one", "same", textOutput, false)
|
|
base := prepareContentHash(t, baseEngine, promptkit.RunRequest{PromptID: "first"})
|
|
repeated := prepareContentHash(t, baseEngine, promptkit.RunRequest{PromptID: "first"})
|
|
otherPrompt := contentHashEngine(t, "second", "alpha", "one", "same", textOutput, false)
|
|
otherProfile := contentHashEngine(t, "first", "beta", "one", "same", textOutput, false)
|
|
otherBackend := contentHashEngine(t, "first", "alpha", "two", "same", textOutput, false)
|
|
credential := prepareContentHash(t, baseEngine, promptkit.RunRequest{PromptID: "first", APIKey: "synthetic-key"})
|
|
settings := prepareContentHash(t, baseEngine, promptkit.RunRequest{PromptID: "first", Execution: &promptkit.ExecutionTargetOverride{Model: "other-model"}})
|
|
session := prepareContentHash(t, baseEngine, promptkit.RunRequest{PromptID: "first", SessionID: "other-session"})
|
|
cacheEngine := contentHashEngine(t, "first", "alpha", "one", "same", textOutput, true)
|
|
cached := prepareContentHash(t, cacheEngine, promptkit.RunRequest{PromptID: "first"})
|
|
schemaEngine := contentHashEngine(t, "first", "alpha", "one", "same", map[string]any{"format": "json", "validation_mode": "json_schema", "schema": map[string]any{"type": "object"}}, false)
|
|
schema := prepareContentHash(t, schemaEngine, promptkit.RunRequest{PromptID: "first"})
|
|
for name, prepared := range map[string]*promptkit.PreparedRun{
|
|
"repeat": repeated, "prompt": prepareContentHash(t, otherPrompt, promptkit.RunRequest{PromptID: "second"}),
|
|
"profile": prepareContentHash(t, otherProfile, promptkit.RunRequest{PromptID: "first"}),
|
|
"backend": prepareContentHash(t, otherBackend, promptkit.RunRequest{PromptID: "first"}),
|
|
"credential": credential, "settings": settings, "session": session, "cache": cached, "schema": schema,
|
|
} {
|
|
if prepared.MessageContentHash != base.MessageContentHash {
|
|
t.Errorf("%s changed content hash", name)
|
|
}
|
|
}
|
|
if session.RenderedPromptHash == base.RenderedPromptHash || cached.RenderedPromptHash == base.RenderedPromptHash {
|
|
t.Fatal("session and cache should each change rendered prompt hash")
|
|
}
|
|
}
|
|
|
|
func TestMessageContentHashUsesRenderedInputBytes(t *testing.T) {
|
|
reader := &mutablePreparedArtifactReader{body: "body", hash: "first-hash"}
|
|
engine := contentHashEngine(t, "prompt", "profile", "backend", `{{input "input"}} {{.label}}`, map[string]any{"format": "text", "validation_mode": "none"}, false, promptkit.WithArtifactReader(reader))
|
|
request := promptkit.RunRequest{PromptID: "prompt", Inputs: map[string]promptkit.ArtifactRef{"input": promptkit.File("first")}, Vars: map[string]string{"label": "label", "unused": "first"}}
|
|
base := prepareContentHash(t, engine, request)
|
|
reader.set("body", "other-hash")
|
|
request.Inputs["input"] = promptkit.File("other")
|
|
request.Vars["unused"] = "other"
|
|
unchanged := prepareContentHash(t, engine, request)
|
|
if unchanged.MessageContentHash != base.MessageContentHash {
|
|
t.Fatal("unused variable or artifact metadata changed hash")
|
|
}
|
|
request.Vars["label"] = "changed"
|
|
if prepareContentHash(t, engine, request).MessageContentHash == base.MessageContentHash {
|
|
t.Fatal("rendered variable did not change hash")
|
|
}
|
|
request.Vars["label"] = "label"
|
|
reader.set("changed body", "other-hash")
|
|
if prepareContentHash(t, engine, request).MessageContentHash == base.MessageContentHash {
|
|
t.Fatal("rendered input did not change hash")
|
|
}
|
|
}
|
|
|
|
func TestMessageContentHashIncludesHistoryAndAppendedMessages(t *testing.T) {
|
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}
|
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
|
initial, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
request := promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange, AppendedMessages: []promptkit.RenderedMessage{{Role: "user", Content: "extra"}}}
|
|
continued := prepareContentHash(t, engine, request)
|
|
explicit := prepareContentHash(t, engine, promptkit.RunRequest{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{{Role: "assistant", Content: "answer"}, {Role: "user", Content: "question"}, {Role: "user", Content: "extra"}}})
|
|
if continued.MessageContentHash != explicit.MessageContentHash {
|
|
t.Fatal("equivalent effective messages hashed differently")
|
|
}
|
|
for _, msg := range []promptkit.RenderedMessage{{Role: "assistant", Content: "extra"}, {Role: "user", Content: "changed"}} {
|
|
request.AppendedMessages = []promptkit.RenderedMessage{msg}
|
|
if prepareContentHash(t, engine, request).MessageContentHash == continued.MessageContentHash {
|
|
t.Fatal("appended message change did not affect hash")
|
|
}
|
|
}
|
|
otherClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "different answer"}}
|
|
otherEngine := newSimpleEngine(t, "question", promptkit.WithLLMClient(otherClient))
|
|
other, err := otherEngine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
request.ContinueFrom = other.Exchange
|
|
request.AppendedMessages = []promptkit.RenderedMessage{{Role: "user", Content: "extra"}}
|
|
if prepareContentHash(t, engine, request).MessageContentHash == continued.MessageContentHash {
|
|
t.Fatal("history text change did not affect hash")
|
|
}
|
|
firstHistory, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true, AppendedMessages: []promptkit.RenderedMessage{{Role: "user", Content: "history"}}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
secondHistory, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true, AppendedMessages: []promptkit.RenderedMessage{{Role: "assistant", Content: "history"}}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
request.ContinueFrom = firstHistory.Exchange
|
|
firstHash := prepareContentHash(t, engine, request).MessageContentHash
|
|
request.ContinueFrom = secondHistory.Exchange
|
|
if prepareContentHash(t, engine, request).MessageContentHash == firstHash {
|
|
t.Fatal("history role change did not affect hash")
|
|
}
|
|
if len(client.requests) != 3 || len(otherClient.requests) != 1 {
|
|
t.Fatal("preparation invoked the model")
|
|
}
|
|
}
|