246 lines
11 KiB
Go
246 lines
11 KiB
Go
package promptkit_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"path/filepath"
|
|
"sync/atomic"
|
|
"testing"
|
|
"testing/fstest"
|
|
|
|
"gitea.maximumdirect.net/eric/promptkit"
|
|
)
|
|
|
|
func TestValidateRequestMatchesPrepareContract(t *testing.T) {
|
|
var modelCalls atomic.Int64
|
|
profiles := map[string]any{
|
|
"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
|
|
}
|
|
newEngine := func(t *testing.T, definition map[string]any) *promptkit.Engine {
|
|
t.Helper()
|
|
return requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), profiles,
|
|
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
|
|
}
|
|
testParity := func(t *testing.T, engine *promptkit.Engine, request promptkit.RunRequest, want error) {
|
|
t.Helper()
|
|
prepared, prepareErr := engine.Prepare(context.Background(), request)
|
|
validateErr := engine.ValidateRequest(context.Background(), request)
|
|
if want == nil {
|
|
if prepareErr != nil || prepared == nil {
|
|
t.Fatalf("Prepare() = (%+v, %v), want a prepared result", prepared, prepareErr)
|
|
}
|
|
if validateErr != nil {
|
|
t.Fatalf("ValidateRequest() error = %v, want nil", validateErr)
|
|
}
|
|
return
|
|
}
|
|
if prepared != nil || !errors.Is(prepareErr, want) {
|
|
t.Fatalf("Prepare() = (%+v, %v), want error matching %v", prepared, prepareErr, want)
|
|
}
|
|
if !errors.Is(validateErr, want) {
|
|
t.Fatalf("ValidateRequest() error = %v, want error matching %v", validateErr, want)
|
|
}
|
|
}
|
|
|
|
t.Run("successful request", func(t *testing.T) {
|
|
engine := newEngine(t, contractPrompt("offline", "hello", nil, "", noValidationOutput))
|
|
testParity(t, engine, promptkit.RunRequest{PromptID: "synthetic"}, nil)
|
|
})
|
|
|
|
t.Run("nil engine", func(t *testing.T) {
|
|
var engine *promptkit.Engine
|
|
request := promptkit.RunRequest{PromptID: "synthetic"}
|
|
prepared, prepareErr := engine.Prepare(context.Background(), request)
|
|
validateErr := engine.ValidateRequest(context.Background(), request)
|
|
if prepared != nil || !errors.Is(prepareErr, promptkit.ErrInvalidConfig) {
|
|
t.Fatalf("Prepare() = (%+v, %v), want ErrInvalidConfig", prepared, prepareErr)
|
|
}
|
|
if !errors.Is(validateErr, promptkit.ErrInvalidConfig) {
|
|
t.Fatalf("ValidateRequest() error = %v, want ErrInvalidConfig", validateErr)
|
|
}
|
|
})
|
|
|
|
t.Run("invalid appended message", func(t *testing.T) {
|
|
engine := newEngine(t, contractPrompt("offline", "hello", nil, "", noValidationOutput))
|
|
request := promptkit.RunRequest{
|
|
PromptID: "synthetic",
|
|
AppendedMessages: []promptkit.RenderedMessage{{
|
|
Role: "unsupported-role", Content: "not sent to a model",
|
|
}},
|
|
}
|
|
testParity(t, engine, request, promptkit.ErrInvalidRequest)
|
|
})
|
|
|
|
t.Run("missing required input", func(t *testing.T) {
|
|
definition := contractPrompt("offline", `{{input "required"}}`, []any{
|
|
map[string]any{"name": "required", "required": true},
|
|
}, "", noValidationOutput)
|
|
engine := newEngine(t, definition)
|
|
testParity(t, engine, promptkit.RunRequest{PromptID: "synthetic"}, promptkit.ErrPromptRender)
|
|
})
|
|
|
|
if got := modelCalls.Load(); got != 0 {
|
|
t.Fatalf("preparation operations made %d model calls, want none", got)
|
|
}
|
|
}
|
|
|
|
func assertValidateRequestParity(t *testing.T, engine *promptkit.Engine, request promptkit.RunRequest, want error) *promptkit.PreparedRun {
|
|
t.Helper()
|
|
prepared, prepareErr := engine.Prepare(context.Background(), request)
|
|
validateErr := engine.ValidateRequest(context.Background(), request)
|
|
if want == nil {
|
|
if prepareErr != nil || prepared == nil {
|
|
t.Fatalf("Prepare() = (%+v, %v), want a prepared result", prepared, prepareErr)
|
|
}
|
|
if validateErr != nil {
|
|
t.Fatalf("ValidateRequest() error = %v, want nil", validateErr)
|
|
}
|
|
return prepared
|
|
}
|
|
if prepared != nil || !errors.Is(prepareErr, want) {
|
|
t.Fatalf("Prepare() = (%+v, %v), want error matching %v", prepared, prepareErr, want)
|
|
}
|
|
if !errors.Is(validateErr, want) {
|
|
t.Fatalf("ValidateRequest() error = %v, want error matching %v", validateErr, want)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func TestValidateRequestUsesSelectedTemplateBranchesAndVariablePresence(t *testing.T) {
|
|
var modelCalls atomic.Int64
|
|
profiles := map[string]any{
|
|
"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
|
|
}
|
|
branchContent := `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`
|
|
optionalDefinition := contractPrompt("offline", branchContent, []any{
|
|
map[string]any{"name": "details", "required": false},
|
|
}, "", noValidationOutput)
|
|
requiredDefinition := contractPrompt("offline", branchContent, []any{
|
|
map[string]any{"name": "details", "required": true},
|
|
}, "", noValidationOutput)
|
|
engine := requirementsEngine(t, map[string]any{
|
|
"optional": oneVersionPrompt("optional", "v1", optionalDefinition)["optional"],
|
|
"required": oneVersionPrompt("required", "v1", requiredDefinition)["required"],
|
|
"direct": oneVersionPrompt("direct", "v1", contractPrompt("offline", `{{.audience}}`, nil, "", noValidationOutput))["direct"],
|
|
"index": oneVersionPrompt("index", "v1", contractPrompt("offline", `{{index . "audience"}}`, nil, "", noValidationOutput))["index"],
|
|
}, profiles, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
|
|
|
|
cases := []struct {
|
|
name string
|
|
request promptkit.RunRequest
|
|
want error
|
|
}{
|
|
{name: "brief branch omits optional input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "brief"}}},
|
|
{name: "detailed branch needs optional input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "detailed"}}, want: promptkit.ErrPromptRender},
|
|
{name: "detailed branch accepts inline input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "detailed"}, Inputs: map[string]promptkit.ArtifactRef{"details": promptkit.Inline("Long form details")}}},
|
|
{name: "declared required input is needed on brief branch", request: promptkit.RunRequest{PromptID: "required", Vars: map[string]string{"mode": "brief"}}, want: promptkit.ErrPromptRender},
|
|
{name: "evaluated variable needs a supplied key", request: promptkit.RunRequest{PromptID: "direct"}, want: promptkit.ErrPromptRender},
|
|
{name: "empty value and unrelated variable are accepted", request: promptkit.RunRequest{PromptID: "direct", Vars: map[string]string{"audience": "", "unrelated": "accepted"}}},
|
|
{name: "missing index lookup remains optional", request: promptkit.RunRequest{PromptID: "index"}},
|
|
}
|
|
for _, test := range cases {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
assertValidateRequestParity(t, engine, test.request, test.want)
|
|
})
|
|
}
|
|
if got := modelCalls.Load(); got != 0 {
|
|
t.Fatalf("preparation operations made %d model calls, want none", got)
|
|
}
|
|
}
|
|
|
|
type canceledValidationArtifactReader struct{}
|
|
|
|
func (canceledValidationArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
|
return nil, context.Canceled
|
|
}
|
|
|
|
func TestValidateRequestPreservesResourceAndSelectionFailures(t *testing.T) {
|
|
var modelCalls atomic.Int64
|
|
profiles := map[string]any{
|
|
"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
|
|
}
|
|
branch := `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`
|
|
definition := contractPrompt("offline", branch, []any{
|
|
map[string]any{"name": "details", "required": false},
|
|
}, "", noValidationOutput)
|
|
profilePrompt := contractPrompt("offline", "hello", nil, "", noValidationOutput)
|
|
schemaPrompt := contractPrompt("offline", "{}", nil, "", map[string]any{
|
|
"format": "json", "validation_mode": "json_schema", "schema_path": "missing-schema.json",
|
|
})
|
|
prompts := map[string]any{
|
|
"branch": oneVersionPrompt("branch", "v1", definition)["branch"],
|
|
"profile": oneVersionPrompt("profile", "v1", profilePrompt)["profile"],
|
|
"schema": oneVersionPrompt("schema", "v1", schemaPrompt)["schema"],
|
|
}
|
|
engine := requirementsEngine(t, prompts, profiles,
|
|
promptkit.WithSchemaFS(fstest.MapFS{}, "."),
|
|
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}),
|
|
)
|
|
|
|
t.Run("unselected supplied input is still loaded", func(t *testing.T) {
|
|
missingPath := filepath.Join(t.TempDir(), "missing-input.txt")
|
|
request := promptkit.RunRequest{
|
|
PromptID: "branch",
|
|
Vars: map[string]string{"mode": "brief"},
|
|
Inputs: map[string]promptkit.ArtifactRef{"details": promptkit.File(missingPath)},
|
|
}
|
|
assertValidateRequestParity(t, engine, request, promptkit.ErrArtifactLoad)
|
|
})
|
|
t.Run("missing selected profile", func(t *testing.T) {
|
|
request := promptkit.RunRequest{PromptID: "profile", ProfileID: "not-configured"}
|
|
assertValidateRequestParity(t, engine, request, promptkit.ErrProfileNotFound)
|
|
})
|
|
t.Run("unavailable selected schema", func(t *testing.T) {
|
|
request := promptkit.RunRequest{PromptID: "schema"}
|
|
assertValidateRequestParity(t, engine, request, promptkit.ErrValidation)
|
|
})
|
|
if got := modelCalls.Load(); got != 0 {
|
|
t.Fatalf("preparation operations made %d model calls, want none", got)
|
|
}
|
|
}
|
|
|
|
func TestValidateRequestPreservesCanceledLoaderCategory(t *testing.T) {
|
|
var modelCalls atomic.Int64
|
|
definition := contractPrompt("offline", `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`, []any{
|
|
map[string]any{"name": "details", "required": false},
|
|
}, "", noValidationOutput)
|
|
engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{
|
|
"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
|
|
}, promptkit.WithArtifactReader(canceledValidationArtifactReader{}),
|
|
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
|
|
request := promptkit.RunRequest{
|
|
PromptID: "synthetic",
|
|
Vars: map[string]string{"mode": "brief"},
|
|
Inputs: map[string]promptkit.ArtifactRef{"details": promptkit.File("controlled-canceled-loader")},
|
|
}
|
|
assertValidateRequestParity(t, engine, request, promptkit.ErrArtifactLoad)
|
|
if got := modelCalls.Load(); got != 0 {
|
|
t.Fatalf("preparation operations made %d model calls, want none", got)
|
|
}
|
|
}
|
|
|
|
func TestValidateRequestForwardsProfileOverridesAndRequestSettings(t *testing.T) {
|
|
var modelCalls atomic.Int64
|
|
definition := contractPrompt("partial", "base message", nil, "", noValidationOutput)
|
|
engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{
|
|
"partial": map[string]any{"model": "partial-model"},
|
|
}, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
|
|
request := promptkit.RunRequest{
|
|
PromptID: "synthetic",
|
|
SessionID: "request-session",
|
|
Execution: &promptkit.ExecutionTargetOverride{Endpoint: "http://override.example/v1"},
|
|
AppendedMessages: []promptkit.RenderedMessage{{
|
|
Role: promptkit.RoleAssistant, Content: "caller suffix",
|
|
}},
|
|
}
|
|
prepared := assertValidateRequestParity(t, engine, request, nil)
|
|
if prepared.SelectedProfileID != "partial" || prepared.EffectiveModelParams.Model != "partial-model" ||
|
|
prepared.EffectiveModelParams.Endpoint != "http://override.example/v1" || prepared.SessionID != "request-session" ||
|
|
len(prepared.Messages) != 2 || prepared.Messages[1].Content != "caller suffix" {
|
|
t.Fatalf("preparation fixture did not exercise profile and request settings: %+v", prepared)
|
|
}
|
|
if got := modelCalls.Load(); got != 0 {
|
|
t.Fatalf("preparation operations made %d model calls, want none", got)
|
|
}
|
|
}
|