package promptkit_test import ( "context" "errors" "path/filepath" "sync/atomic" "testing" "testing/fstest" "gitea.maximumdirect.net/eric/promptkit" ) func TestValidateRequestMatchesPrepareContract(t *testing.T) { var modelCalls atomic.Int64 profiles := map[string]any{ "offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"}, } newEngine := func(t *testing.T, definition map[string]any) *promptkit.Engine { t.Helper() return requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), profiles, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls})) } t.Run("successful request", func(t *testing.T) { engine := newEngine(t, contractPrompt("offline", "hello", nil, "", noValidationOutput)) assertValidateRequestParity(t, engine, promptkit.RunRequest{PromptID: "synthetic"}, nil) }) t.Run("nil engine", func(t *testing.T) { var engine *promptkit.Engine request := promptkit.RunRequest{PromptID: "synthetic"} assertValidateRequestParity(t, engine, request, promptkit.ErrInvalidConfig) }) t.Run("invalid appended message", func(t *testing.T) { engine := newEngine(t, contractPrompt("offline", "hello", nil, "", noValidationOutput)) request := promptkit.RunRequest{ PromptID: "synthetic", AppendedMessages: []promptkit.RenderedMessage{{ Role: "unsupported-role", Content: "not sent to a model", }}, } assertValidateRequestParity(t, engine, request, promptkit.ErrInvalidRequest) }) t.Run("missing required input", func(t *testing.T) { definition := contractPrompt("offline", `{{input "required"}}`, []any{ map[string]any{"name": "required", "required": true}, }, "", noValidationOutput) engine := newEngine(t, definition) assertValidateRequestParity(t, engine, promptkit.RunRequest{PromptID: "synthetic"}, promptkit.ErrPromptRender) }) if got := modelCalls.Load(); got != 0 { t.Fatalf("preparation operations made %d model calls, want none", got) } } func assertValidateRequestParity(t *testing.T, engine *promptkit.Engine, request promptkit.RunRequest, want error) *promptkit.PreparedRun { t.Helper() prepared, prepareErr := engine.Prepare(context.Background(), request) validateErr := engine.ValidateRequest(context.Background(), request) if want == nil { if prepareErr != nil || prepared == nil { t.Fatalf("Prepare() = (%+v, %v), want a prepared result", prepared, prepareErr) } if validateErr != nil { t.Fatalf("ValidateRequest() error = %v, want nil", validateErr) } return prepared } if prepared != nil || !errors.Is(prepareErr, want) { t.Fatalf("Prepare() = (%+v, %v), want error matching %v", prepared, prepareErr, want) } if !errors.Is(validateErr, want) { t.Fatalf("ValidateRequest() error = %v, want error matching %v", validateErr, want) } return nil } func TestValidateRequestUsesSelectedTemplateBranchesAndVariablePresence(t *testing.T) { var modelCalls atomic.Int64 profiles := map[string]any{ "offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"}, } branchContent := `{{if eq .mode "detailed"}}{{input "details"}}{{end}}` optionalDefinition := contractPrompt("offline", branchContent, []any{ map[string]any{"name": "details", "required": false}, }, "", noValidationOutput) requiredDefinition := contractPrompt("offline", branchContent, []any{ map[string]any{"name": "details", "required": true}, }, "", noValidationOutput) engine := requirementsEngine(t, map[string]any{ "optional": oneVersionPrompt("optional", "v1", optionalDefinition)["optional"], "required": oneVersionPrompt("required", "v1", requiredDefinition)["required"], "direct": oneVersionPrompt("direct", "v1", contractPrompt("offline", `{{.audience}}`, nil, "", noValidationOutput))["direct"], "index": oneVersionPrompt("index", "v1", contractPrompt("offline", `{{index . "audience"}}`, nil, "", noValidationOutput))["index"], }, profiles, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls})) cases := []struct { name string request promptkit.RunRequest want error }{ {name: "brief branch omits optional input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "brief"}}}, {name: "detailed branch needs optional input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "detailed"}}, want: promptkit.ErrPromptRender}, {name: "detailed branch accepts inline input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "detailed"}, Inputs: map[string]promptkit.ArtifactRef{"details": promptkit.Inline("Long form details")}}}, {name: "declared required input is needed on brief branch", request: promptkit.RunRequest{PromptID: "required", Vars: map[string]string{"mode": "brief"}}, want: promptkit.ErrPromptRender}, {name: "evaluated variable needs a supplied key", request: promptkit.RunRequest{PromptID: "direct"}, want: promptkit.ErrPromptRender}, {name: "empty value and unrelated variable are accepted", request: promptkit.RunRequest{PromptID: "direct", Vars: map[string]string{"audience": "", "unrelated": "accepted"}}}, {name: "missing index lookup remains optional", request: promptkit.RunRequest{PromptID: "index"}}, } for _, test := range cases { t.Run(test.name, func(t *testing.T) { assertValidateRequestParity(t, engine, test.request, test.want) }) } if got := modelCalls.Load(); got != 0 { t.Fatalf("preparation operations made %d model calls, want none", got) } } type canceledValidationArtifactReader struct{} func (canceledValidationArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) { return nil, context.Canceled } func TestValidateRequestPreservesResourceAndSelectionFailures(t *testing.T) { var modelCalls atomic.Int64 profiles := map[string]any{ "offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"}, } branch := `{{if eq .mode "detailed"}}{{input "details"}}{{end}}` definition := contractPrompt("offline", branch, []any{ map[string]any{"name": "details", "required": false}, }, "", noValidationOutput) profilePrompt := contractPrompt("offline", "hello", nil, "", noValidationOutput) schemaPrompt := contractPrompt("offline", "{}", nil, "", map[string]any{ "format": "json", "validation_mode": "json_schema", "schema_path": "missing-schema.json", }) prompts := map[string]any{ "branch": oneVersionPrompt("branch", "v1", definition)["branch"], "profile": oneVersionPrompt("profile", "v1", profilePrompt)["profile"], "schema": oneVersionPrompt("schema", "v1", schemaPrompt)["schema"], } engine := requirementsEngine(t, prompts, profiles, promptkit.WithSchemaFS(fstest.MapFS{}, "."), promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}), ) t.Run("unselected supplied input is still loaded", func(t *testing.T) { missingPath := filepath.Join(t.TempDir(), "missing-input.txt") request := promptkit.RunRequest{ PromptID: "branch", Vars: map[string]string{"mode": "brief"}, Inputs: map[string]promptkit.ArtifactRef{"details": promptkit.File(missingPath)}, } assertValidateRequestParity(t, engine, request, promptkit.ErrArtifactLoad) }) t.Run("missing selected profile", func(t *testing.T) { request := promptkit.RunRequest{PromptID: "profile", ProfileID: "not-configured"} assertValidateRequestParity(t, engine, request, promptkit.ErrProfileNotFound) }) t.Run("unavailable selected schema", func(t *testing.T) { request := promptkit.RunRequest{PromptID: "schema"} assertValidateRequestParity(t, engine, request, promptkit.ErrValidation) }) if got := modelCalls.Load(); got != 0 { t.Fatalf("preparation operations made %d model calls, want none", got) } } func TestValidateRequestPreservesCanceledLoaderCategory(t *testing.T) { var modelCalls atomic.Int64 definition := contractPrompt("offline", `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`, []any{ map[string]any{"name": "details", "required": false}, }, "", noValidationOutput) engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{ "offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"}, }, promptkit.WithArtifactReader(canceledValidationArtifactReader{}), promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls})) request := promptkit.RunRequest{ PromptID: "synthetic", Vars: map[string]string{"mode": "brief"}, Inputs: map[string]promptkit.ArtifactRef{"details": promptkit.File("controlled-canceled-loader")}, } assertValidateRequestParity(t, engine, request, promptkit.ErrArtifactLoad) if got := modelCalls.Load(); got != 0 { t.Fatalf("preparation operations made %d model calls, want none", got) } } func TestValidateRequestForwardsProfileOverridesAndRequestSettings(t *testing.T) { var modelCalls atomic.Int64 definition := contractPrompt("partial", "base message", nil, "", noValidationOutput) engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{ "partial": map[string]any{"model": "partial-model"}, }, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls})) request := promptkit.RunRequest{ PromptID: "synthetic", SessionID: "request-session", Execution: &promptkit.ExecutionTargetOverride{Endpoint: "http://override.example/v1"}, AppendedMessages: []promptkit.RenderedMessage{{ Role: promptkit.RoleAssistant, Content: "caller suffix", }}, } prepared := assertValidateRequestParity(t, engine, request, nil) if prepared.SelectedProfileID != "partial" || prepared.EffectiveModelParams.Model != "partial-model" || prepared.EffectiveModelParams.Endpoint != "http://override.example/v1" || prepared.SessionID != "request-session" || len(prepared.Messages) != 2 || prepared.Messages[1].Content != "caller suffix" { t.Fatalf("preparation fixture did not exercise profile and request settings: %+v", prepared) } if got := modelCalls.Load(); got != 0 { t.Fatalf("preparation operations made %d model calls, want none", got) } }