From f3148bfe52ff9d574c42232acd07f26468e9ab42 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 12 Oct 2026 17:45:18 +0000 Subject: [PATCH] Verify request validation parity --- docs/roadmap/implementation.md | 2 + validate_request_contract_test.go | 162 ++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 093c58b..a7d8dce 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -141,6 +141,8 @@ required inputs, variable presence, all supplied artifact bindings, representati resource/selection errors, and ordinary request forwarding. No provider or capacity-readiness interpretation has been introduced. +**Completed: 2026-10-12.** + ## Stage 3: Document Consumer Use And Verify The Integrated Feature Scope: canonical documentation, the maintained preparation example, and final diff --git a/validate_request_contract_test.go b/validate_request_contract_test.go index 431ab4f..b547af3 100644 --- a/validate_request_contract_test.go +++ b/validate_request_contract_test.go @@ -3,8 +3,10 @@ package promptkit_test import ( "context" "errors" + "path/filepath" "sync/atomic" "testing" + "testing/fstest" "gitea.maximumdirect.net/eric/promptkit" ) @@ -81,3 +83,163 @@ func TestValidateRequestMatchesPrepareContract(t *testing.T) { t.Fatalf("preparation operations made %d model calls, want none", got) } } + +func assertValidateRequestParity(t *testing.T, engine *promptkit.Engine, request promptkit.RunRequest, want error) *promptkit.PreparedRun { + t.Helper() + prepared, prepareErr := engine.Prepare(context.Background(), request) + validateErr := engine.ValidateRequest(context.Background(), request) + if want == nil { + if prepareErr != nil || prepared == nil { + t.Fatalf("Prepare() = (%+v, %v), want a prepared result", prepared, prepareErr) + } + if validateErr != nil { + t.Fatalf("ValidateRequest() error = %v, want nil", validateErr) + } + return prepared + } + if prepared != nil || !errors.Is(prepareErr, want) { + t.Fatalf("Prepare() = (%+v, %v), want error matching %v", prepared, prepareErr, want) + } + if !errors.Is(validateErr, want) { + t.Fatalf("ValidateRequest() error = %v, want error matching %v", validateErr, want) + } + return nil +} + +func TestValidateRequestUsesSelectedTemplateBranchesAndVariablePresence(t *testing.T) { + var modelCalls atomic.Int64 + profiles := map[string]any{ + "offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"}, + } + branchContent := `{{if eq .mode "detailed"}}{{input "details"}}{{end}}` + optionalDefinition := contractPrompt("offline", branchContent, []any{ + map[string]any{"name": "details", "required": false}, + }, "", noValidationOutput) + requiredDefinition := contractPrompt("offline", branchContent, []any{ + map[string]any{"name": "details", "required": true}, + }, "", noValidationOutput) + engine := requirementsEngine(t, map[string]any{ + "optional": oneVersionPrompt("optional", "v1", optionalDefinition)["optional"], + "required": oneVersionPrompt("required", "v1", requiredDefinition)["required"], + "direct": oneVersionPrompt("direct", "v1", contractPrompt("offline", `{{.audience}}`, nil, "", noValidationOutput))["direct"], + "index": oneVersionPrompt("index", "v1", contractPrompt("offline", `{{index . "audience"}}`, nil, "", noValidationOutput))["index"], + }, profiles, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls})) + + cases := []struct { + name string + request promptkit.RunRequest + want error + }{ + {name: "brief branch omits optional input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "brief"}}}, + {name: "detailed branch needs optional input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "detailed"}}, want: promptkit.ErrPromptRender}, + {name: "detailed branch accepts inline input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "detailed"}, Inputs: map[string]promptkit.ArtifactRef{"details": promptkit.Inline("Long form details")}}}, + {name: "declared required input is needed on brief branch", request: promptkit.RunRequest{PromptID: "required", Vars: map[string]string{"mode": "brief"}}, want: promptkit.ErrPromptRender}, + {name: "evaluated variable needs a supplied key", request: promptkit.RunRequest{PromptID: "direct"}, want: promptkit.ErrPromptRender}, + {name: "empty value and unrelated variable are accepted", request: promptkit.RunRequest{PromptID: "direct", Vars: map[string]string{"audience": "", "unrelated": "accepted"}}}, + {name: "missing index lookup remains optional", request: promptkit.RunRequest{PromptID: "index"}}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + assertValidateRequestParity(t, engine, test.request, test.want) + }) + } + if got := modelCalls.Load(); got != 0 { + t.Fatalf("preparation operations made %d model calls, want none", got) + } +} + +type canceledValidationArtifactReader struct{} + +func (canceledValidationArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) { + return nil, context.Canceled +} + +func TestValidateRequestPreservesResourceAndSelectionFailures(t *testing.T) { + var modelCalls atomic.Int64 + profiles := map[string]any{ + "offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"}, + } + branch := `{{if eq .mode "detailed"}}{{input "details"}}{{end}}` + definition := contractPrompt("offline", branch, []any{ + map[string]any{"name": "details", "required": false}, + }, "", noValidationOutput) + profilePrompt := contractPrompt("offline", "hello", nil, "", noValidationOutput) + schemaPrompt := contractPrompt("offline", "{}", nil, "", map[string]any{ + "format": "json", "validation_mode": "json_schema", "schema_path": "missing-schema.json", + }) + prompts := map[string]any{ + "branch": oneVersionPrompt("branch", "v1", definition)["branch"], + "profile": oneVersionPrompt("profile", "v1", profilePrompt)["profile"], + "schema": oneVersionPrompt("schema", "v1", schemaPrompt)["schema"], + } + engine := requirementsEngine(t, prompts, profiles, + promptkit.WithSchemaFS(fstest.MapFS{}, "."), + promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}), + ) + + t.Run("unselected supplied input is still loaded", func(t *testing.T) { + missingPath := filepath.Join(t.TempDir(), "missing-input.txt") + request := promptkit.RunRequest{ + PromptID: "branch", + Vars: map[string]string{"mode": "brief"}, + Inputs: map[string]promptkit.ArtifactRef{"details": promptkit.File(missingPath)}, + } + assertValidateRequestParity(t, engine, request, promptkit.ErrArtifactLoad) + }) + t.Run("missing selected profile", func(t *testing.T) { + request := promptkit.RunRequest{PromptID: "profile", ProfileID: "not-configured"} + assertValidateRequestParity(t, engine, request, promptkit.ErrProfileNotFound) + }) + t.Run("unavailable selected schema", func(t *testing.T) { + request := promptkit.RunRequest{PromptID: "schema"} + assertValidateRequestParity(t, engine, request, promptkit.ErrValidation) + }) + if got := modelCalls.Load(); got != 0 { + t.Fatalf("preparation operations made %d model calls, want none", got) + } +} + +func TestValidateRequestPreservesCanceledLoaderCategory(t *testing.T) { + var modelCalls atomic.Int64 + definition := contractPrompt("offline", `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`, []any{ + map[string]any{"name": "details", "required": false}, + }, "", noValidationOutput) + engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{ + "offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"}, + }, promptkit.WithArtifactReader(canceledValidationArtifactReader{}), + promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls})) + request := promptkit.RunRequest{ + PromptID: "synthetic", + Vars: map[string]string{"mode": "brief"}, + Inputs: map[string]promptkit.ArtifactRef{"details": promptkit.File("controlled-canceled-loader")}, + } + assertValidateRequestParity(t, engine, request, promptkit.ErrArtifactLoad) + if got := modelCalls.Load(); got != 0 { + t.Fatalf("preparation operations made %d model calls, want none", got) + } +} + +func TestValidateRequestForwardsProfileOverridesAndRequestSettings(t *testing.T) { + var modelCalls atomic.Int64 + definition := contractPrompt("partial", "base message", nil, "", noValidationOutput) + engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{ + "partial": map[string]any{"model": "partial-model"}, + }, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls})) + request := promptkit.RunRequest{ + PromptID: "synthetic", + SessionID: "request-session", + Execution: &promptkit.ExecutionTargetOverride{Endpoint: "http://override.example/v1"}, + AppendedMessages: []promptkit.RenderedMessage{{ + Role: promptkit.RoleAssistant, Content: "caller suffix", + }}, + } + prepared := assertValidateRequestParity(t, engine, request, nil) + if prepared.SelectedProfileID != "partial" || prepared.EffectiveModelParams.Model != "partial-model" || + prepared.EffectiveModelParams.Endpoint != "http://override.example/v1" || prepared.SessionID != "request-session" || + len(prepared.Messages) != 2 || prepared.Messages[1].Content != "caller suffix" { + t.Fatalf("preparation fixture did not exercise profile and request settings: %+v", prepared) + } + if got := modelCalls.Load(); got != 0 { + t.Fatalf("preparation operations made %d model calls, want none", got) + } +}