Verify request validation parity

2026-10-12 17:45:18 +00:00
parent f315620bf7
commit f3148bfe52
2 changed files with 164 additions and 0 deletions
--- a/docs/roadmap/implementation.md
+++ b/docs/roadmap/implementation.md
@@ -141,6 +141,8 @@ required inputs, variable presence, all supplied artifact bindings, representati
 resource/selection errors, and ordinary request forwarding. No provider or
 capacity-readiness interpretation has been introduced.

+**Completed: 2026-10-12.**
+
 ## Stage 3: Document Consumer Use And Verify The Integrated Feature

 Scope: canonical documentation, the maintained preparation example, and final
--- a/validate_request_contract_test.go
+++ b/validate_request_contract_test.go
@@ -3,8 +3,10 @@ package promptkit_test
 import (
 	"context"
 	"errors"
+	"path/filepath"
 	"sync/atomic"
 	"testing"
+	"testing/fstest"

 	"gitea.maximumdirect.net/eric/promptkit"
 )
@@ -81,3 +83,163 @@ func TestValidateRequestMatchesPrepareContract(t *testing.T) {
 		t.Fatalf("preparation operations made %d model calls, want none", got)
 	}
 }
+
+func assertValidateRequestParity(t *testing.T, engine *promptkit.Engine, request promptkit.RunRequest, want error) *promptkit.PreparedRun {
+	t.Helper()
+	prepared, prepareErr := engine.Prepare(context.Background(), request)
+	validateErr := engine.ValidateRequest(context.Background(), request)
+	if want == nil {
+		if prepareErr != nil || prepared == nil {
+			t.Fatalf("Prepare() = (%+v, %v), want a prepared result", prepared, prepareErr)
+		}
+		if validateErr != nil {
+			t.Fatalf("ValidateRequest() error = %v, want nil", validateErr)
+		}
+		return prepared
+	}
+	if prepared != nil || !errors.Is(prepareErr, want) {
+		t.Fatalf("Prepare() = (%+v, %v), want error matching %v", prepared, prepareErr, want)
+	}
+	if !errors.Is(validateErr, want) {
+		t.Fatalf("ValidateRequest() error = %v, want error matching %v", validateErr, want)
+	}
+	return nil
+}
+
+func TestValidateRequestUsesSelectedTemplateBranchesAndVariablePresence(t *testing.T) {
+	var modelCalls atomic.Int64
+	profiles := map[string]any{
+		"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
+	}
+	branchContent := `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`
+	optionalDefinition := contractPrompt("offline", branchContent, []any{
+		map[string]any{"name": "details", "required": false},
+	}, "", noValidationOutput)
+	requiredDefinition := contractPrompt("offline", branchContent, []any{
+		map[string]any{"name": "details", "required": true},
+	}, "", noValidationOutput)
+	engine := requirementsEngine(t, map[string]any{
+		"optional": oneVersionPrompt("optional", "v1", optionalDefinition)["optional"],
+		"required": oneVersionPrompt("required", "v1", requiredDefinition)["required"],
+		"direct":   oneVersionPrompt("direct", "v1", contractPrompt("offline", `{{.audience}}`, nil, "", noValidationOutput))["direct"],
+		"index":    oneVersionPrompt("index", "v1", contractPrompt("offline", `{{index . "audience"}}`, nil, "", noValidationOutput))["index"],
+	}, profiles, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
+
+	cases := []struct {
+		name    string
+		request promptkit.RunRequest
+		want    error
+	}{
+		{name: "brief branch omits optional input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "brief"}}},
+		{name: "detailed branch needs optional input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "detailed"}}, want: promptkit.ErrPromptRender},
+		{name: "detailed branch accepts inline input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "detailed"}, Inputs: map[string]promptkit.ArtifactRef{"details": promptkit.Inline("Long form details")}}},
+		{name: "declared required input is needed on brief branch", request: promptkit.RunRequest{PromptID: "required", Vars: map[string]string{"mode": "brief"}}, want: promptkit.ErrPromptRender},
+		{name: "evaluated variable needs a supplied key", request: promptkit.RunRequest{PromptID: "direct"}, want: promptkit.ErrPromptRender},
+		{name: "empty value and unrelated variable are accepted", request: promptkit.RunRequest{PromptID: "direct", Vars: map[string]string{"audience": "", "unrelated": "accepted"}}},
+		{name: "missing index lookup remains optional", request: promptkit.RunRequest{PromptID: "index"}},
+	}
+	for _, test := range cases {
+		t.Run(test.name, func(t *testing.T) {
+			assertValidateRequestParity(t, engine, test.request, test.want)
+		})
+	}
+	if got := modelCalls.Load(); got != 0 {
+		t.Fatalf("preparation operations made %d model calls, want none", got)
+	}
+}
+
+type canceledValidationArtifactReader struct{}
+
+func (canceledValidationArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
+	return nil, context.Canceled
+}
+
+func TestValidateRequestPreservesResourceAndSelectionFailures(t *testing.T) {
+	var modelCalls atomic.Int64
+	profiles := map[string]any{
+		"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
+	}
+	branch := `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`
+	definition := contractPrompt("offline", branch, []any{
+		map[string]any{"name": "details", "required": false},
+	}, "", noValidationOutput)
+	profilePrompt := contractPrompt("offline", "hello", nil, "", noValidationOutput)
+	schemaPrompt := contractPrompt("offline", "{}", nil, "", map[string]any{
+		"format": "json", "validation_mode": "json_schema", "schema_path": "missing-schema.json",
+	})
+	prompts := map[string]any{
+		"branch":  oneVersionPrompt("branch", "v1", definition)["branch"],
+		"profile": oneVersionPrompt("profile", "v1", profilePrompt)["profile"],
+		"schema":  oneVersionPrompt("schema", "v1", schemaPrompt)["schema"],
+	}
+	engine := requirementsEngine(t, prompts, profiles,
+		promptkit.WithSchemaFS(fstest.MapFS{}, "."),
+		promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}),
+	)
+
+	t.Run("unselected supplied input is still loaded", func(t *testing.T) {
+		missingPath := filepath.Join(t.TempDir(), "missing-input.txt")
+		request := promptkit.RunRequest{
+			PromptID: "branch",
+			Vars:     map[string]string{"mode": "brief"},
+			Inputs:   map[string]promptkit.ArtifactRef{"details": promptkit.File(missingPath)},
+		}
+		assertValidateRequestParity(t, engine, request, promptkit.ErrArtifactLoad)
+	})
+	t.Run("missing selected profile", func(t *testing.T) {
+		request := promptkit.RunRequest{PromptID: "profile", ProfileID: "not-configured"}
+		assertValidateRequestParity(t, engine, request, promptkit.ErrProfileNotFound)
+	})
+	t.Run("unavailable selected schema", func(t *testing.T) {
+		request := promptkit.RunRequest{PromptID: "schema"}
+		assertValidateRequestParity(t, engine, request, promptkit.ErrValidation)
+	})
+	if got := modelCalls.Load(); got != 0 {
+		t.Fatalf("preparation operations made %d model calls, want none", got)
+	}
+}
+
+func TestValidateRequestPreservesCanceledLoaderCategory(t *testing.T) {
+	var modelCalls atomic.Int64
+	definition := contractPrompt("offline", `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`, []any{
+		map[string]any{"name": "details", "required": false},
+	}, "", noValidationOutput)
+	engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{
+		"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
+	}, promptkit.WithArtifactReader(canceledValidationArtifactReader{}),
+		promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
+	request := promptkit.RunRequest{
+		PromptID: "synthetic",
+		Vars:     map[string]string{"mode": "brief"},
+		Inputs:   map[string]promptkit.ArtifactRef{"details": promptkit.File("controlled-canceled-loader")},
+	}
+	assertValidateRequestParity(t, engine, request, promptkit.ErrArtifactLoad)
+	if got := modelCalls.Load(); got != 0 {
+		t.Fatalf("preparation operations made %d model calls, want none", got)
+	}
+}
+
+func TestValidateRequestForwardsProfileOverridesAndRequestSettings(t *testing.T) {
+	var modelCalls atomic.Int64
+	definition := contractPrompt("partial", "base message", nil, "", noValidationOutput)
+	engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{
+		"partial": map[string]any{"model": "partial-model"},
+	}, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
+	request := promptkit.RunRequest{
+		PromptID:  "synthetic",
+		SessionID: "request-session",
+		Execution: &promptkit.ExecutionTargetOverride{Endpoint: "http://override.example/v1"},
+		AppendedMessages: []promptkit.RenderedMessage{{
+			Role: promptkit.RoleAssistant, Content: "caller suffix",
+		}},
+	}
+	prepared := assertValidateRequestParity(t, engine, request, nil)
+	if prepared.SelectedProfileID != "partial" || prepared.EffectiveModelParams.Model != "partial-model" ||
+		prepared.EffectiveModelParams.Endpoint != "http://override.example/v1" || prepared.SessionID != "request-session" ||
+		len(prepared.Messages) != 2 || prepared.Messages[1].Content != "caller suffix" {
+		t.Fatalf("preparation fixture did not exercise profile and request settings: %+v", prepared)
+	}
+	if got := modelCalls.Load(); got != 0 {
+		t.Fatalf("preparation operations made %d model calls, want none", got)
+	}
+}