Resolve smoke-test targets

2026-10-13 16:14:30 +00:00
parent ece764f19e
commit 27ff580dae
4 changed files with 565 additions and 3 deletions
--- a/docs/roadmap/implementation.md
+++ b/docs/roadmap/implementation.md
@@ -289,6 +289,9 @@ selection failure, default-reference failure only on use, cross-backend/default
 identity, endpoint-only profiles, and HTTP model incompleteness. Use real captured
 configuration and failing external readers to prove no application preparation.

+**Completed: 2026-10-13.** Validation passed: `go test ./...`, `go vet ./...`,
+and `git diff --check`.
+
 ## Stage 4: Add OpenRouter Catalog Diagnostic Assets

 Scope: `../promptkit-backend-openrouter` only, plus this plan's completion record.
--- a/internal/usecase/profile_inspection.go
+++ b/internal/usecase/profile_inspection.go
@@ -76,14 +76,22 @@ func (r *Runner) resolveProfileIdentity(ctx context.Context, profileID string) (
 }

 func normalizeResolvedExecutionTarget(target domain.ExecutionTarget) (domain.ExecutionTarget, error) {
+	target, err := normalizeResolvedExecutionSettings(target)
+	if err != nil {
+		return domain.ExecutionTarget{}, err
+	}
+	if strings.TrimSpace(target.Model) == "" {
+		return domain.ExecutionTarget{}, errors.New("execution model is required")
+	}
+	return target, nil
+}
+
+func normalizeResolvedExecutionSettings(target domain.ExecutionTarget) (domain.ExecutionTarget, error) {
 	endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(target.Endpoint)
 	if err != nil {
 		return domain.ExecutionTarget{}, fmt.Errorf("execution endpoint: %w", err)
 	}
 	target.Endpoint = endpoint
-	if strings.TrimSpace(target.Model) == "" {
-		return domain.ExecutionTarget{}, errors.New("execution model is required")
-	}
 	if err := domain.ValidateExecutionTargetSettings(target); err != nil {
 		return domain.ExecutionTarget{}, err
 	}
--- a/internal/usecase/smoke_selection.go
+++ b/internal/usecase/smoke_selection.go
@@ -0,0 +1,193 @@
+package usecase
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
+	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
+)
+
+const (
+	smokeSelectionSourceBackendDefault = "backend_default"
+	smokeSelectionSourceProfile        = "profile"
+	smokeSelectionSourcePrompt         = "prompt"
+	smokeSelectionSourceBackend        = "backend"
+)
+
+type smokeBackendLookup interface {
+	GetBackend(string) (domain.Backend, error)
+}
+
+type smokeTestSelection struct {
+	target           domain.SmokeTestTarget
+	backend          domain.Backend
+	transientRetries uint64
+}
+
+func (r *Runner) resolveSmokeTestTarget(ctx context.Context, req domain.SmokeTestRequest) (*smokeTestSelection, error) {
+	if !validSmokeTestKind(req.Kind) {
+		return nil, fmt.Errorf("%w: unknown smoke-test kind %q", ErrInvalidRequest, req.Kind)
+	}
+	transientRetries, err := domain.ResolveTransientRetries(0, req.TransientRetries)
+	if err != nil {
+		return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
+	}
+
+	promptID := strings.TrimSpace(req.PromptID)
+	profileID := strings.TrimSpace(req.ProfileID)
+	backendID := strings.TrimSpace(req.BackendID)
+	if promptID == "" && strings.TrimSpace(req.PromptVersion) != "" {
+		return nil, fmt.Errorf("%w: prompt version requires a prompt id", ErrInvalidRequest)
+	}
+
+	var promptInspection *domain.PromptInspection
+	if promptID != "" {
+		promptInspection, err = r.resolvePromptInspection(ctx, req.PromptID, req.PromptVersion)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	selection := &smokeTestSelection{transientRetries: transientRetries}
+	var selectedProfileID string
+	switch {
+	case profileID != "":
+		selectedProfileID = req.ProfileID
+		selection.target.SelectionSource = smokeSelectionSourceProfile
+	case promptInspection != nil && (promptInspection.DefaultProfileID != "" || isInferenceSmokeTest(req.Kind)):
+		selectedProfileID = promptInspection.DefaultProfileID
+		selection.target.SelectionSource = smokeSelectionSourcePrompt
+	case backendID != "" && isInferenceSmokeTest(req.Kind):
+		backend, err := r.resolveSmokeBackend(req.BackendID)
+		if err != nil {
+			return nil, err
+		}
+		if backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil {
+			return nil, fmt.Errorf("%w: backend %q has no default smoke-test profile", ErrProfileRequired, req.BackendID)
+		}
+		selectedProfileID = *backend.SmokeTests.DefaultProfile
+		selection.backend = backend
+		selection.target.DefaultSourceBackendID = backend.ID
+		selection.target.SelectionSource = smokeSelectionSourceBackendDefault
+	case backendID != "":
+		backend, err := r.resolveSmokeBackend(req.BackendID)
+		if err != nil {
+			return nil, err
+		}
+		selection.backend = backend
+		selection.target.SelectionSource = smokeSelectionSourceBackend
+	default:
+		if isInferenceSmokeTest(req.Kind) {
+			return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, ErrProfileRequired)
+		}
+		return nil, fmt.Errorf("%w: backend id, profile id, or prompt id is required", ErrInvalidRequest)
+	}
+	if selection.target.SelectionSource != smokeSelectionSourceBackend && selectedProfileID == "" {
+		return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, ErrProfileRequired)
+	}
+
+	if selectedProfileID != "" {
+		selection.backend = domain.Backend{}
+		profileSelection, err := r.resolveProfileSelection(ctx, selectedProfileID)
+		if err != nil {
+			return nil, err
+		}
+		settings, presence := resolveExecutionTarget(profileSelection.profile, req.Execution)
+		settings.APIKey = req.APIKey
+		if isInferenceSmokeTest(req.Kind) {
+			settings, err = normalizeResolvedExecutionTarget(settings)
+		} else {
+			settings, err = normalizeResolvedExecutionSettings(settings)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
+		}
+		selection.target.ProfileID = profileSelection.id
+		selection.target.BackendID = settings.BackendID
+		selection.target.EffectiveSettings = settings
+		selection.target.SettingsPresence = presence
+		if settings.BackendID != "" {
+			selection.backend, err = r.resolveSmokeBackend(settings.BackendID)
+			if err != nil {
+				return nil, err
+			}
+		}
+	} else {
+		settings := defaults.ExecutionTargetDefault()
+		settings.BackendID = selection.backend.ID
+		settings.Endpoint = selection.backend.Endpoint
+		settings.APIKeyEnv = selection.backend.APIKeyEnv
+		settings.ExtraParams = copyExtraParams(selection.backend.ExtraParams)
+		presence := domain.ExecutionTargetPresence{}
+		if req.Execution != nil {
+			settings, presence = mergeExecutionTargetOverride(settings, presence, *req.Execution)
+		}
+		settings.APIKey = req.APIKey
+		settings, err = normalizeResolvedExecutionSettings(settings)
+		if err != nil {
+			return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
+		}
+		selection.target.BackendID = settings.BackendID
+		selection.target.EffectiveSettings = settings
+		selection.target.SettingsPresence = presence
+	}
+
+	if promptInspection != nil {
+		selection.target.PromptID = promptInspection.PromptID
+		selection.target.PromptVersion = promptInspection.PromptVersion
+	}
+	selection.target.Endpoint, err = resolveSmokeTestEndpoint(req.Kind, selection.target.EffectiveSettings.Endpoint, selection.backend.SmokeTests)
+	if err != nil {
+		return nil, fmt.Errorf("%w: diagnostic endpoint: %w", ErrInvalidRequest, err)
+	}
+	return selection, nil
+}
+
+func (r *Runner) resolveSmokeBackend(id string) (domain.Backend, error) {
+	lookup, ok := r.profiles.(smokeBackendLookup)
+	if !ok {
+		return domain.Backend{}, fmt.Errorf("%w: backend lookup is not configured", ErrProfileLoad)
+	}
+	backend, err := lookup.GetBackend(id)
+	if err != nil {
+		return domain.Backend{}, fmt.Errorf("%w: backend %q: %v", ErrProfileLoad, id, err)
+	}
+	return backend, nil
+}
+
+func validSmokeTestKind(kind domain.SmokeTestKind) bool {
+	switch kind {
+	case domain.SmokeTestReachability, domain.SmokeTestAuthentication,
+		domain.SmokeTestTextInference, domain.SmokeTestStructuredOutput:
+		return true
+	default:
+		return false
+	}
+}
+
+func isInferenceSmokeTest(kind domain.SmokeTestKind) bool {
+	return kind == domain.SmokeTestTextInference || kind == domain.SmokeTestStructuredOutput
+}
+
+func resolveSmokeTestEndpoint(kind domain.SmokeTestKind, baseEndpoint string, smokeTests *domain.BackendSmokeTestConfig) (string, error) {
+	switch kind {
+	case domain.SmokeTestReachability:
+		if smokeTests != nil && smokeTests.Reachability != nil && smokeTests.Reachability.Endpoint != nil {
+			return *smokeTests.Reachability.Endpoint, nil
+		}
+	case domain.SmokeTestAuthentication:
+		if smokeTests != nil && smokeTests.Authentication != nil {
+			authentication := smokeTests.Authentication
+			if authentication.Endpoint != nil {
+				return *authentication.Endpoint, nil
+			}
+			if authentication.Kind != nil && *authentication.Kind == "openrouter_key" {
+				return url.JoinPath(baseEndpoint, "key")
+			}
+		}
+	}
+	return baseEndpoint, nil
+}
--- a/internal/usecase/smoke_selection_test.go
+++ b/internal/usecase/smoke_selection_test.go
@@ -0,0 +1,358 @@
+package usecase
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/ConfigKit"
+	"gitea.maximumdirect.net/eric/promptkit/internal/configuration"
+	"gitea.maximumdirect.net/eric/promptkit/internal/domain"
+	"gitea.maximumdirect.net/eric/promptkit/internal/profile"
+)
+
+type selectionTestCollaborators struct {
+	artifactReads   int
+	renderCalls     int
+	generationCalls int
+	validationCalls int
+}
+
+type selectionTestArtifactReader struct{ state *selectionTestCollaborators }
+
+func (r selectionTestArtifactReader) Read(context.Context, domain.ArtifactRef) (*domain.Artifact, error) {
+	r.state.artifactReads++
+	return nil, errors.New("artifact access must not occur during target selection")
+}
+
+type selectionTestRenderer struct{ state *selectionTestCollaborators }
+
+func (r selectionTestRenderer) Render(context.Context, *domain.PromptDefinition, map[string]*domain.Artifact, map[string]string) (*domain.RenderedPrompt, error) {
+	r.state.renderCalls++
+	return nil, errors.New("prompt rendering must not occur during target selection")
+}
+
+type selectionTestLLM struct{ state *selectionTestCollaborators }
+
+func (c selectionTestLLM) Generate(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) {
+	c.state.generationCalls++
+	return nil, errors.New("generation must not occur during target selection")
+}
+
+type selectionTestValidator struct{ state *selectionTestCollaborators }
+
+func (v selectionTestValidator) Validate(context.Context, *domain.Artifact, domain.OutputContract) (domain.ValidationResult, error) {
+	v.state.validationCalls++
+	return domain.ValidationResult{}, errors.New("validation must not occur during target selection")
+}
+
+type selectionTestRepositories struct {
+	profiles     profile.Repository
+	backends     smokeBackendLookup
+	profileCalls int
+}
+
+func (r *selectionTestRepositories) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
+	r.profileCalls++
+	return r.profiles.GetProfile(ctx, id)
+}
+
+func (r *selectionTestRepositories) GetBackend(id string) (domain.Backend, error) {
+	return r.backends.GetBackend(id)
+}
+
+func TestResolveSmokeTestTargetSelectionAndRouting(t *testing.T) {
+	runner, collaborators := newSmokeSelectionRunner(t)
+
+	for _, test := range []struct {
+		name                 string
+		request              domain.SmokeTestRequest
+		profileID            string
+		backendID            string
+		selectionSource      string
+		defaultSourceBackend string
+		endpoint             string
+		settingsEndpoint     string
+		wantModel            string
+	}{
+		{
+			name:             "explicit profile",
+			request:          domain.SmokeTestRequest{Kind: domain.SmokeTestTextInference, ProfileID: "alpha-profile"},
+			profileID:        "alpha-profile",
+			backendID:        "alpha",
+			selectionSource:  smokeSelectionSourceProfile,
+			endpoint:         "https://alpha.example/v1",
+			settingsEndpoint: "https://alpha.example/v1",
+			wantModel:        "alpha-model",
+		},
+		{
+			name: "prompt default",
+			request: domain.SmokeTestRequest{
+				Kind: domain.SmokeTestTextInference, PromptID: "selection.prompt", PromptVersion: "v1",
+			},
+			profileID:        "prompt-profile",
+			backendID:        "beta",
+			selectionSource:  smokeSelectionSourcePrompt,
+			endpoint:         "https://beta.example/v1",
+			settingsEndpoint: "https://beta.example/v1",
+			wantModel:        "prompt-model",
+		},
+		{
+			name: "explicit profile overrides prompt default",
+			request: domain.SmokeTestRequest{
+				Kind: domain.SmokeTestTextInference, PromptID: "selection.prompt", PromptVersion: "v1",
+				ProfileID: "alpha-profile", BackendID: "beta",
+			},
+			profileID:        "alpha-profile",
+			backendID:        "alpha",
+			selectionSource:  smokeSelectionSourceProfile,
+			endpoint:         "https://alpha.example/v1",
+			settingsEndpoint: "https://alpha.example/v1",
+			wantModel:        "alpha-model",
+		},
+		{
+			name: "backend default routes to a different backend",
+			request: domain.SmokeTestRequest{
+				Kind: domain.SmokeTestStructuredOutput, BackendID: "alpha",
+			},
+			profileID:            "cross-backend-profile",
+			backendID:            "beta",
+			selectionSource:      smokeSelectionSourceBackendDefault,
+			defaultSourceBackend: "alpha",
+			endpoint:             "https://beta.example/v1",
+			settingsEndpoint:     "https://beta.example/v1",
+			wantModel:            "cross-model",
+		},
+		{
+			name: "backend default may select endpoint-only profile",
+			request: domain.SmokeTestRequest{
+				Kind: domain.SmokeTestTextInference, BackendID: "epsilon",
+			},
+			profileID:            "endpoint-profile",
+			selectionSource:      smokeSelectionSourceBackendDefault,
+			defaultSourceBackend: "epsilon",
+			endpoint:             "https://endpoint-only.example/v1",
+			settingsEndpoint:     "https://endpoint-only.example/v1",
+			wantModel:            "endpoint-model",
+		},
+		{
+			name: "prompt profile supplies backend-only target over backend hint",
+			request: domain.SmokeTestRequest{
+				Kind: domain.SmokeTestReachability, PromptID: "selection.prompt", PromptVersion: "v1", BackendID: "alpha",
+			},
+			profileID:        "prompt-profile",
+			backendID:        "beta",
+			selectionSource:  smokeSelectionSourcePrompt,
+			endpoint:         "https://beta.example/v1",
+			settingsEndpoint: "https://beta.example/v1",
+			wantModel:        "prompt-model",
+		},
+		{
+			name: "backend-only probe uses backend when prompt has no profile default",
+			request: domain.SmokeTestRequest{
+				Kind: domain.SmokeTestReachability, PromptID: "backend.prompt", PromptVersion: "v1", BackendID: "delta",
+			},
+			backendID:        "delta",
+			selectionSource:  smokeSelectionSourceBackend,
+			endpoint:         "https://delta.example/v1",
+			settingsEndpoint: "https://delta.example/v1",
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			selected, err := runner.resolveSmokeTestTarget(context.Background(), test.request)
+			if err != nil {
+				t.Fatalf("resolve target: %v", err)
+			}
+			if selected.target.ProfileID != test.profileID || selected.target.BackendID != test.backendID ||
+				selected.target.SelectionSource != test.selectionSource ||
+				selected.target.DefaultSourceBackendID != test.defaultSourceBackend ||
+				selected.target.Endpoint != test.endpoint ||
+				selected.target.EffectiveSettings.Endpoint != test.settingsEndpoint ||
+				selected.target.EffectiveSettings.Model != test.wantModel {
+				t.Fatalf("unexpected target: %+v", selected.target)
+			}
+			if test.request.PromptID != "" && (selected.target.PromptID != test.request.PromptID || selected.target.PromptVersion != test.request.PromptVersion) {
+				t.Fatalf("prompt identity was not retained: %+v", selected.target)
+			}
+		})
+	}
+
+	if collaborators.artifactReads != 0 || collaborators.renderCalls != 0 ||
+		collaborators.generationCalls != 0 || collaborators.validationCalls != 0 {
+		t.Fatalf("selection entered application preparation: %+v", collaborators)
+	}
+}
+
+func TestResolveSmokeTestTargetBackendOnlyDoesNotLoadDefaultProfile(t *testing.T) {
+	loaded, err := configuration.Load(configkit.Input{Source: configkit.Mapping("selection", smokeSelectionConfig())}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	repositories := &selectionTestRepositories{profiles: loaded, backends: loaded}
+	runner := NewRunner(nil, repositories, nil, nil, nil, nil, nil)
+
+	zeroTemperature := 0.0
+	selected, err := runner.resolveSmokeTestTarget(context.Background(), domain.SmokeTestRequest{
+		Kind: domain.SmokeTestReachability, BackendID: "delta",
+		Execution: &domain.ExecutionTargetOverride{Temperature: &zeroTemperature},
+	})
+	if err != nil {
+		t.Fatalf("backend-only probe should not resolve its dangling default profile: %v", err)
+	}
+	if selected.target.SelectionSource != smokeSelectionSourceBackend || selected.target.BackendID != "delta" ||
+		selected.target.ProfileID != "" || selected.target.Endpoint != "https://delta.example/v1" {
+		t.Fatalf("unexpected direct backend target: %+v", selected.target)
+	}
+	if !selected.target.SettingsPresence.Temperature || selected.target.EffectiveSettings.Temperature != zeroTemperature {
+		t.Fatalf("explicit zero override presence was not retained: %+v", selected.target)
+	}
+	if repositories.profileCalls != 0 {
+		t.Fatalf("backend-only selection loaded %d profiles", repositories.profileCalls)
+	}
+}
+
+func TestResolveSmokeTestTargetPreservesDiagnosticAndBaseEndpoints(t *testing.T) {
+	runner, _ := newSmokeSelectionRunner(t)
+	override := domain.ExecutionTargetOverride{Endpoint: " https://override.example/v1 "}
+	selected, err := runner.resolveSmokeTestTarget(context.Background(), domain.SmokeTestRequest{
+		Kind: domain.SmokeTestReachability, ProfileID: "incomplete-profile", Execution: &override,
+	})
+	if err != nil {
+		t.Fatalf("HTTP diagnostic should not require a model: %v", err)
+	}
+	if selected.target.BackendID != "alpha" || selected.target.EffectiveSettings.Model != "" || selected.target.EffectiveSettings.Endpoint != "https://override.example/v1" ||
+		selected.target.Endpoint != "https://probe.example/ready?check=1" {
+		t.Fatalf("diagnostic destination and base settings were not kept separately: %+v", selected.target)
+	}
+
+	selected, err = runner.resolveSmokeTestTarget(context.Background(), domain.SmokeTestRequest{
+		Kind: domain.SmokeTestAuthentication, BackendID: "beta",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if selected.target.Endpoint != "https://beta.example/v1/key" {
+		t.Fatalf("authentication destination = %q", selected.target.Endpoint)
+	}
+}
+
+func TestResolveSmokeTestTargetSelectionAndSettingsErrors(t *testing.T) {
+	runner, _ := newSmokeSelectionRunner(t)
+	negativeRetries := -1
+	negativeTimeout := -1
+	for _, test := range []struct {
+		name    string
+		request domain.SmokeTestRequest
+		wantErr error
+	}{
+		{
+			name:    "unknown kind",
+			request: domain.SmokeTestRequest{Kind: "unknown", BackendID: "epsilon"},
+			wantErr: ErrInvalidRequest,
+		},
+		{
+			name:    "negative retry override",
+			request: domain.SmokeTestRequest{Kind: domain.SmokeTestReachability, BackendID: "epsilon", TransientRetries: &negativeRetries},
+			wantErr: ErrInvalidRequest,
+		},
+		{
+			name: "explicit selection does not fall back",
+			request: domain.SmokeTestRequest{
+				Kind: domain.SmokeTestTextInference, ProfileID: "missing-profile", BackendID: "epsilon",
+			},
+			wantErr: ErrProfileLoad,
+		},
+		{
+			name:    "dangling backend default fails only when selected",
+			request: domain.SmokeTestRequest{Kind: domain.SmokeTestTextInference, BackendID: "delta"},
+			wantErr: ErrProfileLoad,
+		},
+		{
+			name: "unused negative setting is still validated for HTTP",
+			request: domain.SmokeTestRequest{
+				Kind: domain.SmokeTestReachability, BackendID: "epsilon",
+				Execution: &domain.ExecutionTargetOverride{TimeoutSeconds: &negativeTimeout},
+			},
+			wantErr: ErrInvalidRequest,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			if _, err := runner.resolveSmokeTestTarget(context.Background(), test.request); !errors.Is(err, test.wantErr) {
+				t.Fatalf("selection error = %v, want errors.Is(_, %v)", err, test.wantErr)
+			}
+		})
+	}
+}
+
+func newSmokeSelectionRunner(t *testing.T) (*Runner, *selectionTestCollaborators) {
+	t.Helper()
+	loaded, err := configuration.Load(configkit.Input{Source: configkit.Mapping("selection", smokeSelectionConfig())}, nil)
+	if err != nil {
+		t.Fatalf("load captured selection configuration: %v", err)
+	}
+	collaborators := &selectionTestCollaborators{}
+	return NewRunner(
+		loaded,
+		loaded,
+		selectionTestArtifactReader{state: collaborators},
+		selectionTestRenderer{state: collaborators},
+		selectionTestLLM{state: collaborators},
+		selectionTestValidator{state: collaborators},
+		nil,
+	), collaborators
+}
+
+func smokeSelectionConfig() map[string]any {
+	return map[string]any{
+		"backends": map[string]any{
+			"alpha": map[string]any{
+				"endpoint": "https://alpha.example/v1",
+				"smoke_tests": map[string]any{
+					"default_profile": "cross-backend-profile",
+					"reachability":    map[string]any{"endpoint": "https://probe.example/ready?check=1"},
+					"authentication":  map[string]any{"kind": "openrouter_key"},
+				},
+			},
+			"beta": map[string]any{
+				"endpoint":    "https://beta.example/v1",
+				"smoke_tests": map[string]any{"authentication": map[string]any{"kind": "openrouter_key"}},
+			},
+			"delta": map[string]any{
+				"endpoint":    "https://delta.example/v1",
+				"smoke_tests": map[string]any{"default_profile": "not-yet-defined"},
+			},
+			"epsilon": map[string]any{
+				"endpoint":    "https://epsilon.example/v1",
+				"smoke_tests": map[string]any{"default_profile": "endpoint-profile"},
+			},
+		},
+		"profiles": map[string]any{
+			"alpha-profile":         map[string]any{"backend": "alpha", "model": "alpha-model"},
+			"prompt-profile":        map[string]any{"backend": "beta", "model": "prompt-model"},
+			"cross-backend-profile": map[string]any{"backend": "beta", "model": "cross-model"},
+			"endpoint-profile":      map[string]any{"endpoint": "https://endpoint-only.example/v1", "model": "endpoint-model"},
+			"incomplete-profile":    map[string]any{"backend": "alpha"},
+		},
+		"prompts": map[string]any{
+			"selection.prompt": map[string]any{
+				"v1": map[string]any{
+					"default_profile": "prompt-profile",
+					"inputs":          []any{map[string]any{"name": "required_input", "required": true}},
+					"messages":        []any{map[string]any{"role": "user", "content": "Hello {{.required_input}}"}},
+					"session_id":      "{{.vars.session}}",
+					"output": map[string]any{
+						"format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json",
+					},
+				},
+			},
+			"backend.prompt": map[string]any{
+				"v1": map[string]any{
+					"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
+					"output": map[string]any{
+						"format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json",
+					},
+				},
+			},
+		},
+	}
+}