promptkit/public_contract_test.go

package promptkit_test

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"reflect"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"testing/fstest"

	"gitea.maximumdirect.net/eric/promptkit"
)

type inspectionCountingFS struct {
	opens atomic.Int64
}

func (f *inspectionCountingFS) Open(string) (fs.File, error) {
	f.opens.Add(1)
	return nil, fs.ErrNotExist
}

func TestInspectProfileResolvesCredentialStatesWithoutPromptOrGeneration(t *testing.T) {
	const environmentName = "PROMPTKIT_INSPECTION_ABSENT_KEY"
	t.Setenv(environmentName, "")
	client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
	engine, err := promptkit.NewEngine(promptkit.Config{},
		promptkit.WithPromptFS(fstest.MapFS{}, "."),
		promptkit.WithProfileFS(fstest.MapFS{
			"environment.yaml": &fstest.MapFile{Data: []byte(`id: environment
endpoint: http://environment.example/v1
model: environment-model
api_key_env: PROMPTKIT_INSPECTION_ABSENT_KEY
`)},
		}, "."),
		promptkit.WithProfiles(
			promptkit.Profile{ID: "direct", Endpoint: "http://direct.example/v1", Model: "direct-model", APIKeyRequired: true},
			promptkit.Profile{ID: "none", Endpoint: "http://none.example/v1", Model: "none-model"},
		),
		promptkit.WithLLMClient(client),
	)
	if err != nil {
		t.Fatalf("construct inspection engine: %v", err)
	}

	for _, tc := range []struct {
		profileID     string
		wantEnv       string
		wantDirectKey bool
		wantEndpoint  string
	}{
		{profileID: " environment ", wantEnv: environmentName, wantEndpoint: "http://environment.example/v1"},
		{profileID: "direct", wantDirectKey: true, wantEndpoint: "http://direct.example/v1"},
		{profileID: "none", wantEndpoint: "http://none.example/v1"},
	} {
		t.Run(tc.profileID, func(t *testing.T) {
			inspection, err := engine.InspectProfile(context.Background(), tc.profileID)
			if err != nil {
				t.Fatalf("inspect profile: %v", err)
			}
			if inspection.ProfileID != strings.TrimSpace(tc.profileID) ||
				inspection.EffectiveModelParams.Endpoint != tc.wantEndpoint ||
				inspection.EffectiveModelParams.BackendID != "" ||
				inspection.EffectiveModelParams.APIKeyEnv != tc.wantEnv ||
				inspection.APIKeyRequired != tc.wantDirectKey {
				t.Fatalf("unexpected inspection: %#v", inspection)
			}
		})
	}
	if len(client.requests) != 0 {
		t.Fatalf("inspection invoked the model client %d times", len(client.requests))
	}
}

func TestFileProfileNormalizedIDMatchesInspectionAndPreparation(t *testing.T) {
	engine, err := promptkit.NewEngine(promptkit.Config{},
		promptkit.WithPromptFS(contractPromptFS("prompt", "normalized-profile", "message"), "."),
		promptkit.WithProfileFS(fstest.MapFS{
			"profile.yaml": &fstest.MapFile{Data: []byte(`
id: "  normalized-profile  "
endpoint: http://profile.example/v1
model: normalized-model
`)},
		}, "."),
	)
	if err != nil {
		t.Fatalf("construct engine: %v", err)
	}

	inspection, err := engine.InspectProfile(context.Background(), " normalized-profile ")
	if err != nil {
		t.Fatalf("inspect normalized profile: %v", err)
	}
	prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
		PromptID:  "prompt",
		ProfileID: " normalized-profile ",
	})
	if err != nil {
		t.Fatalf("prepare with normalized profile: %v", err)
	}

	if inspection.ProfileID != "normalized-profile" ||
		prepared.SelectedProfileID != inspection.ProfileID ||
		inspection.EffectiveModelParams.Model != "normalized-model" ||
		prepared.EffectiveModelParams.Model != inspection.EffectiveModelParams.Model {
		t.Fatalf("inspection=%#v prepared=%#v", inspection, prepared)
	}
}

func TestInspectProfilePreservesPublicErrorIdentities(t *testing.T) {
	newEngine := func(t *testing.T, options ...promptkit.Option) *promptkit.Engine {
		t.Helper()
		engine, err := promptkit.NewEngine(
			promptkit.Config{},
			append([]promptkit.Option{promptkit.WithPromptFS(fstest.MapFS{}, ".")}, options...)...,
		)
		if err != nil {
			t.Fatalf("construct inspection engine: %v", err)
		}
		return engine
	}

	var nilEngine *promptkit.Engine
	if result, err := nilEngine.InspectProfile(context.Background(), "profile"); result != nil ||
		!errors.Is(err, promptkit.ErrInvalidConfig) {
		t.Fatalf("nil engine result=(%#v, %v), want ErrInvalidConfig", result, err)
	}

	valid := newEngine(t, promptkit.WithProfiles(promptkit.Profile{
		ID: "profile", Endpoint: "http://profile.example/v1", Model: "model",
	}))
	if result, err := valid.InspectProfile(context.Background(), " \t "); result != nil ||
		!errors.Is(err, promptkit.ErrInvalidRequest) {
		t.Fatalf("blank profile result=(%#v, %v), want ErrInvalidRequest", result, err)
	}
	if result, err := valid.InspectProfile(context.Background(), "missing"); result != nil ||
		!errors.Is(err, promptkit.ErrProfileNotFound) || errors.Is(err, promptkit.ErrProfileLoad) {
		t.Fatalf("missing profile result=(%#v, %v), want only ErrProfileNotFound", result, err)
	}

	malformed := newEngine(t, promptkit.WithProfileFS(fstest.MapFS{
		"broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nendpoint: http://broken.example/v1\nmodel: model\nextra_params:\n  invalid: .nan\n")},
	}, "."))
	if result, err := malformed.InspectProfile(context.Background(), "broken"); result != nil ||
		!errors.Is(err, promptkit.ErrProfileLoad) {
		t.Fatalf("malformed profile result=(%#v, %v), want ErrProfileLoad", result, err)
	}

	unknownBackend := newEngine(t, promptkit.WithProfiles(promptkit.Profile{
		ID: "unknown-backend", BackendID: "unknown", Model: "model",
	}))
	if result, err := unknownBackend.InspectProfile(context.Background(), "unknown-backend"); result != nil ||
		!errors.Is(err, promptkit.ErrProfileLoad) {
		t.Fatalf("unknown backend result=(%#v, %v), want ErrProfileLoad", result, err)
	}

	countingFS := &inspectionCountingFS{}
	canceled := newEngine(t, promptkit.WithProfileFS(countingFS, "."))
	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	if result, err := canceled.InspectProfile(ctx, "profile"); result != nil ||
		!errors.Is(err, promptkit.ErrProfileLoad) || !errors.Is(err, context.Canceled) || countingFS.opens.Load() != 0 {
		t.Fatalf("canceled inspection result=(%#v, %v), opens=%d", result, err, countingFS.opens.Load())
	}
}

func TestInspectProfileReturnsIndependentTargetMatchingPreparation(t *testing.T) {
	extraParams := map[string]any{
		"nested": map[string]any{"value": "original"},
	}
	engine, err := promptkit.NewEngine(promptkit.Config{},
		promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
		promptkit.WithProfiles(promptkit.Profile{
			ID: "profile", Endpoint: "http://profile.example/v1", Model: "model", ExtraParams: extraParams,
		}),
	)
	if err != nil {
		t.Fatalf("construct inspection engine: %v", err)
	}

	first, err := engine.InspectProfile(context.Background(), "profile")
	if err != nil {
		t.Fatalf("first inspection: %v", err)
	}
	first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] = "changed"
	first.EffectiveModelParams.ExtraParams["later"] = true

	second, err := engine.InspectProfile(context.Background(), "profile")
	if err != nil {
		t.Fatalf("second inspection: %v", err)
	}
	prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
	if err != nil {
		t.Fatalf("prepare after inspection mutation: %v", err)
	}
	for _, target := range []promptkit.ExecutionTarget{second.EffectiveModelParams, prepared.EffectiveModelParams} {
		if target.ExtraParams["nested"].(map[string]any)["value"] != "original" || target.ExtraParams["later"] != nil {
			t.Fatalf("inspection mutation reached engine-owned target: %#v", target.ExtraParams)
		}
	}
	if !reflect.DeepEqual(second.EffectiveModelParams, prepared.EffectiveModelParams) {
		t.Fatalf("inspection target=%#v, preparation target=%#v", second.EffectiveModelParams, prepared.EffectiveModelParams)
	}
}

func TestInspectPromptReturnsDeclaredMetadataWithoutExecutionWork(t *testing.T) {
	client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
	engine, err := promptkit.NewEngine(promptkit.Config{},
		promptkit.WithPromptFS(fstest.MapFS{
			"report-v1.yaml": &fstest.MapFile{Data: []byte(`id: report
version: "1.0.0"
messages:
  - role: user
    content: old report
output:
  format: text
  validation_mode: none
`)},
			"report-v2.yaml": &fstest.MapFile{Data: []byte(`id: report
version: "2.0.0"
default_profile: missing-profile
inputs:
  - name: location
    required: true
    content_type: text/plain
    description: Forecast location.
  - name: units
    content_type: text/plain
    description: Unit preference.
messages:
  - role: user
    content_file: messages/report.md
output:
  format: json
  validation_mode: json_schema
  schema_path: schemas/report.json
`)},
			"messages/report.md": &fstest.MapFile{Data: []byte("rendered report body is not returned")},
		}, "."),
		promptkit.WithLLMClient(client),
	)
	if err != nil {
		t.Fatalf("construct prompt-inspection engine: %v", err)
	}

	inspection, err := engine.InspectPrompt(context.Background(), "report", "2.0.0")
	if err != nil {
		t.Fatalf("inspect prompt: %v", err)
	}
	if inspection.PromptID != "report" ||
		inspection.PromptVersion != "2.0.0" ||
		inspection.PromptHash == "" ||
		inspection.DefaultProfileID != "missing-profile" ||
		inspection.OutputContract != (promptkit.OutputContract{
			Format:         promptkit.FormatJSON,
			ValidationMode: promptkit.ValidationJSONSchema,
			SchemaPath:     "schemas/report.json",
		}) {
		t.Fatalf("unexpected inspection metadata: %#v", inspection)
	}
	wantInputs := []promptkit.PromptInputDefinition{
		{Name: "location", Required: true, ContentType: "text/plain", Description: "Forecast location."},
		{Name: "units", ContentType: "text/plain", Description: "Unit preference."},
	}
	if !reflect.DeepEqual(inspection.Inputs, wantInputs) {
		t.Fatalf("inspection inputs=%#v, want %#v", inspection.Inputs, wantInputs)
	}
	if len(client.requests) != 0 {
		t.Fatalf("inspection invoked the model client %d times", len(client.requests))
	}
}

func TestInspectPromptPreservesPublicErrorIdentities(t *testing.T) {
	newEngine := func(t *testing.T, source fstest.MapFS) *promptkit.Engine {
		t.Helper()
		engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(source, "."))
		if err != nil {
			t.Fatalf("construct prompt-inspection engine: %v", err)
		}
		return engine
	}
	validSource := fstest.MapFS{
		"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prompt
version: "1"
messages:
  - role: user
    content: body
output:
  format: text
  validation_mode: none
`)},
	}

	var nilEngine *promptkit.Engine
	if result, err := nilEngine.InspectPrompt(context.Background(), "prompt", "1"); result != nil ||
		!errors.Is(err, promptkit.ErrInvalidConfig) {
		t.Fatalf("nil engine result=(%#v, %v), want ErrInvalidConfig", result, err)
	}

	valid := newEngine(t, validSource)
	if result, err := valid.InspectPrompt(context.Background(), " \t ", "1"); result != nil ||
		!errors.Is(err, promptkit.ErrInvalidRequest) {
		t.Fatalf("blank prompt result=(%#v, %v), want ErrInvalidRequest", result, err)
	}
	if result, err := valid.InspectPrompt(context.Background(), "missing", "1"); result != nil ||
		!errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrPromptLoad) {
		t.Fatalf("missing prompt result=(%#v, %v), want only ErrPromptNotFound", result, err)
	}
	if result, err := valid.InspectPrompt(context.Background(), "prompt", "missing"); result != nil ||
		!errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrPromptLoad) {
		t.Fatalf("missing version result=(%#v, %v), want only ErrPromptNotFound", result, err)
	}

	ambiguous := newEngine(t, fstest.MapFS{
		"one.yaml": &fstest.MapFile{Data: []byte(`id: prompt
version: "1"
messages:
  - role: user
    content: first
output:
  format: text
  validation_mode: none
`)},
		"two.yaml": &fstest.MapFile{Data: []byte(`id: prompt
version: "2"
messages:
  - role: user
    content: second
output:
  format: text
  validation_mode: none
`)},
	})
	if result, err := ambiguous.InspectPrompt(context.Background(), "prompt", ""); result != nil ||
		!errors.Is(err, promptkit.ErrPromptLoad) {
		t.Fatalf("ambiguous prompt result=(%#v, %v), want ErrPromptLoad", result, err)
	}

	for name, source := range map[string]fstest.MapFS{
		"malformed definition": {
			"broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nversion: \"1\"\nunknown: value\n")},
		},
		"missing content file": {
			"broken.yaml": &fstest.MapFile{Data: []byte(`id: broken
version: "1"
messages:
  - role: user
    content_file: missing.md
output:
  format: text
  validation_mode: none
`)},
		},
	} {
		t.Run(name, func(t *testing.T) {
			if result, err := newEngine(t, source).InspectPrompt(context.Background(), "broken", "1"); result != nil ||
				!errors.Is(err, promptkit.ErrPromptLoad) {
				t.Fatalf("broken prompt result=(%#v, %v), want ErrPromptLoad", result, err)
			}
		})
	}

	countingFS := &inspectionCountingFS{}
	canceled, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(countingFS, "."))
	if err != nil {
		t.Fatalf("construct canceled prompt-inspection engine: %v", err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	if result, err := canceled.InspectPrompt(ctx, "prompt", "1"); result != nil ||
		!errors.Is(err, promptkit.ErrPromptLoad) || !errors.Is(err, context.Canceled) || countingFS.opens.Load() != 0 {
		t.Fatalf("canceled inspection result=(%#v, %v), opens=%d", result, err, countingFS.opens.Load())
	}
}

func TestInspectPromptReturnsIndependentMetadataMatchingPreparation(t *testing.T) {
	engine, err := promptkit.NewEngine(promptkit.Config{},
		promptkit.WithPromptFS(fstest.MapFS{
			"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prompt
version: "1"
default_profile: profile
inputs:
  - name: subject
    content_type: text/plain
    description: Summary subject.
messages:
  - role: user
    content: summarize
output:
  format: markdown
  validation_mode: basic
`)},
		}, "."),
		promptkit.WithProfiles(promptkit.Profile{
			ID: "profile", Endpoint: "http://profile.example/v1", Model: "model",
		}),
	)
	if err != nil {
		t.Fatalf("construct prompt-inspection engine: %v", err)
	}

	first, err := engine.InspectPrompt(context.Background(), "prompt", "1")
	if err != nil {
		t.Fatalf("first inspection: %v", err)
	}
	first.Inputs[0].Name = "changed"
	first.OutputContract.SchemaPath = "changed.json"

	second, err := engine.InspectPrompt(context.Background(), "prompt", "1")
	if err != nil {
		t.Fatalf("second inspection: %v", err)
	}
	prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", PromptVersion: "1"})
	if err != nil {
		t.Fatalf("prepare after inspection mutation: %v", err)
	}
	if second.Inputs[0].Name != "subject" || second.OutputContract.SchemaPath != "" ||
		prepared.OutputContract.SchemaPath != "" || second.PromptHash != prepared.PromptHash {
		t.Fatalf("inspection mutation reached engine-owned prompt metadata: inspection=%#v prepared=%#v", second, prepared)
	}
}

func TestBackendIdentityJSONNamesAndOmission(t *testing.T) {
	t.Run("execution target round trip", func(t *testing.T) {
		value := promptkit.ExecutionTarget{BackendID: promptkit.BackendOpenRouter}
		payload, err := json.Marshal(value)
		if err != nil {
			t.Fatalf("marshal execution target: %v", err)
		}
		var decoded promptkit.ExecutionTarget
		if err := json.Unmarshal(payload, &decoded); err != nil {
			t.Fatalf("unmarshal execution target: %v", err)
		}
		if decoded.BackendID != value.BackendID {
			t.Fatalf("backend identity did not round trip: got %q want %q", decoded.BackendID, value.BackendID)
		}
	})

	t.Run("prepared run round trip", func(t *testing.T) {
		value := promptkit.PreparedRun{SelectedBackendID: promptkit.BackendOpenRouter}
		payload, err := json.Marshal(value)
		if err != nil {
			t.Fatalf("marshal prepared run: %v", err)
		}
		var decoded promptkit.PreparedRun
		if err := json.Unmarshal(payload, &decoded); err != nil {
			t.Fatalf("unmarshal prepared run: %v", err)
		}
		if decoded.SelectedBackendID != value.SelectedBackendID {
			t.Fatalf("backend identity did not round trip: got %q want %q", decoded.SelectedBackendID, value.SelectedBackendID)
		}
	})

	t.Run("run result round trip", func(t *testing.T) {
		value := promptkit.RunResult{SelectedBackendID: promptkit.BackendOpenRouter}
		payload, err := json.Marshal(value)
		if err != nil {
			t.Fatalf("marshal run result: %v", err)
		}
		var decoded promptkit.RunResult
		if err := json.Unmarshal(payload, &decoded); err != nil {
			t.Fatalf("unmarshal run result: %v", err)
		}
		if decoded.SelectedBackendID != value.SelectedBackendID {
			t.Fatalf("backend identity did not round trip: got %q want %q", decoded.SelectedBackendID, value.SelectedBackendID)
		}
	})

	payload, err := json.Marshal(promptkit.ExecutionTarget{})
	if err != nil {
		t.Fatalf("marshal empty execution target: %v", err)
	}
	if strings.Contains(string(payload), `"backend_id"`) {
		t.Fatalf("empty backend identity was not omitted: %s", payload)
	}
}

func TestEndpointOnlyProfileOmitsBackendIdentityFromStableJSON(t *testing.T) {
	client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
	engine, err := promptkit.NewEngine(promptkit.Config{},
		promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
		promptkit.WithProfiles(promptkit.Profile{
			ID: "profile", Endpoint: "http://example.test/v1", Model: "model",
		}),
		promptkit.WithLLMClient(client),
	)
	if err != nil {
		t.Fatalf("construct engine: %v", err)
	}

	prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
	if err != nil {
		t.Fatalf("prepare endpoint-only profile: %v", err)
	}
	result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
	if err != nil {
		t.Fatalf("run endpoint-only profile: %v", err)
	}
	if prepared.SelectedBackendID != "" ||
		prepared.EffectiveModelParams.BackendID != "" ||
		result.SelectedBackendID != "" ||
		result.EffectiveModelParams.BackendID != "" {
		t.Fatalf("endpoint-only profile acquired backend identity: prepared=%+v result=%+v", prepared, result)
	}
	for _, value := range []any{prepared, result} {
		payload, err := json.Marshal(value)
		if err != nil {
			t.Fatalf("marshal endpoint-only value: %v", err)
		}
		if strings.Contains(string(payload), `"backend_id"`) || strings.Contains(string(payload), `"selected_backend_id"`) {
			t.Fatalf("endpoint-only backend identity was not omitted: %s", payload)
		}
	}
}

func TestUnknownProfileBackendHasProfileLoadIdentity(t *testing.T) {
	engine, err := promptkit.NewEngine(promptkit.Config{},
		promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
		promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "unknown", Model: "model"}),
	)
	if err != nil {
		t.Fatalf("construct engine: %v", err)
	}

	_, err = engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
	if !errors.Is(err, promptkit.ErrProfileLoad) {
		t.Fatalf("expected ErrProfileLoad, got %v", err)
	}
	if errors.Is(err, promptkit.ErrInvalidRequest) {
		t.Fatalf("unknown backend should not have invalid-request identity: %v", err)
	}
}

func TestLocalBackendConstructsAndRegistersConventionalBackend(t *testing.T) {
	const (
		localBackendID = "local"
		limit          = 2
	)
	if promptkit.BackendLocal != localBackendID {
		t.Fatalf("BackendLocal=%q, want %q", promptkit.BackendLocal, localBackendID)
	}

	endpoint := "http://local.example/v1"
	backend := promptkit.LocalBackend(endpoint, limit)
	want := promptkit.Backend{
		ID:               localBackendID,
		Endpoint:         endpoint,
		ConcurrencyLimit: limit,
	}
	if !reflect.DeepEqual(backend, want) {
		t.Fatalf("LocalBackend()=%+v, want %+v", backend, want)
	}

	engine, err := promptkit.NewEngine(promptkit.Config{},
		promptkit.WithPromptFS(contractPromptFS("prompt", "local-profile", "message"), "."),
		promptkit.WithBackend(backend),
		promptkit.WithProfiles(promptkit.Profile{
			ID:        "local-profile",
			BackendID: localBackendID,
			Model:     "model",
		}),
	)
	if err != nil {
		t.Fatalf("construct engine with local backend: %v", err)
	}

	prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
	if err != nil {
		t.Fatalf("prepare with local backend: %v", err)
	}
	if prepared.SelectedBackendID != localBackendID ||
		prepared.EffectiveModelParams.Endpoint != endpoint {
		t.Fatalf("unexpected local backend preparation: %+v", prepared)
	}
}

func TestCustomBackendFlowsThroughProfilesOverridesAndInjectedClient(t *testing.T) {
	t.Setenv("CUSTOM_LLM_KEY", "test-key")
	client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
	engine, err := promptkit.NewEngine(promptkit.Config{},
		promptkit.WithPromptFS(contractPromptFS("prompt", "backend-profile", "message"), "."),
		promptkit.WithBackend(promptkit.Backend{
			ID:        "  custom  ",
			Endpoint:  "  http://backend.example/v1  ",
			APIKeyEnv: "  CUSTOM_LLM_KEY  ",
			ExtraParams: map[string]any{
				"provider": "custom",
			},
		}),
		promptkit.WithProfiles(
			promptkit.Profile{ID: "backend-profile", BackendID: "custom", Model: "backend-model"},
			promptkit.Profile{ID: "profile-endpoint", BackendID: "custom", Endpoint: "http://profile.example/v1", Model: "profile-model"},
			promptkit.Profile{ID: "blank-profile-endpoint", BackendID: "custom", Endpoint: " \t ", Model: "profile-model"},
		),
		promptkit.WithLLMClient(client),
	)
	if err != nil {
		t.Fatalf("construct engine: %v", err)
	}

	result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
	if err != nil {
		t.Fatalf("run with custom backend: %v", err)
	}
	if len(client.requests) != 1 {
		t.Fatalf("expected one injected-client request, got %d", len(client.requests))
	}
	target := client.requests[0].Target
	if target.BackendID != "custom" ||
		target.Endpoint != "http://backend.example/v1" ||
		target.APIKeyEnv != "CUSTOM_LLM_KEY" ||
		target.Model != "backend-model" ||
		target.ExtraParams["provider"] != "custom" ||
		result.SelectedBackendID != "custom" {
		t.Fatalf("unexpected custom backend settings: target=%+v result_backend=%q", target, result.SelectedBackendID)
	}

	prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
		PromptID: "prompt", ProfileID: "profile-endpoint",
	})
	if err != nil {
		t.Fatalf("prepare profile endpoint override: %v", err)
	}
	if prepared.SelectedBackendID != "custom" || prepared.EffectiveModelParams.Endpoint != "http://profile.example/v1" {
		t.Fatalf("profile endpoint override changed backend identity: %+v", prepared)
	}

	prepared, err = engine.Prepare(context.Background(), promptkit.RunRequest{
		PromptID: "prompt", ProfileID: "blank-profile-endpoint",
	})
	if err != nil {
		t.Fatalf("prepare blank profile endpoint: %v", err)
	}
	if prepared.SelectedBackendID != "custom" || prepared.EffectiveModelParams.Endpoint != "http://backend.example/v1" {
		t.Fatalf("blank profile endpoint did not inherit backend endpoint: %+v", prepared)
	}

	prepared, err = engine.Prepare(context.Background(), promptkit.RunRequest{
		PromptID: "prompt",
		Execution: &promptkit.ExecutionTargetOverride{
			Endpoint: "http://request.example/v1",
		},
	})
	if err != nil {
		t.Fatalf("prepare request endpoint override: %v", err)
	}
	if prepared.SelectedBackendID != "custom" || prepared.EffectiveModelParams.Endpoint != "http://request.example/v1" {
		t.Fatalf("request endpoint override changed backend identity: %+v", prepared)
	}
}

func TestCustomBackendSupportsFileProfileAndBothSelectionPaths(t *testing.T) {
	engine, err := promptkit.NewEngine(promptkit.Config{},
		promptkit.WithPromptFS(contractPromptFS("prompt", "file-profile", "message"), "."),
		promptkit.WithProfileFS(fstest.MapFS{
			"profile.yaml": &fstest.MapFile{Data: []byte(`id: file-profile
backend: file-backend
endpoint: "  "
model: file-model
`)},
		}, "."),
		promptkit.WithBackend(promptkit.Backend{
			ID:       "file-backend",
			Endpoint: "http://file-backend.example/v1",
		}),
	)
	if err != nil {
		t.Fatalf("construct engine: %v", err)
	}

	for _, request := range []promptkit.RunRequest{
		{PromptID: "prompt"},
		{PromptID: "prompt", ProfileID: "file-profile"},
	} {
		prepared, err := engine.Prepare(context.Background(), request)
		if err != nil {
			t.Fatalf("prepare file profile: %v", err)
		}
		if prepared.SelectedBackendID != "file-backend" ||
			prepared.EffectiveModelParams.Endpoint != "http://file-backend.example/v1" {
			t.Fatalf("unexpected file-profile backend resolution: %+v", prepared)
		}
	}
}

func TestBackendOptionsAccumulateAndRegistrationsAreEngineLocal(t *testing.T) {
	engine, err := promptkit.NewEngine(promptkit.Config{},
		promptkit.WithPromptFS(contractPromptFS("prompt", "first-profile", "message"), "."),
		promptkit.WithBackend(promptkit.Backend{ID: "first", Endpoint: "http://first.example/v1"}),
		promptkit.WithBackend(promptkit.Backend{ID: "second", Endpoint: "http://second.example/v1"}),
		promptkit.WithProfiles(
			promptkit.Profile{ID: "first-profile", BackendID: "first", Model: "model"},
			promptkit.Profile{ID: "second-profile", BackendID: "second", Model: "model"},
		),
	)
	if err != nil {
		t.Fatalf("construct engine with accumulated registrations: %v", err)
	}
	for profileID, wantEndpoint := range map[string]string{
		"first-profile":  "http://first.example/v1",
		"second-profile": "http://second.example/v1",
	} {
		prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
			PromptID: "prompt", ProfileID: profileID,
		})
		if err != nil {
			t.Fatalf("prepare %s: %v", profileID, err)
		}
		if prepared.EffectiveModelParams.Endpoint != wantEndpoint {
			t.Fatalf("profile %s endpoint=%q, want %q", profileID, prepared.EffectiveModelParams.Endpoint, wantEndpoint)
		}
	}

	newEngine := func(endpoint string) *promptkit.Engine {
		t.Helper()
		value, err := promptkit.NewEngine(promptkit.Config{},
			promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
			promptkit.WithBackend(promptkit.Backend{ID: "same-id", Endpoint: endpoint}),
			promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "same-id", Model: "model"}),
		)
		if err != nil {
			t.Fatalf("construct isolated engine: %v", err)
		}
		return value
	}
	firstEngine := newEngine("http://one.example/v1")
	secondEngine := newEngine("http://two.example/v1")
	for engine, wantEndpoint := range map[*promptkit.Engine]string{
		firstEngine:  "http://one.example/v1",
		secondEngine: "http://two.example/v1",
	} {
		prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
		if err != nil {
			t.Fatalf("prepare isolated engine: %v", err)
		}
		if prepared.EffectiveModelParams.Endpoint != wantEndpoint {
			t.Fatalf("isolated engine endpoint=%q, want %q", prepared.EffectiveModelParams.Endpoint, wantEndpoint)
		}
	}
}

func TestWithBackendCopiesQueueCapacity(t *testing.T) {
	queueCapacity := 4
	option := promptkit.WithBackend(promptkit.Backend{
		ID:               "custom",
		Endpoint:         "http://custom.example/v1",
		ConcurrencyLimit: 1,
		QueueCapacity:    &queueCapacity,
	})
	queueCapacity = -1

	_, err := promptkit.NewEngine(promptkit.Config{},
		promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
		option,
		promptkit.WithProfiles(promptkit.Profile{
			ID: "profile", BackendID: "custom", Model: "model",
		}),
	)
	if err != nil {
		t.Fatalf("construct engine after mutating queue pointer: %v", err)
	}
}

func TestBackendRegistrationRejectsInvalidAndDuplicateDefinitions(t *testing.T) {
	cycle := map[string]any{}
	cycle["self"] = cycle
	tests := []struct {
		name     string
		backends []promptkit.Backend
	}{
		{name: "blank id", backends: []promptkit.Backend{{Endpoint: "http://example.test/v1"}}},
		{name: "invalid endpoint", backends: []promptkit.Backend{{ID: "custom", Endpoint: "ftp://example.test/v1"}}},
		{name: "invalid environment", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", APIKeyEnv: "BAD-NAME"}}},
		{name: "reserved extra parameter", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"model": "override"}}}},
		{name: "cyclic extra parameter", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: cycle}}},
		{name: "malformed JSON number", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"value": json.Number("01")}}}},
		{name: "excessively deep extra parameter", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"value": excessivelyDeepJSONValue()}}}},
		{name: "duplicate consumer id", backends: []promptkit.Backend{
			{ID: " custom ", Endpoint: "http://one.example/v1"},
			{ID: "custom", Endpoint: "http://two.example/v1"},
		}},
		{name: "reserved built-in id", backends: []promptkit.Backend{{
			ID: promptkit.BackendOpenRouter, Endpoint: "http://replacement.example/v1",
		}}},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			options := []promptkit.Option{
				promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
			}
			for _, backend := range tt.backends {
				options = append(options, promptkit.WithBackend(backend))
			}
			_, err := promptkit.NewEngine(promptkit.Config{}, options...)
			if !errors.Is(err, promptkit.ErrInvalidConfig) {
				t.Fatalf("expected ErrInvalidConfig, got %v", err)
			}
		})
	}
}

func TestBackendExtraParamsAreDeeplyCopiedAtConstructionAndLookup(t *testing.T) {
	nested := map[string]any{"value": "original"}
	extraParams := map[string]any{"nested": nested}
	engine, err := promptkit.NewEngine(promptkit.Config{},
		promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
		promptkit.WithBackend(promptkit.Backend{
			ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: extraParams,
		}),
		promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "custom", Model: "model"}),
	)
	if err != nil {
		t.Fatalf("construct engine: %v", err)
	}
	nested["value"] = "mutated input"
	extraParams["later"] = true

	prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
	if err != nil {
		t.Fatalf("first prepare: %v", err)
	}
	gotNested := prepared.EffectiveModelParams.ExtraParams["nested"].(map[string]any)
	if gotNested["value"] != "original" || prepared.EffectiveModelParams.ExtraParams["later"] != nil {
		t.Fatalf("backend retained caller mutations: %#v", prepared.EffectiveModelParams.ExtraParams)
	}
	gotNested["value"] = "mutated lookup"

	prepared, err = engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
	if err != nil {
		t.Fatalf("second prepare: %v", err)
	}
	gotNested = prepared.EffectiveModelParams.ExtraParams["nested"].(map[string]any)
	if gotNested["value"] != "original" {
		t.Fatalf("backend retained lookup mutation: %#v", prepared.EffectiveModelParams.ExtraParams)
	}
}

func TestEngineValidationIsSinglePass(t *testing.T) {
	client := &fakeLLMClient{
		response: &promptkit.GenerateResponse{Content: "not-json"},
	}
	engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(client))

	result, err := engine.Run(context.Background(), promptkit.RunRequest{
		PromptID: frameworkMarkdownSummaryPromptID,
		Inputs: map[string]promptkit.ArtifactRef{
			"transcript": promptkit.Inline("Rin opens the gate."),
			"glossary":   promptkit.Inline("gate: A guarded passage."),
		},
		Validation: &promptkit.OutputContract{
			Format:         promptkit.FormatJSON,
			ValidationMode: promptkit.ValidationJSON,
			RepairAttempts: 3,
		},
	})
	if err != nil {
		t.Fatalf("run with failed content validation: %v", err)
	}
	if result.Validation.Status != promptkit.ValidationFailed ||
		result.Validation.RepairAttempts != 0 {
		t.Fatalf("expected failed single-pass validation, got %#v", result.Validation)
	}
	if len(client.requests) != 1 {
		t.Fatalf("expected one model generation, got %d", len(client.requests))
	}
}

func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
	profile := promptkit.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}

	t.Run("prompt source", func(t *testing.T) {
		engine, err := promptkit.NewEngine(promptkit.Config{},
			promptkit.WithPromptFS(contractPromptFS("first", "profile", "first"), "."),
			promptkit.WithPromptFS(contractPromptFS("second", "profile", "second"), "."),
			promptkit.WithProfiles(profile),
		)
		if err != nil {
			t.Fatalf("construct engine: %v", err)
		}
		prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "second"})
		if err != nil {
			t.Fatalf("prepare from last prompt source: %v", err)
		}
		if prepared.Messages[0].Content != "second" {
			t.Fatalf("expected last prompt source, got %#v", prepared.Messages)
		}
	})

	t.Run("profile source", func(t *testing.T) {
		engine, err := promptkit.NewEngine(promptkit.Config{},
			promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
			promptkit.WithProfileFS(contractProfileFS("profile", "first-model"), "."),
			promptkit.WithProfileFS(contractProfileFS("profile", "second-model"), "."),
		)
		if err != nil {
			t.Fatalf("construct engine: %v", err)
		}
		prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
		if err != nil {
			t.Fatalf("prepare from last profile source: %v", err)
		}
		if prepared.EffectiveModelParams.Model != "second-model" {
			t.Fatalf("expected last profile source, got %q", prepared.EffectiveModelParams.Model)
		}
	})

	t.Run("fallback profile source", func(t *testing.T) {
		engine, err := promptkit.NewEngine(promptkit.Config{},
			promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
			promptkit.WithFallbackProfileFS(contractProfileFS("profile", "first-model"), "."),
			promptkit.WithFallbackProfileFS(contractProfileFS("profile", "second-model"), "."),
		)
		if err != nil {
			t.Fatalf("construct engine: %v", err)
		}
		prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
		if err != nil {
			t.Fatalf("prepare from last fallback profile source: %v", err)
		}
		if prepared.EffectiveModelParams.Model != "second-model" {
			t.Fatalf("expected last fallback profile source, got %q", prepared.EffectiveModelParams.Model)
		}
	})

	t.Run("in-memory profiles", func(t *testing.T) {
		first := profile
		first.Model = "first-model"
		second := profile
		second.Model = "second-model"
		engine, err := promptkit.NewEngine(promptkit.Config{},
			promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
			promptkit.WithProfiles(first),
			promptkit.WithProfiles(second),
		)
		if err != nil {
			t.Fatalf("construct engine: %v", err)
		}
		prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
		if err != nil {
			t.Fatalf("prepare from last in-memory profile option: %v", err)
		}
		if prepared.EffectiveModelParams.Model != "second-model" {
			t.Fatalf("expected last in-memory profiles, got %q", prepared.EffectiveModelParams.Model)
		}
	})

	t.Run("schema source", func(t *testing.T) {
		engine, err := promptkit.NewEngine(promptkit.Config{},
			promptkit.WithPromptFS(contractSchemaPromptFS(), "."),
			promptkit.WithProfiles(profile),
			promptkit.WithSchemaFS(contractSchemaFS("first"), "."),
			promptkit.WithSchemaFS(contractSchemaFS("second"), "."),
		)
		if err != nil {
			t.Fatalf("construct engine: %v", err)
		}
		prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "schema-prompt"})
		if err != nil {
			t.Fatalf("prepare from last schema source: %v", err)
		}
		schema := prepared.StructuredOutput.JSONSchema.Schema.(map[string]any)
		if schema["title"] != "second" {
			t.Fatalf("expected last schema source, got %#v", schema)
		}
	})

	t.Run("model client", func(t *testing.T) {
		var firstCalls, secondCalls atomic.Int64
		engine, err := promptkit.NewEngine(promptkit.Config{},
			promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
			promptkit.WithProfiles(profile),
			promptkit.WithLLMClient(countingLLMClient{calls: &firstCalls}),
			promptkit.WithLLMClient(countingLLMClient{calls: &secondCalls}),
		)
		if err != nil {
			t.Fatalf("construct engine: %v", err)
		}
		if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); err != nil {
			t.Fatalf("run with last model client: %v", err)
		}
		if firstCalls.Load() != 0 || secondCalls.Load() != 1 {
			t.Fatalf("expected only last client call, got first=%d second=%d", firstCalls.Load(), secondCalls.Load())
		}
	})

	t.Run("artifact reader", func(t *testing.T) {
		engine, err := promptkit.NewEngine(promptkit.Config{},
			promptkit.WithPromptFS(contractInputPromptFS(), "."),
			promptkit.WithProfiles(profile),
			promptkit.WithArtifactReader(fixedArtifactReader("first")),
			promptkit.WithArtifactReader(fixedArtifactReader("second")),
		)
		if err != nil {
			t.Fatalf("construct engine: %v", err)
		}
		prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
			PromptID: "input-prompt",
			Inputs:   map[string]promptkit.ArtifactRef{"input": promptkit.Inline("ignored")},
		})
		if err != nil {
			t.Fatalf("prepare with last artifact reader: %v", err)
		}
		if prepared.Messages[0].Content != "second" {
			t.Fatalf("expected last artifact reader, got %#v", prepared.Messages)
		}
	})
}

func TestFallbackProfileSourcePrecedence(t *testing.T) {
	const profileID = "application-profile"

	prepareModel := func(t *testing.T, engine *promptkit.Engine, promptID string) string {
		t.Helper()
		prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: promptID})
		if err != nil {
			t.Fatalf("prepare: %v", err)
		}
		return prepared.EffectiveModelParams.Model
	}

	t.Run("in-memory profiles override ordinary and fallback profiles", func(t *testing.T) {
		engine, err := promptkit.NewEngine(promptkit.Config{},
			promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
			promptkit.WithProfiles(promptkit.Profile{ID: profileID, Endpoint: "http://example.test/v1", Model: "memory-model"}),
			promptkit.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."),
			promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
		)
		if err != nil {
			t.Fatalf("construct engine: %v", err)
		}
		if model := prepareModel(t, engine, "prompt"); model != "memory-model" {
			t.Fatalf("expected in-memory profile, got %q", model)
		}
	})

	t.Run("ordinary filesystem source overrides fallback profile", func(t *testing.T) {
		engine, err := promptkit.NewEngine(promptkit.Config{},
			promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
			promptkit.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."),
			promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
		)
		if err != nil {
			t.Fatalf("construct engine: %v", err)
		}
		if model := prepareModel(t, engine, "prompt"); model != "ordinary-model" {
			t.Fatalf("expected ordinary profile, got %q", model)
		}
	})

	t.Run("ordinary option replaces configured directory", func(t *testing.T) {
		profileDir := t.TempDir()
		writePublicProfileFile(t, profileDir, profileID, "http://example.test/v1", "directory-model")
		engine, err := promptkit.NewEngine(promptkit.Config{ProfileDir: profileDir},
			promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
			promptkit.WithProfileFS(contractProfileFS(profileID, "option-model"), "."),
		)
		if err != nil {
			t.Fatalf("construct engine: %v", err)
		}
		if model := prepareModel(t, engine, "prompt"); model != "option-model" {
			t.Fatalf("expected ordinary option profile, got %q", model)
		}
	})

	t.Run("configured directory overrides fallback profile", func(t *testing.T) {
		profileDir := t.TempDir()
		writePublicProfileFile(t, profileDir, profileID, "http://example.test/v1", "directory-model")
		engine, err := promptkit.NewEngine(promptkit.Config{ProfileDir: profileDir},
			promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
			promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
		)
		if err != nil {
			t.Fatalf("construct engine: %v", err)
		}
		if model := prepareModel(t, engine, "prompt"); model != "directory-model" {
			t.Fatalf("expected configured directory profile, got %q", model)
		}
	})

	t.Run("fallback profile overrides built-in profile", func(t *testing.T) {
		engine, err := promptkit.NewEngine(promptkit.Config{},
			promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
			promptkit.WithFallbackProfileFS(contractProfileFS("mistral-small-3", "fallback-model"), "."),
		)
		if err != nil {
			t.Fatalf("construct engine: %v", err)
		}
		if model := prepareModel(t, engine, "prompt"); model != "fallback-model" {
			t.Fatalf("expected fallback profile, got %q", model)
		}
	})

	t.Run("missing fallback profile uses built-in profile", func(t *testing.T) {
		t.Setenv("OPENROUTER_API_KEY", "test-key")
		baseline, err := promptkit.NewEngine(promptkit.Config{},
			promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
		)
		if err != nil {
			t.Fatalf("construct baseline engine: %v", err)
		}
		want := prepareModel(t, baseline, "prompt")

		engine, err := promptkit.NewEngine(promptkit.Config{},
			promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
			promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
		)
		if err != nil {
			t.Fatalf("construct engine: %v", err)
		}
		if model := prepareModel(t, engine, "prompt"); model != want {
			t.Fatalf("expected built-in profile model %q, got %q", want, model)
		}
	})
}

func TestFallbackProfileSourcePreservesLazyLoadingAndErrors(t *testing.T) {
	const profileID = "application-profile"

	t.Run("construction defers malformed fallback profiles", func(t *testing.T) {
		_, err := promptkit.NewEngine(promptkit.Config{},
			promptkit.WithPromptFS(fstest.MapFS{}, "."),
			promptkit.WithFallbackProfileFS(fstest.MapFS{
				"broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nunknown: value\n")},
			}, "."),
		)
		if err != nil {
			t.Fatalf("construct engine with malformed fallback profile: %v", err)
		}
	})

	t.Run("unrelated malformed fallback profile does not block matching definition", func(t *testing.T) {
		engine, err := promptkit.NewEngine(promptkit.Config{},
			promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
			promptkit.WithFallbackProfileFS(fstest.MapFS{
				"broken.yaml": &fstest.MapFile{Data: []byte("id: unrelated\nunknown: value\n")},
				"valid.yaml":  &fstest.MapFile{Data: []byte("id: application-profile\nendpoint: http://example.test/v1\nmodel: fallback-model\n")},
			}, "."),
		)
		if err != nil {
			t.Fatalf("construct engine: %v", err)
		}
		prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
		if err != nil {
			t.Fatalf("prepare from valid fallback profile: %v", err)
		}
		if prepared.EffectiveModelParams.Model != "fallback-model" {
			t.Fatalf("unexpected fallback profile model: %q", prepared.EffectiveModelParams.Model)
		}
	})

	t.Run("matching malformed fallback profile does not reach built-in profile", func(t *testing.T) {
		engine, err := promptkit.NewEngine(promptkit.Config{},
			promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
			promptkit.WithFallbackProfileFS(fstest.MapFS{
				"mistral-small-3.yaml": &fstest.MapFile{Data: []byte("id: mistral-small-3\nendpoint: http://example.test/v1\nmodel: fallback-model\nunknown: value\n")},
			}, "."),
		)
		if err != nil {
			t.Fatalf("construct engine: %v", err)
		}
		if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); !errors.Is(err, promptkit.ErrProfileLoad) {
			t.Fatalf("expected ErrProfileLoad, got %v", err)
		}
	})

	t.Run("matching malformed ordinary profile does not reach fallback profile", func(t *testing.T) {
		engine, err := promptkit.NewEngine(promptkit.Config{},
			promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
			promptkit.WithProfileFS(fstest.MapFS{
				"application-profile.yaml": &fstest.MapFile{Data: []byte("id: application-profile\nendpoint: http://example.test/v1\nmodel: ordinary-model\nunknown: value\n")},
			}, "."),
			promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
		)
		if err != nil {
			t.Fatalf("construct engine: %v", err)
		}
		if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); !errors.Is(err, promptkit.ErrProfileLoad) {
			t.Fatalf("expected ErrProfileLoad, got %v", err)
		}
	})
}

func TestFallbackProfileSourceWorksAcrossWorkflows(t *testing.T) {
	const profileID = "application-profile"
	client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
	engine, err := promptkit.NewEngine(promptkit.Config{},
		promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
		promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
		promptkit.WithLLMClient(client),
	)
	if err != nil {
		t.Fatalf("construct engine: %v", err)
	}

	inspection, err := engine.InspectProfile(context.Background(), profileID)
	if err != nil {
		t.Fatalf("inspect fallback profile: %v", err)
	}
	prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
	if err != nil {
		t.Fatalf("prepare fallback profile: %v", err)
	}
	preparedExecution, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
	if err != nil {
		t.Fatalf("prepare execution with fallback profile: %v", err)
	}
	preparedDetails := preparedExecution.Details()
	preparedResult, err := engine.RunPrepared(context.Background(), preparedExecution)
	if err != nil {
		t.Fatalf("run prepared fallback profile: %v", err)
	}
	runResult, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
	if err != nil {
		t.Fatalf("run fallback profile: %v", err)
	}

	for name, model := range map[string]string{
		"inspection":         inspection.EffectiveModelParams.Model,
		"preparation":        prepared.EffectiveModelParams.Model,
		"prepared execution": preparedDetails.EffectiveModelParams.Model,
		"prepared result":    preparedResult.EffectiveModelParams.Model,
		"run result":         runResult.EffectiveModelParams.Model,
	} {
		if model != "fallback-model" {
			t.Fatalf("%s model=%q, want fallback-model", name, model)
		}
	}
}

func TestEngineSupportsConcurrentPrepareAndRun(t *testing.T) {
	engine, err := promptkit.NewEngine(promptkit.Config{},
		promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
		promptkit.WithBackend(promptkit.Backend{
			ID:       "concurrent",
			Endpoint: "http://example.test/v1",
		}),
		promptkit.WithProfiles(promptkit.Profile{
			ID:        "profile",
			BackendID: "concurrent",
			Model:     "model",
		}),
		promptkit.WithLLMClient(countingLLMClient{}),
	)
	if err != nil {
		t.Fatalf("construct engine: %v", err)
	}

	const calls = 40
	errs := make(chan error, calls)
	var wg sync.WaitGroup
	for i := 0; i < calls; i++ {
		wg.Add(1)
		go func(run bool) {
			defer wg.Done()
			request := promptkit.RunRequest{PromptID: "prompt"}
			if run {
				_, err := engine.Run(context.Background(), request)
				errs <- err
				return
			}
			_, err := engine.Prepare(context.Background(), request)
			errs <- err
		}(i%2 == 0)
	}
	wg.Wait()
	close(errs)
	for err := range errs {
		if err != nil {
			t.Fatalf("concurrent call failed: %v", err)
		}
	}
}

type countingLLMClient struct {
	calls *atomic.Int64
}

func (c countingLLMClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
	if c.calls != nil {
		c.calls.Add(1)
	}
	return &promptkit.GenerateResponse{Content: "ok"}, nil
}

type fixedArtifactReader string

func (r fixedArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
	return &promptkit.Artifact{Body: []byte(r)}, nil
}

func contractPromptFS(id, profileID, message string) fstest.MapFS {
	return fstest.MapFS{
		"prompt.yaml": &fstest.MapFile{Data: []byte(fmt.Sprintf(`id: %s
version: "1"
default_profile: %s
messages:
  - role: user
    content: %q
output:
  format: text
  validation_mode: none
`, id, profileID, message))},
	}
}

func contractInputPromptFS() fstest.MapFS {
	return fstest.MapFS{
		"prompt.yaml": &fstest.MapFile{Data: []byte(`id: input-prompt
version: "1"
default_profile: profile
inputs:
  - name: input
    required: true
messages:
  - role: user
    content: '{{input "input"}}'
output:
  format: text
  validation_mode: none
`)},
	}
}

func contractProfileFS(id, model string) fstest.MapFS {
	return fstest.MapFS{
		"profile.yaml": &fstest.MapFile{Data: []byte(fmt.Sprintf(`id: %s
endpoint: http://example.test/v1
model: %s
`, id, model))},
	}
}

func contractSchemaPromptFS() fstest.MapFS {
	return fstest.MapFS{
		"prompt.yaml": &fstest.MapFile{Data: []byte(`id: schema-prompt
version: "1"
default_profile: profile
messages:
  - role: user
    content: message
output:
  format: json
  validation_mode: json_schema
  schema_path: schema.json
`)},
	}
}

func contractSchemaFS(title string) fstest.MapFS {
	return fstest.MapFS{
		"schema.json": &fstest.MapFile{Data: []byte(fmt.Sprintf(`{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "title": %q,
  "type": "object"
}`, title))},
	}
}